diff --git a/.github/workflows/e2e-test-mysql.yml b/.github/workflows/e2e-test-mysql.yml deleted file mode 100644 index 07fe6e3189..0000000000 --- a/.github/workflows/e2e-test-mysql.yml +++ /dev/null @@ -1,83 +0,0 @@ -# End to end integration test which deploys the Tomcat operator to a Kubernetes -# (Kind) cluster and creates custom resources to verify the operator's functionality -name: MySQL Schema Operator End to End test -on: - pull_request: - branches: [ main, v1 ] - push: - branches: - - main -jobs: - mysql_e2e_test: - runs-on: ubuntu-latest - env: - KIND_CL_NAME: e2e-test - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: clean resident local docker - if: ${{ env.ACT }} - continue-on-error: true - run: | - for DIMG in "$KIND_CL_NAME-control-plane "; do - docker stop $DIMG ; docker rm $DIMG ; - done ; - sleep 1 - - - name: Create Kubernetes KinD Cluster - uses: container-tools/kind-action@v1.7.0 - with: - cluster_name: e2e-test - registry: false - - - name: Deploy MySQL DB - working-directory: sample-operators/mysql-schema - run: | - kubectl create namespace mysql - kubectl apply -f k8s/mysql-deployment.yaml - kubectl apply -f k8s/mysql-service.yaml - - - name: Set up Java and Maven - uses: actions/setup-java@v2 - with: - java-version: 11 - distribution: adopt-hotspot - cache: 'maven' - - - name: Build SDK - run: mvn install -DskipTests - - - name: build jib - working-directory: sample-operators/mysql-schema - run: | - mvn --version - mvn -B package jib:dockerBuild jib:buildTar -Djib-maven-image=mysql-schema-operator -DskipTests - kind load image-archive target/jib-image.tar --name=${{ env.KIND_CL_NAME }} - - - name: Apply CRDs - working-directory: sample-operators/mysql-schema - run: | - kubectl apply -f target/classes/META-INF/fabric8/mysqlschemas.mysql.sample.javaoperatorsdk-v1.yml - - - name: Deploy MySQL Operator - working-directory: sample-operators/mysql-schema - run: | - kubectl apply -f k8s/operator.yaml - - - name: Run E2E Tests - working-directory: sample-operators/mysql-schema - run: mvn -B test -P end-to-end-tests - - - name: Dump state - if: ${{ failure() }} - run: | - set +e - echo "All namespaces" - kubectl get ns - echo "All objects in mysql" - kubectl get all -n mysql-schema-test" -o yaml - echo "Output of mysql pod" - kubectl logs -l app=mysql-schema-operator -n mysql-schema - echo "All objects in mysql-schema-test" - kubectl get deployment,pod,tomcat,webapp -n mysql-schema-test" -o yaml diff --git a/.github/workflows/e2e-test-tomcat.yml b/.github/workflows/e2e-test-tomcat.yml deleted file mode 100644 index 70e01660d0..0000000000 --- a/.github/workflows/e2e-test-tomcat.yml +++ /dev/null @@ -1,79 +0,0 @@ -# End to end integration test which deploys the Tomcat operator to a Kubernetes -# (Kind) cluster and creates custom resources to verify the operator's functionality -name: Tomcat Operator End to End test -on: - pull_request: - branches: [ main, v1 ] - push: - branches: - - main -jobs: - tomcat_e2e_test: - runs-on: ubuntu-latest - env: - KIND_CL_NAME: e2e-test - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: clean resident local docker - if: ${{ env.ACT }} - continue-on-error: true - run: | - for DIMG in "$KIND_CL_NAME-control-plane "; do - docker stop $DIMG ; docker rm $DIMG ; - done ; - sleep 1 - - - name: Create Kubernetes KinD Cluster - uses: container-tools/kind-action@v1.7.0 - with: - cluster_name: e2e-test - registry: false - - - name: Set up Java and Maven - uses: actions/setup-java@v2 - with: - java-version: 11 - distribution: adopt-hotspot - cache: 'maven' - - - name: Build SDK - run: mvn install -DskipTests - - - name: build jib - working-directory: sample-operators/tomcat-operator - run: | - mvn --version - mvn -B package jib:dockerBuild jib:buildTar -Djib-maven-image=tomcat-operator -DskipTests - kind load image-archive target/jib-image.tar --name=${{ env.KIND_CL_NAME }} - - - name: Apply CRDs - working-directory: sample-operators/tomcat-operator - run: | - kubectl apply -f target/classes/META-INF/fabric8/tomcats.tomcatoperator.io-v1.yml - kubectl apply -f target/classes/META-INF/fabric8/webapps.tomcatoperator.io-v1.yml - - - name: Deploy Tomcat Operator - working-directory: sample-operators/tomcat-operator - run: | - kubectl apply -f k8s/operator.yaml - - - name: Run E2E Tests - working-directory: sample-operators/tomcat-operator - run: mvn -B test -P end-to-end-tests - - - name: Dump state - if: ${{ failure() }} - run: | - set +e - echo "All namespaces" - kubectl get ns - echo "All objects in tomcat-operator" - kubectl get all -n tomcat-operator -o yaml - echo "Output of tomcat-operator pod" - kubectl logs -l app=tomcat-operator -n tomcat-operator - echo "All objects in tomcat-test" - kubectl get deployment,pod,tomcat,webapp -n tomcat-test -o yaml - echo "Output of curl command" - kubectl logs curl -n tomcat-test diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml new file mode 100644 index 0000000000..31b89e1b20 --- /dev/null +++ b/.github/workflows/e2e-test.yml @@ -0,0 +1,50 @@ +# Integration and end to end tests which runs locally and deploys the Operator to a Kubernetes +# (Minikube) cluster and creates custom resources to verify the operator's functionality +name: Integration & End to End tests +on: + pull_request: + branches: [ main ] + push: + branches: + - main + +jobs: + sample_operators_tests: + strategy: + matrix: + sample_dir: + - "sample-operators/mysql-schema" + - "sample-operators/tomcat-operator" + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Setup Minikube-Kubernetes + uses: manusa/actions-setup-minikube@v2.4.3 + with: + minikube version: v1.24.0 + kubernetes version: v1.23.0 + github token: ${{ secrets.GITHUB_TOKEN }} + driver: docker + + - name: Set up Java and Maven + uses: actions/setup-java@v2 + with: + java-version: 17 + distribution: temurin + cache: 'maven' + + - name: Build SDK + run: mvn install -DskipTests + + - name: Run integration tests in local mode + working-directory: ${{ matrix.sample_dir }} + run: | + mvn test -P end-to-end-tests + + - name: Run E2E tests as a deployment + working-directory: ${{ matrix.sample_dir }} + run: | + eval $(minikube -p minikube docker-env) + mvn jib:dockerBuild test -P end-to-end-tests -Dtest.deployment=remote diff --git a/operator-framework-junit5/src/main/java/io/javaoperatorsdk/operator/junit/AbstractOperatorExtension.java b/operator-framework-junit5/src/main/java/io/javaoperatorsdk/operator/junit/AbstractOperatorExtension.java new file mode 100644 index 0000000000..fb204e8c57 --- /dev/null +++ b/operator-framework-junit5/src/main/java/io/javaoperatorsdk/operator/junit/AbstractOperatorExtension.java @@ -0,0 +1,176 @@ +package io.javaoperatorsdk.operator.junit; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.UUID; + +import org.junit.jupiter.api.extension.*; + +import io.fabric8.kubernetes.api.model.HasMetadata; +import io.fabric8.kubernetes.api.model.KubernetesResourceList; +import io.fabric8.kubernetes.client.DefaultKubernetesClient; +import io.fabric8.kubernetes.client.KubernetesClient; +import io.fabric8.kubernetes.client.dsl.NonNamespaceOperation; +import io.fabric8.kubernetes.client.dsl.Resource; +import io.fabric8.kubernetes.client.utils.KubernetesResourceUtil; +import io.fabric8.kubernetes.client.utils.Utils; +import io.javaoperatorsdk.operator.api.config.BaseConfigurationService; +import io.javaoperatorsdk.operator.api.config.ConfigurationService; +import io.javaoperatorsdk.operator.api.config.Version; + +public abstract class AbstractOperatorExtension implements HasKubernetesClient, + BeforeAllCallback, + BeforeEachCallback, + AfterAllCallback, + AfterEachCallback { + + protected final KubernetesClient kubernetesClient; + protected final ConfigurationService configurationService; + protected final List infrastructure; + protected Duration infrastructureTimeout; + protected final boolean oneNamespacePerClass; + protected final boolean preserveNamespaceOnError; + protected final boolean waitForNamespaceDeletion; + + protected String namespace; + + protected AbstractOperatorExtension( + ConfigurationService configurationService, + List infrastructure, + Duration infrastructureTimeout, + boolean oneNamespacePerClass, + boolean preserveNamespaceOnError, + boolean waitForNamespaceDeletion) { + + this.kubernetesClient = new DefaultKubernetesClient(); + this.configurationService = configurationService; + this.infrastructure = infrastructure; + this.infrastructureTimeout = infrastructureTimeout; + this.oneNamespacePerClass = oneNamespacePerClass; + this.preserveNamespaceOnError = preserveNamespaceOnError; + this.waitForNamespaceDeletion = waitForNamespaceDeletion; + } + + + @Override + public void beforeAll(ExtensionContext context) throws Exception { + beforeAllImpl(context); + } + + @Override + public void beforeEach(ExtensionContext context) throws Exception { + beforeEachImpl(context); + } + + @Override + public void afterAll(ExtensionContext context) throws Exception { + afterAllImpl(context); + } + + @Override + public void afterEach(ExtensionContext context) throws Exception { + afterEachImpl(context); + } + + @Override + public KubernetesClient getKubernetesClient() { + return kubernetesClient; + } + + public String getNamespace() { + return namespace; + } + + public NonNamespaceOperation, Resource> resources( + Class type) { + return kubernetesClient.resources(type).inNamespace(namespace); + } + + public T get(Class type, String name) { + return kubernetesClient.resources(type).inNamespace(namespace).withName(name).get(); + } + + public T create(Class type, T resource) { + return kubernetesClient.resources(type).inNamespace(namespace).create(resource); + } + + public T replace(Class type, T resource) { + return kubernetesClient.resources(type).inNamespace(namespace).replace(resource); + } + + public boolean delete(Class type, T resource) { + return kubernetesClient.resources(type).inNamespace(namespace).delete(resource); + } + + protected void beforeAllImpl(ExtensionContext context) { + if (oneNamespacePerClass) { + namespace = context.getRequiredTestClass().getSimpleName(); + namespace += "-"; + namespace += UUID.randomUUID(); + namespace = KubernetesResourceUtil.sanitizeName(namespace).toLowerCase(Locale.US); + namespace = namespace.substring(0, Math.min(namespace.length(), 63)); + + before(context); + } + } + + protected void beforeEachImpl(ExtensionContext context) { + if (!oneNamespacePerClass) { + namespace = context.getRequiredTestClass().getSimpleName(); + namespace += "-"; + namespace += context.getRequiredTestMethod().getName(); + namespace += "-"; + namespace += UUID.randomUUID(); + namespace = KubernetesResourceUtil.sanitizeName(namespace).toLowerCase(Locale.US); + namespace = namespace.substring(0, Math.min(namespace.length(), 63)); + + before(context); + } + } + + protected abstract void before(ExtensionContext context); + + protected void afterAllImpl(ExtensionContext context) { + if (oneNamespacePerClass) { + after(context); + } + } + + protected void afterEachImpl(ExtensionContext context) { + if (!oneNamespacePerClass) { + after(context); + } + } + + protected abstract void after(ExtensionContext context); + + public static abstract class AbstractBuilder { + protected ConfigurationService configurationService; + protected final List infrastructure; + protected Duration infrastructureTimeout; + protected boolean preserveNamespaceOnError; + protected boolean waitForNamespaceDeletion; + protected boolean oneNamespacePerClass; + + protected AbstractBuilder() { + this.configurationService = new BaseConfigurationService(Version.UNKNOWN); + + this.infrastructure = new ArrayList<>(); + this.infrastructureTimeout = Duration.ofMinutes(1); + + this.preserveNamespaceOnError = Utils.getSystemPropertyOrEnvVar( + "josdk.it.preserveNamespaceOnError", + false); + + this.waitForNamespaceDeletion = Utils.getSystemPropertyOrEnvVar( + "josdk.it.waitForNamespaceDeletion", + true); + + this.oneNamespacePerClass = Utils.getSystemPropertyOrEnvVar( + "josdk.it.oneNamespacePerClass", + false); + } + } +} diff --git a/operator-framework-junit5/src/main/java/io/javaoperatorsdk/operator/junit/E2EOperatorExtension.java b/operator-framework-junit5/src/main/java/io/javaoperatorsdk/operator/junit/E2EOperatorExtension.java new file mode 100644 index 0000000000..6710b1fc92 --- /dev/null +++ b/operator-framework-junit5/src/main/java/io/javaoperatorsdk/operator/junit/E2EOperatorExtension.java @@ -0,0 +1,200 @@ +package io.javaoperatorsdk.operator.junit; + +import java.io.File; +import java.io.FileInputStream; +import java.io.InputStream; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.concurrent.TimeUnit; + +import org.awaitility.Awaitility; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import io.fabric8.kubernetes.api.model.HasMetadata; +import io.fabric8.kubernetes.api.model.NamespaceBuilder; +import io.fabric8.kubernetes.api.model.rbac.ClusterRoleBinding; +import io.javaoperatorsdk.operator.api.config.ConfigurationService; + +public class E2EOperatorExtension extends AbstractOperatorExtension { + + private static final Logger LOGGER = LoggerFactory.getLogger(E2EOperatorExtension.class); + + private final List operatorDeployment; + private final Duration operatorDeploymentTimeout; + + private E2EOperatorExtension( + ConfigurationService configurationService, + List operatorDeployment, + Duration operatorDeploymentTimeout, + List infrastructure, + Duration infrastructureTimeout, + boolean preserveNamespaceOnError, + boolean waitForNamespaceDeletion, + boolean oneNamespacePerClass) { + super(configurationService, infrastructure, infrastructureTimeout, oneNamespacePerClass, + preserveNamespaceOnError, + waitForNamespaceDeletion); + this.operatorDeployment = operatorDeployment; + this.operatorDeploymentTimeout = operatorDeploymentTimeout; + } + + /** + * Creates a {@link Builder} to set up an {@link E2EOperatorExtension} instance. + * + * @return the builder. + */ + public static Builder builder() { + return new Builder(); + } + + @SuppressWarnings("unchecked") + protected void before(ExtensionContext context) { + LOGGER.info("Initializing integration test in namespace {}", namespace); + + kubernetesClient + .namespaces() + .create(new NamespaceBuilder().withNewMetadata().withName(namespace).endMetadata().build()); + + kubernetesClient + .resourceList(infrastructure) + .createOrReplace(); + kubernetesClient + .resourceList(infrastructure) + .waitUntilReady(infrastructureTimeout.toMillis(), TimeUnit.MILLISECONDS); + + final var crdPath = "./target/classes/META-INF/fabric8/"; + final var crdSuffix = "-v1.yml"; + + for (var crdFile : new File(crdPath).listFiles((ignored, name) -> name.endsWith(crdSuffix))) { + try (InputStream is = new FileInputStream(crdFile)) { + final var crd = kubernetesClient.load(is); + crd.createOrReplace(); + crd.waitUntilReady(2, TimeUnit.SECONDS); + LOGGER.debug("Applied CRD with name: {}", crd.get().get(0).getMetadata().getName()); + } catch (Exception ex) { + throw new IllegalStateException("Cannot apply CRD yaml: " + crdFile.getAbsolutePath(), ex); + } + } + + LOGGER.debug("Deploying the operator into Kubernetes"); + operatorDeployment.stream().forEach(hm -> { + hm.getMetadata().setNamespace(namespace); + if (hm.getKind().toLowerCase(Locale.ROOT).equals("clusterrolebinding")) { + var crb = (ClusterRoleBinding) hm; + for (var subject : crb.getSubjects()) { + subject.setNamespace(namespace); + } + } + }); + + kubernetesClient + .resourceList(operatorDeployment) + .inNamespace(namespace) + .createOrReplace(); + kubernetesClient + .resourceList(operatorDeployment) + .waitUntilReady(operatorDeploymentTimeout.toMillis(), TimeUnit.MILLISECONDS); + } + + protected void after(ExtensionContext context) { + if (namespace != null) { + if (preserveNamespaceOnError && context.getExecutionException().isPresent()) { + LOGGER.info("Preserving namespace {}", namespace); + } else { + kubernetesClient.resourceList(infrastructure).delete(); + kubernetesClient.resourceList(operatorDeployment).inNamespace(namespace).delete(); + LOGGER.info("Deleting namespace {} and stopping operator", namespace); + kubernetesClient.namespaces().withName(namespace).delete(); + if (waitForNamespaceDeletion) { + LOGGER.info("Waiting for namespace {} to be deleted", namespace); + Awaitility.await("namespace deleted") + .pollInterval(50, TimeUnit.MILLISECONDS) + .atMost(90, TimeUnit.SECONDS) + .until(() -> kubernetesClient.namespaces().withName(namespace).get() == null); + } + } + } + } + + @SuppressWarnings("rawtypes") + public static class Builder extends AbstractBuilder { + private final List operatorDeployment; + private Duration deploymentTimeout; + + protected Builder() { + super();; + this.operatorDeployment = new ArrayList<>(); + this.deploymentTimeout = Duration.ofMinutes(1); + } + + public Builder preserveNamespaceOnError(boolean value) { + this.preserveNamespaceOnError = value; + return this; + } + + public Builder waitForNamespaceDeletion(boolean value) { + this.waitForNamespaceDeletion = value; + return this; + } + + public Builder oneNamespacePerClass(boolean value) { + this.oneNamespacePerClass = value; + return this; + } + + public Builder withConfigurationService(ConfigurationService value) { + configurationService = value; + return this; + } + + public Builder withDeploymentTimeout(Duration value) { + deploymentTimeout = value; + return this; + } + + public Builder withInfrastructureTimeout(Duration value) { + infrastructureTimeout = value; + return this; + } + + public Builder withInfrastructure(List hm) { + infrastructure.addAll(hm); + return this; + } + + public Builder withInfrastructure(HasMetadata... hms) { + for (HasMetadata hm : hms) { + infrastructure.add(hm); + } + return this; + } + + public Builder withOperatorDeployment(List hm) { + operatorDeployment.addAll(hm); + return this; + } + + public Builder withOperatorDeployment(HasMetadata... hms) { + for (HasMetadata hm : hms) { + operatorDeployment.add(hm); + } + return this; + } + + public E2EOperatorExtension build() { + return new E2EOperatorExtension( + configurationService, + operatorDeployment, + deploymentTimeout, + infrastructure, + infrastructureTimeout, + preserveNamespaceOnError, + waitForNamespaceDeletion, + oneNamespacePerClass); + } + } +} diff --git a/operator-framework-junit5/src/main/java/io/javaoperatorsdk/operator/junit/OperatorExtension.java b/operator-framework-junit5/src/main/java/io/javaoperatorsdk/operator/junit/OperatorExtension.java index 3393de7a55..247ec9043b 100644 --- a/operator-framework-junit5/src/main/java/io/javaoperatorsdk/operator/junit/OperatorExtension.java +++ b/operator-framework-junit5/src/main/java/io/javaoperatorsdk/operator/junit/OperatorExtension.java @@ -1,30 +1,19 @@ package io.javaoperatorsdk.operator.junit; import java.io.InputStream; +import java.time.Duration; import java.util.ArrayList; import java.util.List; -import java.util.Locale; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import org.awaitility.Awaitility; -import org.junit.jupiter.api.extension.AfterAllCallback; -import org.junit.jupiter.api.extension.AfterEachCallback; -import org.junit.jupiter.api.extension.BeforeAllCallback; -import org.junit.jupiter.api.extension.BeforeEachCallback; import org.junit.jupiter.api.extension.ExtensionContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.fabric8.kubernetes.api.model.HasMetadata; -import io.fabric8.kubernetes.api.model.KubernetesResourceList; import io.fabric8.kubernetes.api.model.NamespaceBuilder; -import io.fabric8.kubernetes.client.DefaultKubernetesClient; -import io.fabric8.kubernetes.client.KubernetesClient; -import io.fabric8.kubernetes.client.dsl.NonNamespaceOperation; -import io.fabric8.kubernetes.client.dsl.Resource; -import io.fabric8.kubernetes.client.utils.KubernetesResourceUtil; -import io.fabric8.kubernetes.client.utils.Utils; import io.javaoperatorsdk.operator.Operator; import io.javaoperatorsdk.operator.api.config.BaseConfigurationService; import io.javaoperatorsdk.operator.api.config.ConfigurationService; @@ -35,36 +24,26 @@ import static io.javaoperatorsdk.operator.api.config.ControllerConfigurationOverrider.override; -public class OperatorExtension - implements HasKubernetesClient, - BeforeAllCallback, - BeforeEachCallback, - AfterAllCallback, - AfterEachCallback { +public class OperatorExtension extends AbstractOperatorExtension { private static final Logger LOGGER = LoggerFactory.getLogger(OperatorExtension.class); - private final KubernetesClient kubernetesClient; - private final ConfigurationService configurationService; private final Operator operator; private final List reconcilers; - private final boolean preserveNamespaceOnError; - private final boolean waitForNamespaceDeletion; - - private String namespace; private OperatorExtension( ConfigurationService configurationService, List reconcilers, + List infrastructure, + Duration infrastructureTimeout, boolean preserveNamespaceOnError, - boolean waitForNamespaceDeletion) { - - this.kubernetesClient = new DefaultKubernetesClient(); - this.configurationService = configurationService; + boolean waitForNamespaceDeletion, + boolean oneNamespacePerClass) { + super(configurationService, infrastructure, infrastructureTimeout, oneNamespacePerClass, + preserveNamespaceOnError, + waitForNamespaceDeletion); this.reconcilers = reconcilers; this.operator = new Operator(this.kubernetesClient, this.configurationService); - this.preserveNamespaceOnError = preserveNamespaceOnError; - this.waitForNamespaceDeletion = waitForNamespaceDeletion; } /** @@ -76,35 +55,6 @@ public static Builder builder() { return new Builder(); } - @Override - public void beforeAll(ExtensionContext context) throws Exception { - before(context); - } - - @Override - public void beforeEach(ExtensionContext context) throws Exception { - before(context); - } - - @Override - public void afterAll(ExtensionContext context) throws Exception { - after(context); - } - - @Override - public void afterEach(ExtensionContext context) throws Exception { - after(context); - } - - @Override - public KubernetesClient getKubernetesClient() { - return kubernetesClient; - } - - public String getNamespace() { - return namespace; - } - @SuppressWarnings({"rawtypes"}) public List getReconcilers() { return operator.getControllers().stream() @@ -129,41 +79,20 @@ public T getControllerOfType(Class type) { () -> new IllegalArgumentException("Unable to find a reconciler of type: " + type)); } - public NonNamespaceOperation, Resource> resources( - Class type) { - return kubernetesClient.resources(type).inNamespace(namespace); - } - - public T get(Class type, String name) { - return kubernetesClient.resources(type).inNamespace(namespace).withName(name).get(); - } - - public T create(Class type, T resource) { - return kubernetesClient.resources(type).inNamespace(namespace).create(resource); - } - - public T replace(Class type, T resource) { - return kubernetesClient.resources(type).inNamespace(namespace).replace(resource); - } - - public boolean delete(Class type, T resource) { - return kubernetesClient.resources(type).inNamespace(namespace).delete(resource); - } - @SuppressWarnings("unchecked") protected void before(ExtensionContext context) { - namespace = context.getRequiredTestClass().getSimpleName(); - namespace += "-"; - namespace += context.getRequiredTestMethod().getName(); - namespace = KubernetesResourceUtil.sanitizeName(namespace).toLowerCase(Locale.US); - namespace = namespace.substring(0, Math.min(namespace.length(), 63)); - LOGGER.info("Initializing integration test in namespace {}", namespace); kubernetesClient .namespaces() .create(new NamespaceBuilder().withNewMetadata().withName(namespace).endMetadata().build()); + kubernetesClient + .resourceList(infrastructure) + .createOrReplace(); + kubernetesClient + .resourceList(infrastructure) + .waitUntilReady(infrastructureTimeout.toMillis(), TimeUnit.MILLISECONDS); for (var ref : reconcilers) { final var config = configurationService.getConfigurationFor(ref.reconciler); @@ -191,6 +120,7 @@ protected void before(ExtensionContext context) { this.operator.register(ref.reconciler, oconfig.build()); } + LOGGER.debug("Starting the operator locally"); this.operator.start(); } @@ -199,6 +129,7 @@ protected void after(ExtensionContext context) { if (preserveNamespaceOnError && context.getExecutionException().isPresent()) { LOGGER.info("Preserving namespace {}", namespace); } else { + kubernetesClient.resourceList(infrastructure).delete(); LOGGER.info("Deleting namespace {} and stopping operator", namespace); kubernetesClient.namespaces().withName(namespace).delete(); if (waitForNamespaceDeletion) { @@ -219,23 +150,14 @@ protected void after(ExtensionContext context) { } @SuppressWarnings("rawtypes") - public static class Builder { + public static class Builder extends AbstractBuilder { private final List reconcilers; private ConfigurationService configurationService; - private boolean preserveNamespaceOnError; - private boolean waitForNamespaceDeletion; protected Builder() { + super(); this.configurationService = new BaseConfigurationService(Version.UNKNOWN); this.reconcilers = new ArrayList<>(); - - this.preserveNamespaceOnError = Utils.getSystemPropertyOrEnvVar( - "josdk.it.preserveNamespaceOnError", - false); - - this.waitForNamespaceDeletion = Utils.getSystemPropertyOrEnvVar( - "josdk.it.waitForNamespaceDeletion", - true); } public Builder preserveNamespaceOnError(boolean value) { @@ -248,11 +170,33 @@ public Builder waitForNamespaceDeletion(boolean value) { return this; } + public Builder oneNamespacePerClass(boolean value) { + this.oneNamespacePerClass = value; + return this; + } + public Builder withConfigurationService(ConfigurationService value) { configurationService = value; return this; } + public Builder withInfrastructureTimeout(Duration value) { + infrastructureTimeout = value; + return this; + } + + public Builder withInfrastructure(List hm) { + infrastructure.addAll(hm); + return this; + } + + public Builder withInfrastructure(HasMetadata... hms) { + for (HasMetadata hm : hms) { + infrastructure.add(hm); + } + return this; + } + @SuppressWarnings("rawtypes") public Builder withReconciler(Reconciler value) { reconcilers.add(new ReconcilerSpec(value, null)); @@ -279,8 +223,11 @@ public OperatorExtension build() { return new OperatorExtension( configurationService, reconcilers, + infrastructure, + infrastructureTimeout, preserveNamespaceOnError, - waitForNamespaceDeletion); + waitForNamespaceDeletion, + oneNamespacePerClass); } } diff --git a/sample-operators/mysql-schema/pom.xml b/sample-operators/mysql-schema/pom.xml index f38aa9bc17..bda0efa060 100644 --- a/sample-operators/mysql-schema/pom.xml +++ b/sample-operators/mysql-schema/pom.xml @@ -72,6 +72,12 @@ jackson-dataformat-yaml 2.13.1 + + io.javaoperatorsdk + operator-framework-junit-5 + ${project.version} + test + diff --git a/sample-operators/mysql-schema/src/main/java/io/javaoperatorsdk/operator/sample/MySQLSchemaReconciler.java b/sample-operators/mysql-schema/src/main/java/io/javaoperatorsdk/operator/sample/MySQLSchemaReconciler.java index af77b6ce97..18a2c7e518 100644 --- a/sample-operators/mysql-schema/src/main/java/io/javaoperatorsdk/operator/sample/MySQLSchemaReconciler.java +++ b/sample-operators/mysql-schema/src/main/java/io/javaoperatorsdk/operator/sample/MySQLSchemaReconciler.java @@ -23,7 +23,7 @@ import static java.lang.String.format; -@ControllerConfiguration +@ControllerConfiguration(finalizerName = Constants.NO_FINALIZER) public class MySQLSchemaReconciler implements Reconciler, ErrorStatusHandler, EventSourceInitializer { diff --git a/sample-operators/mysql-schema/src/test/java/io/javaoperatorsdk/operator/sample/MySQLSchemaOperatorE2E.java b/sample-operators/mysql-schema/src/test/java/io/javaoperatorsdk/operator/sample/MySQLSchemaOperatorE2E.java index f4c529426b..cf8284f179 100644 --- a/sample-operators/mysql-schema/src/test/java/io/javaoperatorsdk/operator/sample/MySQLSchemaOperatorE2E.java +++ b/sample-operators/mysql-schema/src/test/java/io/javaoperatorsdk/operator/sample/MySQLSchemaOperatorE2E.java @@ -1,18 +1,26 @@ package io.javaoperatorsdk.operator.sample; +import java.io.FileInputStream; +import java.io.FileNotFoundException; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import io.fabric8.kubernetes.api.model.*; -import io.fabric8.kubernetes.api.model.apps.*; -import io.fabric8.kubernetes.client.*; -import io.fabric8.kubernetes.client.Config; -import io.fabric8.kubernetes.client.ConfigBuilder; -import io.javaoperatorsdk.operator.Operator; +import io.fabric8.kubernetes.api.model.HasMetadata; +import io.fabric8.kubernetes.api.model.NamespaceBuilder; +import io.fabric8.kubernetes.api.model.ObjectMetaBuilder; +import io.fabric8.kubernetes.client.DefaultKubernetesClient; +import io.fabric8.kubernetes.client.KubernetesClient; +import io.fabric8.kubernetes.client.LocalPortForward; import io.javaoperatorsdk.operator.config.runtime.DefaultConfigurationService; +import io.javaoperatorsdk.operator.junit.AbstractOperatorExtension; +import io.javaoperatorsdk.operator.junit.E2EOperatorExtension; +import io.javaoperatorsdk.operator.junit.OperatorExtension; import static java.util.concurrent.TimeUnit.MINUTES; import static org.awaitility.Awaitility.await; @@ -23,61 +31,102 @@ public class MySQLSchemaOperatorE2E { - final static String TEST_NS = "mysql-schema-test"; + final static Logger log = LoggerFactory.getLogger(MySQLSchemaOperatorE2E.class); + + final static KubernetesClient client = new DefaultKubernetesClient(); + final static String MY_SQL_NS = "mysql"; - final static Logger log = LoggerFactory.getLogger(MySQLSchemaOperatorE2E.class); + private static List infrastructure = new ArrayList<>(); + static { + infrastructure + .add(new NamespaceBuilder() + .withNewMetadata() + .withName(MY_SQL_NS) + .endMetadata() + .build()); + try { + infrastructure.addAll( + client.load(new FileInputStream("k8s/mysql-deployment.yaml")).get()); + infrastructure.addAll( + client.load(new FileInputStream("k8s/mysql-service.yaml")).get()); + } catch (FileNotFoundException e) { + e.printStackTrace(); + } + } + + boolean isLocal() { + String deployment = System.getProperty("test.deployment"); + boolean remote = (deployment != null && deployment.equals("remote")); + log.info("Running the operator " + (remote ? "remote" : "locally")); + return !remote; + } + + @RegisterExtension + AbstractOperatorExtension operator = isLocal() ? OperatorExtension.builder() + .withConfigurationService(DefaultConfigurationService.instance()) + .withReconciler(new MySQLSchemaReconciler(client, + new MySQLDbConfig("127.0.0.1", "3306", "root", "password"))) + .withInfrastructure(infrastructure) + .build() + : E2EOperatorExtension.builder() + .withConfigurationService(DefaultConfigurationService.instance()) + .withOperatorDeployment( + client.load(new FileInputStream("k8s/operator.yaml")).get()) + .withInfrastructure(infrastructure) + .build(); + + + + public MySQLSchemaOperatorE2E() throws FileNotFoundException {} @Test public void test() throws IOException { - Config config = new ConfigBuilder().withNamespace(null).build(); - KubernetesClient client = new DefaultKubernetesClient(config); - - // Use this if you want to run the test without deploying the Operator to Kubernetes - if ("true".equals(System.getenv("RUN_OPERATOR_IN_TEST"))) { - Operator operator = new Operator(client, DefaultConfigurationService.instance()); - MySQLDbConfig dbConfig = new MySQLDbConfig("mysql", null, "root", "password"); - operator.register(new MySQLSchemaReconciler(client, dbConfig)); - operator.start(); + // Opening a port-forward if running locally + LocalPortForward portForward = null; + if (isLocal()) { + String podName = client + .pods() + .inNamespace(MY_SQL_NS) + .withLabel("app", "mysql") + .list() + .getItems() + .get(0) + .getMetadata() + .getName(); + + portForward = client + .pods() + .inNamespace(MY_SQL_NS) + .withName(podName) + .portForward(3306, 3306); } MySQLSchema testSchema = new MySQLSchema(); testSchema.setMetadata(new ObjectMetaBuilder() .withName("mydb1") - .withNamespace(TEST_NS) + .withNamespace(operator.getNamespace()) .build()); testSchema.setSpec(new SchemaSpec()); testSchema.getSpec().setEncoding("utf8"); - Namespace testNs = new NamespaceBuilder().withMetadata( - new ObjectMetaBuilder().withName(TEST_NS).build()).build(); - - if (testNs != null) { - // We perform a pre-run cleanup instead of a post-run cleanup. This is to help with debugging - // test results when running against a persistent cluster. The test namespace would stay - // after the test run so we can check what's there, but it would be cleaned up during the next - // test run. - log.info("Cleanup: deleting test namespace {}", TEST_NS); - client.namespaces().delete(testNs); - await().atMost(5, MINUTES) - .until(() -> client.namespaces().withName(TEST_NS).get() == null); - } - - log.info("Creating test namespace {}", TEST_NS); - client.namespaces().create(testNs); - log.info("Creating test MySQLSchema object: {}", testSchema); client.resource(testSchema).createOrReplace(); log.info("Waiting 5 minutes for expected resources to be created and updated"); - await().atMost(5, MINUTES).untilAsserted(() -> { - MySQLSchema updatedSchema = client.resources(MySQLSchema.class).inNamespace(TEST_NS) - .withName(testSchema.getMetadata().getName()).get(); + await().atMost(1, MINUTES).ignoreExceptions().untilAsserted(() -> { + MySQLSchema updatedSchema = + client.resources(MySQLSchema.class).inNamespace(operator.getNamespace()) + .withName(testSchema.getMetadata().getName()).get(); assertThat(updatedSchema.getStatus(), is(notNullValue())); assertThat(updatedSchema.getStatus().getStatus(), equalTo("CREATED")); assertThat(updatedSchema.getStatus().getSecretName(), is(notNullValue())); assertThat(updatedSchema.getStatus().getUserName(), is(notNullValue())); }); + + if (portForward != null) { + portForward.close(); + } } } diff --git a/sample-operators/tomcat-operator/pom.xml b/sample-operators/tomcat-operator/pom.xml index 9ab41d0e57..4d95bcee4e 100644 --- a/sample-operators/tomcat-operator/pom.xml +++ b/sample-operators/tomcat-operator/pom.xml @@ -57,6 +57,12 @@ 4.1.1 test + + io.javaoperatorsdk + operator-framework-junit-5 + ${project.version} + test + diff --git a/sample-operators/tomcat-operator/src/main/java/io/javaoperatorsdk/operator/sample/WebappReconciler.java b/sample-operators/tomcat-operator/src/main/java/io/javaoperatorsdk/operator/sample/WebappReconciler.java index 0f785feb02..719c435b08 100644 --- a/sample-operators/tomcat-operator/src/main/java/io/javaoperatorsdk/operator/sample/WebappReconciler.java +++ b/sample-operators/tomcat-operator/src/main/java/io/javaoperatorsdk/operator/sample/WebappReconciler.java @@ -28,7 +28,9 @@ import io.javaoperatorsdk.operator.processing.event.source.EventSource; import io.javaoperatorsdk.operator.processing.event.source.informer.InformerEventSource; -@ControllerConfiguration +import static io.javaoperatorsdk.operator.api.reconciler.Constants.NO_FINALIZER; + +@ControllerConfiguration(finalizerName = NO_FINALIZER) public class WebappReconciler implements Reconciler, EventSourceInitializer { private KubernetesClient kubernetesClient; diff --git a/sample-operators/tomcat-operator/src/test/java/io/javaoperatorsdk/operator/sample/TomcatOperatorE2E.java b/sample-operators/tomcat-operator/src/test/java/io/javaoperatorsdk/operator/sample/TomcatOperatorE2E.java index e803b70aba..042f4973cf 100644 --- a/sample-operators/tomcat-operator/src/test/java/io/javaoperatorsdk/operator/sample/TomcatOperatorE2E.java +++ b/sample-operators/tomcat-operator/src/test/java/io/javaoperatorsdk/operator/sample/TomcatOperatorE2E.java @@ -1,18 +1,22 @@ package io.javaoperatorsdk.operator.sample; +import java.io.FileInputStream; +import java.io.FileNotFoundException; + import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.fabric8.kubernetes.api.model.*; import io.fabric8.kubernetes.client.*; -import io.fabric8.kubernetes.client.Config; -import io.fabric8.kubernetes.client.ConfigBuilder; import io.fabric8.kubernetes.client.extended.run.RunConfigBuilder; -import io.javaoperatorsdk.operator.Operator; import io.javaoperatorsdk.operator.config.runtime.DefaultConfigurationService; +import io.javaoperatorsdk.operator.junit.AbstractOperatorExtension; +import io.javaoperatorsdk.operator.junit.E2EOperatorExtension; +import io.javaoperatorsdk.operator.junit.OperatorExtension; -import static java.util.concurrent.TimeUnit.*; +import static java.util.concurrent.TimeUnit.MINUTES; import static org.awaitility.Awaitility.await; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.is; @@ -21,75 +25,82 @@ public class TomcatOperatorE2E { - final static String TEST_NS = "tomcat-test"; - final static Logger log = LoggerFactory.getLogger(TomcatOperatorE2E.class); - @Test - public void test() { - Config config = new ConfigBuilder().withNamespace(null).build(); - KubernetesClient client = new DefaultKubernetesClient(config); + final static KubernetesClient client = new DefaultKubernetesClient(); - // Use this if you want to run the test without deploying the Operator to Kubernetes - if ("true".equals(System.getenv("RUN_OPERATOR_IN_TEST"))) { - Operator operator = new Operator(client, DefaultConfigurationService.instance()); - operator.register(new TomcatReconciler(client)); - operator.register(new WebappReconciler(client)); - operator.start(); - } + public TomcatOperatorE2E() throws FileNotFoundException {} + + final static int tomcatReplicas = 2; + + boolean isLocal() { + String deployment = System.getProperty("test.deployment"); + boolean remote = (deployment != null && deployment.equals("remote")); + log.info("Running the operator " + (remote ? "remote" : "locally")); + return !remote; + } + @RegisterExtension + AbstractOperatorExtension operator = isLocal() ? OperatorExtension.builder() + .waitForNamespaceDeletion(false) + .withConfigurationService(DefaultConfigurationService.instance()) + .withReconciler(new TomcatReconciler(client)) + .withReconciler(new WebappReconciler(client)) + .build() + : E2EOperatorExtension.builder() + .waitForNamespaceDeletion(false) + .withConfigurationService(DefaultConfigurationService.instance()) + .withOperatorDeployment( + client.load(new FileInputStream("k8s/operator.yaml")).get()) + .build(); + + Tomcat getTomcat() { Tomcat tomcat = new Tomcat(); tomcat.setMetadata(new ObjectMetaBuilder() .withName("test-tomcat1") - .withNamespace(TEST_NS) + .withNamespace(operator.getNamespace()) .build()); tomcat.setSpec(new TomcatSpec()); - tomcat.getSpec().setReplicas(3); + tomcat.getSpec().setReplicas(tomcatReplicas); tomcat.getSpec().setVersion(9); + return tomcat; + } + Webapp getWebapp() { Webapp webapp1 = new Webapp(); webapp1.setMetadata(new ObjectMetaBuilder() .withName("test-webapp1") - .withNamespace(TEST_NS) + .withNamespace(operator.getNamespace()) .build()); webapp1.setSpec(new WebappSpec()); webapp1.getSpec().setContextPath("webapp1"); - webapp1.getSpec().setTomcat(tomcat.getMetadata().getName()); + webapp1.getSpec().setTomcat(getTomcat().getMetadata().getName()); webapp1.getSpec().setUrl("http://tomcat.apache.org/tomcat-7.0-doc/appdev/sample/sample.war"); + return webapp1; + } - var tomcatClient = client.customResources(Tomcat.class); - var webappClient = client.customResources(Webapp.class); - - Namespace testNs = new NamespaceBuilder().withMetadata( - new ObjectMetaBuilder().withName(TEST_NS).build()).build(); - - if (testNs != null) { - // We perform a pre-run cleanup instead of a post-run cleanup. This is to help with debugging - // test results when running against a persistent cluster. The test namespace would stay - // after the test run so we can check what's there, but it would be cleaned up during the next - // test run. - log.info("Cleanup: deleting test namespace {}", TEST_NS); - client.namespaces().delete(testNs); - await().atMost(5, MINUTES) - .until(() -> client.namespaces().withName("tomcat-test").get() == null); - } - - log.info("Creating test namespace {}", TEST_NS); - client.namespaces().create(testNs); + @Test + public void test() { + var tomcat = getTomcat(); + var webapp1 = getWebapp(); + var tomcatClient = client.resources(Tomcat.class); + var webappClient = client.resources(Webapp.class); log.info("Creating test Tomcat object: {}", tomcat); - tomcatClient.inNamespace(TEST_NS).create(tomcat); + tomcatClient.inNamespace(operator.getNamespace()).create(tomcat); log.info("Creating test Webapp object: {}", webapp1); - webappClient.inNamespace(TEST_NS).create(webapp1); + webappClient.inNamespace(operator.getNamespace()).create(webapp1); log.info("Waiting 5 minutes for Tomcat and Webapp CR statuses to be updated"); await().atMost(5, MINUTES).untilAsserted(() -> { Tomcat updatedTomcat = - tomcatClient.inNamespace(TEST_NS).withName(tomcat.getMetadata().getName()).get(); + tomcatClient.inNamespace(operator.getNamespace()).withName(tomcat.getMetadata().getName()) + .get(); Webapp updatedWebapp = - webappClient.inNamespace(TEST_NS).withName(webapp1.getMetadata().getName()).get(); + webappClient.inNamespace(operator.getNamespace()) + .withName(webapp1.getMetadata().getName()).get(); assertThat(updatedTomcat.getStatus(), is(notNullValue())); - assertThat(updatedTomcat.getStatus().getReadyReplicas(), equalTo(3)); + assertThat(updatedTomcat.getStatus().getReadyReplicas(), equalTo(tomcatReplicas)); assertThat(updatedWebapp.getStatus(), is(notNullValue())); assertThat(updatedWebapp.getStatus().getDeployedArtifact(), is(notNullValue())); }); @@ -102,7 +113,7 @@ public void test() { try { log.info("Starting curl Pod to test if webapp was deployed correctly"); - Pod curlPod = client.run().inNamespace(TEST_NS) + Pod curlPod = client.run().inNamespace(operator.getNamespace()) .withRunConfig(new RunConfigBuilder() .withArgs("-s", "-o", "/dev/null", "-w", "%{http_code}", url) .withName("curl") @@ -114,21 +125,24 @@ public void test() { await("wait-for-curl-pod-run").atMost(2, MINUTES) .until(() -> { String phase = - client.pods().inNamespace(TEST_NS).withName("curl").get().getStatus().getPhase(); + client.pods().inNamespace(operator.getNamespace()).withName("curl").get() + .getStatus().getPhase(); return phase.equals("Succeeded") || phase.equals("Failed"); }); String curlOutput = - client.pods().inNamespace(TEST_NS).withName(curlPod.getMetadata().getName()).getLog(); + client.pods().inNamespace(operator.getNamespace()) + .withName(curlPod.getMetadata().getName()).getLog(); log.info("Output from curl: '{}'", curlOutput); assertThat(curlOutput, equalTo("200")); } catch (KubernetesClientException ex) { throw new AssertionError(ex); } finally { log.info("Deleting curl Pod"); - client.pods().inNamespace(TEST_NS).withName("curl").delete(); + client.pods().inNamespace(operator.getNamespace()).withName("curl").delete(); await("wait-for-curl-pod-stop").atMost(1, MINUTES) - .until(() -> client.pods().inNamespace(TEST_NS).withName("curl").get() == null); + .until(() -> client.pods().inNamespace(operator.getNamespace()).withName("curl") + .get() == null); } }); }