Do nothing on storage finalizing when namespace is terminating or PVC does not exist

Fix error when namespace is not terminated in time

Requeue storage finalizer each 10 seconds after job is flushed

Wait until devworkspace CRs are removed on uninstall

Author: sleshchenko

Makefile

@@ -211,10 +211,9 @@ restart_webhook:
 uninstall: _kustomize
 # It's safer to delete all workspaces before deleting the controller; otherwise we could
 # leave workspaces in a hanging state if we add finalizers.
-	$(K8S_CLI) delete devworkspaces.workspace.devfile.io --all-namespaces --all | true
-	$(K8S_CLI) delete devworkspacetemplates.workspace.devfile.io --all-namespaces --all | true
-# Have to wait for routings to be deleted in case there are finalizers
-	$(K8S_CLI) delete workspaceroutings.controller.devfile.io --all-namespaces --all --wait | true
+	$(K8S_CLI) delete devworkspaces.workspace.devfile.io --all-namespaces --all --wait || true
+	$(K8S_CLI) delete devworkspacetemplates.workspace.devfile.io --all-namespaces --all || true
+	$(K8S_CLI) delete workspaceroutings.controller.devfile.io --all-namespaces --all --wait || true
 ifeq ($(PLATFORM),kubernetes)
 	$(KUSTOMIZE) build config/cert-manager | $(K8S_CLI) delete --ignore-not-found -f -
 else

config/components/rbac/role.yaml

@@ -22,6 +22,8 @@ rules:
   - namespaces
   verbs:
   - get
+  - list
+  - watch
 - apiGroups:
   - ""
   resources:

controllers/workspace/finalize.go

@@ -16,6 +16,7 @@ import (
 	"context"
 	"fmt"
 	"path"
+	"time"
 
 	"github.com/devfile/api/pkg/apis/workspaces/v1alpha2"
 	"github.com/devfile/devworkspace-operator/controllers/workspace/provision"
@@ -68,10 +69,33 @@ func (r *DevWorkspaceReconciler) finalize(ctx context.Context, log logr.Logger,
 	if wait {
 		return reconcile.Result{Requeue: true}, nil
 	}
+
+	terminating, err := r.namespaceIsTerminating(ctx, workspace.Namespace)
+	if err != nil {
+		return reconcile.Result{}, err
+	} else if terminating {
+		//Namespace is terminating, it's redundant to clean PVC files since it's going to be removed
+		log.Info("Namespace is terminating; clearing storage finalizer")
+		clearFinalizer(workspace)
+		return reconcile.Result{}, r.Update(ctx, workspace)
+	}
+
+	pvcExists, err := r.pvcExists(ctx, workspace)
+	if err != nil {
+		return reconcile.Result{}, err
+	} else if !pvcExists {
+		//PVC does not exist. nothing to clean up
+		log.Info("PVC does not exit; clearing storage finalizer")
+		clearFinalizer(workspace)
+		// job will be clean up by k8s garbage collector
+		return reconcile.Result{}, r.Update(ctx, workspace)
+	}
+
 	specJob, err := r.getSpecCleanupJob(workspace)
 	if err != nil {
 		return reconcile.Result{}, err
 	}
+
 	clusterJob, err := r.getClusterCleanupJob(ctx, workspace)
 	if err != nil {
 		return reconcile.Result{}, err
@@ -111,7 +135,8 @@ func (r *DevWorkspaceReconciler) finalize(ctx context.Context, log logr.Logger,
 			return r.updateWorkspaceStatus(workspace, r.Log, failedStatus, reconcile.Result{}, nil)
 		}
 	}
-	return reconcile.Result{}, nil
+	// Requeue at least each 10 seconds to check if PVC is not removed by someone else
+	return reconcile.Result{RequeueAfter: 10 * time.Second}, nil
 }
 
 func (r *DevWorkspaceReconciler) getSpecCleanupJob(workspace *v1alpha2.DevWorkspace) (*batchv1.Job, error) {
@@ -222,3 +247,33 @@ func clearFinalizer(workspace *v1alpha2.DevWorkspace) {
 	}
 	workspace.SetFinalizers(newFinalizers)
 }
+
+func (r *DevWorkspaceReconciler) namespaceIsTerminating(ctx context.Context, namespace string) (bool, error) {
+	namespacedName := types.NamespacedName{
+		Name: namespace,
+	}
+	n := &corev1.Namespace{}
+
+	err := r.Get(ctx, namespacedName, n)
+	if err != nil {
+		return false, err
+	}
+
+	return n.Status.Phase == corev1.NamespaceTerminating, nil
+}
+
+func (r *DevWorkspaceReconciler) pvcExists(ctx context.Context, workspace *v1alpha2.DevWorkspace) (bool, error) {
+	namespacedName := types.NamespacedName{
+		Name:      config.ControllerCfg.GetWorkspacePVCName(),
+		Namespace: workspace.Namespace,
+	}
+	err := r.Get(ctx, namespacedName, &corev1.PersistentVolumeClaim{})
+	if err != nil {
+		if k8sErrors.IsNotFound(err) {
+			return false, nil
+		}
+
+		return false, err
+	}
+	return true, nil
+}

test/e2e/pkg/client/namespace.go

@@ -51,7 +51,7 @@ func (w *K8sClient) WaitNamespaceIsTerminated(namespace string) (err error) {
 	for {
 		select {
 		case <-timeoutC:
-			return errors.New(fmt.Sprintf("The namespace %s is not terminated and removed after %d.", namespace, timeoutC))
+			return errors.New(fmt.Sprintf("The namespace %s is not terminated and removed after %ds.", namespace, timeout))
 		case <-tickC:
 			var ns *corev1.Namespace
 			ns, err = w.kubeClient.CoreV1().Namespaces().Get(context.TODO(), namespace, metav1.GetOptions{})