Add support for active health checks for Plus

examples/customization/README.md

@@ -60,6 +60,9 @@ The table below summarizes all of the options. For some of them, there are examp
 | `nginx.com/jwt-realm` | N/A | Specifies a realm. | N/A | [Support for JSON Web Tokens (JWTs)](../jwt). |
 | `nginx.com/jwt-token` | N/A | Specifies a variable that contains JSON Web Token. | By default, a JWT is expected in the `Authorization` header as a Bearer Token. | [Support for JSON Web Tokens (JWTs)](../jwt). |
 | `nginx.com/jwt-login-url` | N/A | Specifies a URL to which a client is redirected in case of an invalid or missing JWT. | N/A | [Support for JSON Web Tokens (JWTs)](../jwt). |
+| `nginx.com/health-checks` | N/A | Enables active health checks. | `False` | [Support for Active Health Checks](../health-checks). |
+| `nginx.com/health-checks-mandatory` | N/A | Configures active health checks as mandatory. | `False` | [Support for Active Health Checks](../health-checks). |
+| `nginx.com/health-checks-mandatory-queue` | N/A | When active health checks are mandatory, configures a queue for temporary storing incoming requests during the time when NGINX Plus is checking the health of the endpoints after a configuration reload. | `0` | [Support for Active Health Checks](../health-checks). |
 
 ## Using ConfigMaps
 

examples/health-checks/README.md

@@ -0,0 +1,79 @@
+# Support for Active Health Checks
+
+NGINX Plus supports [active health checks](https://docs.nginx.com/nginx/admin-guide/load-balancer/http-health-check/#active-health-checks). To use active health checks in the Ingress controller:
+
+1. Define health checks ([HTTP Readiness Probe](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/#define-readiness-probes)) in the templates of your application pods.
+2. Enable heath checks in the Ingress controller using the annotations.
+
+The Ingress controller provides the following annotations for configuring active health checks:
+
+* Required: `nginx.com/health-checks: "true"` -- enables active health checks. The default is `false`.
+* Optional: `nginx.com/health-checks-mandatory: "true"` -- configures active health checks as mandatory. With the default active health checks, when an endpoint is added to NGINX Plus via the API or after a configuration reload, NGINX Plus considers the endpoint to be healthy. With mandatory health checks, when an endpoint is added to NGINX Plus or after a configuration reload, NGINX Plus considers the endpoint to be unhealthy until its health check passes. The default is `false`.
+* Optional: `nginx.com/health-checks-mandatory-queue: "500"` -- configures a [queue](http://nginx.org/en/docs/http/ngx_http_upstream_module.html#queue) for temporary storing incoming requests during the time when NGINX Plus is checking the health of the endpoints after a configuration reload. If the queue is not configured or the queue is full, NGINX Plus will drop an incoming request returning the `502` code to the client. The queue is configured only when health checks are mandatory. The timeout parameter of the queue is configured with the value of the timeoutSeconds field of the corresponding Readiness Probe. Choose the size of the queue according with your requirements such as the expected number of requests per second and the timeout. The default is `0`.
+
+# Example
+
+In the following example we enable active health checks in the cafe-ingress Ingress:
+```yaml
+apiVersion: extensions/v1beta1
+kind: Ingress
+metadata:
+  name: cafe-ingress
+  annotations:
+     kubernetes.io/ingress.class: "nginx"
+     nginx.com/health-checks: "true"
+spec:
+  rules:
+  - host: "cafe.example.com"
+    http:
+      paths:
+      - path: /tea
+        backend:
+          serviceName: tea-svc
+          servicePort: 80
+      - path: /coffee
+        backend:
+          serviceName: coffee-svc
+          servicePort: 80
+      - path: /beer
+        backend:
+          serviceName: beer-svc
+          servicePort: 80
+```
+
+Note that a Readiness Probe must be configured in the pod template:
+```yaml
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: tea
+spec:
+  replicas: 3
+  selector:
+    matchLabels:
+      app: tea
+  template: 
+    metadata:
+      labels:
+        app: tea
+    spec:
+      containers:
+      - name: tea
+        image: nginxdemos/hello:plain-text
+        ports:
+        - containerPort: 80
+        readinessProbe:
+          httpGet:
+            port: 80
+            path: /healthz/tea
+            httpHeaders:
+            - name: header1
+              value: "some value"
+            - name: header2
+              value: "123"
+          initialDelaySeconds: 1
+          periodSeconds: 5
+          timeoutSeconds: 4
+          successThreshold: 2
+          failureThreshold: 3
+```

examples/mergeable-ingress-types/README.md

@@ -15,6 +15,9 @@ Masters cannot contain the following annotations:
 * nginx.org/ssl-services
 * nginx.org/websocket-services
 * nginx.com/sticky-cookie-services
+* nginx.com/health-checks
+* nginx.com/health-checks-mandatory
+* nginx.com/health-checks-mandatory-queue
 
 A Minion is declared using `nginx.org/mergeable-ingress-type: minion`. A Minion will be used to append different
 locations to an ingress resource with the Master value. TLS configurations are not allowed. Multiple minions can be

nginx-controller/controller/controller.go

@@ -1040,6 +1040,8 @@ func (lbc *LoadBalancerController) createIngress(ing *extensions.Ingress) (*ngin
 	}
 
 	ingEx.Endpoints = make(map[string][]string)
+	ingEx.HealthChecks = make(map[string]*api_v1.Probe)
+
 	if ing.Spec.Backend != nil {
 		endps, err := lbc.getEndpointsForIngressBackend(ing.Spec.Backend, ing.Namespace)
 		if err != nil {
@@ -1048,10 +1050,15 @@ func (lbc *LoadBalancerController) createIngress(ing *extensions.Ingress) (*ngin
 		} else {
 			ingEx.Endpoints[ing.Spec.Backend.ServiceName+ing.Spec.Backend.ServicePort.String()] = endps
 		}
+		if lbc.nginxPlus && lbc.isHealthCheckEnabled(ing) {
+			healthCheck := lbc.getHealthChecksForIngressBackend(ing.Spec.Backend, ing.Namespace)
+			if healthCheck != nil {
+				ingEx.HealthChecks[ing.Spec.Backend.ServiceName+ing.Spec.Backend.ServicePort.String()] = healthCheck
+			}
+		}
 	}
 
 	validRules := 0
-
 	for _, rule := range ing.Spec.Rules {
 		if rule.IngressRuleValue.HTTP == nil {
 			continue
@@ -1069,8 +1076,14 @@ func (lbc *LoadBalancerController) createIngress(ing *extensions.Ingress) (*ngin
 			} else {
 				ingEx.Endpoints[path.Backend.ServiceName+path.Backend.ServicePort.String()] = endps
 			}
+			if lbc.nginxPlus && lbc.isHealthCheckEnabled(ing) {
+				// Pull active health checks from k8 api
+				healthCheck := lbc.getHealthChecksForIngressBackend(&path.Backend, ing.Namespace)
+				if healthCheck != nil {
+					ingEx.HealthChecks[path.Backend.ServiceName+path.Backend.ServicePort.String()] = healthCheck
+				}
+			}
 		}
-
 		validRules++
 	}
 
@@ -1081,6 +1094,63 @@ func (lbc *LoadBalancerController) createIngress(ing *extensions.Ingress) (*ngin
 	return ingEx, nil
 }
 
+func (lbc *LoadBalancerController) getPodsForIngressBackend(svc *api_v1.Service, namespace string) *api_v1.PodList {
+	pods, err := lbc.client.CoreV1().Pods(svc.Namespace).List(meta_v1.ListOptions{LabelSelector: labels.Set(svc.Spec.Selector).String()})
+	if err != nil {
+		glog.V(3).Infof("Error fetching pods for namespace %v: %v", svc.Namespace, err)
+		return nil
+	}
+	return pods
+}
+
+func (lbc *LoadBalancerController) getHealthChecksForIngressBackend(backend *extensions.IngressBackend, namespace string) *api_v1.Probe {
+	svc, err := lbc.getServiceForIngressBackend(backend, namespace)
+	if err != nil {
+		glog.V(3).Infof("Error getting service %v: %v", backend.ServiceName, err)
+		return nil
+	}
+	svcPort := lbc.getServicePortForIngressPort(backend.ServicePort, svc)
+	if svcPort == nil {
+		return nil
+	}
+	pods := lbc.getPodsForIngressBackend(svc, namespace)
+	if pods == nil {
+		return nil
+	}
+	return findProbeForPods(pods.Items, svcPort)
+}
+
+func findProbeForPods(pods []api_v1.Pod, svcPort *api_v1.ServicePort) *api_v1.Probe {
+	if len(pods) > 0 {
+		pod := pods[0]
+		for _, container := range pod.Spec.Containers {
+			for _, port := range container.Ports {
+				if compareContainerPortAndServicePort(port, *svcPort) {
+					// only http ReadinessProbes are useful for us
+					if container.ReadinessProbe.Handler.HTTPGet != nil && container.ReadinessProbe.PeriodSeconds > 0 {
+						return container.ReadinessProbe
+					}
+				}
+			}
+		}
+	}
+	return nil
+}
+
+func compareContainerPortAndServicePort(containerPort api_v1.ContainerPort, svcPort api_v1.ServicePort) bool {
+	targetPort := svcPort.TargetPort
+	if (targetPort == intstr.IntOrString{}) {
+		return svcPort.Port > 0 && svcPort.Port == containerPort.ContainerPort
+	}
+	switch targetPort.Type {
+	case intstr.String:
+		return targetPort.StrVal == containerPort.Name && svcPort.Protocol == containerPort.Protocol
+	case intstr.Int:
+		return targetPort.IntVal > 0 && targetPort.IntVal == containerPort.ContainerPort
+	}
+	return false
+}
+
 func (lbc *LoadBalancerController) getEndpointsForIngressBackend(backend *extensions.IngressBackend, namespace string) ([]string, error) {
 	svc, err := lbc.getServiceForIngressBackend(backend, namespace)
 	if err != nil {
@@ -1138,6 +1208,15 @@ func (lbc *LoadBalancerController) getEndpointsForPort(endps api_v1.Endpoints, i
 	return nil, fmt.Errorf("No endpoints for target port %v in service %s", targetPort, svc.Name)
 }
 
+func (lbc *LoadBalancerController) getServicePortForIngressPort(ingSvcPort intstr.IntOrString, svc *api_v1.Service) *api_v1.ServicePort {
+	for _, port := range svc.Spec.Ports {
+		if (ingSvcPort.Type == intstr.Int && port.Port == int32(ingSvcPort.IntValue())) || (ingSvcPort.Type == intstr.String && port.Name == ingSvcPort.String()) {
+			return &port
+		}
+	}
+	return nil
+}
+
 func (lbc *LoadBalancerController) getTargetPort(svcPort *api_v1.ServicePort, svc *api_v1.Service) (int32, error) {
 	if (svcPort.TargetPort == intstr.IntOrString{}) {
 		return svcPort.Port, nil
@@ -1203,6 +1282,17 @@ func (lbc *LoadBalancerController) isNginxIngress(ing *extensions.Ingress) bool
 	}
 }
 
+// isHealthCheckEnabled checks if health checks are enabled so we can only query pods if enabled.
+func (lbc *LoadBalancerController) isHealthCheckEnabled(ing *extensions.Ingress) bool {
+	if healthCheckEnabled, exists, err := nginx.GetMapKeyAsBool(ing.Annotations, "nginx.com/health-checks", ing); exists {
+		if err != nil {
+			glog.Error(err)
+		}
+		return healthCheckEnabled
+	}
+	return false
+}
+
 // ValidateSecret validates that the secret follows the TLS Secret format.
 // For NGINX Plus, it also checks if the secret follows the JWK Secret format.
 func (lbc *LoadBalancerController) ValidateSecret(secret *api_v1.Secret) error {