Skip to content

Commit 76c6da5

Browse files
authored
feat: add livenessProbe and readinessProbe to all services (#88)
Closes HDX-1992 Adds livenessProbe and readinessProbe to mongo, clickhouse, otel-collector, and hyperdx services. Tested on cluster and chart rendering via ```bash helm template test ./charts/hdx-oss-v2 --set otel.enabled=true --set clickhouse.enabled=true --set mongodb.enabled=true ```
1 parent 2a8dac4 commit 76c6da5

10 files changed

+573
-4
lines changed

.changeset/pretty-ants-arrive.md

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
"helm-charts": patch
3+
---
4+
5+
feat: add livenessProbe and readinessProbe for services

charts/hdx-oss-v2/templates/clickhouse-deployment.yaml

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,26 @@ spec:
3535
env:
3636
- name: CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT
3737
value: "1"
38+
{{- if .Values.clickhouse.livenessProbe.enabled }}
39+
livenessProbe:
40+
httpGet:
41+
path: /ping
42+
port: {{ .Values.clickhouse.port }}
43+
initialDelaySeconds: {{ .Values.clickhouse.livenessProbe.initialDelaySeconds }}
44+
periodSeconds: {{ .Values.clickhouse.livenessProbe.periodSeconds }}
45+
timeoutSeconds: {{ .Values.clickhouse.livenessProbe.timeoutSeconds }}
46+
failureThreshold: {{ .Values.clickhouse.livenessProbe.failureThreshold }}
47+
{{- end }}
48+
{{- if .Values.clickhouse.readinessProbe.enabled }}
49+
readinessProbe:
50+
httpGet:
51+
path: /ping
52+
port: {{ .Values.clickhouse.port }}
53+
initialDelaySeconds: {{ .Values.clickhouse.readinessProbe.initialDelaySeconds }}
54+
periodSeconds: {{ .Values.clickhouse.readinessProbe.periodSeconds }}
55+
timeoutSeconds: {{ .Values.clickhouse.readinessProbe.timeoutSeconds }}
56+
failureThreshold: {{ .Values.clickhouse.readinessProbe.failureThreshold }}
57+
{{- end }}
3858
volumeMounts:
3959
- name: config
4060
mountPath: /etc/clickhouse-server/config.xml
@@ -153,4 +173,4 @@ spec:
153173
requests:
154174
storage: {{ .Values.clickhouse.persistence.logSize }}
155175
{{- end }}
156-
{{- end }}
176+
{{- end }}

charts/hdx-oss-v2/templates/hyperdx-deployment.yaml

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,26 @@ spec:
6464
resources:
6565
{{- toYaml .Values.hyperdx.resources | nindent 12 }}
6666
{{- end }}
67+
{{- if .Values.hyperdx.livenessProbe.enabled }}
68+
livenessProbe:
69+
httpGet:
70+
path: /health
71+
port: {{ .Values.hyperdx.apiPort }}
72+
initialDelaySeconds: {{ .Values.hyperdx.livenessProbe.initialDelaySeconds }}
73+
periodSeconds: {{ .Values.hyperdx.livenessProbe.periodSeconds }}
74+
timeoutSeconds: {{ .Values.hyperdx.livenessProbe.timeoutSeconds }}
75+
failureThreshold: {{ .Values.hyperdx.livenessProbe.failureThreshold }}
76+
{{- end }}
77+
{{- if .Values.hyperdx.readinessProbe.enabled }}
78+
readinessProbe:
79+
httpGet:
80+
path: /health
81+
port: {{ .Values.hyperdx.apiPort }}
82+
initialDelaySeconds: {{ .Values.hyperdx.readinessProbe.initialDelaySeconds }}
83+
periodSeconds: {{ .Values.hyperdx.readinessProbe.periodSeconds }}
84+
timeoutSeconds: {{ .Values.hyperdx.readinessProbe.timeoutSeconds }}
85+
failureThreshold: {{ .Values.hyperdx.readinessProbe.failureThreshold }}
86+
{{- end }}
6787
envFrom:
6888
- configMapRef:
6989
name: {{ include "hdx-oss.fullname" . }}-app-config

charts/hdx-oss-v2/templates/mongodb-deployment.yaml

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,24 @@ spec:
3131
image: "{{ .Values.mongodb.image }}"
3232
ports:
3333
- containerPort: {{ .Values.mongodb.port }}
34+
{{- if .Values.mongodb.livenessProbe.enabled }}
35+
livenessProbe:
36+
tcpSocket:
37+
port: {{ .Values.mongodb.port }}
38+
initialDelaySeconds: {{ .Values.mongodb.livenessProbe.initialDelaySeconds }}
39+
periodSeconds: {{ .Values.mongodb.livenessProbe.periodSeconds }}
40+
timeoutSeconds: {{ .Values.mongodb.livenessProbe.timeoutSeconds }}
41+
failureThreshold: {{ .Values.mongodb.livenessProbe.failureThreshold }}
42+
{{- end }}
43+
{{- if .Values.mongodb.readinessProbe.enabled }}
44+
readinessProbe:
45+
tcpSocket:
46+
port: {{ .Values.mongodb.port }}
47+
initialDelaySeconds: {{ .Values.mongodb.readinessProbe.initialDelaySeconds }}
48+
periodSeconds: {{ .Values.mongodb.readinessProbe.periodSeconds }}
49+
timeoutSeconds: {{ .Values.mongodb.readinessProbe.timeoutSeconds }}
50+
failureThreshold: {{ .Values.mongodb.readinessProbe.failureThreshold }}
51+
{{- end }}
3452
volumeMounts:
3553
- name: mongodb-data
3654
mountPath: /data/db

charts/hdx-oss-v2/templates/otel-collector-deployment.yaml

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,26 @@ spec:
4646
resources:
4747
{{- toYaml .Values.otel.resources | nindent 12 }}
4848
{{- end }}
49+
{{- if .Values.otel.livenessProbe.enabled }}
50+
livenessProbe:
51+
httpGet:
52+
path: /
53+
port: {{ .Values.otel.port }}
54+
initialDelaySeconds: {{ .Values.otel.livenessProbe.initialDelaySeconds }}
55+
periodSeconds: {{ .Values.otel.livenessProbe.periodSeconds }}
56+
timeoutSeconds: {{ .Values.otel.livenessProbe.timeoutSeconds }}
57+
failureThreshold: {{ .Values.otel.livenessProbe.failureThreshold }}
58+
{{- end }}
59+
{{- if .Values.otel.readinessProbe.enabled }}
60+
readinessProbe:
61+
httpGet:
62+
path: /
63+
port: {{ .Values.otel.port }}
64+
initialDelaySeconds: {{ .Values.otel.readinessProbe.initialDelaySeconds }}
65+
periodSeconds: {{ .Values.otel.readinessProbe.periodSeconds}}
66+
timeoutSeconds: {{ .Values.otel.readinessProbe.timeoutSeconds}}
67+
failureThreshold: {{ .Values.otel.readinessProbe.failureThreshold }}
68+
{{- end }}
4969
env:
5070
- name: CLICKHOUSE_ENDPOINT
5171
value: "{{ .Values.otel.clickhouseEndpoint | default (printf "tcp://%s-clickhouse:%v?dial_timeout=10s" (include "hdx-oss.fullname" .) .Values.clickhouse.nativePort) }}"

charts/hdx-oss-v2/tests/clickhouse-deployment_test.yaml

Lines changed: 122 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -145,4 +145,125 @@ tests:
145145
path: spec.template.spec.volumes[3].emptyDir
146146
- documentIndex: 0
147147
isNull:
148-
path: spec.template.spec.volumes[3].persistentVolumeClaim
148+
path: spec.template.spec.volumes[3].persistentVolumeClaim
149+
150+
- it: should include livenessProbe with default values when enabled
151+
set:
152+
clickhouse:
153+
enabled: true
154+
port: 8123
155+
asserts:
156+
- documentIndex: 0
157+
isSubset:
158+
path: spec.template.spec.containers[0].livenessProbe
159+
content:
160+
httpGet:
161+
path: /ping
162+
port: 8123
163+
initialDelaySeconds: 10
164+
periodSeconds: 30
165+
timeoutSeconds: 5
166+
failureThreshold: 3
167+
168+
- it: should include readinessProbe with default values when enabled
169+
set:
170+
clickhouse:
171+
enabled: true
172+
port: 8123
173+
asserts:
174+
- documentIndex: 0
175+
isSubset:
176+
path: spec.template.spec.containers[0].readinessProbe
177+
content:
178+
httpGet:
179+
path: /ping
180+
port: 8123
181+
initialDelaySeconds: 1
182+
periodSeconds: 10
183+
timeoutSeconds: 5
184+
failureThreshold: 3
185+
186+
- it: should not include livenessProbe when disabled
187+
set:
188+
clickhouse:
189+
enabled: true
190+
livenessProbe:
191+
enabled: false
192+
asserts:
193+
- documentIndex: 0
194+
isNull:
195+
path: spec.template.spec.containers[0].livenessProbe
196+
197+
- it: should not include readinessProbe when disabled
198+
set:
199+
clickhouse:
200+
enabled: true
201+
readinessProbe:
202+
enabled: false
203+
asserts:
204+
- documentIndex: 0
205+
isNull:
206+
path: spec.template.spec.containers[0].readinessProbe
207+
208+
- it: should use custom livenessProbe values when provided
209+
set:
210+
clickhouse:
211+
enabled: true
212+
port: 8123
213+
livenessProbe:
214+
enabled: true
215+
initialDelaySeconds: 20
216+
periodSeconds: 60
217+
timeoutSeconds: 10
218+
failureThreshold: 5
219+
asserts:
220+
- documentIndex: 0
221+
isSubset:
222+
path: spec.template.spec.containers[0].livenessProbe
223+
content:
224+
httpGet:
225+
path: /ping
226+
port: 8123
227+
initialDelaySeconds: 20
228+
periodSeconds: 60
229+
timeoutSeconds: 10
230+
failureThreshold: 5
231+
232+
- it: should use custom readinessProbe values when provided
233+
set:
234+
clickhouse:
235+
enabled: true
236+
port: 8123
237+
readinessProbe:
238+
enabled: true
239+
initialDelaySeconds: 5
240+
periodSeconds: 20
241+
timeoutSeconds: 3
242+
failureThreshold: 2
243+
asserts:
244+
- documentIndex: 0
245+
isSubset:
246+
path: spec.template.spec.containers[0].readinessProbe
247+
content:
248+
httpGet:
249+
path: /ping
250+
port: 8123
251+
initialDelaySeconds: 5
252+
periodSeconds: 20
253+
timeoutSeconds: 3
254+
failureThreshold: 2
255+
256+
- it: should use custom port in probes when provided
257+
set:
258+
clickhouse:
259+
enabled: true
260+
port: 8124
261+
asserts:
262+
- documentIndex: 0
263+
equal:
264+
path: spec.template.spec.containers[0].livenessProbe.httpGet.port
265+
value: 8124
266+
- documentIndex: 0
267+
equal:
268+
path: spec.template.spec.containers[0].readinessProbe.httpGet.port
269+
value: 8124

charts/hdx-oss-v2/tests/hyperdx-deployment_test.yaml

Lines changed: 99 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -128,4 +128,102 @@ tests:
128128
content: -c
129129
- matchRegex:
130130
path: spec.template.spec.initContainers[0].command[2]
131-
pattern: "until nc -z .+-mongodb [0-9]+; do echo waiting for mongodb; sleep 2; done;"
131+
pattern: "until nc -z .+-mongodb [0-9]+; do echo waiting for mongodb; sleep 2; done;"
132+
133+
- it: should include livenessProbe with default values when enabled
134+
asserts:
135+
- isSubset:
136+
path: spec.template.spec.containers[0].livenessProbe
137+
content:
138+
httpGet:
139+
path: /health
140+
port: 8000
141+
initialDelaySeconds: 10
142+
periodSeconds: 30
143+
timeoutSeconds: 5
144+
failureThreshold: 3
145+
146+
- it: should include readinessProbe with default values when enabled
147+
asserts:
148+
- isSubset:
149+
path: spec.template.spec.containers[0].readinessProbe
150+
content:
151+
httpGet:
152+
path: /health
153+
port: 8000
154+
initialDelaySeconds: 1
155+
periodSeconds: 10
156+
timeoutSeconds: 5
157+
failureThreshold: 3
158+
159+
- it: should not include livenessProbe when disabled
160+
set:
161+
hyperdx:
162+
livenessProbe:
163+
enabled: false
164+
asserts:
165+
- isNull:
166+
path: spec.template.spec.containers[0].livenessProbe
167+
168+
- it: should not include readinessProbe when disabled
169+
set:
170+
hyperdx:
171+
readinessProbe:
172+
enabled: false
173+
asserts:
174+
- isNull:
175+
path: spec.template.spec.containers[0].readinessProbe
176+
177+
- it: should use custom livenessProbe values when provided
178+
set:
179+
hyperdx:
180+
livenessProbe:
181+
enabled: true
182+
initialDelaySeconds: 20
183+
periodSeconds: 60
184+
timeoutSeconds: 10
185+
failureThreshold: 5
186+
asserts:
187+
- isSubset:
188+
path: spec.template.spec.containers[0].livenessProbe
189+
content:
190+
httpGet:
191+
path: /health
192+
port: 8000
193+
initialDelaySeconds: 20
194+
periodSeconds: 60
195+
timeoutSeconds: 10
196+
failureThreshold: 5
197+
198+
- it: should use custom readinessProbe values when provided
199+
set:
200+
hyperdx:
201+
readinessProbe:
202+
enabled: true
203+
initialDelaySeconds: 5
204+
periodSeconds: 20
205+
timeoutSeconds: 3
206+
failureThreshold: 2
207+
asserts:
208+
- isSubset:
209+
path: spec.template.spec.containers[0].readinessProbe
210+
content:
211+
httpGet:
212+
path: /health
213+
port: 8000
214+
initialDelaySeconds: 5
215+
periodSeconds: 20
216+
timeoutSeconds: 3
217+
failureThreshold: 2
218+
219+
- it: should use custom apiPort in probes when provided
220+
set:
221+
hyperdx:
222+
apiPort: 9000
223+
asserts:
224+
- equal:
225+
path: spec.template.spec.containers[0].livenessProbe.httpGet.port
226+
value: 9000
227+
- equal:
228+
path: spec.template.spec.containers[0].readinessProbe.httpGet.port
229+
value: 9000

0 commit comments

Comments
 (0)