Skip to content

Commit 7bfc19c

Browse files
dnplkndllclaude
dnplkndll
and
claude
committed
feat(helm): add backup CronJobs and configurable image registry
Add nightly S3 backup automation (CockroachDB pg_dump, MongoDB mongodump, S3-to-S3 file sync) with retention cleanup and credential rotation support. Add hulyRegistry value to allow overriding the Docker image registry prefix for all Huly service deployments (e.g. for GAR/private registry). Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1 parent c5f0d68 commit 7bfc19c

File tree

15 files changed

+382
-9
lines changed

15 files changed

+382
-9
lines changed

‎kube/helm/huly/templates/_helpers.tpl‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,25 @@ Init container that waits for Redpanda to accept connections.
112112
{{- end }}
113113
{{- end }}
114114

115+
{{/*
116+
Backup secret resource name.
117+
*/}}
118+
{{- define "huly.backupSecretName" -}}
119+
{{- printf "%s-backup-secret" (include "huly.fullname" .) }}
120+
{{- end }}
121+
122+
{{/*
123+
Env var from backup Secret helper.
124+
Usage: {{- include "huly.envBackupSecret" (dict "name" "BACKUP_S3_ENDPOINT" "key" "BACKUP_S3_ENDPOINT" "root" .) }}
125+
*/}}
126+
{{- define "huly.envBackupSecret" -}}
127+
- name: {{ .name }}
128+
valueFrom:
129+
secretKeyRef:
130+
name: {{ include "huly.backupSecretName" .root }}
131+
key: {{ .key }}
132+
{{- end }}
133+
115134
{{/*
116135
Env var from Secret helper — reduces boilerplate.
117136
Usage: {{- include "huly.envSecret" (dict "name" "SERVER_SECRET" "key" "SERVER_SECRET" "root" .) }}

‎kube/helm/huly/templates/account/deployment.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ spec:
2424
{{- include "huly.waitForRedpanda" . | nindent 8 }}
2525
containers:
2626
- name: account
27-
image: hardcoreeng/account:{{ .Values.hulyVersion }}
27+
image: {{ .Values.hulyRegistry }}/account:{{ .Values.hulyVersion }}
2828
ports:
2929
- name: http
3030
containerPort: 3000

‎kube/helm/huly/templates/backup/cronjob-cockroachdb.yaml‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
{{- if and .Values.backup.enabled .Values.backup.cockroachdb.enabled .Values.cockroach.enabled }}
2+
apiVersion: batch/v1
3+
kind: CronJob
4+
metadata:
5+
name: {{ include "huly.fullname" . }}-backup-cockroachdb
6+
labels:
7+
{{- include "huly.labels" . | nindent 4 }}
8+
app: backup-cockroachdb
9+
spec:
10+
schedule: {{ .Values.backup.cockroachdb.schedule | default .Values.backup.schedule | quote }}
11+
concurrencyPolicy: Forbid
12+
successfulJobsHistoryLimit: 3
13+
failedJobsHistoryLimit: 3
14+
jobTemplate:
15+
spec:
16+
backoffLimit: 2
17+
template:
18+
metadata:
19+
labels:
20+
{{- include "huly.labels" . | nindent 12 }}
21+
app: backup-cockroachdb
22+
spec:
23+
{{- include "huly.scheduling" . | nindent 10 }}
24+
restartPolicy: OnFailure
25+
volumes:
26+
- name: backup
27+
emptyDir: {}
28+
initContainers:
29+
- name: pg-dump
30+
image: postgres:16-alpine
31+
volumeMounts:
32+
- name: backup
33+
mountPath: /backup
34+
env:
35+
{{- include "huly.envSecret" (dict "name" "CR_DB_URL" "key" "CR_DB_URL" "root" .) | nindent 16 }}
36+
command:
37+
- sh
38+
- -c
39+
- |
40+
set -e
41+
STAMP=$(date +%Y%m%d-%H%M%S)
42+
DEST="/backup/cockroachdb_${STAMP}.sql.gz"
43+
echo "Dumping CockroachDB to ${DEST}..."
44+
pg_dump "$CR_DB_URL" | gzip > "$DEST"
45+
ls -lh "$DEST"
46+
echo "pg_dump complete."
47+
containers:
48+
- name: rclone-upload
49+
image: {{ .Values.backup.rcloneImage }}
50+
volumeMounts:
51+
- name: backup
52+
mountPath: /backup
53+
env:
54+
{{- include "huly.envBackupSecret" (dict "name" "BACKUP_S3_ENDPOINT" "key" "BACKUP_S3_ENDPOINT" "root" .) | nindent 16 }}
55+
{{- include "huly.envBackupSecret" (dict "name" "BACKUP_S3_REGION" "key" "BACKUP_S3_REGION" "root" .) | nindent 16 }}
56+
{{- include "huly.envBackupSecret" (dict "name" "BACKUP_S3_BUCKET" "key" "BACKUP_S3_BUCKET" "root" .) | nindent 16 }}
57+
{{- include "huly.envBackupSecret" (dict "name" "BACKUP_S3_PATH_PREFIX" "key" "BACKUP_S3_PATH_PREFIX" "root" .) | nindent 16 }}
58+
{{- include "huly.envBackupSecret" (dict "name" "BACKUP_S3_ACCESS_KEY" "key" "BACKUP_S3_ACCESS_KEY" "root" .) | nindent 16 }}
59+
{{- include "huly.envBackupSecret" (dict "name" "BACKUP_S3_SECRET_KEY" "key" "BACKUP_S3_SECRET_KEY" "root" .) | nindent 16 }}
60+
command:
61+
- sh
62+
- -c
63+
- |
64+
set -e
65+
# Configure rclone remote
66+
rclone config create backup s3 \
67+
provider=Other \
68+
env_auth=false \
69+
access_key_id="$BACKUP_S3_ACCESS_KEY" \
70+
secret_access_key="$BACKUP_S3_SECRET_KEY" \
71+
endpoint="$BACKUP_S3_ENDPOINT" \
72+
region="$BACKUP_S3_REGION" \
73+
--non-interactive
74+
75+
REMOTE_PATH="backup:${BACKUP_S3_BUCKET}/${BACKUP_S3_PATH_PREFIX}/cockroachdb/"
76+
77+
echo "Uploading to ${REMOTE_PATH}..."
78+
rclone copy /backup/ "$REMOTE_PATH" --include "*.sql.gz" -v
79+
80+
echo "Cleaning up backups older than {{ .Values.backup.retentionDays }} days..."
81+
rclone delete "$REMOTE_PATH" --min-age {{ .Values.backup.retentionDays }}d -v
82+
83+
echo "Backup complete."
84+
rclone ls "$REMOTE_PATH" | tail -5
85+
{{- end }}

‎kube/helm/huly/templates/backup/cronjob-files.yaml‎

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
{{- if and .Values.backup.enabled .Values.backup.files.enabled }}
2+
apiVersion: batch/v1
3+
kind: CronJob
4+
metadata:
5+
name: {{ include "huly.fullname" . }}-backup-files
6+
labels:
7+
{{- include "huly.labels" . | nindent 4 }}
8+
app: backup-files
9+
spec:
10+
schedule: {{ .Values.backup.files.schedule | default .Values.backup.schedule | quote }}
11+
concurrencyPolicy: Forbid
12+
successfulJobsHistoryLimit: 3
13+
failedJobsHistoryLimit: 3
14+
jobTemplate:
15+
spec:
16+
backoffLimit: 2
17+
template:
18+
metadata:
19+
labels:
20+
{{- include "huly.labels" . | nindent 12 }}
21+
app: backup-files
22+
spec:
23+
{{- include "huly.scheduling" . | nindent 10 }}
24+
restartPolicy: OnFailure
25+
containers:
26+
- name: rclone-sync
27+
image: {{ .Values.backup.rcloneImage }}
28+
env:
29+
{{- include "huly.envSecret" (dict "name" "STORAGE_CONFIG" "key" "STORAGE_CONFIG" "root" .) | nindent 16 }}
30+
{{- include "huly.envBackupSecret" (dict "name" "BACKUP_S3_ENDPOINT" "key" "BACKUP_S3_ENDPOINT" "root" .) | nindent 16 }}
31+
{{- include "huly.envBackupSecret" (dict "name" "BACKUP_S3_REGION" "key" "BACKUP_S3_REGION" "root" .) | nindent 16 }}
32+
{{- include "huly.envBackupSecret" (dict "name" "BACKUP_S3_BUCKET" "key" "BACKUP_S3_BUCKET" "root" .) | nindent 16 }}
33+
{{- include "huly.envBackupSecret" (dict "name" "BACKUP_S3_PATH_PREFIX" "key" "BACKUP_S3_PATH_PREFIX" "root" .) | nindent 16 }}
34+
{{- include "huly.envBackupSecret" (dict "name" "BACKUP_S3_ACCESS_KEY" "key" "BACKUP_S3_ACCESS_KEY" "root" .) | nindent 16 }}
35+
{{- include "huly.envBackupSecret" (dict "name" "BACKUP_S3_SECRET_KEY" "key" "BACKUP_S3_SECRET_KEY" "root" .) | nindent 16 }}
36+
command:
37+
- sh
38+
- -c
39+
- |
40+
set -e
41+
42+
# Parse STORAGE_CONFIG to extract source S3 details.
43+
# Format: s3|https://endpoint?accessKey=X&secretKey=Y&region=Z&rootBucket=B
44+
# or: minio|minio?accessKey=X&secretKey=Y
45+
46+
PROTO=$(echo "$STORAGE_CONFIG" | cut -d'|' -f1)
47+
REST=$(echo "$STORAGE_CONFIG" | cut -d'|' -f2)
48+
49+
if [ "$PROTO" = "minio" ]; then
50+
# MinIO: host is "minio", creds in query string
51+
SRC_ENDPOINT="http://minio:9000"
52+
SRC_PROVIDER="Minio"
53+
else
54+
# External S3: endpoint is the URL before '?'
55+
SRC_ENDPOINT=$(echo "$REST" | cut -d'?' -f1)
56+
SRC_PROVIDER="Other"
57+
fi
58+
59+
PARAMS=$(echo "$REST" | cut -d'?' -f2)
60+
SRC_ACCESS_KEY=$(echo "$PARAMS" | tr '&' '\n' | grep '^accessKey=' | cut -d= -f2)
61+
SRC_SECRET_KEY=$(echo "$PARAMS" | tr '&' '\n' | grep '^secretKey=' | cut -d= -f2)
62+
SRC_REGION=$(echo "$PARAMS" | tr '&' '\n' | grep '^region=' | cut -d= -f2)
63+
SRC_ROOT_BUCKET=$(echo "$PARAMS" | tr '&' '\n' | grep '^rootBucket=' | cut -d= -f2)
64+
SRC_BUCKET_PREFIX=$(echo "$PARAMS" | tr '&' '\n' | grep '^bucketPrefix=' | cut -d= -f2)
65+
66+
# Configure source remote
67+
rclone config create source s3 \
68+
provider="$SRC_PROVIDER" \
69+
env_auth=false \
70+
access_key_id="$SRC_ACCESS_KEY" \
71+
secret_access_key="$SRC_SECRET_KEY" \
72+
endpoint="$SRC_ENDPOINT" \
73+
region="${SRC_REGION:-us-east-1}" \
74+
--non-interactive
75+
76+
# Configure backup remote
77+
rclone config create backup s3 \
78+
provider=Other \
79+
env_auth=false \
80+
access_key_id="$BACKUP_S3_ACCESS_KEY" \
81+
secret_access_key="$BACKUP_S3_SECRET_KEY" \
82+
endpoint="$BACKUP_S3_ENDPOINT" \
83+
region="$BACKUP_S3_REGION" \
84+
--non-interactive
85+
86+
# Determine source path
87+
if [ -n "$SRC_ROOT_BUCKET" ]; then
88+
SRC_PATH="source:${SRC_ROOT_BUCKET}"
89+
elif [ -n "$SRC_BUCKET_PREFIX" ]; then
90+
echo "Warning: bucketPrefix mode — syncing all buckets with prefix '${SRC_BUCKET_PREFIX}'"
91+
SRC_PATH="source:${SRC_BUCKET_PREFIX}"
92+
else
93+
echo "Error: cannot determine source bucket from STORAGE_CONFIG"
94+
exit 1
95+
fi
96+
97+
DEST_PATH="backup:${BACKUP_S3_BUCKET}/${BACKUP_S3_PATH_PREFIX}/files/"
98+
99+
echo "Syncing files from ${SRC_PATH} to ${DEST_PATH}..."
100+
rclone sync "$SRC_PATH" "$DEST_PATH" \
101+
--transfers 8 \
102+
--checkers 16 \
103+
--fast-list \
104+
-v
105+
106+
echo "File sync complete."
107+
{{- end }}

‎kube/helm/huly/templates/backup/cronjob-mongodb.yaml‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
{{- if and .Values.backup.enabled .Values.backup.mongodb.enabled .Values.aibot.enabled }}
2+
apiVersion: batch/v1
3+
kind: CronJob
4+
metadata:
5+
name: {{ include "huly.fullname" . }}-backup-mongodb
6+
labels:
7+
{{- include "huly.labels" . | nindent 4 }}
8+
app: backup-mongodb
9+
spec:
10+
schedule: {{ .Values.backup.mongodb.schedule | default .Values.backup.schedule | quote }}
11+
concurrencyPolicy: Forbid
12+
successfulJobsHistoryLimit: 3
13+
failedJobsHistoryLimit: 3
14+
jobTemplate:
15+
spec:
16+
backoffLimit: 2
17+
template:
18+
metadata:
19+
labels:
20+
{{- include "huly.labels" . | nindent 12 }}
21+
app: backup-mongodb
22+
spec:
23+
{{- include "huly.scheduling" . | nindent 10 }}
24+
restartPolicy: OnFailure
25+
volumes:
26+
- name: backup
27+
emptyDir: {}
28+
initContainers:
29+
- name: mongodump
30+
image: {{ .Values.mongodb.image }}
31+
volumeMounts:
32+
- name: backup
33+
mountPath: /backup
34+
env:
35+
{{- include "huly.envConfig" (dict "name" "MONGO_URL" "key" "MONGO_URL" "root" .) | nindent 16 }}
36+
command:
37+
- sh
38+
- -c
39+
- |
40+
set -e
41+
STAMP=$(date +%Y%m%d-%H%M%S)
42+
DEST="/backup/mongodb_${STAMP}.gz"
43+
echo "Dumping MongoDB to ${DEST}..."
44+
mongodump --uri="$MONGO_URL" --archive="$DEST" --gzip
45+
ls -lh "$DEST"
46+
echo "mongodump complete."
47+
containers:
48+
- name: rclone-upload
49+
image: {{ .Values.backup.rcloneImage }}
50+
volumeMounts:
51+
- name: backup
52+
mountPath: /backup
53+
env:
54+
{{- include "huly.envBackupSecret" (dict "name" "BACKUP_S3_ENDPOINT" "key" "BACKUP_S3_ENDPOINT" "root" .) | nindent 16 }}
55+
{{- include "huly.envBackupSecret" (dict "name" "BACKUP_S3_REGION" "key" "BACKUP_S3_REGION" "root" .) | nindent 16 }}
56+
{{- include "huly.envBackupSecret" (dict "name" "BACKUP_S3_BUCKET" "key" "BACKUP_S3_BUCKET" "root" .) | nindent 16 }}
57+
{{- include "huly.envBackupSecret" (dict "name" "BACKUP_S3_PATH_PREFIX" "key" "BACKUP_S3_PATH_PREFIX" "root" .) | nindent 16 }}
58+
{{- include "huly.envBackupSecret" (dict "name" "BACKUP_S3_ACCESS_KEY" "key" "BACKUP_S3_ACCESS_KEY" "root" .) | nindent 16 }}
59+
{{- include "huly.envBackupSecret" (dict "name" "BACKUP_S3_SECRET_KEY" "key" "BACKUP_S3_SECRET_KEY" "root" .) | nindent 16 }}
60+
command:
61+
- sh
62+
- -c
63+
- |
64+
set -e
65+
rclone config create backup s3 \
66+
provider=Other \
67+
env_auth=false \
68+
access_key_id="$BACKUP_S3_ACCESS_KEY" \
69+
secret_access_key="$BACKUP_S3_SECRET_KEY" \
70+
endpoint="$BACKUP_S3_ENDPOINT" \
71+
region="$BACKUP_S3_REGION" \
72+
--non-interactive
73+
74+
REMOTE_PATH="backup:${BACKUP_S3_BUCKET}/${BACKUP_S3_PATH_PREFIX}/mongodb/"
75+
76+
echo "Uploading to ${REMOTE_PATH}..."
77+
rclone copy /backup/ "$REMOTE_PATH" --include "*.gz" -v
78+
79+
echo "Cleaning up backups older than {{ .Values.backup.retentionDays }} days..."
80+
rclone delete "$REMOTE_PATH" --min-age {{ .Values.backup.retentionDays }}d -v
81+
82+
echo "Backup complete."
83+
rclone ls "$REMOTE_PATH" | tail -5
84+
{{- end }}

‎kube/helm/huly/templates/backup/secret.yaml‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
{{- if .Values.backup.enabled }}
2+
{{- $secretName := include "huly.backupSecretName" . -}}
3+
{{- $existing := lookup "v1" "Secret" .Release.Namespace $secretName -}}
4+
{{- $hasExisting := not (empty $existing) -}}
5+
6+
{{- /* Resolve active credential set */ -}}
7+
{{- $accessKey := "" -}}
8+
{{- $secretKey := "" -}}
9+
{{- if eq .Values.backup.s3.activeCredential "secondary" -}}
10+
{{- $accessKey = .Values.backup.s3.secondaryAccessKey -}}
11+
{{- $secretKey = .Values.backup.s3.secondarySecretKey -}}
12+
{{- else -}}
13+
{{- $accessKey = .Values.backup.s3.accessKey -}}
14+
{{- $secretKey = .Values.backup.s3.secretKey -}}
15+
{{- end -}}
16+
17+
{{- /* Fall back to existing secret if keys are empty */ -}}
18+
{{- if and (not $accessKey) $hasExisting (hasKey $existing.data "BACKUP_S3_ACCESS_KEY") -}}
19+
{{- $accessKey = index $existing.data "BACKUP_S3_ACCESS_KEY" | b64dec -}}
20+
{{- end -}}
21+
{{- if and (not $secretKey) $hasExisting (hasKey $existing.data "BACKUP_S3_SECRET_KEY") -}}
22+
{{- $secretKey = index $existing.data "BACKUP_S3_SECRET_KEY" | b64dec -}}
23+
{{- end -}}
24+
25+
{{- if or (not $accessKey) (not $secretKey) -}}
26+
{{- fail "backup.s3.accessKey and backup.s3.secretKey are required when backup.enabled=true" }}
27+
{{- end -}}
28+
29+
apiVersion: v1
30+
kind: Secret
31+
metadata:
32+
name: {{ $secretName }}
33+
labels:
34+
{{- include "huly.labels" . | nindent 4 }}
35+
type: Opaque
36+
data:
37+
BACKUP_S3_ENDPOINT: {{ .Values.backup.s3.endpoint | b64enc | quote }}
38+
BACKUP_S3_REGION: {{ .Values.backup.s3.region | b64enc | quote }}
39+
BACKUP_S3_BUCKET: {{ .Values.backup.s3.bucket | b64enc | quote }}
40+
BACKUP_S3_PATH_PREFIX: {{ .Values.backup.s3.pathPrefix | b64enc | quote }}
41+
BACKUP_S3_ACCESS_KEY: {{ $accessKey | b64enc | quote }}
42+
BACKUP_S3_SECRET_KEY: {{ $secretKey | b64enc | quote }}
43+
{{- end }}

‎kube/helm/huly/templates/collaborator/deployment.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ spec:
2121
{{- include "huly.scheduling" . | nindent 6 }}
2222
containers:
2323
- name: collaborator
24-
image: hardcoreeng/collaborator:{{ .Values.hulyVersion }}
24+
image: {{ .Values.hulyRegistry }}/collaborator:{{ .Values.hulyVersion }}
2525
ports:
2626
- name: http
2727
containerPort: 3078

‎kube/helm/huly/templates/front/deployment.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ spec:
2121
{{- include "huly.scheduling" . | nindent 6 }}
2222
containers:
2323
- name: front
24-
image: hardcoreeng/front:{{ .Values.hulyVersion }}
24+
image: {{ .Values.hulyRegistry }}/front:{{ .Values.hulyVersion }}
2525
ports:
2626
- name: http
2727
containerPort: 8080

‎kube/helm/huly/templates/fulltext/deployment.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ spec:
2424
{{- include "huly.waitForRedpanda" . | nindent 8 }}
2525
containers:
2626
- name: fulltext
27-
image: hardcoreeng/fulltext:{{ .Values.hulyVersion }}
27+
image: {{ .Values.hulyRegistry }}/fulltext:{{ .Values.hulyVersion }}
2828
ports:
2929
- name: http
3030
containerPort: 4700

‎kube/helm/huly/templates/kvs/deployment.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ spec:
2424
{{- include "huly.waitForCockroach" . | nindent 8 }}
2525
containers:
2626
- name: kvs
27-
image: hardcoreeng/hulykvs:{{ .Values.hulyVersion }}
27+
image: {{ .Values.hulyRegistry }}/hulykvs:{{ .Values.hulyVersion }}
2828
ports:
2929
- name: http
3030
containerPort: 8094

0 commit comments

Comments
 (0)