K8SPSMDB-1425: Add pitr-to-new-cluster e2e test

e2e-tests/pitr-to-new-cluster/compare/find-2nd.json

@@ -0,0 +1,6 @@
+switched to db myApp
+{ "_id" : , "x" : 100500 }
+{ "_id" : , "x" : 100501 }
+{ "_id" : , "x" : 100502 }
+{ "_id" : , "x" : 100503 }
+bye

e2e-tests/pitr-to-new-cluster/compare/find.json

@@ -0,0 +1,3 @@
+switched to db myApp
+{ "_id" : , "x" : 100500 }
+bye

e2e-tests/pitr-to-new-cluster/compare/pbm_config.yml

@@ -0,0 +1,21 @@
+storage:
+  type: s3
+  s3:
+    region: us-east-1
+    endpointUrl: http://minio-service:9000/
+    forcePathStyle: true
+    bucket: operator-testing
+    prefix: some-name-target
+    credentials:
+      access-key-id: '***'
+      secret-access-key: '***'
+    maxUploadParts: 10000
+    storageClass: STANDARD
+    insecureSkipTLSVerify: false
+pitr:
+  enabled: false
+  compression: s2
+backup:
+  oplogSpanMin: 0
+  compression: s2
+restore: {}

e2e-tests/pitr-to-new-cluster/conf/backup.yml

@@ -0,0 +1,8 @@
+apiVersion: psmdb.percona.com/v1
+kind: PerconaServerMongoDBBackup
+metadata:
+  name:
+spec:
+  clusterName:
+  storageName: minio
+  type: 

e2e-tests/pitr-to-new-cluster/conf/restore.yml

@@ -0,0 +1,19 @@
+apiVersion: psmdb.percona.com/v1
+kind: PerconaServerMongoDBRestore
+metadata:
+  name:
+spec:
+  clusterName:
+  pitr:
+    pitrType: latest
+  backupSource:
+    type:
+    destination: s3://DESTINATION
+    s3:
+      credentialsSecret: minio-secret
+      bucket: operator-testing
+      prefix: some-name-source
+      endpointUrl: http://minio-service:9000/
+      region: us-east-1
+      insecureSkipTLSVerify: false
+

e2e-tests/pitr-to-new-cluster/conf/some-name-source.yml

@@ -0,0 +1,64 @@
+apiVersion: psmdb.percona.com/v1
+kind: PerconaServerMongoDB
+metadata:
+  name: some-name-source
+spec:
+  #platform: openshift
+  image:
+  imagePullPolicy: Always
+  backup:
+    enabled: true
+    image: perconalab/percona-server-mongodb-operator:main-backup
+    storages:
+      minio:
+        type: s3
+        s3:
+          credentialsSecret: minio-secret
+          region: us-east-1
+          bucket: operator-testing
+          prefix: some-name-source
+          endpointUrl: http://minio-service:9000/
+          insecureSkipTLSVerify: false
+    pitr:
+      enabled: true
+      oplogSpanMin: 1
+  sharding:
+    enabled: false
+  replsets:
+  - name: rs0
+    affinity:
+      antiAffinityTopologyKey: none
+    resources:
+      limits:
+        memory: 4G
+      requests:
+        memory: 1G
+    volumeSpec:
+      persistentVolumeClaim:
+        resources:
+          requests:
+            storage: 1Gi
+    size: 3
+    configuration: |
+      operationProfiling:
+        mode: slowOp
+        slowOpThresholdMs: 100
+      security:
+        enableEncryption: false
+        redactClientLogData: false
+      setParameter:
+        ttlMonitorSleepSecs: 60
+        wiredTigerConcurrentReadTransactions: 128
+        wiredTigerConcurrentWriteTransactions: 128
+      storage:
+        engine: wiredTiger
+        wiredTiger:
+          collectionConfig:
+            blockCompressor: snappy
+          engineConfig:
+            directoryForIndexes: false
+            journalCompressor: snappy
+          indexConfig:
+            prefixCompression: true
+  secrets:
+    users: some-users

e2e-tests/pitr-to-new-cluster/conf/some-name-target.yml

@@ -0,0 +1,64 @@
+apiVersion: psmdb.percona.com/v1
+kind: PerconaServerMongoDB
+metadata:
+  name: some-name-target
+spec:
+  #platform: openshift
+  image:
+  imagePullPolicy: Always
+  backup:
+    enabled: true
+    image: perconalab/percona-server-mongodb-operator:main-backup
+    storages:
+      minio:
+        type: s3
+        s3:
+          credentialsSecret: minio-secret
+          region: us-east-1
+          bucket: operator-testing
+          prefix: some-name-target
+          endpointUrl: http://minio-service:9000/
+          insecureSkipTLSVerify: false
+    pitr:
+      enabled: false
+      oplogSpanMin: 1
+  sharding:
+    enabled: false
+  replsets:
+  - name: rs0
+    affinity:
+      antiAffinityTopologyKey: none
+    resources:
+      limits:
+        memory: 4G
+      requests:
+        memory: 1G
+    volumeSpec:
+      persistentVolumeClaim:
+        resources:
+          requests:
+            storage: 1Gi
+    size: 3
+    configuration: |
+      operationProfiling:
+        mode: slowOp
+        slowOpThresholdMs: 100
+      security:
+        enableEncryption: false
+        redactClientLogData: false
+      setParameter:
+        ttlMonitorSleepSecs: 60
+        wiredTigerConcurrentReadTransactions: 128
+        wiredTigerConcurrentWriteTransactions: 128
+      storage:
+        engine: wiredTiger
+        wiredTiger:
+          collectionConfig:
+            blockCompressor: snappy
+          engineConfig:
+            directoryForIndexes: false
+            journalCompressor: snappy
+          indexConfig:
+            prefixCompression: true
+  secrets:
+    users: some-users

e2e-tests/pitr-to-new-cluster/run

@@ -0,0 +1,144 @@
+#!/bin/bash
+
+# Regression test for K8SPSMDB-1425
+
+set -o errexit
+set -o pipefail
+
+test_dir=$(realpath $(dirname $0))
+. ${test_dir}/../functions
+set_debug
+
+create_user() {
+	local cluster=$1
+
+	log 'create user myApp'
+	run_mongo \
+		'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' \
+		"userAdmin:userAdmin123456@${cluster}-rs0.${namespace}" \
+		"mongodb"
+}
+
+write_document() {
+	local cluster=$1
+	local x=$2
+
+	log "write document: ${x}"
+	run_mongo \
+		"use myApp\n db.test.insert({ x: ${x} })" \
+		"myApp:myPass@${cluster}-rs0.${namespace}" \
+		"mongodb"
+}
+
+run_backup() {
+	local cluster=$1
+	local name=$2
+	local type=$3
+
+	desc "run backup ${name}-${type}"
+	cat ${test_dir}/conf/backup.yml \
+		| $sed -e "s/name:/name: ${name}-${type}/" \
+		| $sed -e "s/clusterName:/clusterName: ${cluster}/" \
+		| $sed -e "s/type:/type: ${type}/" \
+		| kubectl_bin apply -f -
+
+	wait_backup "${name}-${type}"
+}
+
+run_restore() {
+	local cluster=$1
+	local backup_name=$2
+	local backup_type=$3
+	local destination=$(get_backup_dest ${backup_name}-${backup_type})
+
+	cat ${test_dir}/conf/restore.yml \
+		| $sed -e "s/name:/name: restore-${backup_name}-${backup_type}/" \
+		| $sed -e "s/clusterName:/clusterName: ${cluster}/" \
+		| $sed -e "s/type:/type: ${backup_type}/" \
+		| $sed -e "s/pitrType:/type:/" \
+		| $sed -e "s|DESTINATION|${destination}|" \
+		| kubectl_bin apply -f -
+}
+
+compare_pbm_config() {
+	local cluster=$1
+
+	kubectl_bin exec ${cluster}-rs0-0 -c backup-agent -- pbm config > ${tmp_dir}/pbm_config.yml
+
+	diff -u ${test_dir}/compare/pbm_config.yml ${tmp_dir}/pbm_config.yml
+}
+
+source_cluster="some-name-source"
+target_cluster="some-name-target"
+
+create_infra $namespace
+deploy_minio
+
+desc 'create secrets and start client'
+kubectl_bin apply \
+	-f ${conf_dir}/secrets.yml \
+	-f ${conf_dir}/client.yml \
+	-f ${conf_dir}/minio-secret.yml
+
+desc "create PSMDB cluster: ${source_cluster}"
+apply_cluster ${test_dir}/conf/${source_cluster}.yml
+
+wait_for_running ${source_cluster}-rs0 3
+
+wait_backup_agent ${source_cluster}-rs0-0
+wait_backup_agent ${source_cluster}-rs0-1
+wait_backup_agent ${source_cluster}-rs0-2
+
+create_user ${source_cluster}
+write_document ${source_cluster} 100500
+
+sleep_with_log 360 "wait for PBM-1265 workaround"
+
+compare_mongo_cmd "find" "myApp:myPass@${source_cluster}-rs0.${namespace}"
+
+backup_name_minio="backup-minio"
+
+run_backup ${source_cluster} ${backup_name_minio} logical
+
+sleep_with_log 70 "wait for oplog chunks to be uploaded"
+
+run_backup ${source_cluster} ${backup_name_minio} physical
+
+write_document ${source_cluster} 100501
+write_document ${source_cluster} 100502
+write_document ${source_cluster} 100503
+
+sleep_with_log 70 "wait for oplog chunks to be uploaded"
+
+compare_mongo_cmd "find" "myApp:myPass@${source_cluster}-rs0.${namespace}" "-2nd"
+
+desc "create PSMDB cluster: ${target_cluster}"
+apply_cluster ${test_dir}/conf/${target_cluster}.yml
+
+wait_for_running ${target_cluster}-rs0 3
+
+wait_backup_agent ${target_cluster}-rs0-0
+wait_backup_agent ${target_cluster}-rs0-1
+wait_backup_agent ${target_cluster}-rs0-2
+
+run_restore ${target_cluster} ${backup_name_minio} logical
+wait_restore "${backup_name_minio}-logical" "${target_cluster}" "ready"
+
+compare_mongo_cmd "find" "myApp:myPass@${target_cluster}-rs0.${namespace}" "-2nd"
+log "Data is restored from ${backup_name_minio}-logical: OK"
+
+compare_pbm_config ${target_cluster}
+log "Main storage is reverted after ${backup_name_minio}-logical: OK"
+
+run_restore ${target_cluster} ${backup_name_minio} physical
+wait_restore "${backup_name_minio}-physical" "${target_cluster}" "ready"
+
+compare_mongo_cmd "find" "myApp:myPass@${target_cluster}-rs0.${namespace}" "-2nd"
+log "Data is restored from ${backup_name_minio}-physical: OK"
+
+compare_pbm_config ${target_cluster}
+log "Main storage is reverted after ${backup_name_minio}-physical: OK"
+
+desc "test passed"
+
+destroy $namespace

e2e-tests/run-backups.csv

@@ -1,4 +1,5 @@
 demand-backup
+demand-backup-eks-credentials
 demand-backup-eks-credentials-irsa
 demand-backup-fs
 demand-backup-incremental
@@ -14,4 +15,8 @@ demand-backup-physical-sharded-azure
 demand-backup-physical-sharded-gcp
 demand-backup-physical-sharded-minio
 demand-backup-sharded
+pitr-physical
 pitr-physical-backup-source
+pitr-sharded
+pitr-to-new-cluster
+scheduled-backup

e2e-tests/run-pr.csv

@@ -43,6 +43,7 @@ operator-self-healing-chaos
 pitr
 pitr-physical
 pitr-sharded
+pitr-to-new-cluster
 pitr-physical-backup-source
 preinit-updates
 pvc-resize

e2e-tests/run-release.csv

@@ -43,6 +43,7 @@ operator-self-healing-chaos
 pitr
 pitr-physical
 pitr-sharded
+pitr-to-new-cluster
 pitr-physical-backup-source
 preinit-updates
 pvc-resize