Add retention flag specific for s3

* Add retention flag specific for s3
* Add retention for the unit tests:

Signed-off-by: Vitor Savian <[email protected]>

Author: vitorsavian

pkg/cli/cmds/etcd_snapshot.go

@@ -124,6 +124,13 @@ var EtcdSnapshotFlags = []cli.Flag{
 		Destination: &ServerConfig.EtcdS3Region,
 		Value:       "us-east-1",
 	},
+	&cli.IntFlag{
+		Name:        "s3-retention",
+		Aliases:     []string{"etcd-s3-retention"},
+		Usage:       "(db) Number of s3 snapshots to retain.",
+		Destination: &ServerConfig.EtcdS3Retention,
+		Value:       defaultSnapshotRentention,
+	},
 	&cli.StringFlag{
 		Name:        "s3-folder",
 		Aliases:     []string{"etcd-s3-folder"},

pkg/cli/cmds/server.go

@@ -105,6 +105,7 @@ type Server struct {
 	EtcdS3SecretKey          string
 	EtcdS3SessionToken       string
 	EtcdS3BucketName         string
+	EtcdS3Retention          int
 	EtcdS3BucketLookupType   string
 	EtcdS3Region             string
 	EtcdS3Folder             string
@@ -480,6 +481,12 @@ var ServerFlags = []cli.Flag{
 		Usage:       "(db) S3 folder",
 		Destination: &ServerConfig.EtcdS3Folder,
 	},
+	&cli.IntFlag{
+		Name:        "etcd-s3-retention",
+		Usage:       "(db) S3 retention limit",
+		Destination: &ServerConfig.EtcdS3Retention,
+		Value:       defaultSnapshotRentention,
+	},
 	&cli.StringFlag{
 		Name:        "etcd-s3-proxy",
 		Usage:       "(db) Proxy server to use when connecting to S3, overriding any proxy-releated environment variables",

pkg/cli/etcdsnapshot/etcd_snapshot.go

@@ -65,6 +65,7 @@ func commandSetup(app *cli.Context, cfg *cmds.Server) (*etcd.SnapshotRequest, *c
 			Region:        cfg.EtcdS3Region,
 			SecretKey:     cfg.EtcdS3SecretKey,
 			SkipSSLVerify: cfg.EtcdS3SkipSSLVerify,
+			Retention:     cfg.EtcdS3Retention,
 			Timeout:       metav1.Duration{Duration: cfg.EtcdS3Timeout},
 		}
 		// extend request timeout to allow the S3 operation to complete

pkg/cli/server/server.go

@@ -210,6 +210,7 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
 				SecretKey:     cfg.EtcdS3SecretKey,
 				SessionToken:  cfg.EtcdS3SessionToken,
 				SkipSSLVerify: cfg.EtcdS3SkipSSLVerify,
+				Retention:     cfg.EtcdS3Retention,
 				Timeout:       metav1.Duration{Duration: cfg.EtcdS3Timeout},
 			}
 		}

pkg/daemons/config/types.go

@@ -76,6 +76,7 @@ type EtcdS3 struct {
 	SessionToken  string          `json:"sessionToken,omitempty"`
 	Insecure      bool            `json:"insecure,omitempty"`
 	SkipSSLVerify bool            `json:"skipSSLVerify,omitempty"`
+	Retention     int             `json:"retention,omitempty"`
 	Timeout       metav1.Duration `json:"timeout,omitempty"`
 }
 

pkg/etcd/s3/config_secret.go

@@ -53,6 +53,7 @@ func (c *Controller) getConfigFromSecret(secretName string) (*config.EtcdS3, err
 		Region:       defaultEtcdS3.Region,
 		SecretKey:    string(secret.Data["etcd-s3-secret-key"]),
 		SessionToken: string(secret.Data["etcd-s3-session-token"]),
+		Retention:    defaultEtcdS3.Retention,
 		Timeout:      *defaultEtcdS3.Timeout.DeepCopy(),
 	}
 
@@ -75,6 +76,14 @@ func (c *Controller) getConfigFromSecret(secretName string) (*config.EtcdS3, err
 		}
 	}
 
+	if v, ok := secret.Data["etcd-s3-retention"]; ok {
+		if retention, err := strconv.Atoi(string(v)); err != nil {
+			logrus.Warnf("Failed to parse etcd-s3-retention value from S3 config secret %s: %v", secretName, err)
+		} else {
+			etcdS3.Retention = retention
+		}
+	}
+
 	// configure ssl verification, if value can be parsed
 	if v, ok := secret.Data["etcd-s3-skip-ssl-verify"]; ok {
 		if b, err := strconv.ParseBool(string(v)); err != nil {

pkg/etcd/s3/s3.go

@@ -48,6 +48,7 @@ var defaultEtcdS3 = &config.EtcdS3{
 	Timeout: metav1.Duration{
 		Duration: 5 * time.Minute,
 	},
+	Retention: 5,
 }
 
 var (
@@ -386,13 +387,13 @@ func (c *Client) downloadSnapshotMetadata(ctx context.Context, key, file string)
 
 // SnapshotRetention prunes snapshots in the configured S3 compatible backend for this specific node.
 // Returns a list of pruned snapshot names.
-func (c *Client) SnapshotRetention(ctx context.Context, retention int, prefix string) ([]string, error) {
-	if retention < 1 {
+func (c *Client) SnapshotRetention(ctx context.Context, prefix string) ([]string, error) {
+	if c.etcdS3.Retention < 1 {
 		return nil, nil
 	}
 
 	prefix = path.Join(c.etcdS3.Folder, prefix)
-	logrus.Infof("Applying snapshot retention=%d to snapshots stored in s3://%s/%s", retention, c.etcdS3.Bucket, prefix)
+	logrus.Infof("Applying snapshot retention=%d to snapshots stored in s3://%s/%s", c.etcdS3.Retention, c.etcdS3.Bucket, prefix)
 
 	var snapshotFiles []minio.ObjectInfo
 
@@ -416,7 +417,7 @@ func (c *Client) SnapshotRetention(ctx context.Context, retention int, prefix st
 		snapshotFiles = append(snapshotFiles, info)
 	}
 
-	if len(snapshotFiles) <= retention {
+	if len(snapshotFiles) <= c.etcdS3.Retention {
 		return nil, nil
 	}
 
@@ -426,7 +427,7 @@ func (c *Client) SnapshotRetention(ctx context.Context, retention int, prefix st
 	})
 
 	deleted := []string{}
-	for _, df := range snapshotFiles[retention:] {
+	for _, df := range snapshotFiles[c.etcdS3.Retention:] {
 		logrus.Infof("Removing S3 snapshot: s3://%s/%s", c.etcdS3.Bucket, df.Key)
 
 		key := path.Base(df.Key)