add support for banning standalone remotes and dont proceed with upload if any of them are banned

Author: l3uddz

cache/get.go

@@ -4,7 +4,8 @@ import (
 	"time"
 )
 
-func Get(key string) bool {
+func Get(key string) (bool, time.Time) {
+	var expires time.Time
 	changes := false
 	exists := false
 
@@ -24,6 +25,7 @@ func Get(key string) bool {
 		} else {
 			// the item has not expired
 			exists = true
+			expires = expiry
 		}
 	}
 
@@ -35,5 +37,5 @@ func Get(key string) bool {
 		}
 	}
 
-	return exists
+	return exists, expires
 }

cmd/upload.go

@@ -1,9 +1,11 @@
 package cmd
 
 import (
+	"github.com/dustin/go-humanize"
 	"github.com/l3uddz/crop/config"
 	"github.com/l3uddz/crop/uploader"
 	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"github.com/yale8848/gorpool"
 )
@@ -27,7 +29,6 @@ var uploadCmd = &cobra.Command{
 			}
 
 			log := log.WithField("uploader", uploaderName)
-			log.Info("Uploader commencing...")
 
 			// create uploader
 			upload, err := uploader.New(config.Config, &uploaderConfig, uploaderName)
@@ -40,8 +41,32 @@ var uploadCmd = &cobra.Command{
 			if upload.ServiceAccountCount > 0 {
 				upload.Log.WithField("found_files", upload.ServiceAccountCount).
 					Info("Loaded service accounts")
+			} else {
+				// no service accounts were loaded
+				// check to see if any of the copy or move remote(s) are banned
+				banned, expiry := upload.RemotesBanned(upload.Config.Remotes.Copy)
+				if banned && !expiry.IsZero() {
+					// one of the copy remotes is banned, abort
+					upload.Log.WithFields(logrus.Fields{
+						"expires_time": expiry,
+						"expires_in":   humanize.Time(expiry),
+					}).Warn("Cannot proceed with upload as a copy remote is banned")
+					continue
+				}
+
+				banned, expiry = upload.RemotesBanned([]string{upload.Config.Remotes.Move})
+				if banned && !expiry.IsZero() {
+					// the move remote is banned, abort
+					upload.Log.WithFields(logrus.Fields{
+						"expires_time": expiry,
+						"expires_in":   humanize.Time(expiry),
+					}).Warn("Cannot proceed with upload as the move remote is banned")
+					continue
+				}
 			}
 
+			log.Info("Uploader commencing...")
+
 			// refresh details about files to upload
 			if err := upload.RefreshLocalFiles(); err != nil {
 				upload.Log.WithError(err).Error("Failed refreshing details of files to upload")

stringutils/until.go

@@ -0,0 +1,12 @@
+package stringutils
+
+import "strings"
+
+func FromLeftUntil(from string, sub string) string {
+	pos := strings.Index(from, sub)
+	if pos < 0 {
+		return from
+	}
+
+	return from[:pos]
+}

uploader/sa.go uploader/cache.gouploader/sa.go renamed to uploader/cache.go

@@ -3,7 +3,9 @@ package uploader
 import (
 	"github.com/l3uddz/crop/cache"
 	"github.com/l3uddz/crop/pathutils"
+	"github.com/l3uddz/crop/stringutils"
 	"github.com/pkg/errors"
+	"time"
 )
 
 /* Private */
@@ -13,7 +15,7 @@ func (u *Uploader) getAvailableServiceAccount() (*pathutils.Path, error) {
 	// find an unbanned service account
 	for _, sa := range u.ServiceAccountFiles {
 		// does the cache already contain this service account?
-		if cache.Get(sa.RealPath) {
+		if exists, _ := cache.Get(sa.RealPath); exists {
 			// service account is currently banned
 			continue
 		}
@@ -30,3 +32,30 @@ func (u *Uploader) getAvailableServiceAccount() (*pathutils.Path, error) {
 
 	return serviceAccountFile, nil
 }
+
+/* Public */
+func (u *Uploader) RemotesBanned(remotes []string) (bool, time.Time) {
+	var banned bool
+	var expires time.Time
+
+	// ignore empty remotes slice
+	if remotes == nil {
+		return banned, expires
+	}
+
+	// format remotes into remote names if possible
+	var checkRemotes []string
+	for _, remote := range remotes {
+		checkRemotes = append(checkRemotes, stringutils.FromLeftUntil(remote, ":"))
+	}
+
+	// iterate remotes
+	for _, remote := range checkRemotes {
+		banned, expires = cache.Get(remote)
+		if banned {
+			break
+		}
+	}
+
+	return banned, expires
+}

uploader/copy.go

@@ -5,6 +5,7 @@ import (
 	"github.com/l3uddz/crop/cache"
 	"github.com/l3uddz/crop/pathutils"
 	"github.com/l3uddz/crop/rclone"
+	"github.com/l3uddz/crop/stringutils"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"time"
@@ -58,14 +59,20 @@ func (u *Uploader) Copy(additionalRcloneParams []string) error {
 				break
 			}
 
-			// are we using service accounts?
-			if u.ServiceAccountCount == 0 {
-				return fmt.Errorf("copy failed with exit code: %v", exitCode)
-			}
-
 			// is this an exit code we can retry?
 			switch exitCode {
 			case rclone.EXIT_FATAL_ERROR:
+				// are we using service accounts?
+				if u.ServiceAccountCount == 0 {
+					// we are not using service accounts, so mark this remote as banned
+					if err := cache.Set(stringutils.FromLeftUntil(remotePath, ":"),
+						time.Now().UTC().Add(25*time.Hour)); err != nil {
+						rLog.WithError(err).Errorf("Failed banning remote")
+					}
+
+					return fmt.Errorf("copy failed with exit code: %v", exitCode)
+				}
+
 				// ban this service account
 				if err := cache.Set(serviceAccount.RealPath, time.Now().UTC().Add(25*time.Hour)); err != nil {
 					rLog.WithError(err).Error("Failed banning service account, cannot try again...")

uploader/move.go

@@ -5,6 +5,7 @@ import (
 	"github.com/l3uddz/crop/cache"
 	"github.com/l3uddz/crop/pathutils"
 	"github.com/l3uddz/crop/rclone"
+	"github.com/l3uddz/crop/stringutils"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"time"
@@ -89,14 +90,22 @@ func (u *Uploader) Move(serverSide bool, additionalRcloneParams []string) error
 				return fmt.Errorf("failed and cannot proceed with exit code: %v", exitCode)
 			}
 
-			// are we using service accounts?
-			if u.ServiceAccountCount == 0 {
-				return fmt.Errorf("move failed with exit code: %v", exitCode)
-			}
-
 			// is this an exit code we can retry?
 			switch exitCode {
 			case rclone.EXIT_FATAL_ERROR:
+				// are we using service accounts?
+				if u.ServiceAccountCount == 0 {
+					// we are not using service accounts, so mark this remote as banned (if non server side move)
+					if !serverSide {
+						if err := cache.Set(stringutils.FromLeftUntil(move.To, ":"),
+							time.Now().UTC().Add(25*time.Hour)); err != nil {
+							rLog.WithError(err).Errorf("Failed banning remote")
+						}
+					}
+
+					return fmt.Errorf("move failed with exit code: %v", exitCode)
+				}
+
 				// ban this service account
 				if err := cache.Set(serviceAccount.RealPath, time.Now().UTC().Add(25*time.Hour)); err != nil {
 					rLog.WithError(err).Error("Failed banning service account, cannot try again...")