change(syncer/uploader/deduper): decouple from gclone, deduper will use a random service account when there are service accounts associated with it

Author: l3uddz

cmd/sync.go

@@ -130,14 +130,13 @@ func worker(wg *sync.WaitGroup, jobs <-chan *syncer.Syncer) {
 func performSync(s *syncer.Syncer) error {
 	s.Log.Info("Running...")
 
-	var gcloneParams []string
-	if strings.Contains(s.GlobalConfig.Rclone.Path, "gclone") &&
-		s.RemoteServiceAccountFiles.ServiceAccountsCount() > 0 {
+	var liveRotateParams []string
+	if s.GlobalConfig.Rclone.LiveRotate && s.RemoteServiceAccountFiles.ServiceAccountsCount() > 0 {
 		// start web-server
 		s.Ws.Run()
 		defer s.Ws.Stop()
 
-		gcloneParams = append(gcloneParams,
+		liveRotateParams = append(liveRotateParams,
 			"--drive-service-account-url",
 			fmt.Sprintf("http://%s:%d", s.Ws.Host, s.Ws.Port),
 		)
@@ -147,7 +146,7 @@ func performSync(s *syncer.Syncer) error {
 	if len(s.Config.Remotes.Copy) > 0 {
 		s.Log.Info("Running copies...")
 
-		if err := s.Copy(gcloneParams); err != nil {
+		if err := s.Copy(liveRotateParams); err != nil {
 			return errors.WithMessage(err, "failed performing all copies")
 		}
 
@@ -158,7 +157,7 @@ func performSync(s *syncer.Syncer) error {
 	if len(s.Config.Remotes.Sync) > 0 {
 		s.Log.Info("Running syncs...")
 
-		if err := s.Sync(gcloneParams); err != nil {
+		if err := s.Sync(liveRotateParams); err != nil {
 			return errors.WithMessage(err, "failed performing all syncs")
 		}
 

cmd/upload.go

@@ -122,15 +122,14 @@ func init() {
 func performUpload(u *uploader.Uploader) error {
 	u.Log.Info("Running...")
 
-	var gcloneParams []string
+	var liveRotateParams []string
 
-	if strings.Contains(u.GlobalConfig.Rclone.Path, "gclone") &&
-		u.RemoteServiceAccountFiles.ServiceAccountsCount() > 0 {
+	if u.GlobalConfig.Rclone.LiveRotate && u.RemoteServiceAccountFiles.ServiceAccountsCount() > 0 {
 		// start web-server
 		u.Ws.Run()
 		defer u.Ws.Stop()
 
-		gcloneParams = append(gcloneParams,
+		liveRotateParams = append(liveRotateParams,
 			"--drive-service-account-url",
 			fmt.Sprintf("http://%s:%d", u.Ws.Host, u.Ws.Port),
 		)
@@ -152,9 +151,9 @@ func performUpload(u *uploader.Uploader) error {
 		additionalRcloneParams = u.CheckRcloneParams()
 	}
 
-	// add gclone params set
-	if len(gcloneParams) > 0 {
-		additionalRcloneParams = append(additionalRcloneParams, gcloneParams...)
+	// add live rotate params set
+	if len(liveRotateParams) > 0 {
+		additionalRcloneParams = append(additionalRcloneParams, liveRotateParams...)
 	}
 
 	/* Copies */

config/config.go

@@ -123,6 +123,7 @@ func setConfigDefaults(check bool) error {
 	// rclone settings
 	added += setConfigDefault("rclone.path", "/usr/bin/rclone", check)
 	added += setConfigDefault("rclone.config", "/Users/l3uddz/.config/rclone/rclone.conf", check)
+	added += setConfigDefault("rclone.live_rotate", false, check)
 
 	// were new settings added?
 	if check && added > 0 {

config/rclone.go

@@ -4,6 +4,7 @@ type RcloneConfig struct {
 	Path                  string
 	Config                string
 	Stats                 string
+	LiveRotate            bool                    `mapstructure:"live_rotate"`
 	DryRun                bool                    `mapstructure:"dry_run"`
 	ServiceAccountRemotes map[string]string       `mapstructure:"service_account_remotes"`
 	GlobalParams          map[string]RcloneParams `mapstructure:"global_params"`

rclone/sa.go

@@ -11,6 +11,7 @@ import (
 	"github.com/l3uddz/crop/stringutils"
 	"github.com/sirupsen/logrus"
 	"go/types"
+	"math/rand"
 	"sort"
 	"strconv"
 	"strings"
@@ -145,6 +146,30 @@ func (m *ServiceAccountManager) LoadServiceAccounts(remotePaths []string) error
 	return nil
 }
 
+func (m *ServiceAccountManager) GetRandomServiceAccount(remotePath string) (string, error) {
+	// parse remote name
+	remoteName := stringutils.FromLeftUntil(remotePath, ":")
+	if remoteName == "" {
+		// no remote name was parsed, so ignore this request
+		m.log.Tracef("No remote determined for: %q, not providing service account", remotePath)
+		return "", nil
+	}
+
+	// service accounts loaded for this remote?
+	remote, ok := m.remoteServiceAccounts[remoteName]
+	if !ok || len(remote.ServiceAccounts) == 0 {
+		// no service accounts found for this remote
+		m.log.Tracef("No service accounts loaded for remote: %q, not providing service account", remoteName)
+		return "", nil
+	}
+
+	// random service account
+	rand.Seed(time.Now().Unix())
+	sa := remote.ServiceAccounts[rand.Intn(len(remote.ServiceAccounts))]
+
+	return sa.RealPath, nil
+}
+
 func (m *ServiceAccountManager) GetServiceAccount(remotePaths ...string) ([]*RemoteServiceAccount, error) {
 	var serviceAccounts []*RemoteServiceAccount
 	var err error

syncer/copy.go

@@ -63,7 +63,7 @@ func (s *Syncer) Copy(additionalRcloneParams []string) error {
 			} else if success {
 				// successful exit code
 				if !s.Ws.Running {
-					// web service is not running (no gclone)
+					// web service is not running (no live rotate)
 					rclone.RemoveServiceAccountsFromTempCache(serviceAccounts)
 				}
 				break

syncer/dedupe.go

@@ -24,6 +24,14 @@ func (s *Syncer) Dedupe(additionalRcloneParams []string) error {
 			"dedupe_remote": dedupeRemote,
 		})
 
+		// service account
+		if s.RemoteServiceAccountFiles.ServiceAccountsCount() > 0 {
+			sa, err := s.RemoteServiceAccountFiles.GetRandomServiceAccount(dedupeRemote)
+			if err == nil && sa != "" {
+				extraParams = append(extraParams, "--drive-service-account-file", sa)
+			}
+		}
+
 		// dedupe remote
 		rLog.Info("Deduping...")
 		success, exitCode, err := rclone.Dedupe(dedupeRemote, extraParams)

syncer/sync.go

@@ -63,7 +63,7 @@ func (s *Syncer) Sync(additionalRcloneParams []string) error {
 			} else if success {
 				// successful exit code
 				if !s.Ws.Running {
-					// web service is not running (no gclone)
+					// web service is not running (no live rotate)
 					rclone.RemoveServiceAccountsFromTempCache(serviceAccounts)
 				}
 				break

uploader/dedupe.go

@@ -24,6 +24,14 @@ func (u *Uploader) Dedupe(additionalRcloneParams []string) error {
 			"dedupe_remote": dedupeRemote,
 		})
 
+		// service account
+		if u.RemoteServiceAccountFiles.ServiceAccountsCount() > 0 {
+			sa, err := u.RemoteServiceAccountFiles.GetRandomServiceAccount(dedupeRemote)
+			if err == nil && sa != "" {
+				extraParams = append(extraParams, "--drive-service-account-file", sa)
+			}
+		}
+
 		// dedupe remote
 		rLog.Info("Deduping...")
 		success, exitCode, err := rclone.Dedupe(dedupeRemote, extraParams)

web/handler.go

@@ -17,7 +17,7 @@ func (ws *Server) ServiceAccountHandler(c *fiber.Ctx) {
 	// parse body
 	req := new(ServiceAccountRequest)
 	if err := c.BodyParser(req); err != nil {
-		ws.log.WithError(err).Error("Failed parsing service account request from gclone...")
+		ws.log.WithError(err).Error("Failed parsing service account request from rclone...")
 		c.SendStatus(500)
 		return
 	}
@@ -80,9 +80,10 @@ func (ws *Server) ServiceAccountHandler(c *fiber.Ctx) {
 	ws.saCache.cache[req.OldServiceAccount] = cacheEntry
 
 	// store cache entry for the new account
-	// (so if another gclone transfer routine requests within N duration, re-issue the same sa)
+	// (so if another transfer routine requests within N duration, re-issue the same sa)
 	ws.saCache.cache[sa[0].ServiceAccountPath] = cacheEntry
 
 	// return service account
+	ws.log.Warnf("New service account for remote %q, sa: %v", req.Remote, sa[0].ServiceAccountPath)
 	c.SendString(sa[0].ServiceAccountPath)
 }