Merge pull request #2 from l3uddz/feat/service-account-env-vars

Feat/service account env vars

Author: l3uddz

README.md

@@ -16,6 +16,13 @@ rclone:
   config: /home/seed/.config/rclone/rclone.conf
   path: /usr/bin/rclone
   stats: 30s
+  service_account_remotes:
+    tv: /opt/rclone/service_accounts/crop
+    movies: /opt/rclone/service_accounts/crop
+    music: /opt/rclone/service_accounts/crop
+    4k_movies: /opt/rclone/service_accounts/crop
+    source_4k_movies: /opt/rclone/service_accounts/crop
+    staging: /opt/rclone/service_accounts/staging
 uploader:
   google:
     check:
@@ -36,7 +43,6 @@ uploader:
       move_server_side:
         - from: 'staging:/Media'
           to: 'gdrive:/Media'
-    sa_folder: /opt/service_account_maker/service_accounts/crop
     rclone_params:
       move:
         - '--transfers=8'
@@ -45,10 +51,35 @@ uploader:
         - '--delete-empty-src-dirs'
       dedupe:
         - '--tpslimit=50'
+  tv:
+    enabled: true
+    check:
+      limit: 1440
+      type: age
+    local_folder: /mnt/local/Media/TV
+    remotes:
+      move: 'tv:/Media/TV'
+    rclone_params:
+      move:
+        - '--order-by=modtime,ascending'
+        - '--transfers=8'
+        - '--delete-empty-src-dirs'
+  movies:
+    enabled: true
+    check:
+      limit: 720
+      type: age
+    local_folder: /mnt/local/Media/Movies
+    remotes:
+      move: 'movies:/Media/Movies'
+    rclone_params:
+      move:
+        - '--order-by=modtime,ascending'
+        - '--transfers=8'
+        - '--delete-empty-src-dirs'
 syncer:
   4k_movies:
     enabled: true
-    sa_folder: /opt/service_account_maker/service_accounts/crop
     source_remote: 'source_4k_movies:/'
     remotes:
       sync:

cache/cache.go

@@ -1,38 +1,51 @@
 package cache
 
 import (
-	jsoniter "github.com/json-iterator/go"
 	"github.com/l3uddz/crop/logger"
 	"github.com/l3uddz/crop/stringutils"
-	"sync"
-	"time"
+	"github.com/pkg/errors"
+	"github.com/zippoxer/bow"
 )
 
 var (
 	log           = logger.GetLogger("cache")
 	cacheFilePath string
-	mtx           sync.Mutex
-	vault         map[string]time.Time
 
 	// Internal
-	json = jsoniter.ConfigCompatibleWithStandardLibrary
+	db *bow.DB
 )
 
 /* Public */
 
-func Init(cachePath string) error {
+func Init(cachePath string, logLevel int) error {
 	// set globals
 	cacheFilePath = cachePath
-	vault = make(map[string]time.Time)
 
-	// load cache from disk
-	if err := loadFromFile(cachePath); err != nil {
-		return err
+	// set badger options
+	opts := make([]bow.Option, 0)
+
+	if logLevel < 2 {
+		// disable badger logging for non trace log level
+		opts = append(opts, bow.SetLogger(nil))
+	}
+
+	// init database
+	v, err := bow.Open(cachePath, opts...)
+	if err != nil {
+		return errors.WithMessage(err, "failed opening cache")
 	}
 
+	db = v
+
 	return nil
 }
 
+func Close() {
+	if err := db.Close(); err != nil {
+		log.WithError(err).Error("Failed closing cache gracefully...")
+	}
+}
+
 func ShowUsing() {
 	log.Infof("Using %s = %q", stringutils.LeftJust("CACHE", " ", 10), cacheFilePath)
 }

cache/dump.go

@@ -0,0 +1,52 @@
+package cache
+
+import (
+	"github.com/zippoxer/bow"
+	"time"
+)
+
+type Banned struct {
+	Path    string `bow:"key"`
+	Expires time.Time
+}
+
+func IsBanned(key string) (bool, time.Time) {
+	// check if key was found in banned bucket
+	var item Banned
+	err := db.Bucket("banned").Get(key, &item)
+
+	// was key not found
+	if err == bow.ErrNotFound {
+		// this key is not banned
+		return false, time.Time{}
+	} else if err != nil {
+		log.WithError(err).Errorf("Failed checking banned bucket for: %q", key)
+		return false, time.Time{}
+	}
+
+	// check if the ban has expired
+	if item.Expires.Before(time.Now().UTC()) {
+		// the ban has expired, remove
+		log.Warnf("Expired %q: %v", key, item.Expires)
+
+		err := db.Bucket("banned").Delete(key)
+		if err != nil {
+			log.WithError(err).Errorf("Failed removing from banned bucket: %q", key)
+			return false, time.Time{}
+		}
+
+		return false, time.Time{}
+	}
+
+	// this key is still banned
+	return true, item.Expires
+}
+
+func SetBanned(key string, hours int) error {
+	expiry := time.Now().UTC().Add(time.Duration(hours) * time.Hour)
+
+	return db.Bucket("banned").Put(Banned{
+		Path:    key,
+		Expires: expiry,
+	})
+}

cache/get.go

@@ -20,6 +20,7 @@ var cleanCmd = &cobra.Command{
 
 		// iterate uploader's
 		for uploaderName, uploaderConfig := range config.Config.Uploader {
+			uploaderConfig := uploaderConfig
 			log := log.WithField("uploader", uploaderName)
 
 			// skip disabled uploader(s)

cache/load.go

@@ -20,7 +20,7 @@ var (
 	flagLogLevel     = 0
 	flagConfigFolder = pathutils.GetCurrentBinaryPath()
 	flagConfigFile   = "config.yaml"
-	flagCacheFile    = "cache.json"
+	flagCachePath    = "cache"
 	flagLogFile      = "activity.log"
 
 	flagDryRun bool
@@ -50,7 +50,7 @@ func init() {
 	// Parse persistent flags
 	rootCmd.PersistentFlags().StringVar(&flagConfigFolder, "config-dir", flagConfigFolder, "Config folder")
 	rootCmd.PersistentFlags().StringVarP(&flagConfigFile, "config", "c", flagConfigFile, "Config file")
-	rootCmd.PersistentFlags().StringVarP(&flagCacheFile, "cache", "d", flagCacheFile, "Cache file")
+	rootCmd.PersistentFlags().StringVarP(&flagCachePath, "cache", "d", flagCachePath, "Cache path")
 	rootCmd.PersistentFlags().StringVarP(&flagLogFile, "log", "l", flagLogFile, "Log file")
 	rootCmd.PersistentFlags().CountVarP(&flagLogLevel, "verbose", "v", "Verbose level")
 
@@ -63,7 +63,7 @@ func initCore(showAppInfo bool) {
 		flagConfigFile = filepath.Join(flagConfigFolder, flagConfigFile)
 	}
 	if !rootCmd.PersistentFlags().Changed("cache") {
-		flagCacheFile = filepath.Join(flagConfigFolder, flagCacheFile)
+		flagCachePath = filepath.Join(flagConfigFolder, flagCachePath)
 	}
 	if !rootCmd.PersistentFlags().Changed("log") {
 		flagLogFile = filepath.Join(flagConfigFolder, flagLogFile)
@@ -84,7 +84,7 @@ func initCore(showAppInfo bool) {
 	setConfigOverrides()
 
 	// Init Cache
-	if err := cache.Init(flagCacheFile); err != nil {
+	if err := cache.Init(flagCachePath, flagLogLevel); err != nil {
 		log.WithError(err).Fatal("Failed to initialize cache")
 	}
 

cache/sa.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"github.com/dustin/go-humanize"
+	"github.com/l3uddz/crop/cache"
 	"github.com/l3uddz/crop/config"
 	"github.com/l3uddz/crop/rclone"
 	"github.com/l3uddz/crop/syncer"
@@ -23,9 +24,11 @@ var syncCmd = &cobra.Command{
 	Run: func(cmd *cobra.Command, args []string) {
 		// init core
 		initCore(true)
+		defer cache.Close()
 
 		// iterate syncer's
 		for syncerName, syncerConfig := range config.Config.Syncer {
+			syncerConfig := syncerConfig
 			log := log.WithField("syncer", syncerName)
 
 			// skip disabled syncer(s)
@@ -47,9 +50,9 @@ var syncCmd = &cobra.Command{
 				continue
 			}
 
-			if sync.ServiceAccountCount > 0 {
-				sync.Log.WithField("found_files", sync.ServiceAccountCount).
-					Info("Loaded service accounts")
+			serviceAccountCount := sync.RemoteServiceAccountFiles.ServiceAccountsCount()
+			if serviceAccountCount > 0 {
+				sync.Log.WithField("found_files", serviceAccountCount).Info("Loaded service accounts")
 			} else {
 				// no service accounts were loaded
 				// check to see if any of the copy or move remote(s) are banned