feat: --daisy-chain flag for sync command and standalone dedupe command (#27)

* build: use latest goreleaser

* Add systemd services

* Update README.md

* feat: add dedupe command

--no-dedupe also added to upload command.

* build: remove artifact cleanup action

* feat(sync): add daisy-chain parameter

* chore: module updates

Co-authored-by: Eric Nemchik <[email protected]>

Author: l3uddz

.github/workflows/build.yml

@@ -21,21 +21,19 @@ jobs:
       - name: dependencies
         run: |
           curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sudo sh -s -- -b /usr/local/bin v1.24.0
-          curl -sfL https://install.goreleaser.com/github.com/goreleaser/goreleaser.sh | sudo sh -s -- -b /usr/local/bin v0.139.0
+          curl -sfL https://install.goreleaser.com/github.com/goreleaser/goreleaser.sh | sudo sh -s -- -b /usr/local/bin
 
       # checkout
       - name: checkout
         uses: actions/checkout@v2
-
-      # unshallow (for goreleaser)
-      - name: unshallow
-        run: git fetch --prune --unshallow
+        with:
+          fetch-depth: 0
 
       # setup go
       - name: go
         uses: actions/setup-go@v1
         with:
-          go-version: 1.14
+          go-version: 1.15
 
       - name: go_version
         run: go version

.github/workflows/remove-old-artifacts.yml

@@ -42,3 +42,4 @@ changelog:
     exclude:
       - "^docs:"
       - "^test:"
+      - "^Merge branch"

.goreleaser.yml

@@ -39,7 +39,6 @@ rclone:
         - '--use-mmap'
         - '--no-update-modtime'
         - '--drive-chunk-size=128M'
-        - '--drive-use-trash=false'
       dedupe:
         - '--dedupe-mode=newest'
         - '--tpslimit=5'
@@ -134,9 +133,9 @@ syncer:
 
 - Manual - Perform manual sync/copy job(s)
 
-`crop manual --copy --src remote1:/Backups --dst remote2:/Backups --sa /opt/service_accounts -- --dry-run --drive-use-trash=false`
+`crop manual --copy --src remote1:/Backups --dst remote2:/Backups --sa /opt/service_accounts -- --dry-run`
 
-`crop manual --sync --src remote1:/Backups --dst remote2:/Backups --sa /opt/service_accounts --dedupe -- --drive-use-trash=false`
+`crop manual --sync --src remote1:/Backups --dst remote2:/Backups --sa /opt/service_accounts --dedupe --`
 
 ***
 

README.md

@@ -0,0 +1,78 @@
+package cmd
+
+import (
+	"github.com/dustin/go-humanize"
+	"github.com/l3uddz/crop/config"
+	"github.com/l3uddz/crop/uploader"
+	"github.com/pkg/errors"
+	"github.com/spf13/cobra"
+	"strings"
+	"time"
+)
+
+var dedupeCmd = &cobra.Command{
+	Use:   "dedupe",
+	Short: "Perform dedupe associated with uploader(s)",
+	Long:  `This command can be used to trigger a dedupe associated with uploader(s).`,
+
+	Run: func(cmd *cobra.Command, args []string) {
+		// init core
+		initCore(true)
+		defer releaseFileLock()
+
+		// iterate uploader's
+		started := time.Now().UTC()
+
+		for _, uploaderConfig := range config.Config.Uploader {
+			log := log.WithField("uploader", uploaderConfig.Name)
+
+			// skip disabled uploader(s)
+			if !uploaderConfig.Enabled {
+				log.Debug("Skipping disabled uploader")
+				continue
+			}
+
+			// skip uploader specific chosen
+			if flagUploader != "" && !strings.EqualFold(uploaderConfig.Name, flagUploader) {
+				log.Debugf("Skipping uploader as not: %q", flagUploader)
+				continue
+			}
+
+			// create uploader
+			upload, err := uploader.New(config.Config, &uploaderConfig, uploaderConfig.Name)
+			if err != nil {
+				log.WithError(err).Error("Failed initializing uploader, skipping...")
+				continue
+			}
+
+			log.Info("Dedupe commencing...")
+
+			// perform upload
+			if err := performDedupe(upload); err != nil {
+				upload.Log.WithError(err).Error("Error occurred while running dedupe, skipping...")
+				continue
+			}
+		}
+
+		log.Infof("Finished in: %v", humanize.RelTime(started, time.Now().UTC(), "", ""))
+	},
+}
+
+func init() {
+	rootCmd.AddCommand(dedupeCmd)
+
+	dedupeCmd.Flags().StringVarP(&flagUploader, "uploader", "u", "", "Run for a specific uploader")
+}
+
+func performDedupe(u *uploader.Uploader) error {
+	u.Log.Info("Running dedupe...")
+
+	/* Dedupe */
+	err := u.Dedupe(nil)
+	if err != nil {
+		return errors.Wrap(err, "failed dedupe remotes")
+	}
+
+	u.Log.Info("Finished dedupe!")
+	return nil
+}

cmd/dedupe.go

@@ -26,6 +26,7 @@ var (
 	flagLogFile      = "activity.log"
 	flagLockFile     = "crop.lock"
 	flagDryRun       bool
+	flagNoDedupe     bool
 
 	// Global command specific
 	flagUploader string

cmd/root.go

@@ -18,7 +18,7 @@ import (
 var (
 	flagSyncer      string
 	flagParallelism int
-	flagNoDedupe    bool
+	flagDaisyChain  bool
 )
 
 var syncCmd = &cobra.Command{
@@ -114,6 +114,7 @@ func init() {
 	syncCmd.Flags().StringVarP(&flagSyncer, "syncer", "s", "", "Run for a specific syncer")
 	syncCmd.Flags().IntVarP(&flagParallelism, "parallelism", "p", 1, "Max parallel syncers")
 
+	syncCmd.Flags().BoolVar(&flagDaisyChain, "daisy-chain", false, "Daisy chain source remotes")
 	syncCmd.Flags().BoolVar(&flagNoDedupe, "no-dedupe", false, "Ignore dedupe tasks for syncer")
 }
 
@@ -147,7 +148,7 @@ func performSync(s *syncer.Syncer) error {
 	if len(s.Config.Remotes.Copy) > 0 {
 		s.Log.Info("Running copies...")
 
-		if err := s.Copy(liveRotateParams); err != nil {
+		if err := s.Copy(liveRotateParams, flagDaisyChain); err != nil {
 			return errors.WithMessage(err, "failed performing all copies")
 		}
 
@@ -158,7 +159,7 @@ func performSync(s *syncer.Syncer) error {
 	if len(s.Config.Remotes.Sync) > 0 {
 		s.Log.Info("Running syncs...")
 
-		if err := s.Sync(liveRotateParams); err != nil {
+		if err := s.Sync(liveRotateParams, flagDaisyChain); err != nil {
 			return errors.WithMessage(err, "failed performing all syncs")
 		}
 

cmd/sync.go

@@ -162,6 +162,7 @@ func init() {
 	uploadCmd.Flags().StringVarP(&flagUploader, "uploader", "u", "", "Run for a specific uploader")
 
 	uploadCmd.Flags().BoolVar(&flagNoCheck, "no-check", false, "Ignore check and run")
+	uploadCmd.Flags().BoolVar(&flagNoDedupe, "no-dedupe", false, "Ignore dedupe tasks for uploader")
 }
 
 func performUpload(u *uploader.Uploader, forced bool) error {
@@ -240,7 +241,7 @@ func performUpload(u *uploader.Uploader, forced bool) error {
 	}
 
 	/* Dedupe */
-	if len(u.Config.Remotes.Dedupe) > 0 {
+	if !flagNoDedupe && len(u.Config.Remotes.Dedupe) > 0 {
 		u.Log.Info("Running dedupes...")
 
 		if err := u.Dedupe(nil); err != nil {

cmd/upload.go

@@ -4,19 +4,20 @@ go 1.14
 
 require (
 	github.com/DataDog/zstd v1.4.5 // indirect
-	github.com/ReneKroon/ttlcache v1.6.0
+	github.com/ReneKroon/ttlcache v1.7.0
 	github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect
 	github.com/blang/semver v3.5.1+incompatible
-	github.com/dgraph-io/badger/v2 v2.0.3 // indirect
+	github.com/dgraph-io/badger/v2 v2.2007.2 // indirect
 	github.com/dgraph-io/ristretto v0.0.3 // indirect
 	github.com/dustin/go-humanize v1.0.0
 	github.com/fsnotify/fsnotify v1.4.9 // indirect
 	github.com/go-cmd/cmd v1.2.1
 	github.com/go-ole/go-ole v1.2.4 // indirect
-	github.com/gofiber/fiber v1.12.6
-	github.com/gofiber/recover v0.1.1
-	github.com/golang/protobuf v1.4.2 // indirect
-	github.com/klauspost/compress v1.10.10 // indirect
+	github.com/gofiber/fiber v1.14.6
+	github.com/gofiber/recover v0.1.2
+	github.com/golang/snappy v0.0.2 // indirect
+	github.com/gorilla/schema v1.2.0 // indirect
+	github.com/klauspost/compress v1.11.0 // indirect
 	github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect
 	github.com/natefinch/lumberjack v2.0.0+incompatible
 	github.com/nightlyone/lockfile v1.0.0
@@ -25,21 +26,19 @@ require (
 	github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2
 	github.com/pkg/errors v0.9.1
 	github.com/rhysd/go-github-selfupdate v1.2.2
-	github.com/shirou/gopsutil v2.20.6+incompatible
+	github.com/shirou/gopsutil v2.20.8+incompatible
 	github.com/sirupsen/logrus v1.6.0
 	github.com/sony/sonyflake v1.0.0 // indirect
 	github.com/spf13/cobra v1.0.0
 	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/ulikunitz/xz v0.5.7 // indirect
+	github.com/ulikunitz/xz v0.5.8 // indirect
 	github.com/x-cray/logrus-prefixed-formatter v0.5.2
 	github.com/yale8848/gorpool v0.1.0
 	github.com/zippoxer/bow v0.0.0-20200229231453-bf1012ae7ab9
-	golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899 // indirect
-	golang.org/x/net v0.0.0-20200707034311-ab3426394381 // indirect
-	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d // indirect
-	golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae
-	google.golang.org/appengine v1.6.6 // indirect
-	google.golang.org/protobuf v1.25.0 // indirect
+	golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a // indirect
+	golang.org/x/net v0.0.0-20200925080053-05aa5d4ee321 // indirect
+	golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43 // indirect
+	golang.org/x/sys v0.0.0-20200926100807-9d91bd62050c
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
 	gopkg.in/yaml.v2 v2.3.0
 )