change(cache): use badgerdb

Author: l3uddz

cache/cache.go

@@ -1,38 +1,54 @@
 package cache
 
 import (
-	jsoniter "github.com/json-iterator/go"
 	"github.com/l3uddz/crop/logger"
 	"github.com/l3uddz/crop/stringutils"
-	"sync"
-	"time"
+	"github.com/pkg/errors"
+	"github.com/zippoxer/bow"
+	"github.com/zippoxer/bow/codec/json"
 )
 
 var (
 	log           = logger.GetLogger("cache")
 	cacheFilePath string
-	mtx           sync.Mutex
-	vault         map[string]time.Time
 
 	// Internal
-	json = jsoniter.ConfigCompatibleWithStandardLibrary
+	db *bow.DB
 )
 
 /* Public */
 
-func Init(cachePath string) error {
+func Init(cachePath string, logLevel int) error {
 	// set globals
 	cacheFilePath = cachePath
-	vault = make(map[string]time.Time)
 
-	// load cache from disk
-	if err := loadFromFile(cachePath); err != nil {
-		return err
+	// set badger options
+	opts := []bow.Option{
+		// codec
+		bow.SetCodec(json.Codec{}),
 	}
 
+	if logLevel < 2 {
+		// disable badger logging for non trace log level
+		opts = append(opts, bow.SetLogger(nil))
+	}
+
+	// init database
+	v, err := bow.Open(cachePath, opts...)
+	if err != nil {
+		return errors.WithMessage(err, "failed opening cache")
+	}
+
+	db = v
 	return nil
 }
 
+func Close() {
+	if err := db.Close(); err != nil {
+		log.WithError(err).Error("Failed closing cache gracefully...")
+	}
+}
+
 func ShowUsing() {
 	log.Infof("Using %s = %q", stringutils.LeftJust("CACHE", " ", 10), cacheFilePath)
 }

cache/dump.go

@@ -0,0 +1,53 @@
+package cache
+
+import (
+	"github.com/zippoxer/bow"
+	"time"
+)
+
+type Banned struct {
+	Path    string `bow:"key"`
+	Expires time.Time
+}
+
+func IsBanned(key string) (bool, time.Time) {
+	// check if key was found in banned bucket
+	var item Banned
+	err := db.Bucket("banned").Get(key, &item)
+
+	// was key not found
+	if err == bow.ErrNotFound {
+		// this key is not banned
+		return false, time.Time{}
+	} else if err != nil {
+		log.WithError(err).Errorf("Failed checking banned bucket for: %q", key)
+		return false, time.Time{}
+	}
+
+	// check if the ban has expired
+	if item.Expires.Before(time.Now().UTC()) {
+		// the ban has expired, remove
+		log.Warnf("Expired %q: %v", key, item.Expires)
+
+		err := db.Bucket("banned").Delete(key)
+		if err != nil {
+			log.WithError(err).Errorf("Failed removing from banned bucket: %q", key)
+			return false, time.Time{}
+		}
+
+		return false, time.Time{}
+	}
+
+	// this key is still banned
+	return true, item.Expires
+
+}
+
+func SetBanned(key string, hours int) error {
+	expiry := time.Now().UTC().Add(time.Duration(hours) * time.Hour)
+
+	return db.Bucket("banned").Put(Banned{
+		Path:    key,
+		Expires: expiry,
+	})
+}

cache/get.go

@@ -20,7 +20,7 @@ var (
 	flagLogLevel     = 0
 	flagConfigFolder = pathutils.GetCurrentBinaryPath()
 	flagConfigFile   = "config.yaml"
-	flagCacheFile    = "cache.json"
+	flagCachePath    = "cache"
 	flagLogFile      = "activity.log"
 
 	flagDryRun bool
@@ -50,7 +50,7 @@ func init() {
 	// Parse persistent flags
 	rootCmd.PersistentFlags().StringVar(&flagConfigFolder, "config-dir", flagConfigFolder, "Config folder")
 	rootCmd.PersistentFlags().StringVarP(&flagConfigFile, "config", "c", flagConfigFile, "Config file")
-	rootCmd.PersistentFlags().StringVarP(&flagCacheFile, "cache", "d", flagCacheFile, "Cache file")
+	rootCmd.PersistentFlags().StringVarP(&flagCachePath, "cache", "d", flagCachePath, "Cache path")
 	rootCmd.PersistentFlags().StringVarP(&flagLogFile, "log", "l", flagLogFile, "Log file")
 	rootCmd.PersistentFlags().CountVarP(&flagLogLevel, "verbose", "v", "Verbose level")
 
@@ -63,7 +63,7 @@ func initCore(showAppInfo bool) {
 		flagConfigFile = filepath.Join(flagConfigFolder, flagConfigFile)
 	}
 	if !rootCmd.PersistentFlags().Changed("cache") {
-		flagCacheFile = filepath.Join(flagConfigFolder, flagCacheFile)
+		flagCachePath = filepath.Join(flagConfigFolder, flagCachePath)
 	}
 	if !rootCmd.PersistentFlags().Changed("log") {
 		flagLogFile = filepath.Join(flagConfigFolder, flagLogFile)
@@ -84,7 +84,7 @@ func initCore(showAppInfo bool) {
 	setConfigOverrides()
 
 	// Init Cache
-	if err := cache.Init(flagCacheFile); err != nil {
+	if err := cache.Init(flagCachePath, flagLogLevel); err != nil {
 		log.WithError(err).Fatal("Failed to initialize cache")
 	}
 

cache/load.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"github.com/dustin/go-humanize"
+	"github.com/l3uddz/crop/cache"
 	"github.com/l3uddz/crop/config"
 	"github.com/l3uddz/crop/rclone"
 	"github.com/l3uddz/crop/syncer"
@@ -23,6 +24,7 @@ var syncCmd = &cobra.Command{
 	Run: func(cmd *cobra.Command, args []string) {
 		// init core
 		initCore(true)
+		defer cache.Close()
 
 		// iterate syncer's
 		for syncerName, syncerConfig := range config.Config.Syncer {

cache/sa.go

@@ -3,6 +3,7 @@ package cmd
 import (
 	"bufio"
 	"github.com/blang/semver"
+	"github.com/l3uddz/crop/cache"
 	"github.com/l3uddz/crop/runtime"
 	"github.com/rhysd/go-github-selfupdate/selfupdate"
 	"github.com/spf13/cobra"
@@ -17,6 +18,7 @@ var updateCmd = &cobra.Command{
 	Run: func(cmd *cobra.Command, args []string) {
 		// init core
 		initCore(false)
+		defer cache.Close()
 
 		// parse current version
 		v, err := semver.Parse(runtime.Version)

cache/set.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"github.com/dustin/go-humanize"
+	"github.com/l3uddz/crop/cache"
 	"github.com/l3uddz/crop/config"
 	"github.com/l3uddz/crop/rclone"
 	"github.com/l3uddz/crop/uploader"
@@ -23,6 +24,7 @@ var uploadCmd = &cobra.Command{
 	Run: func(cmd *cobra.Command, args []string) {
 		// init core
 		initCore(true)
+		defer cache.Close()
 
 		// iterate uploader's
 		for uploaderName, uploaderConfig := range config.Config.Uploader {