feat: config reloading (#1771)

## What kind of change does this PR introduce?

File based configuration reloading using fsnotify. 

## What is the current behavior?

Currently the Auth config is loaded once from the environment or file
(-c flag) and persists until the service is restarted.

## What is the new behavior?

A new optional flag (long: `--watch-dir`, short: `-w`) has been added.
When present any files with a ".env" suffix will be loaded into the
environment before the `*GlobalConfiguration` is created, otherwise
existing behavior is preserved.

In addition when the watch-dir flag is present a goroutine will be
started in serve_cmd.go and begin blocking on a call to
`(*Reloader).Watch` with a callback function that accepts a
`*conf.GlobalConfiguration object`. Each time this function is called we
create a new `*api.API` object and store it within our `AtomicHandler`,
previously given as the root handler to the `*http.Server`.

The Reloader uses some simple heuristics to deal with a few edge cases,
an overview:

- At most 1 configuration reload may happen per 10 seconds with a +-1s
margin of error.
- After a file within `--watch-dir` has changed the 10 second grace
period begins. After that it will reload the config.
- Config reloads first sort each file by name then processes them in
sequence.
- Directories within `--watch-dir` are ignored during config reloading.
- Implementation quirk: directory changes can trigger a config reload,
as I don't stat fsnotify events. This and similar superfulous reloads
could be easily fixed by storing a snapshot of `os.Environ()` after
successful reloads to compare with the latest via `slices.Equal()`
before reloading.
  - Files that do not end with a `.env` suffix are ignored.
- It handles the removal or renaming of the `-watch-dir` during runtime,
but an error message will be printed every 10 seconds as long as it's
missing.
- The config file passed with -c is only loaded once. Live reloads only
read the config dir. Meaning it would be possible to create a config dir
change that results in a new final configuration on the next reload due
to the persistence of `os.Environ()`.

---------

Co-authored-by: Chris Stockton <[email protected]>

Author: 2 people

cmd/root_cmd.go

@@ -9,7 +9,10 @@ import (
 	"github.com/supabase/auth/internal/observability"
 )
 
-var configFile = ""
+var (
+	configFile = ""
+	watchDir   = ""
+)
 
 var rootCmd = cobra.Command{
 	Use: "gotrue",
@@ -22,8 +25,8 @@ var rootCmd = cobra.Command{
 // RootCommand will setup and return the root command
 func RootCommand() *cobra.Command {
 	rootCmd.AddCommand(&serveCmd, &migrateCmd, &versionCmd, adminCmd())
-	rootCmd.PersistentFlags().StringVarP(&configFile, "config", "c", "", "the config file to use")
-
+	rootCmd.PersistentFlags().StringVarP(&configFile, "config", "c", "", "base configuration file to load")
+	rootCmd.PersistentFlags().StringVarP(&watchDir, "config-dir", "d", "", "directory containing a sorted list of config files to watch for changes")
 	return &rootCmd
 }
 

cmd/serve_cmd.go

@@ -3,11 +3,16 @@ package cmd
 import (
 	"context"
 	"net"
+	"net/http"
+	"sync"
+	"time"
 
+	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"github.com/supabase/auth/internal/api"
 	"github.com/supabase/auth/internal/conf"
+	"github.com/supabase/auth/internal/reloader"
 	"github.com/supabase/auth/internal/storage"
 	"github.com/supabase/auth/internal/utilities"
 )
@@ -21,7 +26,15 @@ var serveCmd = cobra.Command{
 }
 
 func serve(ctx context.Context) {
-	config, err := conf.LoadGlobal(configFile)
+	if err := conf.LoadFile(configFile); err != nil {
+		logrus.WithError(err).Fatal("unable to load config")
+	}
+
+	if err := conf.LoadDirectory(watchDir); err != nil {
+		logrus.WithError(err).Fatal("unable to load config from watch dir")
+	}
+
+	config, err := conf.LoadGlobalFromEnv()
 	if err != nil {
 		logrus.WithError(err).Fatal("unable to load config")
 	}
@@ -32,10 +45,63 @@ func serve(ctx context.Context) {
 	}
 	defer db.Close()
 
-	api := api.NewAPIWithVersion(config, db, utilities.Version)
-
 	addr := net.JoinHostPort(config.API.Host, config.API.Port)
 	logrus.Infof("GoTrue API started on: %s", addr)
 
-	api.ListenAndServe(ctx, addr)
+	a := api.NewAPIWithVersion(config, db, utilities.Version)
+	ah := reloader.NewAtomicHandler(a)
+
+	baseCtx, baseCancel := context.WithCancel(context.Background())
+	defer baseCancel()
+
+	httpSrv := &http.Server{
+		Addr:              addr,
+		Handler:           ah,
+		ReadHeaderTimeout: 2 * time.Second, // to mitigate a Slowloris attack
+		BaseContext: func(net.Listener) context.Context {
+			return baseCtx
+		},
+	}
+	log := logrus.WithField("component", "api")
+
+	var wg sync.WaitGroup
+	defer wg.Wait() // Do not return to caller until this goroutine is done.
+
+	if watchDir != "" {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+
+			fn := func(latestCfg *conf.GlobalConfiguration) {
+				log.Info("reloading api with new configuration")
+				latestAPI := api.NewAPIWithVersion(latestCfg, db, utilities.Version)
+				ah.Store(latestAPI)
+			}
+
+			rl := reloader.NewReloader(watchDir)
+			if err := rl.Watch(ctx, fn); err != nil {
+				log.WithError(err).Error("watcher is exiting")
+			}
+		}()
+	}
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+
+		<-ctx.Done()
+
+		defer baseCancel() // close baseContext
+
+		shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), time.Minute)
+		defer shutdownCancel()
+
+		if err := httpSrv.Shutdown(shutdownCtx); err != nil && !errors.Is(err, context.Canceled) {
+			log.WithError(err).Error("shutdown failed")
+		}
+	}()
+
+	if err := httpSrv.ListenAndServe(); err != http.ErrServerClosed {
+		log.WithError(err).Fatal("http server listen failed")
+	}
 }

go.mod

@@ -36,6 +36,7 @@ require (
 require (
 	github.com/bits-and-blooms/bitset v1.10.0 // indirect
 	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 // indirect
+	github.com/fsnotify/fsnotify v1.7.0 // indirect
 	github.com/go-jose/go-jose/v3 v3.0.3 // indirect
 	github.com/gobuffalo/nulls v0.4.2 // indirect
 	github.com/goccy/go-json v0.10.3 // indirect

go.sum

@@ -61,6 +61,8 @@ github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo=
 github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
 github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
+github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
+github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
 github.com/go-chi/chi/v5 v5.0.12 h1:9euLV5sTrTNTRUU9POmDUvfxyj6LAABLUcEWO+JJb4s=
 github.com/go-chi/chi/v5 v5.0.12/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8=
 github.com/go-jose/go-jose/v3 v3.0.3 h1:fFKWeig/irsp7XD2zBxvnmA/XaRWp5V3CBsZXJF7G7k=

internal/api/api.go

@@ -357,3 +357,9 @@ func (a *API) Mailer() mailer.Mailer {
 	config := a.config
 	return mailer.NewMailer(config)
 }
+
+// ServeHTTP implements the http.Handler interface by passing the request along
+// to its underlying Handler.
+func (a *API) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	a.handler.ServeHTTP(w, r)
+}

internal/api/cleanup.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"net/url"
 	"os"
+	"path/filepath"
 	"regexp"
 	"strings"
 	"text/template"
@@ -647,59 +648,140 @@ func (e *ExtensibilityPointConfiguration) PopulateExtensibilityPoint() error {
 	return nil
 }
 
+// LoadFile calls godotenv.Load() when the given filename is empty ignoring any
+// errors loading, otherwise it calls godotenv.Overload(filename).
+//
+// godotenv.Load: preserves env, ".env" path is optional
+// godotenv.Overload: overrides env, "filename" path must exist
+func LoadFile(filename string) error {
+	var err error
+	if filename != "" {
+		err = godotenv.Overload(filename)
+	} else {
+		err = godotenv.Load()
+		// handle if .env file does not exist, this is OK
+		if os.IsNotExist(err) {
+			return nil
+		}
+	}
+	return err
+}
+
+// LoadDirectory does nothing when configDir is empty, otherwise it will attempt
+// to load a list of configuration files located in configDir by using ReadDir
+// to obtain a sorted list of files containing a .env suffix.
+//
+// When the list is empty it will do nothing, otherwise it passes the file list
+// to godotenv.Overload to pull them into the current environment.
+func LoadDirectory(configDir string) error {
+	if configDir == "" {
+		return nil
+	}
+
+	// Returns entries sorted by filename
+	ents, err := os.ReadDir(configDir)
+	if err != nil {
+		// We mimic the behavior of LoadGlobal here, if an explicit path is
+		// provided we return an error.
+		return err
+	}
+
+	var paths []string
+	for _, ent := range ents {
+		if ent.IsDir() {
+			continue // ignore directories
+		}
+
+		// We only read files ending in .env
+		name := ent.Name()
+		if !strings.HasSuffix(name, ".env") {
+			continue
+		}
+
+		// ent.Name() does not include the watch dir.
+		paths = append(paths, filepath.Join(configDir, name))
+	}
+
+	// If at least one path was found we load the configuration files in the
+	// directory. We don't call override without config files because it will
+	// override the env vars previously set with a ".env", if one exists.
+	if len(paths) > 0 {
+		if err := godotenv.Overload(paths...); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// LoadGlobalFromEnv will return a new *GlobalConfiguration value from the
+// currently configured environment.
+func LoadGlobalFromEnv() (*GlobalConfiguration, error) {
+	config := new(GlobalConfiguration)
+	if err := loadGlobal(config); err != nil {
+		return nil, err
+	}
+	return config, nil
+}
+
 func LoadGlobal(filename string) (*GlobalConfiguration, error) {
 	if err := loadEnvironment(filename); err != nil {
 		return nil, err
 	}
 
 	config := new(GlobalConfiguration)
+	if err := loadGlobal(config); err != nil {
+		return nil, err
+	}
+	return config, nil
+}
 
+func loadGlobal(config *GlobalConfiguration) error {
 	// although the package is called "auth" it used to be called "gotrue"
 	// so environment configs will remain to be called "GOTRUE"
 	if err := envconfig.Process("gotrue", config); err != nil {
-		return nil, err
+		return err
 	}
 
 	if err := config.ApplyDefaults(); err != nil {
-		return nil, err
+		return err
 	}
 
 	if err := config.Validate(); err != nil {
-		return nil, err
+		return err
 	}
 
 	if config.Hook.PasswordVerificationAttempt.Enabled {
 		if err := config.Hook.PasswordVerificationAttempt.PopulateExtensibilityPoint(); err != nil {
-			return nil, err
+			return err
 		}
 	}
 
 	if config.Hook.SendSMS.Enabled {
 		if err := config.Hook.SendSMS.PopulateExtensibilityPoint(); err != nil {
-			return nil, err
+			return err
 		}
 	}
 	if config.Hook.SendEmail.Enabled {
 		if err := config.Hook.SendEmail.PopulateExtensibilityPoint(); err != nil {
-			return nil, err
+			return err
 		}
 	}
 
 	if config.Hook.MFAVerificationAttempt.Enabled {
 		if err := config.Hook.MFAVerificationAttempt.PopulateExtensibilityPoint(); err != nil {
-			return nil, err
+			return err
 		}
 	}
 
 	if config.Hook.CustomAccessToken.Enabled {
 		if err := config.Hook.CustomAccessToken.PopulateExtensibilityPoint(); err != nil {
-			return nil, err
+			return err
 		}
 	}
 
 	if config.SAML.Enabled {
 		if err := config.SAML.PopulateFields(config.API.ExternalURL); err != nil {
-			return nil, err
+			return err
 		}
 	} else {
 		config.SAML.PrivateKey = ""
@@ -712,7 +794,7 @@ func LoadGlobal(filename string) (*GlobalConfiguration, error) {
 		}
 		template, err := template.New("").Parse(SMSTemplate)
 		if err != nil {
-			return nil, err
+			return err
 		}
 		config.Sms.SMSTemplate = template
 	}
@@ -724,12 +806,12 @@ func LoadGlobal(filename string) (*GlobalConfiguration, error) {
 		}
 		template, err := template.New("").Parse(smsTemplate)
 		if err != nil {
-			return nil, err
+			return err
 		}
 		config.MFA.Phone.SMSTemplate = template
 	}
 
-	return config, nil
+	return nil
 }
 
 // ApplyDefaults sets defaults for a GlobalConfiguration