Default pgroll pull to write in YAML format and add a --json flag (#810)

Make `pgroll pull` default to pulling migrations in YAML format, rather
than JSON.

Add a `--json` flag to the `pull` command to allow users to output
migrations in JSON format instead of YAML.

We want to default to YAML as the output format, but provide an option
for users to choose JSON if they prefer it. This is useful for users who
may want to integrate the output with other tools or systems that work
better with JSON.

Author: andrew-farries

cli-definition.json

@@ -102,6 +102,12 @@
       "use": "pull <target directory>",
       "example": "",
       "flags": [
+        {
+          "name": "json",
+          "shorthand": "j",
+          "description": "output each migration in JSON format instead of YAML",
+          "default": "false"
+        },
         {
           "name": "with-prefixes",
           "shorthand": "p",

cmd/pull.go

@@ -14,8 +14,9 @@ import (
 func pullCmd() *cobra.Command {
 	opts := map[string]string{
 		"p": "prefix each migration filename with its position in the schema history",
+		"j": "output each migration in JSON format instead of YAML",
 	}
-	var withPrefixes bool
+	var withPrefixes, useJSON bool
 
 	pullCmd := &cobra.Command{
 		Use:       "pull <target directory>",
@@ -51,7 +52,7 @@ func pullCmd() *cobra.Command {
 				if withPrefixes {
 					prefix = fmt.Sprintf("%04d", i+1) + "_"
 				}
-				err := mig.WriteToFile(targetDir, prefix)
+				err := mig.WriteToFile(targetDir, prefix, useJSON)
 				if err != nil {
 					return fmt.Errorf("failed to write migration %q: %w", mig.Migration.Name, err)
 				}
@@ -61,6 +62,7 @@ func pullCmd() *cobra.Command {
 	}
 
 	pullCmd.Flags().BoolVarP(&withPrefixes, "with-prefixes", "p", false, opts["p"])
+	pullCmd.Flags().BoolVarP(&useJSON, "json", "j", false, opts["j"])
 
 	return pullCmd
 }

docs/cli/pull.mdx

@@ -1,6 +1,6 @@
 ---
 title: Pull
-description: Pull the complete schema history of applied migrations from the target database and write the migrations to disk.
+description: Pull migrations from the target database into a local migrations directory
 ---
 
 ## Command
@@ -11,17 +11,17 @@ Assuming that all [example migrations](https://github.com/xataio/pgroll/tree/mai
 $ pgroll pull migrations/
 ```
 
-will write the complete schema history as `.json` files to the `migrations/` directory:
+will write the complete schema history as `.yaml` files to the `migrations/` directory:
 
 ```
 $ ls migrations/
 
-01_create_tables.json
-02_create_another_table.json
-03_add_column_to_products.json
-04_rename_table.json
-05_sql.json
-06_add_column_to_sql_table.json
+01_create_tables.yaml
+02_create_another_table.yaml
+03_add_column_to_products.yaml
+04_rename_table.yaml
+05_sql.yaml
+06_add_column_to_sql_table.yaml
 ...
 ```
 
@@ -30,15 +30,19 @@ The command takes an optional `--with-prefixes` flag which will write each filen
 ```
 $ ls migrations/
 
-0001_01_create_tables.json
-0002_02_create_another_table.json
-0003_03_add_column_to_products.json
-0004_04_rename_table.json
-0005_05_sql.json
-0006_06_add_column_to_sql_table.json
+0001_01_create_tables.yaml
+0002_02_create_another_table.yaml
+0003_03_add_column_to_products.yaml
+0004_04_rename_table.yaml
+0005_05_sql.yaml
+0006_06_add_column_to_sql_table.yaml
 ...
 ```
 
 The `--with-prefixes` flag ensures that files are sorted lexicographically by their time of application.
 
-If the directory specified as the required argument to `pgroll pull` does not exist, `pgroll pull` will create it.
+Use the `--json` flag to pull migrations in JSON format rather than YAML.
+
+If the target directory given to `pgroll pull` does not exist, `pgroll pull` will create it.
+
+If the target directory is empty, `pgroll pull` will pull all migrations from the target database. If the target directory contains migration files, `pgroll pull` will pull only those migrations that don't already exist in the directory.

pkg/state/history.go

@@ -6,11 +6,13 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"io"
 	"os"
 	"path/filepath"
 	"time"
 
 	"github.com/lib/pq"
+	"sigs.k8s.io/yaml"
 
 	"github.com/xataio/pgroll/pkg/migrations"
 )
@@ -63,14 +65,20 @@ func (s *State) SchemaHistory(ctx context.Context, schema string) ([]Migration,
 }
 
 // WriteToFile writes the migration to a file in `targetDir`, prefixing the
-// filename with `prefix`.
-func (m *Migration) WriteToFile(targetDir, prefix string) error {
+// filename with `prefix`. The output format defaults to YAML, but can
+// be changed to JSON by setting `useJSON` to true.
+func (m *Migration) WriteToFile(targetDir, prefix string, useJSON bool) error {
 	err := os.MkdirAll(targetDir, 0o755)
 	if err != nil {
 		return err
 	}
 
-	fileName := fmt.Sprintf("%s%s.json", prefix, m.Migration.Name)
+	suffix := "yaml"
+	if useJSON {
+		suffix = "json"
+	}
+
+	fileName := fmt.Sprintf("%s%s.%s", prefix, m.Migration.Name, suffix)
 	filePath := filepath.Join(targetDir, fileName)
 
 	file, err := os.Create(filePath)
@@ -79,8 +87,26 @@ func (m *Migration) WriteToFile(targetDir, prefix string) error {
 	}
 	defer file.Close()
 
-	encoder := json.NewEncoder(file)
+	if useJSON {
+		return m.writeAsJSON(file)
+	} else {
+		return m.writeAsYAML(file)
+	}
+}
+
+func (m *Migration) writeAsJSON(w io.Writer) error {
+	encoder := json.NewEncoder(w)
 	encoder.SetIndent("", "  ")
 
 	return encoder.Encode(m.Migration)
 }
+
+func (m *Migration) writeAsYAML(w io.Writer) error {
+	yml, err := yaml.Marshal(m.Migration)
+	if err != nil {
+		return err
+	}
+
+	_, err = w.Write(yml)
+	return err
+}