alpha: attempt at proper skip/target apps

Author: 23doors

cmd/deploy.go

@@ -30,32 +30,20 @@ func (e *Executor) newDeployCmd() *cobra.Command {
 			for _, t := range targetApps {
 				tsplit := strings.SplitN(t, ".", 2)
 				if len(tsplit) != 2 {
-					return fmt.Errorf("wrong format for 'target' %s: specify in a form of <app type>.<name>, e.g.: static.website", t)
+					return fmt.Errorf("wrong format for target '%s': specify in a form of <app type>.<name>, e.g.: static.website", t)
 				}
 
-				app := e.cfg.AppMap[config.ComputeAppID(tsplit[0], tsplit[1])]
-				if app == nil {
-					return fmt.Errorf("target app '%s' not found", t)
-				}
-
-				opts.TargetApps = append(opts.TargetApps, app)
+				opts.TargetApps = append(opts.TargetApps, config.ComputeAppID(tsplit[0], tsplit[1]))
 			}
 
-			if opts.SkipAllApps {
-				opts.SkipApps = e.cfg.Apps
-			} else {
+			if !opts.SkipAllApps {
 				for _, t := range skipApps {
 					tsplit := strings.SplitN(t, ".", 2)
 					if len(tsplit) != 2 {
-						return fmt.Errorf("wrong format for 'skip' %s: specify in a form of <app type>.<name>, e.g.: static.website", t)
-					}
-
-					app := e.cfg.AppMap[config.ComputeAppID(tsplit[0], tsplit[1])]
-					if app == nil {
-						return fmt.Errorf("app '%s' not found", t)
+						return fmt.Errorf("wrong format for skip '%s': specify in a form of <app type>.<name>, e.g.: static.website", t)
 					}
 
-					opts.SkipApps = append(opts.SkipApps, app)
+					opts.SkipApps = append(opts.SkipApps, config.ComputeAppID(tsplit[0], tsplit[1]))
 				}
 			}
 

go.mod

@@ -15,7 +15,7 @@ require (
 	github.com/mholt/archiver/v3 v3.5.0
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/otiai10/copy v1.6.0
-	github.com/outblocks/outblocks-plugin-go v0.0.0-20211103180603-373c5d4c00f1
+	github.com/outblocks/outblocks-plugin-go v0.0.0-20211103194421-5088c86b17e0
 	github.com/pterm/pterm v0.12.32
 	github.com/spf13/cobra v1.1.3
 	github.com/spf13/pflag v1.0.5

go.sum

@@ -627,8 +627,8 @@ github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6
 github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo=
 github.com/otiai10/mint v1.3.2 h1:VYWnrP5fXmz1MXvjuUvcBrXSjGE6xjON+axB/UrpO3E=
 github.com/otiai10/mint v1.3.2/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc=
-github.com/outblocks/outblocks-plugin-go v0.0.0-20211103180603-373c5d4c00f1 h1:JqlgaWtFJZ4GblEDqGvejUkjWjpa+i1iO1eObOaKOSE=
-github.com/outblocks/outblocks-plugin-go v0.0.0-20211103180603-373c5d4c00f1/go.mod h1:xosanOIc+3GlRA9b90/xRoYWzRgdhbew9T7+yTeI/TQ=
+github.com/outblocks/outblocks-plugin-go v0.0.0-20211103194421-5088c86b17e0 h1:nbIoc/Ee/aM03vZPy/8gbVs2OzIT9UUEgo2OzdlT9Eg=
+github.com/outblocks/outblocks-plugin-go v0.0.0-20211103194421-5088c86b17e0/go.mod h1:xosanOIc+3GlRA9b90/xRoYWzRgdhbew9T7+yTeI/TQ=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc=

internal/util/util.go

@@ -56,3 +56,13 @@ func FlattenEnvMap(m map[string]string) []string {
 
 	return out
 }
+
+func StringArrayToSet(in []string) map[string]bool {
+	outMap := make(map[string]bool, len(in))
+
+	for _, i := range in {
+		outMap[i] = true
+	}
+
+	return outMap
+}

pkg/actions/deploy.go

@@ -4,12 +4,14 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
+	"fmt"
 	"sort"
 	"sync"
 	"time"
 
 	dockerclient "github.com/docker/docker/client"
 	"github.com/outblocks/outblocks-cli/internal/urlutil"
+	"github.com/outblocks/outblocks-cli/internal/util"
 	"github.com/outblocks/outblocks-cli/pkg/config"
 	"github.com/outblocks/outblocks-cli/pkg/logger"
 	"github.com/outblocks/outblocks-cli/pkg/plugins"
@@ -48,7 +50,7 @@ type DeployOptions struct {
 	SkipBuild            bool
 	Lock                 bool
 	AutoApprove          bool
-	TargetApps, SkipApps []config.App
+	TargetApps, SkipApps []string
 	SkipAllApps          bool
 }
 
@@ -86,7 +88,12 @@ func (d *Deploy) Run(ctx context.Context) error {
 	stateBeforeStr, _ := json.Marshal(stateRes.State)
 
 	// Plan and apply.
-	planMap := calculatePlanMap(d.cfg, d.opts.TargetApps, d.opts.SkipApps)
+	apps, deps, skipAppIDs := filterApps(d.cfg, stateRes.State, d.opts.TargetApps, d.opts.SkipApps, d.opts.SkipAllApps)
+
+	planMap, err := calculatePlanMap(d.cfg, apps, deps, d.opts.TargetApps, skipAppIDs)
+	if err != nil {
+		return err
+	}
 
 	// Start plugins.
 	for plug := range planMap {
@@ -106,7 +113,7 @@ func (d *Deploy) Run(ctx context.Context) error {
 		return err
 	}
 
-	deployChanges, dnsChanges := computeChange(d.cfg.AppMap, d.cfg.DependencyMap, stateRes.State, planRetMap)
+	deployChanges, dnsChanges := computeChange(d.cfg, stateRes.State, planRetMap)
 
 	_ = spinner.Stop()
 
@@ -333,39 +340,122 @@ func getState(ctx context.Context, cfg *config.Project, lock bool) (*plugin_go.G
 	return ret, nil
 }
 
-func calculatePlanMap(cfg *config.Project, targetApps, skipApps []config.App) map[*plugins.Plugin]*planParams {
-	planMap := make(map[*plugins.Plugin]*planParams)
+func filterApps(cfg *config.Project, state *types.StateData, targetAppIDs, skipAppIDs []string, skipAllApps bool) (apps []*types.App, deps []*types.Dependency, retSkipAppIDs []string) {
+	if skipAllApps {
+		retSkipAppIDs := make([]string, 0, len(state.Apps))
+		apps = make([]*types.App, 0, len(state.Apps))
+
+		for _, app := range state.Apps {
+			apps = append(apps, app)
+			retSkipAppIDs = append(retSkipAppIDs, app.ID)
+		}
+
+		deps = make([]*types.Dependency, 0, len(state.Dependencies))
+		for _, dep := range state.Dependencies {
+			deps = append(deps, dep)
+		}
+
+		return apps, deps, retSkipAppIDs
+	}
+
+	// In non target and non skip mode, use config apps and deps.
+	if len(skipAppIDs) == 0 && len(targetAppIDs) == 0 {
+		apps = make([]*types.App, 0, len(cfg.Apps))
+		for _, app := range cfg.Apps {
+			apps = append(apps, app.PluginType())
+		}
+
+		deps = make([]*types.Dependency, 0, len(cfg.Dependencies))
+		for _, dep := range cfg.Dependencies {
+			deps = append(deps, dep.PluginType())
+		}
+
+		return apps, deps, nil
+	}
+
+	appsMap := make(map[string]*types.App, len(state.Apps))
+	dependenciesMap := make(map[string]*types.Dependency, len(state.Dependencies))
+	targetAppIDsMap := util.StringArrayToSet(targetAppIDs)
+	skipAppIDsMap := util.StringArrayToSet(skipAppIDs)
+
+	for key, app := range state.Apps {
+		appsMap[key] = app
+	}
+
+	for key, dep := range state.Dependencies {
+		dependenciesMap[key] = dep
+	}
 
 	for _, app := range cfg.Apps {
-		dnsPlugin := app.DNSPlugin()
-		deployPlugin := app.DeployPlugin()
+		if len(targetAppIDsMap) > 0 && !targetAppIDsMap[app.ID()] {
+			continue
+		}
+
+		if !skipAppIDsMap[app.ID()] {
+			continue
+		}
+
 		appType := app.PluginType()
+		appType.Properties = plugin_util.MergeMaps(cfg.Defaults.Deploy.Other, appType.Properties, app.DeployInfo().Other)
+		appType.Env = plugin_util.MergeStringMaps(cfg.Defaults.Run.Env, appType.Env, app.DeployInfo().Env)
 
-		includeDNS := dnsPlugin != nil && dnsPlugin == deployPlugin
+		appsMap[app.ID()] = appType
+	}
+
+	for _, dep := range cfg.Dependencies {
+		dependenciesMap[dep.ID()] = dep.PluginType()
+	}
+
+	// Flatten maps to list.
+	apps = make([]*types.App, 0, len(appsMap))
+	for _, app := range appsMap {
+		apps = append(apps, app)
+	}
+
+	deps = make([]*types.Dependency, 0, len(dependenciesMap))
+	for _, dep := range dependenciesMap {
+		deps = append(deps, dep)
+	}
+
+	return apps, deps, skipAppIDs
+}
+
+func calculatePlanMap(cfg *config.Project, apps []*types.App, deps []*types.Dependency, targetAppIDs, skipAppIDs []string) (map[*plugins.Plugin]*planParams, error) {
+	planMap := make(map[*plugins.Plugin]*planParams)
+
+	for _, app := range apps {
+		includeDNS := app.DNSPlugin != "" && app.DNSPlugin == app.DeployPlugin
+
+		deployPlugin := cfg.FindLoadedPlugin(app.DeployPlugin)
+		if deployPlugin == nil {
+			return nil, fmt.Errorf("missing deploy plugin: %s used for app: %s", app.DeployPlugin, app.Name)
+		}
 
 		if _, ok := planMap[deployPlugin]; !ok {
 			planMap[deployPlugin] = &planParams{
 				args: deployPlugin.CommandArgs(deployCommand),
 			}
 		}
 
-		appType.Properties = plugin_util.MergeMaps(cfg.Defaults.Deploy.Other, appType.Properties, app.DeployInfo().Other)
-		appType.Env = plugin_util.MergeStringMaps(cfg.Defaults.Run.Env, appType.Env, app.DeployInfo().Env)
-
 		appReq := &types.AppPlan{
 			IsDeploy: true,
 			IsDNS:    includeDNS,
-			App:      appType,
+			App:      app,
 		}
 
 		planMap[deployPlugin].apps = append(planMap[deployPlugin].apps, appReq)
 		planMap[deployPlugin].firstPass = !includeDNS // if dns is handled by different plugin, plan this as a first pass
 
 		// Add DNS plugin if not already included (handled by same plugin).
-		if includeDNS || dnsPlugin == nil {
+		if includeDNS || app.DNSPlugin == "" {
 			continue
 		}
 
+		dnsPlugin := cfg.FindLoadedPlugin(app.DNSPlugin)
+		if dnsPlugin == nil {
+			return nil, fmt.Errorf("missing dns plugin: %s used for app: %s", app.DNSPlugin, app.Name)
+		}
+
 		if _, ok := planMap[dnsPlugin]; !ok {
 			planMap[dnsPlugin] = &planParams{
 				args: dnsPlugin.CommandArgs(deployCommand),
@@ -375,57 +465,49 @@ func calculatePlanMap(cfg *config.Project, targetApps, skipApps []config.App) ma
 		appReq = &types.AppPlan{
 			IsDeploy: false,
 			IsDNS:    true,
-			App:      appType,
+			App:      app,
 		}
 
 		planMap[dnsPlugin].apps = append(planMap[dnsPlugin].apps, appReq)
 	}
 
 	// Process dependencies.
-	for _, dep := range cfg.Dependencies {
-		t := dep.PluginType()
+	for _, dep := range deps {
+		deployPlugin := cfg.FindLoadedPlugin(dep.DeployPlugin)
+		if deployPlugin == nil {
+			return nil, fmt.Errorf("missing deploy plugin: %s used for dependency: %s", dep.DeployPlugin, dep.Name)
+		}
 
-		p := dep.DeployPlugin()
-		if _, ok := planMap[p]; !ok {
-			planMap[p] = &planParams{
-				args: p.CommandArgs(deployCommand),
+		if _, ok := planMap[deployPlugin]; !ok {
+			planMap[deployPlugin] = &planParams{
+				args: deployPlugin.CommandArgs(deployCommand),
 			}
 		}
 
-		planMap[p].deps = append(planMap[p].deps, &types.DependencyPlan{
-			Dependency: t,
+		planMap[deployPlugin].deps = append(planMap[deployPlugin].deps, &types.DependencyPlan{
+			Dependency: dep,
 		})
 	}
 
-	return addPlanTargetAndSkipApps(planMap, targetApps, skipApps)
+	return addPlanTargetAndSkipApps(planMap, targetAppIDs, skipAppIDs), nil
 }
 
-func addPlanTargetAndSkipApps(planMap map[*plugins.Plugin]*planParams, targetApps, skipApps []config.App) map[*plugins.Plugin]*planParams {
-	if len(targetApps) == 0 && len(skipApps) == 0 {
+func addPlanTargetAndSkipApps(planMap map[*plugins.Plugin]*planParams, targetAppIDs, skipAppIDs []string) map[*plugins.Plugin]*planParams {
+	if len(targetAppIDs) == 0 && len(skipAppIDs) == 0 {
 		return planMap
 	}
 
 	// Add target and skip app ids.
-	targetAppIDsMap := make(map[string]struct{}, len(targetApps))
-	skipAppIDsMap := make(map[string]struct{}, len(skipApps))
-
-	for _, app := range targetApps {
-		appID := app.ID()
-		targetAppIDsMap[appID] = struct{}{}
-	}
-
-	for _, app := range skipApps {
-		appID := app.ID()
-		skipAppIDsMap[appID] = struct{}{}
-	}
+	targetIDsMap := util.StringArrayToSet(targetAppIDs)
+	skipIDsMap := util.StringArrayToSet(skipAppIDs)
 
 	for _, planParam := range planMap {
 		for _, app := range planParam.apps {
-			if _, ok := targetAppIDsMap[app.App.ID]; ok {
+			if _, ok := targetIDsMap[app.App.ID]; ok {
 				planParam.targetApps = append(planParam.targetApps, app.App.ID)
 			}
 
-			if _, ok := skipAppIDsMap[app.App.ID]; ok {
+			if _, ok := skipIDsMap[app.App.ID]; ok {
 				planParam.skipApps = append(planParam.skipApps, app.App.ID)
 			}
 		}

pkg/actions/deploy_build.go

@@ -188,20 +188,26 @@ func (d *Deploy) buildApps(ctx context.Context) error {
 	g, _ := errgroup.WithConcurrency(ctx, defaultConcurrency)
 
 	apps := d.cfg.Apps
-	if len(d.opts.TargetApps) > 0 {
-		apps = d.opts.TargetApps
-	}
+	targetAppIDsMap := util.StringArrayToSet(d.opts.TargetApps)
+	skipAppIDsMap := util.StringArrayToSet(d.opts.SkipApps)
+
+	var appsTemp []config.App
 
-	appsMap := make(map[config.App]struct{})
 	for _, app := range apps {
-		appsMap[app] = struct{}{}
-	}
+		if len(targetAppIDsMap) > 0 && !targetAppIDsMap[app.ID()] {
+			continue
+		}
 
-	for _, app := range d.opts.SkipApps {
-		delete(appsMap, app)
+		if !skipAppIDsMap[app.ID()] {
+			continue
+		}
+
+		appsTemp = append(appsTemp, app)
 	}
 
-	for app := range appsMap {
+	apps = appsTemp
+
+	for _, app := range apps {
 		// TODO: add build app function
 		switch app.Type() {
 		case config.AppTypeStatic:

pkg/actions/run.go

@@ -419,7 +419,7 @@ func formatRunOutput(log logger.Logger, cfg *config.Project, r *plugin_go.RunOut
 
 	switch r.Source {
 	case plugin_go.RunOutpoutSourceApp:
-		app := cfg.AppMap[r.ID]
+		app := cfg.AppByID(r.ID)
 		prefix = fmt.Sprintf("APP:%s:%s:", app.Type(), app.Name())
 
 	case plugin_go.RunOutpoutSourceDependency: