Merge pull request #1552 from krissetto/better-no-provider-err

Show better errors if `dmr` not found or no providers for `model: "auto"`

Author: dgageot

pkg/config/auto.go

@@ -2,11 +2,54 @@ package config
 
 import (
 	"context"
+	"fmt"
+	"strings"
 
 	"github.com/docker/cagent/pkg/config/latest"
 	"github.com/docker/cagent/pkg/environment"
 )
 
+// providerConfig defines a cloud provider and how to detect/describe its API keys.
+type providerConfig struct {
+	name    string   // provider name (e.g., "anthropic")
+	envVars []string // env vars to check - provider is available if ANY is set
+	hint    string   // description for error messages
+}
+
+// cloudProviders defines the available cloud providers in priority order.
+// The first provider with a configured API key will be selected by AutoModelConfig.
+// DMR is always appended as the final fallback (not listed here).
+var cloudProviders = []providerConfig{
+	{"anthropic", []string{"ANTHROPIC_API_KEY"}, "ANTHROPIC_API_KEY"},
+	{"openai", []string{"OPENAI_API_KEY"}, "OPENAI_API_KEY"},
+	{"google", []string{"GOOGLE_API_KEY"}, "GOOGLE_API_KEY"},
+	{"mistral", []string{"MISTRAL_API_KEY"}, "MISTRAL_API_KEY"},
+	{"amazon-bedrock", []string{
+		"AWS_BEARER_TOKEN_BEDROCK",
+		"AWS_ACCESS_KEY_ID",
+		"AWS_PROFILE",
+		"AWS_ROLE_ARN",
+	}, "AWS_ACCESS_KEY_ID (or AWS_PROFILE, AWS_ROLE_ARN, AWS_BEARER_TOKEN_BEDROCK)"},
+}
+
+// ErrAutoModelFallback is returned when auto model selection fails because
+// no providers are available (no API keys configured and DMR not installed).
+type ErrAutoModelFallback struct{}
+
+func (e *ErrAutoModelFallback) Error() string {
+	var hints []string
+	for _, p := range cloudProviders {
+		hints = append(hints, fmt.Sprintf("    - %s: %s", p.name, p.hint))
+	}
+
+	return fmt.Sprintf(`No model providers available.
+
+To fix this, you can:
+  - Install Docker Model Runner: https://docs.docker.com/ai/model-runner/get-started/
+  - Configure an API key for a cloud provider:
+%s`, strings.Join(hints, "\n"))
+}
+
 var DefaultModels = map[string]string{
 	"openai":         "gpt-5-mini",
 	"anthropic":      "claude-sonnet-4-0",
@@ -24,29 +67,16 @@ func AvailableProviders(ctx context.Context, modelsGateway string, env environme
 
 	var providers []string
 
-	if key, _ := env.Get(ctx, "ANTHROPIC_API_KEY"); key != "" {
-		providers = append(providers, "anthropic")
-	}
-	if key, _ := env.Get(ctx, "OPENAI_API_KEY"); key != "" {
-		providers = append(providers, "openai")
-	}
-	if key, _ := env.Get(ctx, "GOOGLE_API_KEY"); key != "" {
-		providers = append(providers, "google")
-	}
-	if key, _ := env.Get(ctx, "MISTRAL_API_KEY"); key != "" {
-		providers = append(providers, "mistral")
-	}
-	// AWS Bedrock supports multiple authentication methods (API key, IAM credentials, profile, role)
-	if key, _ := env.Get(ctx, "AWS_BEARER_TOKEN_BEDROCK"); key != "" {
-		providers = append(providers, "amazon-bedrock")
-	} else if key, _ := env.Get(ctx, "AWS_ACCESS_KEY_ID"); key != "" {
-		providers = append(providers, "amazon-bedrock")
-	} else if key, _ := env.Get(ctx, "AWS_PROFILE"); key != "" {
-		providers = append(providers, "amazon-bedrock")
-	} else if key, _ := env.Get(ctx, "AWS_ROLE_ARN"); key != "" {
-		providers = append(providers, "amazon-bedrock")
+	for _, p := range cloudProviders {
+		for _, envVar := range p.envVars {
+			if key, _ := env.Get(ctx, envVar); key != "" {
+				providers = append(providers, p.name)
+				break // found one, no need to check other env vars for this provider
+			}
+		}
 	}
 
+	// DMR is always the final fallback
 	providers = append(providers, "dmr")
 
 	return providers

pkg/creator/agent_test.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/docker/cagent/pkg/config"
 	"github.com/docker/cagent/pkg/config/latest"
+	"github.com/docker/cagent/pkg/environment"
 )
 
 func TestAgentConfigYAML(t *testing.T) {
@@ -102,14 +103,17 @@ func TestAgent(t *testing.T) {
 
 	ctx := t.Context()
 
-	// Create a minimal runtime config
+	// Create a runtime config with a mock env provider that has a dummy API key
+	// so the auto model can resolve to a provider without needing real credentials
 	runConfig := &config.RuntimeConfig{
 		Config: config.Config{
 			WorkingDir: t.TempDir(),
 		},
+		EnvProviderForTests: environment.NewEnvListProvider([]string{
+			"OPENAI_API_KEY=dummy-key-for-testing",
+		}),
 	}
 
-	// Test with a mock model override to avoid needing real API keys
 	// The auto model will be resolved based on available providers
 	team, err := Agent(ctx, runConfig, "")
 	require.NoError(t, err)

pkg/model/provider/dmr/client.go

@@ -44,6 +44,9 @@ const (
 	connectivityTimeout = 2 * time.Second
 )
 
+// ErrNotInstalled is returned when Docker Model Runner is not installed.
+var ErrNotInstalled = errors.New("docker model runner is not available\nplease install it and try again (https://docs.docker.com/ai/model-runner/get-started/)")
+
 const (
 	// dmrInferencePrefix mirrors github.com/docker/model-runner/pkg/inference.InferencePrefix.
 	dmrInferencePrefix = "/engines"
@@ -87,7 +90,11 @@ func NewClient(ctx context.Context, cfg *latest.ModelConfig, opts ...options.Opt
 		var err error
 		endpoint, engine, err = getDockerModelEndpointAndEngine(ctx)
 		if err != nil {
-			slog.Debug("docker model status query failed", "error", err)
+			if err.Error() == "unknown flag: --json\n\nUsage:  docker [OPTIONS] COMMAND [ARG...]\n\nRun 'docker --help' for more information" {
+				slog.Debug("docker model status query failed", "error", err)
+				return nil, ErrNotInstalled
+			}
+			slog.Error("docker model status query failed", "error", err)
 		} else {
 			// Auto-pull the model if needed
 			if err := pullDockerModelIfNeeded(ctx, cfg.Model); err != nil {

pkg/model/provider/dmr/client_test.go

@@ -5,6 +5,9 @@ import (
 	"io"
 	"net/http"
 	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"runtime"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -27,6 +30,31 @@ func TestNewClientWithExplicitBaseURL(t *testing.T) {
 	assert.Equal(t, "https://custom.example.com:8080/api/v1", client.baseURL)
 }
 
+func TestNewClientReturnsErrNotInstalledWhenDockerModelUnsupported(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("Skipping docker CLI shim test on Windows")
+	}
+
+	tempDir := t.TempDir()
+	dockerPath := filepath.Join(tempDir, "docker")
+	script := "#!/bin/sh\n" +
+		"printf 'unknown flag: --json\\n\\nUsage:  docker [OPTIONS] COMMAND [ARG...]\\n\\nRun '\\''docker --help'\\'' for more information\\n' >&2\n" +
+		"exit 1\n"
+	require.NoError(t, os.WriteFile(dockerPath, []byte(script), 0o755))
+
+	t.Setenv("PATH", tempDir)
+	t.Setenv("MODEL_RUNNER_HOST", "")
+
+	cfg := &latest.ModelConfig{
+		Provider: "dmr",
+		Model:    "ai/qwen3",
+	}
+
+	_, err := NewClient(t.Context(), cfg)
+	require.Error(t, err)
+	require.ErrorIs(t, err, ErrNotInstalled)
+}
+
 func TestGetDMRFallbackURLs(t *testing.T) {
 	t.Parallel()
 

pkg/teamloader/teamloader.go

@@ -3,6 +3,7 @@ package teamloader
 import (
 	"cmp"
 	"context"
+	"errors"
 	"fmt"
 	"log/slog"
 	"strings"
@@ -13,6 +14,7 @@ import (
 	"github.com/docker/cagent/pkg/config/latest"
 	"github.com/docker/cagent/pkg/js"
 	"github.com/docker/cagent/pkg/model/provider"
+	"github.com/docker/cagent/pkg/model/provider/dmr"
 	"github.com/docker/cagent/pkg/model/provider/options"
 	"github.com/docker/cagent/pkg/modelsdev"
 	"github.com/docker/cagent/pkg/permissions"
@@ -157,6 +159,12 @@ func LoadWithConfig(ctx context.Context, agentSource config.Source, runConfig *c
 
 		models, thinkingConfigured, err := getModelsForAgent(ctx, cfg, &agentConfig, autoModel, runConfig)
 		if err != nil {
+			// Return auto model fallback errors and DMR not installed errors directly
+			// without wrapping to provide cleaner messages
+			var autoErr *config.ErrAutoModelFallback
+			if errors.As(err, &autoErr) || errors.Is(err, dmr.ErrNotInstalled) {
+				return nil, err
+			}
 			return nil, fmt.Errorf("failed to get models: %w", err)
 		}
 		for _, model := range models {
@@ -238,9 +246,11 @@ func getModelsForAgent(ctx context.Context, cfg *latest.Config, a *latest.AgentC
 
 	for name := range strings.SplitSeq(a.Model, ",") {
 		modelCfg, exists := cfg.Models[name]
+		isAutoModel := false
 		if !exists {
 			if name == "auto" {
 				modelCfg = autoModelFn()
+				isAutoModel = true
 			} else {
 				return nil, false, fmt.Errorf("model '%s' not found in configuration", name)
 			}
@@ -286,6 +296,10 @@ func getModelsForAgent(ctx context.Context, cfg *latest.Config, a *latest.AgentC
 			opts...,
 		)
 		if err != nil {
+			// Return a cleaner error message for auto model selection failures
+			if isAutoModel {
+				return nil, false, &config.ErrAutoModelFallback{}
+			}
 			return nil, false, err
 		}
 		models = append(models, model)

pkg/teamloader/teamloader_test.go

@@ -2,8 +2,11 @@ package teamloader
 
 import (
 	"context"
+	"errors"
 	"io/fs"
+	"os"
 	"path/filepath"
+	"runtime"
 	"testing"
 
 	"github.com/goccy/go-yaml"
@@ -12,6 +15,8 @@ import (
 
 	"github.com/docker/cagent/pkg/config"
 	"github.com/docker/cagent/pkg/config/latest"
+	"github.com/docker/cagent/pkg/environment"
+	"github.com/docker/cagent/pkg/model/provider/dmr"
 )
 
 // skipExamples contains example files that require cloud-specific configurations
@@ -119,6 +124,11 @@ func TestLoadExamples(t *testing.T) {
 
 			// Then make sure the config loads successfully
 			teams, err := Load(t.Context(), agentSource, runConfig)
+			if err != nil {
+				if errors.Is(err, dmr.ErrNotInstalled) && filepath.Base(agentFilename) == "dmr.yaml" {
+					t.Skip("Skipping DMR example: Docker Model Runner not installed")
+				}
+			}
 			require.NoError(t, err)
 			assert.NotEmpty(t, teams)
 		})
@@ -131,7 +141,13 @@ func TestLoadDefaultAgent(t *testing.T) {
 	agentSource, err := config.Resolve("../../pkg/config/default-agent.yaml")
 	require.NoError(t, err)
 
-	teams, err := Load(t.Context(), agentSource, &config.RuntimeConfig{})
+	runConfig := &config.RuntimeConfig{
+		EnvProviderForTests: environment.NewEnvListProvider([]string{
+			"OPENAI_API_KEY=dummy",
+		}),
+	}
+
+	teams, err := Load(t.Context(), agentSource, runConfig)
 	require.NoError(t, err)
 	require.NotEmpty(t, teams)
 }
@@ -199,6 +215,37 @@ func TestToolsetInstructions(t *testing.T) {
 	require.Equal(t, expected, instructions)
 }
 
+func TestAutoModelFallbackError(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("Skipping docker CLI shim test on Windows")
+	}
+
+	tempDir := t.TempDir()
+	dockerPath := filepath.Join(tempDir, "docker")
+	script := "#!/bin/sh\n" +
+		"printf 'unknown flag: --json\\n\\nUsage:  docker [OPTIONS] COMMAND [ARG...]\\n\\nRun '\\''docker --help'\\'' for more information\\n' >&2\n" +
+		"exit 1\n"
+	require.NoError(t, os.WriteFile(dockerPath, []byte(script), 0o755))
+
+	t.Setenv("PATH", tempDir+string(os.PathListSeparator)+os.Getenv("PATH"))
+	t.Setenv("MODEL_RUNNER_HOST", "")
+
+	agentSource, err := config.Resolve("testdata/auto-model.yaml")
+	require.NoError(t, err)
+
+	// Use noEnvProvider to ensure no API keys are available,
+	// so DMR is the only fallback option.
+	runConfig := &config.RuntimeConfig{
+		EnvProviderForTests: &noEnvProvider{},
+	}
+
+	_, err = Load(t.Context(), agentSource, runConfig)
+	require.Error(t, err)
+
+	var autoErr *config.ErrAutoModelFallback
+	require.ErrorAs(t, err, &autoErr, "expected ErrAutoModelFallback when auto model selection fails")
+}
+
 func TestIsThinkingBudgetDisabled(t *testing.T) {
 	t.Parallel()
 

pkg/teamloader/testdata/auto-model.yaml

@@ -0,0 +1,4 @@
+agents:
+  root:
+    model: auto
+    instruction: Test agent with auto model selection