Fix missing data when Caddy isn't available at startup

Author: alexandre-daubois

docs/caddy-configuration.md

@@ -46,7 +46,7 @@ If auto-detection does not work, you can specify the process ID manually:
 ember --pid 12345
 ```
 
-When no `--pid` is provided, Ember scans the process list for a FrankenPHP process. If none is found, it falls back to scanning for a Caddy process for CPU/RSS monitoring.
+When no `--pid` is provided, Ember scans the process list for a FrankenPHP process. If none is found, it falls back to scanning for a Caddy process for CPU/RSS monitoring. When process scanning is unavailable, Ember derives CPU and RSS from the standard Go `process_*` Prometheus metrics exposed by Caddy's `/metrics` endpoint.
 
 ## Remote Caddy Instances
 

internal/fetcher/fetcher.go

@@ -67,6 +67,11 @@ type MetricsSnapshot struct {
 
 	// Per-host Caddy HTTP metrics
 	Hosts map[string]*HostMetrics `json:"hosts,omitempty"`
+
+	// Go runtime process metrics (from standard Prometheus collector)
+	ProcessCPUSecondsTotal  float64 `json:"processCpuSecondsTotal,omitempty"`
+	ProcessRSSBytes         float64 `json:"processRssBytes,omitempty"`
+	ProcessStartTimeSeconds float64 `json:"processStartTimeSeconds,omitempty"`
 }
 
 type HistogramBucket struct {
@@ -83,11 +88,12 @@ type ProcessMetrics struct {
 }
 
 type Snapshot struct {
-	Threads   ThreadsResponse `json:"threads"`
-	Metrics   MetricsSnapshot `json:"metrics"`
-	Process   ProcessMetrics  `json:"process"`
-	FetchedAt time.Time       `json:"fetchedAt"`
-	Errors    []string        `json:"errors,omitempty"`
+	Threads       ThreadsResponse `json:"threads"`
+	Metrics       MetricsSnapshot `json:"metrics"`
+	Process       ProcessMetrics  `json:"process"`
+	FetchedAt     time.Time       `json:"fetchedAt"`
+	Errors        []string        `json:"errors,omitempty"`
+	HasFrankenPHP bool            `json:"hasFrankenPHP"`
 }
 
 type Fetcher interface {

internal/fetcher/http.go

@@ -26,6 +26,9 @@ type HTTPFetcher struct {
 	procHandle    *processHandle
 	hasFrankenPHP bool
 	serverNames   []string
+
+	lastPromCPU    float64
+	lastPromSample time.Time
 }
 
 func NewHTTPFetcher(baseURL string, pid int32) *HTTPFetcher {
@@ -105,18 +108,19 @@ func (f *HTTPFetcher) ServerNames() []string {
 
 func (f *HTTPFetcher) Fetch(ctx context.Context) (*Snapshot, error) {
 	var (
-		threads ThreadsResponse
-		metrics MetricsSnapshot
-		proc    ProcessMetrics
-		mu      sync.Mutex
-		errs    []string
+		threads   ThreadsResponse
+		metrics   MetricsSnapshot
+		proc      ProcessMetrics
+		mu        sync.Mutex
+		errs      []string
+		metricsOK bool
 	)
 
-	g, ctx := errgroup.WithContext(ctx)
+	g, gCtx := errgroup.WithContext(ctx)
 
 	if f.hasFrankenPHP {
 		g.Go(func() error {
-			t, err := f.fetchThreads(ctx)
+			t, err := f.fetchThreads(gCtx)
 			if err != nil {
 				mu.Lock()
 				errs = append(errs, err.Error())
@@ -129,19 +133,22 @@ func (f *HTTPFetcher) Fetch(ctx context.Context) (*Snapshot, error) {
 	}
 
 	g.Go(func() error {
-		m, err := f.fetchMetrics(ctx)
+		m, err := f.fetchMetrics(gCtx)
 		if err != nil {
 			mu.Lock()
 			errs = append(errs, err.Error())
 			mu.Unlock()
 			return nil
 		}
+		mu.Lock()
+		metricsOK = true
+		mu.Unlock()
 		metrics = m
 		return nil
 	})
 
 	g.Go(func() error {
-		p, err := f.procHandle.fetch(ctx)
+		p, err := f.procHandle.fetch(gCtx)
 		if err != nil {
 			mu.Lock()
 			errs = append(errs, err.Error())
@@ -154,6 +161,34 @@ func (f *HTTPFetcher) Fetch(ctx context.Context) (*Snapshot, error) {
 
 	_ = g.Wait()
 
+	if metricsOK {
+		f.onConnected(ctx)
+
+		// Derive process metrics from Prometheus when gopsutil has no data
+		if proc.RSS == 0 && metrics.ProcessRSSBytes > 0 {
+			proc.RSS = uint64(metrics.ProcessRSSBytes)
+		}
+		if proc.CPUPercent == 0 && metrics.ProcessCPUSecondsTotal > 0 {
+			now := time.Now()
+			if !f.lastPromSample.IsZero() {
+				elapsed := now.Sub(f.lastPromSample).Seconds()
+				if elapsed > 0 {
+					proc.CPUPercent = (metrics.ProcessCPUSecondsTotal - f.lastPromCPU) / elapsed * 100
+					if proc.CPUPercent < 0 {
+						proc.CPUPercent = 0
+					}
+				}
+			}
+			f.lastPromCPU = metrics.ProcessCPUSecondsTotal
+			f.lastPromSample = now
+		}
+		if proc.CreateTime == 0 && metrics.ProcessStartTimeSeconds > 0 {
+			startSec := int64(metrics.ProcessStartTimeSeconds)
+			proc.CreateTime = startSec * 1000
+			proc.Uptime = time.Since(time.Unix(startSec, 0))
+		}
+	}
+
 	// Seed known server names as empty host entries so they appear immediately
 	if len(f.serverNames) > 0 && metrics.Hosts != nil {
 		for _, name := range f.serverNames {
@@ -168,14 +203,24 @@ func (f *HTTPFetcher) Fetch(ctx context.Context) (*Snapshot, error) {
 	}
 
 	return &Snapshot{
-		Threads:   threads,
-		Metrics:   metrics,
-		Process:   proc,
-		FetchedAt: time.Now(),
-		Errors:    errs,
+		Threads:       threads,
+		Metrics:       metrics,
+		Process:       proc,
+		FetchedAt:     time.Now(),
+		Errors:        errs,
+		HasFrankenPHP: f.hasFrankenPHP,
 	}, nil
 }
 
+func (f *HTTPFetcher) onConnected(ctx context.Context) {
+	if !f.hasFrankenPHP {
+		f.DetectFrankenPHP(ctx)
+	}
+	if len(f.serverNames) == 0 {
+		f.FetchServerNames(ctx)
+	}
+}
+
 func (f *HTTPFetcher) RestartWorkers(ctx context.Context) error {
 	ctx, cancel := context.WithTimeout(ctx, requestTimeout)
 	defer cancel()

internal/fetcher/http_test.go

@@ -406,6 +406,222 @@ caddy_http_request_duration_seconds_count{server="main"} 100
 	assert.Equal(t, float64(100), main.RequestsTotal, "seeding should not overwrite existing host data")
 }
 
+func TestOnConnected_DetectsFrankenPHP(t *testing.T) {
+	frankenPHPAvailable := false
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.Path {
+		case "/frankenphp/threads":
+			if frankenPHPAvailable {
+				w.WriteHeader(200)
+				json.NewEncoder(w).Encode(ThreadsResponse{
+					ThreadDebugStates: []ThreadDebugState{{Index: 0, State: "ready"}},
+				})
+			} else {
+				w.WriteHeader(404)
+			}
+		case "/metrics":
+			w.WriteHeader(200)
+		default:
+			w.WriteHeader(404)
+		}
+	}))
+	defer srv.Close()
+
+	f := NewHTTPFetcher(srv.URL, 0)
+
+	snap, err := f.Fetch(context.Background())
+	require.NoError(t, err)
+	assert.False(t, snap.HasFrankenPHP, "should not detect FrankenPHP when unavailable")
+	assert.Empty(t, snap.Threads.ThreadDebugStates)
+
+	frankenPHPAvailable = true
+
+	snap, err = f.Fetch(context.Background())
+	require.NoError(t, err)
+	assert.True(t, snap.HasFrankenPHP, "should detect FrankenPHP on next successful fetch")
+	// Threads are fetched on the NEXT Fetch() after detection
+	snap, err = f.Fetch(context.Background())
+	require.NoError(t, err)
+	assert.Len(t, snap.Threads.ThreadDebugStates, 1)
+}
+
+func TestOnConnected_FetchesServerNames(t *testing.T) {
+	serverNamesAvailable := false
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.Path {
+		case "/config/apps/http/servers":
+			if serverNamesAvailable {
+				w.WriteHeader(200)
+				json.NewEncoder(w).Encode(map[string]any{
+					"main": map[string]any{"listen": []string{":443"}},
+				})
+			} else {
+				w.WriteHeader(404)
+			}
+		case "/metrics":
+			w.WriteHeader(200)
+		default:
+			w.WriteHeader(404)
+		}
+	}))
+	defer srv.Close()
+
+	f := NewHTTPFetcher(srv.URL, 0)
+
+	f.Fetch(context.Background())
+	assert.Empty(t, f.ServerNames())
+
+	serverNamesAvailable = true
+
+	snap, err := f.Fetch(context.Background())
+	require.NoError(t, err)
+	assert.Equal(t, []string{"main"}, f.ServerNames())
+	require.NotNil(t, snap.Metrics.Hosts)
+	assert.Contains(t, snap.Metrics.Hosts, "main")
+}
+
+func TestOnConnected_NoRetryWhenMetricsFail(t *testing.T) {
+	var detectCalls atomic.Int32
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.Path {
+		case "/frankenphp/threads":
+			detectCalls.Add(1)
+			w.WriteHeader(404)
+		case "/metrics":
+			w.WriteHeader(500)
+		default:
+			w.WriteHeader(404)
+		}
+	}))
+	defer srv.Close()
+
+	f := NewHTTPFetcher(srv.URL, 0)
+
+	f.Fetch(context.Background())
+	f.Fetch(context.Background())
+	f.Fetch(context.Background())
+
+	assert.Equal(t, int32(0), detectCalls.Load(), "should not attempt detection when metrics fail")
+}
+
+func TestOnConnected_StopsAfterSuccess(t *testing.T) {
+	var detectCalls atomic.Int32
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.Path {
+		case "/frankenphp/threads":
+			detectCalls.Add(1)
+			w.WriteHeader(200)
+			json.NewEncoder(w).Encode(ThreadsResponse{})
+		case "/metrics":
+			w.WriteHeader(200)
+		default:
+			w.WriteHeader(404)
+		}
+	}))
+	defer srv.Close()
+
+	f := NewHTTPFetcher(srv.URL, 0)
+
+	f.Fetch(context.Background())
+	f.Fetch(context.Background())
+	f.Fetch(context.Background())
+
+	// Detection is called once (first fetch), then stops because hasFrankenPHP is true
+	// But fetchThreads also hits /frankenphp/threads in subsequent fetches
+	assert.True(t, f.HasFrankenPHP())
+}
+
+func TestFetch_HasFrankenPHPInSnapshot(t *testing.T) {
+	srv := newTestServer(200, ThreadsResponse{}, 200, "")
+	defer srv.Close()
+
+	f := NewHTTPFetcher(srv.URL, 0)
+	f.hasFrankenPHP = true
+
+	snap, err := f.Fetch(context.Background())
+	require.NoError(t, err)
+	assert.True(t, snap.HasFrankenPHP)
+}
+
+func TestFetch_NoFrankenPHPInSnapshot(t *testing.T) {
+	srv := newTestServer(404, nil, 200, "")
+	defer srv.Close()
+
+	f := NewHTTPFetcher(srv.URL, 0)
+
+	snap, err := f.Fetch(context.Background())
+	require.NoError(t, err)
+	assert.False(t, snap.HasFrankenPHP)
+}
+
+func TestFetch_PrometheusProcessFallback_RSS(t *testing.T) {
+	metricsText := `# TYPE process_resident_memory_bytes gauge
+process_resident_memory_bytes 1.048576e+07
+# TYPE process_cpu_seconds_total counter
+process_cpu_seconds_total 12.5
+# TYPE process_start_time_seconds gauge
+process_start_time_seconds 1.7e+09
+`
+	srv := newTestServer(404, nil, 200, metricsText)
+	defer srv.Close()
+
+	f := NewHTTPFetcher(srv.URL, 0)
+
+	snap, err := f.Fetch(context.Background())
+	require.NoError(t, err)
+	assert.Equal(t, uint64(10485760), snap.Process.RSS, "RSS should come from Prometheus metrics")
+	assert.True(t, snap.Process.Uptime > 0, "Uptime should be derived from process_start_time_seconds")
+	assert.True(t, snap.Process.CreateTime > 0, "CreateTime should be derived from process_start_time_seconds")
+}
+
+func TestFetch_PrometheusProcessFallback_CPU(t *testing.T) {
+	metricsText := `# TYPE process_cpu_seconds_total counter
+process_cpu_seconds_total 10.0
+`
+	srv := newTestServer(404, nil, 200, metricsText)
+	defer srv.Close()
+
+	f := NewHTTPFetcher(srv.URL, 0)
+
+	// First fetch: records baseline, CPU=0 (no previous sample)
+	snap, err := f.Fetch(context.Background())
+	require.NoError(t, err)
+	assert.Equal(t, float64(0), snap.Process.CPUPercent, "first fetch has no delta yet")
+
+	// Simulate time passing and CPU usage increasing
+	f.lastPromSample = f.lastPromSample.Add(-1 * time.Second)
+	f.lastPromCPU = 10.0
+
+	metricsText2 := `# TYPE process_cpu_seconds_total counter
+process_cpu_seconds_total 10.5
+`
+	srv.Close()
+	srv2 := newTestServer(404, nil, 200, metricsText2)
+	defer srv2.Close()
+	f.baseURL = srv2.URL
+
+	snap, err = f.Fetch(context.Background())
+	require.NoError(t, err)
+	assert.True(t, snap.Process.CPUPercent > 0, "CPU should be derived from Prometheus delta")
+}
+
+func TestFetch_PrometheusProcessFallback_NotUsedWhenGopsutilWorks(t *testing.T) {
+	metricsText := `# TYPE process_resident_memory_bytes gauge
+process_resident_memory_bytes 1.048576e+07
+`
+	srv := newTestServer(404, nil, 200, metricsText)
+	defer srv.Close()
+
+	f := NewHTTPFetcher(srv.URL, 0)
+	// Simulate gopsutil having found the process and returned real RSS
+	f.procHandle.proc = nil // no proc, but we'll set proc directly
+	// We can't easily simulate gopsutil success in tests, so just verify
+	// that the fallback IS used when proc.RSS == 0
+	snap, err := f.Fetch(context.Background())
+	require.NoError(t, err)
+	assert.Equal(t, uint64(10485760), snap.Process.RSS)
+}
+
 func TestFetchThreads_PerRequestTimeout(t *testing.T) {
 	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		time.Sleep(requestTimeout + time.Second)