remove gw from k8s manifest, add agent fix ctrl vault for certs

Author: Flo4604

deployment/docker-compose.yaml

@@ -273,11 +273,17 @@ services:
       UNKEY_CLICKHOUSE_URL: "clickhouse://default:password@clickhouse:9000?secure=false&skip_verify=true"
       UNKEY_REDIS_URL: "redis://redis:6379"
 
+      # Vault - General secrets (env vars, API keys)
       UNKEY_VAULT_S3_URL: "http://s3:3902"
-      UNKEY_VAULT_S3_BUCKET: "acme-vault"
+      UNKEY_VAULT_S3_BUCKET: "vault"
       UNKEY_VAULT_S3_ACCESS_KEY_ID: "minio_root_user"
       UNKEY_VAULT_S3_ACCESS_KEY_SECRET: "minio_root_password"
       UNKEY_VAULT_MASTER_KEYS: "Ch9rZWtfMmdqMFBJdVhac1NSa0ZhNE5mOWlLSnBHenFPENTt7an5MRogENt9Si6wms4pQ2XIvqNSIgNpaBenJmXgcInhu6Nfv2U="
+      # ACME Vault - Let's Encrypt certificates
+      UNKEY_ACME_VAULT_S3_URL: "http://s3:3902"
+      UNKEY_ACME_VAULT_S3_BUCKET: "acme-vault"
+      UNKEY_ACME_VAULT_S3_ACCESS_KEY_ID: "minio_root_user"
+      UNKEY_ACME_VAULT_S3_ACCESS_KEY_SECRET: "minio_root_password"
 
   krane:
     build:
@@ -367,11 +373,17 @@ services:
       UNKEY_RESTATE_HTTP_PORT: "9080"
       UNKEY_RESTATE_REGISTER_AS: "http://ctrl:9080"
 
+      # Vault - General secrets (env vars, API keys)
       UNKEY_VAULT_S3_URL: "http://s3:3902"
-      UNKEY_VAULT_S3_BUCKET: "acme-vault"
+      UNKEY_VAULT_S3_BUCKET: "vault"
       UNKEY_VAULT_S3_ACCESS_KEY_ID: "minio_root_user"
       UNKEY_VAULT_S3_ACCESS_KEY_SECRET: "minio_root_password"
       UNKEY_VAULT_MASTER_KEYS: "Ch9rZWtfMmdqMFBJdVhac1NSa0ZhNE5mOWlLSnBHenFPENTt7an5MRogENt9Si6wms4pQ2XIvqNSIgNpaBenJmXgcInhu6Nfv2U="
+      # ACME Vault - Let's Encrypt certificates
+      UNKEY_ACME_VAULT_S3_URL: "http://s3:3902"
+      UNKEY_ACME_VAULT_S3_BUCKET: "acme-vault"
+      UNKEY_ACME_VAULT_S3_ACCESS_KEY_ID: "minio_root_user"
+      UNKEY_ACME_VAULT_S3_ACCESS_KEY_SECRET: "minio_root_password"
 
       # Build configuration
       UNKEY_BUILD_S3_URL: "${UNKEY_BUILD_S3_URL:-http://s3:3902}"

go/Tiltfile

@@ -26,10 +26,10 @@ start_observability = 'all' in services or 'observability' in services
 start_planetscale = 'all' in services or 'planetscale' in services
 start_restate = 'all' in services or 'restate' in services
 start_api = 'all' in services or 'api' in services
-start_gw = 'all' in services or 'gateway' in services or 'gw' in services
 start_ctrl = 'all' in services or 'ctrl' in services
 start_krane = 'all' in services or 'krane' in services
 start_dashboard = 'all' in services or 'dashboard' in services
+start_agent = 'all' in services or 'agent' in services
 
 # Apply RBAC
 k8s_yaml('k8s/manifests/rbac.yaml')
@@ -95,7 +95,7 @@ if start_s3:
     k8s_yaml('k8s/manifests/s3.yaml')
     k8s_resource(
         's3',
-        port_forwards=['9000:9000', '9001:9001'],
+        port_forwards=['3902:3902', '3903:3903'],
         resource_deps=[],
         labels=['storage']
     )
@@ -141,7 +141,7 @@ if start_observability:
     )
 
 # Build Unkey binary locally (independent of infrastructure)
-if start_api or start_gw or start_ctrl or start_krane:
+if start_api or start_ctrl or start_krane:
     print("Building Unkey binary...")
     # Build locally first for faster updates
     local_resource(
@@ -187,39 +187,6 @@ if start_api:
         trigger_mode=TRIGGER_MODE_MANUAL if debug_mode else TRIGGER_MODE_AUTO
     )
 
-# Gateway service (1 replica)
-if start_gw:
-    print("Setting up Gateway service...")
-
-    docker_build_with_restart(
-        'unkey-gw:latest',
-        '.',
-        dockerfile='Dockerfile.tilt',
-        entrypoint=['/unkey', 'run', 'gw'],
-        only=['./bin'],
-        live_update=[
-            sync('./bin/unkey', '/unkey'),
-        ],
-        ignore=['./cmd/api', './cmd/ctrl', './apps/api', './apps/ctrl']
-    )
-
-    k8s_yaml('k8s/manifests/gw.yaml')
-
-    # Build dependency list
-    gw_deps = []
-    if start_mysql: gw_deps.append('mysql')
-    # Add compilation dependency for Unkey services
-    gw_deps.append('unkey-compile')
-
-    k8s_resource(
-        'gw',
-        port_forwards=['8080:8080', '8443:8443'],
-        resource_deps=gw_deps,
-        labels=['unkey'],
-        auto_init=True,
-        trigger_mode=TRIGGER_MODE_MANUAL if debug_mode else TRIGGER_MODE_AUTO
-    )
-
 # Ctrl service (1 replica)
 if start_ctrl:
     print("Setting up Ctrl service...")
@@ -287,6 +254,30 @@ if start_krane:
         trigger_mode=TRIGGER_MODE_AUTO
     )
 
+# Agent service
+if start_agent:
+    print("Setting up Agent service...")
+    docker_build(
+        'unkey-agent:latest',
+        '../apps/agent',
+        dockerfile='../apps/agent/Dockerfile',
+    )
+    k8s_yaml('k8s/manifests/agent.yaml')
+
+    # Build dependency list
+    agent_deps = []
+    if start_s3: agent_deps.append('s3')
+    if start_clickhouse: agent_deps.append('clickhouse')
+
+    k8s_resource(
+        'agent',
+        port_forwards='8082:8080',
+        resource_deps=agent_deps,
+        labels=['unkey'],
+        auto_init=True,
+        trigger_mode=TRIGGER_MODE_AUTO
+    )
+
 # Dashboard service
 if start_dashboard:
     print("Setting up Dashboard service...")
@@ -306,6 +297,7 @@ if start_dashboard:
     dashboard_deps = []
     if start_planetscale: dashboard_deps.append('planetscale')
     if start_clickhouse: dashboard_deps.append('clickhouse')
+    if start_agent: dashboard_deps.append('agent')
 
     k8s_resource(
         'dashboard',
@@ -326,9 +318,9 @@ if start_planetscale: active_services.append('planetscale')
 if start_observability: active_services.extend(['prometheus', 'otel-collector'])
 if start_restate: active_services.append('restate')
 if start_api: active_services.append('api')
-if start_gw: active_services.append('gw')
 if start_ctrl: active_services.append('ctrl')
 if start_dashboard: active_services.append('dashboard')
+if start_agent: active_services.append('agent')
 
 print("""
 Tilt is ready!
@@ -340,13 +332,14 @@ Web UI: http://localhost:10350
 Services available via Tilt port forwards:
 Dashboard: http://localhost:3000
 API: http://localhost:7070
-Gateway: http://localhost:8080
 Ctrl: http://localhost:7091
 Krane: http://localhost:8090
+Agent: http://localhost:8082
 Restate Ingress: http://localhost:8081
 Restate Admin: http://localhost:9070
 Prometheus: http://localhost:9090
-S3 Console: http://localhost:9000
+S3 API: http://localhost:3902
+S3 Console: http://localhost:3903
 ClickHouse: http://localhost:8123
 
 Tips:

go/apps/ctrl/config.go

@@ -125,8 +125,12 @@ type Config struct {
 	Clock clock.Clock
 
 	// --- Vault Configuration ---
+	// VaultMasterKeys are the master encryption keys for both vaults
 	VaultMasterKeys []string
-	VaultS3         S3Config
+	// VaultS3 is used for general secrets (env vars, API keys, etc.)
+	VaultS3 S3Config
+	// AcmeVaultS3 is used specifically for ACME/Let's Encrypt certificate storage
+	AcmeVaultS3 S3Config
 
 	// --- ACME/Cloudflare Configuration ---
 	Acme AcmeConfig

go/apps/ctrl/run.go

@@ -82,18 +82,18 @@ func Run(ctx context.Context, cfg Config) error {
 		logger.Info("TLS is enabled, server will use HTTPS")
 	}
 
+	// Create vault service for general secrets (env vars, API keys, etc.)
 	var vaultSvc *vault.Service
-	if len(cfg.VaultMasterKeys) > 0 {
-		var vaultStorage storage.Storage
-		vaultStorage, err = storage.NewS3(storage.S3Config{
+	if len(cfg.VaultMasterKeys) > 0 && cfg.VaultS3.URL != "" {
+		vaultStorage, vaultStorageErr := storage.NewS3(storage.S3Config{
 			Logger:            logger,
 			S3URL:             cfg.VaultS3.URL,
 			S3Bucket:          cfg.VaultS3.Bucket,
 			S3AccessKeyID:     cfg.VaultS3.AccessKeyID,
 			S3AccessKeySecret: cfg.VaultS3.AccessKeySecret,
 		})
-		if err != nil {
-			return fmt.Errorf("unable to create vault storage: %w", err)
+		if vaultStorageErr != nil {
+			return fmt.Errorf("unable to create vault storage: %w", vaultStorageErr)
 		}
 
 		vaultSvc, err = vault.New(vault.Config{
@@ -104,6 +104,32 @@ func Run(ctx context.Context, cfg Config) error {
 		if err != nil {
 			return fmt.Errorf("unable to create vault service: %w", err)
 		}
+		logger.Info("Vault service initialized", "bucket", cfg.VaultS3.Bucket)
+	}
+
+	// Create separate vault service for ACME certificates
+	var acmeVaultSvc *vault.Service
+	if len(cfg.VaultMasterKeys) > 0 && cfg.AcmeVaultS3.URL != "" {
+		acmeVaultStorage, acmeStorageErr := storage.NewS3(storage.S3Config{
+			Logger:            logger,
+			S3URL:             cfg.AcmeVaultS3.URL,
+			S3Bucket:          cfg.AcmeVaultS3.Bucket,
+			S3AccessKeyID:     cfg.AcmeVaultS3.AccessKeyID,
+			S3AccessKeySecret: cfg.AcmeVaultS3.AccessKeySecret,
+		})
+		if acmeStorageErr != nil {
+			return fmt.Errorf("unable to create ACME vault storage: %w", acmeStorageErr)
+		}
+
+		acmeVaultSvc, err = vault.New(vault.Config{
+			Logger:     logger,
+			Storage:    acmeVaultStorage,
+			MasterKeys: cfg.VaultMasterKeys,
+		})
+		if err != nil {
+			return fmt.Errorf("unable to create ACME vault service: %w", err)
+		}
+		logger.Info("ACME vault service initialized", "bucket", cfg.AcmeVaultS3.Bucket)
 	}
 
 	// Initialize database
@@ -247,7 +273,7 @@ func Run(ctx context.Context, cfg Config) error {
 	restateSrv.Bind(hydrav1.NewCertificateServiceServer(certificate.New(certificate.Config{
 		Logger: logger,
 		DB:     database,
-		Vault:  vaultSvc,
+		Vault:  acmeVaultSvc,
 	})))
 	restateSrv.Bind(hydrav1.NewProjectServiceServer(projectWorkflow.New(projectWorkflow.Config{
 		Logger: logger,

go/cmd/ctrl/main.go

@@ -60,17 +60,27 @@ var Cmd = &cli.Command{
 		cli.String("spiffe-socket-path", "Path to SPIFFE agent socket for mTLS authentication. Default: /var/lib/spire/agent/agent.sock",
 			cli.Default("/var/lib/spire/agent/agent.sock"), cli.EnvVar("UNKEY_SPIFFE_SOCKET_PATH")),
 
-		// Vault Configuration
-		cli.StringSlice("vault-master-keys", "Vault master keys for encryption",
+		// Vault Configuration - General secrets (env vars, API keys)
+		cli.StringSlice("vault-master-keys", "Vault master keys for encryption (shared by both vaults)",
 			cli.Required(), cli.EnvVar("UNKEY_VAULT_MASTER_KEYS")),
-		cli.String("vault-s3-url", "S3 Compatible Endpoint URL",
-			cli.Required(), cli.EnvVar("UNKEY_VAULT_S3_URL")),
-		cli.String("vault-s3-bucket", "S3 bucket name",
-			cli.Required(), cli.EnvVar("UNKEY_VAULT_S3_BUCKET")),
-		cli.String("vault-s3-access-key-id", "S3 access key ID",
-			cli.Required(), cli.EnvVar("UNKEY_VAULT_S3_ACCESS_KEY_ID")),
-		cli.String("vault-s3-access-key-secret", "S3 secret access key",
-			cli.Required(), cli.EnvVar("UNKEY_VAULT_S3_ACCESS_KEY_SECRET")),
+		cli.String("vault-s3-url", "S3 endpoint URL for general vault",
+			cli.EnvVar("UNKEY_VAULT_S3_URL")),
+		cli.String("vault-s3-bucket", "S3 bucket for general vault (env vars, API keys)",
+			cli.EnvVar("UNKEY_VAULT_S3_BUCKET")),
+		cli.String("vault-s3-access-key-id", "S3 access key ID for general vault",
+			cli.EnvVar("UNKEY_VAULT_S3_ACCESS_KEY_ID")),
+		cli.String("vault-s3-access-key-secret", "S3 secret access key for general vault",
+			cli.EnvVar("UNKEY_VAULT_S3_ACCESS_KEY_SECRET")),
+
+		// ACME Vault Configuration - Let's Encrypt certificates
+		cli.String("acme-vault-s3-url", "S3 endpoint URL for ACME vault",
+			cli.EnvVar("UNKEY_ACME_VAULT_S3_URL")),
+		cli.String("acme-vault-s3-bucket", "S3 bucket for ACME vault (Let's Encrypt certs)",
+			cli.EnvVar("UNKEY_ACME_VAULT_S3_BUCKET")),
+		cli.String("acme-vault-s3-access-key-id", "S3 access key ID for ACME vault",
+			cli.EnvVar("UNKEY_ACME_VAULT_S3_ACCESS_KEY_ID")),
+		cli.String("acme-vault-s3-access-key-secret", "S3 secret access key for ACME vault",
+			cli.EnvVar("UNKEY_ACME_VAULT_S3_ACCESS_KEY_SECRET")),
 
 		// Build Configuration
 		cli.String("build-backend", "Build backend to use: 'docker' for local, 'depot' for production. Default: depot",
@@ -167,14 +177,20 @@ func action(ctx context.Context, cmd *cli.Command) error {
 		APIKey:           cmd.String("api-key"),
 		SPIFFESocketPath: cmd.String("spiffe-socket-path"),
 
-		// Vault configuration
+		// Vault configuration - General secrets
 		VaultMasterKeys: cmd.StringSlice("vault-master-keys"),
 		VaultS3: ctrl.S3Config{
-			ExternalURL:     cmd.String(""),
 			URL:             cmd.String("vault-s3-url"),
 			Bucket:          cmd.String("vault-s3-bucket"),
-			AccessKeySecret: cmd.String("vault-s3-access-key-secret"),
 			AccessKeyID:     cmd.String("vault-s3-access-key-id"),
+			AccessKeySecret: cmd.String("vault-s3-access-key-secret"),
+		},
+		// ACME Vault configuration - Let's Encrypt certificates
+		AcmeVaultS3: ctrl.S3Config{
+			URL:             cmd.String("acme-vault-s3-url"),
+			Bucket:          cmd.String("acme-vault-s3-bucket"),
+			AccessKeyID:     cmd.String("acme-vault-s3-access-key-id"),
+			AccessKeySecret: cmd.String("acme-vault-s3-access-key-secret"),
 		},
 
 		// Build configuration

go/k8s/manifests/agent.yaml

@@ -0,0 +1,76 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: agent
+  namespace: unkey
+  labels:
+    app: agent
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: agent
+  template:
+    metadata:
+      labels:
+        app: agent
+    spec:
+      containers:
+        - name: agent
+          image: unkey-agent:latest
+          imagePullPolicy: Never
+          ports:
+            - containerPort: 8080
+            - containerPort: 9095
+          env:
+            - name: PORT
+              value: "8080"
+            - name: RPC_PORT
+              value: "9095"
+            - name: AUTH_TOKEN
+              value: "agent-auth-secret"
+            - name: VAULT_S3_URL
+              value: "http://s3:3902"
+            - name: VAULT_S3_BUCKET
+              value: "vault"
+            - name: VAULT_S3_ACCESS_KEY_ID
+              value: "minio_root_user"
+            - name: VAULT_S3_ACCESS_KEY_SECRET
+              value: "minio_root_password"
+            - name: VAULT_MASTER_KEYS
+              value: "Ch9rZWtfMmdqMFBJdVhac1NSa0ZhNE5mOWlLSnBHenFPENTt7an5MRogENt9Si6wms4pQ2XIvqNSIgNpaBenJmXgcInhu6Nfv2U="
+            - name: CLICKHOUSE_URL
+              value: "clickhouse://default:password@clickhouse:9000"
+          command: ["/usr/local/bin/unkey", "agent", "--config", "config.docker.json"]
+      initContainers:
+        - name: wait-for-dependencies
+          image: busybox:1.36
+          command:
+            [
+              "sh",
+              "-c",
+              "until nc -z s3 3902 && nc -z clickhouse 9000; do echo waiting for dependencies; sleep 2; done;",
+            ]
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: agent
+  namespace: unkey
+  labels:
+    app: agent
+spec:
+  selector:
+    app: agent
+  ports:
+    - name: http
+      port: 8080
+      targetPort: 8080
+      protocol: TCP
+    - name: rpc
+      port: 9095
+      targetPort: 9095
+      protocol: TCP
+  type: ClusterIP