feat: enable Azure Container Apps as optional execution method (#5646)

packages/evals/azure/deploy.sh

@@ -0,0 +1,77 @@
+#!/bin/bash
+
+# Azure Container Apps Deployment Script for Roo Code Evals
+# This script deploys the Azure infrastructure needed for Azure Container Apps execution
+
+set -e
+
+# Configuration
+RESOURCE_GROUP_NAME="${AZURE_RESOURCE_GROUP_NAME:-roo-code-evals}"
+LOCATION="${AZURE_LOCATION:-eastus}"
+SUBSCRIPTION_ID="${AZURE_SUBSCRIPTION_ID}"
+
+# Validate required environment variables
+if [ -z "$SUBSCRIPTION_ID" ]; then
+    echo "Error: AZURE_SUBSCRIPTION_ID environment variable is required"
+    exit 1
+fi
+
+echo "🚀 Starting Azure Container Apps deployment..."
+echo "📍 Resource Group: $RESOURCE_GROUP_NAME"
+echo "🌍 Location: $LOCATION"
+echo "🔑 Subscription: $SUBSCRIPTION_ID"
+
+# Login to Azure (if not already logged in)
+echo "🔐 Checking Azure login status..."
+if ! az account show &>/dev/null; then
+    echo "Please log in to Azure:"
+    az login
+fi
+
+# Set the subscription
+echo "📋 Setting Azure subscription..."
+az account set --subscription "$SUBSCRIPTION_ID"
+
+# Create resource group if it doesn't exist
+echo "📦 Creating resource group..."
+az group create \
+    --name "$RESOURCE_GROUP_NAME" \
+    --location "$LOCATION" \
+    --output table
+
+# Deploy the Bicep template
+echo "🏗️ Deploying Azure Container Apps infrastructure..."
+az deployment group create \
+    --resource-group "$RESOURCE_GROUP_NAME" \
+    --template-file main.bicep \
+    --parameters main.bicepparam \
+    --output table
+
+# Get deployment outputs
+echo "📤 Retrieving deployment outputs..."
+CONTAINER_APP_ENV_ID=$(az deployment group show \
+    --resource-group "$RESOURCE_GROUP_NAME" \
+    --name main \
+    --query 'properties.outputs.containerAppEnvironmentId.value' \
+    --output tsv)
+
+CONTAINER_APP_ENV_NAME=$(az deployment group show \
+    --resource-group "$RESOURCE_GROUP_NAME" \
+    --name main \
+    --query 'properties.outputs.containerAppEnvironmentName.value' \
+    --output tsv)
+
+echo "✅ Deployment completed successfully!"
+echo ""
+echo "📋 Deployment Summary:"
+echo "   Resource Group: $RESOURCE_GROUP_NAME"
+echo "   Container App Environment: $CONTAINER_APP_ENV_NAME"
+echo "   Container App Environment ID: $CONTAINER_APP_ENV_ID"
+echo ""
+echo "🔧 To use Azure Container Apps execution, set these environment variables:"
+echo "   export AZURE_SUBSCRIPTION_ID=$SUBSCRIPTION_ID"
+echo "   export AZURE_RESOURCE_GROUP_NAME=$RESOURCE_GROUP_NAME"
+echo "   export AZURE_CONTAINER_APP_ENVIRONMENT_NAME=$CONTAINER_APP_ENV_NAME"
+echo "   export HOST_EXECUTION_METHOD=azure-container-apps"
+echo ""
+echo "📚 For more information, see the Azure Container Apps documentation."

packages/evals/azure/main.bicep

@@ -0,0 +1,174 @@
+@description('The name of the Container App Environment')
+param containerAppEnvironmentName string = 'evals-env'
+
+@description('The name of the Container App')
+param containerAppName string = 'evals-runner'
+
+@description('The location for all resources')
+param location string = resourceGroup().location
+
+@description('The container registry server')
+param containerRegistryServer string
+
+@description('The container image')
+param containerImage string
+
+@description('The container registry username')
+@secure()
+param containerRegistryUsername string
+
+@description('The container registry password')
+@secure()
+param containerRegistryPassword string
+
+@description('The database connection string')
+@secure()
+param databaseUrl string
+
+@description('The Redis connection string')
+@secure()
+param redisUrl string
+
+@description('The OpenRouter API key')
+@secure()
+param openRouterApiKey string
+
+// Log Analytics Workspace
+resource logAnalyticsWorkspace 'Microsoft.OperationalInsights/workspaces@2022-10-01' = {
+  name: '${containerAppEnvironmentName}-logs'
+  location: location
+  properties: {
+    sku: {
+      name: 'PerGB2018'
+    }
+    retentionInDays: 30
+  }
+}
+
+// Container App Environment
+resource containerAppEnvironment 'Microsoft.App/managedEnvironments@2023-05-01' = {
+  name: containerAppEnvironmentName
+  location: location
+  properties: {
+    appLogsConfiguration: {
+      destination: 'log-analytics'
+      logAnalyticsConfiguration: {
+        customerId: logAnalyticsWorkspace.properties.customerId
+        sharedKey: logAnalyticsWorkspace.listKeys().primarySharedKey
+      }
+    }
+  }
+}
+
+// Container Registry Secret
+resource containerRegistrySecret 'Microsoft.App/managedEnvironments/secrets@2023-05-01' = {
+  parent: containerAppEnvironment
+  name: 'container-registry-password'
+  properties: {
+    value: containerRegistryPassword
+  }
+}
+
+// Database URL Secret
+resource databaseUrlSecret 'Microsoft.App/managedEnvironments/secrets@2023-05-01' = {
+  parent: containerAppEnvironment
+  name: 'database-url'
+  properties: {
+    value: databaseUrl
+  }
+}
+
+// Redis URL Secret
+resource redisUrlSecret 'Microsoft.App/managedEnvironments/secrets@2023-05-01' = {
+  parent: containerAppEnvironment
+  name: 'redis-url'
+  properties: {
+    value: redisUrl
+  }
+}
+
+// OpenRouter API Key Secret
+resource openRouterApiKeySecret 'Microsoft.App/managedEnvironments/secrets@2023-05-01' = {
+  parent: containerAppEnvironment
+  name: 'openrouter-api-key'
+  properties: {
+    value: openRouterApiKey
+  }
+}
+
+// Container App for Jobs (this will be used as a template for job executions)
+resource containerApp 'Microsoft.App/containerApps@2023-05-01' = {
+  name: containerAppName
+  location: location
+  properties: {
+    managedEnvironmentId: containerAppEnvironment.id
+    configuration: {
+      secrets: [
+        {
+          name: 'container-registry-password'
+          value: containerRegistryPassword
+        }
+        {
+          name: 'database-url'
+          value: databaseUrl
+        }
+        {
+          name: 'redis-url'
+          value: redisUrl
+        }
+        {
+          name: 'openrouter-api-key'
+          value: openRouterApiKey
+        }
+      ]
+      registries: [
+        {
+          server: containerRegistryServer
+          username: containerRegistryUsername
+          passwordSecretRef: 'container-registry-password'
+        }
+      ]
+    }
+    template: {
+      containers: [
+        {
+          name: 'evals-runner'
+          image: containerImage
+          env: [
+            {
+              name: 'HOST_EXECUTION_METHOD'
+              value: 'azure-container-apps'
+            }
+            {
+              name: 'DATABASE_URL'
+              secretRef: 'database-url'
+            }
+            {
+              name: 'REDIS_URL'
+              secretRef: 'redis-url'
+            }
+            {
+              name: 'OPENROUTER_API_KEY'
+              secretRef: 'openrouter-api-key'
+            }
+          ]
+          resources: {
+            cpu: 1
+            memory: '2Gi'
+          }
+        }
+      ]
+      scale: {
+        minReplicas: 0
+        maxReplicas: 10
+      }
+    }
+  }
+}
+
+// Output values
+output containerAppEnvironmentId string = containerAppEnvironment.id
+output containerAppEnvironmentName string = containerAppEnvironment.name
+output containerAppId string = containerApp.id
+output containerAppName string = containerApp.name
+output logAnalyticsWorkspaceId string = logAnalyticsWorkspace.id

packages/evals/azure/main.bicepparam

@@ -0,0 +1,12 @@
+using './main.bicep'
+
+param containerAppEnvironmentName = 'evals-env'
+param containerAppName = 'evals-runner'
+param location = 'East US'
+param containerRegistryServer = 'your-registry.azurecr.io'
+param containerImage = 'your-registry.azurecr.io/evals-runner:latest'
+param containerRegistryUsername = 'your-registry-username'
+param containerRegistryPassword = 'your-registry-password'
+param databaseUrl = 'postgres://username:password@hostname:5432/database'
+param redisUrl = 'redis://hostname:6379'
+param openRouterApiKey = 'your-openrouter-api-key'

packages/evals/docker-compose.yml

@@ -12,6 +12,9 @@
 #
 # To enable docker execution, run:
 # docker run -it --rm --network evals_default -v /var/run/docker.sock:/var/run/docker.sock -e HOST_EXECUTION_METHOD=docker evals-runner bash
+#
+# To enable Azure Container Apps execution, run:
+# docker run -it --rm --network evals_default -e HOST_EXECUTION_METHOD=azure-container-apps evals-runner bash
 
 services:
     db: