Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 3 additions & 5 deletions .github/workflows/ci-all.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,27 +11,25 @@ on:

permissions:
contents: read
# Lets the test workflows mint the GitHub OIDC token they exchange for a
# Claude API access token (workload identity federation). See docs/setup.md.
id-token: write

jobs:
ci:
uses: ./.github/workflows/ci.yml

test-base-action:
uses: ./.github/workflows/test-base-action.yml
secrets: inherit # Required for ANTHROPIC_API_KEY

test-custom-executables:
uses: ./.github/workflows/test-custom-executables.yml
secrets: inherit

test-mcp-servers:
uses: ./.github/workflows/test-mcp-servers.yml
secrets: inherit

test-settings:
uses: ./.github/workflows/test-settings.yml
secrets: inherit

test-structured-output:
uses: ./.github/workflows/test-structured-output.yml
secrets: inherit
15 changes: 13 additions & 2 deletions .github/workflows/test-base-action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,13 @@ on:
default: "List the files in the current directory starting with 'package'"
workflow_call:

# The Claude API is authenticated via workload identity federation: id-token
# lets the action mint the GitHub OIDC token it exchanges for a short-lived
# access token. See docs/setup.md.
permissions:
contents: read
id-token: write

jobs:
test-inline-prompt:
runs-on: ubuntu-latest
Expand All @@ -21,7 +28,9 @@ jobs:
uses: ./base-action
with:
prompt: ${{ github.event.inputs.test_prompt || 'List the files in the current directory starting with "package"' }}
anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }}
anthropic_federation_rule_id: ${{ vars.ANTHROPIC_FEDERATION_RULE_ID }}
anthropic_organization_id: ${{ vars.ANTHROPIC_ORGANIZATION_ID }}
anthropic_service_account_id: ${{ vars.ANTHROPIC_SERVICE_ACCOUNT_ID }}
allowed_tools: "LS,Read"

- name: Verify inline prompt output
Expand Down Expand Up @@ -78,7 +87,9 @@ jobs:
uses: ./base-action
with:
prompt_file: "test-prompt.txt"
anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }}
anthropic_federation_rule_id: ${{ vars.ANTHROPIC_FEDERATION_RULE_ID }}
anthropic_organization_id: ${{ vars.ANTHROPIC_ORGANIZATION_ID }}
anthropic_service_account_id: ${{ vars.ANTHROPIC_SERVICE_ACCOUNT_ID }}
allowed_tools: "LS,Read"

- name: Verify prompt file output
Expand Down
11 changes: 10 additions & 1 deletion .github/workflows/test-custom-executables.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,13 @@ on:
workflow_dispatch:
workflow_call:

# The Claude API is authenticated via workload identity federation: id-token
# lets the action mint the GitHub OIDC token it exchanges for a short-lived
# access token. See docs/setup.md.
permissions:
contents: read
id-token: write

jobs:
test-custom-executables:
runs-on: ubuntu-latest
Expand Down Expand Up @@ -47,7 +54,9 @@ jobs:
with:
prompt: |
List the files in the current directory starting with "package"
anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }}
anthropic_federation_rule_id: ${{ vars.ANTHROPIC_FEDERATION_RULE_ID }}
anthropic_organization_id: ${{ vars.ANTHROPIC_ORGANIZATION_ID }}
anthropic_service_account_id: ${{ vars.ANTHROPIC_SERVICE_ACCOUNT_ID }}
path_to_claude_code_executable: /home/runner/.local/bin/claude
path_to_bun_executable: /home/runner/.bun/bin/bun
allowed_tools: "LS,Read"
Expand Down
80 changes: 55 additions & 25 deletions .github/workflows/test-mcp-servers.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,13 @@ on:
workflow_dispatch:
workflow_call:

# The Claude API is authenticated via workload identity federation: id-token
# lets the action mint the GitHub OIDC token it exchanges for a short-lived
# access token. See docs/setup.md.
permissions:
contents: read
id-token: write

jobs:
test-mcp-integration:
runs-on: ubuntu-latest
Expand All @@ -25,8 +32,11 @@ jobs:
uses: ./base-action
id: claude-test
with:
prompt: "List all available tools"
anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }}
prompt: "Call the test_tool tool and report its response."
anthropic_federation_rule_id: ${{ vars.ANTHROPIC_FEDERATION_RULE_ID }}
anthropic_organization_id: ${{ vars.ANTHROPIC_ORGANIZATION_ID }}
anthropic_service_account_id: ${{ vars.ANTHROPIC_SERVICE_ACCOUNT_ID }}
claude_args: --allowedTools mcp__test-server__test_tool
env:
# Change to test directory so it finds .mcp.json
CLAUDE_WORKING_DIR: ${{ github.workspace }}/base-action/test/mcp-test
Expand All @@ -50,21 +60,29 @@ jobs:
if jq -e '.[] | select(.type == "system" and .subtype == "init") | .mcp_servers' "$OUTPUT_FILE" > /dev/null; then
echo "✓ Found mcp_servers in output"

# Check if test-server is connected
if jq -e '.[] | select(.type == "system" and .subtype == "init") | .mcp_servers[] | select(.name == "test-server" and .status == "connected")' "$OUTPUT_FILE" > /dev/null; then
echo "✓ test-server is connected"
# MCP servers can connect asynchronously, so the init event may
# report the server as pending — check registration there, then
# verify the tool actually ran.
if jq -e '.[] | select(.type == "system" and .subtype == "init") | .mcp_servers[] | select(.name == "test-server")' "$OUTPUT_FILE" > /dev/null; then
echo "✓ test-server is registered"
else
echo "✗ test-server not found or not connected"
echo "✗ test-server not found"
jq '.[] | select(.type == "system" and .subtype == "init") | .mcp_servers' "$OUTPUT_FILE"
exit 1
fi

# Check if mcp tools are available
if jq -e '.[] | select(.type == "system" and .subtype == "init") | .tools[] | select(. == "mcp__test-server__test_tool")' "$OUTPUT_FILE" > /dev/null; then
echo "✓ MCP test tool found"

if jq -e '.[] | select(.type == "assistant") | .message.content[]? | select(.type == "tool_use" and .name == "mcp__test-server__test_tool")' "$OUTPUT_FILE" > /dev/null; then
echo "✓ MCP test tool was called"
else
echo "✗ MCP test tool was not called"
jq '[.[] | select(.type == "assistant") | .message.content[]? | select(.type == "tool_use") | .name]' "$OUTPUT_FILE"
exit 1
fi

if jq -e '.[] | select(.type == "user") | .message.content[]? | select(.type == "tool_result") | select(.content | tostring | contains("Test tool response"))' "$OUTPUT_FILE" > /dev/null; then
echo "✓ MCP test tool returned its response"
else
echo "✗ MCP test tool not found"
jq '.[] | select(.type == "system" and .subtype == "init") | .tools' "$OUTPUT_FILE"
echo "✗ MCP test tool response not found"
exit 1
fi
else
Expand Down Expand Up @@ -106,9 +124,13 @@ jobs:
uses: ./base-action
id: claude-config-test
with:
prompt: "List all available tools"
anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }}
mcp_config: '{"mcpServers":{"test-server":{"type":"stdio","command":"bun","args":["simple-mcp-server.ts"],"env":{}}}}'
prompt: "Call the test_tool tool and report its response."
anthropic_federation_rule_id: ${{ vars.ANTHROPIC_FEDERATION_RULE_ID }}
anthropic_organization_id: ${{ vars.ANTHROPIC_ORGANIZATION_ID }}
anthropic_service_account_id: ${{ vars.ANTHROPIC_SERVICE_ACCOUNT_ID }}
claude_args: |
--allowedTools mcp__test-server__test_tool
--mcp-config '{"mcpServers":{"test-server":{"type":"stdio","command":"bun","args":["simple-mcp-server.ts"],"env":{}}}}'
env:
# Change to test directory so bun can find the MCP server script
CLAUDE_WORKING_DIR: ${{ github.workspace }}/base-action/test/mcp-test
Expand All @@ -132,21 +154,29 @@ jobs:
if jq -e '.[] | select(.type == "system" and .subtype == "init") | .mcp_servers' "$OUTPUT_FILE" > /dev/null; then
echo "✓ Found mcp_servers in output"

# Check if test-server is connected
if jq -e '.[] | select(.type == "system" and .subtype == "init") | .mcp_servers[] | select(.name == "test-server" and .status == "connected")' "$OUTPUT_FILE" > /dev/null; then
echo "✓ test-server is connected"
# MCP servers can connect asynchronously, so the init event may
# report the server as pending — check registration there, then
# verify the tool actually ran.
if jq -e '.[] | select(.type == "system" and .subtype == "init") | .mcp_servers[] | select(.name == "test-server")' "$OUTPUT_FILE" > /dev/null; then
echo "✓ test-server is registered"
else
echo "✗ test-server not found or not connected"
echo "✗ test-server not found"
jq '.[] | select(.type == "system" and .subtype == "init") | .mcp_servers' "$OUTPUT_FILE"
exit 1
fi

# Check if mcp tools are available
if jq -e '.[] | select(.type == "system" and .subtype == "init") | .tools[] | select(. == "mcp__test-server__test_tool")' "$OUTPUT_FILE" > /dev/null; then
echo "✓ MCP test tool found"

if jq -e '.[] | select(.type == "assistant") | .message.content[]? | select(.type == "tool_use" and .name == "mcp__test-server__test_tool")' "$OUTPUT_FILE" > /dev/null; then
echo "✓ MCP test tool was called"
else
echo "✗ MCP test tool was not called"
jq '[.[] | select(.type == "assistant") | .message.content[]? | select(.type == "tool_use") | .name]' "$OUTPUT_FILE"
exit 1
fi

if jq -e '.[] | select(.type == "user") | .message.content[]? | select(.type == "tool_result") | select(.content | tostring | contains("Test tool response"))' "$OUTPUT_FILE" > /dev/null; then
echo "✓ MCP test tool returned its response"
else
echo "✗ MCP test tool not found"
jq '.[] | select(.type == "system" and .subtype == "init") | .tools' "$OUTPUT_FILE"
echo "✗ MCP test tool response not found"
exit 1
fi
else
Expand Down
23 changes: 19 additions & 4 deletions .github/workflows/test-settings.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,13 @@ on:
workflow_dispatch:
workflow_call:

# The Claude API is authenticated via workload identity federation: id-token
# lets the action mint the GitHub OIDC token it exchanges for a short-lived
# access token. See docs/setup.md.
permissions:
contents: read
id-token: write

jobs:
test-settings-inline-allow:
runs-on: ubuntu-latest
Expand All @@ -17,7 +24,9 @@ jobs:
with:
prompt: |
Use Bash to echo "Hello from settings test"
anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }}
anthropic_federation_rule_id: ${{ vars.ANTHROPIC_FEDERATION_RULE_ID }}
anthropic_organization_id: ${{ vars.ANTHROPIC_ORGANIZATION_ID }}
anthropic_service_account_id: ${{ vars.ANTHROPIC_SERVICE_ACCOUNT_ID }}
settings: |
{
"permissions": {
Expand Down Expand Up @@ -66,7 +75,9 @@ jobs:
with:
prompt: |
Run the command `echo $HOME` to check the home directory path
anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }}
anthropic_federation_rule_id: ${{ vars.ANTHROPIC_FEDERATION_RULE_ID }}
anthropic_organization_id: ${{ vars.ANTHROPIC_ORGANIZATION_ID }}
anthropic_service_account_id: ${{ vars.ANTHROPIC_SERVICE_ACCOUNT_ID }}
settings: |
{
"permissions": {
Expand Down Expand Up @@ -108,7 +119,9 @@ jobs:
with:
prompt: |
Use Bash to echo "Hello from settings file test"
anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }}
anthropic_federation_rule_id: ${{ vars.ANTHROPIC_FEDERATION_RULE_ID }}
anthropic_organization_id: ${{ vars.ANTHROPIC_ORGANIZATION_ID }}
anthropic_service_account_id: ${{ vars.ANTHROPIC_SERVICE_ACCOUNT_ID }}
settings: "test-settings.json"

- name: Verify echo worked
Expand Down Expand Up @@ -162,7 +175,9 @@ jobs:
with:
prompt: |
Run the command `echo $HOME` to check the home directory path
anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }}
anthropic_federation_rule_id: ${{ vars.ANTHROPIC_FEDERATION_RULE_ID }}
anthropic_organization_id: ${{ vars.ANTHROPIC_ORGANIZATION_ID }}
anthropic_service_account_id: ${{ vars.ANTHROPIC_SERVICE_ACCOUNT_ID }}
settings: "test-settings.json"

- name: Verify echo was denied
Expand Down
24 changes: 19 additions & 5 deletions .github/workflows/test-structured-output.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,12 @@ on:
workflow_dispatch:
workflow_call:

# The Claude API is authenticated via workload identity federation: id-token
# lets the action mint the GitHub OIDC token it exchanges for a short-lived
# access token. See docs/setup.md.
permissions:
contents: read
id-token: write

jobs:
test-basic-types:
Expand All @@ -28,7 +32,9 @@ jobs:
- number_field: 42
- boolean_true: true
- boolean_false: false
anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }}
anthropic_federation_rule_id: ${{ vars.ANTHROPIC_FEDERATION_RULE_ID }}
anthropic_organization_id: ${{ vars.ANTHROPIC_ORGANIZATION_ID }}
anthropic_service_account_id: ${{ vars.ANTHROPIC_SERVICE_ACCOUNT_ID }}
claude_args: |
--allowedTools Bash
--json-schema '{"type":"object","properties":{"text_field":{"type":"string"},"number_field":{"type":"number"},"boolean_true":{"type":"boolean"},"boolean_false":{"type":"boolean"}},"required":["text_field","number_field","boolean_true","boolean_false"]}'
Expand Down Expand Up @@ -86,7 +92,9 @@ jobs:
- items: ["apple", "banana", "cherry"]
- config: {"key": "value", "count": 3}
- empty_array: []
anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }}
anthropic_federation_rule_id: ${{ vars.ANTHROPIC_FEDERATION_RULE_ID }}
anthropic_organization_id: ${{ vars.ANTHROPIC_ORGANIZATION_ID }}
anthropic_service_account_id: ${{ vars.ANTHROPIC_SERVICE_ACCOUNT_ID }}
claude_args: |
--allowedTools Bash
--json-schema '{"type":"object","properties":{"items":{"type":"array","items":{"type":"string"}},"config":{"type":"object"},"empty_array":{"type":"array"}},"required":["items","config","empty_array"]}'
Expand Down Expand Up @@ -138,7 +146,9 @@ jobs:
- empty_string: ""
- negative: -5
- decimal: 3.14
anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }}
anthropic_federation_rule_id: ${{ vars.ANTHROPIC_FEDERATION_RULE_ID }}
anthropic_organization_id: ${{ vars.ANTHROPIC_ORGANIZATION_ID }}
anthropic_service_account_id: ${{ vars.ANTHROPIC_SERVICE_ACCOUNT_ID }}
claude_args: |
--allowedTools Bash
--json-schema '{"type":"object","properties":{"zero":{"type":"number"},"empty_string":{"type":"string"},"negative":{"type":"number"},"decimal":{"type":"number"}},"required":["zero","empty_string","negative","decimal"]}'
Expand Down Expand Up @@ -192,7 +202,9 @@ jobs:
prompt: |
Run: echo "test"
Return EXACTLY: {test-result: "passed", item_count: 10}
anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }}
anthropic_federation_rule_id: ${{ vars.ANTHROPIC_FEDERATION_RULE_ID }}
anthropic_organization_id: ${{ vars.ANTHROPIC_ORGANIZATION_ID }}
anthropic_service_account_id: ${{ vars.ANTHROPIC_SERVICE_ACCOUNT_ID }}
claude_args: |
--allowedTools Bash
--json-schema '{"type":"object","properties":{"test-result":{"type":"string"},"item_count":{"type":"number"}},"required":["test-result","item_count"]}'
Expand Down Expand Up @@ -230,7 +242,9 @@ jobs:
uses: ./base-action
with:
prompt: "Run: echo 'complete'. Return: {done: true}"
anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }}
anthropic_federation_rule_id: ${{ vars.ANTHROPIC_FEDERATION_RULE_ID }}
anthropic_organization_id: ${{ vars.ANTHROPIC_ORGANIZATION_ID }}
anthropic_service_account_id: ${{ vars.ANTHROPIC_SERVICE_ACCOUNT_ID }}
claude_args: |
--allowedTools Bash
--json-schema '{"type":"object","properties":{"done":{"type":"boolean"}},"required":["done"]}'
Expand Down
Loading
Loading