Claude Mini NL Test Suite (Unity live) #1
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Claude Mini NL Test Suite (Unity live) | |
| on: | |
| workflow_dispatch: {} | |
| permissions: | |
| contents: read | |
| checks: write | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| UNITY_VERSION: 2021.3.45f1 | |
| UNITY_IMAGE: unityci/editor:ubuntu-2021.3.45f1-linux-il2cpp-3 | |
| UNITY_CACHE_ROOT: /home/runner/work/_temp/_github_home | |
| jobs: | |
| nl-suite: | |
| if: github.event_name == 'workflow_dispatch' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 60 | |
| env: | |
| JUNIT_OUT: reports/junit-nl-suite.xml | |
| MD_OUT: reports/junit-nl-suite.md | |
| steps: | |
| # ---------- Detect secrets ---------- | |
| - name: Detect secrets (outputs) | |
| id: detect | |
| env: | |
| UNITY_LICENSE: ${{ secrets.UNITY_LICENSE }} | |
| UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }} | |
| UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }} | |
| UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }} | |
| ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} | |
| run: | | |
| set -e | |
| if [ -n "$ANTHROPIC_API_KEY" ]; then echo "anthropic_ok=true" >> "$GITHUB_OUTPUT"; else echo "anthropic_ok=false" >> "$GITHUB_OUTPUT"; fi | |
| if [ -n "$UNITY_LICENSE" ] || { [ -n "$UNITY_EMAIL" ] && [ -n "$UNITY_PASSWORD" ]; } || [ -n "$UNITY_SERIAL" ]; then | |
| echo "unity_ok=true" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "unity_ok=false" >> "$GITHUB_OUTPUT" | |
| fi | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| # ---------- Python env for MCP server (uv) ---------- | |
| - uses: astral-sh/setup-uv@v4 | |
| with: | |
| python-version: '3.11' | |
| - name: Install MCP server | |
| run: | | |
| set -eux | |
| uv venv | |
| echo "VIRTUAL_ENV=$GITHUB_WORKSPACE/.venv" >> "$GITHUB_ENV" | |
| echo "$GITHUB_WORKSPACE/.venv/bin" >> "$GITHUB_PATH" | |
| if [ -f UnityMcpBridge/UnityMcpServer~/src/pyproject.toml ]; then | |
| uv pip install -e UnityMcpBridge/UnityMcpServer~/src | |
| elif [ -f UnityMcpBridge/UnityMcpServer~/src/requirements.txt ]; then | |
| uv pip install -r UnityMcpBridge/UnityMcpServer~/src/requirements.txt | |
| elif [ -f UnityMcpBridge/UnityMcpServer~/pyproject.toml ]; then | |
| uv pip install -e UnityMcpBridge/UnityMcpServer~/ | |
| elif [ -f UnityMcpBridge/UnityMcpServer~/requirements.txt ]; then | |
| uv pip install -r UnityMcpBridge/UnityMcpServer~/requirements.txt | |
| else | |
| echo "No MCP Python deps found (skipping)" | |
| fi | |
| # ---------- License prime on host (handles ULF or EBL) ---------- | |
| - name: Prime Unity license on host (GameCI) | |
| if: steps.detect.outputs.unity_ok == 'true' | |
| uses: game-ci/unity-test-runner@v4 | |
| env: | |
| UNITY_LICENSE: ${{ secrets.UNITY_LICENSE }} | |
| UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }} | |
| UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }} | |
| UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }} | |
| with: | |
| projectPath: TestProjects/UnityMCPTests | |
| testMode: EditMode | |
| customParameters: -runTests -testFilter __NoSuchTest__ -batchmode -nographics | |
| unityVersion: ${{ env.UNITY_VERSION }} | |
| # (Optional) Show where the license actually got written | |
| - name: Inspect GameCI license caches (host) | |
| if: steps.detect.outputs.unity_ok == 'true' | |
| run: | | |
| set -eux | |
| find "${{ env.UNITY_CACHE_ROOT }}" -maxdepth 4 \( -path "*/.cache" -prune -o -type f \( -name '*.ulf' -o -name 'user.json' \) -print \) 2>/dev/null || true | |
| # ---------- Clean any stale MCP status from previous runs ---------- | |
| - name: Clean old MCP status | |
| run: | | |
| set -eux | |
| mkdir -p "$HOME/.unity-mcp" | |
| rm -f "$HOME/.unity-mcp"/unity-mcp-status-*.json || true | |
| # ---------- Start headless Unity that stays up (bridge enabled) ---------- | |
| - name: Start Unity (persistent bridge) | |
| if: steps.detect.outputs.unity_ok == 'true' | |
| env: | |
| UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }} | |
| UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }} | |
| UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }} | |
| run: | | |
| set -eu | |
| if [ ! -d "${{ github.workspace }}/TestProjects/UnityMCPTests/ProjectSettings" ]; then | |
| echo "Unity project not found; failing fast." | |
| exit 1 | |
| fi | |
| mkdir -p "$HOME/.unity-mcp" | |
| MANUAL_ARG=() | |
| if [ -f "${UNITY_CACHE_ROOT}/.local/share/unity3d/Unity_lic.ulf" ]; then | |
| MANUAL_ARG=(-manualLicenseFile /root/.local/share/unity3d/Unity_lic.ulf) | |
| fi | |
| EBL_ARGS=() | |
| [ -n "${UNITY_SERIAL:-}" ] && EBL_ARGS+=(-serial "$UNITY_SERIAL") | |
| [ -n "${UNITY_EMAIL:-}" ] && EBL_ARGS+=(-username "$UNITY_EMAIL") | |
| [ -n "${UNITY_PASSWORD:-}" ] && EBL_ARGS+=(-password "$UNITY_PASSWORD") | |
| docker rm -f unity-mcp >/dev/null 2>&1 || true | |
| docker run -d --name unity-mcp --network host \ | |
| -e HOME=/root \ | |
| -e UNITY_MCP_ALLOW_BATCH=1 -e UNITY_MCP_STATUS_DIR=/root/.unity-mcp \ | |
| -e UNITY_MCP_BIND_HOST=127.0.0.1 \ | |
| -v "${{ github.workspace }}:/workspace" -w /workspace \ | |
| -v "${{ env.UNITY_CACHE_ROOT }}:/root" \ | |
| -v "$HOME/.unity-mcp:/root/.unity-mcp" \ | |
| ${{ env.UNITY_IMAGE }} /opt/unity/Editor/Unity -batchmode -nographics -logFile - \ | |
| -stackTraceLogType Full \ | |
| -projectPath /workspace/TestProjects/UnityMCPTests \ | |
| "${MANUAL_ARG[@]}" \ | |
| "${EBL_ARGS[@]}" \ | |
| -executeMethod MCPForUnity.Editor.MCPForUnityBridge.StartAutoConnect | |
| # ---------- Wait for Unity bridge (fail fast if not running/ready) ---------- | |
| - name: Wait for Unity bridge (robust) | |
| if: steps.detect.outputs.unity_ok == 'true' | |
| run: | | |
| set -euo pipefail | |
| if ! docker ps --format '{{.Names}}' | grep -qx 'unity-mcp'; then | |
| echo "Unity container failed to start"; docker ps -a || true; exit 1 | |
| fi | |
| docker logs -f unity-mcp 2>&1 | sed -E 's/((serial|license|password|token)[^[:space:]]*)/[REDACTED]/ig' & LOGPID=$! | |
| deadline=$((SECONDS+420)); READY=0 | |
| try_connect_host() { | |
| P="$1" | |
| timeout 1 bash -lc "exec 3<>/dev/tcp/127.0.0.1/$P; head -c 8 <&3 >/dev/null" && return 0 || true | |
| if command -v nc >/dev/null 2>&1; then nc -6 -z ::1 "$P" && return 0 || true; fi | |
| return 1 | |
| } | |
| # in-container probe will try IPv4 then IPv6 via nc or /dev/tcp | |
| while [ $SECONDS -lt $deadline ]; do | |
| if docker logs unity-mcp 2>&1 | grep -qE "MCP Bridge listening|Bridge ready|Server started"; then | |
| READY=1; echo "Bridge ready (log markers)"; break | |
| fi | |
| PORT=$(python -c "import os,glob,json,sys,time; b=os.path.expanduser('~/.unity-mcp'); fs=sorted(glob.glob(os.path.join(b,'unity-mcp-status-*.json')), key=os.path.getmtime, reverse=True); print(next((json.load(open(f,'r',encoding='utf-8')).get('unity_port') for f in fs if time.time()-os.path.getmtime(f)<=300 and json.load(open(f,'r',encoding='utf-8')).get('unity_port')), '' ))" 2>/dev/null || true) | |
| if [ -n "${PORT:-}" ] && { try_connect_host "$PORT" || docker exec unity-mcp bash -lc "timeout 1 bash -lc 'exec 3<>/dev/tcp/127.0.0.1/$PORT' || (command -v nc >/dev/null 2>&1 && nc -6 -z ::1 $PORT)"; }; then | |
| READY=1; echo "Bridge ready on port $PORT"; break | |
| fi | |
| if docker logs unity-mcp 2>&1 | grep -qE "No valid Unity Editor license|Token not found in cache|com\.unity\.editor\.headless"; then | |
| echo "Licensing error detected"; break | |
| fi | |
| sleep 2 | |
| done | |
| kill $LOGPID || true | |
| if [ "$READY" != "1" ]; then | |
| echo "Bridge not ready; diagnostics:" | |
| echo "== status files =="; ls -la "$HOME/.unity-mcp" || true | |
| echo "== status contents =="; for f in "$HOME"/.unity-mcp/unity-mcp-status-*.json; do [ -f "$f" ] && { echo "--- $f"; sed -n '1,120p' "$f"; }; done | |
| echo "== sockets (inside container) =="; docker exec unity-mcp bash -lc 'ss -lntp || netstat -tulpen || true' | |
| echo "== tail of Unity log ==" | |
| docker logs --tail 200 unity-mcp | sed -E 's/((serial|license|password|token)[^[:space:]]*)/[REDACTED]/ig' || true | |
| exit 1 | |
| fi | |
| # ---------- Make MCP config available to the action ---------- | |
| - name: Write MCP config (.claude/mcp.json) | |
| run: | | |
| set -eux | |
| mkdir -p .claude | |
| cat > .claude/mcp.json <<JSON | |
| { | |
| "mcpServers": { | |
| "unity": { | |
| "command": "uv", | |
| "args": ["run","--active","--directory","UnityMcpBridge/UnityMcpServer~/src","python","server.py"], | |
| "transport": { "type": "stdio" }, | |
| "env": { | |
| "PYTHONUNBUFFERED": "1", | |
| "MCP_LOG_LEVEL": "debug", | |
| "UNITY_PROJECT_ROOT": "$GITHUB_WORKSPACE/TestProjects/UnityMCPTests" | |
| } | |
| } | |
| } | |
| } | |
| JSON | |
| # ---------- Ensure reports dir exists ---------- | |
| - name: Prepare reports | |
| run: | | |
| set -eux | |
| mkdir -p reports | |
| # ---------- Run full NL suite once ---------- | |
| - name: Run Claude NL suite (single pass) | |
| uses: anthropics/claude-code-base-action@beta | |
| if: steps.detect.outputs.anthropic_ok == 'true' && steps.detect.outputs.unity_ok == 'true' | |
| env: | |
| JUNIT_OUT: reports/junit-nl-suite.xml | |
| MD_OUT: reports/junit-nl-suite.md | |
| with: | |
| use_node_cache: false | |
| prompt_file: .claude/prompts/nl-unity-claude-tests-mini.md | |
| mcp_config: .claude/mcp.json | |
| allowed_tools: "Write,mcp__unity__manage_editor,mcp__unity__list_resources,mcp__unity__read_resource,mcp__unity__apply_text_edits,mcp__unity__script_apply_edits,mcp__unity__validate_script,mcp__unity__find_in_file, mcp__unity__read_console" | |
| disallowed_tools: "TodoWrite,Task" | |
| model: "claude-3-7-sonnet-latest" | |
| timeout_minutes: "30" | |
| anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }} | |
| - name: Normalize JUnit for consumer actions (strong) | |
| if: always() | |
| shell: bash | |
| run: | | |
| python3 - <<'PY' | |
| from pathlib import Path | |
| import xml.etree.ElementTree as ET | |
| import os | |
| def localname(tag: str) -> str: | |
| return tag.rsplit('}', 1)[-1] if '}' in tag else tag | |
| src = Path(os.environ.get('JUNIT_OUT', 'reports/junit-nl-suite.xml')) | |
| out = Path('reports/junit-for-actions.xml') | |
| out.parent.mkdir(parents=True, exist_ok=True) | |
| if not src.exists(): | |
| # Try to use any existing XML as a source (e.g., claude-nl-tests.xml) | |
| candidates = sorted(Path('reports').glob('*.xml')) | |
| if candidates: | |
| src = candidates[0] | |
| else: | |
| print("WARN: no XML source found for normalization") | |
| if src.exists(): | |
| try: | |
| root = ET.parse(src).getroot() | |
| rtag = localname(root.tag) | |
| if rtag == 'testsuites' and len(root) == 1 and localname(root[0].tag) == 'testsuite': | |
| ET.ElementTree(root[0]).write(out, encoding='utf-8', xml_declaration=True) | |
| else: | |
| out.write_bytes(src.read_bytes()) | |
| except Exception as e: | |
| print("Normalization error:", e) | |
| out.write_bytes(src.read_bytes()) | |
| # Always create a second copy with a junit-* name so wildcard patterns match too | |
| if out.exists(): | |
| Path('reports/junit-nl-suite-copy.xml').write_bytes(out.read_bytes()) | |
| PY | |
| - name: "Debug: list report files" | |
| if: always() | |
| shell: bash | |
| run: | | |
| set -eux | |
| ls -la reports || true | |
| shopt -s nullglob | |
| for f in reports/*.xml; do | |
| echo "===== $f =====" | |
| head -n 40 "$f" || true | |
| done | |
| # sanitize only the markdown (does not touch JUnit xml) | |
| - name: Sanitize markdown (all shards) | |
| if: always() | |
| run: | | |
| set -eu | |
| python - <<'PY' | |
| from pathlib import Path | |
| rp=Path('reports') | |
| rp.mkdir(parents=True, exist_ok=True) | |
| for p in rp.glob('*.md'): | |
| b=p.read_bytes().replace(b'\x00', b'') | |
| s=b.decode('utf-8','replace').replace('\r\n','\n') | |
| p.write_text(s, encoding='utf-8', newline='\n') | |
| PY | |
| - name: NL/T details → Job Summary | |
| if: always() | |
| run: | | |
| echo "## Unity NL/T Editing Suite — Full Coverage" >> $GITHUB_STEP_SUMMARY | |
| python - <<'PY' >> $GITHUB_STEP_SUMMARY | |
| from pathlib import Path | |
| p = Path('reports/junit-nl-suite.md') if Path('reports/junit-nl-suite.md').exists() else Path('reports/claude-nl-tests.md') | |
| if p.exists(): | |
| text = p.read_bytes().decode('utf-8', 'replace') | |
| MAX = 65000 | |
| print(text[:MAX]) | |
| if len(text) > MAX: | |
| print("\n\n_…truncated in summary; full report is in artifacts._") | |
| else: | |
| print("_No markdown report found._") | |
| PY | |
| - name: Fallback JUnit if missing | |
| if: always() | |
| run: | | |
| set -eu | |
| mkdir -p reports | |
| if [ ! -f reports/junit-for-actions.xml ]; then | |
| printf '%s\n' \ | |
| '<?xml version="1.0" encoding="UTF-8"?>' \ | |
| '<testsuite name="UnityMCP.NL-T" tests="1" failures="1" time="0">' \ | |
| ' <testcase classname="UnityMCP.NL-T" name="NL-Suite.Execution" time="0.0">' \ | |
| ' <failure><![CDATA[No JUnit was produced by the NL suite step. See the '"'"'Run Claude NL suite (single pass)'"'"' logs.]]></failure>' \ | |
| ' </testcase>' \ | |
| '</testsuite>' \ | |
| > reports/junit-for-actions.xml | |
| fi | |
| - name: Publish JUnit reports | |
| if: always() | |
| uses: mikepenz/action-junit-report@v5 | |
| with: | |
| report_paths: 'reports/junit-for-actions.xml' | |
| include_passed: true | |
| detailed_summary: true | |
| annotate_notice: true | |
| require_tests: false | |
| fail_on_parse_error: true | |
| - name: Upload artifacts | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: claude-nl-suite-artifacts | |
| path: reports/** | |
| # ---------- Always stop Unity ---------- | |
| - name: Stop Unity | |
| if: always() | |
| run: | | |
| docker logs --tail 400 unity-mcp | sed -E 's/((serial|license|password|token)[^[:space:]]*)/[REDACTED]/ig' || true | |
| docker rm -f unity-mcp || true |