Meeting Asset Pipeline #927
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "Meeting Asset Pipeline" | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| FORCE_SERIES: | |
| description: "Optional: Series code (e.g., 'acde', 'trustlesslogindex')" | |
| required: false | |
| type: string | |
| FORCE_NUMBER: | |
| description: "Optional: Call number (e.g., 226 for ACDE #226)" | |
| required: false | |
| type: string | |
| schedule: | |
| - cron: "0 * * * *" # Run every hour at minute 0 | |
| jobs: | |
| run-pipeline: | |
| runs-on: ubuntu-latest | |
| env: | |
| refresh_failed: 'false' | |
| INPUT_FORCE_SERIES: ${{ github.event.inputs.FORCE_SERIES }} | |
| INPUT_FORCE_NUMBER: ${{ github.event.inputs.FORCE_NUMBER }} | |
| steps: | |
| - name: Check out code | |
| uses: actions/checkout@v3 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.11" | |
| - name: Install dependencies | |
| run: | | |
| pip install --upgrade pip | |
| pip install -e .github/ACDbot/ | |
| pip install -r .github/ACDbot/requirements.txt | |
| - name: Refresh YouTube Token | |
| env: | |
| YOUTUBE_REFRESH_TOKEN: ${{ secrets.YOUTUBE_REFRESH_TOKEN }} | |
| GOOGLE_CLIENT_ID: ${{ secrets.GOOGLE_CLIENT_ID }} | |
| GOOGLE_CLIENT_SECRET: ${{ secrets.GOOGLE_CLIENT_SECRET }} | |
| run: | | |
| python .github/ACDbot/scripts/refresh_youtube_token.py || echo "refresh_failed=true" >> $GITHUB_ENV | |
| - name: Poll Zoom for recordings | |
| run: | | |
| if [ -n "$INPUT_FORCE_SERIES" ] && [ -n "$INPUT_FORCE_NUMBER" ]; then | |
| # Look up meeting_id and issue_number from mapping file | |
| LOOKUP=$(python -c " | |
| import json, re, sys, os | |
| with open('.github/ACDbot/meeting_topic_mapping.json') as f: | |
| mapping = json.load(f) | |
| series = os.environ['INPUT_FORCE_SERIES'] | |
| call_num = int(os.environ['INPUT_FORCE_NUMBER']) | |
| if series not in mapping: | |
| print(f'Series {series} not found', file=sys.stderr) | |
| sys.exit(1) | |
| data = mapping[series] | |
| meeting_id = data.get('meeting_id', '') | |
| issue_number = '' | |
| for occ in data.get('occurrences', []): | |
| title = occ.get('issue_title', '') | |
| # Match patterns: '#123', '# 123', '#0123', 'Meeting 123', 'Call 123', 'Breakout 123' | |
| pattern = rf'(?:#\s*0*{call_num}|(?:Meeting|Call|Breakout)\s+0*{call_num})(?:\D|$)' | |
| if re.search(pattern, title, re.IGNORECASE): | |
| issue_number = str(occ.get('issue_number', '')) | |
| break | |
| print(f'{meeting_id}|{issue_number}') | |
| ") | |
| MEETING_ID=$(echo "$LOOKUP" | cut -d'|' -f1) | |
| ISSUE_NUMBER=$(echo "$LOOKUP" | cut -d'|' -f2) | |
| echo "Found meeting_id=$MEETING_ID, issue_number=$ISSUE_NUMBER for $INPUT_FORCE_SERIES #$INPUT_FORCE_NUMBER" | |
| if [ -n "$MEETING_ID" ] && [ -n "$ISSUE_NUMBER" ]; then | |
| python .github/ACDbot/scripts/poll_zoom_recordings.py \ | |
| --force_meeting_id "$MEETING_ID" \ | |
| --force_issue_number "$ISSUE_NUMBER" | |
| else | |
| echo "Could not find meeting_id or issue_number for $INPUT_FORCE_SERIES #$INPUT_FORCE_NUMBER" | |
| exit 1 | |
| fi | |
| else | |
| python .github/ACDbot/scripts/poll_zoom_recordings.py | |
| fi | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| # Zoom credentials | |
| ZOOM_CLIENT_ID: ${{ secrets.ZOOM_CLIENT_ID }} | |
| ZOOM_CLIENT_SECRET: ${{ secrets.ZOOM_CLIENT_SECRET }} | |
| ZOOM_REFRESH_TOKEN: ${{ secrets.ZOOM_REFRESH_TOKEN }} | |
| # Discourse credentials | |
| DISCOURSE_API_KEY: ${{ secrets.DISCOURSE_API_KEY }} | |
| DISCOURSE_API_USERNAME: ${{ secrets.DISCOURSE_API_USERNAME }} | |
| DISCOURSE_BASE_URL: ${{ vars.DISCOURSE_BASE_URL }} | |
| # Telegram credentials | |
| TELEGRAM_BOT_TOKEN: ${{ secrets.TELEGRAM_BOT_TOKEN }} | |
| TELEGRAM_CHAT_ID: ${{ vars.TELEGRAM_CHAT_ID }} | |
| - name: Upload Zoom recordings to YouTube | |
| continue-on-error: true | |
| run: | | |
| if [ -n "$INPUT_FORCE_SERIES" ]; then | |
| MEETING_ID=$(python -c " | |
| import json, sys, os | |
| with open('.github/ACDbot/meeting_topic_mapping.json') as f: | |
| mapping = json.load(f) | |
| series = os.environ['INPUT_FORCE_SERIES'] | |
| if series not in mapping: | |
| print(f'Series {series} not found', file=sys.stderr) | |
| sys.exit(1) | |
| print(mapping[series].get('meeting_id', '')) | |
| ") | |
| if [ -n "$MEETING_ID" ]; then | |
| python .github/ACDbot/scripts/upload_zoom_recording.py \ | |
| --meeting_id "$MEETING_ID" | |
| else | |
| echo "No meeting_id found for series $INPUT_FORCE_SERIES" | |
| python .github/ACDbot/scripts/upload_zoom_recording.py | |
| fi | |
| else | |
| python .github/ACDbot/scripts/upload_zoom_recording.py | |
| fi | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| ZOOM_CLIENT_ID: ${{ secrets.ZOOM_CLIENT_ID }} | |
| ZOOM_CLIENT_SECRET: ${{ secrets.ZOOM_CLIENT_SECRET }} | |
| ZOOM_REFRESH_TOKEN: ${{ secrets.ZOOM_REFRESH_TOKEN }} | |
| YOUTUBE_API_KEY: ${{ secrets.YOUTUBE_API_KEY }} | |
| YOUTUBE_REFRESH_TOKEN: ${{ secrets.YOUTUBE_REFRESH_TOKEN }} | |
| GOOGLE_CLIENT_ID: ${{ secrets.GOOGLE_CLIENT_ID }} | |
| GOOGLE_CLIENT_SECRET: ${{ secrets.GOOGLE_CLIENT_SECRET }} | |
| DISCOURSE_API_KEY: ${{ secrets.DISCOURSE_API_KEY }} | |
| DISCOURSE_API_USERNAME: ${{ secrets.DISCOURSE_API_USERNAME }} | |
| DISCOURSE_BASE_URL: ${{ vars.DISCOURSE_BASE_URL }} | |
| TELEGRAM_BOT_TOKEN: ${{ secrets.TELEGRAM_BOT_TOKEN }} | |
| TELEGRAM_CHAT_ID: ${{ vars.TELEGRAM_CHAT_ID }} | |
| MATTERMOST_BOT_WEBHOOK_URL: ${{ secrets.MATTERMOST_BOT_WEBHOOK_URL }} | |
| - name: Run asset pipeline | |
| if: success() | |
| continue-on-error: true | |
| env: | |
| ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} | |
| ZOOM_CLIENT_ID: ${{ secrets.ZOOM_CLIENT_ID }} | |
| ZOOM_CLIENT_SECRET: ${{ secrets.ZOOM_CLIENT_SECRET }} | |
| ZOOM_REFRESH_TOKEN: ${{ secrets.ZOOM_REFRESH_TOKEN }} | |
| run: | | |
| cd .github/ACDbot/scripts/asset_pipeline | |
| if [ -n "$INPUT_FORCE_SERIES" ] && [ -n "$INPUT_FORCE_NUMBER" ]; then | |
| # Process specific meeting | |
| echo "Processing $INPUT_FORCE_SERIES #$INPUT_FORCE_NUMBER..." | |
| python run_pipeline.py -c "$INPUT_FORCE_SERIES" -n "$INPUT_FORCE_NUMBER" --auto-approve --summarize | |
| else | |
| # No forced meeting - process all recent meetings | |
| SERIES=$(python -c "import json; m=json.load(open('../../meeting_topic_mapping.json')); print(' '.join(m.keys()))") | |
| for series in $SERIES; do | |
| python run_pipeline.py -c "$series" --recent --max-age-days 3 --auto-approve --summarize || true | |
| done | |
| fi | |
| - name: Generate manifest and commit all changes | |
| id: commit | |
| if: always() | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| git config --local user.email "action@github.com" | |
| git config --local user.name "GitHub Action" | |
| # Generate manifest | |
| cd .github/ACDbot/scripts/asset_pipeline | |
| python generate_manifest.py | |
| cd - | |
| # Stage all changed files | |
| git add .github/ACDbot/meeting_topic_mapping.json | |
| git add .github/ACDbot/artifacts/ | |
| # Check if there are any staged changes | |
| if git diff --cached --quiet; then | |
| echo "No changes to commit" | |
| else | |
| git commit -m "Update meeting mapping, artifacts, and manifest" | |
| git pull --rebase | |
| git push | |
| echo "has_changes=true" >> $GITHUB_OUTPUT | |
| echo "Committed all changes" | |
| fi | |
| - name: Notify Forkcast | |
| if: steps.commit.outputs.has_changes == 'true' | |
| uses: actions/github-script@v7 | |
| with: | |
| github-token: ${{ secrets.FORKCAST_DISPATCH_TOKEN }} | |
| script: | | |
| await github.rest.repos.createDispatchEvent({ | |
| owner: 'ethereum', | |
| repo: 'forkcast', | |
| event_type: 'pm-assets-updated' | |
| }); |