gpt_data_combined.yml

name: Combine Markdown Files for AirRepsGPT

on:
  push:
    branches:
      - main
    paths:
      # Only trigger on English content changes (excluding locale directories)
      - 'docs/*.md'
      - 'docs/introduction/**/*.md'
      - 'docs/version-info/**/*.md'
      - 'docs/ordering/**/*.md'
      - 'docs/links/**/*.md'
      - 'docs/troubleshooting/**/*.md'

permissions:
  contents: read
  actions: read

jobs:
  combine-markdown:
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v6

      - name: Combine Markdown Files
        run: |
          # Clear output file
          > ultimateguide.md
          
          # Function to add files from a directory (English only)
          add_files_from_dir() {
            local dir="$1"
            local description="$2"
            
            if [ -d "$dir" ]; then
              echo "Adding $description files..."
              find "$dir" -name "*.md" -type f ! -path "*/pt/*" ! -path "*/es/*" ! -path "*/da/*" ! -path "*/fr/*" ! -path "*/pl/*" ! -path "*/ru/*" ! -path "*/de/*" ! -path "*/tr/*" | sort | while read -r file; do
                echo "  - Adding: $file"
                cat "$file" >> ultimateguide.md
              done
            else
              echo "Directory $dir not found, skipping..."
            fi
          }
          
          # Add files in logical order (English content only)
          add_files_from_dir "docs/introduction" "Introduction"
          add_files_from_dir "docs/version-info" "Version Info"
          add_files_from_dir "docs/ordering" "Ordering"
          add_files_from_dir "docs/links" "Links"
          add_files_from_dir "docs/troubleshooting" "Troubleshooting"
          
          # Add remaining top-level files
          find docs -maxdepth 1 -name "*.md" -not -name "index.md" -type f | sort | while read -r file; do
            echo "  - Adding: $file"
            cat "$file" >> ultimateguide.md
          done
          
          echo "✅ Successfully combined all markdown files into ultimateguide.md"
          echo "📊 Total size: $(wc -l < ultimateguide.md) lines"

      - name: Setup Bun
        uses: oven-sh/setup-bun@v2
        with:
          bun-version: 1.3.11

      - name: Install dependencies
        run: bun install --frozen-lockfile

      - name: Convert markdown tables to JSONL
        run: |
          bun -e '
          const fs = require("fs");
          const input = fs.readFileSync("ultimateguide.md", "utf-8");
          const lines = input.split("\n");
          const output = [];

          for (const line of lines) {
            const trimmed = line.trim();

            // Skip separator rows like |---|---|
            if (/^\|[\s\-:|]+\|$/.test(trimmed)) continue;

            // Convert table data rows (lines that start and end with |)
            if (trimmed.startsWith("|") && trimmed.endsWith("|")) {
              const cells = trimmed
                .slice(1, -1)        // strip leading/trailing pipes
                .split("|")
                .map(c => c.trim());
              output.push(JSON.stringify(cells));
            } else {
              output.push(line);
            }
          }

          fs.writeFileSync("ultimateguide.md", output.join("\n"));
          '

      - name: Upload Combined File as Artifact
        uses: actions/upload-artifact@v7
        with:
          retention-days: 14
          path: ultimateguide.md

      - name: Upload Combined File to OpenAI Assistant
        env:
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
          VECTOR_STORE_ID: ${{ secrets.VECTOR_STORE_ID }}
        run: bun .github/scripts/upload-to-openai.js