Cancel Pulumi Lock #12
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Cancel Pulumi Lock | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| environment: | |
| description: 'Environment to cancel lock for' | |
| required: true | |
| type: choice | |
| options: | |
| - staging | |
| - production | |
| action: | |
| description: 'Action to perform' | |
| required: true | |
| type: choice | |
| options: | |
| - cancel-pulumi-lock | |
| - check-migrations | |
| - run-migration-010-dry-run | |
| - reset-staging-db | |
| permissions: | |
| contents: read | |
| env: | |
| PULUMI_VERSION: "3.188.0" | |
| jobs: | |
| cancel-lock: | |
| name: Cancel Pulumi Lock | |
| runs-on: ubuntu-latest | |
| environment: ${{ inputs.environment }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 | |
| - name: Setup Pulumi | |
| uses: pulumi/actions@d7ceb0215da5a14ec84f50b703365ddf0194a9c8 | |
| with: | |
| pulumi-version: ${{ env.PULUMI_VERSION }} | |
| - name: Authenticate to Google Cloud (Staging) | |
| if: inputs.environment == 'staging' | |
| uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 | |
| with: | |
| credentials_json: ${{ secrets.GCP_STAGING_SERVICE_ACCOUNT_KEY }} | |
| - name: Authenticate to Google Cloud (Production) | |
| if: inputs.environment == 'production' | |
| uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 | |
| with: | |
| credentials_json: ${{ secrets.GCP_PROD_SERVICE_ACCOUNT_KEY }} | |
| - name: Setup Google Cloud SDK (Staging) | |
| if: inputs.environment == 'staging' | |
| uses: google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db | |
| with: | |
| project_id: mcp-registry-staging | |
| - name: Setup Google Cloud SDK (Production) | |
| if: inputs.environment == 'production' | |
| uses: google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db | |
| with: | |
| project_id: mcp-registry-prod | |
| - name: Cancel Pulumi Lock (Staging) | |
| if: inputs.environment == 'staging' && inputs.action == 'cancel-pulumi-lock' | |
| working-directory: ./deploy | |
| env: | |
| PULUMI_STAGING_PASSPHRASE: ${{ secrets.PULUMI_STAGING_PASSPHRASE }} | |
| run: | | |
| echo "$PULUMI_STAGING_PASSPHRASE" > passphrase.staging.txt | |
| pulumi login gs://mcp-registry-staging-pulumi-state | |
| PULUMI_CONFIG_PASSPHRASE_FILE=passphrase.staging.txt pulumi cancel --stack gcpStaging --yes | |
| - name: Cancel Pulumi Lock (Production) | |
| if: inputs.environment == 'production' && inputs.action == 'cancel-pulumi-lock' | |
| working-directory: ./deploy | |
| env: | |
| PULUMI_PROD_PASSPHRASE: ${{ secrets.PULUMI_PROD_PASSPHRASE }} | |
| run: | | |
| echo "$PULUMI_PROD_PASSPHRASE" > passphrase.prod.txt | |
| pulumi login gs://mcp-registry-prod-pulumi-state | |
| PULUMI_CONFIG_PASSPHRASE_FILE=passphrase.prod.txt pulumi cancel --stack gcpProd --yes | |
| - name: Configure kubectl (Staging) | |
| if: inputs.environment == 'staging' && (inputs.action == 'check-migrations' || inputs.action == 'run-migration-010-dry-run' || inputs.action == 'reset-staging-db') | |
| run: | | |
| gcloud container clusters get-credentials mcp-registry-staging \ | |
| --zone=us-central1-b \ | |
| --project=mcp-registry-staging | |
| gcloud components install gke-gcloud-auth-plugin | |
| - name: Configure kubectl (Production) | |
| if: inputs.environment == 'production' && (inputs.action == 'check-migrations' || inputs.action == 'run-migration-010-dry-run') | |
| run: | | |
| gcloud container clusters get-credentials mcp-registry-prod \ | |
| --zone=us-central1-b \ | |
| --project=mcp-registry-prod | |
| gcloud components install gke-gcloud-auth-plugin | |
| - name: Check Migrations | |
| if: inputs.action == 'check-migrations' | |
| run: | | |
| echo "=== Listing all postgres secrets in default namespace ===" | |
| kubectl get secrets -n default | grep registry-pg | |
| echo "" | |
| echo "=== Creating temporary postgres client pod ===" | |
| # Use the app user credentials from registry-pg-app secret | |
| DB_USER=$(kubectl get secret registry-pg-app -n default -o jsonpath='{.data.username}' | base64 -d) | |
| DB_PASS=$(kubectl get secret registry-pg-app -n default -o jsonpath='{.data.password}' | base64 -d) | |
| echo "Using database user: $DB_USER" | |
| # Create pod without --rm so we can get full logs | |
| POD_NAME="pg-check-$(date +%s)" | |
| kubectl run -n default $POD_NAME \ | |
| --image=postgres:15 \ | |
| --restart=Never \ | |
| --env="PGPASSWORD=$DB_PASS" \ | |
| -- bash -c " | |
| echo '=== Waiting for database to accept connections ===' | |
| for i in {1..30}; do | |
| if pg_isready -h registry-pg-rw -U $DB_USER 2>/dev/null; then | |
| echo 'Database is ready!' | |
| break | |
| fi | |
| echo \"Waiting... (\$i/30)\" | |
| sleep 2 | |
| done | |
| echo '' | |
| echo '=== ALL Applied Migrations in ${{ inputs.environment }} ===' | |
| psql -h registry-pg-rw -U $DB_USER -d app -c 'SELECT version, name, applied_at FROM schema_migrations ORDER BY version;' | |
| echo '' | |
| echo '=== Checking for migration 009 specifically ===' | |
| MIGRATION_009_COUNT=\$(psql -h registry-pg-rw -U $DB_USER -d app -tAc 'SELECT COUNT(*) FROM schema_migrations WHERE version = 9;' 2>/dev/null | tr -d '[:space:]') | |
| echo "Migration 009 count: '$MIGRATION_009_COUNT'" | |
| if [ -z "$MIGRATION_009_COUNT" ] || [ "$MIGRATION_009_COUNT" = "0" ]; then | |
| echo "❌ Migration 009 NOT FOUND - this explains why packages still have old format" | |
| else | |
| echo "✅ Migration 009 IS APPLIED" | |
| fi | |
| echo '' | |
| echo '=== Sample Package Data (first 3 servers with packages) ===' | |
| psql -h registry-pg-rw -U $DB_USER -d app -c \" | |
| SELECT | |
| server_name, | |
| jsonb_pretty(value->'packages') as packages | |
| FROM servers | |
| WHERE value ? 'packages' | |
| AND jsonb_array_length(value->'packages') > 0 | |
| LIMIT 3; | |
| \" | |
| echo '' | |
| echo '=== Database Schema/Layout ===' | |
| psql -h registry-pg-rw -U $DB_USER -d app -c '\dt+' | |
| echo '' | |
| echo '=== Schema for servers table ===' | |
| psql -h registry-pg-rw -U $DB_USER -d app -c '\d+ servers' | |
| echo '' | |
| echo '=== Schema for schema_migrations table ===' | |
| psql -h registry-pg-rw -U $DB_USER -d app -c '\d+ schema_migrations' | |
| " | |
| # Wait for pod to complete | |
| echo "" | |
| echo "Waiting for pod to complete..." | |
| kubectl wait --for=condition=Ready pod/$POD_NAME -n default --timeout=60s || true | |
| kubectl wait --for=jsonpath='{.status.phase}'=Succeeded pod/$POD_NAME -n default --timeout=60s || true | |
| # Get the logs | |
| echo "" | |
| echo "=== Database Query Results ===" | |
| kubectl logs $POD_NAME -n default | |
| # Cleanup | |
| kubectl delete pod $POD_NAME -n default | |
| echo "" | |
| echo "=== Registry Server Logs (first 100 lines) ===" | |
| REGISTRY_POD=$(kubectl get pods -l app=mcp-registry -n default -o jsonpath='{.items[0].metadata.name}') | |
| if [ -n "$REGISTRY_POD" ]; then | |
| echo "Found registry pod: $REGISTRY_POD" | |
| kubectl logs -n default $REGISTRY_POD --tail=100 | |
| else | |
| echo "⚠️ No registry pod found" | |
| fi | |
| - name: Run Migration 010 Dry Run | |
| if: inputs.action == 'run-migration-010-dry-run' | |
| run: | | |
| set -e | |
| echo "=== Running Migration 010 DRY RUN ===" | |
| echo "This will show what changes WOULD be made WITHOUT committing" | |
| echo "" | |
| # Get database credentials | |
| DB_USER=$(kubectl get secret registry-pg-app -n default -o jsonpath='{.data.username}' | base64 -d) | |
| DB_PASS=$(kubectl get secret registry-pg-app -n default -o jsonpath='{.data.password}' | base64 -d) | |
| echo "Database user: $DB_USER" | |
| # Create a ConfigMap with the SQL file | |
| echo "Creating ConfigMap with migration SQL..." | |
| kubectl create configmap migration-010-dry-run \ | |
| --from-file=migration.sql=apply_migration_010_to_prod.sql \ | |
| -n default | |
| # Create pod to run the migration SQL | |
| POD_NAME="pg-migration-dry-run-$(date +%s)" | |
| echo "Creating pod: $POD_NAME" | |
| cat <<EOF | kubectl apply -f - | |
| apiVersion: v1 | |
| kind: Pod | |
| metadata: | |
| name: $POD_NAME | |
| namespace: default | |
| spec: | |
| restartPolicy: Never | |
| containers: | |
| - name: postgres-client | |
| image: postgres:15 | |
| env: | |
| - name: PGPASSWORD | |
| value: "$DB_PASS" | |
| - name: PGUSER | |
| value: "$DB_USER" | |
| command: | |
| - bash | |
| - -c | |
| - | | |
| echo "Waiting for database..." | |
| pg_isready -h registry-pg-rw -U \$PGUSER | |
| echo "Database is ready!" | |
| echo "" | |
| echo "Running migration dry run..." | |
| psql -h registry-pg-rw -U \$PGUSER -d app -f /migration/migration.sql | |
| volumeMounts: | |
| - name: migration-sql | |
| mountPath: /migration | |
| volumes: | |
| - name: migration-sql | |
| configMap: | |
| name: migration-010-dry-run | |
| EOF | |
| # Wait for completion | |
| echo "" | |
| echo "Waiting for migration dry run to complete..." | |
| kubectl wait --for=jsonpath='{.status.phase}'=Succeeded pod/$POD_NAME -n default --timeout=120s || { | |
| echo "Pod did not complete successfully, checking status..." | |
| kubectl get pod $POD_NAME -n default | |
| } | |
| # Get logs regardless of success/failure | |
| echo "" | |
| echo "=== Migration Dry Run Results ===" | |
| kubectl logs $POD_NAME -n default || echo "Failed to get logs" | |
| # Cleanup (always run, even if previous steps failed) | |
| echo "" | |
| echo "Cleaning up..." | |
| kubectl delete pod $POD_NAME -n default || true | |
| kubectl delete configmap migration-010-dry-run -n default || true | |
| echo "Cleanup complete" | |
| - name: Reset Staging Database | |
| if: inputs.environment == 'staging' && inputs.action == 'reset-staging-db' | |
| run: | | |
| set -e | |
| echo "=== RESETTING STAGING DATABASE ===" | |
| echo "⚠️ WARNING: This will DROP ALL TABLES in the staging database" | |
| echo "Environment: ${{ inputs.environment }}" | |
| echo "" | |
| # Safety check: Ensure we're on staging | |
| if [ "${{ inputs.environment }}" != "staging" ]; then | |
| echo "❌ ERROR: This action can only be run on staging!" | |
| exit 1 | |
| fi | |
| # Get database credentials | |
| DB_USER=$(kubectl get secret registry-pg-app -n default -o jsonpath='{.data.username}' | base64 -d) | |
| DB_PASS=$(kubectl get secret registry-pg-app -n default -o jsonpath='{.data.password}' | base64 -d) | |
| echo "Database user: $DB_USER" | |
| # Create pod to reset the database | |
| POD_NAME="pg-reset-$(date +%s)" | |
| echo "Creating pod: $POD_NAME" | |
| cat <<EOF | kubectl apply -f - | |
| apiVersion: v1 | |
| kind: Pod | |
| metadata: | |
| name: $POD_NAME | |
| namespace: default | |
| spec: | |
| restartPolicy: Never | |
| containers: | |
| - name: postgres-client | |
| image: postgres:15 | |
| env: | |
| - name: PGPASSWORD | |
| value: "$DB_PASS" | |
| - name: PGUSER | |
| value: "$DB_USER" | |
| command: | |
| - bash | |
| - -c | |
| - | | |
| echo "Waiting for database..." | |
| pg_isready -h registry-pg-rw -U \$PGUSER | |
| echo "Database is ready!" | |
| echo "" | |
| echo "=== Listing tables before reset ===" | |
| psql -h registry-pg-rw -U \$PGUSER -d app -c '\dt' | |
| echo "" | |
| echo "=== Dropping all tables ===" | |
| psql -h registry-pg-rw -U \$PGUSER -d app -c " | |
| DROP TABLE IF EXISTS servers CASCADE; | |
| DROP TABLE IF EXISTS schema_migrations CASCADE; | |
| " | |
| echo "" | |
| echo "=== Verifying tables are dropped ===" | |
| psql -h registry-pg-rw -U \$PGUSER -d app -c '\dt' | |
| echo "" | |
| echo "✅ Database reset complete!" | |
| echo "The registry application will recreate the schema on next startup." | |
| EOF | |
| # Wait for completion | |
| echo "" | |
| echo "Waiting for database reset to complete..." | |
| kubectl wait --for=jsonpath='{.status.phase}'=Succeeded pod/$POD_NAME -n default --timeout=60s || { | |
| echo "Pod did not complete successfully, checking status..." | |
| kubectl get pod $POD_NAME -n default | |
| } | |
| # Get logs | |
| echo "" | |
| echo "=== Database Reset Results ===" | |
| kubectl logs $POD_NAME -n default || echo "Failed to get logs" | |
| # Cleanup | |
| echo "" | |
| echo "Cleaning up..." | |
| kubectl delete pod $POD_NAME -n default || true | |
| echo "Cleanup complete" | |
| echo "" | |
| echo "⚠️ Next steps:" | |
| echo "1. Restart the registry deployment to trigger migrations" | |
| echo "2. The application will recreate all tables with the correct schema" |