Added unit tests for parquet-dataformat plugin #171
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: DataFusion E2E Integration Test | |
| on: | |
| pull_request: | |
| branches: | |
| - feature/datafusion | |
| push: | |
| branches: | |
| - feature/datafusion | |
| workflow_dispatch: | |
| jobs: | |
| test: | |
| runs-on: ubuntu-latest | |
| env: | |
| RUSTFLAGS: "-A unused_variables -A unused_mut" | |
| steps: | |
| - name: Checkout OpenSearch | |
| uses: actions/checkout@v4 | |
| with: | |
| path: OpenSearch | |
| - name: Set up JDK | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: 'temurin' | |
| java-version: '21' | |
| - name: Setup Rust | |
| uses: actions-rust-lang/setup-rust-toolchain@v1 | |
| with: | |
| components: rustfmt | |
| - name: Install Protocol Buffers | |
| if: runner.os == 'Linux' | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y protobuf-compiler | |
| - name: Install Protocol Buffers | |
| if: runner.os == 'macOS' | |
| run: brew install protobuf | |
| - name: Install Protocol Buffers | |
| if: runner.os == 'Windows' | |
| run: choco install protoc | |
| - name: Setup Gradle | |
| uses: gradle/actions/setup-gradle@v3 | |
| - name: Publish OpenSearch to Maven Local | |
| working-directory: OpenSearch | |
| run: ./gradlew publishToMavenLocal -x test -PrustDebug=true | |
| - name: Checkout SQL Plugin | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: vinaykpud/sql | |
| ref: feature/substrait-plan | |
| path: opensearch-sql | |
| - name: Publish SQL Plugin to Maven Local | |
| working-directory: opensearch-sql | |
| run: ./gradlew publishToMavenLocal -x test | |
| - name: Run DataFusionReaderManager Tests | |
| working-directory: OpenSearch | |
| run: ./gradlew :plugins:engine-datafusion:test --tests "org.opensearch.datafusion.DataFusionReaderManagerTests" | |
| - name: Run IndexFileDeleter Tests | |
| working-directory: OpenSearch | |
| run: ./gradlew :server:test --tests "org.opensearch.index.engine.exec.coord.IndexFileDeleterTests" | |
| - name: Run Native(Rust) UTs for ParquetDataFormat plugin | |
| working-directory: OpenSearch | |
| run: ./gradlew runNativeUnitTests | |
| - name: Run OpenSearch with DataFusion Plugin | |
| working-directory: OpenSearch | |
| run: | | |
| ./gradlew run \ | |
| --preserve-data \ | |
| -PremotePlugins="['org.opensearch.plugin:opensearch-job-scheduler:3.3.0.0-SNAPSHOT', 'org.opensearch.plugin:opensearch-sql-plugin:3.3.0.0-SNAPSHOT']" \ | |
| -PinstalledPlugins="['engine-datafusion']" & | |
| # Wait for OpenSearch to start | |
| timeout 300 bash -c 'until curl -s http://localhost:9200; do sleep 5; done' | |
| - name: Run SQL CalcitePPLClickBenchIT | |
| working-directory: opensearch-sql | |
| run: ./gradlew :integ-test:integTest --tests "org.opensearch.sql.calcite.clickbench.CalcitePPLClickBenchIT" -Dtests.method="testDataFusion" -Dtests.cluster=localhost:9200 -Dtests.rest.cluster=localhost:9200 -DignorePrometheus=true -Dtests.clustername=opensearch -Dtests.output=true |