Skip to content

Commit e16df8d

Browse files
vnghiavnghia
authored
add test for filesystem plugins (#1221)
* add initial tests for s3 * add test for az * bump `azurite` to `3.12.0` * add `test_filesystem` to `api`
1 parent 9310fd4 commit e16df8d

File tree

5 files changed

+442
-6
lines changed

5 files changed

+442
-6
lines changed

‎.github/workflows/api.yml

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -42,14 +42,15 @@ jobs:
4242
echo ${{ matrix.version }} | awk -F: '{print $1}' | xargs python -m pip install -U
4343
echo ${{ matrix.version }} | awk -F: '{print $2}' | xargs python -m pip install --no-deps -U
4444
if [[ "${{ matrix.version }}" == *"tensorflow-io-nightly"* ]]; then python -m pip install tensorflow-io-plugin-gs-nightly ; fi
45-
python -m pip install pytest-benchmark boto3 google-cloud-storage==1.32.0
45+
python -m pip install pytest-benchmark boto3 google-cloud-storage==1.32.0 azure-storage-blob==12.7.1
4646
python -m pip freeze
4747
python -c 'import tensorflow as tf; print(tf.version.VERSION)'
4848
python -c 'import tensorflow_io as tfio; print(tfio.version.VERSION)'
4949
python -m pytest -s -v tests/test_http.py
5050
python -m pytest -s -v tests/test_s3.py
5151
python -m pytest -s -v tests/test_azure.py
5252
python -m pytest -s -v tests/test_gcs.py
53+
python -m pytest -s -v tests/test_filesystem.py
5354
5455
linux:
5556
name: Linux ${{ matrix.python }} + ${{ matrix.version }}
@@ -80,14 +81,15 @@ jobs:
8081
echo ${{ matrix.version }} | awk -F: '{print $1}' | xargs python -m pip install -U
8182
echo ${{ matrix.version }} | awk -F: '{print $2}' | xargs python -m pip install --no-deps -U
8283
if [[ "${{ matrix.version }}" == *"tensorflow-io-nightly"* ]]; then python -m pip install tensorflow-io-plugin-gs-nightly ; fi
83-
python -m pip install pytest-benchmark boto3 google-cloud-storage==1.32.0
84+
python -m pip install pytest-benchmark boto3 google-cloud-storage==1.32.0 azure-storage-blob==12.7.1
8485
python -m pip freeze
8586
python -c 'import tensorflow as tf; print(tf.version.VERSION)'
8687
python -c 'import tensorflow_io as tfio; print(tfio.version.VERSION)'
8788
python -m pytest -s -v tests/test_http.py
8889
python -m pytest -s -v tests/test_s3.py
8990
python -m pytest -s -v tests/test_azure.py
9091
if [[ "${{ matrix.version }}" != "tf-nightly:tensorflow-io" ]]; then python -m pytest -s -v tests/test_gcs.py ; fi
92+
python -m pytest -s -v tests/test_filesystem.py
9193
9294
windows:
9395
name: Windows ${{ matrix.python }} + ${{ matrix.version }}

‎.github/workflows/benchmarks.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ jobs:
3333
echo ${{ matrix.version }} | awk -F: '{print $1}' | xargs python -m pip install -U
3434
echo ${{ matrix.version }} | awk -F: '{print $2}' | xargs python -m pip install --no-deps -U
3535
if [[ "${{ matrix.version }}" == *"tensorflow-io-nightly"* ]]; then python -m pip install tensorflow-io-plugin-gs-nightly ; fi
36-
python -m pip install -q scikit-image pytest pytest-benchmark boto3 fastavro avro-python3 scikit-image pandas pyarrow==2.0.0 google-cloud-pubsub==2.1.0 google-cloud-bigquery-storage==1.1.0 google-cloud-bigquery==2.3.1 google-cloud-storage==1.32.0
36+
python -m pip install -q scikit-image pytest pytest-benchmark boto3 fastavro avro-python3 scikit-image pandas pyarrow==2.0.0 google-cloud-pubsub==2.1.0 google-cloud-bigquery-storage==1.1.0 google-cloud-bigquery==2.3.1 google-cloud-storage==1.32.0 azure-storage-blob==12.7.1
3737
python -m pip freeze
3838
python -c 'import tensorflow as tf; print(tf.version.VERSION)'
3939
python -c 'import tensorflow_io as tfio; print(tfio.version.VERSION)'
@@ -68,7 +68,7 @@ jobs:
6868
echo ${{ matrix.version }} | awk -F: '{print $1}' | xargs python -m pip install -U
6969
echo ${{ matrix.version }} | awk -F: '{print $2}' | xargs python -m pip install --no-deps -U
7070
if [[ "${{ matrix.version }}" == *"tensorflow-io-nightly"* ]]; then python -m pip install tensorflow-io-plugin-gs-nightly ; fi
71-
python -m pip install -q scikit-image pytest pytest-benchmark boto3 fastavro avro-python3 scikit-image pandas pyarrow==2.0.0 google-cloud-pubsub==2.1.0 google-cloud-bigquery-storage==1.1.0 google-cloud-bigquery==2.3.1 google-cloud-storage==1.32.0
71+
python -m pip install -q scikit-image pytest pytest-benchmark boto3 fastavro avro-python3 scikit-image pandas pyarrow==2.0.0 google-cloud-pubsub==2.1.0 google-cloud-bigquery-storage==1.1.0 google-cloud-bigquery==2.3.1 google-cloud-storage==1.32.0 azure-storage-blob==12.7.1
7272
python -m pip freeze
7373
python -c 'import tensorflow as tf; print(tf.version.VERSION)'
7474
python -c 'import tensorflow_io as tfio; print(tfio.version.VERSION)'

‎.github/workflows/build.wheel.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ run_test() {
88
CPYTHON_VERSION=$($entry -c 'import sys; print(str(sys.version_info[0])+str(sys.version_info[1]))')
99
(cd wheelhouse && $entry -m pip install tensorflow_io_plugin_gs-*-cp${CPYTHON_VERSION}-*.whl)
1010
(cd wheelhouse && $entry -m pip install tensorflow_io-*-cp${CPYTHON_VERSION}-*.whl)
11-
$entry -m pip install -q pytest pytest-benchmark boto3 fastavro avro-python3 scikit-image pandas pyarrow==3.0.0 google-cloud-pubsub==2.1.0 google-cloud-bigtable==1.6.0 google-cloud-bigquery-storage==1.1.0 google-cloud-bigquery==2.3.1 google-cloud-storage==1.32.0 PyYAML==5.3.1
11+
$entry -m pip install -q pytest pytest-benchmark boto3 fastavro avro-python3 scikit-image pandas pyarrow==3.0.0 google-cloud-pubsub==2.1.0 google-cloud-bigtable==1.6.0 google-cloud-bigquery-storage==1.1.0 google-cloud-bigquery==2.3.1 google-cloud-storage==1.32.0 PyYAML==5.3.1 azure-storage-blob==12.8.1
1212
(cd tests && $entry -m pytest --benchmark-disable -v --import-mode=append $(find . -type f \( -iname "test_*_v1.py" \)))
1313
(cd tests && $entry -m pytest --benchmark-disable -v --import-mode=append $(find . -type f \( -iname "test_*.py" ! \( -iname "test_*_v1.py" -o -iname "test_bigquery.py" \) \)))
1414
# GRPC and test_bigquery tests have to be executed separately because of https://github.com/grpc/grpc/issues/20034

‎tests/test_azure/start_azure.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ set -e
22
set -o pipefail
33

44

5-
npm install azurite@2.7.0
5+
npm install azurite@3.12.0
66
echo starting azurite-blob
77
$(npm bin)/azurite-blob &
88
sleep 10 # Wait for storage emulator to start

0 commit comments

Comments
 (0)