Adjust CI tests coverage in different pipelines. (#2281)

* Adjust CI tests coverage in different pipelines.
* Correct the way of installing tensorflow-text.
* Ignore a test in windows because of tensorflow-text issue.

Signed-off-by: Jay Zhang <[email protected]>

---------

Signed-off-by: Jay Zhang <[email protected]>

Author: fatcat-z

ci_build/azure_pipelines/pretrained_model_test-matrix.yml

@@ -41,7 +41,7 @@ jobs:
     platforms: ['linux', 'windows']
     python_versions: ['3.8']
     tf_versions: ['2.8.0', '2.13.0']
-    onnx_opsets: ['14']
+    onnx_opsets: ['17', '16', '14']
     job:
       steps:
       - template: 'pretrained_model_test.yml'

ci_build/azure_pipelines/pretrained_model_test.yml

@@ -18,7 +18,17 @@ jobs:
     platforms: ['linux']
     python_versions: ['3.8']
     tf_versions: ['2.8.0', '2.13.0']
-    onnx_opsets: ['18', '17', '16', '15']
+    onnx_opsets: ['18', '15']
     job:
       steps:
       - template: 'pretrained_model_test.yml'
+
+- template: 'templates/job_generator.yml'
+  parameters:
+    platforms: ['linux']
+    python_versions: ['3.10']
+    tf_versions: ['2.13.0']
+    onnx_opsets: ['18', '15']
+    job:
+      steps:
+      - template: 'pretrained_model_test.yml'

ci_build/azure_pipelines/templates/setup.yml

@@ -4,7 +4,6 @@ steps:
 - bash: |
     set -ex
     pip install pytest pytest-cov pytest-runner coverage graphviz requests pyyaml pillow pandas parameterized sympy coloredlogs flatbuffers timeout-decorator
-    pip install $(CI_PIP_TF_NAME)
     pip install $(CI_PIP_ONNX_NAME)
 
     # TF < 2.7 reuires numpy <= 1.19, but onnxruntime >= 1.11 requires numpy >= 1.21
@@ -24,41 +23,7 @@ steps:
     if [[ $CI_TF_VERSION == 2.* ]] ;
     then
       pip install onnxruntime-extensions
-
-      if [[ $CI_TF_VERSION == 2.3* ]] ;
-      then
-        pip install tensorflow-text==${CI_TF_VERSION}
-      fi
-      if [[ $CI_TF_VERSION == 2.4* ]] ;
-      then
-        pip install tensorflow-text==${CI_TF_VERSION}
-      fi
-      if [[ $CI_TF_VERSION == 2.5* ]] ;
-      then
-        pip install "tensorflow-text>=2.5,<2.6"
-      fi
-      if [[ $CI_TF_VERSION == 2.6* ]] ;
-      then
-        pip install "tensorflow-text>=2.6,<2.7"
-      fi
-      if [[ $CI_TF_VERSION == 2.7* ]] ;
-      then
-        pip install "tensorflow-text>=2.7,<2.8"
-      fi
-      if [[ $CI_TF_VERSION == 2.8* ]] ;
-      then
-        pip install "tensorflow-text>=2.8,<2.9"
-      fi
-      if [[ $CI_TF_VERSION == 2.9* ]] ;
-        then
-          pip install "tensorflow-text>=2.9,<2.10"
-      fi
-      if [[ $CI_TF_VERSION == 2.10* ]] || [[ $CI_TF_VERSION == 2.11* ]] ;
-        then
-          pip install "tensorflow-text>=2.10,<2.14"
-      else
-        pip install tensorflow-text
-      fi
+      pip install "tensorflow-text<=$(CI_TF_VERSION)"
 
       if [[ $CI_TF_VERSION < 2.8 ]] ;
       then
@@ -74,6 +39,8 @@ steps:
       pip install -i https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/ORT-Nightly/pypi/simple/ ort-nightly
     fi
 
+    pip uninstall -y tensorflow
+    pip install $(CI_PIP_TF_NAME)
     pip uninstall -y protobuf
     pip install "protobuf~=3.20"
 

ci_build/azure_pipelines/unit_test-matrix.yml

@@ -8,7 +8,7 @@ stages:
         python_versions: ['3.8']
         platforms: ['linux', 'windows']
         tf_versions: ['2.8.0', '2.13.0']
-        onnx_opsets: ['14']
+        onnx_opsets: ['17', '16', '14']
         skip_tflite_tests: 'False'
         skip_tf_tests: 'False'
         skip_tfjs_tests: 'True'

ci_build/azure_pipelines/unit_test.yml

@@ -8,7 +8,21 @@ stages:
         python_versions: ['3.8']
         platforms: ['linux', 'windows']
         tf_versions: ['2.8.0', '2.13.0']
-        onnx_opsets: ['18', '17', '16', '15']
+        onnx_opsets: ['18', '15']
+        skip_tflite_tests: 'False'
+        skip_tf_tests: 'False'
+        skip_tfjs_tests: 'True'
+        job:
+          steps:
+          - template: 'unit_test.yml'
+        report_coverage: 'True'
+
+    - template: 'templates/job_generator.yml'
+      parameters:
+        python_versions: ['3.10']
+        platforms: ['linux']
+        tf_versions: ['2.13.0']
+        onnx_opsets: ['18', '15']
         skip_tflite_tests: 'False'
         skip_tf_tests: 'False'
         skip_tfjs_tests: 'True'

tests/test_string_ops.py

@@ -8,7 +8,7 @@
 import tensorflow as tf
 
 from backend_test_base import Tf2OnnxBackendTestBase
-from common import requires_custom_ops, check_tf_min_version, check_opset_min_version
+from common import requires_custom_ops, check_tf_min_version, check_opset_min_version, get_test_config
 from tf2onnx import utils
 from tf2onnx import constants
 
@@ -118,6 +118,7 @@ def func(x1, x2):
             return tf.identity(mi, name=_TFOUTPUT)
         self._run_test_case(func, [_OUTPUT], {_INPUT: x_val1, _INPUT1: x_val2})
 
+    @unittest.skipIf(get_test_config().is_windows, "tensorflow-text lacks versions in windows.")
     @requires_custom_ops("RegexSplitWithOffsets")
     @check_tf_min_version("2.3", "tensorflow_text")
     def test_regex_split_with_offsets(self):