Reuse coerce in engines.utils (#645)

* reuse coerce logic in engines.utils

* add test_coerce_error

* rename coerce to try_coerce and _coerce to coerce

Author: Jean-Francois Zinque

pandera/dtypes.py

@@ -31,6 +31,14 @@ def coerce(self, data_container: Any):
         """Coerce data container to the data type."""
         raise NotImplementedError()
 
+    def try_coerce(self, data_container: Any):
+        """Coerce data container to the data type,
+        raises a `~pandera.errors.ParserError` if the coercion fails
+
+        :raises: :class:`~pandera.errors.ParserError`: if coercion fails
+        """
+        raise NotImplementedError()
+
     def __call__(self, data_container: Any):
         """Coerce data container to the data type."""
         return self.coerce(data_container)

pandera/engines/numpy_engine.py

@@ -43,12 +43,16 @@ def __post_init__(self):
             self, "type", np.dtype(self.type)
         )  # pragma: no cover
 
-    def coerce(
+    def coerce(self, data_container: PandasObject) -> PandasObject:
+        """Pure coerce without catching exceptions."""
+        return data_container.astype(self.type)
+
+    def try_coerce(
         self, data_container: Union[PandasObject, np.ndarray]
     ) -> Union[PandasObject, np.ndarray]:
         try:
-            return data_container.astype(self.type)
-        except (ValueError, TypeError) as exc:
+            return self.coerce(data_container)
+        except Exception as exc:  # pylint:disable=broad-except
             raise errors.ParserError(
                 f"Could not coerce {type(data_container)} data_container "
                 f"into type {self.type}",

pandera/engines/pandas_engine.py

@@ -72,14 +72,18 @@ def __post_init__(self):
         )  # pragma: no cover
 
     def coerce(self, data_container: PandasObject) -> PandasObject:
+        """Pure coerce without catching exceptions."""
+        return data_container.astype(self.type)
+
+    def try_coerce(self, data_container: PandasObject) -> PandasObject:
         try:
-            return data_container.astype(self.type)
-        except (ValueError, TypeError) as exc:
+            return self.coerce(data_container)
+        except Exception as exc:  # pylint:disable=broad-except
             raise errors.ParserError(
                 f"Could not coerce {type(data_container)} data_container "
                 f"into type {self.type}",
                 failure_cases=utils.numpy_pandas_coerce_failure_cases(
-                    data_container, self.type
+                    data_container, self
                 ),
             ) from exc
 
@@ -376,9 +380,7 @@ class Category(DataType, dtypes.Category):
     type: pd.CategoricalDtype = dataclasses.field(default=None, init=False)
 
     def __init__(  # pylint:disable=super-init-not-called
-        self,
-        categories: Optional[Iterable[Any]] = None,
-        ordered: bool = False,
+        self, categories: Optional[Iterable[Any]] = None, ordered: bool = False
     ) -> None:
         dtypes.Category.__init__(self, categories, ordered)
         object.__setattr__(

pandera/engines/utils.py

@@ -15,6 +15,10 @@ def numpy_pandas_coercible(series: pd.Series, type_: Any) -> pd.Series:
 
     Bisects the series until all the failure cases are found.
     """
+    # pylint: disable=import-outside-toplevel,cyclic-import
+    from pandera.engines import pandas_engine
+
+    data_type = pandas_engine.Engine.dtype(type_)
 
     def _bisect(series):
         assert (
@@ -25,9 +29,9 @@ def _bisect(series):
 
     def _coercible(series):
         try:
-            series.astype(type_)
+            data_type.coerce(series)
             return True
-        except (ValueError, TypeError):
+        except Exception:  # pylint:disable=broad-except
             return False
 
     search_list = [series] if series.size == 1 else _bisect(series)
@@ -60,6 +64,9 @@ def numpy_pandas_coerce_failure_cases(
     """
     # pylint: disable=import-outside-toplevel,cyclic-import
     from pandera import error_formatters
+    from pandera.engines import pandas_engine
+
+    data_type = pandas_engine.Engine.dtype(type_)
 
     if isinstance(data_container, np.ndarray):
         if len(data_container.shape) == 1:
@@ -76,16 +83,15 @@ def numpy_pandas_coerce_failure_cases(
 
     if isinstance(data_container, pd.DataFrame):
         check_output = data_container.apply(
-            numpy_pandas_coercible,
-            args=(type_,),
+            numpy_pandas_coercible, args=(data_type,)
         )
         _, failure_cases = check_utils.prepare_dataframe_check_output(
             data_container,
             check_output,
             ignore_na=False,
         )
     elif isinstance(data_container, pd.Series):
-        check_output = numpy_pandas_coercible(data_container, type_)
+        check_output = numpy_pandas_coercible(data_container, data_type)
         _, failure_cases = check_utils.prepare_series_check_output(
             data_container,
             check_output,

pandera/schema_components.py

@@ -376,7 +376,7 @@ def validate(
             check_obj.index = self.coerce_dtype(check_obj.index)
             # handles case where pandas native string type is not supported
             # by index.
-            obj_to_validate = self.dtype.coerce(
+            obj_to_validate = self.dtype.try_coerce(
                 pd.Series(check_obj.index, name=check_obj.index.name)
             )
         else:

pandera/schemas.py

@@ -326,7 +326,7 @@ def _coerce_dtype(self, obj: pd.DataFrame) -> pd.DataFrame:
             )
 
         try:
-            return self.dtype.coerce(obj)
+            return self.dtype.try_coerce(obj)
         except errors.ParserError as exc:
             raise errors.SchemaError(
                 self,
@@ -1689,7 +1689,7 @@ def coerce_dtype(self, obj: Union[pd.Series, pd.Index]) -> pd.Series:
             return obj
 
         try:
-            return self.dtype.coerce(obj)
+            return self.dtype.try_coerce(obj)
         except errors.ParserError as exc:
             msg = (
                 f"Error while coercing '{self.name}' to type "

tests/core/test_dtypes.py

@@ -363,6 +363,36 @@ def test_coerce_cast(dtypes, examples, data):
     assert expected_dtype.check(pandas_engine.Engine.dtype(coerced_dtype))
 
 
+@pytest.mark.parametrize(
+    "examples, type_, failure_indices",
+    [
+        (["a", 0, "b"], int, [0, 2]),
+        (
+            [
+                "2021-09-01",
+                datetime.datetime(2021, 1, 7),
+                pd.NaT,
+                "not_a_date",
+            ],
+            datetime.datetime,
+            [3],
+        ),
+    ],
+)
+def test_try_coerce(examples, type_, failure_indices):
+    """Test that try_coerce raises a ParseError."""
+    data_type = pandas_engine.Engine.dtype(type_)
+    data = pd.Series(examples)
+
+    with pytest.raises(pa.errors.ParserError):
+        data_type.try_coerce(data)
+
+    try:
+        data_type.try_coerce(data)
+    except pa.errors.ParserError as exc:
+        assert exc.failure_cases["index"].to_list() == failure_indices
+
+
 def test_coerce_string():
     """Test that strings can be coerced."""
     data = pd.Series([1, None], dtype="Int32")
@@ -487,8 +517,7 @@ def test_is_float(float_dtype: Any, expected: bool):
 
 @pytest.mark.parametrize(
     "complex_dtype, expected",
-    [(dtype, True) for dtype in complex_dtypes]
-    + [("string", False)],  # type: ignore
+    [(dtype, True) for dtype in complex_dtypes] + [("string", False)],  # type: ignore
 )
 def test_is_complex(complex_dtype: Any, expected: bool):
     """Test is_complex."""

tests/core/test_pandas_engine.py

@@ -39,6 +39,6 @@ def test_pandas_data_type_coerce(data_type):
         # don't test data types that require parameters e.g. Category
         return
     try:
-        data_type().coerce(pd.Series(["1", "2", "a"]))
+        data_type().try_coerce(pd.Series(["1", "2", "a"]))
     except ParserError as exc:
         assert exc.failure_cases.shape[0] > 0