drop_invalid_rows still raises schema-level errors (#2000)

* drop_invalid_rows still raises schema-level errors

Signed-off-by: cosmicBboy <[email protected]>

* update drop invalid rows docs

Signed-off-by: cosmicBboy <[email protected]>

---------

Signed-off-by: cosmicBboy <[email protected]>

Author: cosmicBboy

.pylintrc

@@ -57,5 +57,6 @@ disable=
     multiple-statements,
     cyclic-import,
     too-many-positional-arguments,
+    too-many-function-args,
     # Due to custom `immutable` decorator replacing `dataclasses.dataclass`
     invalid-field-call

docs/source/drop_invalid_rows.md

@@ -32,7 +32,7 @@ import pandas as pd
 import pandera.pandas as pa
 
 
-df = pd.DataFrame({"counter": ["1", "2", "3"]})
+df = pd.DataFrame({"counter": [1, 2, 3]})
 schema = pa.DataFrameSchema(
     {"counter": pa.Column(int, checks=[pa.Check(lambda x: x >= 3)])},
     drop_invalid_rows=True,
@@ -48,7 +48,7 @@ import pandas as pd
 import pandera.pandas as pa
 
 
-series = pd.Series(["1", "2", "3"])
+series = pd.Series([1, 2, 3])
 schema = pa.SeriesSchema(
     int,
     checks=[pa.Check(lambda x: x >= 3)],
@@ -65,7 +65,7 @@ import pandas as pd
 import pandera.pandas as pa
 
 
-df = pd.DataFrame({"counter": ["1", "2", "3"]})
+df = pd.DataFrame({"counter": [1, 2, 3]})
 schema = pa.Column(
     int,
     name="counter",

pandera/backends/pandas/base.py

@@ -189,27 +189,16 @@ def defaultdict_to_dict(d):
 
     def drop_invalid_rows(self, check_obj, error_handler: ErrorHandler):
         """Remove invalid elements in a check obj according to failures caught by the error handler."""
-        errors = error_handler.schema_errors
-        for err in errors:
+        for err in error_handler.schema_errors:
+            if isinstance(err.failure_cases, str):
+                # if the failure cases are a string, it means the error is
+                # a schema-level error.
+                continue
             if isinstance(check_obj.index, pd.MultiIndex):
-                # MultiIndex values are saved on the error as strings so need to be cast back
-                # to their original types
-                if isinstance(err.failure_cases, str):
-                    # string type failure cases indicates that the entire column/dataframe
-                    # being checked didn't pass validation, meaning the entire dataframe
-                    # is invalid.
-                    index_values = check_obj.index
-                else:
-                    index_tuples = err.failure_cases["index"].apply(eval)
-                    index_values = pd.MultiIndex.from_tuples(index_tuples)
+                index_tuples = err.failure_cases["index"].apply(eval)
+                index_values = pd.MultiIndex.from_tuples(index_tuples)
             else:
-                if isinstance(err.failure_cases, str):
-                    # string type failure cases indicates that the entire column/dataframe
-                    # being checked didn't pass validation, meaning the entire dataframe
-                    # is invalid.
-                    index_values = check_obj.index
-                else:
-                    index_values = err.failure_cases["index"]
+                index_values = err.failure_cases["index"]
 
             mask = ~check_obj.index.isin(index_values)
             check_obj = check_obj.loc[mask]

pandera/backends/pandas/container.py

@@ -116,8 +116,18 @@ def validate(
 
         if error_handler.collected_errors:
             if getattr(schema, "drop_invalid_rows", False):
+                # if the failure cases are a string, it means the error is
+                # a schema-level error.
+                if any(
+                    isinstance(err.failure_cases, str)
+                    for err in error_handler.schema_errors
+                ):
+                    raise SchemaErrors(
+                        schema=schema,
+                        schema_errors=error_handler.schema_errors,
+                        data=check_obj,
+                    )
                 check_obj = self.drop_invalid_rows(check_obj, error_handler)
-                return check_obj
             else:
                 raise SchemaErrors(
                     schema=schema,
@@ -129,7 +139,7 @@ def validate(
 
     def run_checks_and_handle_errors(
         self,
-        error_handler,
+        error_handler: ErrorHandler,
         schema,
         check_obj,
         column_info,
@@ -139,7 +149,7 @@ def run_checks_and_handle_errors(
         head,
         tail,
         random_state,
-    ):
+    ) -> ErrorHandler:
         """Run checks on schema"""
         # pylint: disable=too-many-locals