Allow strict='filter' and coerce='True' at the same time for PySpark schemas. Generalize test to validate this case. (#2032)

gfilaci · web-flow · commit 971cae877ba6 · 2025-07-03T10:27:01.000-04:00
Signed-off-by: Gianluca Filaci &lt;gianluca.filaci@gmail.com&gt;
Co-authored-by: Gianluca Filaci &lt;&gt;
diff --git a/pandera/backends/pyspark/container.py b/pandera/backends/pyspark/container.py
@@ -390,7 +390,9 @@ def strict_filter_columns(
                     )
 
         if schema.strict == "filter":
+            schema = check_obj.pandera.schema
             check_obj = check_obj.drop(*filter_out_columns)
+            check_obj.pandera.add_schema(schema)
 
         return check_obj
 
diff --git a/tests/pyspark/test_pyspark_model.py b/tests/pyspark/test_pyspark_model.py
@@ -370,6 +370,7 @@ def test_dataframe_schema_strict(
             if df_out.pandera.errors:
                 raise pa.PysparkSchemaError
 
+        schema.coerce = True
         schema.strict = "filter"
         assert isinstance(schema.validate(df), DataFrame)
 

Original file line number	Diff line number	Diff line change
`@@ -390,7 +390,9 @@ def strict_filter_columns(`
`390`	`390`	`)`
`391`	`391`
`392`	`392`	`if schema.strict == "filter":`
	`393`	`+ schema = check_obj.pandera.schema`
`393`	`394`	`check_obj = check_obj.drop(*filter_out_columns)`
	`395`	`+ check_obj.pandera.add_schema(schema)`
`394`	`396`
`395`	`397`	`return check_obj`
`396`	`398`