Implement regex option and add additional checks (#2061)

* Run `pre-commit` on all files to fix linter issues

Signed-off-by: Deepyaman Datta <[email protected]>

* Implement `regex` option and add additional checks

Signed-off-by: Deepyaman Datta <[email protected]>

* Support validating set of columns matching a regex

Signed-off-by: Deepyaman Datta <[email protected]>

---------

Signed-off-by: Deepyaman Datta <[email protected]>

Author: deepyaman

pandera/api/ibis/components.py

@@ -1,5 +1,6 @@
 """Core Ibis schema component specifications."""
 
+import logging
 from typing import Any, Optional, Type
 
 import ibis
@@ -11,6 +12,8 @@
 from pandera.engines import ibis_engine
 from pandera.utils import is_regex
 
+logger = logging.getLogger(__name__)
+
 
 class Column(ComponentSchema[ibis.Table]):
     """Validate types and properties of table columns."""
@@ -97,7 +100,7 @@ def __init__(
         self.regex = regex
         self.name = name
 
-        # self.set_regex()  # TODO(deepyaman): Implement method.
+        self.set_regex()
 
     # pylint: disable=unused-argument
     @staticmethod
@@ -118,10 +121,22 @@ def selector(self):
             return f"^{self.name}$"
         return self.name
 
+    def set_regex(self):
+        if self.name is None:
+            return
+
+        if is_regex(self.name) and not self.regex:
+            logger.info(
+                f"Column schema '{self.name}' is a regex expression. "
+                "Setting regex=True."
+            )
+            self.regex = True
+
     def set_name(self, name: str):
         """Set or modify the name of a column object.
 
         :param str name: the name of the column object
         """
         self.name = name
+        self.set_regex()
         return self

pandera/backends/ibis/components.py

@@ -2,9 +2,10 @@
 
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, List, Optional, cast
+from typing import TYPE_CHECKING, Iterable, List, Optional, cast
 
 import ibis
+import ibis.selectors as s
 
 from pandera.api.base.error_handler import ErrorHandler
 from pandera.backends.base import CoreCheckResult
@@ -36,45 +37,68 @@ def validate(
         """Validation backend implementation for Ibis table columns."""
         error_handler = ErrorHandler(lazy)
 
-        # TODO(deepyaman): subsample the check object if head, tail, or sample are specified
-        sample = check_obj[schema.name]
-
-        # run the checks
-        core_checks = [
-            self.check_dtype,
-            self.run_checks,
-        ]
-
-        args = (sample, schema)
-        for check in core_checks:
-            results = check(*args)
-            if isinstance(results, CoreCheckResult):
-                results = [results]
-
-            for result in results:
-                if result.passed:
-                    continue
-                # Why cast `results` only in components.py, not in container.py?
-                results = cast(List[CoreCheckResult], results)
-                if result.schema_error is not None:
-                    error = result.schema_error
-                else:
-                    error = SchemaError(
-                        schema=schema,
-                        data=check_obj,
-                        message=result.message,
-                        failure_cases=result.failure_cases,
-                        check=result.check,
-                        check_index=result.check_index,
-                        check_output=result.check_output,
-                        reason_code=result.reason_code,
-                    )
-                    error_handler.collect_error(  # Why indent (unlike in container.py)?
-                        validation_type(result.reason_code),
-                        result.reason_code,
-                        error,
-                        original_exc=result.original_exc,
-                    )
+        def validate_column(check_obj, column_name):
+            # make sure the schema component mutations are reverted after
+            # validation
+            _orig_name = schema.name
+            _orig_regex = schema.regex
+
+            # set the column name and regex flag for a single column
+            schema.name = column_name
+            schema.regex = False
+
+            # TODO(deepyaman): subsample the check object if head, tail, or sample are specified
+            sample = check_obj[column_name]
+
+            # run the checks
+            core_checks = [
+                self.check_dtype,
+                self.run_checks,
+            ]
+
+            args = (sample, schema)
+            for check in core_checks:
+                results = check(*args)
+                if isinstance(results, CoreCheckResult):
+                    results = [results]
+
+                for result in results:
+                    if result.passed:
+                        continue
+                    # Why cast `results` only in components.py, not in container.py?
+                    results = cast(List[CoreCheckResult], results)
+                    if result.schema_error is not None:
+                        error = result.schema_error
+                    else:
+                        error = SchemaError(
+                            schema=schema,
+                            data=check_obj,
+                            message=result.message,
+                            failure_cases=result.failure_cases,
+                            check=result.check,
+                            check_index=result.check_index,
+                            check_output=result.check_output,
+                            reason_code=result.reason_code,
+                        )
+                        error_handler.collect_error(  # Why indent (unlike in container.py)?
+                            validation_type(result.reason_code),
+                            result.reason_code,
+                            error,
+                            original_exc=result.original_exc,
+                        )
+
+            # revert the schema component mutations
+            schema.name = _orig_name
+            schema.regex = _orig_regex
+
+        column_keys_to_check = (
+            self.get_regex_columns(schema, check_obj)
+            if schema.regex
+            else [schema.name]
+        )
+
+        for column_name in column_keys_to_check:
+            validate_column(check_obj, column_name)
 
         if lazy and error_handler.collected_errors:
             raise SchemaErrors(
@@ -85,6 +109,9 @@ def validate(
 
         return check_obj
 
+    def get_regex_columns(self, schema, check_obj) -> Iterable:
+        return check_obj.select(s.matches(schema.selector)).columns
+
     @validate_scope(scope=ValidationScope.SCHEMA)
     def check_dtype(
         self, check_obj: ibis.Column, schema: Column

pandera/backends/ibis/container.py

@@ -7,6 +7,8 @@
 from typing import TYPE_CHECKING, Any, Iterable, List, Optional
 
 import ibis
+import ibis.selectors as s
+from ibis.common.exceptions import IbisError
 
 from pandera.api.base.error_handler import ErrorHandler
 from pandera.config import ValidationScope
@@ -203,7 +205,7 @@ def collect_column_info(
             ):
                 absent_column_names.append(col_name)
 
-            if col_schema.regex:  # TODO(deepyaman): Implement functionality.
+            if col_schema.regex:
                 try:
                     column_names.extend(
                         col_schema.get_backend(check_obj).get_regex_columns(
@@ -291,7 +293,14 @@ def check_column_presence(
         if column_info.absent_column_names and not schema.add_missing_columns:
             for colname in column_info.absent_column_names:
                 if is_regex(colname):
-                    continue  # TODO(deepyaman): Support regex colnames.
+                    try:
+                        # don't raise an error if the column schema name is a
+                        # regex pattern
+                        check_obj.select(s.matches(colname))
+                        continue
+                    except IbisError:
+                        # regex pattern didn't match any columns
+                        pass
                 results.append(
                     CoreCheckResult(
                         passed=False,

pandera/engines/ibis_engine.py

@@ -25,7 +25,7 @@ class DataType(dtypes.DataType):
     type: Any = dataclasses.field(repr=False, init=False)
     """Native Ibis dtype boxed by the data type."""
 
-    def __init__(self, dtype: Any):
+    def __init__(self, dtype: Optional[Any] = None):
         super().__init__()
         object.__setattr__(self, "type", ibis.dtype(dtype))
         dtype_cls = dtype if inspect.isclass(dtype) else dtype.__class__
@@ -220,6 +220,8 @@ class UInt64(DataType, dtypes.UInt64):
 class Float32(DataType, dtypes.Float32):
     """Semantic representation of a :class:`dt.Float32`."""
 
+    type = dt.float32
+
 
 @Engine.register_dtype(
     equivalents=[

tests/ibis/test_ibis_builtin_checks.py

@@ -1367,7 +1367,7 @@ def pytest_generate_tests(self, metafunc):
         )
 
     def get_data_param(self):
-        """Generate the params which will be used to test this function. All the accpetable
+        """Generate the params which will be used to test this function. All the acceptable
         data types would be tested"""
         return {
             "test_unique_values_eq_check": [