From cc29b94e3d8d5f5e1b5bff9137aa9369d933ab1b Mon Sep 17 00:00:00 2001 From: lucas-nelson-uiuc Date: Thu, 26 Dec 2024 16:54:15 -0600 Subject: [PATCH] dev: temporarily remove option for TDF --- src/tidy_tools/model/validate.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/tidy_tools/model/validate.py b/src/tidy_tools/model/validate.py index a46c750..e972f05 100644 --- a/src/tidy_tools/model/validate.py +++ b/src/tidy_tools/model/validate.py @@ -6,7 +6,6 @@ from pyspark.sql import Column from pyspark.sql import DataFrame from pyspark.sql import functions as F -from tidy_tools.dataframe import TidyDataFrame from tidy_tools.error import TidyError @@ -67,12 +66,9 @@ def validate_field(cls_field: attrs.Attribute, data: DataFrame) -> TidyError: is returned for further logging. """ validate_func = _mapper(cls_field.validator) - if isinstance(data, TidyDataFrame): - invalid_entries = data.filter( - operator.inv(validate_func(cls_field.name)), disable=True - ) - else: - invalid_entries = data.filter(operator.inv(validate_func(cls_field.name))) + # TODO: add support for TidyDataFrame; + # should disable messages to avoid unnecessary filter messages + invalid_entries = data.filter(operator.inv(validate_func(cls_field.name))) try: assert invalid_entries.isEmpty() error = None