Skip to content

Commit

Permalink
dev: temporarily remove option for TDF
Browse files Browse the repository at this point in the history
  • Loading branch information
lucas-nelson-uiuc committed Dec 26, 2024
1 parent b79c63d commit cc29b94
Showing 1 changed file with 3 additions and 7 deletions.
10 changes: 3 additions & 7 deletions src/tidy_tools/model/validate.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@
from pyspark.sql import Column
from pyspark.sql import DataFrame
from pyspark.sql import functions as F
from tidy_tools.dataframe import TidyDataFrame
from tidy_tools.error import TidyError


Expand Down Expand Up @@ -67,12 +66,9 @@ def validate_field(cls_field: attrs.Attribute, data: DataFrame) -> TidyError:
is returned for further logging.
"""
validate_func = _mapper(cls_field.validator)
if isinstance(data, TidyDataFrame):
invalid_entries = data.filter(
operator.inv(validate_func(cls_field.name)), disable=True
)
else:
invalid_entries = data.filter(operator.inv(validate_func(cls_field.name)))
# TODO: add support for TidyDataFrame;
# should disable messages to avoid unnecessary filter messages
invalid_entries = data.filter(operator.inv(validate_func(cls_field.name)))
try:
assert invalid_entries.isEmpty()
error = None
Expand Down

0 comments on commit cc29b94

Please sign in to comment.