Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix domain tests #73

Merged
merged 2 commits into from
Mar 13, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions datamimic_ce/contexts/context.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
import random
import re
import statistics
import traceback
import types
import uuid
from abc import ABC
Expand Down Expand Up @@ -205,6 +206,7 @@ def recursion_data_dict(recursion_dict):
"(e.g., 'True' vs 'true') adhere to the required formats."
) from e
except Exception as e:
traceback.print_exc()
raise ValueError(f"Failed while evaluate '{expr}': {str(e)}") from e

@staticmethod
Expand Down
11 changes: 11 additions & 0 deletions datamimic_ce/domains/common/generators/address_generator.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@

import random

from datamimic_ce.domains.common.literal_generators.company_name_generator import CompanyNameGenerator
from datamimic_ce.logger import logger
from datamimic_ce.domain_core.base_domain_generator import BaseDomainGenerator
from datamimic_ce.domains.common.generators.city_generator import CityGenerator
Expand Down Expand Up @@ -36,6 +37,7 @@ def __init__(self, country_code: str = _DEFAULT_DATASET):
self._country_generator = CountryGenerator()
self._phone_number_generator = PhoneNumberGenerator()
self._current_street_name_generator = None
self._company_name_generator = CompanyNameGenerator()

@property
def country_code(self) -> str:
Expand Down Expand Up @@ -72,7 +74,16 @@ def phone_number_generator(self) -> PhoneNumberGenerator:
The phone number generator.
"""
return self._phone_number_generator

@property
def company_name_generator(self) -> CompanyNameGenerator:
"""Get the company name generator.

Returns:
The company name generator.
"""
return self._company_name_generator

def generate_street_name(self) -> str:
"""Generate a street name.

Expand Down
2 changes: 1 addition & 1 deletion datamimic_ce/domains/common/generators/city_generator.py
Original file line number Diff line number Diff line change
Expand Up @@ -119,7 +119,7 @@ def get_random_city(self) -> dict[str, Any]:
"name": city_row[city_header_dict.get("name")],
"postal_code": city_row[city_header_dict.get("postalCode")],
"area_code": city_row[city_header_dict.get("areaCode")],
"state_id": city_row[city_header_dict.get("state.id")],
"state_id": city_row[city_header_dict.get("state.id")] if "state.id" in city_header_dict else None,
"state": city_row[city_header_dict.get("state")] if "state" in city_header_dict else None,
"language": city_row[city_header_dict.get("language")] if "language" in city_header_dict else None,
"population": city_row[city_header_dict.get("population")] if "population" in city_header_dict else None,
Expand Down
5 changes: 4 additions & 1 deletion datamimic_ce/domains/common/generators/country_generator.py
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,10 @@ def get_country_by_iso_code(self, iso_code: str) -> dict[str, Any] | None:
# Load country data
country_data = self.load_country_data()

return country_data.get(iso_code)
return_value = country_data.get(iso_code)
if return_value is None:
raise ValueError(f"Country with ISO code {iso_code} not found")
return return_value

def get_random_country(self) -> dict[str, Any]:
"""Get a random country.
Expand Down
6 changes: 3 additions & 3 deletions datamimic_ce/domains/common/generators/person_generator.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,16 +25,16 @@ class PersonGenerator(BaseDomainGenerator):
Provides methods to generate person-related attributes such as
first name, last name, email address, phone number, and address.
"""
def __init__(self, country_code: str = "US"):
def __init__(self, country_code: str = "US", min_age: int = 18, max_age: int = 65, female_quota: float = 0.5):
self._country_code = country_code
self._gender_generator = GenderGenerator()
self._gender_generator = GenderGenerator(female_quota=female_quota)
self._given_name_generator = GivenNameGenerator(dataset=country_code)
self._family_name_generator = FamilyNameGenerator(dataset=country_code)
self._email_generator = EmailAddressGenerator(dataset=country_code)
self._phone_generator = PhoneNumberGenerator(dataset=country_code)
self._address_generator = AddressGenerator(country_code=country_code)
from datamimic_ce.utils.class_factory_ce_util import ClassFactoryCEUtil
self._birthdate_generator = BirthdateGenerator(class_factory_util=ClassFactoryCEUtil())
self._birthdate_generator = BirthdateGenerator(class_factory_util=ClassFactoryCEUtil(), min_age=min_age, max_age=max_age)
self._academic_title_generator = AcademicTitleGenerator(dataset=country_code)
self._nobility_title_generator = NobilityTitleGenerator(dataset=country_code)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,10 @@
# See LICENSE file for the full text of the license.
# For questions and support, contact: [email protected]

from datamimic_ce.core.interfaces import Generator
from datamimic_ce.generators.data_faker_generator import DataFakerGenerator
from datamimic_ce.domain_core.base_literal_generator import BaseLiteralGenerator
from datamimic_ce.domains.common.literal_generators.data_faker_generator import DataFakerGenerator


class CNPJGenerator(Generator):
class CNPJGenerator(BaseLiteralGenerator):
"""
Generates Brazilian CNPJ numbers.
"""
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,10 @@
# See LICENSE file for the full text of the license.
# For questions and support, contact: [email protected]

from datamimic_ce.core.interfaces import Generator
from datamimic_ce.generators.data_faker_generator import DataFakerGenerator
from datamimic_ce.domain_core.base_literal_generator import BaseLiteralGenerator
from datamimic_ce.domains.common.literal_generators.data_faker_generator import DataFakerGenerator


class CPFGenerator(Generator):
class CPFGenerator(BaseLiteralGenerator):
"""
Generate Brazilian SSN also known in Brazil as CPF.
Can also use SSN Generator but The SSN returns a valid number with numbers
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,10 @@

from typing import Any

from datamimic_ce.core.interfaces import Generator
from datamimic_ce.generators.data_faker_generator import DataFakerGenerator
from datamimic_ce.domain_core.base_literal_generator import BaseLiteralGenerator
from datamimic_ce.domains.common.literal_generators.data_faker_generator import DataFakerGenerator


class EANGenerator(Generator):
class EANGenerator(BaseLiteralGenerator):
def __init__(self, locale: str | None = "en_US") -> None:
self._gen = DataFakerGenerator(method="ean", locale=locale)

Expand Down
46 changes: 19 additions & 27 deletions datamimic_ce/domains/common/literal_generators/generator_util.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,11 @@
# JobTitleGenerator,
# )
# from datamimic_ce.domains.common.literal_generators.cnpj_generator import CNPJGenerator
from datamimic_ce.domains.common.literal_generators.cnpj_generator import CNPJGenerator
from datamimic_ce.domains.common.literal_generators.color_generators import ColorGenerator
from datamimic_ce.domains.common.literal_generators.company_name_generator import CompanyNameGenerator
# from datamimic_ce.domains.common.literal_generators.cpf_generator import CPFGenerator
from datamimic_ce.domains.common.literal_generators.cpf_generator import CPFGenerator
from datamimic_ce.domains.common.literal_generators.data_faker_generator import DataFakerGenerator
from datamimic_ce.domains.common.literal_generators.department_name_generator import DepartmentNameGenerator
# from datamimic_ce.domains.common.literal_generators.document_generators import (
Expand All @@ -33,6 +35,7 @@
# )
from datamimic_ce.domains.common.literal_generators.domain_generator import DomainGenerator
# from datamimic_ce.domains.common.literal_generators.ean_generator import EANGenerator
from datamimic_ce.domains.common.literal_generators.ean_generator import EANGenerator
from datamimic_ce.domains.common.literal_generators.email_address_generator import EmailAddressGenerator
from datamimic_ce.domains.common.literal_generators.family_name_generator import FamilyNameGenerator
from datamimic_ce.domains.common.literal_generators.float_generator import FloatGenerator
Expand Down Expand Up @@ -66,6 +69,9 @@
from datamimic_ce.domains.common.literal_generators.street_name_generator import StreetNameGenerator
# from datamimic_ce.domains.common.literal_generators.text_generators import ParagraphGenerator
# from datamimic_ce.domains.common.literal_generators.url_generator import UrlGenerator
from datamimic_ce.domains.common.literal_generators.uuid_generator import UUIDGenerator
from datamimic_ce.domains.common.literal_generators.ssn_generator import SSNGenerator
from datamimic_ce.domains.healthcare.generators.healthcare_generators import DiagnosisGenerator
from datamimic_ce.logger import logger
from datamimic_ce.statements.statement import Statement

Expand Down Expand Up @@ -94,10 +100,10 @@ def __init__(self, context: Context):
"BooleanGenerator": BooleanGenerator,
"DataFakerGenerator": DataFakerGenerator,
# Identity and Personal Information
# "SSNGenerator": SSNGenerator,
# "CNPJGenerator": CNPJGenerator,
# "CPFGenerator": CPFGenerator,
# "EANGenerator": EANGenerator,
"SSNGenerator": SSNGenerator,
"CNPJGenerator": CNPJGenerator,
"CPFGenerator": CPFGenerator,
"EANGenerator": EANGenerator,
"GenderGenerator": GenderGenerator,
"BirthdateGenerator": BirthdateGenerator,
"PhoneNumberGenerator": PhoneNumberGenerator,
Expand All @@ -124,7 +130,7 @@ def __init__(self, context: Context):
# "FilePathGenerator": FilePathGenerator,
# "MIMETypeGenerator": MIMETypeGenerator,
# Security
# "UUIDGenerator": UUIDGenerator,
"UUIDGenerator": UUIDGenerator,
# "HashGenerator": HashGenerator,
# "TokenGenerator": TokenGenerator,
# "MnemonicPhraseGenerator": MnemonicPhraseGenerator,
Expand All @@ -140,7 +146,7 @@ def __init__(self, context: Context):
# "ScientificUnitGenerator": ScientificUnitGenerator,
# Healthcare
# "AllergyGenerator": AllergyGenerator,
# "DiagnosisGenerator": DiagnosisGenerator,
"DiagnosisGenerator": DiagnosisGenerator,
# "ImmunizationGenerator": ImmunizationGenerator,
# "LabResultGenerator": LabResultGenerator,
# "MedicalAppointmentGenerator": MedicalAppointmentGenerator,
Expand All @@ -153,12 +159,6 @@ def __init__(self, context: Context):
"SequenceTableGenerator": SequenceTableGenerator,
}

self._generator_with_count = (
"DomainGenerator",
"EmailAddressGenerator",
"FamilyNameGenerator",
"GivenNameGenerator",
)
self._generator_with_class_factory_util = (
"IntegerGenerator",
"StringGenerator",
Expand Down Expand Up @@ -382,31 +382,23 @@ def create_generator(
# Handle other generators with constructor parameters
if class_name != generator_str:
local_ns = copy.deepcopy(self._class_dict)
if class_name in self._generator_with_count or class_name in self._generator_with_class_factory_util:
if class_name in self._generator_with_class_factory_util:
class_name, args_str = generator_str[:-1].split("(")
# Filtering out empty arguments to avoid extra commas
args = [arg.strip() for arg in args_str.split(",") if arg.strip()]

if class_name in self._generator_with_count:
args.append(f"generated_count={generated_count}")
else:
args.append("class_factory_util=class_factory_util")
local_ns.update({"class_factory_util": self._context.root.class_factory_util})
args.append("class_factory_util=class_factory_util")
local_ns.update({"class_factory_util": self._context.root.class_factory_util})

new_args_str = ", ".join(args)
generator_str = f"{class_name}({new_args_str})"

result = self._context.evaluate_python_expression(generator_str, local_ns)
else:
# Handle simple instance initialization
if class_name in self._generator_with_count:
if class_name == "DomainGenerator":
result = cls(generated_count=generated_count)
else:
result = cls(
dataset=self._context.root.default_dataset,
generated_count=generated_count,
)
if class_name in ["EmailAddressGenerator"]:
result = cls(
dataset=self._context.root.default_dataset,
)
elif class_name in self._generator_with_class_factory_util:
result = cls(class_factory_util=self._context.root.class_factory_util)
else:
Expand Down
18 changes: 11 additions & 7 deletions datamimic_ce/domains/common/literal_generators/sector_generator.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,18 +25,22 @@ def __init__(self, dataset: str | None = "US", locale: str | None = None) -> Non
# Ensure country_code is never None by defaulting to "US"
country_code = locale if locale is not None else (dataset if dataset is not None else "US")

file_path = Path(__file__).parent.joinpath(f"data/organization/sector_{country_code}.csv")
file_path = Path(__file__).parent.parent.parent.parent.joinpath(f"domain_data/common/organization/sector_{country_code}.csv")

try:
# Use the file content storage to cache the data
self._sector_data_load = FileContentStorage.load_file_with_custom_func(
cache_key=str(file_path),
read_func=lambda: FileUtil.read_csv_to_list_of_tuples_without_header(file_path)
)
except FileNotFoundError as e:
raise ValueError(f"Sector data does not exist for country code '{country_code}': {e}") from e

# Use the file content storage to cache the data
self._sector_data_load = FileContentStorage.load_file_with_custom_func(
cache_key=str(file_path),
read_func=lambda: FileUtil.read_mutil_column_wgt_file(file_path)
)

def generate(self) -> str:
"""Generate a random sector.

Returns:
A randomly chosen sector.
"""
return random.choice(self._sector_data_load)
return random.choice(self._sector_data_load)[0]
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,11 @@

from typing import Any

from datamimic_ce.core.interfaces import Generator
from datamimic_ce.generators.data_faker_generator import DataFakerGenerator
from datamimic_ce.domain_core.base_literal_generator import BaseLiteralGenerator
from datamimic_ce.domains.common.literal_generators.data_faker_generator import DataFakerGenerator


class SSNGenerator(Generator):
class SSNGenerator(BaseLiteralGenerator):
def __init__(self, locale: str | None = "en_US") -> None:
self._gen = DataFakerGenerator(method="ssn", locale=locale)

Expand Down
6 changes: 6 additions & 0 deletions datamimic_ce/domains/common/models/address.py
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,11 @@ def private_phone(self) -> str:
@property_cache
def fax(self) -> str:
return self._address_generator.phone_number_generator.generate()

@property
@property_cache
def organization(self) -> str:
return self._address_generator.company_name_generator.generate()

def to_dict(self) -> dict[str, Any]:
return {
Expand All @@ -114,4 +119,5 @@ def to_dict(self) -> dict[str, Any]:
"phone": self.phone,
"mobile_phone": self.mobile_phone,
"fax": self.fax,
"organization": self.organization,
}
5 changes: 4 additions & 1 deletion datamimic_ce/domains/common/models/person.py
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,10 @@ def nobility_title(self) -> str:
def salutation(self) -> str:
"""Get the salutation of the person.
"""
return self._person_generator.load_salutation_data()[self.gender.upper()]
gender = self.gender.upper()

return self._person_generator.load_salutation_data()[gender] if gender != "OTHER" else ""

def to_dict(self) -> dict[str, Any]:
"""Convert the person to a dictionary.

Expand Down
4 changes: 2 additions & 2 deletions datamimic_ce/domains/common/services/city_service.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,6 @@ class CityService(BaseDomainService[City]):

This class provides methods for creating, retrieving, and managing city data.
"""
def __init__(self):
super().__init__(CityGenerator(), City)
def __init__(self, dataset: str = "US"):
super().__init__(CityGenerator(dataset), City)

4 changes: 2 additions & 2 deletions datamimic_ce/domains/common/services/person_service.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,5 +16,5 @@ class PersonService(BaseDomainService[Person]):
This class provides methods for creating, retrieving, and managing person data.
"""

def __init__(self):
super().__init__(PersonGenerator(), Person)
def __init__(self, min_age: int = 18, max_age: int = 65, female_quota: float = 0.5):
super().__init__(PersonGenerator(min_age=min_age, max_age=max_age, female_quota=female_quota), Person)
Loading