From 06417dc9a2871ecccaf7f3735ee566d0553578d7 Mon Sep 17 00:00:00 2001 From: risto Date: Fri, 13 Dec 2024 10:21:06 +0100 Subject: [PATCH] fix: remove debug print statement in consent_manager.py refactor: comment out profile ID mismatch warning in event_computer.py style: remove extra newline in config.py perf: adjust timeout in event_source.py to 1 second feat: add multiple indexes to mysql schema tables for optimization chore: remove unused code from mysql schema table definitions fix: simplify error message in source_validation.py --- tracardi/config.py | 1 - tracardi/service/cache/event_source.py | 2 +- .../service/storage/mysql/schema/table.py | 86 +++---------------- .../service/tracking/source_validation.py | 4 +- 4 files changed, 15 insertions(+), 78 deletions(-) diff --git a/tracardi/config.py b/tracardi/config.py index 831df255..f4d9c1ac 100644 --- a/tracardi/config.py +++ b/tracardi/config.py @@ -46,7 +46,6 @@ def __init__(self): logger.info(f"EVENT_TO_PROFILE_COPY_CACHE_TTL={self.event_to_profile_coping_ttl}") logger.info(f"SOURCE_CACHE_TTL={self.source_ttl}") - class MysqlConfig: def __init__(self, env): diff --git a/tracardi/service/cache/event_source.py b/tracardi/service/cache/event_source.py index d7c25c4a..22be98ef 100644 --- a/tracardi/service/cache/event_source.py +++ b/tracardi/service/cache/event_source.py @@ -7,7 +7,7 @@ @AsyncCache(memory_cache.source_ttl, - timeout=.5, + timeout=1, max_one_cache_fill_every=.1, return_cache_on_error=True ) diff --git a/tracardi/service/storage/mysql/schema/table.py b/tracardi/service/storage/mysql/schema/table.py index 544eea52..d4713e05 100644 --- a/tracardi/service/storage/mysql/schema/table.py +++ b/tracardi/service/storage/mysql/schema/table.py @@ -123,6 +123,7 @@ class WorkflowTriggerTable(Base): __table_args__ = ( PrimaryKeyConstraint('id', 'tenant', 'production'), + Index('idx_collector', 'event_type_id', 'source_id', 'enabled', 'tenant', 'production'), ) running: bool = False @@ -237,6 +238,8 @@ class DestinationTable(Base): __table_args__ = ( PrimaryKeyConstraint('id', 'tenant', 'production'), + Index('idx_collector', 'event_type_id', 'source_id', 'enabled', 'tenant', 'production'), + ) running: bool = False @@ -298,6 +301,7 @@ class IdentificationPointTable(Base): __table_args__ = ( PrimaryKeyConstraint('id', 'tenant', 'production'), + Index('idx_collector', 'tenant', 'production', 'enabled', 'event_type_id', 'source_id') ) running: bool = False @@ -335,6 +339,7 @@ class EventRedirectTable(Base): __table_args__ = ( PrimaryKeyConstraint('id', 'tenant', 'production'), + Index('idx_collector', 'event_type', 'source_id', 'tenant', 'production'), ) running: bool = False @@ -356,6 +361,7 @@ class EventValidationTable(Base): __table_args__ = ( PrimaryKeyConstraint('id', 'tenant', 'production'), + Index('idx_collector', 'event_type', 'enabled', 'tenant', 'production'), ) running: bool = False @@ -402,6 +408,7 @@ class EventReshapingTable(Base): __table_args__ = ( PrimaryKeyConstraint('id', 'tenant', 'production'), + Index('idx_collector', 'event_type', 'tenant', 'production', 'enabled') ) running: bool = False @@ -425,7 +432,8 @@ class EventMappingTable(Base): __table_args__ = ( PrimaryKeyConstraint('id', 'tenant', 'production'), - UniqueConstraint('event_type', name='uiq_event_type') + UniqueConstraint('event_type', name='uiq_event_type'), + Index('idx_collector', 'event_type', 'tenant', 'production', 'enabled') ) running: bool = False @@ -496,6 +504,7 @@ class EventToProfileMappingTable(Base): __table_args__ = ( PrimaryKeyConstraint('id', 'tenant', 'production'), + Index('idx_collector', 'event_type_id', 'tenant', 'production', 'enabled') ) running: bool = False @@ -517,6 +526,7 @@ class EventDataComplianceTable(Base): __table_args__ = ( PrimaryKeyConstraint('id', 'tenant', 'production'), + Index('idx_collector', 'event_type_id', 'tenant', 'production', 'enabled') ) running: bool = False @@ -546,27 +556,6 @@ class ActivationTable(Base): running: bool = False -# class SegmentTable(Base): -# __tablename__ = 'segment' -# -# id = Column(String(40)) -# name = Column(Text) -# description = Column(Text) -# event_type = Column(String(64), default=None) -# condition = Column(Text) -# enabled = Column(Boolean, default=False) -# machine_name = Column(String(128)) -# -# tenant = Column(String(40)) -# production = Column(Boolean) -# -# __table_args__ = ( -# PrimaryKeyConstraint('id', 'tenant', 'production'), -# ) -# -# running: bool = False - - class ReportTable(Base): __tablename__ = 'report' @@ -587,31 +576,7 @@ class ReportTable(Base): running: bool = False - -# class ContentTable(Base): -# __tablename__ = 'content' -# -# id = Column(String(48)) -# profile_id = Column(String(40)) -# timestamp = Column(DateTime) -# type = Column(String(64)) -# url = Column(String(255)) -# source = Column(String(128)) -# author = Column(String(96)) -# copyright = Column(String(128)) -# content = Column(BLOB) -# text = Column(Text) -# properties = Column(JSON) -# traits = Column(JSON) -# -# tenant = Column(String(40)) -# production = Column(Boolean) -# -# __table_args__ = ( -# PrimaryKeyConstraint('id', 'tenant', 'production'), -# ) - - +# TODO probably not used class ImportTable(Base): __tablename__ = 'import' @@ -637,32 +602,6 @@ class ImportTable(Base): running: bool = False -# class WorkflowSegmentationTriggerTable(Base): -# __tablename__ = 'workflow_segmentation_trigger' -# -# id = Column(String(40)) -# timestamp = Column(DateTime) -# name = Column(String(128)) -# description = Column(Text) -# enabled = Column(Boolean, default=False) -# type = Column(String(32)) -# condition = Column(String(255)) -# operation = Column(String(32)) -# segment = Column(String(128)) -# code = Column(Text) -# workflow_id = Column(String(40)) -# workflow_name = Column(String(128)) -# -# tenant = Column(String(40)) -# production = Column(Boolean) -# -# __table_args__ = ( -# PrimaryKeyConstraint('id', 'tenant', 'production'), -# ) -# -# running: bool = False - - class TaskTable(Base): __tablename__ = 'task' @@ -681,6 +620,7 @@ class TaskTable(Base): __table_args__ = ( PrimaryKeyConstraint('id', 'tenant', 'production'), + Index('idx_gui', 'type', 'tenant', 'production') ) running: bool = False diff --git a/tracardi/service/tracking/source_validation.py b/tracardi/service/tracking/source_validation.py index fcb2f446..a6c9e324 100644 --- a/tracardi/service/tracking/source_validation.py +++ b/tracardi/service/tracking/source_validation.py @@ -66,9 +66,7 @@ async def _validate_source(tracker_payload: TrackerPayload, allowed_bridges) -> if source is None: context = get_context() raise BlockedException(f"Invalid event source `{source_id}` for tenant `{context.tenant}`. " - f"Tenant or event source may not exit. " - f"Request came from IP: `{ip}` " - f"width payload: {tracker_payload}") + f"Tenant or event source may not exit. ") if source.has_restricted_domain(): origin = tracker_payload.get_origin_or_referer()