diff --git a/sdk/python/feast/inference.py b/sdk/python/feast/inference.py index b5efdd338ad..e625500e549 100644 --- a/sdk/python/feast/inference.py +++ b/sdk/python/feast/inference.py @@ -283,18 +283,17 @@ def _infer_on_demand_features_and_entities( batch_source = getattr(source_feature_view, "batch_source") batch_field_mapping = getattr(batch_source or None, "field_mapping") - if batch_field_mapping: - for ( - original_col, - mapped_col, - ) in batch_field_mapping.items(): - if mapped_col in columns_to_exclude: - columns_to_exclude.remove(mapped_col) - columns_to_exclude.add(original_col) + for ( + original_col, + mapped_col, + ) in batch_field_mapping.items(): + if mapped_col in columns_to_exclude: + columns_to_exclude.remove(mapped_col) + columns_to_exclude.add(original_col) - table_column_names_and_types = ( - batch_source.get_table_column_names_and_types(config) - ) + table_column_names_and_types = batch_source.get_table_column_names_and_types( + config + ) for col_name, col_datatype in table_column_names_and_types: if col_name in columns_to_exclude: continue diff --git a/sdk/python/feast/infra/online_stores/sqlite.py b/sdk/python/feast/infra/online_stores/sqlite.py index 7d29266ba9e..21769b55a6a 100644 --- a/sdk/python/feast/infra/online_stores/sqlite.py +++ b/sdk/python/feast/infra/online_stores/sqlite.py @@ -197,16 +197,16 @@ def online_write_batch( ), ) else: - # except Exception as e: - print( - f"error writing online batch for {table_name} - {feature_name} = {val}\n {e}" - ) + # print( + # f"error writing online batch for {table_name} - {feature_name} = {val}\n {e}" + # ) print( f'querying all records for table: {conn.execute(f"select * from {table_name}").fetchall()}' ) r = conn.execute(""" SELECT * FROM sqlite_master WHERE type='table' and name = 'test_on_demand_python_transformation_python_stored_writes_feature_view'; """) + print(f"table exists: {r.fetchall()}") if progress: progress(1) diff --git a/sdk/python/feast/protos/feast/core/FeatureViewProjection_pb2.py b/sdk/python/feast/protos/feast/core/FeatureViewProjection_pb2.py index 286f5116584..b47d4fe392f 100644 --- a/sdk/python/feast/protos/feast/core/FeatureViewProjection_pb2.py +++ b/sdk/python/feast/protos/feast/core/FeatureViewProjection_pb2.py @@ -13,9 +13,10 @@ from feast.protos.feast.core import Feature_pb2 as feast_dot_core_dot_Feature__pb2 +from feast.protos.feast.core import DataSource_pb2 as feast_dot_core_dot_DataSource__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n&feast/core/FeatureViewProjection.proto\x12\nfeast.core\x1a\x18\x66\x65\x61st/core/Feature.proto\"\x83\x02\n\x15\x46\x65\x61tureViewProjection\x12\x19\n\x11\x66\x65\x61ture_view_name\x18\x01 \x01(\t\x12\x1f\n\x17\x66\x65\x61ture_view_name_alias\x18\x03 \x01(\t\x12\x32\n\x0f\x66\x65\x61ture_columns\x18\x02 \x03(\x0b\x32\x19.feast.core.FeatureSpecV2\x12G\n\x0cjoin_key_map\x18\x04 \x03(\x0b\x32\x31.feast.core.FeatureViewProjection.JoinKeyMapEntry\x1a\x31\n\x0fJoinKeyMapEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x42Z\n\x10\x66\x65\x61st.proto.coreB\x15\x46\x65\x61tureReferenceProtoZ/github.com/feast-dev/feast/go/protos/feast/coreb\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n&feast/core/FeatureViewProjection.proto\x12\nfeast.core\x1a\x18\x66\x65\x61st/core/Feature.proto\x1a\x1b\x66\x65\x61st/core/DataSource.proto\"\xba\x03\n\x15\x46\x65\x61tureViewProjection\x12\x19\n\x11\x66\x65\x61ture_view_name\x18\x01 \x01(\t\x12\x1f\n\x17\x66\x65\x61ture_view_name_alias\x18\x03 \x01(\t\x12\x32\n\x0f\x66\x65\x61ture_columns\x18\x02 \x03(\x0b\x32\x19.feast.core.FeatureSpecV2\x12G\n\x0cjoin_key_map\x18\x04 \x03(\x0b\x32\x31.feast.core.FeatureViewProjection.JoinKeyMapEntry\x12\x17\n\x0ftimestamp_field\x18\x05 \x01(\t\x12\x1d\n\x15\x64\x61te_partition_column\x18\x06 \x01(\t\x12 \n\x18\x63reated_timestamp_column\x18\x07 \x01(\t\x12,\n\x0c\x62\x61tch_source\x18\x08 \x01(\x0b\x32\x16.feast.core.DataSource\x12-\n\rstream_source\x18\t \x01(\x0b\x32\x16.feast.core.DataSource\x1a\x31\n\x0fJoinKeyMapEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x42Z\n\x10\x66\x65\x61st.proto.coreB\x15\x46\x65\x61tureReferenceProtoZ/github.com/feast-dev/feast/go/protos/feast/coreb\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -25,8 +26,8 @@ _globals['DESCRIPTOR']._serialized_options = b'\n\020feast.proto.coreB\025FeatureReferenceProtoZ/github.com/feast-dev/feast/go/protos/feast/core' _globals['_FEATUREVIEWPROJECTION_JOINKEYMAPENTRY']._options = None _globals['_FEATUREVIEWPROJECTION_JOINKEYMAPENTRY']._serialized_options = b'8\001' - _globals['_FEATUREVIEWPROJECTION']._serialized_start=81 - _globals['_FEATUREVIEWPROJECTION']._serialized_end=340 - _globals['_FEATUREVIEWPROJECTION_JOINKEYMAPENTRY']._serialized_start=291 - _globals['_FEATUREVIEWPROJECTION_JOINKEYMAPENTRY']._serialized_end=340 + _globals['_FEATUREVIEWPROJECTION']._serialized_start=110 + _globals['_FEATUREVIEWPROJECTION']._serialized_end=552 + _globals['_FEATUREVIEWPROJECTION_JOINKEYMAPENTRY']._serialized_start=503 + _globals['_FEATUREVIEWPROJECTION_JOINKEYMAPENTRY']._serialized_end=552 # @@protoc_insertion_point(module_scope) diff --git a/sdk/python/feast/protos/feast/core/FeatureViewProjection_pb2.pyi b/sdk/python/feast/protos/feast/core/FeatureViewProjection_pb2.pyi index 2c0a298e140..6b44ad4a931 100644 --- a/sdk/python/feast/protos/feast/core/FeatureViewProjection_pb2.pyi +++ b/sdk/python/feast/protos/feast/core/FeatureViewProjection_pb2.pyi @@ -4,6 +4,7 @@ isort:skip_file """ import builtins import collections.abc +import feast.core.DataSource_pb2 import feast.core.Feature_pb2 import google.protobuf.descriptor import google.protobuf.internal.containers @@ -43,6 +44,11 @@ class FeatureViewProjection(google.protobuf.message.Message): FEATURE_VIEW_NAME_ALIAS_FIELD_NUMBER: builtins.int FEATURE_COLUMNS_FIELD_NUMBER: builtins.int JOIN_KEY_MAP_FIELD_NUMBER: builtins.int + TIMESTAMP_FIELD_FIELD_NUMBER: builtins.int + DATE_PARTITION_COLUMN_FIELD_NUMBER: builtins.int + CREATED_TIMESTAMP_COLUMN_FIELD_NUMBER: builtins.int + BATCH_SOURCE_FIELD_NUMBER: builtins.int + STREAM_SOURCE_FIELD_NUMBER: builtins.int feature_view_name: builtins.str """The feature view name""" feature_view_name_alias: builtins.str @@ -53,6 +59,15 @@ class FeatureViewProjection(google.protobuf.message.Message): @property def join_key_map(self) -> google.protobuf.internal.containers.ScalarMap[builtins.str, builtins.str]: """Map for entity join_key overrides of feature data entity join_key to entity data join_key""" + timestamp_field: builtins.str + date_partition_column: builtins.str + created_timestamp_column: builtins.str + @property + def batch_source(self) -> feast.core.DataSource_pb2.DataSource: + """Batch/Offline DataSource where this view can retrieve offline feature data.""" + @property + def stream_source(self) -> feast.core.DataSource_pb2.DataSource: + """Streaming DataSource from where this view can consume "online" feature data.""" def __init__( self, *, @@ -60,7 +75,13 @@ class FeatureViewProjection(google.protobuf.message.Message): feature_view_name_alias: builtins.str = ..., feature_columns: collections.abc.Iterable[feast.core.Feature_pb2.FeatureSpecV2] | None = ..., join_key_map: collections.abc.Mapping[builtins.str, builtins.str] | None = ..., + timestamp_field: builtins.str = ..., + date_partition_column: builtins.str = ..., + created_timestamp_column: builtins.str = ..., + batch_source: feast.core.DataSource_pb2.DataSource | None = ..., + stream_source: feast.core.DataSource_pb2.DataSource | None = ..., ) -> None: ... - def ClearField(self, field_name: typing_extensions.Literal["feature_columns", b"feature_columns", "feature_view_name", b"feature_view_name", "feature_view_name_alias", b"feature_view_name_alias", "join_key_map", b"join_key_map"]) -> None: ... + def HasField(self, field_name: typing_extensions.Literal["batch_source", b"batch_source", "stream_source", b"stream_source"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["batch_source", b"batch_source", "created_timestamp_column", b"created_timestamp_column", "date_partition_column", b"date_partition_column", "feature_columns", b"feature_columns", "feature_view_name", b"feature_view_name", "feature_view_name_alias", b"feature_view_name_alias", "join_key_map", b"join_key_map", "stream_source", b"stream_source", "timestamp_field", b"timestamp_field"]) -> None: ... global___FeatureViewProjection = FeatureViewProjection diff --git a/sdk/python/feast/protos/feast/core/OnDemandFeatureView_pb2.py b/sdk/python/feast/protos/feast/core/OnDemandFeatureView_pb2.py index 4be551724c7..a27c4fba3b7 100644 --- a/sdk/python/feast/protos/feast/core/OnDemandFeatureView_pb2.py +++ b/sdk/python/feast/protos/feast/core/OnDemandFeatureView_pb2.py @@ -20,7 +20,7 @@ from feast.protos.feast.core import Transformation_pb2 as feast_dot_core_dot_Transformation__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n$feast/core/OnDemandFeatureView.proto\x12\nfeast.core\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1c\x66\x65\x61st/core/FeatureView.proto\x1a&feast/core/FeatureViewProjection.proto\x1a\x18\x66\x65\x61st/core/Feature.proto\x1a\x1b\x66\x65\x61st/core/DataSource.proto\x1a\x1f\x66\x65\x61st/core/Transformation.proto\"{\n\x13OnDemandFeatureView\x12\x31\n\x04spec\x18\x01 \x01(\x0b\x32#.feast.core.OnDemandFeatureViewSpec\x12\x31\n\x04meta\x18\x02 \x01(\x0b\x32#.feast.core.OnDemandFeatureViewMeta\"\x99\x04\n\x17OnDemandFeatureViewSpec\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12+\n\x08\x66\x65\x61tures\x18\x03 \x03(\x0b\x32\x19.feast.core.FeatureSpecV2\x12\x41\n\x07sources\x18\x04 \x03(\x0b\x32\x30.feast.core.OnDemandFeatureViewSpec.SourcesEntry\x12\x42\n\x15user_defined_function\x18\x05 \x01(\x0b\x32\x1f.feast.core.UserDefinedFunctionB\x02\x18\x01\x12\x43\n\x16\x66\x65\x61ture_transformation\x18\n \x01(\x0b\x32#.feast.core.FeatureTransformationV2\x12\x13\n\x0b\x64\x65scription\x18\x06 \x01(\t\x12;\n\x04tags\x18\x07 \x03(\x0b\x32-.feast.core.OnDemandFeatureViewSpec.TagsEntry\x12\r\n\x05owner\x18\x08 \x01(\t\x12\x0c\n\x04mode\x18\x0b \x01(\t\x1aJ\n\x0cSourcesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12)\n\x05value\x18\x02 \x01(\x0b\x32\x1a.feast.core.OnDemandSource:\x02\x38\x01\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x8c\x01\n\x17OnDemandFeatureViewMeta\x12\x35\n\x11\x63reated_timestamp\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12:\n\x16last_updated_timestamp\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\"\xc8\x01\n\x0eOnDemandSource\x12/\n\x0c\x66\x65\x61ture_view\x18\x01 \x01(\x0b\x32\x17.feast.core.FeatureViewH\x00\x12\x44\n\x17\x66\x65\x61ture_view_projection\x18\x03 \x01(\x0b\x32!.feast.core.FeatureViewProjectionH\x00\x12\x35\n\x13request_data_source\x18\x02 \x01(\x0b\x32\x16.feast.core.DataSourceH\x00\x42\x08\n\x06source\"H\n\x13UserDefinedFunction\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0c\n\x04\x62ody\x18\x02 \x01(\x0c\x12\x11\n\tbody_text\x18\x03 \x01(\t:\x02\x18\x01\x42]\n\x10\x66\x65\x61st.proto.coreB\x18OnDemandFeatureViewProtoZ/github.com/feast-dev/feast/go/protos/feast/coreb\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n$feast/core/OnDemandFeatureView.proto\x12\nfeast.core\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1c\x66\x65\x61st/core/FeatureView.proto\x1a&feast/core/FeatureViewProjection.proto\x1a\x18\x66\x65\x61st/core/Feature.proto\x1a\x1b\x66\x65\x61st/core/DataSource.proto\x1a\x1f\x66\x65\x61st/core/Transformation.proto\"{\n\x13OnDemandFeatureView\x12\x31\n\x04spec\x18\x01 \x01(\x0b\x32#.feast.core.OnDemandFeatureViewSpec\x12\x31\n\x04meta\x18\x02 \x01(\x0b\x32#.feast.core.OnDemandFeatureViewMeta\"\xfd\x04\n\x17OnDemandFeatureViewSpec\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12+\n\x08\x66\x65\x61tures\x18\x03 \x03(\x0b\x32\x19.feast.core.FeatureSpecV2\x12\x41\n\x07sources\x18\x04 \x03(\x0b\x32\x30.feast.core.OnDemandFeatureViewSpec.SourcesEntry\x12\x42\n\x15user_defined_function\x18\x05 \x01(\x0b\x32\x1f.feast.core.UserDefinedFunctionB\x02\x18\x01\x12\x43\n\x16\x66\x65\x61ture_transformation\x18\n \x01(\x0b\x32#.feast.core.FeatureTransformationV2\x12\x13\n\x0b\x64\x65scription\x18\x06 \x01(\t\x12;\n\x04tags\x18\x07 \x03(\x0b\x32-.feast.core.OnDemandFeatureViewSpec.TagsEntry\x12\r\n\x05owner\x18\x08 \x01(\t\x12\x0c\n\x04mode\x18\x0b \x01(\t\x12\x1d\n\x15write_to_online_store\x18\x0c \x01(\x08\x12\x10\n\x08\x65ntities\x18\r \x03(\t\x12\x31\n\x0e\x65ntity_columns\x18\x0e \x03(\x0b\x32\x19.feast.core.FeatureSpecV2\x1aJ\n\x0cSourcesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12)\n\x05value\x18\x02 \x01(\x0b\x32\x1a.feast.core.OnDemandSource:\x02\x38\x01\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x8c\x01\n\x17OnDemandFeatureViewMeta\x12\x35\n\x11\x63reated_timestamp\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12:\n\x16last_updated_timestamp\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\"\xc8\x01\n\x0eOnDemandSource\x12/\n\x0c\x66\x65\x61ture_view\x18\x01 \x01(\x0b\x32\x17.feast.core.FeatureViewH\x00\x12\x44\n\x17\x66\x65\x61ture_view_projection\x18\x03 \x01(\x0b\x32!.feast.core.FeatureViewProjectionH\x00\x12\x35\n\x13request_data_source\x18\x02 \x01(\x0b\x32\x16.feast.core.DataSourceH\x00\x42\x08\n\x06source\"H\n\x13UserDefinedFunction\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0c\n\x04\x62ody\x18\x02 \x01(\x0c\x12\x11\n\tbody_text\x18\x03 \x01(\t:\x02\x18\x01\x42]\n\x10\x66\x65\x61st.proto.coreB\x18OnDemandFeatureViewProtoZ/github.com/feast-dev/feast/go/protos/feast/coreb\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -39,15 +39,15 @@ _globals['_ONDEMANDFEATUREVIEW']._serialized_start=243 _globals['_ONDEMANDFEATUREVIEW']._serialized_end=366 _globals['_ONDEMANDFEATUREVIEWSPEC']._serialized_start=369 - _globals['_ONDEMANDFEATUREVIEWSPEC']._serialized_end=906 - _globals['_ONDEMANDFEATUREVIEWSPEC_SOURCESENTRY']._serialized_start=787 - _globals['_ONDEMANDFEATUREVIEWSPEC_SOURCESENTRY']._serialized_end=861 - _globals['_ONDEMANDFEATUREVIEWSPEC_TAGSENTRY']._serialized_start=863 - _globals['_ONDEMANDFEATUREVIEWSPEC_TAGSENTRY']._serialized_end=906 - _globals['_ONDEMANDFEATUREVIEWMETA']._serialized_start=909 - _globals['_ONDEMANDFEATUREVIEWMETA']._serialized_end=1049 - _globals['_ONDEMANDSOURCE']._serialized_start=1052 - _globals['_ONDEMANDSOURCE']._serialized_end=1252 - _globals['_USERDEFINEDFUNCTION']._serialized_start=1254 - _globals['_USERDEFINEDFUNCTION']._serialized_end=1326 + _globals['_ONDEMANDFEATUREVIEWSPEC']._serialized_end=1006 + _globals['_ONDEMANDFEATUREVIEWSPEC_SOURCESENTRY']._serialized_start=887 + _globals['_ONDEMANDFEATUREVIEWSPEC_SOURCESENTRY']._serialized_end=961 + _globals['_ONDEMANDFEATUREVIEWSPEC_TAGSENTRY']._serialized_start=963 + _globals['_ONDEMANDFEATUREVIEWSPEC_TAGSENTRY']._serialized_end=1006 + _globals['_ONDEMANDFEATUREVIEWMETA']._serialized_start=1009 + _globals['_ONDEMANDFEATUREVIEWMETA']._serialized_end=1149 + _globals['_ONDEMANDSOURCE']._serialized_start=1152 + _globals['_ONDEMANDSOURCE']._serialized_end=1352 + _globals['_USERDEFINEDFUNCTION']._serialized_start=1354 + _globals['_USERDEFINEDFUNCTION']._serialized_end=1426 # @@protoc_insertion_point(module_scope) diff --git a/sdk/python/feast/protos/feast/core/OnDemandFeatureView_pb2.pyi b/sdk/python/feast/protos/feast/core/OnDemandFeatureView_pb2.pyi index d72a8f98629..b2ec15b1867 100644 --- a/sdk/python/feast/protos/feast/core/OnDemandFeatureView_pb2.pyi +++ b/sdk/python/feast/protos/feast/core/OnDemandFeatureView_pb2.pyi @@ -104,6 +104,9 @@ class OnDemandFeatureViewSpec(google.protobuf.message.Message): TAGS_FIELD_NUMBER: builtins.int OWNER_FIELD_NUMBER: builtins.int MODE_FIELD_NUMBER: builtins.int + WRITE_TO_ONLINE_STORE_FIELD_NUMBER: builtins.int + ENTITIES_FIELD_NUMBER: builtins.int + ENTITY_COLUMNS_FIELD_NUMBER: builtins.int name: builtins.str """Name of the feature view. Must be unique. Not updated.""" project: builtins.str @@ -127,6 +130,13 @@ class OnDemandFeatureViewSpec(google.protobuf.message.Message): owner: builtins.str """Owner of the on demand feature view.""" mode: builtins.str + write_to_online_store: builtins.bool + @property + def entities(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]: + """List of names of entities associated with this feature view.""" + @property + def entity_columns(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[feast.core.Feature_pb2.FeatureSpecV2]: + """List of specifications for each entity defined as part of this feature view.""" def __init__( self, *, @@ -140,9 +150,12 @@ class OnDemandFeatureViewSpec(google.protobuf.message.Message): tags: collections.abc.Mapping[builtins.str, builtins.str] | None = ..., owner: builtins.str = ..., mode: builtins.str = ..., + write_to_online_store: builtins.bool = ..., + entities: collections.abc.Iterable[builtins.str] | None = ..., + entity_columns: collections.abc.Iterable[feast.core.Feature_pb2.FeatureSpecV2] | None = ..., ) -> None: ... def HasField(self, field_name: typing_extensions.Literal["feature_transformation", b"feature_transformation", "user_defined_function", b"user_defined_function"]) -> builtins.bool: ... - def ClearField(self, field_name: typing_extensions.Literal["description", b"description", "feature_transformation", b"feature_transformation", "features", b"features", "mode", b"mode", "name", b"name", "owner", b"owner", "project", b"project", "sources", b"sources", "tags", b"tags", "user_defined_function", b"user_defined_function"]) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["description", b"description", "entities", b"entities", "entity_columns", b"entity_columns", "feature_transformation", b"feature_transformation", "features", b"features", "mode", b"mode", "name", b"name", "owner", b"owner", "project", b"project", "sources", b"sources", "tags", b"tags", "user_defined_function", b"user_defined_function", "write_to_online_store", b"write_to_online_store"]) -> None: ... global___OnDemandFeatureViewSpec = OnDemandFeatureViewSpec