From 9820b8508470f46864d321ada1d9b0c5f2668310 Mon Sep 17 00:00:00 2001 From: James Estevez Date: Fri, 14 Apr 2023 15:26:30 -0700 Subject: [PATCH 01/36] Lint history file before committing it --- .github/workflows/pre-release.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/pre-release.yml b/.github/workflows/pre-release.yml index 43f91a1e9c..13bbdcf51f 100644 --- a/.github/workflows/pre-release.yml +++ b/.github/workflows/pre-release.yml @@ -52,6 +52,10 @@ jobs: sed -e 's_\(https.*\/\)\([0-9]*\)$_[#\2](\1\2)_' \ -e 's_by @\(.*\) in_by [@\1](https://github.com/\1) in_' >> changelog.md python utility/update-history.py + - name: Lint history + run: | + npm install prettier + npx prettier --write docs/history.md - name: Commit changes run: | git config user.name github-actions From e379f87886703bc79e4f89fa90174bb91025840e Mon Sep 17 00:00:00 2001 From: aditya-balachander Date: Tue, 9 Jan 2024 13:03:07 +0530 Subject: [PATCH 02/36] Successfully able to retrieve polymorphic lookups --- cumulusci/tasks/bulkdata/extract.py | 83 ++++++++++++++-------- cumulusci/tasks/bulkdata/mapping_parser.py | 78 +++++++++++++++++++- cumulusci/tasks/bulkdata/utils.py | 2 +- 3 files changed, 133 insertions(+), 30 deletions(-) diff --git a/cumulusci/tasks/bulkdata/extract.py b/cumulusci/tasks/bulkdata/extract.py index 6f6aa4b333..3ef714735b 100644 --- a/cumulusci/tasks/bulkdata/extract.py +++ b/cumulusci/tasks/bulkdata/extract.py @@ -70,6 +70,7 @@ def _init_options(self, kwargs): self.options["drop_missing_schema"] = process_bool_arg( self.options.get("drop_missing_schema") or False ) + self._id_generators = {} def _run_task(self): self._init_mapping() @@ -127,7 +128,7 @@ def _init_mapping(self): def _soql_for_mapping(self, mapping): """Return a SOQL query suitable for extracting data for this mapping.""" sf_object = mapping.sf_object - fields = mapping.get_complete_field_map(include_id=True).keys() + fields = mapping.get_extract_field_list() soql = f"SELECT {', '.join(fields)} FROM {sf_object}" if mapping.record_type: @@ -146,7 +147,7 @@ def _run_query(self, soql, mapping): step = get_query_operation( sobject=mapping.sf_object, api=mapping.api, - fields=list(mapping.get_complete_field_map(include_id=True).keys()), + fields=list(mapping.get_extract_field_list()), api_options={}, context=self, query=soql, @@ -225,19 +226,23 @@ def strip_name_field(record): # If using the autogenerated id field, split out the returned records # into two separate streams and load into the main table and the sf_id_table values, ids = itertools.tee(record_iterator) - f_values = (row[1:] for row in values) - f_ids = (row[:1] for row in ids) + + id_source_sobj, id_source_global = itertools.tee( + self._id_generator_for_object(mapping.sf_object) + ) + f_ids = ((row[0], next(id_source_global)) for row in ids) + f_values = ([next(id_source_sobj)] + row[1:] for row in values) values_chunks = sql_bulk_insert_from_records_incremental( connection=conn, table=self.metadata.tables[mapping.table], - columns=columns[1:], # Strip off the Id column + columns=["id"] + columns[1:], record_iterable=f_values, ) ids_chunks = sql_bulk_insert_from_records_incremental( connection=conn, table=self.metadata.tables[mapping.get_sf_id_table()], - columns=["sf_id"], + columns=["sf_id", "id"], record_iterable=f_ids, ) @@ -255,6 +260,19 @@ def strip_name_field(record): self.session.commit() + def _id_generator_for_object(self, sobject: str): + if sobject not in self._id_generators: + + def _generate_ids(): + counter = 0 + while True: + yield f"{sobject}-{counter}" + counter += 1 + + self._id_generators[sobject] = _generate_ids() + + return self._id_generators[sobject] + def _map_autopks(self): # Convert Salesforce Ids to autopks for m in self.mapping.values(): @@ -270,9 +288,17 @@ def _map_autopks(self): def _get_mapping_for_table(self, table): """Return the first mapping for a table name""" - for mapping in self.mapping.values(): - if mapping["table"] == table: - return mapping + mappings = [ + mapping for mapping in self.mapping.values() + if (isinstance(table, str) and mapping["table"] == table) or + (isinstance(table, list) and mapping["table"] in table) + ] + + if isinstance(table, list) and len(mappings) != len(table): + missing_tables = set(table) - set(mapping["table"] for mapping in mappings) + raise ValueError(f"The following tables are missing in the mapping file: {missing_tables}") + + return mappings def _convert_lookups_to_id(self, mapping, lookup_keys): """Rewrite persisted Salesforce Ids to refer to auto-PKs.""" @@ -286,29 +312,30 @@ def throw(string): # pragma: no cover ) model = self.models.get(mapping.table) - lookup_mapping = self._get_mapping_for_table(lookup_info.table) or throw( + lookup_mappings = self._get_mapping_for_table(lookup_info.table) or throw( f"Cannot find lookup mapping for {lookup_info.table}" ) - lookup_model = self.models.get(lookup_mapping.get_sf_id_table()) + for lookup_mapping in lookup_mappings: + lookup_model = self.models.get(lookup_mapping.get_sf_id_table()) - key_field = lookup_info.get_lookup_key_field() + key_field = lookup_info.get_lookup_key_field() - key_attr = getattr(model, key_field, None) or throw( - f"key_field {key_field} not found in table {mapping.table}" - ) - try: - self.session.query(model).filter( - key_attr.isnot(None), key_attr == lookup_model.sf_id - ).update({key_attr: lookup_model.id}, synchronize_session=False) - except NotImplementedError: - # Some databases such as sqlite don't support multitable update - mappings = [] - for row, lookup_id in self.session.query(model, lookup_model.id).join( - lookup_model, key_attr == lookup_model.sf_id - ): - mappings.append({"id": row.id, key_field: lookup_id}) - self.session.bulk_update_mappings(model, mappings) + key_attr = getattr(model, key_field, None) or throw( + f"key_field {key_field} not found in table {mapping.table}" + ) + try: + self.session.query(model).filter( + key_attr.isnot(None), key_attr == lookup_model.sf_id + ).update({key_attr: lookup_model.id}, synchronize_session=False) + except NotImplementedError: + # Some databases such as sqlite don't support multitable update + mappings = [] + for row, lookup_id in self.session.query(model, lookup_model.id).join( + lookup_model, key_attr == lookup_model.sf_id + ): + mappings.append({"id": row.id, key_field: lookup_id}) + self.session.bulk_update_mappings(model, mappings) self.session.commit() def _create_tables(self): @@ -339,7 +366,7 @@ def _create_table(self, mapping): sf_id_model_name, (object,), {} ) sf_id_fields = [ - Column("id", Integer(), primary_key=True, autoincrement=True), + Column("id", Unicode(255), primary_key=True), Column("sf_id", Unicode(24)), ] id_t = Table(mapping.get_sf_id_table(), self.metadata, *sf_id_fields) diff --git a/cumulusci/tasks/bulkdata/mapping_parser.py b/cumulusci/tasks/bulkdata/mapping_parser.py index 2b4db8ef92..3473b8b1d4 100644 --- a/cumulusci/tasks/bulkdata/mapping_parser.py +++ b/cumulusci/tasks/bulkdata/mapping_parser.py @@ -36,7 +36,7 @@ def __setitem__(self, key, value): class MappingLookup(CCIDictModel): "Lookup relationship between two tables." - table: str + table: Union[str, List[str]] key_field: Optional[str] = None value_field: Optional[str] = None join_field: Optional[str] = None @@ -194,6 +194,26 @@ def get_load_field_list(self): return columns + def get_extract_field_list(self): + """Build a flat list of Salesforce fields for the given mapping, including fields, lookups, and record types, + for an extraction operation. + The Id field is guaranteed to come first in the list.""" + + # Build the list of fields to import + fields = ["Id"] + fields.extend([f for f in self.fields.keys() if f != "Id"]) + fields.extend(self.lookups.keys()) + + # If we're using Record Type mapping, `RecordTypeId` goes at the end. + # This makes it easier to manage the relationship with database columns. + if "RecordTypeId" in fields: + fields.remove("RecordTypeId") + + if "RecordTypeId" in self.fields: + fields.append("RecordTypeId") + + return fields + def get_relative_date_context(self, fields: List[str], sf: Salesforce): date_fields = [ fields.index(f) @@ -584,6 +604,59 @@ def parse_from_yaml(source: Union[str, Path, IO]) -> Dict: "Parse from a path, url, path-like or file-like" return MappingSteps.parse_from_yaml(source) +def _infer_and_validate_lookups(mapping: Dict, sf: Salesforce): + sf_objects = [m.sf_object for m in mapping.values()] + + fail = False + + for idx, m in enumerate(mapping.values()): + describe = CaseInsensitiveDict( + { + f["name"]: f + for f in getattr(sf, m.sf_object).describe()[ + "fields" + ] + } + ) + + for lookup in m.lookups.values(): + if lookup.after: + # If configured by the user, skip. + # TODO: do we need more validation here? + continue + + field_describe = describe[lookup.name] + target_objects = field_describe["referenceTo"] + if len(target_objects) == 1: + # This is a non-polymorphic lookup. + try: + target_index = sf_objects.index(target_objects[0]) + except ValueError: + fail = True + logger.error( + f"The field {m.sf_object}.{lookup.name} looks up to {target_objects[0]}, which is not included in the operation" + ) + continue + + if target_index > idx or target_index == idx: + # This is a non-polymorphic after step. + lookup.after = mapping.keys()[idx] + else: + # This is a polymorphic lookup. + # Make sure that any lookup targets present in the operation precede this step. + target_indices = [sf_objects.index(t) for t in target_objects] + if not all([target_index < idx for target_index in target_indices]): + logger.error( + f"All included target objects ({','.join(target_objects)}) for the field {m.sf_object}.{lookup.name} " + f"must precede {m.sf_object} in the mapping." + ) + fail = True + continue + + if fail: + raise BulkDataException( + "One or more relationship errors blocked the operation." + ) def validate_and_inject_mapping( *, @@ -634,6 +707,9 @@ def validate_and_inject_mapping( "due to missing permissions." ) + # Synthesize `after` declarations and validate lookup references. + # _infer_and_validate_lookups(mapping, sf) + # If the org has person accounts enable, add a field mapping to track "IsPersonAccount". # IsPersonAccount field values are used to properly load person account records. if org_has_person_accounts_enabled and data_operation == DataOperationType.QUERY: diff --git a/cumulusci/tasks/bulkdata/utils.py b/cumulusci/tasks/bulkdata/utils.py index ea09ba49df..aa2fb51189 100644 --- a/cumulusci/tasks/bulkdata/utils.py +++ b/cumulusci/tasks/bulkdata/utils.py @@ -82,7 +82,7 @@ def _handle_primary_key(mapping, fields): id_column = mapping.fields["Id"] fields.append(Column(id_column, Unicode(255), primary_key=True)) else: - fields.append(Column("id", Integer(), primary_key=True, autoincrement=True)) + fields.append(Column("id", Unicode(255), primary_key=True)) def create_table(mapping, metadata) -> Table: From 2c5d0056e4e65457a5d6b5f2c2bab18de7047090 Mon Sep 17 00:00:00 2001 From: aditya-balachander Date: Tue, 9 Jan 2024 15:54:21 +0530 Subject: [PATCH 03/36] Updated id numbering --- cumulusci/tasks/bulkdata/extract.py | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/cumulusci/tasks/bulkdata/extract.py b/cumulusci/tasks/bulkdata/extract.py index 3ef714735b..881652368e 100644 --- a/cumulusci/tasks/bulkdata/extract.py +++ b/cumulusci/tasks/bulkdata/extract.py @@ -227,11 +227,9 @@ def strip_name_field(record): # into two separate streams and load into the main table and the sf_id_table values, ids = itertools.tee(record_iterator) - id_source_sobj, id_source_global = itertools.tee( - self._id_generator_for_object(mapping.sf_object) - ) + id_source_global = self._id_generator_for_object(mapping.sf_object) f_ids = ((row[0], next(id_source_global)) for row in ids) - f_values = ([next(id_source_sobj)] + row[1:] for row in values) + f_values = ([record_number] + row[1:] for record_number, row in enumerate(values, start=1)) values_chunks = sql_bulk_insert_from_records_incremental( connection=conn, @@ -264,7 +262,7 @@ def _id_generator_for_object(self, sobject: str): if sobject not in self._id_generators: def _generate_ids(): - counter = 0 + counter = 1 while True: yield f"{sobject}-{counter}" counter += 1 @@ -316,14 +314,17 @@ def throw(string): # pragma: no cover f"Cannot find lookup mapping for {lookup_info.table}" ) - for lookup_mapping in lookup_mappings: - lookup_model = self.models.get(lookup_mapping.get_sf_id_table()) + key_field = lookup_info.get_lookup_key_field() - key_field = lookup_info.get_lookup_key_field() + key_attr = getattr(model, key_field, None) or throw( + f"key_field {key_field} not found in table {mapping.table}" + ) - key_attr = getattr(model, key_field, None) or throw( - f"key_field {key_field} not found in table {mapping.table}" - ) + # Keep track of total mapping operations + total_mapping_operations = 0 + + for lookup_mapping in lookup_mappings: + lookup_model = self.models.get(lookup_mapping.get_sf_id_table()) try: self.session.query(model).filter( key_attr.isnot(None), key_attr == lookup_model.sf_id @@ -335,7 +336,13 @@ def throw(string): # pragma: no cover lookup_model, key_attr == lookup_model.sf_id ): mappings.append({"id": row.id, key_field: lookup_id}) + total_mapping_operations += len(mappings) self.session.bulk_update_mappings(model, mappings) + # Count the total number of rows excluding those with no entry for that field + total_rows = self.session.query(model).filter(key_attr.isnot('')).count() + + if total_mapping_operations != total_rows: + raise ValueError(f"Total mapping operations ({total_mapping_operations}) do not match total non-empty rows ({total_rows}) for lookup_key: {lookup_key}") self.session.commit() def _create_tables(self): From eacb2830da0da6d51cfdbf3f9f706eaa383bb8f7 Mon Sep 17 00:00:00 2001 From: aditya-balachander Date: Tue, 9 Jan 2024 16:13:11 +0530 Subject: [PATCH 04/36] Changed id type from Unicode to Integer --- cumulusci/tasks/bulkdata/tests/test_extract.py | 4 ++-- cumulusci/tasks/bulkdata/utils.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cumulusci/tasks/bulkdata/tests/test_extract.py b/cumulusci/tasks/bulkdata/tests/test_extract.py index a7a336a98a..adc1757d79 100644 --- a/cumulusci/tasks/bulkdata/tests/test_extract.py +++ b/cumulusci/tasks/bulkdata/tests/test_extract.py @@ -285,7 +285,7 @@ def test_run__v2__person_accounts_disabled(self, query_op_mock): assert household.record_type == "HH_Account" contact = next(conn.execute("select * from contacts")) - assert contact.household_id == "1" + assert contact.household_id == "Account-1" assert not hasattr(contact, "IsPersonAccount") @responses.activate @@ -336,7 +336,7 @@ def test_run__v2__person_accounts_enabled(self, query_op_mock): assert household.record_type == "HH_Account" contact = next(conn.execute("select * from contacts")) - assert contact.household_id == "1" + assert contact.household_id == "Account-1" assert contact.IsPersonAccount == "true" @mock.patch("cumulusci.tasks.bulkdata.extract.log_progress") diff --git a/cumulusci/tasks/bulkdata/utils.py b/cumulusci/tasks/bulkdata/utils.py index aa2fb51189..086f92bc45 100644 --- a/cumulusci/tasks/bulkdata/utils.py +++ b/cumulusci/tasks/bulkdata/utils.py @@ -82,7 +82,7 @@ def _handle_primary_key(mapping, fields): id_column = mapping.fields["Id"] fields.append(Column(id_column, Unicode(255), primary_key=True)) else: - fields.append(Column("id", Unicode(255), primary_key=True)) + fields.append(Column("id", Integer(), primary_key=True)) def create_table(mapping, metadata) -> Table: From 4a3d898daaac55190a46d4c8619119f6ca006370 Mon Sep 17 00:00:00 2001 From: aditya-balachander Date: Wed, 10 Jan 2024 11:27:21 +0530 Subject: [PATCH 05/36] Fix existing tests --- cumulusci/tasks/bulkdata/extract.py | 12 ++++++++---- cumulusci/tasks/bulkdata/tests/test_extract.py | 7 +++++-- cumulusci/tasks/bulkdata/utils.py | 2 +- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/cumulusci/tasks/bulkdata/extract.py b/cumulusci/tasks/bulkdata/extract.py index 881652368e..22ece08bd4 100644 --- a/cumulusci/tasks/bulkdata/extract.py +++ b/cumulusci/tasks/bulkdata/extract.py @@ -234,7 +234,7 @@ def strip_name_field(record): values_chunks = sql_bulk_insert_from_records_incremental( connection=conn, table=self.metadata.tables[mapping.table], - columns=["id"] + columns[1:], + columns= columns[1:], record_iterable=f_values, ) ids_chunks = sql_bulk_insert_from_records_incremental( @@ -326,9 +326,10 @@ def throw(string): # pragma: no cover for lookup_mapping in lookup_mappings: lookup_model = self.models.get(lookup_mapping.get_sf_id_table()) try: - self.session.query(model).filter( + update_query = self.session.query(model).filter( key_attr.isnot(None), key_attr == lookup_model.sf_id ).update({key_attr: lookup_model.id}, synchronize_session=False) + total_mapping_operations += update_query.rowcount except NotImplementedError: # Some databases such as sqlite don't support multitable update mappings = [] @@ -336,10 +337,13 @@ def throw(string): # pragma: no cover lookup_model, key_attr == lookup_model.sf_id ): mappings.append({"id": row.id, key_field: lookup_id}) - total_mapping_operations += len(mappings) + total_mapping_operations += len(mappings) self.session.bulk_update_mappings(model, mappings) # Count the total number of rows excluding those with no entry for that field - total_rows = self.session.query(model).filter(key_attr.isnot('')).count() + total_rows = self.session.query(model).filter( + key_attr.isnot(None), # Ensure key_attr is not None + key_attr.isnot('') # Ensure key_attr is not an empty string + ).count() if total_mapping_operations != total_rows: raise ValueError(f"Total mapping operations ({total_mapping_operations}) do not match total non-empty rows ({total_rows}) for lookup_key: {lookup_key}") diff --git a/cumulusci/tasks/bulkdata/tests/test_extract.py b/cumulusci/tasks/bulkdata/tests/test_extract.py index adc1757d79..b249b046ed 100644 --- a/cumulusci/tasks/bulkdata/tests/test_extract.py +++ b/cumulusci/tasks/bulkdata/tests/test_extract.py @@ -570,6 +570,8 @@ def test_convert_lookups_to_id(self): "Opportunity": MappingStep(sf_object="Opportunity"), } + task.session.query.return_value.filter.return_value.count.return_value = 0 + task.session.query.return_value.filter.return_value.update.return_value.rowcount = 0 task._convert_lookups_to_id( MappingStep( sf_object="Opportunity", @@ -607,6 +609,7 @@ def test_convert_lookups_to_id__sqlite(self): item = mock.Mock() task.session.query.return_value.join.return_value = [(item, "1")] + task.session.query.return_value.filter.return_value.count.return_value = 1 task._convert_lookups_to_id( MappingStep( @@ -1019,8 +1022,8 @@ def test_import_results__autopk(self, create_task_fixture): assert len(output_accounts) == 2 output_opportunities = list(conn.execute("select * from Opportunity")) - assert output_opportunities[0].AccountId == "2" - assert output_opportunities[0].ContactId == "1" + assert output_opportunities[0].AccountId == "Account-2" + assert output_opportunities[0].ContactId == "Contact-1" def test_run_soql_filter(self): """This test case is to verify when soql_filter is specified with valid filter in the mapping yml""" diff --git a/cumulusci/tasks/bulkdata/utils.py b/cumulusci/tasks/bulkdata/utils.py index 086f92bc45..ea09ba49df 100644 --- a/cumulusci/tasks/bulkdata/utils.py +++ b/cumulusci/tasks/bulkdata/utils.py @@ -82,7 +82,7 @@ def _handle_primary_key(mapping, fields): id_column = mapping.fields["Id"] fields.append(Column(id_column, Unicode(255), primary_key=True)) else: - fields.append(Column("id", Integer(), primary_key=True)) + fields.append(Column("id", Integer(), primary_key=True, autoincrement=True)) def create_table(mapping, metadata) -> Table: From 3ec630815e3cd454bda654373c851836112de036 Mon Sep 17 00:00:00 2001 From: aditya-balachander Date: Wed, 10 Jan 2024 14:51:55 +0530 Subject: [PATCH 06/36] Tests written for extract --- cumulusci/tasks/bulkdata/extract.py | 2 +- cumulusci/tasks/bulkdata/mapping_parser.py | 56 ---------------- .../tasks/bulkdata/tests/mapping_poly.yml | 33 +++++++++ .../tasks/bulkdata/tests/test_extract.py | 67 +++++++++++++++++++ .../bulkdata/tests/test_mapping_parser.py | 15 +++++ .../GET_sobjects_Event_describe.yaml | 18 +++++ .../GET_sobjects_Global_describe.yaml | 17 ++++- cumulusci/tests/util.py | 1 + 8 files changed, 151 insertions(+), 58 deletions(-) create mode 100644 cumulusci/tasks/bulkdata/tests/mapping_poly.yml create mode 100644 cumulusci/tests/shared_cassettes/GET_sobjects_Event_describe.yaml diff --git a/cumulusci/tasks/bulkdata/extract.py b/cumulusci/tasks/bulkdata/extract.py index 22ece08bd4..421e0d867b 100644 --- a/cumulusci/tasks/bulkdata/extract.py +++ b/cumulusci/tasks/bulkdata/extract.py @@ -229,7 +229,7 @@ def strip_name_field(record): id_source_global = self._id_generator_for_object(mapping.sf_object) f_ids = ((row[0], next(id_source_global)) for row in ids) - f_values = ([record_number] + row[1:] for record_number, row in enumerate(values, start=1)) + f_values = (row[1:] for row in values) values_chunks = sql_bulk_insert_from_records_incremental( connection=conn, diff --git a/cumulusci/tasks/bulkdata/mapping_parser.py b/cumulusci/tasks/bulkdata/mapping_parser.py index 3473b8b1d4..919ba7ec52 100644 --- a/cumulusci/tasks/bulkdata/mapping_parser.py +++ b/cumulusci/tasks/bulkdata/mapping_parser.py @@ -604,59 +604,6 @@ def parse_from_yaml(source: Union[str, Path, IO]) -> Dict: "Parse from a path, url, path-like or file-like" return MappingSteps.parse_from_yaml(source) -def _infer_and_validate_lookups(mapping: Dict, sf: Salesforce): - sf_objects = [m.sf_object for m in mapping.values()] - - fail = False - - for idx, m in enumerate(mapping.values()): - describe = CaseInsensitiveDict( - { - f["name"]: f - for f in getattr(sf, m.sf_object).describe()[ - "fields" - ] - } - ) - - for lookup in m.lookups.values(): - if lookup.after: - # If configured by the user, skip. - # TODO: do we need more validation here? - continue - - field_describe = describe[lookup.name] - target_objects = field_describe["referenceTo"] - if len(target_objects) == 1: - # This is a non-polymorphic lookup. - try: - target_index = sf_objects.index(target_objects[0]) - except ValueError: - fail = True - logger.error( - f"The field {m.sf_object}.{lookup.name} looks up to {target_objects[0]}, which is not included in the operation" - ) - continue - - if target_index > idx or target_index == idx: - # This is a non-polymorphic after step. - lookup.after = mapping.keys()[idx] - else: - # This is a polymorphic lookup. - # Make sure that any lookup targets present in the operation precede this step. - target_indices = [sf_objects.index(t) for t in target_objects] - if not all([target_index < idx for target_index in target_indices]): - logger.error( - f"All included target objects ({','.join(target_objects)}) for the field {m.sf_object}.{lookup.name} " - f"must precede {m.sf_object} in the mapping." - ) - fail = True - continue - - if fail: - raise BulkDataException( - "One or more relationship errors blocked the operation." - ) def validate_and_inject_mapping( *, @@ -707,9 +654,6 @@ def validate_and_inject_mapping( "due to missing permissions." ) - # Synthesize `after` declarations and validate lookup references. - # _infer_and_validate_lookups(mapping, sf) - # If the org has person accounts enable, add a field mapping to track "IsPersonAccount". # IsPersonAccount field values are used to properly load person account records. if org_has_person_accounts_enabled and data_operation == DataOperationType.QUERY: diff --git a/cumulusci/tasks/bulkdata/tests/mapping_poly.yml b/cumulusci/tasks/bulkdata/tests/mapping_poly.yml new file mode 100644 index 0000000000..26454cc8d6 --- /dev/null +++ b/cumulusci/tasks/bulkdata/tests/mapping_poly.yml @@ -0,0 +1,33 @@ +Insert Households: + api: Bulk + sf_object: Account + table: households + fields: + Name: name + record_type: HH_Account +Insert Contacts: + api: BULK + sf_object: Contact + table: contacts + filters: + - "household_id is not null" + fields: + FirstName: first_name + LastName: last_name + Email: email + lookups: + AccountId: + key_field: household_id + table: households +Insert Events: + api: BULK + sf_object: Event + table: events + fields: + Subject: subject + lookups: + WhoId: + key_field: who_id + table: + - households + - contacts diff --git a/cumulusci/tasks/bulkdata/tests/test_extract.py b/cumulusci/tasks/bulkdata/tests/test_extract.py index b249b046ed..c62bc273aa 100644 --- a/cumulusci/tasks/bulkdata/tests/test_extract.py +++ b/cumulusci/tasks/bulkdata/tests/test_extract.py @@ -3,6 +3,7 @@ from datetime import date, timedelta from tempfile import TemporaryDirectory from unittest import mock +from sqlalchemy.orm import Query import pytest import responses @@ -79,6 +80,7 @@ class TestExtractData: mapping_file_v1 = "mapping_v1.yml" mapping_file_v2 = "mapping_v2.yml" + mapping_file_poly = "mapping_poly.yml" mapping_file_vanilla = "mapping_vanilla_sf.yml" @responses.activate @@ -338,6 +340,71 @@ def test_run__v2__person_accounts_enabled(self, query_op_mock): contact = next(conn.execute("select * from contacts")) assert contact.household_id == "Account-1" assert contact.IsPersonAccount == "true" + + @responses.activate + @mock.patch("cumulusci.tasks.bulkdata.extract.get_query_operation") + def test_run__poly__polymorphic_lookups(self, query_op_mock): + base_path = os.path.dirname(__file__) + mapping_path = os.path.join(base_path, self.mapping_file_poly) + mock_describe_calls() + + with temporary_dir() as t: + task = _make_task( + ExtractData, + { + "options": { + "database_url": f"sqlite:///{t}/temp_poly.db", # in memory + "mapping": mapping_path, + } + }, + ) + task.bulk = mock.Mock() + task.sf = mock.Mock() + task.org_config._is_person_accounts_enabled = False + + mock_query_households = MockBulkQueryOperation( + sobject="Account", + api_options={}, + context=task, + query="SELECT Id, Name FROM Account", + ) + mock_query_contacts = MockBulkQueryOperation( + sobject="Contact", + api_options={}, + context=task, + query="SELECT Id, FirstName, LastName, Email, AccountId FROM Contact", + ) + mock_query_events = MockBulkQueryOperation( + sobject="Event", + api_options={}, + context=task, + query="SELECT Id, LastName, WhoId FROM Event", + ) + mock_query_households.results = [["abc123", "TestHousehold"]] + mock_query_contacts.results = [ + ["def456", "First", "Last", "test@example.com", "abc123"] + ] + mock_query_events.results = [ + ["ijk789", "Last1", "abc123"], + ["lmn010", "Last2", "def456"] + ] + + query_op_mock.side_effect = [mock_query_households, mock_query_contacts, mock_query_events] + task() + with create_engine(task.options["database_url"]).connect() as conn: + household = next(conn.execute("select * from households")) + assert household.name == "TestHousehold" + assert not hasattr(household, "IsPersonAccount") + assert household.record_type == "HH_Account" + + contact = next(conn.execute("select * from contacts")) + assert contact.household_id == "Account-1" + assert not hasattr(contact, "IsPersonAccount") + + events = conn.execute("select * from events").fetchall() + assert events[0].who_id == "Account-1" + assert events[1].who_id == "Contact-1" + @mock.patch("cumulusci.tasks.bulkdata.extract.log_progress") def test_import_results__oid_as_pk(self, log_mock): diff --git a/cumulusci/tasks/bulkdata/tests/test_mapping_parser.py b/cumulusci/tasks/bulkdata/tests/test_mapping_parser.py index afa397d09e..ba94834157 100644 --- a/cumulusci/tasks/bulkdata/tests/test_mapping_parser.py +++ b/cumulusci/tasks/bulkdata/tests/test_mapping_parser.py @@ -1260,3 +1260,18 @@ def test_upsert_key_list(self): "FirstName", "LastName", ), mapping["Insert Accounts"]["update_key"] + + def test_get_extract_field_list(self): + m = MappingStep( + sf_object="Account", + fields=["Name", "RecordTypeId", "AccountSite"], + lookups={"ParentId": MappingLookup(table="Account")}, + ) + + assert m.get_extract_field_list() == [ + "Id", + "Name", + "AccountSite", + "ParentId", + "RecordTypeId", + ] diff --git a/cumulusci/tests/shared_cassettes/GET_sobjects_Event_describe.yaml b/cumulusci/tests/shared_cassettes/GET_sobjects_Event_describe.yaml new file mode 100644 index 0000000000..58623f0c3c --- /dev/null +++ b/cumulusci/tests/shared_cassettes/GET_sobjects_Event_describe.yaml @@ -0,0 +1,18 @@ +request: + method: GET + uri: https://orgname.my.salesforce.com/services/data/vxx.0/sobjects/Event/describe + body: null + headers: + Request-Headers: + - Elided +response: + status: + code: 200 + message: OK + headers: + Content-Type: + - application/json;charset=UTF-8 + Others: Elided + body: + string: >- + {"actionOverrides":[],"activateable":false,"associateEntityType":null,"associateParentEntity":null,"childRelationships":[{"cascadeDelete":true,"childSObject":"AIInsightValue","deprecatedAndHidden":false,"field":"SobjectLookupValueId","junctionIdListNames":[],"junctionReferenceTo":[],"relationshipName":null,"restrictedDelete":false},{"cascadeDelete":true,"childSObject":"AIRecordInsight","deprecatedAndHidden":false,"field":"TargetId","junctionIdListNames":[],"junctionReferenceTo":[],"relationshipName":null,"restrictedDelete":false},{"cascadeDelete":false,"childSObject":"AcceptedEventRelation","deprecatedAndHidden":false,"field":"EventId","junctionIdListNames":["AcceptedEventInviteeIds"],"junctionReferenceTo":["Calendar","Contact","Lead","User"],"relationshipName":"AcceptedEventRelations","restrictedDelete":false},{"cascadeDelete":false,"childSObject":"ActivityFieldHistory","deprecatedAndHidden":false,"field":"ActivityId","junctionIdListNames":[],"junctionReferenceTo":[],"relationshipName":"ActivityFieldHistories","restrictedDelete":false},{"cascadeDelete":true,"childSObject":"AttachedContentDocument","deprecatedAndHidden":false,"field":"LinkedEntityId","junctionIdListNames":[],"junctionReferenceTo":[],"relationshipName":"AttachedContentDocuments","restrictedDelete":false},{"cascadeDelete":true,"childSObject":"Attachment","deprecatedAndHidden":false,"field":"ParentId","junctionIdListNames":[],"junctionReferenceTo":[],"relationshipName":"Attachments","restrictedDelete":false},{"cascadeDelete":true,"childSObject":"CombinedAttachment","deprecatedAndHidden":false,"field":"ParentId","junctionIdListNames":[],"junctionReferenceTo":[],"relationshipName":"CombinedAttachments","restrictedDelete":false},{"cascadeDelete":true,"childSObject":"ContentDocumentLink","deprecatedAndHidden":false,"field":"LinkedEntityId","junctionIdListNames":[],"junctionReferenceTo":[],"relationshipName":"ContentDocumentLinks","restrictedDelete":false},{"cascadeDelete":false,"childSObject":"ContentDocumentLinkChangeEvent","deprecatedAndHidden":false,"field":"LinkedEntityId","junctionIdListNames":[],"junctionReferenceTo":[],"relationshipName":null,"restrictedDelete":false},{"cascadeDelete":false,"childSObject":"ContentVersion","deprecatedAndHidden":false,"field":"FirstPublishLocationId","junctionIdListNames":[],"junctionReferenceTo":[],"relationshipName":null,"restrictedDelete":false},{"cascadeDelete":false,"childSObject":"ContentVersionChangeEvent","deprecatedAndHidden":false,"field":"FirstPublishLocationId","junctionIdListNames":[],"junctionReferenceTo":[],"relationshipName":null,"restrictedDelete":false},{"cascadeDelete":false,"childSObject":"DeclinedEventRelation","deprecatedAndHidden":false,"field":"EventId","junctionIdListNames":["DeclinedEventInviteeIds"],"junctionReferenceTo":["Calendar","Contact","Lead","User"],"relationshipName":"DeclinedEventRelations","restrictedDelete":false},{"cascadeDelete":true,"childSObject":"EntitySubscription","deprecatedAndHidden":false,"field":"ParentId","junctionIdListNames":[],"junctionReferenceTo":[],"relationshipName":"FeedSubscriptionsForEntity","restrictedDelete":false},{"cascadeDelete":false,"childSObject":"Event","deprecatedAndHidden":false,"field":"RecurrenceActivityId","junctionIdListNames":[],"junctionReferenceTo":[],"relationshipName":"RecurringEvents","restrictedDelete":false},{"cascadeDelete":false,"childSObject":"EventChangeEvent","deprecatedAndHidden":false,"field":"RecurrenceActivityId","junctionIdListNames":[],"junctionReferenceTo":[],"relationshipName":null,"restrictedDelete":false},{"cascadeDelete":true,"childSObject":"EventFeed","deprecatedAndHidden":false,"field":"ParentId","junctionIdListNames":[],"junctionReferenceTo":[],"relationshipName":"Feeds","restrictedDelete":false},{"cascadeDelete":true,"childSObject":"EventRelation","deprecatedAndHidden":false,"field":"EventId","junctionIdListNames":[],"junctionReferenceTo":[],"relationshipName":"EventRelations","restrictedDelete":false},{"cascadeDelete":false,"childSObject":"EventRelationChangeEvent","deprecatedAndHidden":false,"field":"EventId","junctionIdListNames":[],"junctionReferenceTo":[],"relationshipName":null,"restrictedDelete":false},{"cascadeDelete":true,"childSObject":"ExternalEventMapping","deprecatedAndHidden":false,"field":"EventId","junctionIdListNames":[],"junctionReferenceTo":[],"relationshipName":null,"restrictedDelete":false},{"cascadeDelete":false,"childSObject":"FeedComment","deprecatedAndHidden":false,"field":"ParentId","junctionIdListNames":[],"junctionReferenceTo":[],"relationshipName":null,"restrictedDelete":false},{"cascadeDelete":true,"childSObject":"FeedItem","deprecatedAndHidden":false,"field":"ParentId","junctionIdListNames":[],"junctionReferenceTo":[],"relationshipName":null,"restrictedDelete":false},{"cascadeDelete":false,"childSObject":"FlowExecutionErrorEvent","deprecatedAndHidden":false,"field":"ContextRecordId","junctionIdListNames":[],"junctionReferenceTo":[],"relationshipName":null,"restrictedDelete":false},{"cascadeDelete":false,"childSObject":"FlowRecordRelation","deprecatedAndHidden":false,"field":"RelatedRecordId","junctionIdListNames":[],"junctionReferenceTo":[],"relationshipName":null,"restrictedDelete":false},{"cascadeDelete":false,"childSObject":"NetworkActivityAudit","deprecatedAndHidden":false,"field":"ParentEntityId","junctionIdListNames":[],"junctionReferenceTo":[],"relationshipName":"ParentEntities","restrictedDelete":false},{"cascadeDelete":false,"childSObject":"NetworkFeedResponseMetric","deprecatedAndHidden":false,"field":"ParentRecordId","junctionIdListNames":[],"junctionReferenceTo":[],"relationshipName":null,"restrictedDelete":false},{"cascadeDelete":true,"childSObject":"TopicAssignment","deprecatedAndHidden":false,"field":"EntityId","junctionIdListNames":[],"junctionReferenceTo":[],"relationshipName":"TopicAssignments","restrictedDelete":false},{"cascadeDelete":false,"childSObject":"UndecidedEventRelation","deprecatedAndHidden":false,"field":"EventId","junctionIdListNames":["UndecidedEventInviteeIds"],"junctionReferenceTo":["Calendar","Contact","Lead","User"],"relationshipName":"UndecidedEventRelations","restrictedDelete":false}],"compactLayoutable":true,"createable":true,"custom":false,"customSetting":false,"deepCloneable":false,"defaultImplementation":null,"deletable":true,"deprecatedAndHidden":false,"extendedBy":null,"extendsInterfaces":null,"feedEnabled":false,"fields":[{"aggregatable":true,"aiPredictionField":false,"autoNumber":false,"byteLength":18,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":false,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":true,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":true,"highScaleNumber":false,"htmlFormatted":false,"idLookup":true,"inlineHelpText":null,"label":"Activity ID","length":18,"mask":null,"maskType":null,"name":"Id","nameField":false,"namePointing":false,"nillable":false,"permissionable":false,"picklistValues":[],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"searchPrefilterable":false,"soapType":"tns:ID","sortable":true,"type":"id","unique":false,"updateable":false,"writeRequiresMasterRead":false},{"aggregatable":true,"aiPredictionField":false,"autoNumber":false,"byteLength":18,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":true,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":false,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":true,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Name ID","length":18,"mask":null,"maskType":null,"name":"WhoId","nameField":false,"namePointing":true,"nillable":true,"permissionable":true,"picklistValues":[],"polymorphicForeignKey":true,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":["Contact","Lead"],"relationshipName":"Who","relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"searchPrefilterable":true,"soapType":"tns:ID","sortable":true,"type":"reference","unique":false,"updateable":true,"writeRequiresMasterRead":false},{"aggregatable":true,"aiPredictionField":false,"autoNumber":false,"byteLength":18,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":true,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":false,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":true,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Related To ID","length":18,"mask":null,"maskType":null,"name":"WhatId","nameField":false,"namePointing":true,"nillable":true,"permissionable":true,"picklistValues":[],"polymorphicForeignKey":true,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":["Account","ApptBundleAggrDurDnscale","ApptBundleAggrPolicy","ApptBundleConfig","ApptBundlePolicy","ApptBundlePolicySvcTerr","ApptBundlePropagatePolicy","ApptBundleRestrictPolicy","ApptBundleSortPolicy","Asset","AssetRelationship","AssetWarranty","AssignedResource","AttributeDefinition","AttributePicklist","AttributePicklistValue","Campaign","Case","ChannelProgram","ChannelProgramLevel","CommSubscriptionConsent","ContactRequest","Contract","ContractLineItem","CreditMemo","DelegatedAccount","Entitlement","Expense","ExpenseReport","ExpenseReportEntry","Image","Invoice","LegalEntity","ListEmail","Location","MaintenanceAsset","MaintenancePlan","OperatingHoursHoliday","Opportunity","Order","PartnerFundAllocation","PartnerFundClaim","PartnerFundRequest","PartnerMarketingBudget","PartyConsent","ProcessException","Product2","ProductConsumed","ProductItem","ProductRequest","ProductRequestLineItem","ProductServiceCampaign","ProductServiceCampaignItem","ProductTransfer","ResourceAbsence","ReturnOrder","ReturnOrderLineItem","ServiceAppointment","ServiceContract","ServiceCrew","ServiceCrewMember","ServiceResource","Shift","ShiftPattern","ShiftPatternEntry","Shipment","ShipmentItem","Solution","TimeSheet","TimeSheetEntry","TravelMode","WarrantyTerm","WorkOrder","WorkOrderLineItem","WorkPlan","WorkPlanSelectionRule","WorkPlanTemplate","WorkPlanTemplateEntry","WorkStep","WorkStepTemplate"],"relationshipName":"What","relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"searchPrefilterable":true,"soapType":"tns:ID","sortable":true,"type":"reference","unique":false,"updateable":true,"writeRequiresMasterRead":false},{"aggregatable":true,"aiPredictionField":false,"autoNumber":false,"byteLength":765,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":true,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":false,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":false,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Subject","length":255,"mask":null,"maskType":null,"name":"Subject","nameField":true,"namePointing":false,"nillable":true,"permissionable":false,"picklistValues":[{"active":true,"defaultValue":false,"label":"Call","validFor":null,"value":"Call"},{"active":true,"defaultValue":false,"label":"Email","validFor":null,"value":"Email"},{"active":true,"defaultValue":false,"label":"Meeting","validFor":null,"value":"Meeting"},{"active":true,"defaultValue":false,"label":"Send Letter/Quote","validFor":null,"value":"Send Letter/Quote"},{"active":true,"defaultValue":false,"label":"Other","validFor":null,"value":"Other"}],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"searchPrefilterable":false,"soapType":"xsd:string","sortable":true,"type":"combobox","unique":false,"updateable":true,"writeRequiresMasterRead":false},{"aggregatable":true,"aiPredictionField":false,"autoNumber":false,"byteLength":765,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":true,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":false,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":true,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Location","length":255,"mask":null,"maskType":null,"name":"Location","nameField":false,"namePointing":false,"nillable":true,"permissionable":true,"picklistValues":[],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"searchPrefilterable":false,"soapType":"xsd:string","sortable":true,"type":"string","unique":false,"updateable":true,"writeRequiresMasterRead":false},{"aggregatable":false,"aiPredictionField":false,"autoNumber":false,"byteLength":0,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":true,"custom":false,"defaultValue":false,"defaultValueFormula":null,"defaultedOnCreate":true,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":true,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"All-Day Event","length":0,"mask":null,"maskType":null,"name":"IsAllDayEvent","nameField":false,"namePointing":false,"nillable":false,"permissionable":true,"picklistValues":[],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"searchPrefilterable":false,"soapType":"xsd:boolean","sortable":true,"type":"boolean","unique":false,"updateable":true,"writeRequiresMasterRead":false},{"aggregatable":true,"aiPredictionField":false,"autoNumber":false,"byteLength":0,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":true,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":false,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":false,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Due Date Time","length":0,"mask":null,"maskType":null,"name":"ActivityDateTime","nameField":false,"namePointing":false,"nillable":true,"permissionable":false,"picklistValues":[],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"searchPrefilterable":false,"soapType":"xsd:dateTime","sortable":true,"type":"datetime","unique":false,"updateable":true,"writeRequiresMasterRead":false},{"aggregatable":false,"aiPredictionField":false,"autoNumber":false,"byteLength":0,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":true,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":false,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":true,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Due Date Only","length":0,"mask":null,"maskType":null,"name":"ActivityDate","nameField":false,"namePointing":false,"nillable":true,"permissionable":false,"picklistValues":[],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"searchPrefilterable":false,"soapType":"xsd:date","sortable":true,"type":"date","unique":false,"updateable":true,"writeRequiresMasterRead":false},{"aggregatable":true,"aiPredictionField":false,"autoNumber":false,"byteLength":0,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":true,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":false,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":8,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":true,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Duration","length":0,"mask":null,"maskType":null,"name":"DurationInMinutes","nameField":false,"namePointing":false,"nillable":true,"permissionable":false,"picklistValues":[],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"searchPrefilterable":false,"soapType":"xsd:int","sortable":true,"type":"int","unique":false,"updateable":true,"writeRequiresMasterRead":false},{"aggregatable":true,"aiPredictionField":false,"autoNumber":false,"byteLength":0,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":true,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":false,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":false,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Start Date Time","length":0,"mask":null,"maskType":null,"name":"StartDateTime","nameField":false,"namePointing":false,"nillable":true,"permissionable":false,"picklistValues":[],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"searchPrefilterable":false,"soapType":"xsd:dateTime","sortable":true,"type":"datetime","unique":false,"updateable":true,"writeRequiresMasterRead":false},{"aggregatable":true,"aiPredictionField":false,"autoNumber":false,"byteLength":0,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":true,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":false,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":false,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"End Date Time","length":0,"mask":null,"maskType":null,"name":"EndDateTime","nameField":false,"namePointing":false,"nillable":true,"permissionable":false,"picklistValues":[],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"searchPrefilterable":false,"soapType":"xsd:dateTime","sortable":true,"type":"datetime","unique":false,"updateable":true,"writeRequiresMasterRead":false},{"aggregatable":true,"aiPredictionField":false,"autoNumber":false,"byteLength":0,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":false,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":false,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":true,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"End Date","length":0,"mask":null,"maskType":null,"name":"EndDate","nameField":false,"namePointing":false,"nillable":true,"permissionable":false,"picklistValues":[],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"searchPrefilterable":false,"soapType":"xsd:date","sortable":true,"type":"date","unique":false,"updateable":false,"writeRequiresMasterRead":false},{"aggregatable":false,"aiPredictionField":false,"autoNumber":false,"byteLength":96000,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":true,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":false,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":"plaintextarea","filterable":false,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":false,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Description","length":32000,"mask":null,"maskType":null,"name":"Description","nameField":false,"namePointing":false,"nillable":true,"permissionable":true,"picklistValues":[],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"searchPrefilterable":false,"soapType":"xsd:string","sortable":false,"type":"textarea","unique":false,"updateable":true,"writeRequiresMasterRead":false},{"aggregatable":true,"aiPredictionField":false,"autoNumber":false,"byteLength":18,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":false,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":false,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":true,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Account ID","length":18,"mask":null,"maskType":null,"name":"AccountId","nameField":false,"namePointing":false,"nillable":true,"permissionable":false,"picklistValues":[],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":["Account"],"relationshipName":"Account","relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"searchPrefilterable":true,"soapType":"tns:ID","sortable":true,"type":"reference","unique":false,"updateable":false,"writeRequiresMasterRead":false},{"aggregatable":true,"aiPredictionField":false,"autoNumber":false,"byteLength":18,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":true,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":true,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":true,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Assigned To ID","length":18,"mask":null,"maskType":null,"name":"OwnerId","nameField":false,"namePointing":true,"nillable":false,"permissionable":false,"picklistValues":[],"polymorphicForeignKey":true,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":["Calendar","User"],"relationshipName":"Owner","relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"searchPrefilterable":false,"soapType":"tns:ID","sortable":true,"type":"reference","unique":false,"updateable":true,"writeRequiresMasterRead":false},{"aggregatable":false,"aiPredictionField":false,"autoNumber":false,"byteLength":0,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":true,"custom":false,"defaultValue":false,"defaultValueFormula":null,"defaultedOnCreate":true,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":true,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Private","length":0,"mask":null,"maskType":null,"name":"IsPrivate","nameField":false,"namePointing":false,"nillable":false,"permissionable":false,"picklistValues":[],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"searchPrefilterable":false,"soapType":"xsd:boolean","sortable":true,"type":"boolean","unique":false,"updateable":true,"writeRequiresMasterRead":false},{"aggregatable":true,"aiPredictionField":false,"autoNumber":false,"byteLength":120,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":true,"custom":false,"defaultValue":"Busy","defaultValueFormula":null,"defaultedOnCreate":true,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":true,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Show Time As","length":40,"mask":null,"maskType":null,"name":"ShowAs","nameField":false,"namePointing":false,"nillable":true,"permissionable":false,"picklistValues":[{"active":true,"defaultValue":true,"label":"Busy","validFor":null,"value":"Busy"},{"active":true,"defaultValue":false,"label":"Out of Office","validFor":null,"value":"OutOfOffice"},{"active":true,"defaultValue":false,"label":"Free","validFor":null,"value":"Free"}],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":true,"scale":0,"searchPrefilterable":false,"soapType":"xsd:string","sortable":true,"type":"picklist","unique":false,"updateable":true,"writeRequiresMasterRead":false},{"aggregatable":false,"aiPredictionField":false,"autoNumber":false,"byteLength":0,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":false,"custom":false,"defaultValue":false,"defaultValueFormula":null,"defaultedOnCreate":true,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":true,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Deleted","length":0,"mask":null,"maskType":null,"name":"IsDeleted","nameField":false,"namePointing":false,"nillable":false,"permissionable":false,"picklistValues":[],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"searchPrefilterable":false,"soapType":"xsd:boolean","sortable":true,"type":"boolean","unique":false,"updateable":false,"writeRequiresMasterRead":false},{"aggregatable":false,"aiPredictionField":false,"autoNumber":false,"byteLength":0,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":false,"custom":false,"defaultValue":false,"defaultValueFormula":null,"defaultedOnCreate":true,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":true,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Is Child","length":0,"mask":null,"maskType":null,"name":"IsChild","nameField":false,"namePointing":false,"nillable":false,"permissionable":false,"picklistValues":[],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"searchPrefilterable":false,"soapType":"xsd:boolean","sortable":true,"type":"boolean","unique":false,"updateable":false,"writeRequiresMasterRead":false},{"aggregatable":false,"aiPredictionField":false,"autoNumber":false,"byteLength":0,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":false,"custom":false,"defaultValue":false,"defaultValueFormula":null,"defaultedOnCreate":true,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":true,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Is Group Event","length":0,"mask":null,"maskType":null,"name":"IsGroupEvent","nameField":false,"namePointing":false,"nillable":false,"permissionable":false,"picklistValues":[],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"searchPrefilterable":false,"soapType":"xsd:boolean","sortable":true,"type":"boolean","unique":false,"updateable":false,"writeRequiresMasterRead":false},{"aggregatable":true,"aiPredictionField":false,"autoNumber":false,"byteLength":120,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":false,"custom":false,"defaultValue":"0","defaultValueFormula":null,"defaultedOnCreate":true,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":true,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Group Event Type","length":40,"mask":null,"maskType":null,"name":"GroupEventType","nameField":false,"namePointing":false,"nillable":true,"permissionable":false,"picklistValues":[{"active":true,"defaultValue":true,"label":"Non-group Event","validFor":null,"value":"0"},{"active":true,"defaultValue":false,"label":"Group Event","validFor":null,"value":"1"},{"active":true,"defaultValue":false,"label":"Proposed Event","validFor":null,"value":"2"},{"active":true,"defaultValue":false,"label":"IsRecurrence2 Series Pattern","validFor":null,"value":"3"}],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":true,"scale":0,"searchPrefilterable":false,"soapType":"xsd:string","sortable":true,"type":"picklist","unique":false,"updateable":false,"writeRequiresMasterRead":false},{"aggregatable":true,"aiPredictionField":false,"autoNumber":false,"byteLength":0,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":false,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":true,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":false,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Created Date","length":0,"mask":null,"maskType":null,"name":"CreatedDate","nameField":false,"namePointing":false,"nillable":false,"permissionable":false,"picklistValues":[],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"searchPrefilterable":false,"soapType":"xsd:dateTime","sortable":true,"type":"datetime","unique":false,"updateable":false,"writeRequiresMasterRead":false},{"aggregatable":true,"aiPredictionField":false,"autoNumber":false,"byteLength":18,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":false,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":true,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":true,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Created By ID","length":18,"mask":null,"maskType":null,"name":"CreatedById","nameField":false,"namePointing":false,"nillable":false,"permissionable":false,"picklistValues":[],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":["User"],"relationshipName":"CreatedBy","relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"searchPrefilterable":false,"soapType":"tns:ID","sortable":true,"type":"reference","unique":false,"updateable":false,"writeRequiresMasterRead":false},{"aggregatable":true,"aiPredictionField":false,"autoNumber":false,"byteLength":0,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":false,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":true,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":false,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Last Modified Date","length":0,"mask":null,"maskType":null,"name":"LastModifiedDate","nameField":false,"namePointing":false,"nillable":false,"permissionable":false,"picklistValues":[],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"searchPrefilterable":false,"soapType":"xsd:dateTime","sortable":true,"type":"datetime","unique":false,"updateable":false,"writeRequiresMasterRead":false},{"aggregatable":true,"aiPredictionField":false,"autoNumber":false,"byteLength":18,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":false,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":true,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":true,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Last Modified By ID","length":18,"mask":null,"maskType":null,"name":"LastModifiedById","nameField":false,"namePointing":false,"nillable":false,"permissionable":false,"picklistValues":[],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":["User"],"relationshipName":"LastModifiedBy","relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"searchPrefilterable":false,"soapType":"tns:ID","sortable":true,"type":"reference","unique":false,"updateable":false,"writeRequiresMasterRead":false},{"aggregatable":true,"aiPredictionField":false,"autoNumber":false,"byteLength":0,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":false,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":true,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":false,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"System Modstamp","length":0,"mask":null,"maskType":null,"name":"SystemModstamp","nameField":false,"namePointing":false,"nillable":false,"permissionable":false,"picklistValues":[],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"searchPrefilterable":false,"soapType":"xsd:dateTime","sortable":true,"type":"datetime","unique":false,"updateable":false,"writeRequiresMasterRead":false},{"aggregatable":false,"aiPredictionField":false,"autoNumber":false,"byteLength":0,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":false,"custom":false,"defaultValue":false,"defaultValueFormula":null,"defaultedOnCreate":true,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":true,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Archived","length":0,"mask":null,"maskType":null,"name":"IsArchived","nameField":false,"namePointing":false,"nillable":false,"permissionable":false,"picklistValues":[],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"searchPrefilterable":false,"soapType":"xsd:boolean","sortable":true,"type":"boolean","unique":false,"updateable":false,"writeRequiresMasterRead":false},{"aggregatable":true,"aiPredictionField":false,"autoNumber":false,"byteLength":18,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":false,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":false,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":true,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Recurrence Activity ID","length":18,"mask":null,"maskType":null,"name":"RecurrenceActivityId","nameField":false,"namePointing":false,"nillable":true,"permissionable":false,"picklistValues":[],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":["Event"],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"searchPrefilterable":false,"soapType":"tns:ID","sortable":true,"type":"reference","unique":false,"updateable":false,"writeRequiresMasterRead":false},{"aggregatable":false,"aiPredictionField":false,"autoNumber":false,"byteLength":0,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":true,"custom":false,"defaultValue":false,"defaultValueFormula":null,"defaultedOnCreate":true,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":true,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Create Recurring Series of Events","length":0,"mask":null,"maskType":null,"name":"IsRecurrence","nameField":false,"namePointing":false,"nillable":false,"permissionable":false,"picklistValues":[],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"searchPrefilterable":false,"soapType":"xsd:boolean","sortable":true,"type":"boolean","unique":false,"updateable":false,"writeRequiresMasterRead":false},{"aggregatable":true,"aiPredictionField":false,"autoNumber":false,"byteLength":0,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":true,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":false,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":false,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Recurrence Start","length":0,"mask":null,"maskType":null,"name":"RecurrenceStartDateTime","nameField":false,"namePointing":false,"nillable":true,"permissionable":false,"picklistValues":[],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"searchPrefilterable":false,"soapType":"xsd:dateTime","sortable":true,"type":"datetime","unique":false,"updateable":true,"writeRequiresMasterRead":false},{"aggregatable":true,"aiPredictionField":false,"autoNumber":false,"byteLength":0,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":true,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":false,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":true,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Recurrence End","length":0,"mask":null,"maskType":null,"name":"RecurrenceEndDateOnly","nameField":false,"namePointing":false,"nillable":true,"permissionable":false,"picklistValues":[],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"searchPrefilterable":false,"soapType":"xsd:date","sortable":true,"type":"date","unique":false,"updateable":true,"writeRequiresMasterRead":false},{"aggregatable":true,"aiPredictionField":false,"autoNumber":false,"byteLength":120,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":true,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":false,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":true,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Recurrence Time Zone","length":40,"mask":null,"maskType":null,"name":"RecurrenceTimeZoneSidKey","nameField":false,"namePointing":false,"nillable":true,"permissionable":false,"picklistValues":[{"active":true,"defaultValue":false,"label":"(GMT+14:00) Line Islands Time (Pacific/Kiritimati)","validFor":null,"value":"Pacific/Kiritimati"},{"active":true,"defaultValue":false,"label":"(GMT+13:45) Chatham Daylight Time (Pacific/Chatham)","validFor":null,"value":"Pacific/Chatham"},{"active":true,"defaultValue":false,"label":"(GMT+13:00) New Zealand Daylight Time (Antarctica/McMurdo)","validFor":null,"value":"Antarctica/McMurdo"},{"active":true,"defaultValue":false,"label":"(GMT+13:00) Apia Standard Time (Pacific/Apia)","validFor":null,"value":"Pacific/Apia"},{"active":true,"defaultValue":false,"label":"(GMT+13:00) New Zealand Daylight Time (Pacific/Auckland)","validFor":null,"value":"Pacific/Auckland"},{"active":true,"defaultValue":false,"label":"(GMT+13:00) Phoenix Islands Time (Pacific/Enderbury)","validFor":null,"value":"Pacific/Enderbury"},{"active":true,"defaultValue":false,"label":"(GMT+13:00) Tokelau Time (Pacific/Fakaofo)","validFor":null,"value":"Pacific/Fakaofo"},{"active":true,"defaultValue":false,"label":"(GMT+13:00) Tonga Standard Time (Pacific/Tongatapu)","validFor":null,"value":"Pacific/Tongatapu"},{"active":true,"defaultValue":false,"label":"(GMT+12:00) Anadyr Standard Time (Asia/Anadyr)","validFor":null,"value":"Asia/Anadyr"},{"active":true,"defaultValue":false,"label":"(GMT+12:00) Petropavlovsk-Kamchatski Standard Time (Asia/Kamchatka)","validFor":null,"value":"Asia/Kamchatka"},{"active":true,"defaultValue":false,"label":"(GMT+12:00) Fiji Standard Time (Pacific/Fiji)","validFor":null,"value":"Pacific/Fiji"},{"active":true,"defaultValue":false,"label":"(GMT+12:00) Tuvalu Time (Pacific/Funafuti)","validFor":null,"value":"Pacific/Funafuti"},{"active":true,"defaultValue":false,"label":"(GMT+12:00) Marshall Islands Time (Pacific/Kwajalein)","validFor":null,"value":"Pacific/Kwajalein"},{"active":true,"defaultValue":false,"label":"(GMT+12:00) Marshall Islands Time (Pacific/Majuro)","validFor":null,"value":"Pacific/Majuro"},{"active":true,"defaultValue":false,"label":"(GMT+12:00) Nauru Time (Pacific/Nauru)","validFor":null,"value":"Pacific/Nauru"},{"active":true,"defaultValue":false,"label":"(GMT+12:00) Norfolk Island Daylight Time (Pacific/Norfolk)","validFor":null,"value":"Pacific/Norfolk"},{"active":true,"defaultValue":false,"label":"(GMT+12:00) Gilbert Islands Time (Pacific/Tarawa)","validFor":null,"value":"Pacific/Tarawa"},{"active":true,"defaultValue":false,"label":"(GMT+12:00) Wake Island Time (Pacific/Wake)","validFor":null,"value":"Pacific/Wake"},{"active":true,"defaultValue":false,"label":"(GMT+12:00) Wallis & Futuna Time (Pacific/Wallis)","validFor":null,"value":"Pacific/Wallis"},{"active":true,"defaultValue":false,"label":"(GMT+11:00) Casey Time (Antarctica/Casey)","validFor":null,"value":"Antarctica/Casey"},{"active":true,"defaultValue":false,"label":"(GMT+11:00) Australian Eastern Daylight Time (Antarctica/Macquarie)","validFor":null,"value":"Antarctica/Macquarie"},{"active":true,"defaultValue":false,"label":"(GMT+11:00) Magadan Standard Time (Asia/Magadan)","validFor":null,"value":"Asia/Magadan"},{"active":true,"defaultValue":false,"label":"(GMT+11:00) Sakhalin Standard Time (Asia/Sakhalin)","validFor":null,"value":"Asia/Sakhalin"},{"active":true,"defaultValue":false,"label":"(GMT+11:00) Magadan Standard Time (Asia/Srednekolymsk)","validFor":null,"value":"Asia/Srednekolymsk"},{"active":true,"defaultValue":false,"label":"(GMT+11:00) Australian Eastern Daylight Time (Australia/Currie)","validFor":null,"value":"Australia/Currie"},{"active":true,"defaultValue":false,"label":"(GMT+11:00) Australian Eastern Daylight Time (Australia/Hobart)","validFor":null,"value":"Australia/Hobart"},{"active":true,"defaultValue":false,"label":"(GMT+11:00) Lord Howe Daylight Time (Australia/Lord_Howe)","validFor":null,"value":"Australia/Lord_Howe"},{"active":true,"defaultValue":false,"label":"(GMT+11:00) Australian Eastern Daylight Time (Australia/Melbourne)","validFor":null,"value":"Australia/Melbourne"},{"active":true,"defaultValue":false,"label":"(GMT+11:00) Australian Eastern Daylight Time (Australia/Sydney)","validFor":null,"value":"Australia/Sydney"},{"active":true,"defaultValue":false,"label":"(GMT+11:00) Bougainville Standard Time (Pacific/Bougainville)","validFor":null,"value":"Pacific/Bougainville"},{"active":true,"defaultValue":false,"label":"(GMT+11:00) Vanuatu Standard Time (Pacific/Efate)","validFor":null,"value":"Pacific/Efate"},{"active":true,"defaultValue":false,"label":"(GMT+11:00) Solomon Islands Time (Pacific/Guadalcanal)","validFor":null,"value":"Pacific/Guadalcanal"},{"active":true,"defaultValue":false,"label":"(GMT+11:00) Kosrae Time (Pacific/Kosrae)","validFor":null,"value":"Pacific/Kosrae"},{"active":true,"defaultValue":false,"label":"(GMT+11:00) New Caledonia Standard Time (Pacific/Noumea)","validFor":null,"value":"Pacific/Noumea"},{"active":true,"defaultValue":false,"label":"(GMT+11:00) Ponape Time (Pacific/Ponape)","validFor":null,"value":"Pacific/Ponape"},{"active":true,"defaultValue":false,"label":"(GMT+10:30) Australian Central Daylight Time (Australia/Adelaide)","validFor":null,"value":"Australia/Adelaide"},{"active":true,"defaultValue":false,"label":"(GMT+10:30) Australian Central Daylight Time (Australia/Broken_Hill)","validFor":null,"value":"Australia/Broken_Hill"},{"active":true,"defaultValue":false,"label":"(GMT+10:00) Dumont-d’Urville Time (Antarctica/DumontDUrville)","validFor":null,"value":"Antarctica/DumontDUrville"},{"active":true,"defaultValue":false,"label":"(GMT+10:00) Vladivostok Standard Time (Asia/Ust-Nera)","validFor":null,"value":"Asia/Ust-Nera"},{"active":true,"defaultValue":false,"label":"(GMT+10:00) Vladivostok Standard Time (Asia/Vladivostok)","validFor":null,"value":"Asia/Vladivostok"},{"active":true,"defaultValue":false,"label":"(GMT+10:00) Australian Eastern Standard Time (Australia/Brisbane)","validFor":null,"value":"Australia/Brisbane"},{"active":true,"defaultValue":false,"label":"(GMT+10:00) Australian Eastern Standard Time (Australia/Lindeman)","validFor":null,"value":"Australia/Lindeman"},{"active":true,"defaultValue":false,"label":"(GMT+10:00) Chamorro Standard Time (Pacific/Guam)","validFor":null,"value":"Pacific/Guam"},{"active":true,"defaultValue":false,"label":"(GMT+10:00) Papua New Guinea Time (Pacific/Port_Moresby)","validFor":null,"value":"Pacific/Port_Moresby"},{"active":true,"defaultValue":false,"label":"(GMT+10:00) Chamorro Standard Time (Pacific/Saipan)","validFor":null,"value":"Pacific/Saipan"},{"active":true,"defaultValue":false,"label":"(GMT+10:00) Chuuk Time (Pacific/Truk)","validFor":null,"value":"Pacific/Truk"},{"active":true,"defaultValue":false,"label":"(GMT+09:30) Australian Central Standard Time (Australia/Darwin)","validFor":null,"value":"Australia/Darwin"},{"active":true,"defaultValue":false,"label":"(GMT+09:00) Yakutsk Standard Time (Asia/Chita)","validFor":null,"value":"Asia/Chita"},{"active":true,"defaultValue":false,"label":"(GMT+09:00) East Timor Time (Asia/Dili)","validFor":null,"value":"Asia/Dili"},{"active":true,"defaultValue":false,"label":"(GMT+09:00) Eastern Indonesia Time (Asia/Jayapura)","validFor":null,"value":"Asia/Jayapura"},{"active":true,"defaultValue":false,"label":"(GMT+09:00) Yakutsk Standard Time (Asia/Khandyga)","validFor":null,"value":"Asia/Khandyga"},{"active":true,"defaultValue":false,"label":"(GMT+09:00) Korean Standard Time (Asia/Seoul)","validFor":null,"value":"Asia/Seoul"},{"active":true,"defaultValue":false,"label":"(GMT+09:00) Japan Standard Time (Asia/Tokyo)","validFor":null,"value":"Asia/Tokyo"},{"active":true,"defaultValue":false,"label":"(GMT+09:00) Yakutsk Standard Time (Asia/Yakutsk)","validFor":null,"value":"Asia/Yakutsk"},{"active":true,"defaultValue":false,"label":"(GMT+09:00) Palau Time (Pacific/Palau)","validFor":null,"value":"Pacific/Palau"},{"active":true,"defaultValue":false,"label":"(GMT+08:45) Australian Central Western Standard Time (Australia/Eucla)","validFor":null,"value":"Australia/Eucla"},{"active":true,"defaultValue":false,"label":"(GMT+08:00) Brunei Darussalam Time (Asia/Brunei)","validFor":null,"value":"Asia/Brunei"},{"active":true,"defaultValue":false,"label":"(GMT+08:00) Ulaanbaatar Standard Time (Asia/Choibalsan)","validFor":null,"value":"Asia/Choibalsan"},{"active":true,"defaultValue":false,"label":"(GMT+08:00) Hong Kong Standard Time (Asia/Hong_Kong)","validFor":null,"value":"Asia/Hong_Kong"},{"active":true,"defaultValue":false,"label":"(GMT+08:00) Irkutsk Standard Time (Asia/Irkutsk)","validFor":null,"value":"Asia/Irkutsk"},{"active":true,"defaultValue":false,"label":"(GMT+08:00) Malaysia Time (Asia/Kuala_Lumpur)","validFor":null,"value":"Asia/Kuala_Lumpur"},{"active":true,"defaultValue":false,"label":"(GMT+08:00) Malaysia Time (Asia/Kuching)","validFor":null,"value":"Asia/Kuching"},{"active":true,"defaultValue":false,"label":"(GMT+08:00) China Standard Time (Asia/Macau)","validFor":null,"value":"Asia/Macau"},{"active":true,"defaultValue":false,"label":"(GMT+08:00) Central Indonesia Time (Asia/Makassar)","validFor":null,"value":"Asia/Makassar"},{"active":true,"defaultValue":false,"label":"(GMT+08:00) Philippine Standard Time (Asia/Manila)","validFor":null,"value":"Asia/Manila"},{"active":true,"defaultValue":false,"label":"(GMT+08:00) China Standard Time (Asia/Shanghai)","validFor":null,"value":"Asia/Shanghai"},{"active":true,"defaultValue":false,"label":"(GMT+08:00) Singapore Standard Time (Asia/Singapore)","validFor":null,"value":"Asia/Singapore"},{"active":true,"defaultValue":false,"label":"(GMT+08:00) Taipei Standard Time (Asia/Taipei)","validFor":null,"value":"Asia/Taipei"},{"active":true,"defaultValue":false,"label":"(GMT+08:00) Ulaanbaatar Standard Time (Asia/Ulaanbaatar)","validFor":null,"value":"Asia/Ulaanbaatar"},{"active":true,"defaultValue":false,"label":"(GMT+08:00) Australian Western Standard Time (Australia/Perth)","validFor":null,"value":"Australia/Perth"},{"active":true,"defaultValue":false,"label":"(GMT+07:00) Davis Time (Antarctica/Davis)","validFor":null,"value":"Antarctica/Davis"},{"active":true,"defaultValue":false,"label":"(GMT+07:00) Indochina Time (Asia/Bangkok)","validFor":null,"value":"Asia/Bangkok"},{"active":true,"defaultValue":false,"label":"(GMT+07:00) Moscow Standard Time + 4 (Asia/Barnaul)","validFor":null,"value":"Asia/Barnaul"},{"active":true,"defaultValue":false,"label":"(GMT+07:00) Indochina Time (Asia/Ho_Chi_Minh)","validFor":null,"value":"Asia/Ho_Chi_Minh"},{"active":true,"defaultValue":false,"label":"(GMT+07:00) Hovd Standard Time (Asia/Hovd)","validFor":null,"value":"Asia/Hovd"},{"active":true,"defaultValue":false,"label":"(GMT+07:00) Western Indonesia Time (Asia/Jakarta)","validFor":null,"value":"Asia/Jakarta"},{"active":true,"defaultValue":false,"label":"(GMT+07:00) Krasnoyarsk Standard Time (Asia/Krasnoyarsk)","validFor":null,"value":"Asia/Krasnoyarsk"},{"active":true,"defaultValue":false,"label":"(GMT+07:00) Krasnoyarsk Standard Time (Asia/Novokuznetsk)","validFor":null,"value":"Asia/Novokuznetsk"},{"active":true,"defaultValue":false,"label":"(GMT+07:00) Novosibirsk Standard Time (Asia/Novosibirsk)","validFor":null,"value":"Asia/Novosibirsk"},{"active":true,"defaultValue":false,"label":"(GMT+07:00) Indochina Time (Asia/Phnom_Penh)","validFor":null,"value":"Asia/Phnom_Penh"},{"active":true,"defaultValue":false,"label":"(GMT+07:00) Western Indonesia Time (Asia/Pontianak)","validFor":null,"value":"Asia/Pontianak"},{"active":true,"defaultValue":false,"label":"(GMT+07:00) Moscow Standard Time + 4 (Asia/Tomsk)","validFor":null,"value":"Asia/Tomsk"},{"active":true,"defaultValue":false,"label":"(GMT+07:00) Indochina Time (Asia/Vientiane)","validFor":null,"value":"Asia/Vientiane"},{"active":true,"defaultValue":false,"label":"(GMT+07:00) Christmas Island Time (Indian/Christmas)","validFor":null,"value":"Indian/Christmas"},{"active":true,"defaultValue":false,"label":"(GMT+06:30) Myanmar Time (Asia/Rangoon)","validFor":null,"value":"Asia/Rangoon"},{"active":true,"defaultValue":false,"label":"(GMT+06:30) Cocos Islands Time (Indian/Cocos)","validFor":null,"value":"Indian/Cocos"},{"active":true,"defaultValue":false,"label":"(GMT+06:00) Vostok Time (Antarctica/Vostok)","validFor":null,"value":"Antarctica/Vostok"},{"active":true,"defaultValue":false,"label":"(GMT+06:00) East Kazakhstan Time (Asia/Almaty)","validFor":null,"value":"Asia/Almaty"},{"active":true,"defaultValue":false,"label":"(GMT+06:00) Kyrgyzstan Time (Asia/Bishkek)","validFor":null,"value":"Asia/Bishkek"},{"active":true,"defaultValue":false,"label":"(GMT+06:00) Bangladesh Standard Time (Asia/Dhaka)","validFor":null,"value":"Asia/Dhaka"},{"active":true,"defaultValue":false,"label":"(GMT+06:00) Omsk Standard Time (Asia/Omsk)","validFor":null,"value":"Asia/Omsk"},{"active":true,"defaultValue":false,"label":"(GMT+06:00) East Kazakhstan Time (Asia/Qostanay)","validFor":null,"value":"Asia/Qostanay"},{"active":true,"defaultValue":false,"label":"(GMT+06:00) Bhutan Time (Asia/Thimphu)","validFor":null,"value":"Asia/Thimphu"},{"active":true,"defaultValue":false,"label":"(GMT+06:00) China Standard Time (Asia/Urumqi)","validFor":null,"value":"Asia/Urumqi"},{"active":true,"defaultValue":false,"label":"(GMT+06:00) Indian Ocean Time (Indian/Chagos)","validFor":null,"value":"Indian/Chagos"},{"active":true,"defaultValue":false,"label":"(GMT+05:45) Nepal Time (Asia/Kathmandu)","validFor":null,"value":"Asia/Kathmandu"},{"active":true,"defaultValue":false,"label":"(GMT+05:30) India Standard Time (Asia/Colombo)","validFor":null,"value":"Asia/Colombo"},{"active":true,"defaultValue":false,"label":"(GMT+05:30) India Standard Time (Asia/Kolkata)","validFor":null,"value":"Asia/Kolkata"},{"active":true,"defaultValue":false,"label":"(GMT+05:00) Mawson Time (Antarctica/Mawson)","validFor":null,"value":"Antarctica/Mawson"},{"active":true,"defaultValue":false,"label":"(GMT+05:00) West Kazakhstan Time (Asia/Aqtau)","validFor":null,"value":"Asia/Aqtau"},{"active":true,"defaultValue":false,"label":"(GMT+05:00) West Kazakhstan Time (Asia/Aqtobe)","validFor":null,"value":"Asia/Aqtobe"},{"active":true,"defaultValue":false,"label":"(GMT+05:00) Turkmenistan Standard Time (Asia/Ashgabat)","validFor":null,"value":"Asia/Ashgabat"},{"active":true,"defaultValue":false,"label":"(GMT+05:00) West Kazakhstan Time (Asia/Atyrau)","validFor":null,"value":"Asia/Atyrau"},{"active":true,"defaultValue":false,"label":"(GMT+05:00) Tajikistan Time (Asia/Dushanbe)","validFor":null,"value":"Asia/Dushanbe"},{"active":true,"defaultValue":false,"label":"(GMT+05:00) Pakistan Standard Time (Asia/Karachi)","validFor":null,"value":"Asia/Karachi"},{"active":true,"defaultValue":false,"label":"(GMT+05:00) West Kazakhstan Time (Asia/Oral)","validFor":null,"value":"Asia/Oral"},{"active":true,"defaultValue":false,"label":"(GMT+05:00) West Kazakhstan Time (Asia/Qyzylorda)","validFor":null,"value":"Asia/Qyzylorda"},{"active":true,"defaultValue":false,"label":"(GMT+05:00) Uzbekistan Standard Time (Asia/Samarkand)","validFor":null,"value":"Asia/Samarkand"},{"active":true,"defaultValue":false,"label":"(GMT+05:00) Uzbekistan Standard Time (Asia/Tashkent)","validFor":null,"value":"Asia/Tashkent"},{"active":true,"defaultValue":false,"label":"(GMT+05:00) Yekaterinburg Standard Time (Asia/Yekaterinburg)","validFor":null,"value":"Asia/Yekaterinburg"},{"active":true,"defaultValue":false,"label":"(GMT+05:00) French Southern & Antarctic Time (Indian/Kerguelen)","validFor":null,"value":"Indian/Kerguelen"},{"active":true,"defaultValue":false,"label":"(GMT+05:00) Maldives Time (Indian/Maldives)","validFor":null,"value":"Indian/Maldives"},{"active":true,"defaultValue":false,"label":"(GMT+04:30) Afghanistan Time (Asia/Kabul)","validFor":null,"value":"Asia/Kabul"},{"active":true,"defaultValue":false,"label":"(GMT+04:00) Azerbaijan Standard Time (Asia/Baku)","validFor":null,"value":"Asia/Baku"},{"active":true,"defaultValue":false,"label":"(GMT+04:00) Gulf Standard Time (Asia/Dubai)","validFor":null,"value":"Asia/Dubai"},{"active":true,"defaultValue":false,"label":"(GMT+04:00) Gulf Standard Time (Asia/Muscat)","validFor":null,"value":"Asia/Muscat"},{"active":true,"defaultValue":false,"label":"(GMT+04:00) Georgia Standard Time (Asia/Tbilisi)","validFor":null,"value":"Asia/Tbilisi"},{"active":true,"defaultValue":false,"label":"(GMT+04:00) Armenia Standard Time (Asia/Yerevan)","validFor":null,"value":"Asia/Yerevan"},{"active":true,"defaultValue":false,"label":"(GMT+04:00) Samara Standard Time (Europe/Astrakhan)","validFor":null,"value":"Europe/Astrakhan"},{"active":true,"defaultValue":false,"label":"(GMT+04:00) Samara Standard Time (Europe/Samara)","validFor":null,"value":"Europe/Samara"},{"active":true,"defaultValue":false,"label":"(GMT+04:00) Moscow Standard Time + 1 (Europe/Saratov)","validFor":null,"value":"Europe/Saratov"},{"active":true,"defaultValue":false,"label":"(GMT+04:00) Moscow Standard Time + 1 (Europe/Ulyanovsk)","validFor":null,"value":"Europe/Ulyanovsk"},{"active":true,"defaultValue":false,"label":"(GMT+04:00) Seychelles Time (Indian/Mahe)","validFor":null,"value":"Indian/Mahe"},{"active":true,"defaultValue":false,"label":"(GMT+04:00) Mauritius Standard Time (Indian/Mauritius)","validFor":null,"value":"Indian/Mauritius"},{"active":true,"defaultValue":false,"label":"(GMT+04:00) Réunion Time (Indian/Reunion)","validFor":null,"value":"Indian/Reunion"},{"active":true,"defaultValue":false,"label":"(GMT+03:00) East Africa Time (Africa/Addis_Ababa)","validFor":null,"value":"Africa/Addis_Ababa"},{"active":true,"defaultValue":false,"label":"(GMT+03:00) East Africa Time (Africa/Asmera)","validFor":null,"value":"Africa/Asmera"},{"active":true,"defaultValue":false,"label":"(GMT+03:00) East Africa Time (Africa/Dar_es_Salaam)","validFor":null,"value":"Africa/Dar_es_Salaam"},{"active":true,"defaultValue":false,"label":"(GMT+03:00) East Africa Time (Africa/Djibouti)","validFor":null,"value":"Africa/Djibouti"},{"active":true,"defaultValue":false,"label":"(GMT+03:00) East Africa Time (Africa/Kampala)","validFor":null,"value":"Africa/Kampala"},{"active":true,"defaultValue":false,"label":"(GMT+03:00) East Africa Time (Africa/Mogadishu)","validFor":null,"value":"Africa/Mogadishu"},{"active":true,"defaultValue":false,"label":"(GMT+03:00) East Africa Time (Africa/Nairobi)","validFor":null,"value":"Africa/Nairobi"},{"active":true,"defaultValue":false,"label":"(GMT+03:00) Syowa Time (Antarctica/Syowa)","validFor":null,"value":"Antarctica/Syowa"},{"active":true,"defaultValue":false,"label":"(GMT+03:00) Arabian Standard Time (Asia/Aden)","validFor":null,"value":"Asia/Aden"},{"active":true,"defaultValue":false,"label":"(GMT+03:00) Eastern European Standard Time (Asia/Amman)","validFor":null,"value":"Asia/Amman"},{"active":true,"defaultValue":false,"label":"(GMT+03:00) Arabian Standard Time (Asia/Baghdad)","validFor":null,"value":"Asia/Baghdad"},{"active":true,"defaultValue":false,"label":"(GMT+03:00) Arabian Standard Time (Asia/Bahrain)","validFor":null,"value":"Asia/Bahrain"},{"active":true,"defaultValue":false,"label":"(GMT+03:00) Arabian Standard Time (Asia/Kuwait)","validFor":null,"value":"Asia/Kuwait"},{"active":true,"defaultValue":false,"label":"(GMT+03:00) Arabian Standard Time (Asia/Qatar)","validFor":null,"value":"Asia/Qatar"},{"active":true,"defaultValue":false,"label":"(GMT+03:00) Arabian Standard Time (Asia/Riyadh)","validFor":null,"value":"Asia/Riyadh"},{"active":true,"defaultValue":false,"label":"(GMT+03:00) Eastern European Standard Time (Europe/Istanbul)","validFor":null,"value":"Europe/Istanbul"},{"active":true,"defaultValue":false,"label":"(GMT+03:00) Moscow Standard Time (Europe/Kirov)","validFor":null,"value":"Europe/Kirov"},{"active":true,"defaultValue":false,"label":"(GMT+03:00) Moscow Standard Time (Europe/Minsk)","validFor":null,"value":"Europe/Minsk"},{"active":true,"defaultValue":false,"label":"(GMT+03:00) Moscow Standard Time (Europe/Moscow)","validFor":null,"value":"Europe/Moscow"},{"active":true,"defaultValue":false,"label":"(GMT+03:00) Volgograd Standard Time (Europe/Volgograd)","validFor":null,"value":"Europe/Volgograd"},{"active":true,"defaultValue":false,"label":"(GMT+03:00) East Africa Time (Indian/Antananarivo)","validFor":null,"value":"Indian/Antananarivo"},{"active":true,"defaultValue":false,"label":"(GMT+03:00) East Africa Time (Indian/Comoro)","validFor":null,"value":"Indian/Comoro"},{"active":true,"defaultValue":false,"label":"(GMT+03:00) East Africa Time (Indian/Mayotte)","validFor":null,"value":"Indian/Mayotte"},{"active":true,"defaultValue":false,"label":"(GMT+02:00) Central Africa Time (Africa/Blantyre)","validFor":null,"value":"Africa/Blantyre"},{"active":true,"defaultValue":false,"label":"(GMT+02:00) Central Africa Time (Africa/Bujumbura)","validFor":null,"value":"Africa/Bujumbura"},{"active":true,"defaultValue":false,"label":"(GMT+02:00) Eastern European Standard Time (Africa/Cairo)","validFor":null,"value":"Africa/Cairo"},{"active":true,"defaultValue":false,"label":"(GMT+02:00) Central Africa Time (Africa/Gaborone)","validFor":null,"value":"Africa/Gaborone"},{"active":true,"defaultValue":false,"label":"(GMT+02:00) Central Africa Time (Africa/Harare)","validFor":null,"value":"Africa/Harare"},{"active":true,"defaultValue":false,"label":"(GMT+02:00) South Africa Standard Time (Africa/Johannesburg)","validFor":null,"value":"Africa/Johannesburg"},{"active":true,"defaultValue":false,"label":"(GMT+02:00) Central Africa Time (Africa/Juba)","validFor":null,"value":"Africa/Juba"},{"active":true,"defaultValue":false,"label":"(GMT+02:00) Central Africa Time (Africa/Kigali)","validFor":null,"value":"Africa/Kigali"},{"active":true,"defaultValue":false,"label":"(GMT+02:00) Central Africa Time (Africa/Lubumbashi)","validFor":null,"value":"Africa/Lubumbashi"},{"active":true,"defaultValue":false,"label":"(GMT+02:00) Central Africa Time (Africa/Lusaka)","validFor":null,"value":"Africa/Lusaka"},{"active":true,"defaultValue":false,"label":"(GMT+02:00) Central Africa Time (Africa/Maputo)","validFor":null,"value":"Africa/Maputo"},{"active":true,"defaultValue":false,"label":"(GMT+02:00) South Africa Standard Time (Africa/Maseru)","validFor":null,"value":"Africa/Maseru"},{"active":true,"defaultValue":false,"label":"(GMT+02:00) South Africa Standard Time (Africa/Mbabane)","validFor":null,"value":"Africa/Mbabane"},{"active":true,"defaultValue":false,"label":"(GMT+02:00) Eastern European Standard Time (Africa/Tripoli)","validFor":null,"value":"Africa/Tripoli"},{"active":true,"defaultValue":false,"label":"(GMT+02:00) Central Africa Time (Africa/Windhoek)","validFor":null,"value":"Africa/Windhoek"},{"active":true,"defaultValue":false,"label":"(GMT+02:00) Eastern European Standard Time (Asia/Beirut)","validFor":null,"value":"Asia/Beirut"},{"active":true,"defaultValue":false,"label":"(GMT+02:00) Eastern European Standard Time (Asia/Famagusta)","validFor":null,"value":"Asia/Famagusta"},{"active":true,"defaultValue":false,"label":"(GMT+02:00) Eastern European Standard Time (Asia/Gaza)","validFor":null,"value":"Asia/Gaza"},{"active":true,"defaultValue":false,"label":"(GMT+02:00) Eastern European Standard Time (Asia/Hebron)","validFor":null,"value":"Asia/Hebron"},{"active":true,"defaultValue":false,"label":"(GMT+02:00) Israel Standard Time (Asia/Jerusalem)","validFor":null,"value":"Asia/Jerusalem"},{"active":true,"defaultValue":false,"label":"(GMT+02:00) Eastern European Standard Time (Asia/Nicosia)","validFor":null,"value":"Asia/Nicosia"},{"active":true,"defaultValue":false,"label":"(GMT+02:00) Eastern European Standard Time (Europe/Athens)","validFor":null,"value":"Europe/Athens"},{"active":true,"defaultValue":false,"label":"(GMT+02:00) Eastern European Standard Time (Europe/Bucharest)","validFor":null,"value":"Europe/Bucharest"},{"active":true,"defaultValue":false,"label":"(GMT+02:00) Eastern European Standard Time (Europe/Chisinau)","validFor":null,"value":"Europe/Chisinau"},{"active":true,"defaultValue":false,"label":"(GMT+02:00) Eastern European Standard Time (Europe/Helsinki)","validFor":null,"value":"Europe/Helsinki"},{"active":true,"defaultValue":false,"label":"(GMT+02:00) Eastern European Standard Time (Europe/Kaliningrad)","validFor":null,"value":"Europe/Kaliningrad"},{"active":true,"defaultValue":false,"label":"(GMT+02:00) Eastern European Standard Time (Europe/Kyiv)","validFor":null,"value":"Europe/Kiev"},{"active":true,"defaultValue":false,"label":"(GMT+02:00) Eastern European Standard Time (Europe/Mariehamn)","validFor":null,"value":"Europe/Mariehamn"},{"active":true,"defaultValue":false,"label":"(GMT+02:00) Eastern European Standard Time (Europe/Riga)","validFor":null,"value":"Europe/Riga"},{"active":true,"defaultValue":false,"label":"(GMT+02:00) Eastern European Standard Time (Europe/Sofia)","validFor":null,"value":"Europe/Sofia"},{"active":true,"defaultValue":false,"label":"(GMT+02:00) Eastern European Standard Time (Europe/Tallinn)","validFor":null,"value":"Europe/Tallinn"},{"active":true,"defaultValue":false,"label":"(GMT+02:00) Eastern European Standard Time (Europe/Uzhgorod)","validFor":null,"value":"Europe/Uzhgorod"},{"active":true,"defaultValue":false,"label":"(GMT+02:00) Eastern European Standard Time (Europe/Vilnius)","validFor":null,"value":"Europe/Vilnius"},{"active":true,"defaultValue":false,"label":"(GMT+02:00) Eastern European Standard Time (Europe/Zaporozhye)","validFor":null,"value":"Europe/Zaporozhye"},{"active":true,"defaultValue":false,"label":"(GMT+01:00) Central European Standard Time (Africa/Algiers)","validFor":null,"value":"Africa/Algiers"},{"active":true,"defaultValue":false,"label":"(GMT+01:00) West Africa Standard Time (Africa/Bangui)","validFor":null,"value":"Africa/Bangui"},{"active":true,"defaultValue":false,"label":"(GMT+01:00) West Africa Standard Time (Africa/Brazzaville)","validFor":null,"value":"Africa/Brazzaville"},{"active":true,"defaultValue":false,"label":"(GMT+01:00) Western European Summer Time (Africa/Casablanca)","validFor":null,"value":"Africa/Casablanca"},{"active":true,"defaultValue":false,"label":"(GMT+01:00) Central European Standard Time (Africa/Ceuta)","validFor":null,"value":"Africa/Ceuta"},{"active":true,"defaultValue":false,"label":"(GMT+01:00) West Africa Standard Time (Africa/Douala)","validFor":null,"value":"Africa/Douala"},{"active":true,"defaultValue":false,"label":"(GMT+01:00) Western European Summer Time (Africa/El_Aaiun)","validFor":null,"value":"Africa/El_Aaiun"},{"active":true,"defaultValue":false,"label":"(GMT+01:00) West Africa Standard Time (Africa/Kinshasa)","validFor":null,"value":"Africa/Kinshasa"},{"active":true,"defaultValue":false,"label":"(GMT+01:00) West Africa Standard Time (Africa/Lagos)","validFor":null,"value":"Africa/Lagos"},{"active":true,"defaultValue":false,"label":"(GMT+01:00) West Africa Standard Time (Africa/Libreville)","validFor":null,"value":"Africa/Libreville"},{"active":true,"defaultValue":false,"label":"(GMT+01:00) West Africa Standard Time (Africa/Luanda)","validFor":null,"value":"Africa/Luanda"},{"active":true,"defaultValue":false,"label":"(GMT+01:00) West Africa Standard Time (Africa/Malabo)","validFor":null,"value":"Africa/Malabo"},{"active":true,"defaultValue":false,"label":"(GMT+01:00) West Africa Standard Time (Africa/Ndjamena)","validFor":null,"value":"Africa/Ndjamena"},{"active":true,"defaultValue":false,"label":"(GMT+01:00) West Africa Standard Time (Africa/Niamey)","validFor":null,"value":"Africa/Niamey"},{"active":true,"defaultValue":false,"label":"(GMT+01:00) West Africa Standard Time (Africa/Porto-Novo)","validFor":null,"value":"Africa/Porto-Novo"},{"active":true,"defaultValue":false,"label":"(GMT+01:00) Central European Standard Time (Africa/Tunis)","validFor":null,"value":"Africa/Tunis"},{"active":true,"defaultValue":false,"label":"(GMT+01:00) Central European Standard Time (Arctic/Longyearbyen)","validFor":null,"value":"Arctic/Longyearbyen"},{"active":true,"defaultValue":false,"label":"(GMT+01:00) Central European Standard Time (Europe/Amsterdam)","validFor":null,"value":"Europe/Amsterdam"},{"active":true,"defaultValue":false,"label":"(GMT+01:00) Central European Standard Time (Europe/Andorra)","validFor":null,"value":"Europe/Andorra"},{"active":true,"defaultValue":false,"label":"(GMT+01:00) Central European Standard Time (Europe/Belgrade)","validFor":null,"value":"Europe/Belgrade"},{"active":true,"defaultValue":false,"label":"(GMT+01:00) Central European Standard Time (Europe/Berlin)","validFor":null,"value":"Europe/Berlin"},{"active":true,"defaultValue":false,"label":"(GMT+01:00) Central European Standard Time (Europe/Bratislava)","validFor":null,"value":"Europe/Bratislava"},{"active":true,"defaultValue":false,"label":"(GMT+01:00) Central European Standard Time (Europe/Brussels)","validFor":null,"value":"Europe/Brussels"},{"active":true,"defaultValue":false,"label":"(GMT+01:00) Central European Standard Time (Europe/Budapest)","validFor":null,"value":"Europe/Budapest"},{"active":true,"defaultValue":false,"label":"(GMT+01:00) Central European Standard Time (Europe/Busingen)","validFor":null,"value":"Europe/Busingen"},{"active":true,"defaultValue":false,"label":"(GMT+01:00) Central European Standard Time (Europe/Copenhagen)","validFor":null,"value":"Europe/Copenhagen"},{"active":true,"defaultValue":false,"label":"(GMT+01:00) Central European Standard Time (Europe/Gibraltar)","validFor":null,"value":"Europe/Gibraltar"},{"active":true,"defaultValue":false,"label":"(GMT+01:00) Central European Standard Time (Europe/Ljubljana)","validFor":null,"value":"Europe/Ljubljana"},{"active":true,"defaultValue":false,"label":"(GMT+01:00) Central European Standard Time (Europe/Luxembourg)","validFor":null,"value":"Europe/Luxembourg"},{"active":true,"defaultValue":false,"label":"(GMT+01:00) Central European Standard Time (Europe/Madrid)","validFor":null,"value":"Europe/Madrid"},{"active":true,"defaultValue":false,"label":"(GMT+01:00) Central European Standard Time (Europe/Malta)","validFor":null,"value":"Europe/Malta"},{"active":true,"defaultValue":false,"label":"(GMT+01:00) Central European Standard Time (Europe/Monaco)","validFor":null,"value":"Europe/Monaco"},{"active":true,"defaultValue":false,"label":"(GMT+01:00) Central European Standard Time (Europe/Oslo)","validFor":null,"value":"Europe/Oslo"},{"active":true,"defaultValue":false,"label":"(GMT+01:00) Central European Standard Time (Europe/Paris)","validFor":null,"value":"Europe/Paris"},{"active":true,"defaultValue":false,"label":"(GMT+01:00) Central European Standard Time (Europe/Podgorica)","validFor":null,"value":"Europe/Podgorica"},{"active":true,"defaultValue":false,"label":"(GMT+01:00) Central European Standard Time (Europe/Prague)","validFor":null,"value":"Europe/Prague"},{"active":true,"defaultValue":false,"label":"(GMT+01:00) Central European Standard Time (Europe/Rome)","validFor":null,"value":"Europe/Rome"},{"active":true,"defaultValue":false,"label":"(GMT+01:00) Central European Standard Time (Europe/San_Marino)","validFor":null,"value":"Europe/San_Marino"},{"active":true,"defaultValue":false,"label":"(GMT+01:00) Central European Standard Time (Europe/Sarajevo)","validFor":null,"value":"Europe/Sarajevo"},{"active":true,"defaultValue":false,"label":"(GMT+01:00) Central European Standard Time (Europe/Skopje)","validFor":null,"value":"Europe/Skopje"},{"active":true,"defaultValue":false,"label":"(GMT+01:00) Central European Standard Time (Europe/Stockholm)","validFor":null,"value":"Europe/Stockholm"},{"active":true,"defaultValue":false,"label":"(GMT+01:00) Central European Standard Time (Europe/Tirane)","validFor":null,"value":"Europe/Tirane"},{"active":true,"defaultValue":false,"label":"(GMT+01:00) Central European Standard Time (Europe/Vaduz)","validFor":null,"value":"Europe/Vaduz"},{"active":true,"defaultValue":false,"label":"(GMT+01:00) Central European Standard Time (Europe/Vatican)","validFor":null,"value":"Europe/Vatican"},{"active":true,"defaultValue":false,"label":"(GMT+01:00) Central European Standard Time (Europe/Vienna)","validFor":null,"value":"Europe/Vienna"},{"active":true,"defaultValue":false,"label":"(GMT+01:00) Central European Standard Time (Europe/Warsaw)","validFor":null,"value":"Europe/Warsaw"},{"active":true,"defaultValue":false,"label":"(GMT+01:00) Central European Standard Time (Europe/Zagreb)","validFor":null,"value":"Europe/Zagreb"},{"active":true,"defaultValue":false,"label":"(GMT+01:00) Central European Standard Time (Europe/Zurich)","validFor":null,"value":"Europe/Zurich"},{"active":true,"defaultValue":false,"label":"(GMT+00:00) Greenwich Mean Time (Africa/Abidjan)","validFor":null,"value":"Africa/Abidjan"},{"active":true,"defaultValue":false,"label":"(GMT+00:00) Greenwich Mean Time (Africa/Accra)","validFor":null,"value":"Africa/Accra"},{"active":true,"defaultValue":false,"label":"(GMT+00:00) Greenwich Mean Time (Africa/Bamako)","validFor":null,"value":"Africa/Bamako"},{"active":true,"defaultValue":false,"label":"(GMT+00:00) Greenwich Mean Time (Africa/Banjul)","validFor":null,"value":"Africa/Banjul"},{"active":true,"defaultValue":false,"label":"(GMT+00:00) Greenwich Mean Time (Africa/Bissau)","validFor":null,"value":"Africa/Bissau"},{"active":true,"defaultValue":false,"label":"(GMT+00:00) Greenwich Mean Time (Africa/Conakry)","validFor":null,"value":"Africa/Conakry"},{"active":true,"defaultValue":false,"label":"(GMT+00:00) Greenwich Mean Time (Africa/Dakar)","validFor":null,"value":"Africa/Dakar"},{"active":true,"defaultValue":false,"label":"(GMT+00:00) Greenwich Mean Time (Africa/Freetown)","validFor":null,"value":"Africa/Freetown"},{"active":true,"defaultValue":false,"label":"(GMT+00:00) Greenwich Mean Time (Africa/Lome)","validFor":null,"value":"Africa/Lome"},{"active":true,"defaultValue":false,"label":"(GMT+00:00) Greenwich Mean Time (Africa/Monrovia)","validFor":null,"value":"Africa/Monrovia"},{"active":true,"defaultValue":false,"label":"(GMT+00:00) Greenwich Mean Time (Africa/Nouakchott)","validFor":null,"value":"Africa/Nouakchott"},{"active":true,"defaultValue":false,"label":"(GMT+00:00) Greenwich Mean Time (Africa/Ouagadougou)","validFor":null,"value":"Africa/Ouagadougou"},{"active":true,"defaultValue":false,"label":"(GMT+00:00) Greenwich Mean Time (Africa/Sao_Tome)","validFor":null,"value":"Africa/Sao_Tome"},{"active":true,"defaultValue":false,"label":"(GMT+00:00) Greenwich Mean Time (America/Danmarkshavn)","validFor":null,"value":"America/Danmarkshavn"},{"active":true,"defaultValue":false,"label":"(GMT+00:00) Greenwich Mean Time (Antarctica/Troll)","validFor":null,"value":"Antarctica/Troll"},{"active":true,"defaultValue":false,"label":"(GMT+00:00) Western European Standard Time (Atlantic/Canary)","validFor":null,"value":"Atlantic/Canary"},{"active":true,"defaultValue":false,"label":"(GMT+00:00) Western European Standard Time (Atlantic/Faeroe)","validFor":null,"value":"Atlantic/Faeroe"},{"active":true,"defaultValue":false,"label":"(GMT+00:00) Western European Standard Time (Atlantic/Madeira)","validFor":null,"value":"Atlantic/Madeira"},{"active":true,"defaultValue":false,"label":"(GMT+00:00) Greenwich Mean Time (Atlantic/Reykjavik)","validFor":null,"value":"Atlantic/Reykjavik"},{"active":true,"defaultValue":false,"label":"(GMT+00:00) Greenwich Mean Time (Atlantic/St_Helena)","validFor":null,"value":"Atlantic/St_Helena"},{"active":true,"defaultValue":false,"label":"(GMT+00:00) Greenwich Mean Time (Europe/Dublin)","validFor":null,"value":"Europe/Dublin"},{"active":true,"defaultValue":false,"label":"(GMT+00:00) Greenwich Mean Time (Europe/Guernsey)","validFor":null,"value":"Europe/Guernsey"},{"active":true,"defaultValue":false,"label":"(GMT+00:00) Greenwich Mean Time (Europe/Isle_of_Man)","validFor":null,"value":"Europe/Isle_of_Man"},{"active":true,"defaultValue":false,"label":"(GMT+00:00) Greenwich Mean Time (Europe/Jersey)","validFor":null,"value":"Europe/Jersey"},{"active":true,"defaultValue":false,"label":"(GMT+00:00) Western European Standard Time (Europe/Lisbon)","validFor":null,"value":"Europe/Lisbon"},{"active":true,"defaultValue":false,"label":"(GMT+00:00) Greenwich Mean Time (Europe/London)","validFor":null,"value":"Europe/London"},{"active":true,"defaultValue":false,"label":"(GMT+00:00) Greenwich Mean Time (GMT)","validFor":null,"value":"GMT"},{"active":true,"defaultValue":false,"label":"(GMT-01:00) East Greenland Standard Time (America/Scoresbysund)","validFor":null,"value":"America/Scoresbysund"},{"active":true,"defaultValue":false,"label":"(GMT-01:00) Azores Standard Time (Atlantic/Azores)","validFor":null,"value":"Atlantic/Azores"},{"active":true,"defaultValue":false,"label":"(GMT-01:00) Cape Verde Standard Time (Atlantic/Cape_Verde)","validFor":null,"value":"Atlantic/Cape_Verde"},{"active":true,"defaultValue":false,"label":"(GMT-02:00) West Greenland Standard Time (America/Godthab)","validFor":null,"value":"America/Godthab"},{"active":true,"defaultValue":false,"label":"(GMT-02:00) Fernando de Noronha Standard Time (America/Noronha)","validFor":null,"value":"America/Noronha"},{"active":true,"defaultValue":false,"label":"(GMT-02:00) South Georgia Time (Atlantic/South_Georgia)","validFor":null,"value":"Atlantic/South_Georgia"},{"active":true,"defaultValue":false,"label":"(GMT-03:00) Brasilia Standard Time (America/Araguaina)","validFor":null,"value":"America/Araguaina"},{"active":true,"defaultValue":false,"label":"(GMT-03:00) Argentina Standard Time (America/Argentina/Buenos_Aires)","validFor":null,"value":"America/Argentina/Buenos_Aires"},{"active":true,"defaultValue":false,"label":"(GMT-03:00) Argentina Standard Time (America/Argentina/La_Rioja)","validFor":null,"value":"America/Argentina/La_Rioja"},{"active":true,"defaultValue":false,"label":"(GMT-03:00) Argentina Standard Time (America/Argentina/Rio_Gallegos)","validFor":null,"value":"America/Argentina/Rio_Gallegos"},{"active":true,"defaultValue":false,"label":"(GMT-03:00) Argentina Standard Time (America/Argentina/Salta)","validFor":null,"value":"America/Argentina/Salta"},{"active":true,"defaultValue":false,"label":"(GMT-03:00) Argentina Standard Time (America/Argentina/San_Juan)","validFor":null,"value":"America/Argentina/San_Juan"},{"active":true,"defaultValue":false,"label":"(GMT-03:00) Argentina Standard Time (America/Argentina/San_Luis)","validFor":null,"value":"America/Argentina/San_Luis"},{"active":true,"defaultValue":false,"label":"(GMT-03:00) Argentina Standard Time (America/Argentina/Tucuman)","validFor":null,"value":"America/Argentina/Tucuman"},{"active":true,"defaultValue":false,"label":"(GMT-03:00) Argentina Standard Time (America/Argentina/Ushuaia)","validFor":null,"value":"America/Argentina/Ushuaia"},{"active":true,"defaultValue":false,"label":"(GMT-03:00) Paraguay Summer Time (America/Asuncion)","validFor":null,"value":"America/Asuncion"},{"active":true,"defaultValue":false,"label":"(GMT-03:00) Brasilia Standard Time (America/Bahia)","validFor":null,"value":"America/Bahia"},{"active":true,"defaultValue":false,"label":"(GMT-03:00) Brasilia Standard Time (America/Belem)","validFor":null,"value":"America/Belem"},{"active":true,"defaultValue":false,"label":"(GMT-03:00) Argentina Standard Time (America/Catamarca)","validFor":null,"value":"America/Catamarca"},{"active":true,"defaultValue":false,"label":"(GMT-03:00) French Guiana Time (America/Cayenne)","validFor":null,"value":"America/Cayenne"},{"active":true,"defaultValue":false,"label":"(GMT-03:00) Argentina Standard Time (America/Cordoba)","validFor":null,"value":"America/Cordoba"},{"active":true,"defaultValue":false,"label":"(GMT-03:00) Brasilia Standard Time (America/Fortaleza)","validFor":null,"value":"America/Fortaleza"},{"active":true,"defaultValue":false,"label":"(GMT-03:00) Argentina Standard Time (America/Jujuy)","validFor":null,"value":"America/Jujuy"},{"active":true,"defaultValue":false,"label":"(GMT-03:00) Brasilia Standard Time (America/Maceio)","validFor":null,"value":"America/Maceio"},{"active":true,"defaultValue":false,"label":"(GMT-03:00) Argentina Standard Time (America/Mendoza)","validFor":null,"value":"America/Mendoza"},{"active":true,"defaultValue":false,"label":"(GMT-03:00) St Pierre & Miquelon Standard Time (America/Miquelon)","validFor":null,"value":"America/Miquelon"},{"active":true,"defaultValue":false,"label":"(GMT-03:00) Uruguay Standard Time (America/Montevideo)","validFor":null,"value":"America/Montevideo"},{"active":true,"defaultValue":false,"label":"(GMT-03:00) Suriname Time (America/Paramaribo)","validFor":null,"value":"America/Paramaribo"},{"active":true,"defaultValue":false,"label":"(GMT-03:00) Chile Standard Time (America/Punta_Arenas)","validFor":null,"value":"America/Punta_Arenas"},{"active":true,"defaultValue":false,"label":"(GMT-03:00) Brasilia Standard Time (America/Recife)","validFor":null,"value":"America/Recife"},{"active":true,"defaultValue":false,"label":"(GMT-03:00) Brasilia Standard Time (America/Santarem)","validFor":null,"value":"America/Santarem"},{"active":true,"defaultValue":false,"label":"(GMT-03:00) Chile Summer Time (America/Santiago)","validFor":null,"value":"America/Santiago"},{"active":true,"defaultValue":false,"label":"(GMT-03:00) Brasilia Standard Time (America/Sao_Paulo)","validFor":null,"value":"America/Sao_Paulo"},{"active":true,"defaultValue":false,"label":"(GMT-03:00) Chile Standard Time (Antarctica/Palmer)","validFor":null,"value":"Antarctica/Palmer"},{"active":true,"defaultValue":false,"label":"(GMT-03:00) Rothera Time (Antarctica/Rothera)","validFor":null,"value":"Antarctica/Rothera"},{"active":true,"defaultValue":false,"label":"(GMT-03:00) Falkland Islands Standard Time (Atlantic/Stanley)","validFor":null,"value":"Atlantic/Stanley"},{"active":true,"defaultValue":false,"label":"(GMT-03:30) Newfoundland Standard Time (America/St_Johns)","validFor":null,"value":"America/St_Johns"},{"active":true,"defaultValue":false,"label":"(GMT-04:00) Atlantic Standard Time (America/Anguilla)","validFor":null,"value":"America/Anguilla"},{"active":true,"defaultValue":false,"label":"(GMT-04:00) Atlantic Standard Time (America/Antigua)","validFor":null,"value":"America/Antigua"},{"active":true,"defaultValue":false,"label":"(GMT-04:00) Atlantic Standard Time (America/Aruba)","validFor":null,"value":"America/Aruba"},{"active":true,"defaultValue":false,"label":"(GMT-04:00) Atlantic Standard Time (America/Barbados)","validFor":null,"value":"America/Barbados"},{"active":true,"defaultValue":false,"label":"(GMT-04:00) Atlantic Standard Time (America/Blanc-Sablon)","validFor":null,"value":"America/Blanc-Sablon"},{"active":true,"defaultValue":false,"label":"(GMT-04:00) Amazon Standard Time (America/Boa_Vista)","validFor":null,"value":"America/Boa_Vista"},{"active":true,"defaultValue":false,"label":"(GMT-04:00) Amazon Standard Time (America/Campo_Grande)","validFor":null,"value":"America/Campo_Grande"},{"active":true,"defaultValue":false,"label":"(GMT-04:00) Venezuela Time (America/Caracas)","validFor":null,"value":"America/Caracas"},{"active":true,"defaultValue":false,"label":"(GMT-04:00) Amazon Standard Time (America/Cuiaba)","validFor":null,"value":"America/Cuiaba"},{"active":true,"defaultValue":false,"label":"(GMT-04:00) Atlantic Standard Time (America/Curacao)","validFor":null,"value":"America/Curacao"},{"active":true,"defaultValue":false,"label":"(GMT-04:00) Atlantic Standard Time (America/Dominica)","validFor":null,"value":"America/Dominica"},{"active":true,"defaultValue":false,"label":"(GMT-04:00) Atlantic Standard Time (America/Glace_Bay)","validFor":null,"value":"America/Glace_Bay"},{"active":true,"defaultValue":false,"label":"(GMT-04:00) Atlantic Standard Time (America/Goose_Bay)","validFor":null,"value":"America/Goose_Bay"},{"active":true,"defaultValue":false,"label":"(GMT-04:00) Atlantic Standard Time (America/Grenada)","validFor":null,"value":"America/Grenada"},{"active":true,"defaultValue":false,"label":"(GMT-04:00) Atlantic Standard Time (America/Guadeloupe)","validFor":null,"value":"America/Guadeloupe"},{"active":true,"defaultValue":false,"label":"(GMT-04:00) Guyana Time (America/Guyana)","validFor":null,"value":"America/Guyana"},{"active":true,"defaultValue":false,"label":"(GMT-04:00) Atlantic Standard Time (America/Halifax)","validFor":null,"value":"America/Halifax"},{"active":true,"defaultValue":false,"label":"(GMT-04:00) Atlantic Standard Time (America/Kralendijk)","validFor":null,"value":"America/Kralendijk"},{"active":true,"defaultValue":false,"label":"(GMT-04:00) Bolivia Time (America/La_Paz)","validFor":null,"value":"America/La_Paz"},{"active":true,"defaultValue":false,"label":"(GMT-04:00) Atlantic Standard Time (America/Lower_Princes)","validFor":null,"value":"America/Lower_Princes"},{"active":true,"defaultValue":false,"label":"(GMT-04:00) Amazon Standard Time (America/Manaus)","validFor":null,"value":"America/Manaus"},{"active":true,"defaultValue":false,"label":"(GMT-04:00) Atlantic Standard Time (America/Marigot)","validFor":null,"value":"America/Marigot"},{"active":true,"defaultValue":false,"label":"(GMT-04:00) Atlantic Standard Time (America/Martinique)","validFor":null,"value":"America/Martinique"},{"active":true,"defaultValue":false,"label":"(GMT-04:00) Atlantic Standard Time (America/Moncton)","validFor":null,"value":"America/Moncton"},{"active":true,"defaultValue":false,"label":"(GMT-04:00) Atlantic Standard Time (America/Montserrat)","validFor":null,"value":"America/Montserrat"},{"active":true,"defaultValue":false,"label":"(GMT-04:00) Atlantic Standard Time (America/Port_of_Spain)","validFor":null,"value":"America/Port_of_Spain"},{"active":true,"defaultValue":false,"label":"(GMT-04:00) Amazon Standard Time (America/Porto_Velho)","validFor":null,"value":"America/Porto_Velho"},{"active":true,"defaultValue":false,"label":"(GMT-04:00) Atlantic Standard Time (America/Puerto_Rico)","validFor":null,"value":"America/Puerto_Rico"},{"active":true,"defaultValue":false,"label":"(GMT-04:00) Atlantic Standard Time (America/Santo_Domingo)","validFor":null,"value":"America/Santo_Domingo"},{"active":true,"defaultValue":false,"label":"(GMT-04:00) Atlantic Standard Time (America/St_Barthelemy)","validFor":null,"value":"America/St_Barthelemy"},{"active":true,"defaultValue":false,"label":"(GMT-04:00) Atlantic Standard Time (America/St_Kitts)","validFor":null,"value":"America/St_Kitts"},{"active":true,"defaultValue":false,"label":"(GMT-04:00) Atlantic Standard Time (America/St_Lucia)","validFor":null,"value":"America/St_Lucia"},{"active":true,"defaultValue":false,"label":"(GMT-04:00) Atlantic Standard Time (America/St_Thomas)","validFor":null,"value":"America/St_Thomas"},{"active":true,"defaultValue":false,"label":"(GMT-04:00) Atlantic Standard Time (America/St_Vincent)","validFor":null,"value":"America/St_Vincent"},{"active":true,"defaultValue":false,"label":"(GMT-04:00) Atlantic Standard Time (America/Thule)","validFor":null,"value":"America/Thule"},{"active":true,"defaultValue":false,"label":"(GMT-04:00) Atlantic Standard Time (America/Tortola)","validFor":null,"value":"America/Tortola"},{"active":true,"defaultValue":false,"label":"(GMT-04:00) Atlantic Standard Time (Atlantic/Bermuda)","validFor":null,"value":"Atlantic/Bermuda"},{"active":true,"defaultValue":false,"label":"(GMT-05:00) Colombia Standard Time (America/Bogota)","validFor":null,"value":"America/Bogota"},{"active":true,"defaultValue":false,"label":"(GMT-05:00) Eastern Standard Time (America/Cancun)","validFor":null,"value":"America/Cancun"},{"active":true,"defaultValue":false,"label":"(GMT-05:00) Eastern Standard Time (America/Cayman)","validFor":null,"value":"America/Cayman"},{"active":true,"defaultValue":false,"label":"(GMT-05:00) Eastern Standard Time (America/Coral_Harbour)","validFor":null,"value":"America/Coral_Harbour"},{"active":true,"defaultValue":false,"label":"(GMT-05:00) Eastern Standard Time (America/Detroit)","validFor":null,"value":"America/Detroit"},{"active":true,"defaultValue":false,"label":"(GMT-05:00) Acre Standard Time (America/Eirunepe)","validFor":null,"value":"America/Eirunepe"},{"active":true,"defaultValue":false,"label":"(GMT-05:00) Eastern Standard Time (America/Grand_Turk)","validFor":null,"value":"America/Grand_Turk"},{"active":true,"defaultValue":false,"label":"(GMT-05:00) Ecuador Time (America/Guayaquil)","validFor":null,"value":"America/Guayaquil"},{"active":true,"defaultValue":false,"label":"(GMT-05:00) Eastern Standard Time (America/Indiana/Indianapolis)","validFor":null,"value":"America/Indiana/Indianapolis"},{"active":true,"defaultValue":false,"label":"(GMT-05:00) Eastern Standard Time (America/Indiana/Marengo)","validFor":null,"value":"America/Indiana/Marengo"},{"active":true,"defaultValue":false,"label":"(GMT-05:00) Eastern Standard Time (America/Indiana/Petersburg)","validFor":null,"value":"America/Indiana/Petersburg"},{"active":true,"defaultValue":false,"label":"(GMT-05:00) Eastern Standard Time (America/Indiana/Vevay)","validFor":null,"value":"America/Indiana/Vevay"},{"active":true,"defaultValue":false,"label":"(GMT-05:00) Eastern Standard Time (America/Indiana/Vincennes)","validFor":null,"value":"America/Indiana/Vincennes"},{"active":true,"defaultValue":false,"label":"(GMT-05:00) Eastern Standard Time (America/Indiana/Winamac)","validFor":null,"value":"America/Indiana/Winamac"},{"active":true,"defaultValue":false,"label":"(GMT-05:00) Eastern Standard Time (America/Iqaluit)","validFor":null,"value":"America/Iqaluit"},{"active":true,"defaultValue":false,"label":"(GMT-05:00) Eastern Standard Time (America/Jamaica)","validFor":null,"value":"America/Jamaica"},{"active":true,"defaultValue":false,"label":"(GMT-05:00) Eastern Standard Time (America/Kentucky/Monticello)","validFor":null,"value":"America/Kentucky/Monticello"},{"active":true,"defaultValue":false,"label":"(GMT-05:00) Peru Standard Time (America/Lima)","validFor":null,"value":"America/Lima"},{"active":true,"defaultValue":false,"label":"(GMT-05:00) Eastern Standard Time (America/Louisville)","validFor":null,"value":"America/Louisville"},{"active":true,"defaultValue":false,"label":"(GMT-05:00) Eastern Standard Time (America/Montreal)","validFor":null,"value":"America/Montreal"},{"active":true,"defaultValue":false,"label":"(GMT-05:00) Eastern Standard Time (America/Nassau)","validFor":null,"value":"America/Nassau"},{"active":true,"defaultValue":false,"label":"(GMT-05:00) Eastern Standard Time (America/New_York)","validFor":null,"value":"America/New_York"},{"active":true,"defaultValue":false,"label":"(GMT-05:00) Eastern Standard Time (America/Nipigon)","validFor":null,"value":"America/Nipigon"},{"active":true,"defaultValue":false,"label":"(GMT-05:00) Eastern Standard Time (America/Panama)","validFor":null,"value":"America/Panama"},{"active":true,"defaultValue":false,"label":"(GMT-05:00) Eastern Standard Time (America/Pangnirtung)","validFor":null,"value":"America/Pangnirtung"},{"active":true,"defaultValue":false,"label":"(GMT-05:00) Eastern Standard Time (America/Port-au-Prince)","validFor":null,"value":"America/Port-au-Prince"},{"active":true,"defaultValue":false,"label":"(GMT-05:00) Acre Standard Time (America/Rio_Branco)","validFor":null,"value":"America/Rio_Branco"},{"active":true,"defaultValue":false,"label":"(GMT-05:00) Eastern Standard Time (America/Thunder_Bay)","validFor":null,"value":"America/Thunder_Bay"},{"active":true,"defaultValue":false,"label":"(GMT-05:00) Eastern Standard Time (America/Toronto)","validFor":null,"value":"America/Toronto"},{"active":true,"defaultValue":false,"label":"(GMT-05:00) Easter Island Summer Time (Pacific/Easter)","validFor":null,"value":"Pacific/Easter"},{"active":true,"defaultValue":false,"label":"(GMT-06:00) Central Standard Time (America/Bahia_Banderas)","validFor":null,"value":"America/Bahia_Banderas"},{"active":true,"defaultValue":false,"label":"(GMT-06:00) Central Standard Time (America/Belize)","validFor":null,"value":"America/Belize"},{"active":true,"defaultValue":false,"label":"(GMT-06:00) Central Standard Time (America/Chicago)","validFor":null,"value":"America/Chicago"},{"active":true,"defaultValue":false,"label":"(GMT-06:00) Mexican Pacific Standard Time (America/Chihuahua)","validFor":null,"value":"America/Chihuahua"},{"active":true,"defaultValue":false,"label":"(GMT-06:00) Central Standard Time (America/Costa_Rica)","validFor":null,"value":"America/Costa_Rica"},{"active":true,"defaultValue":false,"label":"(GMT-06:00) Central Standard Time (America/El_Salvador)","validFor":null,"value":"America/El_Salvador"},{"active":true,"defaultValue":false,"label":"(GMT-06:00) Central Standard Time (America/Guatemala)","validFor":null,"value":"America/Guatemala"},{"active":true,"defaultValue":false,"label":"(GMT-06:00) Central Standard Time (America/Indiana/Knox)","validFor":null,"value":"America/Indiana/Knox"},{"active":true,"defaultValue":false,"label":"(GMT-06:00) Central Standard Time (America/Indiana/Tell_City)","validFor":null,"value":"America/Indiana/Tell_City"},{"active":true,"defaultValue":false,"label":"(GMT-06:00) Central Standard Time (America/Managua)","validFor":null,"value":"America/Managua"},{"active":true,"defaultValue":false,"label":"(GMT-06:00) Central Standard Time (America/Matamoros)","validFor":null,"value":"America/Matamoros"},{"active":true,"defaultValue":false,"label":"(GMT-06:00) Central Standard Time (America/Menominee)","validFor":null,"value":"America/Menominee"},{"active":true,"defaultValue":false,"label":"(GMT-06:00) Central Standard Time (America/Merida)","validFor":null,"value":"America/Merida"},{"active":true,"defaultValue":false,"label":"(GMT-06:00) Central Standard Time (America/Mexico_City)","validFor":null,"value":"America/Mexico_City"},{"active":true,"defaultValue":false,"label":"(GMT-06:00) Central Standard Time (America/Monterrey)","validFor":null,"value":"America/Monterrey"},{"active":true,"defaultValue":false,"label":"(GMT-06:00) Central Standard Time (America/North_Dakota/Beulah)","validFor":null,"value":"America/North_Dakota/Beulah"},{"active":true,"defaultValue":false,"label":"(GMT-06:00) Central Standard Time (America/North_Dakota/Center)","validFor":null,"value":"America/North_Dakota/Center"},{"active":true,"defaultValue":false,"label":"(GMT-06:00) Central Standard Time (America/North_Dakota/New_Salem)","validFor":null,"value":"America/North_Dakota/New_Salem"},{"active":true,"defaultValue":false,"label":"(GMT-06:00) Mountain Standard Time (America/Ojinaga)","validFor":null,"value":"America/Ojinaga"},{"active":true,"defaultValue":false,"label":"(GMT-06:00) Central Standard Time (America/Rainy_River)","validFor":null,"value":"America/Rainy_River"},{"active":true,"defaultValue":false,"label":"(GMT-06:00) Central Standard Time (America/Rankin_Inlet)","validFor":null,"value":"America/Rankin_Inlet"},{"active":true,"defaultValue":false,"label":"(GMT-06:00) Central Standard Time (America/Regina)","validFor":null,"value":"America/Regina"},{"active":true,"defaultValue":false,"label":"(GMT-06:00) Central Standard Time (America/Resolute)","validFor":null,"value":"America/Resolute"},{"active":true,"defaultValue":false,"label":"(GMT-06:00) Central Standard Time (America/Swift_Current)","validFor":null,"value":"America/Swift_Current"},{"active":true,"defaultValue":false,"label":"(GMT-06:00) Central Standard Time (America/Tegucigalpa)","validFor":null,"value":"America/Tegucigalpa"},{"active":true,"defaultValue":false,"label":"(GMT-06:00) Central Standard Time (America/Winnipeg)","validFor":null,"value":"America/Winnipeg"},{"active":true,"defaultValue":false,"label":"(GMT-06:00) Galapagos Time (Pacific/Galapagos)","validFor":null,"value":"Pacific/Galapagos"},{"active":true,"defaultValue":false,"label":"(GMT-07:00) Mountain Standard Time (America/Boise)","validFor":null,"value":"America/Boise"},{"active":true,"defaultValue":false,"label":"(GMT-07:00) Mountain Standard Time (America/Cambridge_Bay)","validFor":null,"value":"America/Cambridge_Bay"},{"active":true,"defaultValue":false,"label":"(GMT-07:00) Mountain Standard Time (America/Creston)","validFor":null,"value":"America/Creston"},{"active":true,"defaultValue":false,"label":"(GMT-07:00) Yukon Time (America/Dawson)","validFor":null,"value":"America/Dawson"},{"active":true,"defaultValue":false,"label":"(GMT-07:00) Mountain Standard Time (America/Dawson_Creek)","validFor":null,"value":"America/Dawson_Creek"},{"active":true,"defaultValue":false,"label":"(GMT-07:00) Mountain Standard Time (America/Denver)","validFor":null,"value":"America/Denver"},{"active":true,"defaultValue":false,"label":"(GMT-07:00) Mountain Standard Time (America/Edmonton)","validFor":null,"value":"America/Edmonton"},{"active":true,"defaultValue":false,"label":"(GMT-07:00) Mountain Standard Time (America/Fort_Nelson)","validFor":null,"value":"America/Fort_Nelson"},{"active":true,"defaultValue":false,"label":"(GMT-07:00) Mexican Pacific Standard Time (America/Hermosillo)","validFor":null,"value":"America/Hermosillo"},{"active":true,"defaultValue":false,"label":"(GMT-07:00) Mountain Standard Time (America/Inuvik)","validFor":null,"value":"America/Inuvik"},{"active":true,"defaultValue":false,"label":"(GMT-07:00) Mexican Pacific Standard Time (America/Mazatlan)","validFor":null,"value":"America/Mazatlan"},{"active":true,"defaultValue":false,"label":"(GMT-07:00) Mountain Standard Time (America/Phoenix)","validFor":null,"value":"America/Phoenix"},{"active":true,"defaultValue":false,"label":"(GMT-07:00) Yukon Time (America/Whitehorse)","validFor":null,"value":"America/Whitehorse"},{"active":true,"defaultValue":false,"label":"(GMT-07:00) Mountain Standard Time (America/Yellowknife)","validFor":null,"value":"America/Yellowknife"},{"active":true,"defaultValue":false,"label":"(GMT-08:00) Pacific Standard Time (America/Los_Angeles)","validFor":null,"value":"America/Los_Angeles"},{"active":true,"defaultValue":false,"label":"(GMT-08:00) Northwest Mexico Standard Time (America/Santa_Isabel)","validFor":null,"value":"America/Santa_Isabel"},{"active":true,"defaultValue":false,"label":"(GMT-08:00) Pacific Standard Time (America/Tijuana)","validFor":null,"value":"America/Tijuana"},{"active":true,"defaultValue":false,"label":"(GMT-08:00) Pacific Standard Time (America/Vancouver)","validFor":null,"value":"America/Vancouver"},{"active":true,"defaultValue":false,"label":"(GMT-08:00) Pitcairn Time (Pacific/Pitcairn)","validFor":null,"value":"Pacific/Pitcairn"},{"active":true,"defaultValue":false,"label":"(GMT-09:00) Alaska Standard Time (America/Anchorage)","validFor":null,"value":"America/Anchorage"},{"active":true,"defaultValue":false,"label":"(GMT-09:00) Alaska Standard Time (America/Juneau)","validFor":null,"value":"America/Juneau"},{"active":true,"defaultValue":false,"label":"(GMT-09:00) Alaska Standard Time (America/Metlakatla)","validFor":null,"value":"America/Metlakatla"},{"active":true,"defaultValue":false,"label":"(GMT-09:00) Alaska Standard Time (America/Nome)","validFor":null,"value":"America/Nome"},{"active":true,"defaultValue":false,"label":"(GMT-09:00) Alaska Standard Time (America/Sitka)","validFor":null,"value":"America/Sitka"},{"active":true,"defaultValue":false,"label":"(GMT-09:00) Alaska Standard Time (America/Yakutat)","validFor":null,"value":"America/Yakutat"},{"active":true,"defaultValue":false,"label":"(GMT-09:00) Gambier Time (Pacific/Gambier)","validFor":null,"value":"Pacific/Gambier"},{"active":true,"defaultValue":false,"label":"(GMT-09:30) Marquesas Time (Pacific/Marquesas)","validFor":null,"value":"Pacific/Marquesas"},{"active":true,"defaultValue":false,"label":"(GMT-10:00) Hawaii-Aleutian Standard Time (America/Adak)","validFor":null,"value":"America/Adak"},{"active":true,"defaultValue":false,"label":"(GMT-10:00) Hawaii-Aleutian Standard Time (Pacific/Honolulu)","validFor":null,"value":"Pacific/Honolulu"},{"active":true,"defaultValue":false,"label":"(GMT-10:00) Hawaii-Aleutian Standard Time (Pacific/Johnston)","validFor":null,"value":"Pacific/Johnston"},{"active":true,"defaultValue":false,"label":"(GMT-10:00) Cook Islands Standard Time (Pacific/Rarotonga)","validFor":null,"value":"Pacific/Rarotonga"},{"active":true,"defaultValue":false,"label":"(GMT-10:00) Tahiti Time (Pacific/Tahiti)","validFor":null,"value":"Pacific/Tahiti"},{"active":true,"defaultValue":false,"label":"(GMT-11:00) Samoa Standard Time (Pacific/Midway)","validFor":null,"value":"Pacific/Midway"},{"active":true,"defaultValue":false,"label":"(GMT-11:00) Niue Time (Pacific/Niue)","validFor":null,"value":"Pacific/Niue"},{"active":true,"defaultValue":false,"label":"(GMT-11:00) Samoa Standard Time (Pacific/Pago_Pago)","validFor":null,"value":"Pacific/Pago_Pago"}],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":true,"scale":0,"searchPrefilterable":false,"soapType":"xsd:string","sortable":true,"type":"picklist","unique":false,"updateable":true,"writeRequiresMasterRead":false},{"aggregatable":true,"aiPredictionField":false,"autoNumber":false,"byteLength":120,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":true,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":false,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":true,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Recurrence Type","length":40,"mask":null,"maskType":null,"name":"RecurrenceType","nameField":false,"namePointing":false,"nillable":true,"permissionable":false,"picklistValues":[{"active":true,"defaultValue":false,"label":"Recurs Daily","validFor":null,"value":"RecursDaily"},{"active":true,"defaultValue":false,"label":"Recurs Every Weekday","validFor":null,"value":"RecursEveryWeekday"},{"active":true,"defaultValue":false,"label":"Recurs Monthly","validFor":null,"value":"RecursMonthly"},{"active":true,"defaultValue":false,"label":"Recurs Monthly Nth","validFor":null,"value":"RecursMonthlyNth"},{"active":true,"defaultValue":false,"label":"Recurs Weekly","validFor":null,"value":"RecursWeekly"},{"active":true,"defaultValue":false,"label":"Recurs Yearly","validFor":null,"value":"RecursYearly"},{"active":true,"defaultValue":false,"label":"Recurs Yearly Nth","validFor":null,"value":"RecursYearlyNth"}],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":true,"scale":0,"searchPrefilterable":false,"soapType":"xsd:string","sortable":true,"type":"picklist","unique":false,"updateable":true,"writeRequiresMasterRead":false},{"aggregatable":true,"aiPredictionField":false,"autoNumber":false,"byteLength":0,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":true,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":false,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":9,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":true,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Recurrence Interval","length":0,"mask":null,"maskType":null,"name":"RecurrenceInterval","nameField":false,"namePointing":false,"nillable":true,"permissionable":false,"picklistValues":[],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"searchPrefilterable":false,"soapType":"xsd:int","sortable":true,"type":"int","unique":false,"updateable":true,"writeRequiresMasterRead":false},{"aggregatable":true,"aiPredictionField":false,"autoNumber":false,"byteLength":0,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":true,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":false,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":9,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":true,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Recurrence Day of Week Mask","length":0,"mask":null,"maskType":null,"name":"RecurrenceDayOfWeekMask","nameField":false,"namePointing":false,"nillable":true,"permissionable":false,"picklistValues":[],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"searchPrefilterable":false,"soapType":"xsd:int","sortable":true,"type":"int","unique":false,"updateable":true,"writeRequiresMasterRead":false},{"aggregatable":true,"aiPredictionField":false,"autoNumber":false,"byteLength":0,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":true,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":false,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":9,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":true,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Recurrence Day of Month","length":0,"mask":null,"maskType":null,"name":"RecurrenceDayOfMonth","nameField":false,"namePointing":false,"nillable":true,"permissionable":false,"picklistValues":[],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"searchPrefilterable":false,"soapType":"xsd:int","sortable":true,"type":"int","unique":false,"updateable":true,"writeRequiresMasterRead":false},{"aggregatable":true,"aiPredictionField":false,"autoNumber":false,"byteLength":120,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":true,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":false,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":true,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Recurrence Instance","length":40,"mask":null,"maskType":null,"name":"RecurrenceInstance","nameField":false,"namePointing":false,"nillable":true,"permissionable":false,"picklistValues":[{"active":true,"defaultValue":false,"label":"1st","validFor":null,"value":"First"},{"active":true,"defaultValue":false,"label":"2nd","validFor":null,"value":"Second"},{"active":true,"defaultValue":false,"label":"3rd","validFor":null,"value":"Third"},{"active":true,"defaultValue":false,"label":"4th","validFor":null,"value":"Fourth"},{"active":true,"defaultValue":false,"label":"last","validFor":null,"value":"Last"}],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":true,"scale":0,"searchPrefilterable":false,"soapType":"xsd:string","sortable":true,"type":"picklist","unique":false,"updateable":true,"writeRequiresMasterRead":false},{"aggregatable":true,"aiPredictionField":false,"autoNumber":false,"byteLength":120,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":true,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":false,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":true,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Recurrence Month of Year","length":40,"mask":null,"maskType":null,"name":"RecurrenceMonthOfYear","nameField":false,"namePointing":false,"nillable":true,"permissionable":false,"picklistValues":[{"active":true,"defaultValue":false,"label":"January","validFor":null,"value":"January"},{"active":true,"defaultValue":false,"label":"February","validFor":null,"value":"February"},{"active":true,"defaultValue":false,"label":"March","validFor":null,"value":"March"},{"active":true,"defaultValue":false,"label":"April","validFor":null,"value":"April"},{"active":true,"defaultValue":false,"label":"May","validFor":null,"value":"May"},{"active":true,"defaultValue":false,"label":"June","validFor":null,"value":"June"},{"active":true,"defaultValue":false,"label":"July","validFor":null,"value":"July"},{"active":true,"defaultValue":false,"label":"August","validFor":null,"value":"August"},{"active":true,"defaultValue":false,"label":"September","validFor":null,"value":"September"},{"active":true,"defaultValue":false,"label":"October","validFor":null,"value":"October"},{"active":true,"defaultValue":false,"label":"November","validFor":null,"value":"November"},{"active":true,"defaultValue":false,"label":"December","validFor":null,"value":"December"}],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":true,"scale":0,"searchPrefilterable":false,"soapType":"xsd:string","sortable":true,"type":"picklist","unique":false,"updateable":true,"writeRequiresMasterRead":false},{"aggregatable":true,"aiPredictionField":false,"autoNumber":false,"byteLength":0,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":true,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":false,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":false,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Reminder Date/Time","length":0,"mask":null,"maskType":null,"name":"ReminderDateTime","nameField":false,"namePointing":false,"nillable":true,"permissionable":false,"picklistValues":[],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"searchPrefilterable":false,"soapType":"xsd:dateTime","sortable":true,"type":"datetime","unique":false,"updateable":true,"writeRequiresMasterRead":false},{"aggregatable":false,"aiPredictionField":false,"autoNumber":false,"byteLength":0,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":true,"custom":false,"defaultValue":false,"defaultValueFormula":null,"defaultedOnCreate":true,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":true,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Reminder Set","length":0,"mask":null,"maskType":null,"name":"IsReminderSet","nameField":false,"namePointing":false,"nillable":false,"permissionable":false,"picklistValues":[],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"searchPrefilterable":false,"soapType":"xsd:boolean","sortable":true,"type":"boolean","unique":false,"updateable":true,"writeRequiresMasterRead":false},{"aggregatable":true,"aiPredictionField":false,"autoNumber":false,"byteLength":120,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":true,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":false,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":true,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Event Subtype","length":40,"mask":null,"maskType":null,"name":"EventSubtype","nameField":false,"namePointing":false,"nillable":true,"permissionable":false,"picklistValues":[{"active":true,"defaultValue":false,"label":"Event","validFor":null,"value":"Event"}],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":true,"scale":0,"searchPrefilterable":false,"soapType":"xsd:string","sortable":true,"type":"picklist","unique":false,"updateable":false,"writeRequiresMasterRead":false},{"aggregatable":false,"aiPredictionField":false,"autoNumber":false,"byteLength":0,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":false,"custom":false,"defaultValue":false,"defaultValueFormula":null,"defaultedOnCreate":true,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":true,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Historical Event, Not Following Recurrence","length":0,"mask":null,"maskType":null,"name":"IsRecurrence2Exclusion","nameField":false,"namePointing":false,"nillable":false,"permissionable":false,"picklistValues":[],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"searchPrefilterable":false,"soapType":"xsd:boolean","sortable":true,"type":"boolean","unique":false,"updateable":false,"writeRequiresMasterRead":false},{"aggregatable":false,"aiPredictionField":false,"autoNumber":false,"byteLength":1536,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":true,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":false,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":"plaintextarea","filterable":false,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":false,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Recurrence Pattern","length":512,"mask":null,"maskType":null,"name":"Recurrence2PatternText","nameField":false,"namePointing":false,"nillable":true,"permissionable":false,"picklistValues":[],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"searchPrefilterable":false,"soapType":"xsd:string","sortable":false,"type":"textarea","unique":false,"updateable":false,"writeRequiresMasterRead":false},{"aggregatable":true,"aiPredictionField":false,"autoNumber":false,"byteLength":120,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":false,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":false,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":true,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Pattern Version","length":40,"mask":null,"maskType":null,"name":"Recurrence2PatternVersion","nameField":false,"namePointing":false,"nillable":true,"permissionable":false,"picklistValues":[{"active":true,"defaultValue":false,"label":"RFC 5545 v4 RRULE","validFor":null,"value":"1"},{"active":true,"defaultValue":false,"label":"RFC 5545 v4_2 RRULE","validFor":null,"value":"2"}],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":true,"scale":0,"searchPrefilterable":false,"soapType":"xsd:string","sortable":true,"type":"picklist","unique":false,"updateable":false,"writeRequiresMasterRead":false},{"aggregatable":false,"aiPredictionField":false,"autoNumber":false,"byteLength":0,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":false,"custom":false,"defaultValue":false,"defaultValueFormula":null,"defaultedOnCreate":true,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":true,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Repeat","length":0,"mask":null,"maskType":null,"name":"IsRecurrence2","nameField":false,"namePointing":false,"nillable":false,"permissionable":false,"picklistValues":[],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"searchPrefilterable":false,"soapType":"xsd:boolean","sortable":true,"type":"boolean","unique":false,"updateable":false,"writeRequiresMasterRead":false},{"aggregatable":false,"aiPredictionField":false,"autoNumber":false,"byteLength":0,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":false,"custom":false,"defaultValue":false,"defaultValueFormula":null,"defaultedOnCreate":true,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":true,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Is Exception","length":0,"mask":null,"maskType":null,"name":"IsRecurrence2Exception","nameField":false,"namePointing":false,"nillable":false,"permissionable":false,"picklistValues":[],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"searchPrefilterable":false,"soapType":"xsd:boolean","sortable":true,"type":"boolean","unique":false,"updateable":false,"writeRequiresMasterRead":false},{"aggregatable":true,"aiPredictionField":false,"autoNumber":false,"byteLength":0,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":false,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":false,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":false,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Recurrence Pattern Start Date","length":0,"mask":null,"maskType":null,"name":"Recurrence2PatternStartDate","nameField":false,"namePointing":false,"nillable":true,"permissionable":false,"picklistValues":[],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"searchPrefilterable":false,"soapType":"xsd:dateTime","sortable":true,"type":"datetime","unique":false,"updateable":false,"writeRequiresMasterRead":false},{"aggregatable":true,"aiPredictionField":false,"autoNumber":false,"byteLength":765,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":false,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":false,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":true,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Recurrence Pattern Time Zone Reference","length":255,"mask":null,"maskType":null,"name":"Recurrence2PatternTimeZone","nameField":false,"namePointing":false,"nillable":true,"permissionable":false,"picklistValues":[],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"searchPrefilterable":false,"soapType":"xsd:string","sortable":true,"type":"string","unique":false,"updateable":false,"writeRequiresMasterRead":false}],"hasSubtypes":false,"implementedBy":null,"implementsInterfaces":null,"isInterface":false,"isSubtype":false,"keyPrefix":"00U","label":"Event","labelPlural":"Events","layoutable":true,"listviewable":null,"lookupLayoutable":null,"mergeable":false,"mruEnabled":true,"name":"Event","namedLayoutInfos":[],"networkScopeFieldName":null,"queryable":true,"recordTypeInfos":[{"active":true,"available":true,"defaultRecordTypeMapping":true,"developerName":"Master","master":true,"name":"Master","recordTypeId":"012000000000000AAA","urls":{"layout":"/services/data/vxx.0/sobjects/Event/describe/layouts/012000000000000AAA"}}],"replicateable":true,"retrieveable":true,"searchLayoutable":true,"searchable":true,"sobjectDescribeOption":"FULL","supportedScopes":[{"label":"My delegated events","name":"delegated"},{"label":"All events","name":"everything"},{"label":"My events","name":"mine"},{"label":"Filter by scope","name":"scopingRule"},{"label":"My team's events","name":"team"}],"triggerable":true,"undeletable":true,"updateable":true,"urls":{"compactLayouts":"/services/data/vxx.0/sobjects/Event/describe/compactLayouts","rowTemplate":"/services/data/vxx.0/sobjects/Event/{ID}","uiDetailTemplate":"https://customization-computing-387-dev-ed.scratch.my.salesforce.com/{ID}","uiEditTemplate":"https://customization-computing-387-dev-ed.scratch.my.salesforce.com/{ID}/e","eventSeriesUpdates":"/services/data/vxx.0/sobjects/Event/{ID}/fromThisEventOnwards","describe":"/services/data/vxx.0/sobjects/Event/describe","uiNewRecord":"https://customization-computing-387-dev-ed.scratch.my.salesforce.com/00U/e","quickActions":"/services/data/vxx.0/sobjects/Event/quickActions","layouts":"/services/data/vxx.0/sobjects/Event/describe/layouts","sobject":"/services/data/vxx.0/sobjects/Event"}} diff --git a/cumulusci/tests/shared_cassettes/GET_sobjects_Global_describe.yaml b/cumulusci/tests/shared_cassettes/GET_sobjects_Global_describe.yaml index 687dfec615..78440e4796 100644 --- a/cumulusci/tests/shared_cassettes/GET_sobjects_Global_describe.yaml +++ b/cumulusci/tests/shared_cassettes/GET_sobjects_Global_describe.yaml @@ -1305,4 +1305,19 @@ response: true, "searchable": false, "triggerable": false, "undeletable": false, "updateable": true, "urls": {"rowTemplate": "/services/data/v52.0/sobjects/WorkTypeShare/{ID}", "describe": "/services/data/v52.0/sobjects/WorkTypeShare/describe", "sobject": - "/services/data/v52.0/sobjects/WorkTypeShare"}}]}' + "/services/data/v52.0/sobjects/WorkTypeShare"}}, {"activateable": + false, "associateEntityType": null, "associateParentEntity": null, "createable": + true, "custom": false, "customSetting": false, "deepCloneable": false, + "deletable": true, "deprecatedAndHidden": false, "feedEnabled": true, + "hasSubtypes": false, "isInterface": false, "isSubtype": false, "keyPrefix": + null, "label": "Event", "labelPlural": "Events", "layoutable": true, + "mergeable": true, "mruEnabled": true, "name": "Event", "queryable": + true, "replicateable": true, "retrieveable": true, "searchable": true, + "triggerable": true, "undeletable": true, "updateable": true, "urls": + {"compactLayouts": "/services/data/v52.0/sobjects/Event/describe/compactLayouts", + "rowTemplate": "/services/data/v52.0/sobjects/Event/{ID}", "approvalLayouts": + "/services/data/v52.0/sobjects/Event/describe/approvalLayouts", "listviews": + "/services/data/v52.0/sobjects/Event/listviews", "describe": "/services/data/v52.0/sobjects/Event/describe", + "quickActions": "/services/data/v52.0/sobjects/Event/quickActions", + "layouts": "/services/data/v52.0/sobjects/Event/describe/layouts", "sobject": + "/services/data/v52.0/sobjects/Event"}}]}' diff --git a/cumulusci/tests/util.py b/cumulusci/tests/util.py index ad9bd6620f..526ee54dd1 100644 --- a/cumulusci/tests/util.py +++ b/cumulusci/tests/util.py @@ -238,6 +238,7 @@ def mock_sobject_describe(name: str): "Opportunity", "OpportunityContactRole", "Case", + "Event", ]: mock_sobject_describe(sobject) From 38a6540228f8a69e8ec83d8df16512565496f2fc Mon Sep 17 00:00:00 2001 From: aditya-balachander Date: Wed, 10 Jan 2024 15:14:02 +0530 Subject: [PATCH 07/36] Increased Coverage --- .../TestDatasetsE2E.test_datasets_e2e.yaml | 2 +- ...est_datasets_extract_standard_objects.yaml | 2 +- cumulusci/tasks/bulkdata/extract.py | 6 +- .../tests/mapping_poly_incomplete.yml | 17 +++ .../bulkdata/tests/mapping_poly_wrong.yml | 19 ++++ .../tasks/bulkdata/tests/test_extract.py | 105 +++++++++++++++++- 6 files changed, 144 insertions(+), 7 deletions(-) create mode 100644 cumulusci/tasks/bulkdata/tests/mapping_poly_incomplete.yml create mode 100644 cumulusci/tasks/bulkdata/tests/mapping_poly_wrong.yml diff --git a/cumulusci/core/tests/cassettes/TestDatasetsE2E.test_datasets_e2e.yaml b/cumulusci/core/tests/cassettes/TestDatasetsE2E.test_datasets_e2e.yaml index 5833b12d30..40aeecf89f 100644 --- a/cumulusci/core/tests/cassettes/TestDatasetsE2E.test_datasets_e2e.yaml +++ b/cumulusci/core/tests/cassettes/TestDatasetsE2E.test_datasets_e2e.yaml @@ -36,7 +36,7 @@ interactions: \n } ]\n}" - request: method: GET - uri: https://orgname.my.salesforce.com/services/data/vxx.0/query/?q=SELECT+Id%2C+AccountNumber%2C+AccountSource%2C+AnnualRevenue%2C+BillingCity%2C+BillingCountry%2C+BillingGeocodeAccuracy%2C+BillingLatitude%2C+BillingLongitude%2C+BillingPostalCode%2C+BillingState%2C+BillingStreet%2C+CleanStatus%2C+DunsNumber%2C+Fax%2C+Industry%2C+Jigsaw%2C+NaicsCode%2C+NaicsDesc%2C+Name%2C+NumberOfEmployees%2C+Ownership%2C+Phone%2C+Rating%2C+RecordTypeId%2C+ShippingCity%2C+ShippingCountry%2C+ShippingGeocodeAccuracy%2C+ShippingLatitude%2C+ShippingLongitude%2C+ShippingPostalCode%2C+ShippingState%2C+ShippingStreet%2C+Sic%2C+SicDesc%2C+Site%2C+TickerSymbol%2C+Tradestyle%2C+Type%2C+Website%2C+YearStarted%2C+ParentId+FROM+Account+WHERE+Name+%21%3D+%27Sample+Account+for+Entitlements%27 + uri: https://orgname.my.salesforce.com/services/data/vxx.0/query/?q=SELECT+Id%2C+AccountNumber%2C+AccountSource%2C+AnnualRevenue%2C+BillingCity%2C+BillingCountry%2C+BillingGeocodeAccuracy%2C+BillingLatitude%2C+BillingLongitude%2C+BillingPostalCode%2C+BillingState%2C+BillingStreet%2C+CleanStatus%2C+DunsNumber%2C+Fax%2C+Industry%2C+Jigsaw%2C+NaicsCode%2C+NaicsDesc%2C+Name%2C+NumberOfEmployees%2C+Ownership%2C+Phone%2C+Rating%2C+ShippingCity%2C+ShippingCountry%2C+ShippingGeocodeAccuracy%2C+ShippingLatitude%2C+ShippingLongitude%2C+ShippingPostalCode%2C+ShippingState%2C+ShippingStreet%2C+Sic%2C+SicDesc%2C+Site%2C+TickerSymbol%2C+Tradestyle%2C+Type%2C+Website%2C+YearStarted%2C+ParentId%2C+RecordTypeId+FROM+Account+WHERE+Name+%21%3D+%27Sample+Account+for+Entitlements%27 body: null headers: *id001 response: diff --git a/cumulusci/core/tests/cassettes/TestDatasetsE2E.test_datasets_extract_standard_objects.yaml b/cumulusci/core/tests/cassettes/TestDatasetsE2E.test_datasets_extract_standard_objects.yaml index 9f1f861b89..c2392fc218 100644 --- a/cumulusci/core/tests/cassettes/TestDatasetsE2E.test_datasets_extract_standard_objects.yaml +++ b/cumulusci/core/tests/cassettes/TestDatasetsE2E.test_datasets_extract_standard_objects.yaml @@ -29,7 +29,7 @@ interactions: \n } ]\n}" - request: method: GET - uri: https://orgname.my.salesforce.com/services/data/vxx.0/query/?q=SELECT+Id%2C+AccountNumber%2C+AccountSource%2C+AnnualRevenue%2C+BillingCity%2C+BillingCountry%2C+BillingGeocodeAccuracy%2C+BillingLatitude%2C+BillingLongitude%2C+BillingPostalCode%2C+BillingState%2C+BillingStreet%2C+CleanStatus%2C+Description%2C+DunsNumber%2C+Fax%2C+Industry%2C+Jigsaw%2C+NaicsCode%2C+NaicsDesc%2C+Name%2C+NumberOfEmployees%2C+Ownership%2C+Phone%2C+Rating%2C+RecordTypeId%2C+ShippingCity%2C+ShippingCountry%2C+ShippingGeocodeAccuracy%2C+ShippingLatitude%2C+ShippingLongitude%2C+ShippingPostalCode%2C+ShippingState%2C+ShippingStreet%2C+Sic%2C+SicDesc%2C+Site%2C+TickerSymbol%2C+Tradestyle%2C+Type%2C+Website%2C+YearStarted%2C+ParentId+FROM+Account+WHERE+Name+%21%3D+%27Sample+Account+for+Entitlements%27 + uri: https://orgname.my.salesforce.com/services/data/vxx.0/query/?q=SELECT+Id%2C+AccountNumber%2C+AccountSource%2C+AnnualRevenue%2C+BillingCity%2C+BillingCountry%2C+BillingGeocodeAccuracy%2C+BillingLatitude%2C+BillingLongitude%2C+BillingPostalCode%2C+BillingState%2C+BillingStreet%2C+CleanStatus%2C+Description%2C+DunsNumber%2C+Fax%2C+Industry%2C+Jigsaw%2C+NaicsCode%2C+NaicsDesc%2C+Name%2C+NumberOfEmployees%2C+Ownership%2C+Phone%2C+Rating%2C+ShippingCity%2C+ShippingCountry%2C+ShippingGeocodeAccuracy%2C+ShippingLatitude%2C+ShippingLongitude%2C+ShippingPostalCode%2C+ShippingState%2C+ShippingStreet%2C+Sic%2C+SicDesc%2C+Site%2C+TickerSymbol%2C+Tradestyle%2C+Type%2C+Website%2C+YearStarted%2C+ParentId%2C+RecordTypeId+FROM+Account+WHERE+Name+%21%3D+%27Sample+Account+for+Entitlements%27 body: null headers: *id002 response: diff --git a/cumulusci/tasks/bulkdata/extract.py b/cumulusci/tasks/bulkdata/extract.py index 421e0d867b..9650f2d311 100644 --- a/cumulusci/tasks/bulkdata/extract.py +++ b/cumulusci/tasks/bulkdata/extract.py @@ -5,7 +5,7 @@ from sqlalchemy import Column, Integer, MetaData, Table, Unicode, create_engine from sqlalchemy.orm import create_session, mapper -from cumulusci.core.exceptions import BulkDataException, TaskOptionsError +from cumulusci.core.exceptions import BulkDataException, TaskOptionsError, CumulusCIException, ConfigError from cumulusci.core.utils import process_bool_arg from cumulusci.tasks.bulkdata.dates import adjust_relative_dates from cumulusci.tasks.bulkdata.mapping_parser import ( @@ -294,7 +294,7 @@ def _get_mapping_for_table(self, table): if isinstance(table, list) and len(mappings) != len(table): missing_tables = set(table) - set(mapping["table"] for mapping in mappings) - raise ValueError(f"The following tables are missing in the mapping file: {missing_tables}") + raise CumulusCIException(f"The following tables are missing in the mapping file: {missing_tables}") return mappings @@ -346,7 +346,7 @@ def throw(string): # pragma: no cover ).count() if total_mapping_operations != total_rows: - raise ValueError(f"Total mapping operations ({total_mapping_operations}) do not match total non-empty rows ({total_rows}) for lookup_key: {lookup_key}") + raise ConfigError(f"Total mapping operations ({total_mapping_operations}) do not match total non-empty rows ({total_rows}) for lookup_key: {lookup_key}") self.session.commit() def _create_tables(self): diff --git a/cumulusci/tasks/bulkdata/tests/mapping_poly_incomplete.yml b/cumulusci/tasks/bulkdata/tests/mapping_poly_incomplete.yml new file mode 100644 index 0000000000..79fbb24144 --- /dev/null +++ b/cumulusci/tasks/bulkdata/tests/mapping_poly_incomplete.yml @@ -0,0 +1,17 @@ +Insert Households: + api: Bulk + sf_object: Account + table: households + fields: + Name: name + record_type: HH_Account +Insert Events: + api: BULK + sf_object: Event + table: events + fields: + Subject: subject + lookups: + WhoId: + key_field: who_id + table: households diff --git a/cumulusci/tasks/bulkdata/tests/mapping_poly_wrong.yml b/cumulusci/tasks/bulkdata/tests/mapping_poly_wrong.yml new file mode 100644 index 0000000000..343a989a61 --- /dev/null +++ b/cumulusci/tasks/bulkdata/tests/mapping_poly_wrong.yml @@ -0,0 +1,19 @@ +Insert Households: + api: Bulk + sf_object: Account + table: households + fields: + Name: name + record_type: HH_Account +Insert Events: + api: BULK + sf_object: Event + table: events + fields: + Subject: subject + lookups: + WhoId: + key_field: who_id + table: + - households + - contacts diff --git a/cumulusci/tasks/bulkdata/tests/test_extract.py b/cumulusci/tasks/bulkdata/tests/test_extract.py index c62bc273aa..834cf96dd1 100644 --- a/cumulusci/tasks/bulkdata/tests/test_extract.py +++ b/cumulusci/tasks/bulkdata/tests/test_extract.py @@ -9,7 +9,7 @@ import responses from sqlalchemy import create_engine -from cumulusci.core.exceptions import BulkDataException, TaskOptionsError +from cumulusci.core.exceptions import BulkDataException, TaskOptionsError, CumulusCIException, ConfigError from cumulusci.tasks.bulkdata import ExtractData from cumulusci.tasks.bulkdata.mapping_parser import MappingLookup, MappingStep from cumulusci.tasks.bulkdata.step import ( @@ -81,6 +81,8 @@ class TestExtractData: mapping_file_v1 = "mapping_v1.yml" mapping_file_v2 = "mapping_v2.yml" mapping_file_poly = "mapping_poly.yml" + mapping_file_poly_wrong = "mapping_poly_wrong.yml" + mapping_file_poly_incomplete = "mapping_poly_incomplete.yml" mapping_file_vanilla = "mapping_vanilla_sf.yml" @responses.activate @@ -404,7 +406,106 @@ def test_run__poly__polymorphic_lookups(self, query_op_mock): events = conn.execute("select * from events").fetchall() assert events[0].who_id == "Account-1" assert events[1].who_id == "Contact-1" - + + @responses.activate + @mock.patch("cumulusci.tasks.bulkdata.extract.get_query_operation") + def test_run__poly__wrong_mapping(self, query_op_mock): + base_path = os.path.dirname(__file__) + mapping_path = os.path.join(base_path, self.mapping_file_poly_wrong) + mock_describe_calls() + + with temporary_dir() as t: + task = _make_task( + ExtractData, + { + "options": { + "database_url": f"sqlite:///{t}/temp_poly.db", # in memory + "mapping": mapping_path, + } + }, + ) + task.bulk = mock.Mock() + task.sf = mock.Mock() + task.org_config._is_person_accounts_enabled = False + + mock_query_households = MockBulkQueryOperation( + sobject="Account", + api_options={}, + context=task, + query="SELECT Id, Name FROM Account", + ) + mock_query_contacts = MockBulkQueryOperation( + sobject="Contact", + api_options={}, + context=task, + query="SELECT Id, FirstName, LastName, Email, AccountId FROM Contact", + ) + mock_query_events = MockBulkQueryOperation( + sobject="Event", + api_options={}, + context=task, + query="SELECT Id, LastName, WhoId FROM Event", + ) + mock_query_households.results = [["abc123", "TestHousehold"]] + mock_query_contacts.results = [ + ["def456", "First", "Last", "test@example.com", "abc123"] + ] + mock_query_events.results = [ + ["ijk789", "Last1", "abc123"], + ["lmn010", "Last2", "def456"] + ] + + query_op_mock.side_effect = [mock_query_households, mock_query_contacts, mock_query_events] + with pytest.raises(CumulusCIException) as e: + task() + + assert "The following tables are missing in the mapping file:" in str(e.value) + + @responses.activate + @mock.patch("cumulusci.tasks.bulkdata.extract.get_query_operation") + def test_run__poly__incomplete_mapping(self, query_op_mock): + base_path = os.path.dirname(__file__) + mapping_path = os.path.join(base_path, self.mapping_file_poly_incomplete) + mock_describe_calls() + + with temporary_dir() as t: + task = _make_task( + ExtractData, + { + "options": { + "database_url": f"sqlite:///{t}/temp_poly.db", # in memory + "mapping": mapping_path, + } + }, + ) + task.bulk = mock.Mock() + task.sf = mock.Mock() + task.org_config._is_person_accounts_enabled = False + + mock_query_households = MockBulkQueryOperation( + sobject="Account", + api_options={}, + context=task, + query="SELECT Id, Name FROM Account", + ) + mock_query_events = MockBulkQueryOperation( + sobject="Event", + api_options={}, + context=task, + query="SELECT Id, LastName, WhoId FROM Event", + ) + mock_query_households.results = [["abc123", "TestHousehold"]] + mock_query_events.results = [ + ["ijk789", "Last1", "abc123"], + ["lmn010", "Last2", "def456"] + ] + + query_op_mock.side_effect = [mock_query_households, mock_query_events] + with pytest.raises(ConfigError) as e: + task() + + assert "Total mapping operations" in str(e.value) + assert "do not match total non-empty rows" in str(e.value) @mock.patch("cumulusci.tasks.bulkdata.extract.log_progress") def test_import_results__oid_as_pk(self, log_mock): From 4de7153c48eb46be721a58c7b0cdc9309042dd65 Mon Sep 17 00:00:00 2001 From: lakshmi2506 Date: Fri, 12 Jan 2024 15:45:53 +0530 Subject: [PATCH 08/36] load-data with new format --- cumulusci/tasks/bulkdata/load.py | 69 ++++++------- .../tasks/bulkdata/query_transformers.py | 5 +- cumulusci/tasks/bulkdata/utils.py | 71 ++++++++++++-- datasets/mapping.yml | 23 +++-- datasets/sample.sql | 97 +++++++++++-------- 5 files changed, 166 insertions(+), 99 deletions(-) diff --git a/cumulusci/tasks/bulkdata/load.py b/cumulusci/tasks/bulkdata/load.py index 9fa66c2a4f..cddf93bc91 100644 --- a/cumulusci/tasks/bulkdata/load.py +++ b/cumulusci/tasks/bulkdata/load.py @@ -124,6 +124,7 @@ def _init_options(self, kwargs): self.options["enable_rollback"] = process_bool_arg( self.options.get("enable_rollback", False) ) + self._id_generators = {} def _init_dataset(self): """Find the dataset paths to use with the following sequence: @@ -205,10 +206,11 @@ def _run_task(self): "No data will be loaded because this is a persistent org and no dataset was specified." ) return + self.ID_TABLE_NAME = "cumulusci_id_table" self._init_mapping() with self._init_db(): self._expand_mapping() - + self._initialize_id_table(self.reset_oids) start_step = self.options.get("start_step") started = False results = {} @@ -359,7 +361,8 @@ def check_simple_upsert(self, mapping): def _stream_queried_data(self, mapping, local_ids, query): """Get data from the local db""" - statics = self._get_statics(mapping) + # statics = self._get_statics(mapping) + staticizer = self._add_statics_to_row(mapping) total_rows = 0 if mapping.anchor_date: @@ -372,13 +375,13 @@ def _stream_queried_data(self, mapping, local_ids, query): batch_size = mapping.batch_size or DEFAULT_BULK_BATCH_SIZE for row in query.yield_per(batch_size): total_rows += 1 - # Add static values to row - pkey = row[0] - row = list(row[1:]) + statics + if mapping.anchor_date and (date_context[0] or date_context[1]): row = adjust_relative_dates( mapping, date_context, row, DataOperationType.INSERT ) + pkey = row[0] # FIXME: This is a local-DB ordering assumption. + row = staticizer(list(row[1:])) if mapping.action is DataOperationType.UPDATE: if len(row) > 1 and all([f is None for f in row[1:]]): # Skip update rows that contain no values @@ -389,7 +392,7 @@ def _stream_queried_data(self, mapping, local_ids, query): yield row self.logger.info( - f"Prepared {total_rows} rows for {mapping['action']} to {mapping['sf_object']}." + f"Prepared {total_rows} rows for {mapping.action.value} to {mapping.sf_object}." ) def _load_record_types(self, sobjects, conn): @@ -400,10 +403,9 @@ def _load_record_types(self, sobjects, conn): sobject, table_name, conn, self.org_config.is_person_accounts_enabled ) - def _get_statics(self, mapping): - """Return the static values (not column names) to be appended to - records for this mapping.""" + def _add_statics_to_row(self, mapping): statics = list(mapping.static.values()) + if mapping.record_type: query = ( f"SELECT Id FROM RecordType WHERE SObjectType='{mapping.sf_object}'" @@ -416,7 +418,10 @@ def _get_statics(self, mapping): raise BulkDataException(f"Cannot find RecordType with query `{query}`") statics.append(record_type_id) - return statics + def add_statics(row): + return row + statics + + return add_statics def _query_db(self, mapping): """Build a query to retrieve data from the local db. @@ -479,10 +484,8 @@ def _process_job_results(self, mapping, step, local_ids): DataOperationType.UPSERT, DataOperationType.ETL_UPSERT, ) - if is_insert_or_upsert: - id_table_name = self._initialize_id_table(mapping, self.reset_oids) - conn = self.session.connection() + conn = self.session.connection() sf_id_results = self._generate_results_id_map(step, local_ids) # If we know we have no successful inserts, don't attempt to persist Ids. @@ -490,9 +493,10 @@ def _process_job_results(self, mapping, step, local_ids): if is_insert_or_upsert and ( step.job_result.records_processed - step.job_result.total_row_errors ): + table = self.metadata.tables[self.ID_TABLE_NAME] sql_bulk_insert_from_records( connection=conn, - table=self.metadata.tables[id_table_name], + table=table, columns=("id", "sf_id"), record_iterable=sf_id_results, ) @@ -510,7 +514,7 @@ def _process_job_results(self, mapping, step, local_ids): if account_id_lookup: sql_bulk_insert_from_records( connection=conn, - table=self.metadata.tables[id_table_name], + table=self.metadata.tables[self.ID_TABLE_NAME], columns=("id", "sf_id"), record_iterable=self._generate_contact_id_map_for_person_accounts( mapping, account_id_lookup, conn @@ -554,7 +558,7 @@ def _generate_results_id_map(self, step, local_ids): CreateRollback.prepare_for_rollback(self, step, created_results) return sf_id_results - def _initialize_id_table(self, mapping, should_reset_table): + def _initialize_id_table(self, should_reset_table): """initalize or find table to hold the inserted SF Ids The table has a name like xxx_sf_ids and has just two columns, id and sf_id. @@ -562,29 +566,20 @@ def _initialize_id_table(self, mapping, should_reset_table): If the table already exists, should_reset_table determines whether to drop and recreate it or not. """ - id_table_name = f"{mapping['table']}_sf_ids" - already_exists = id_table_name in self.metadata.tables + already_exists = self.ID_TABLE_NAME in self.metadata.tables if already_exists and not should_reset_table: - return id_table_name - - if not hasattr(self, "_initialized_id_tables"): - self._initialized_id_tables = set() - if id_table_name not in self._initialized_id_tables: - if already_exists: - self.metadata.remove(self.metadata.tables[id_table_name]) - id_table = Table( - id_table_name, - self.metadata, - Column("id", Unicode(255), primary_key=True), - Column("sf_id", Unicode(18)), - ) - if self.inspector.has_table(id_table_name): - id_table.drop() - id_table.create() - self._initialized_id_tables.add(id_table_name) - return id_table_name + return + id_table = Table( + self.ID_TABLE_NAME, + self.metadata, + Column("id", Unicode(255), primary_key=True), + Column("sf_id", Unicode(18)), + ) + if id_table.exists(): + id_table.drop() + id_table.create() def _sqlite_load(self): """Read a SQLite script and initialize the in-memory database.""" @@ -655,7 +650,7 @@ def _init_mapping(self): mapping=self.mapping, sf=self.sf, namespace=self.project_config.project__package__namespace, - data_operation=DataOperationType.INSERT, + data_operation=DataOperationType.QUERY, inject_namespaces=self.options["inject_namespaces"], drop_missing=self.options["drop_missing_schema"], ) diff --git a/cumulusci/tasks/bulkdata/query_transformers.py b/cumulusci/tasks/bulkdata/query_transformers.py index cbc50e389a..cb80b182fa 100644 --- a/cumulusci/tasks/bulkdata/query_transformers.py +++ b/cumulusci/tasks/bulkdata/query_transformers.py @@ -7,6 +7,7 @@ from cumulusci.core.exceptions import BulkDataException Criterion = T.Any +ID_TABLE_NAME = "cumulusci_id_table" class LoadQueryExtender: @@ -59,9 +60,7 @@ def __init__(self, mapping, metadata, model) -> None: @cached_property def columns_to_add(self): for lookup in self.lookups: - lookup.aliased_table = aliased( - self.metadata.tables[f"{lookup.table}_sf_ids"] - ) + lookup.aliased_table = aliased(self.metadata.tables[ID_TABLE_NAME]) return [lookup.aliased_table.columns.sf_id for lookup in self.lookups] @cached_property diff --git a/cumulusci/tasks/bulkdata/utils.py b/cumulusci/tasks/bulkdata/utils.py index ea09ba49df..1d51ddc6cf 100644 --- a/cumulusci/tasks/bulkdata/utils.py +++ b/cumulusci/tasks/bulkdata/utils.py @@ -13,6 +13,8 @@ from cumulusci.core.exceptions import BulkDataException from cumulusci.utils.iterators import iterate_in_chunks +ID_TABLE_NAME = "cumulusci_id_table" + class SqlAlchemyMixin: logger: logging.Logger @@ -73,16 +75,60 @@ def _database_url(self): else: return self._temp_database_url() + def _id_generator_for_object(self, sobject: str): + if sobject not in self._id_generators: + + def _generate_ids(): + counter = 0 + while True: + yield f"{sobject}-{counter}" + counter += 1 + + self._id_generators[sobject] = _generate_ids() + + return self._id_generators[sobject] + + def _update_column( + self, *, source_model, target_model, key_field, join_field, target_field + ): + key_attr = getattr(source_model, key_field) + join_attr = getattr(target_model, join_field) + target_attr = getattr(target_model, target_field) + + id_column = inspect(source_model).primary_key[0].name + + try: + self.session.query(source_model).filter( + key_attr.isnot(None), key_attr == join_attr + ).update({key_attr: target_attr}, synchronize_session=False) + except NotImplementedError: + # Some databases, such as SQLite, don't support multitable update + # TODO: review memory consumption of this routine. + mappings = [] + for row, lookup_id in self.session.query(source_model, target_attr).join( + target_model, key_attr == join_attr + ): + mappings.append( + {id_column: getattr(row, id_column), key_field: lookup_id} + ) + self.session.bulk_update_mappings(source_model, mappings) + + def _update_sf_id_column(self, model, key_field): + self._update_column( + source_model=model, + target_model=self.models[self.ID_TABLE_NAME], + key_field=key_field, + join_field="sf_id", + target_field="id", + ) -def _handle_primary_key(mapping, fields): - """Provide support for legacy mappings which used the OID as the pk but - default to using an autoincrementing int pk and a separate sf_id column""" + def _is_autopk_database(self): + # If the type of the Id column on a mapping is INTEGER, + # this is an autopk database. - if mapping.get_oid_as_pk(): - id_column = mapping.fields["Id"] - fields.append(Column(id_column, Unicode(255), primary_key=True)) - else: - fields.append(Column("id", Integer(), primary_key=True, autoincrement=True)) + mapping = self.mapping.values()[0] + id_field = mapping.fields["Id"] + return isinstance(getattr(self.models[mapping.table], id_field).type, Integer) def create_table(mapping, metadata) -> Table: @@ -92,10 +138,15 @@ def create_table(mapping, metadata) -> Table: Mapping should be a MappingStep instance""" fields = [] - _handle_primary_key(mapping, fields) + # _handle_primary_key(mapping, fields) + id_column = mapping.fields["Id"] # Guaranteed to be present by mapping parser. + fields.append(Column(id_column, Unicode(255), primary_key=True)) # make a field list to create - for field, db in mapping.get_complete_field_map().items(): + # for field, db in mapping.get_complete_field_map().items(): + for field, db in zip( + mapping.get_extract_field_list(), mapping.get_database_column_list() + ).items(): if field == "Id": continue diff --git a/datasets/mapping.yml b/datasets/mapping.yml index ae7952b22c..878c7fa9f7 100644 --- a/datasets/mapping.yml +++ b/datasets/mapping.yml @@ -35,12 +35,23 @@ Contact: lookups: AccountId: table: Account -Opportunity: - sf_object: Opportunity +Lead: + sf_object: Lead + api: rest + batch_size: 2 + fields: + - LastName + - Company +Event: + sf_object: Event api: bulk batch_size: 2 fields: - - Name - - CloseDate - - Amount - - StageName + - Subject + - DurationInMinutes + - ActivityDateTime + lookups: + WhoId: + table: + - Contact + - Lead diff --git a/datasets/sample.sql b/datasets/sample.sql index dc9bc8fdd6..921b10aadb 100644 --- a/datasets/sample.sql +++ b/datasets/sample.sql @@ -1,53 +1,64 @@ BEGIN TRANSACTION; CREATE TABLE "Account" ( - id INTEGER NOT NULL, - "Name" VARCHAR(255), - "Description" VARCHAR(255), - "NumberOfEmployees" VARCHAR(255), - "BillingStreet" VARCHAR(255), - "BillingCity" VARCHAR(255), - "BillingState" VARCHAR(255), - "BillingPostalCode" VARCHAR(255), - "BillingCountry" VARCHAR(255), - "ShippingStreet" VARCHAR(255), - "ShippingCity" VARCHAR(255), - "ShippingState" VARCHAR(255), - "ShippingPostalCode" VARCHAR(255), - "ShippingCountry" VARCHAR(255), - "Phone" VARCHAR(255), - "Fax" VARCHAR(255), - "Website" VARCHAR(255), - "AccountNumber" VARCHAR(255), + "id" VARCHAR(255) NOT NULL, + "Name" VARCHAR(255), + "Description" VARCHAR(255), + "NumberOfEmployees" VARCHAR(255), + "BillingStreet" VARCHAR(255), + "BillingCity" VARCHAR(255), + "BillingState" VARCHAR(255), + "BillingPostalCode" VARCHAR(255), + "BillingCountry" VARCHAR(255), + "ShippingStreet" VARCHAR(255), + "ShippingCity" VARCHAR(255), + "ShippingState" VARCHAR(255), + "ShippingPostalCode" VARCHAR(255), + "ShippingCountry" VARCHAR(255), + "Phone" VARCHAR(255), + "Fax" VARCHAR(255), + "Website" VARCHAR(255), + "AccountNumber" VARCHAR(255), PRIMARY KEY (id) ); -INSERT INTO "Account" VALUES(1,'Sample Account for Entitlements','','','','','','','','','','','','','','','',''); -INSERT INTO "Account" VALUES(2,'The Bluth Company','Solid as a rock','6','','','','','','','','','','','','','',''); -INSERT INTO "Account" VALUES(3,'Camacho PLC','Total logistical task-force','59908','2852 Caleb Village Suite 428','Porterside','Maryland','14525','Canada','6070 Davidson Rapids','Gibsonland','North Dakota','62676','Lithuania','221.285.1033','+1-081-230-6073x31438','http://jenkins.info/category/tag/tag/terms/','2679965'); +INSERT INTO "Account" VALUES("Account-1",'alpha','','','Baker St.','','','','','','','','','','','','',''); +INSERT INTO "Account" VALUES("Account-2",'beta','','','Baker St.','','','','','','','','','','','','',''); +INSERT INTO "Account" VALUES("Account-3",'gamma','','','Baker St.','','','','','','','','','','','','',''); + CREATE TABLE "Contact" ( - id INTEGER NOT NULL, - "FirstName" VARCHAR(255), - "LastName" VARCHAR(255), - "Salutation" VARCHAR(255), - "Email" VARCHAR(255), - "Phone" VARCHAR(255), - "MobilePhone" VARCHAR(255), - "Title" VARCHAR(255), - "Birthdate" VARCHAR(255), - "AccountId" VARCHAR(255), + "id" VARCHAR(255) NOT NULL, + "FirstName" VARCHAR(255), + "LastName" VARCHAR(255), + "Salutation" VARCHAR(255), + "Email" VARCHAR(255), + "Phone" VARCHAR(255), + "MobilePhone" VARCHAR(255), + "Title" VARCHAR(255), + "Birthdate" VARCHAR(255), + "AccountId" VARCHAR(255), PRIMARY KEY (id) ); -INSERT INTO "Contact" VALUES(1,'Michael','Bluth','','','','','','','2'); -INSERT INTO "Contact" VALUES(2,'Jared','Burnett','Ms.','ja-burnett2011@example.net','372.865.5762x5990','033.134.7156x7943','Systems analyst','2000-04-18','3'); -CREATE TABLE "Opportunity" ( - id INTEGER NOT NULL, - "Name" VARCHAR(255), - "CloseDate" VARCHAR(255), - "Amount" VARCHAR(255), - "StageName" VARCHAR(255), +INSERT INTO "Contact" VALUES("Contact-1",'alphass','Mannwereerevhefwingefew','','krithvtffder@example.com','','','','','Account-1'); +INSERT INTO "Contact" VALUES("Contact-2",'betasss','Blackefefererf','','kathjvhryn85@exaerfemple.com','','','','','Account-2'); +INSERT INTO "Contact" VALUES("Contact-3",'deltasss','Hunteererbhjbefrewererfef','','dfdfvgh@example.com','','','','','Account-3'); +INSERT INTO "Contact" VALUES("Contact-4",'gammasss','Carererfbhjhjbrlsonere','','johnmjbbhontddfgfdcsdcsces@example.com','','','','',''); +CREATE TABLE "Event" ( + "id" VARCHAR(255) NOT NULL, + "Subject" VARCHAR(255), + "DurationInMinutes" VARCHAR(255), + "ActivityDateTime" VARCHAR(255), + "WhoId" VARCHAR(255), PRIMARY KEY (id) ); -INSERT INTO "Opportunity" VALUES(1,'democratic Opportunity','2022-07-27','69.0','In Progress'); -INSERT INTO "Opportunity" VALUES(2,'your Opportunity','2022-10-09','76.0','Closed Won'); -INSERT INTO "Opportunity" VALUES(3,'heart Opportunity','2022-11-04','32.0','Closed Won'); -INSERT INTO "Opportunity" VALUES(4,'treat Opportunity','2022-12-12','137.0','Closed Won'); +INSERT INTO "Event" VALUES("Event-1",'helllo','60','2024-01-10T05:30:00.000Z','Contact-1'); +INSERT INTO "Event" VALUES("Event-2",'newer','60','2024-01-10T05:30:00.000Z','Lead-1'); + +CREATE TABLE "Lead" ( + "id" VARCHAR(255) NOT NULL, + "LastName" VARCHAR(255), + "Company" VARCHAR(255), + PRIMARY KEY (id) +); +INSERT INTO "Lead" VALUES("Lead-1",'deltassssds','Farmers Coop. of Florida'); +INSERT INTO "Lead" VALUES("Lead-2",'gauramm','Abbott Insurance'); + COMMIT; From 18947ae788acc897244816e982be92133ea5410b Mon Sep 17 00:00:00 2001 From: lakshmi2506 Date: Fri, 12 Jan 2024 17:19:36 +0530 Subject: [PATCH 09/36] backward compatability enabled --- cumulusci/tasks/bulkdata/load.py | 11 ++++++++++- cumulusci/tasks/bulkdata/query_transformers.py | 18 +++++++++++++----- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/cumulusci/tasks/bulkdata/load.py b/cumulusci/tasks/bulkdata/load.py index cddf93bc91..0eb905ca95 100644 --- a/cumulusci/tasks/bulkdata/load.py +++ b/cumulusci/tasks/bulkdata/load.py @@ -125,6 +125,7 @@ def _init_options(self, kwargs): self.options.get("enable_rollback", False) ) self._id_generators = {} + self._old_format = False def _init_dataset(self): """Find the dataset paths to use with the following sequence: @@ -444,12 +445,14 @@ def _query_db(self, mapping): query = self.session.query(*columns) classes = [ - AddLookupsToQuery, AddRecordTypesToQuery, AddMappingFiltersToQuery, AddUpsertsToQuery, ] transformers = [cls(mapping, self.metadata, model) for cls in classes] + transformers.append( + AddLookupsToQuery(mapping, self.metadata, model, self._old_format) + ) if mapping.sf_object == "Contact" and self._can_load_person_accounts(mapping): transformers.append(AddPersonAccountsToQuery(mapping, self.metadata, model)) @@ -488,6 +491,12 @@ def _process_job_results(self, mapping, step, local_ids): conn = self.session.connection() sf_id_results = self._generate_results_id_map(step, local_ids) + for i in range(len(sf_id_results)): + if str.isdigit(sf_id_results[i][0][0]): + self._old_format = True + sf_id_results[i][0] = mapping.table + "-" + str(sf_id_results[i][0]) + else: + break # If we know we have no successful inserts, don't attempt to persist Ids. # Do, however, drain the generator to get error-checking behavior. if is_insert_or_upsert and ( diff --git a/cumulusci/tasks/bulkdata/query_transformers.py b/cumulusci/tasks/bulkdata/query_transformers.py index cb80b182fa..84c1e66207 100644 --- a/cumulusci/tasks/bulkdata/query_transformers.py +++ b/cumulusci/tasks/bulkdata/query_transformers.py @@ -51,8 +51,9 @@ def add_outerjoins(self, query: Query): class AddLookupsToQuery(LoadQueryExtender): """Adds columns and joins relatinng to lookups""" - def __init__(self, mapping, metadata, model) -> None: + def __init__(self, mapping, metadata, model, _old_format) -> None: super().__init__(mapping, metadata, model) + self._old_format = _old_format self.lookups = [ lookup for lookup in self.mapping.lookups.values() if not lookup.after ] @@ -70,10 +71,17 @@ def outerjoins_to_add(self): def join_for_lookup(lookup): key_field = lookup.get_lookup_key_field(self.model) value_column = getattr(self.model, key_field) - return ( - lookup.aliased_table, - lookup.aliased_table.columns.id == value_column, - ) + if self._old_format: + return ( + lookup.aliased_table, + lookup.aliased_table.columns.id + == str(lookup.table) + "-" + value_column, + ) + else: + return ( + lookup.aliased_table, + lookup.aliased_table.columns.id == value_column, + ) return [join_for_lookup(lookup) for lookup in self.lookups] From 773501fda7ecf47ca2fb19ebfa6a34f541851946 Mon Sep 17 00:00:00 2001 From: lakshmi2506 Date: Tue, 16 Jan 2024 12:18:24 +0530 Subject: [PATCH 10/36] old_format condition changed --- cumulusci/tasks/bulkdata/load.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cumulusci/tasks/bulkdata/load.py b/cumulusci/tasks/bulkdata/load.py index 0eb905ca95..62e8188d09 100644 --- a/cumulusci/tasks/bulkdata/load.py +++ b/cumulusci/tasks/bulkdata/load.py @@ -492,7 +492,7 @@ def _process_job_results(self, mapping, step, local_ids): sf_id_results = self._generate_results_id_map(step, local_ids) for i in range(len(sf_id_results)): - if str.isdigit(sf_id_results[i][0][0]): + if str(sf_id_results[i][0]).isnumeric(): self._old_format = True sf_id_results[i][0] = mapping.table + "-" + str(sf_id_results[i][0]) else: From c563a58f1a64574daa672e36f2991eced492c507 Mon Sep 17 00:00:00 2001 From: aditya-balachander <139134092+aditya-balachander@users.noreply.github.com> Date: Tue, 16 Jan 2024 15:04:09 +0530 Subject: [PATCH 11/36] Revert "Polymorphic Reference - Load (merging into another branch)" --- cumulusci/tasks/bulkdata/load.py | 80 ++++++++------- .../tasks/bulkdata/query_transformers.py | 23 ++--- cumulusci/tasks/bulkdata/utils.py | 71 ++------------ datasets/mapping.yml | 23 ++--- datasets/sample.sql | 97 ++++++++----------- 5 files changed, 105 insertions(+), 189 deletions(-) diff --git a/cumulusci/tasks/bulkdata/load.py b/cumulusci/tasks/bulkdata/load.py index 62e8188d09..9fa66c2a4f 100644 --- a/cumulusci/tasks/bulkdata/load.py +++ b/cumulusci/tasks/bulkdata/load.py @@ -124,8 +124,6 @@ def _init_options(self, kwargs): self.options["enable_rollback"] = process_bool_arg( self.options.get("enable_rollback", False) ) - self._id_generators = {} - self._old_format = False def _init_dataset(self): """Find the dataset paths to use with the following sequence: @@ -207,11 +205,10 @@ def _run_task(self): "No data will be loaded because this is a persistent org and no dataset was specified." ) return - self.ID_TABLE_NAME = "cumulusci_id_table" self._init_mapping() with self._init_db(): self._expand_mapping() - self._initialize_id_table(self.reset_oids) + start_step = self.options.get("start_step") started = False results = {} @@ -362,8 +359,7 @@ def check_simple_upsert(self, mapping): def _stream_queried_data(self, mapping, local_ids, query): """Get data from the local db""" - # statics = self._get_statics(mapping) - staticizer = self._add_statics_to_row(mapping) + statics = self._get_statics(mapping) total_rows = 0 if mapping.anchor_date: @@ -376,13 +372,13 @@ def _stream_queried_data(self, mapping, local_ids, query): batch_size = mapping.batch_size or DEFAULT_BULK_BATCH_SIZE for row in query.yield_per(batch_size): total_rows += 1 - + # Add static values to row + pkey = row[0] + row = list(row[1:]) + statics if mapping.anchor_date and (date_context[0] or date_context[1]): row = adjust_relative_dates( mapping, date_context, row, DataOperationType.INSERT ) - pkey = row[0] # FIXME: This is a local-DB ordering assumption. - row = staticizer(list(row[1:])) if mapping.action is DataOperationType.UPDATE: if len(row) > 1 and all([f is None for f in row[1:]]): # Skip update rows that contain no values @@ -393,7 +389,7 @@ def _stream_queried_data(self, mapping, local_ids, query): yield row self.logger.info( - f"Prepared {total_rows} rows for {mapping.action.value} to {mapping.sf_object}." + f"Prepared {total_rows} rows for {mapping['action']} to {mapping['sf_object']}." ) def _load_record_types(self, sobjects, conn): @@ -404,9 +400,10 @@ def _load_record_types(self, sobjects, conn): sobject, table_name, conn, self.org_config.is_person_accounts_enabled ) - def _add_statics_to_row(self, mapping): + def _get_statics(self, mapping): + """Return the static values (not column names) to be appended to + records for this mapping.""" statics = list(mapping.static.values()) - if mapping.record_type: query = ( f"SELECT Id FROM RecordType WHERE SObjectType='{mapping.sf_object}'" @@ -419,10 +416,7 @@ def _add_statics_to_row(self, mapping): raise BulkDataException(f"Cannot find RecordType with query `{query}`") statics.append(record_type_id) - def add_statics(row): - return row + statics - - return add_statics + return statics def _query_db(self, mapping): """Build a query to retrieve data from the local db. @@ -445,14 +439,12 @@ def _query_db(self, mapping): query = self.session.query(*columns) classes = [ + AddLookupsToQuery, AddRecordTypesToQuery, AddMappingFiltersToQuery, AddUpsertsToQuery, ] transformers = [cls(mapping, self.metadata, model) for cls in classes] - transformers.append( - AddLookupsToQuery(mapping, self.metadata, model, self._old_format) - ) if mapping.sf_object == "Contact" and self._can_load_person_accounts(mapping): transformers.append(AddPersonAccountsToQuery(mapping, self.metadata, model)) @@ -487,25 +479,20 @@ def _process_job_results(self, mapping, step, local_ids): DataOperationType.UPSERT, DataOperationType.ETL_UPSERT, ) + if is_insert_or_upsert: + id_table_name = self._initialize_id_table(mapping, self.reset_oids) + conn = self.session.connection() - conn = self.session.connection() sf_id_results = self._generate_results_id_map(step, local_ids) - for i in range(len(sf_id_results)): - if str(sf_id_results[i][0]).isnumeric(): - self._old_format = True - sf_id_results[i][0] = mapping.table + "-" + str(sf_id_results[i][0]) - else: - break # If we know we have no successful inserts, don't attempt to persist Ids. # Do, however, drain the generator to get error-checking behavior. if is_insert_or_upsert and ( step.job_result.records_processed - step.job_result.total_row_errors ): - table = self.metadata.tables[self.ID_TABLE_NAME] sql_bulk_insert_from_records( connection=conn, - table=table, + table=self.metadata.tables[id_table_name], columns=("id", "sf_id"), record_iterable=sf_id_results, ) @@ -523,7 +510,7 @@ def _process_job_results(self, mapping, step, local_ids): if account_id_lookup: sql_bulk_insert_from_records( connection=conn, - table=self.metadata.tables[self.ID_TABLE_NAME], + table=self.metadata.tables[id_table_name], columns=("id", "sf_id"), record_iterable=self._generate_contact_id_map_for_person_accounts( mapping, account_id_lookup, conn @@ -567,7 +554,7 @@ def _generate_results_id_map(self, step, local_ids): CreateRollback.prepare_for_rollback(self, step, created_results) return sf_id_results - def _initialize_id_table(self, should_reset_table): + def _initialize_id_table(self, mapping, should_reset_table): """initalize or find table to hold the inserted SF Ids The table has a name like xxx_sf_ids and has just two columns, id and sf_id. @@ -575,20 +562,29 @@ def _initialize_id_table(self, should_reset_table): If the table already exists, should_reset_table determines whether to drop and recreate it or not. """ + id_table_name = f"{mapping['table']}_sf_ids" - already_exists = self.ID_TABLE_NAME in self.metadata.tables + already_exists = id_table_name in self.metadata.tables if already_exists and not should_reset_table: - return - id_table = Table( - self.ID_TABLE_NAME, - self.metadata, - Column("id", Unicode(255), primary_key=True), - Column("sf_id", Unicode(18)), - ) - if id_table.exists(): - id_table.drop() - id_table.create() + return id_table_name + + if not hasattr(self, "_initialized_id_tables"): + self._initialized_id_tables = set() + if id_table_name not in self._initialized_id_tables: + if already_exists: + self.metadata.remove(self.metadata.tables[id_table_name]) + id_table = Table( + id_table_name, + self.metadata, + Column("id", Unicode(255), primary_key=True), + Column("sf_id", Unicode(18)), + ) + if self.inspector.has_table(id_table_name): + id_table.drop() + id_table.create() + self._initialized_id_tables.add(id_table_name) + return id_table_name def _sqlite_load(self): """Read a SQLite script and initialize the in-memory database.""" @@ -659,7 +655,7 @@ def _init_mapping(self): mapping=self.mapping, sf=self.sf, namespace=self.project_config.project__package__namespace, - data_operation=DataOperationType.QUERY, + data_operation=DataOperationType.INSERT, inject_namespaces=self.options["inject_namespaces"], drop_missing=self.options["drop_missing_schema"], ) diff --git a/cumulusci/tasks/bulkdata/query_transformers.py b/cumulusci/tasks/bulkdata/query_transformers.py index 84c1e66207..cbc50e389a 100644 --- a/cumulusci/tasks/bulkdata/query_transformers.py +++ b/cumulusci/tasks/bulkdata/query_transformers.py @@ -7,7 +7,6 @@ from cumulusci.core.exceptions import BulkDataException Criterion = T.Any -ID_TABLE_NAME = "cumulusci_id_table" class LoadQueryExtender: @@ -51,9 +50,8 @@ def add_outerjoins(self, query: Query): class AddLookupsToQuery(LoadQueryExtender): """Adds columns and joins relatinng to lookups""" - def __init__(self, mapping, metadata, model, _old_format) -> None: + def __init__(self, mapping, metadata, model) -> None: super().__init__(mapping, metadata, model) - self._old_format = _old_format self.lookups = [ lookup for lookup in self.mapping.lookups.values() if not lookup.after ] @@ -61,7 +59,9 @@ def __init__(self, mapping, metadata, model, _old_format) -> None: @cached_property def columns_to_add(self): for lookup in self.lookups: - lookup.aliased_table = aliased(self.metadata.tables[ID_TABLE_NAME]) + lookup.aliased_table = aliased( + self.metadata.tables[f"{lookup.table}_sf_ids"] + ) return [lookup.aliased_table.columns.sf_id for lookup in self.lookups] @cached_property @@ -71,17 +71,10 @@ def outerjoins_to_add(self): def join_for_lookup(lookup): key_field = lookup.get_lookup_key_field(self.model) value_column = getattr(self.model, key_field) - if self._old_format: - return ( - lookup.aliased_table, - lookup.aliased_table.columns.id - == str(lookup.table) + "-" + value_column, - ) - else: - return ( - lookup.aliased_table, - lookup.aliased_table.columns.id == value_column, - ) + return ( + lookup.aliased_table, + lookup.aliased_table.columns.id == value_column, + ) return [join_for_lookup(lookup) for lookup in self.lookups] diff --git a/cumulusci/tasks/bulkdata/utils.py b/cumulusci/tasks/bulkdata/utils.py index 1d51ddc6cf..ea09ba49df 100644 --- a/cumulusci/tasks/bulkdata/utils.py +++ b/cumulusci/tasks/bulkdata/utils.py @@ -13,8 +13,6 @@ from cumulusci.core.exceptions import BulkDataException from cumulusci.utils.iterators import iterate_in_chunks -ID_TABLE_NAME = "cumulusci_id_table" - class SqlAlchemyMixin: logger: logging.Logger @@ -75,60 +73,16 @@ def _database_url(self): else: return self._temp_database_url() - def _id_generator_for_object(self, sobject: str): - if sobject not in self._id_generators: - - def _generate_ids(): - counter = 0 - while True: - yield f"{sobject}-{counter}" - counter += 1 - - self._id_generators[sobject] = _generate_ids() - - return self._id_generators[sobject] - - def _update_column( - self, *, source_model, target_model, key_field, join_field, target_field - ): - key_attr = getattr(source_model, key_field) - join_attr = getattr(target_model, join_field) - target_attr = getattr(target_model, target_field) - - id_column = inspect(source_model).primary_key[0].name - - try: - self.session.query(source_model).filter( - key_attr.isnot(None), key_attr == join_attr - ).update({key_attr: target_attr}, synchronize_session=False) - except NotImplementedError: - # Some databases, such as SQLite, don't support multitable update - # TODO: review memory consumption of this routine. - mappings = [] - for row, lookup_id in self.session.query(source_model, target_attr).join( - target_model, key_attr == join_attr - ): - mappings.append( - {id_column: getattr(row, id_column), key_field: lookup_id} - ) - self.session.bulk_update_mappings(source_model, mappings) - - def _update_sf_id_column(self, model, key_field): - self._update_column( - source_model=model, - target_model=self.models[self.ID_TABLE_NAME], - key_field=key_field, - join_field="sf_id", - target_field="id", - ) - def _is_autopk_database(self): - # If the type of the Id column on a mapping is INTEGER, - # this is an autopk database. +def _handle_primary_key(mapping, fields): + """Provide support for legacy mappings which used the OID as the pk but + default to using an autoincrementing int pk and a separate sf_id column""" - mapping = self.mapping.values()[0] - id_field = mapping.fields["Id"] - return isinstance(getattr(self.models[mapping.table], id_field).type, Integer) + if mapping.get_oid_as_pk(): + id_column = mapping.fields["Id"] + fields.append(Column(id_column, Unicode(255), primary_key=True)) + else: + fields.append(Column("id", Integer(), primary_key=True, autoincrement=True)) def create_table(mapping, metadata) -> Table: @@ -138,15 +92,10 @@ def create_table(mapping, metadata) -> Table: Mapping should be a MappingStep instance""" fields = [] - # _handle_primary_key(mapping, fields) - id_column = mapping.fields["Id"] # Guaranteed to be present by mapping parser. - fields.append(Column(id_column, Unicode(255), primary_key=True)) + _handle_primary_key(mapping, fields) # make a field list to create - # for field, db in mapping.get_complete_field_map().items(): - for field, db in zip( - mapping.get_extract_field_list(), mapping.get_database_column_list() - ).items(): + for field, db in mapping.get_complete_field_map().items(): if field == "Id": continue diff --git a/datasets/mapping.yml b/datasets/mapping.yml index 878c7fa9f7..ae7952b22c 100644 --- a/datasets/mapping.yml +++ b/datasets/mapping.yml @@ -35,23 +35,12 @@ Contact: lookups: AccountId: table: Account -Lead: - sf_object: Lead - api: rest - batch_size: 2 - fields: - - LastName - - Company -Event: - sf_object: Event +Opportunity: + sf_object: Opportunity api: bulk batch_size: 2 fields: - - Subject - - DurationInMinutes - - ActivityDateTime - lookups: - WhoId: - table: - - Contact - - Lead + - Name + - CloseDate + - Amount + - StageName diff --git a/datasets/sample.sql b/datasets/sample.sql index 921b10aadb..dc9bc8fdd6 100644 --- a/datasets/sample.sql +++ b/datasets/sample.sql @@ -1,64 +1,53 @@ BEGIN TRANSACTION; CREATE TABLE "Account" ( - "id" VARCHAR(255) NOT NULL, - "Name" VARCHAR(255), - "Description" VARCHAR(255), - "NumberOfEmployees" VARCHAR(255), - "BillingStreet" VARCHAR(255), - "BillingCity" VARCHAR(255), - "BillingState" VARCHAR(255), - "BillingPostalCode" VARCHAR(255), - "BillingCountry" VARCHAR(255), - "ShippingStreet" VARCHAR(255), - "ShippingCity" VARCHAR(255), - "ShippingState" VARCHAR(255), - "ShippingPostalCode" VARCHAR(255), - "ShippingCountry" VARCHAR(255), - "Phone" VARCHAR(255), - "Fax" VARCHAR(255), - "Website" VARCHAR(255), - "AccountNumber" VARCHAR(255), + id INTEGER NOT NULL, + "Name" VARCHAR(255), + "Description" VARCHAR(255), + "NumberOfEmployees" VARCHAR(255), + "BillingStreet" VARCHAR(255), + "BillingCity" VARCHAR(255), + "BillingState" VARCHAR(255), + "BillingPostalCode" VARCHAR(255), + "BillingCountry" VARCHAR(255), + "ShippingStreet" VARCHAR(255), + "ShippingCity" VARCHAR(255), + "ShippingState" VARCHAR(255), + "ShippingPostalCode" VARCHAR(255), + "ShippingCountry" VARCHAR(255), + "Phone" VARCHAR(255), + "Fax" VARCHAR(255), + "Website" VARCHAR(255), + "AccountNumber" VARCHAR(255), PRIMARY KEY (id) ); -INSERT INTO "Account" VALUES("Account-1",'alpha','','','Baker St.','','','','','','','','','','','','',''); -INSERT INTO "Account" VALUES("Account-2",'beta','','','Baker St.','','','','','','','','','','','','',''); -INSERT INTO "Account" VALUES("Account-3",'gamma','','','Baker St.','','','','','','','','','','','','',''); - +INSERT INTO "Account" VALUES(1,'Sample Account for Entitlements','','','','','','','','','','','','','','','',''); +INSERT INTO "Account" VALUES(2,'The Bluth Company','Solid as a rock','6','','','','','','','','','','','','','',''); +INSERT INTO "Account" VALUES(3,'Camacho PLC','Total logistical task-force','59908','2852 Caleb Village Suite 428','Porterside','Maryland','14525','Canada','6070 Davidson Rapids','Gibsonland','North Dakota','62676','Lithuania','221.285.1033','+1-081-230-6073x31438','http://jenkins.info/category/tag/tag/terms/','2679965'); CREATE TABLE "Contact" ( - "id" VARCHAR(255) NOT NULL, - "FirstName" VARCHAR(255), - "LastName" VARCHAR(255), - "Salutation" VARCHAR(255), - "Email" VARCHAR(255), - "Phone" VARCHAR(255), - "MobilePhone" VARCHAR(255), - "Title" VARCHAR(255), - "Birthdate" VARCHAR(255), - "AccountId" VARCHAR(255), + id INTEGER NOT NULL, + "FirstName" VARCHAR(255), + "LastName" VARCHAR(255), + "Salutation" VARCHAR(255), + "Email" VARCHAR(255), + "Phone" VARCHAR(255), + "MobilePhone" VARCHAR(255), + "Title" VARCHAR(255), + "Birthdate" VARCHAR(255), + "AccountId" VARCHAR(255), PRIMARY KEY (id) ); -INSERT INTO "Contact" VALUES("Contact-1",'alphass','Mannwereerevhefwingefew','','krithvtffder@example.com','','','','','Account-1'); -INSERT INTO "Contact" VALUES("Contact-2",'betasss','Blackefefererf','','kathjvhryn85@exaerfemple.com','','','','','Account-2'); -INSERT INTO "Contact" VALUES("Contact-3",'deltasss','Hunteererbhjbefrewererfef','','dfdfvgh@example.com','','','','','Account-3'); -INSERT INTO "Contact" VALUES("Contact-4",'gammasss','Carererfbhjhjbrlsonere','','johnmjbbhontddfgfdcsdcsces@example.com','','','','',''); -CREATE TABLE "Event" ( - "id" VARCHAR(255) NOT NULL, - "Subject" VARCHAR(255), - "DurationInMinutes" VARCHAR(255), - "ActivityDateTime" VARCHAR(255), - "WhoId" VARCHAR(255), +INSERT INTO "Contact" VALUES(1,'Michael','Bluth','','','','','','','2'); +INSERT INTO "Contact" VALUES(2,'Jared','Burnett','Ms.','ja-burnett2011@example.net','372.865.5762x5990','033.134.7156x7943','Systems analyst','2000-04-18','3'); +CREATE TABLE "Opportunity" ( + id INTEGER NOT NULL, + "Name" VARCHAR(255), + "CloseDate" VARCHAR(255), + "Amount" VARCHAR(255), + "StageName" VARCHAR(255), PRIMARY KEY (id) ); -INSERT INTO "Event" VALUES("Event-1",'helllo','60','2024-01-10T05:30:00.000Z','Contact-1'); -INSERT INTO "Event" VALUES("Event-2",'newer','60','2024-01-10T05:30:00.000Z','Lead-1'); - -CREATE TABLE "Lead" ( - "id" VARCHAR(255) NOT NULL, - "LastName" VARCHAR(255), - "Company" VARCHAR(255), - PRIMARY KEY (id) -); -INSERT INTO "Lead" VALUES("Lead-1",'deltassssds','Farmers Coop. of Florida'); -INSERT INTO "Lead" VALUES("Lead-2",'gauramm','Abbott Insurance'); - +INSERT INTO "Opportunity" VALUES(1,'democratic Opportunity','2022-07-27','69.0','In Progress'); +INSERT INTO "Opportunity" VALUES(2,'your Opportunity','2022-10-09','76.0','Closed Won'); +INSERT INTO "Opportunity" VALUES(3,'heart Opportunity','2022-11-04','32.0','Closed Won'); +INSERT INTO "Opportunity" VALUES(4,'treat Opportunity','2022-12-12','137.0','Closed Won'); COMMIT; From 28c2006695143bb391ded7e220a026a0f82f07b0 Mon Sep 17 00:00:00 2001 From: aditya-balachander Date: Tue, 16 Jan 2024 16:26:30 +0530 Subject: [PATCH 12/36] Added load changes --- cumulusci/tasks/bulkdata/extract.py | 8 +- cumulusci/tasks/bulkdata/load.py | 82 ++++++++++--------- cumulusci/tasks/bulkdata/mapping_parser.py | 8 -- .../tasks/bulkdata/query_transformers.py | 25 ++++-- cumulusci/tasks/bulkdata/utils.py | 7 +- datasets/mapping.yml | 2 + 6 files changed, 70 insertions(+), 62 deletions(-) diff --git a/cumulusci/tasks/bulkdata/extract.py b/cumulusci/tasks/bulkdata/extract.py index 9650f2d311..ea10317b39 100644 --- a/cumulusci/tasks/bulkdata/extract.py +++ b/cumulusci/tasks/bulkdata/extract.py @@ -227,14 +227,16 @@ def strip_name_field(record): # into two separate streams and load into the main table and the sf_id_table values, ids = itertools.tee(record_iterator) - id_source_global = self._id_generator_for_object(mapping.sf_object) + id_source_sobj, id_source_global = itertools.tee( + self._id_generator_for_object(mapping.sf_object) + ) f_ids = ((row[0], next(id_source_global)) for row in ids) - f_values = (row[1:] for row in values) + f_values = ([next(id_source_sobj)] + row[1:] for row in values) values_chunks = sql_bulk_insert_from_records_incremental( connection=conn, table=self.metadata.tables[mapping.table], - columns= columns[1:], + columns= ["id"] + columns[1:], record_iterable=f_values, ) ids_chunks = sql_bulk_insert_from_records_incremental( diff --git a/cumulusci/tasks/bulkdata/load.py b/cumulusci/tasks/bulkdata/load.py index 9fa66c2a4f..6b519c809f 100644 --- a/cumulusci/tasks/bulkdata/load.py +++ b/cumulusci/tasks/bulkdata/load.py @@ -124,6 +124,8 @@ def _init_options(self, kwargs): self.options["enable_rollback"] = process_bool_arg( self.options.get("enable_rollback", False) ) + self._id_generators = {} + self._old_format = False def _init_dataset(self): """Find the dataset paths to use with the following sequence: @@ -205,10 +207,11 @@ def _run_task(self): "No data will be loaded because this is a persistent org and no dataset was specified." ) return + self.ID_TABLE_NAME = "cumulusci_id_table" self._init_mapping() with self._init_db(): self._expand_mapping() - + self._initialize_id_table(self.reset_oids) start_step = self.options.get("start_step") started = False results = {} @@ -359,7 +362,8 @@ def check_simple_upsert(self, mapping): def _stream_queried_data(self, mapping, local_ids, query): """Get data from the local db""" - statics = self._get_statics(mapping) + # statics = self._get_statics(mapping) + staticizer = self._add_statics_to_row(mapping) total_rows = 0 if mapping.anchor_date: @@ -372,13 +376,13 @@ def _stream_queried_data(self, mapping, local_ids, query): batch_size = mapping.batch_size or DEFAULT_BULK_BATCH_SIZE for row in query.yield_per(batch_size): total_rows += 1 - # Add static values to row - pkey = row[0] - row = list(row[1:]) + statics + if mapping.anchor_date and (date_context[0] or date_context[1]): row = adjust_relative_dates( mapping, date_context, row, DataOperationType.INSERT ) + pkey = row[0] # FIXME: This is a local-DB ordering assumption. + row = staticizer(list(row[1:])) if mapping.action is DataOperationType.UPDATE: if len(row) > 1 and all([f is None for f in row[1:]]): # Skip update rows that contain no values @@ -389,7 +393,7 @@ def _stream_queried_data(self, mapping, local_ids, query): yield row self.logger.info( - f"Prepared {total_rows} rows for {mapping['action']} to {mapping['sf_object']}." + f"Prepared {total_rows} rows for {mapping.action.value} to {mapping.sf_object}." ) def _load_record_types(self, sobjects, conn): @@ -400,10 +404,9 @@ def _load_record_types(self, sobjects, conn): sobject, table_name, conn, self.org_config.is_person_accounts_enabled ) - def _get_statics(self, mapping): - """Return the static values (not column names) to be appended to - records for this mapping.""" + def _add_statics_to_row(self, mapping): statics = list(mapping.static.values()) + if mapping.record_type: query = ( f"SELECT Id FROM RecordType WHERE SObjectType='{mapping.sf_object}'" @@ -416,7 +419,10 @@ def _get_statics(self, mapping): raise BulkDataException(f"Cannot find RecordType with query `{query}`") statics.append(record_type_id) - return statics + def add_statics(row): + return row + statics + + return add_statics def _query_db(self, mapping): """Build a query to retrieve data from the local db. @@ -439,12 +445,14 @@ def _query_db(self, mapping): query = self.session.query(*columns) classes = [ - AddLookupsToQuery, AddRecordTypesToQuery, AddMappingFiltersToQuery, AddUpsertsToQuery, ] transformers = [cls(mapping, self.metadata, model) for cls in classes] + transformers.append( + AddLookupsToQuery(mapping, self.metadata, model, self._old_format) + ) if mapping.sf_object == "Contact" and self._can_load_person_accounts(mapping): transformers.append(AddPersonAccountsToQuery(mapping, self.metadata, model)) @@ -479,20 +487,25 @@ def _process_job_results(self, mapping, step, local_ids): DataOperationType.UPSERT, DataOperationType.ETL_UPSERT, ) - if is_insert_or_upsert: - id_table_name = self._initialize_id_table(mapping, self.reset_oids) - conn = self.session.connection() + conn = self.session.connection() sf_id_results = self._generate_results_id_map(step, local_ids) + for i in range(len(sf_id_results)): + if str(sf_id_results[i][0]).isnumeric(): + self._old_format = True + sf_id_results[i][0] = mapping.table + "-" + str(sf_id_results[i][0]) + else: + break # If we know we have no successful inserts, don't attempt to persist Ids. # Do, however, drain the generator to get error-checking behavior. if is_insert_or_upsert and ( step.job_result.records_processed - step.job_result.total_row_errors ): + table = self.metadata.tables[self.ID_TABLE_NAME] sql_bulk_insert_from_records( connection=conn, - table=self.metadata.tables[id_table_name], + table=table, columns=("id", "sf_id"), record_iterable=sf_id_results, ) @@ -510,7 +523,7 @@ def _process_job_results(self, mapping, step, local_ids): if account_id_lookup: sql_bulk_insert_from_records( connection=conn, - table=self.metadata.tables[id_table_name], + table=self.metadata.tables[self.ID_TABLE_NAME], columns=("id", "sf_id"), record_iterable=self._generate_contact_id_map_for_person_accounts( mapping, account_id_lookup, conn @@ -554,7 +567,7 @@ def _generate_results_id_map(self, step, local_ids): CreateRollback.prepare_for_rollback(self, step, created_results) return sf_id_results - def _initialize_id_table(self, mapping, should_reset_table): + def _initialize_id_table(self, should_reset_table): """initalize or find table to hold the inserted SF Ids The table has a name like xxx_sf_ids and has just two columns, id and sf_id. @@ -562,29 +575,20 @@ def _initialize_id_table(self, mapping, should_reset_table): If the table already exists, should_reset_table determines whether to drop and recreate it or not. """ - id_table_name = f"{mapping['table']}_sf_ids" - already_exists = id_table_name in self.metadata.tables + already_exists = self.ID_TABLE_NAME in self.metadata.tables if already_exists and not should_reset_table: - return id_table_name - - if not hasattr(self, "_initialized_id_tables"): - self._initialized_id_tables = set() - if id_table_name not in self._initialized_id_tables: - if already_exists: - self.metadata.remove(self.metadata.tables[id_table_name]) - id_table = Table( - id_table_name, - self.metadata, - Column("id", Unicode(255), primary_key=True), - Column("sf_id", Unicode(18)), - ) - if self.inspector.has_table(id_table_name): - id_table.drop() - id_table.create() - self._initialized_id_tables.add(id_table_name) - return id_table_name + return + id_table = Table( + self.ID_TABLE_NAME, + self.metadata, + Column("id", Unicode(255), primary_key=True), + Column("sf_id", Unicode(18)), + ) + if id_table.exists(): + id_table.drop() + id_table.create() def _sqlite_load(self): """Read a SQLite script and initialize the in-memory database.""" @@ -655,7 +659,7 @@ def _init_mapping(self): mapping=self.mapping, sf=self.sf, namespace=self.project_config.project__package__namespace, - data_operation=DataOperationType.INSERT, + data_operation=DataOperationType.QUERY, inject_namespaces=self.options["inject_namespaces"], drop_missing=self.options["drop_missing_schema"], ) @@ -1014,4 +1018,4 @@ class SetRecentlyViewedInfo(T.NamedTuple): """Did the set recently succeed or fail?""" sobject: str - error: T.Optional[Exception] + error: T.Optional[Exception] \ No newline at end of file diff --git a/cumulusci/tasks/bulkdata/mapping_parser.py b/cumulusci/tasks/bulkdata/mapping_parser.py index 919ba7ec52..3362202761 100644 --- a/cumulusci/tasks/bulkdata/mapping_parser.py +++ b/cumulusci/tasks/bulkdata/mapping_parser.py @@ -204,14 +204,6 @@ def get_extract_field_list(self): fields.extend([f for f in self.fields.keys() if f != "Id"]) fields.extend(self.lookups.keys()) - # If we're using Record Type mapping, `RecordTypeId` goes at the end. - # This makes it easier to manage the relationship with database columns. - if "RecordTypeId" in fields: - fields.remove("RecordTypeId") - - if "RecordTypeId" in self.fields: - fields.append("RecordTypeId") - return fields def get_relative_date_context(self, fields: List[str], sf: Salesforce): diff --git a/cumulusci/tasks/bulkdata/query_transformers.py b/cumulusci/tasks/bulkdata/query_transformers.py index cbc50e389a..a4861917c7 100644 --- a/cumulusci/tasks/bulkdata/query_transformers.py +++ b/cumulusci/tasks/bulkdata/query_transformers.py @@ -7,6 +7,7 @@ from cumulusci.core.exceptions import BulkDataException Criterion = T.Any +ID_TABLE_NAME = "cumulusci_id_table" class LoadQueryExtender: @@ -50,8 +51,9 @@ def add_outerjoins(self, query: Query): class AddLookupsToQuery(LoadQueryExtender): """Adds columns and joins relatinng to lookups""" - def __init__(self, mapping, metadata, model) -> None: + def __init__(self, mapping, metadata, model, _old_format) -> None: super().__init__(mapping, metadata, model) + self._old_format = _old_format self.lookups = [ lookup for lookup in self.mapping.lookups.values() if not lookup.after ] @@ -59,9 +61,7 @@ def __init__(self, mapping, metadata, model) -> None: @cached_property def columns_to_add(self): for lookup in self.lookups: - lookup.aliased_table = aliased( - self.metadata.tables[f"{lookup.table}_sf_ids"] - ) + lookup.aliased_table = aliased(self.metadata.tables[ID_TABLE_NAME]) return [lookup.aliased_table.columns.sf_id for lookup in self.lookups] @cached_property @@ -71,10 +71,17 @@ def outerjoins_to_add(self): def join_for_lookup(lookup): key_field = lookup.get_lookup_key_field(self.model) value_column = getattr(self.model, key_field) - return ( - lookup.aliased_table, - lookup.aliased_table.columns.id == value_column, - ) + if self._old_format: + return ( + lookup.aliased_table, + lookup.aliased_table.columns.id + == str(lookup.table) + "-" + value_column, + ) + else: + return ( + lookup.aliased_table, + lookup.aliased_table.columns.id == value_column, + ) return [join_for_lookup(lookup) for lookup in self.lookups] @@ -167,4 +174,4 @@ def filters_to_add(self): assert self.mapping.sf_object == "Contact" return [ func.lower(self.model.__table__.columns.get("IsPersonAccount")) == "false" - ] + ] \ No newline at end of file diff --git a/cumulusci/tasks/bulkdata/utils.py b/cumulusci/tasks/bulkdata/utils.py index ea09ba49df..d77738b25f 100644 --- a/cumulusci/tasks/bulkdata/utils.py +++ b/cumulusci/tasks/bulkdata/utils.py @@ -13,6 +13,7 @@ from cumulusci.core.exceptions import BulkDataException from cumulusci.utils.iterators import iterate_in_chunks +ID_TABLE_NAME = "cumulusci_id_table" class SqlAlchemyMixin: logger: logging.Logger @@ -78,11 +79,11 @@ def _handle_primary_key(mapping, fields): """Provide support for legacy mappings which used the OID as the pk but default to using an autoincrementing int pk and a separate sf_id column""" + id_column = "id" if mapping.get_oid_as_pk(): id_column = mapping.fields["Id"] - fields.append(Column(id_column, Unicode(255), primary_key=True)) - else: - fields.append(Column("id", Integer(), primary_key=True, autoincrement=True)) + + fields.append(Column(id_column, Unicode(255), primary_key=True)) def create_table(mapping, metadata) -> Table: diff --git a/datasets/mapping.yml b/datasets/mapping.yml index ae7952b22c..34ee85e32a 100644 --- a/datasets/mapping.yml +++ b/datasets/mapping.yml @@ -3,6 +3,7 @@ Account: api: bulk fields: - Name + - LastName - Description - NumberOfEmployees - BillingStreet @@ -19,6 +20,7 @@ Account: - Fax - Website - AccountNumber + - RecordTypeId Contact: sf_object: Contact From a484821b04e83b2713c8625934cdf640c895fac7 Mon Sep 17 00:00:00 2001 From: lakshmi2506 Date: Tue, 16 Jan 2024 18:12:49 +0530 Subject: [PATCH 13/36] person accounts backward compatability --- cumulusci/tasks/bulkdata/load.py | 34 ++++++++++++++++++++++---------- 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/cumulusci/tasks/bulkdata/load.py b/cumulusci/tasks/bulkdata/load.py index 6b519c809f..b2ef2ff61c 100644 --- a/cumulusci/tasks/bulkdata/load.py +++ b/cumulusci/tasks/bulkdata/load.py @@ -779,17 +779,31 @@ def _generate_contact_id_map_for_person_accounts( # create a Map: Account SF ID --> Contact ID. Outer join the # Account SF IDs table to get each Contact's associated # Account SF ID. - query = ( - self.session.query(contact_id_column, account_sf_id_column) - .filter( - func.lower(contact_model.__table__.columns.get("IsPersonAccount")) - == "true" + if self._old_format: + query = ( + self.session.query(contact_id_column, account_sf_id_column) + .filter( + func.lower(contact_model.__table__.columns.get("IsPersonAccount")) + == "true" + ) + .outerjoin( + account_sf_ids_table, + account_sf_ids_table.columns["id"] + == str(account_id_lookup.table) + "-" + account_id_column, + ) ) - .outerjoin( - account_sf_ids_table, - account_sf_ids_table.columns["id"] == account_id_column, + else: + query = ( + self.session.query(contact_id_column, account_sf_id_column) + .filter( + func.lower(contact_model.__table__.columns.get("IsPersonAccount")) + == "true" + ) + .outerjoin( + account_sf_ids_table, + account_sf_ids_table.columns["id"] == account_id_column, + ) ) - ) # Stream the results so we can process batches of 200 Contacts # in case we have large data volumes. @@ -1018,4 +1032,4 @@ class SetRecentlyViewedInfo(T.NamedTuple): """Did the set recently succeed or fail?""" sobject: str - error: T.Optional[Exception] \ No newline at end of file + error: T.Optional[Exception] From 17bf608a9a157c17dfbb3adeb07a52e1b5ea39d6 Mon Sep 17 00:00:00 2001 From: lakshmi2506 Date: Tue, 16 Jan 2024 20:22:37 +0530 Subject: [PATCH 14/36] person accounts sf_id table corrected --- cumulusci/tasks/bulkdata/load.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cumulusci/tasks/bulkdata/load.py b/cumulusci/tasks/bulkdata/load.py index b2ef2ff61c..ebe4a234b8 100644 --- a/cumulusci/tasks/bulkdata/load.py +++ b/cumulusci/tasks/bulkdata/load.py @@ -831,7 +831,10 @@ def _generate_contact_id_map_for_person_accounts( contact_sf_id = record["Id"] # Join maps together to get tuple (Contact ID, Contact SF ID) to insert into step's ID Table. - yield (contact_id, contact_sf_id) + if self._old_format: + yield (contact_mapping.table + "-" + contact_id, contact_sf_id) + else: + yield (contact_id, contact_sf_id) def _set_viewed(self) -> T.List["SetRecentlyViewedInfo"]: """Set items as recently viewed. Filter out custom objects without custom tabs.""" From 5d3b7bee5e62aa434642061c44d9bc3b07069400 Mon Sep 17 00:00:00 2001 From: aditya-balachander Date: Wed, 17 Jan 2024 11:18:58 +0530 Subject: [PATCH 15/36] Updated load.py --- cumulusci/tasks/bulkdata/load.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cumulusci/tasks/bulkdata/load.py b/cumulusci/tasks/bulkdata/load.py index ebe4a234b8..42eb0e8dad 100644 --- a/cumulusci/tasks/bulkdata/load.py +++ b/cumulusci/tasks/bulkdata/load.py @@ -659,7 +659,7 @@ def _init_mapping(self): mapping=self.mapping, sf=self.sf, namespace=self.project_config.project__package__namespace, - data_operation=DataOperationType.QUERY, + data_operation=DataOperationType.INSERT, inject_namespaces=self.options["inject_namespaces"], drop_missing=self.options["drop_missing_schema"], ) From 17efaddc6d10cd01a98533fa0fdb27fcd7c74450 Mon Sep 17 00:00:00 2001 From: aditya-balachander Date: Wed, 17 Jan 2024 11:53:19 +0530 Subject: [PATCH 16/36] Update mapping_parser.py --- cumulusci/tasks/bulkdata/mapping_parser.py | 57 ++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/cumulusci/tasks/bulkdata/mapping_parser.py b/cumulusci/tasks/bulkdata/mapping_parser.py index 3362202761..2ae1fbe2d7 100644 --- a/cumulusci/tasks/bulkdata/mapping_parser.py +++ b/cumulusci/tasks/bulkdata/mapping_parser.py @@ -597,6 +597,60 @@ def parse_from_yaml(source: Union[str, Path, IO]) -> Dict: return MappingSteps.parse_from_yaml(source) +def _infer_and_validate_lookups(mapping: Dict, sf: Salesforce): + sf_objects = [m.sf_object for m in mapping.values()] + + fail = False + + for idx, m in enumerate(mapping.values()): + describe = CaseInsensitiveDict( + { + f["name"]: f + for f in getattr(sf, m.sf_object).describe()[ + "fields" + ] + } + ) + + for lookup in m.lookups.values(): + if lookup.after: + # If configured by the user, skip. + # TODO: do we need more validation here? + continue + + field_describe = describe[lookup.name] + target_objects = field_describe["referenceTo"] + if len(target_objects) == 1: + # This is a non-polymorphic lookup. + try: + target_index = sf_objects.index(target_objects[0]) + except ValueError: + fail = True + logger.error( + f"The field {m.sf_object}.{lookup.name} looks up to {target_objects[0]}, which is not included in the operation" + ) + continue + + if target_index > idx or target_index == idx: + # This is a non-polymorphic after step. + lookup.after = mapping.keys()[idx] + else: + # This is a polymorphic lookup. + # Make sure that any lookup targets present in the operation precede this step. + target_indices = [sf_objects.index(t) for t in target_objects] + if not all([target_index < idx for target_index in target_indices]): + logger.error( + f"All included target objects ({','.join(target_objects)}) for the field {m.sf_object}.{lookup.name} " + f"must precede {m.sf_object} in the mapping." + ) + fail = True + continue + + if fail: + raise BulkDataException( + "One or more relationship errors blocked the operation." + ) + def validate_and_inject_mapping( *, mapping: Dict, @@ -645,6 +699,9 @@ def validate_and_inject_mapping( f"{describe[field]['referenceTo']} was removed from the operation " "due to missing permissions." ) + + # Synthesize `after` declarations and validate lookup references. + _infer_and_validate_lookups(mapping, sf) # If the org has person accounts enable, add a field mapping to track "IsPersonAccount". # IsPersonAccount field values are used to properly load person account records. From b7b22e0975b10b533f45698cc7af61c0424be324 Mon Sep 17 00:00:00 2001 From: aditya-balachander Date: Wed, 17 Jan 2024 12:39:04 +0530 Subject: [PATCH 17/36] Added validation for mapping --- cumulusci/tasks/bulkdata/mapping_parser.py | 36 +++++++++++++++++++--- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/cumulusci/tasks/bulkdata/mapping_parser.py b/cumulusci/tasks/bulkdata/mapping_parser.py index 2ae1fbe2d7..4fc1ad4400 100644 --- a/cumulusci/tasks/bulkdata/mapping_parser.py +++ b/cumulusci/tasks/bulkdata/mapping_parser.py @@ -599,6 +599,7 @@ def parse_from_yaml(source: Union[str, Path, IO]) -> Dict: def _infer_and_validate_lookups(mapping: Dict, sf: Salesforce): sf_objects = [m.sf_object for m in mapping.values()] + table_sf_obj = {m.table: m.sf_object for m in mapping.values()} fail = False @@ -619,7 +620,32 @@ def _infer_and_validate_lookups(mapping: Dict, sf: Salesforce): continue field_describe = describe[lookup.name] - target_objects = field_describe["referenceTo"] + reference_to_objects = field_describe["referenceTo"] + target_objects = [] + + lookup_tables = [] + if type(lookup.table)==str: + lookup_tables = [lookup.table] + else: + lookup_tables = lookup.table + + for table in lookup_tables: + try: + sf_object = table_sf_obj[table] + if sf_object in reference_to_objects: + target_objects.append(sf_object) + else: + logger.error(f"The table {table} is not a valid lookup for {lookup.name} in sf_object: {m.sf_object}") + fail = True + except KeyError: + logger.error(f"The table {table} does not exists in the mapping file") + fail = True + + if fail: + continue + + + if len(target_objects) == 1: # This is a non-polymorphic lookup. try: @@ -633,7 +659,7 @@ def _infer_and_validate_lookups(mapping: Dict, sf: Salesforce): if target_index > idx or target_index == idx: # This is a non-polymorphic after step. - lookup.after = mapping.keys()[idx] + lookup.after = list(mapping.keys())[idx] else: # This is a polymorphic lookup. # Make sure that any lookup targets present in the operation precede this step. @@ -661,6 +687,7 @@ def validate_and_inject_mapping( drop_missing: bool, org_has_person_accounts_enabled: bool = False, ): + is_load = True if data_operation == DataOperationType.INSERT else False should_continue = [ m.validate_and_inject_namespace( sf, namespace, data_operation, inject_namespaces, drop_missing @@ -700,8 +727,9 @@ def validate_and_inject_mapping( "due to missing permissions." ) - # Synthesize `after` declarations and validate lookup references. - _infer_and_validate_lookups(mapping, sf) + if is_load: + # Synthesize `after` declarations and validate lookup references. + _infer_and_validate_lookups(mapping, sf) # If the org has person accounts enable, add a field mapping to track "IsPersonAccount". # IsPersonAccount field values are used to properly load person account records. From cbdebb54cdc6e3c90f4a183ec592053e70a64f47 Mon Sep 17 00:00:00 2001 From: lakshmi2506 Date: Wed, 17 Jan 2024 15:03:03 +0530 Subject: [PATCH 18/36] test_cases --- cumulusci/tasks/bulkdata/tests/test_load.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/cumulusci/tasks/bulkdata/tests/test_load.py b/cumulusci/tasks/bulkdata/tests/test_load.py index a4293910dd..07583aac97 100644 --- a/cumulusci/tasks/bulkdata/tests/test_load.py +++ b/cumulusci/tasks/bulkdata/tests/test_load.py @@ -129,8 +129,8 @@ def test_run(self, dml_mock): ["Error", "User", "error@example.com", "001000000000000"], ] with create_engine(task.options["database_url"]).connect() as c: - hh_ids = next(c.execute("SELECT * from households_sf_ids")) - assert hh_ids == ("1", "001000000000000") + hh_ids = next(c.execute("SELECT * from cumulusci_id_table")) + assert hh_ids == ("households-1", "001000000000000") @responses.activate @mock.patch("cumulusci.tasks.bulkdata.load.get_dml_operation") @@ -271,6 +271,7 @@ def test_run_task__start_step(self): }, ) task._init_db = mock.Mock(return_value=nullcontext()) + task._initialize_id_table = mock.Mock() task._init_mapping = mock.Mock() task.mapping = {} task.mapping["Insert Households"] = MappingStep(sf_object="one", fields={}) @@ -298,6 +299,7 @@ def test_run_task__after_steps(self): task._init_db = mock.Mock(return_value=nullcontext()) task._init_mapping = mock.Mock() task._expand_mapping = mock.Mock() + task._initialize_id_table = mock.Mock() task.mapping = {} one = task.mapping["Insert Households"] = mock.Mock() two = task.mapping["Insert Contacts"] = mock.Mock() @@ -324,6 +326,7 @@ def test_run_task__after_steps_failure(self): task._init_db = mock.Mock(return_value=nullcontext()) task._init_mapping = mock.Mock() task._expand_mapping = mock.Mock() + task._initialize_id_table = mock.Mock() task.mapping = {} task.mapping["Insert Households"] = 1 task.mapping["Insert Contacts"] = 2 From 9e9a3ed7b63c7a9e55dce1c13a41c0832c8ee507 Mon Sep 17 00:00:00 2001 From: aditya-balachander Date: Wed, 17 Jan 2024 17:16:13 +0530 Subject: [PATCH 19/36] Fix test cases for extract and e2e --- .../tests/cassettes/TestDatasetsE2E.test_datasets_e2e.yaml | 2 +- ...tDatasetsE2E.test_datasets_extract_standard_objects.yaml | 2 +- cumulusci/tasks/bulkdata/tests/test_extract.py | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cumulusci/core/tests/cassettes/TestDatasetsE2E.test_datasets_e2e.yaml b/cumulusci/core/tests/cassettes/TestDatasetsE2E.test_datasets_e2e.yaml index 40aeecf89f..5833b12d30 100644 --- a/cumulusci/core/tests/cassettes/TestDatasetsE2E.test_datasets_e2e.yaml +++ b/cumulusci/core/tests/cassettes/TestDatasetsE2E.test_datasets_e2e.yaml @@ -36,7 +36,7 @@ interactions: \n } ]\n}" - request: method: GET - uri: https://orgname.my.salesforce.com/services/data/vxx.0/query/?q=SELECT+Id%2C+AccountNumber%2C+AccountSource%2C+AnnualRevenue%2C+BillingCity%2C+BillingCountry%2C+BillingGeocodeAccuracy%2C+BillingLatitude%2C+BillingLongitude%2C+BillingPostalCode%2C+BillingState%2C+BillingStreet%2C+CleanStatus%2C+DunsNumber%2C+Fax%2C+Industry%2C+Jigsaw%2C+NaicsCode%2C+NaicsDesc%2C+Name%2C+NumberOfEmployees%2C+Ownership%2C+Phone%2C+Rating%2C+ShippingCity%2C+ShippingCountry%2C+ShippingGeocodeAccuracy%2C+ShippingLatitude%2C+ShippingLongitude%2C+ShippingPostalCode%2C+ShippingState%2C+ShippingStreet%2C+Sic%2C+SicDesc%2C+Site%2C+TickerSymbol%2C+Tradestyle%2C+Type%2C+Website%2C+YearStarted%2C+ParentId%2C+RecordTypeId+FROM+Account+WHERE+Name+%21%3D+%27Sample+Account+for+Entitlements%27 + uri: https://orgname.my.salesforce.com/services/data/vxx.0/query/?q=SELECT+Id%2C+AccountNumber%2C+AccountSource%2C+AnnualRevenue%2C+BillingCity%2C+BillingCountry%2C+BillingGeocodeAccuracy%2C+BillingLatitude%2C+BillingLongitude%2C+BillingPostalCode%2C+BillingState%2C+BillingStreet%2C+CleanStatus%2C+DunsNumber%2C+Fax%2C+Industry%2C+Jigsaw%2C+NaicsCode%2C+NaicsDesc%2C+Name%2C+NumberOfEmployees%2C+Ownership%2C+Phone%2C+Rating%2C+RecordTypeId%2C+ShippingCity%2C+ShippingCountry%2C+ShippingGeocodeAccuracy%2C+ShippingLatitude%2C+ShippingLongitude%2C+ShippingPostalCode%2C+ShippingState%2C+ShippingStreet%2C+Sic%2C+SicDesc%2C+Site%2C+TickerSymbol%2C+Tradestyle%2C+Type%2C+Website%2C+YearStarted%2C+ParentId+FROM+Account+WHERE+Name+%21%3D+%27Sample+Account+for+Entitlements%27 body: null headers: *id001 response: diff --git a/cumulusci/core/tests/cassettes/TestDatasetsE2E.test_datasets_extract_standard_objects.yaml b/cumulusci/core/tests/cassettes/TestDatasetsE2E.test_datasets_extract_standard_objects.yaml index c2392fc218..9f1f861b89 100644 --- a/cumulusci/core/tests/cassettes/TestDatasetsE2E.test_datasets_extract_standard_objects.yaml +++ b/cumulusci/core/tests/cassettes/TestDatasetsE2E.test_datasets_extract_standard_objects.yaml @@ -29,7 +29,7 @@ interactions: \n } ]\n}" - request: method: GET - uri: https://orgname.my.salesforce.com/services/data/vxx.0/query/?q=SELECT+Id%2C+AccountNumber%2C+AccountSource%2C+AnnualRevenue%2C+BillingCity%2C+BillingCountry%2C+BillingGeocodeAccuracy%2C+BillingLatitude%2C+BillingLongitude%2C+BillingPostalCode%2C+BillingState%2C+BillingStreet%2C+CleanStatus%2C+Description%2C+DunsNumber%2C+Fax%2C+Industry%2C+Jigsaw%2C+NaicsCode%2C+NaicsDesc%2C+Name%2C+NumberOfEmployees%2C+Ownership%2C+Phone%2C+Rating%2C+ShippingCity%2C+ShippingCountry%2C+ShippingGeocodeAccuracy%2C+ShippingLatitude%2C+ShippingLongitude%2C+ShippingPostalCode%2C+ShippingState%2C+ShippingStreet%2C+Sic%2C+SicDesc%2C+Site%2C+TickerSymbol%2C+Tradestyle%2C+Type%2C+Website%2C+YearStarted%2C+ParentId%2C+RecordTypeId+FROM+Account+WHERE+Name+%21%3D+%27Sample+Account+for+Entitlements%27 + uri: https://orgname.my.salesforce.com/services/data/vxx.0/query/?q=SELECT+Id%2C+AccountNumber%2C+AccountSource%2C+AnnualRevenue%2C+BillingCity%2C+BillingCountry%2C+BillingGeocodeAccuracy%2C+BillingLatitude%2C+BillingLongitude%2C+BillingPostalCode%2C+BillingState%2C+BillingStreet%2C+CleanStatus%2C+Description%2C+DunsNumber%2C+Fax%2C+Industry%2C+Jigsaw%2C+NaicsCode%2C+NaicsDesc%2C+Name%2C+NumberOfEmployees%2C+Ownership%2C+Phone%2C+Rating%2C+RecordTypeId%2C+ShippingCity%2C+ShippingCountry%2C+ShippingGeocodeAccuracy%2C+ShippingLatitude%2C+ShippingLongitude%2C+ShippingPostalCode%2C+ShippingState%2C+ShippingStreet%2C+Sic%2C+SicDesc%2C+Site%2C+TickerSymbol%2C+Tradestyle%2C+Type%2C+Website%2C+YearStarted%2C+ParentId+FROM+Account+WHERE+Name+%21%3D+%27Sample+Account+for+Entitlements%27 body: null headers: *id002 response: diff --git a/cumulusci/tasks/bulkdata/tests/test_extract.py b/cumulusci/tasks/bulkdata/tests/test_extract.py index 834cf96dd1..7c76a8582c 100644 --- a/cumulusci/tasks/bulkdata/tests/test_extract.py +++ b/cumulusci/tasks/bulkdata/tests/test_extract.py @@ -565,7 +565,7 @@ def test_import_results__no_columns(self): # , query_op_mock): output_Opportunties = list( task.session.execute("select * from Opportunity") ) - assert output_Opportunties == [(1,), (2,)] + assert output_Opportunties == [('Opportunity-1',), ('Opportunity-2',)] @responses.activate def test_import_results__relative_dates(self): @@ -1185,8 +1185,8 @@ def test_import_results__autopk(self, create_task_fixture): task() with create_engine(task.options["database_url"]).connect() as conn: output_accounts = list(conn.execute("select * from Account")) - assert output_accounts[0][0:2] == (1, "Account1") - assert output_accounts[1][0:2] == (2, "Account2") + assert output_accounts[0][0:2] == ('Account-1', "Account1") + assert output_accounts[1][0:2] == ('Account-2', "Account2") assert len(output_accounts) == 2 output_opportunities = list(conn.execute("select * from Opportunity")) From 007f73c0130d4f0fc30e4802f285723faec00b91 Mon Sep 17 00:00:00 2001 From: lakshmi2506 Date: Wed, 17 Jan 2024 17:18:04 +0530 Subject: [PATCH 20/36] failed test cases corrected --- cumulusci/tasks/bulkdata/load.py | 22 +++++----- cumulusci/tasks/bulkdata/tests/test_load.py | 40 +++++++++---------- .../test_query_db__joins_self_lookups.sql | 8 ++-- 3 files changed, 34 insertions(+), 36 deletions(-) diff --git a/cumulusci/tasks/bulkdata/load.py b/cumulusci/tasks/bulkdata/load.py index 42eb0e8dad..ecac47a04c 100644 --- a/cumulusci/tasks/bulkdata/load.py +++ b/cumulusci/tasks/bulkdata/load.py @@ -126,6 +126,7 @@ def _init_options(self, kwargs): ) self._id_generators = {} self._old_format = False + self.ID_TABLE_NAME = "cumulusci_id_table" def _init_dataset(self): """Find the dataset paths to use with the following sequence: @@ -207,7 +208,6 @@ def _run_task(self): "No data will be loaded because this is a persistent org and no dataset was specified." ) return - self.ID_TABLE_NAME = "cumulusci_id_table" self._init_mapping() with self._init_db(): self._expand_mapping() @@ -362,8 +362,7 @@ def check_simple_upsert(self, mapping): def _stream_queried_data(self, mapping, local_ids, query): """Get data from the local db""" - # statics = self._get_statics(mapping) - staticizer = self._add_statics_to_row(mapping) + statics = self._get_statics(mapping) total_rows = 0 if mapping.anchor_date: @@ -376,13 +375,14 @@ def _stream_queried_data(self, mapping, local_ids, query): batch_size = mapping.batch_size or DEFAULT_BULK_BATCH_SIZE for row in query.yield_per(batch_size): total_rows += 1 + pkey = row[0] + row = list(row[1:]) + statics if mapping.anchor_date and (date_context[0] or date_context[1]): row = adjust_relative_dates( mapping, date_context, row, DataOperationType.INSERT ) - pkey = row[0] # FIXME: This is a local-DB ordering assumption. - row = staticizer(list(row[1:])) + if mapping.action is DataOperationType.UPDATE: if len(row) > 1 and all([f is None for f in row[1:]]): # Skip update rows that contain no values @@ -404,9 +404,10 @@ def _load_record_types(self, sobjects, conn): sobject, table_name, conn, self.org_config.is_person_accounts_enabled ) - def _add_statics_to_row(self, mapping): + def _get_statics(self, mapping): + """Return the static values (not column names) to be appended to + records for this mapping.""" statics = list(mapping.static.values()) - if mapping.record_type: query = ( f"SELECT Id FROM RecordType WHERE SObjectType='{mapping.sf_object}'" @@ -419,10 +420,7 @@ def _add_statics_to_row(self, mapping): raise BulkDataException(f"Cannot find RecordType with query `{query}`") statics.append(record_type_id) - def add_statics(row): - return row + statics - - return add_statics + return statics def _query_db(self, mapping): """Build a query to retrieve data from the local db. @@ -580,6 +578,8 @@ def _initialize_id_table(self, should_reset_table): if already_exists and not should_reset_table: return + elif already_exists: + self.metadata.remove(self.metadata.tables[self.ID_TABLE_NAME]) id_table = Table( self.ID_TABLE_NAME, self.metadata, diff --git a/cumulusci/tasks/bulkdata/tests/test_load.py b/cumulusci/tasks/bulkdata/tests/test_load.py index 07583aac97..772dc440ab 100644 --- a/cumulusci/tasks/bulkdata/tests/test_load.py +++ b/cumulusci/tasks/bulkdata/tests/test_load.py @@ -830,11 +830,12 @@ def test_stream_queried_data__adjusts_relative_dates(self): ["001000000003", None], ] ) - local_ids = io.StringIO() + print(local_ids) records = list( task._stream_queried_data(mapping, local_ids, task._query_db(mapping)) ) + print(records) assert [[(date.today() + timedelta(days=9)).isoformat()], [None]] == records def test_get_statics(self): @@ -876,9 +877,9 @@ def test_query_db__joins_self_lookups(self): sql_path=Path(__file__).parent / "test_query_db__joins_self_lookups.sql", mapping=Path(__file__).parent / "test_query_db__joins_self_lookups.yml", mapping_step_name="Update Accounts", - expected="""SELECT accounts.sf_id AS accounts_sf_id, accounts."Name" AS "accounts_Name", accounts_sf_ids_1.sf_id AS accounts_sf_ids_1_sf_id -FROM accounts LEFT OUTER JOIN accounts_sf_ids AS accounts_sf_ids_1 ON accounts_sf_ids_1.id = accounts.parent_id ORDER BY accounts.parent_id - """, + expected="""SELECT accounts.sf_id AS accounts_sf_id, accounts."Name" AS "accounts_Name", cumulusci_id_table_1.sf_id AS cumulusci_id_table_1_sf_id + FROM accounts LEFT OUTER JOIN cumulusci_id_table AS cumulusci_id_table_1 ON cumulusci_id_table_1.id = accounts.parent_id ORDER BY accounts.parent_id + """, ) @responses.activate @@ -949,8 +950,7 @@ def test_query_db__person_accounts_enabled__contact_mapping(self, aliased): task.models = {"contacts": model} task.metadata = mock.Mock() task.metadata.tables = { - "contacts_sf_ids": mock.Mock(), - "accounts_sf_ids": mock.Mock(), + "cumulusci_id_table": mock.Mock(), } task.session = mock.Mock() task._can_load_person_accounts = mock.Mock(return_value=True) @@ -1017,8 +1017,7 @@ def test_query_db__person_accounts_disabled__contact_mapping(self, aliased): task.models = {"contacts": model} task.metadata = mock.Mock() task.metadata.tables = { - "contacts_sf_ids": mock.Mock(), - "accounts_sf_ids": mock.Mock(), + "cumulusci_id_table": mock.Mock(), } task.session = mock.Mock() task._can_load_person_accounts = mock.Mock(return_value=False) @@ -1089,8 +1088,7 @@ def test_query_db__person_accounts_enabled__neither_account_nor_contact_mapping( task.models = {"requests": model} task.metadata = mock.Mock() task.metadata.tables = { - "requests_sf_ids": mock.Mock(), - "accounts_sf_ids": mock.Mock(), + "cumulusci_id_table": mock.Mock(), } task.session = mock.Mock() task._can_load_person_accounts = mock.Mock(return_value=True) @@ -1144,13 +1142,13 @@ def test_initialize_id_table__already_exists(self): task.mapping = {} with task._init_db(): id_table = Table( - "test_sf_ids", + "cumulusci_id_table", task.metadata, Column("id", Unicode(255), primary_key=True), ) id_table.create() - task._initialize_id_table({"table": "test"}, True) - new_id_table = task.metadata.tables["test_sf_ids"] + task._initialize_id_table(True) + new_id_table = task.metadata.tables["cumulusci_id_table"] assert not (new_id_table is id_table) def test_initialize_id_table__already_exists_and_should_not_reset_table(self): @@ -1161,14 +1159,13 @@ def test_initialize_id_table__already_exists_and_should_not_reset_table(self): task.mapping = {} with task._init_db(): id_table = Table( - "test_sf_ids", + "cumulusci_id_table", task.metadata, Column("id", Unicode(255), primary_key=True), ) id_table.create() - table_name = task._initialize_id_table({"table": "test"}, False) - assert table_name == "test_sf_ids" - new_id_table = task.metadata.tables["test_sf_ids"] + task._initialize_id_table(False) + new_id_table = task.metadata.tables["cumulusci_id_table"] assert new_id_table is id_table def test_run_task__exception_failure(self): @@ -1183,6 +1180,7 @@ def test_run_task__exception_failure(self): DataOperationStatus.JOB_FAILURE, [], 0, 0 ) ) + task._initialize_id_table = mock.Mock() task.mapping = {"Test": MappingStep(sf_object="Account")} with pytest.raises(BulkDataException): @@ -1218,7 +1216,6 @@ def test_process_job_results__insert_success(self): task._process_job_results(mapping, step, local_ids) task.session.connection.assert_called_once() - task._initialize_id_table.assert_called_once_with(mapping, True) sql_bulk_insert_from_records.assert_called_once() task.session.commit.assert_called_once() @@ -1268,7 +1265,6 @@ def test_process_job_results__insert_rows_fail(self): task._process_job_results(mapping, step, local_ids) task.session.connection.assert_called_once() - task._initialize_id_table.assert_called_once_with(mapping, True) sql_bulk_insert_from_records.assert_not_called() task.session.commit.assert_called_once() assert len(task.logger.mock_calls) == 4 @@ -2057,8 +2053,8 @@ def test_run__autopk(self, dml_mock): ], step.records with create_engine(task.options["database_url"]).connect() as c: - hh_ids = next(c.execute("SELECT * from households_sf_ids")) - assert hh_ids == ("1", "001000000000000") + hh_ids = next(c.execute("SELECT * from cumulusci_id_table")) + assert hh_ids == ("households-1", "001000000000000") @responses.activate def test_run__complex_lookups(self): @@ -2818,6 +2814,7 @@ def test_set_viewed__exception(self): ) task._init_db = mock.Mock(return_value=nullcontext()) task._init_mapping = mock.Mock() + task._initialize_id_table = mock.Mock() task.mapping = {} task.after_steps = {} @@ -2840,6 +2837,7 @@ def test_no_mapping(self): }, ) task._init_db = mock.Mock(return_value=nullcontext()) + with pytest.raises(TaskOptionsError, match="Mapping file path required"): task() diff --git a/cumulusci/tasks/bulkdata/tests/test_query_db__joins_self_lookups.sql b/cumulusci/tasks/bulkdata/tests/test_query_db__joins_self_lookups.sql index 51f08ac05e..7ccd4285db 100644 --- a/cumulusci/tasks/bulkdata/tests/test_query_db__joins_self_lookups.sql +++ b/cumulusci/tasks/bulkdata/tests/test_query_db__joins_self_lookups.sql @@ -8,10 +8,10 @@ CREATE TABLE "accounts" ( INSERT INTO "accounts" VALUES("001DEADBEEF",'Bluth',''); INSERT INTO "accounts" VALUES("002DEADBEEF",'Funke-Bluth',1); -CREATE TABLE "accounts_sf_ids" ( - id INTEGER NOT NULL, +CREATE TABLE "cumulusci_id_table" ( + id VARCHAR(255) NOT NULL, sf_id VARCHAR(255) ); -INSERT INTO "accounts_sf_ids" VALUES(1,'001DEADBEEF'); -INSERT INTO "accounts_sf_ids" VALUES(2,'002DEADBEEF'); +INSERT INTO "cumulusci_id_table" VALUES("accounts-1",'001DEADBEEF'); +INSERT INTO "cumulusci_id_table" VALUES("accounts-2",'002DEADBEEF'); COMMIT; From 8d386ef6c605f145a24ee6393f59c8875891e040 Mon Sep 17 00:00:00 2001 From: lakshmi2506 Date: Thu, 18 Jan 2024 13:12:20 +0530 Subject: [PATCH 21/36] tests --- .../tasks/bulkdata/tests/person_accounts.sql | 11 ++---- cumulusci/tasks/bulkdata/tests/test_load.py | 28 +++++++++----- .../test_query_db__joins_self_lookups.sql | 8 ++-- .../tests/test_query_db_joins_lookups.sql | 37 +++++++++++++++++++ .../tests/test_query_db_joins_lookups.yml | 27 ++++++++++++++ 5 files changed, 89 insertions(+), 22 deletions(-) create mode 100644 cumulusci/tasks/bulkdata/tests/test_query_db_joins_lookups.sql create mode 100644 cumulusci/tasks/bulkdata/tests/test_query_db_joins_lookups.yml diff --git a/cumulusci/tasks/bulkdata/tests/person_accounts.sql b/cumulusci/tasks/bulkdata/tests/person_accounts.sql index fc128a8fb3..336f847a53 100644 --- a/cumulusci/tasks/bulkdata/tests/person_accounts.sql +++ b/cumulusci/tasks/bulkdata/tests/person_accounts.sql @@ -23,16 +23,11 @@ CREATE TABLE IF NOT EXISTS "Account" ( PRIMARY KEY (id) ); INSERT INTO Account VALUES(1,'Ana','Jacobson','61034 Calderon Point','Robinview','Michigan','USA','38677','AnJacobson1997@example.com','(330)320-6522','001-377-923-2289','1'); -CREATE TABLE IF NOT EXISTS "Account_sf_ids" ( +CREATE TABLE IF NOT EXISTS "cumulusci_id_table" ( id VARCHAR(255) NOT NULL, sf_id VARCHAR(18), PRIMARY KEY (id) ); -INSERT INTO Account_sf_ids VALUES('1','0016300001AKkZyAAL'); -CREATE TABLE IF NOT EXISTS "PersonContact_sf_ids" ( - id VARCHAR(255) NOT NULL, - sf_id VARCHAR(18), - PRIMARY KEY (id) -); -INSERT INTO PersonContact_sf_ids VALUES('1','0036300000wjVUBAA2'); +INSERT INTO cumulusci_id_table VALUES('Account-1','0016300001AKkZyAAL'); +INSERT INTO cumulusci_id_table VALUES('Contact-1','0036300000wjVUBAA2'); COMMIT; diff --git a/cumulusci/tasks/bulkdata/tests/test_load.py b/cumulusci/tasks/bulkdata/tests/test_load.py index 772dc440ab..2c7e97bf00 100644 --- a/cumulusci/tasks/bulkdata/tests/test_load.py +++ b/cumulusci/tasks/bulkdata/tests/test_load.py @@ -877,11 +877,18 @@ def test_query_db__joins_self_lookups(self): sql_path=Path(__file__).parent / "test_query_db__joins_self_lookups.sql", mapping=Path(__file__).parent / "test_query_db__joins_self_lookups.yml", mapping_step_name="Update Accounts", - expected="""SELECT accounts.sf_id AS accounts_sf_id, accounts."Name" AS "accounts_Name", cumulusci_id_table_1.sf_id AS cumulusci_id_table_1_sf_id - FROM accounts LEFT OUTER JOIN cumulusci_id_table AS cumulusci_id_table_1 ON cumulusci_id_table_1.id = accounts.parent_id ORDER BY accounts.parent_id - """, + expected="""SELECT accounts.id AS accounts_id, accounts."Name" AS "accounts_Name", cumulusci_id_table_1.sf_id AS cumulusci_id_table_1_sf_id FROM accounts LEFT OUTER JOIN cumulusci_id_table AS cumulusci_id_table_1 ON cumulusci_id_table_1.id = ? || accounts.parent_id ORDER BY accounts.parent_id""", + old_format= True ) - + + def test_query_db__joins_polymorphic_lookups(self): + _validate_query_for_mapping_step( + sql_path=Path(__file__).parent / "test_query_db_joins_lookups.sql", + mapping=Path(__file__).parent / "test_query_db_joins_lookups.yml", + mapping_step_name="Update Event", + expected="""SELECT events.id AS events_id, events."Subject" AS "events_Subject", cumulusci_id_table_1.sf_id AS cumulusci_id_table_1_sf_id FROM events LEFT OUTER JOIN cumulusci_id_table AS cumulusci_id_table_1 ON cumulusci_id_table_1.id = ? || events."WhoId" ORDER BY events."WhoId" """, + ) + @responses.activate def test_query_db__person_accounts_enabled__account_mapping(self): responses.add( @@ -1297,7 +1304,7 @@ def test_process_job_results__update_success(self): ) as sql_bulk_insert_from_records: task._process_job_results(mapping, step, local_ids) - task.session.connection.assert_not_called() + task.session.connection.assert_called_once() task._initialize_id_table.assert_not_called() sql_bulk_insert_from_records.assert_not_called() task.session.commit.assert_not_called() @@ -1649,7 +1656,7 @@ def test_process_job_results__person_account_contact_ids__updated(self): task._generate_contact_id_map_for_person_accounts.assert_called_once_with( mapping, mapping.lookups["AccountId"], task.session.connection.return_value ) - + assert task._old_format == True sql_bulk_insert_from_records.assert_called_with( connection=task.session.connection.return_value, table=task.metadata.tables[task._initialize_id_table.return_value], @@ -2456,14 +2463,14 @@ def test_generate_contact_id_map_for_person_accounts(self): task.metadata.tables = { "accounts": mock.Mock(), "contacts": mock.Mock(), - "accounts_sf_ids": mock.Mock(), - "contacts_sf_ids": mock.Mock(), + "cumulusci_id_table": mock.Mock(), } task.session = mock.Mock() task.session.query.return_value = task.session.query task.session.query.filter.return_value = task.session.query task.session.query.outerjoin.return_value = task.session.query task.sf = mock.Mock() + # task._old_format= mock.Mock(return_value= True) # Set model mocks account_model.__table__ = mock.Mock() @@ -2476,7 +2483,7 @@ def test_generate_contact_id_map_for_person_accounts(self): account_sf_ids_table = mock.Mock() account_sf_ids_table.columns = {"id": mock.Mock(), "sf_id": mock.Mock()} - + contact_model.__table__ = mock.Mock() contact_model.__table__.primary_key.columns.keys.return_value = ["sf_id"] contact_model.__table__.columns = { @@ -2981,7 +2988,7 @@ def test_recreate_set_recent_bug( ] -def _validate_query_for_mapping_step(sql_path, mapping, mapping_step_name, expected): +def _validate_query_for_mapping_step(sql_path, mapping, mapping_step_name, expected,old_format= False): """Validate the text of a SQL query""" task = _make_task( LoadData, @@ -2997,6 +3004,7 @@ def _validate_query_for_mapping_step(sql_path, mapping, mapping_step_name, expec ), mock.patch.object(task, "sf", create=True): task._init_mapping() with task._init_db(): + task._old_format= mock.Mock(return_value= old_format) query = task._query_db(task.mapping[mapping_step_name]) def normalize(query): diff --git a/cumulusci/tasks/bulkdata/tests/test_query_db__joins_self_lookups.sql b/cumulusci/tasks/bulkdata/tests/test_query_db__joins_self_lookups.sql index 7ccd4285db..818bece666 100644 --- a/cumulusci/tasks/bulkdata/tests/test_query_db__joins_self_lookups.sql +++ b/cumulusci/tasks/bulkdata/tests/test_query_db__joins_self_lookups.sql @@ -1,12 +1,12 @@ BEGIN TRANSACTION; CREATE TABLE "accounts" ( - sf_id VARCHAR(255) NOT NULL, + id VARCHAR(255) NOT NULL, "Name" VARCHAR(255), "parent_id" VARCHAR(255), - PRIMARY KEY (sf_id) + PRIMARY KEY (id) ); -INSERT INTO "accounts" VALUES("001DEADBEEF",'Bluth',''); -INSERT INTO "accounts" VALUES("002DEADBEEF",'Funke-Bluth',1); +INSERT INTO "accounts" VALUES(1,'Bluth',''); +INSERT INTO "accounts" VALUES(2,'Funke-Bluth',1); CREATE TABLE "cumulusci_id_table" ( id VARCHAR(255) NOT NULL, diff --git a/cumulusci/tasks/bulkdata/tests/test_query_db_joins_lookups.sql b/cumulusci/tasks/bulkdata/tests/test_query_db_joins_lookups.sql new file mode 100644 index 0000000000..113e5cebe5 --- /dev/null +++ b/cumulusci/tasks/bulkdata/tests/test_query_db_joins_lookups.sql @@ -0,0 +1,37 @@ +BEGIN TRANSACTION; + +CREATE TABLE "contacts" ( + id VARCHAR(255) NOT NULL, + "FirstName" VARCHAR(255), + "LastName" VARCHAR(255), + PRIMARY KEY (id) +); +INSERT INTO "contacts" VALUES("Contact-1",'Alpha','gamma'); +INSERT INTO "contacts" VALUES("Contact-2",'Temp','Bluth'); + +CREATE TABLE "events" ( + id VARCHAR(255) NOT NULL, + "Subject" VARCHAR(255), + "WhoId" VARCHAR(255), + PRIMARY KEY (id) +); +INSERT INTO "events" VALUES("Event-1",'helllo','Contact-1'); +INSERT INTO "events" VALUES("Event-2",'newer','Lead-2'); + +CREATE TABLE "leads" ( + id VARCHAR(255) NOT NULL, + "LastName" VARCHAR(255), + PRIMARY KEY (id) +); +INSERT INTO "leads" VALUES("Lead-1",'Boxer'); +INSERT INTO "leads" VALUES("Lead-2",'Cotton'); + +CREATE TABLE "cumulusci_id_table" ( + id VARCHAR(255) NOT NULL, + sf_id VARCHAR(255) +); +INSERT INTO "cumulusci_id_table" VALUES("Contact-1",'001DEADBEEF'); +INSERT INTO "cumulusci_id_table" VALUES("Contact-2",'002DEADBEEF'); +INSERT INTO "cumulusci_id_table" VALUES("Lead-1",'001DEADBEEA'); +INSERT INTO "cumulusci_id_table" VALUES("Lead-2",'002DEADBEEE'); +COMMIT; \ No newline at end of file diff --git a/cumulusci/tasks/bulkdata/tests/test_query_db_joins_lookups.yml b/cumulusci/tasks/bulkdata/tests/test_query_db_joins_lookups.yml new file mode 100644 index 0000000000..314e8cf1ff --- /dev/null +++ b/cumulusci/tasks/bulkdata/tests/test_query_db_joins_lookups.yml @@ -0,0 +1,27 @@ +Update Lead: + sf_object: Lead + table: leads + api: bulk + fields: + - LastName + +Update Contact: + sf_object: Contact + table: contacts + api: bulk + fields: + - FirstName + - LastName + +Update Event: + sf_object: Event + table: events + api: rest + fields: + - Subject + lookups: + WhoId: + table: + - Contact + - Lead + From 8708896b22c6e8ba3ac8f568b00d330e92d40e9b Mon Sep 17 00:00:00 2001 From: aditya-balachander Date: Fri, 19 Jan 2024 11:27:00 +0530 Subject: [PATCH 22/36] Fix tests --- .../tests/cassettes/TestDatasetsE2E.test_datasets_e2e.yaml | 3 +++ ...TestUpsert.test_upsert_complex_external_id_field__rest.yaml | 3 +++ cumulusci/tasks/bulkdata/tests/test_mapping_parser.py | 2 +- 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/cumulusci/core/tests/cassettes/TestDatasetsE2E.test_datasets_e2e.yaml b/cumulusci/core/tests/cassettes/TestDatasetsE2E.test_datasets_e2e.yaml index 5833b12d30..596992d830 100644 --- a/cumulusci/core/tests/cassettes/TestDatasetsE2E.test_datasets_e2e.yaml +++ b/cumulusci/core/tests/cassettes/TestDatasetsE2E.test_datasets_e2e.yaml @@ -181,8 +181,11 @@ interactions: include_file: GET_sobjects_Global_describe.yaml - &id007 include_file: GET_sobjects_Account_describe.yaml + - &id010 + include_file: GET_sobjects_Account_describe.yaml - *id006 - *id007 + - *id010 - request: method: POST uri: https://orgname.my.salesforce.com/services/data/vxx.0/composite/sobjects diff --git a/cumulusci/tasks/bulkdata/tests/cassettes/TestUpsert.test_upsert_complex_external_id_field__rest.yaml b/cumulusci/tasks/bulkdata/tests/cassettes/TestUpsert.test_upsert_complex_external_id_field__rest.yaml index 266778b93f..ccbe1961e5 100644 --- a/cumulusci/tasks/bulkdata/tests/cassettes/TestUpsert.test_upsert_complex_external_id_field__rest.yaml +++ b/cumulusci/tasks/bulkdata/tests/cassettes/TestUpsert.test_upsert_complex_external_id_field__rest.yaml @@ -4,6 +4,9 @@ interactions: include_file: GET_sobjects_Global_describe.yaml - &id005 include_file: GET_sobjects_Account_describe.yaml + - &id016 + include_file: GET_sobjects_Account_describe.yaml + - *id016 - *id001 - &id007 include_file: GET_sobjects_Contact_describe.yaml diff --git a/cumulusci/tasks/bulkdata/tests/test_mapping_parser.py b/cumulusci/tasks/bulkdata/tests/test_mapping_parser.py index ba94834157..80a4e071fd 100644 --- a/cumulusci/tasks/bulkdata/tests/test_mapping_parser.py +++ b/cumulusci/tasks/bulkdata/tests/test_mapping_parser.py @@ -1271,7 +1271,7 @@ def test_get_extract_field_list(self): assert m.get_extract_field_list() == [ "Id", "Name", + "RecordTypeId", "AccountSite", "ParentId", - "RecordTypeId", ] From c7edb214fb654283d68dd2a61769d4ddf6fc656a Mon Sep 17 00:00:00 2001 From: aditya-balachander Date: Fri, 19 Jan 2024 11:27:08 +0530 Subject: [PATCH 23/36] Update mapping_parser.py --- cumulusci/tasks/bulkdata/mapping_parser.py | 32 ++++++++++------------ 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/cumulusci/tasks/bulkdata/mapping_parser.py b/cumulusci/tasks/bulkdata/mapping_parser.py index 4fc1ad4400..0930b73fb2 100644 --- a/cumulusci/tasks/bulkdata/mapping_parser.py +++ b/cumulusci/tasks/bulkdata/mapping_parser.py @@ -5,6 +5,7 @@ from logging import getLogger from pathlib import Path from typing import IO, Any, Callable, Dict, List, Mapping, Optional, Tuple, Union +from collections import OrderedDict from pydantic import Field, ValidationError, root_validator, validator from requests.structures import CaseInsensitiveDict as RequestsCaseInsensitiveDict @@ -598,8 +599,8 @@ def parse_from_yaml(source: Union[str, Path, IO]) -> Dict: def _infer_and_validate_lookups(mapping: Dict, sf: Salesforce): - sf_objects = [m.sf_object for m in mapping.values()] - table_sf_obj = {m.table: m.sf_object for m in mapping.values()} + # store the table name and their sobject + sf_objects = OrderedDict((m.table, m.sf_object) for m in mapping.values()) fail = False @@ -619,28 +620,25 @@ def _infer_and_validate_lookups(mapping: Dict, sf: Salesforce): # TODO: do we need more validation here? continue - field_describe = describe[lookup.name] - reference_to_objects = field_describe["referenceTo"] + field_describe = describe.get(lookup.name, {}) + reference_to_objects = field_describe.get("referenceTo", []) target_objects = [] - - lookup_tables = [] - if type(lookup.table)==str: - lookup_tables = [lookup.table] - else: - lookup_tables = lookup.table - + + lookup_tables = [lookup.table] if isinstance(lookup.table, str) else lookup.table + for table in lookup_tables: try: - sf_object = table_sf_obj[table] + sf_object = sf_objects[table] + # Check if sf_object is a valid lookup if sf_object in reference_to_objects: target_objects.append(sf_object) else: - logger.error(f"The table {table} is not a valid lookup for {lookup.name} in sf_object: {m.sf_object}") + logger.error(f"The lookup {sf_object} is not a valid lookup for {lookup.name} in sf_object: {m.sf_object}") fail = True except KeyError: - logger.error(f"The table {table} does not exists in the mapping file") + logger.error(f"The table {table} does not exist in the mapping file") fail = True - + if fail: continue @@ -649,7 +647,7 @@ def _infer_and_validate_lookups(mapping: Dict, sf: Salesforce): if len(target_objects) == 1: # This is a non-polymorphic lookup. try: - target_index = sf_objects.index(target_objects[0]) + target_index = list(sf_objects.values()).index(target_objects[0]) except ValueError: fail = True logger.error( @@ -663,7 +661,7 @@ def _infer_and_validate_lookups(mapping: Dict, sf: Salesforce): else: # This is a polymorphic lookup. # Make sure that any lookup targets present in the operation precede this step. - target_indices = [sf_objects.index(t) for t in target_objects] + target_indices = [list(sf_objects.values()).index(t) for t in target_objects] if not all([target_index < idx for target_index in target_indices]): logger.error( f"All included target objects ({','.join(target_objects)}) for the field {m.sf_object}.{lookup.name} " From a9bdae4d8ddc6c7f79153c9deaa439eb501c5a5f Mon Sep 17 00:00:00 2001 From: aditya-balachander Date: Fri, 19 Jan 2024 12:16:17 +0530 Subject: [PATCH 24/36] Update test_utils.py --- cumulusci/tasks/bulkdata/tests/test_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cumulusci/tasks/bulkdata/tests/test_utils.py b/cumulusci/tasks/bulkdata/tests/test_utils.py index ad3e69b8e0..f0e3f817fb 100644 --- a/cumulusci/tasks/bulkdata/tests/test_utils.py +++ b/cumulusci/tasks/bulkdata/tests/test_utils.py @@ -133,7 +133,7 @@ def test_create_table_modern_id_mapping(self): engine, metadata = create_db_file(tmp_db_path) t = create_table(account_mapping, metadata) assert t.name == "contacts" - assert isinstance(t.columns["id"].type, Integer) + assert isinstance(t.columns["id"].type, Unicode) assert isinstance(t.columns["first_name"].type, Unicode) assert isinstance(t.columns["last_name"].type, Unicode) assert isinstance(t.columns["email"].type, Unicode) From 94974d39e0f68c67846581ebce78e79a1cae27d8 Mon Sep 17 00:00:00 2001 From: aditya-balachander Date: Fri, 19 Jan 2024 12:58:33 +0530 Subject: [PATCH 25/36] Lint checks --- cumulusci/tasks/bulkdata/extract.py | 59 +++++++++++++------ cumulusci/tasks/bulkdata/mapping_parser.py | 43 ++++++++------ .../tasks/bulkdata/query_transformers.py | 2 +- .../tasks/bulkdata/tests/mapping_poly.yml | 1 + .../tests/mapping_poly_incomplete.yml | 3 + .../bulkdata/tests/mapping_poly_wrong.yml | 2 + .../tasks/bulkdata/tests/test_extract.py | 52 +++++++++++----- cumulusci/tasks/bulkdata/tests/test_load.py | 16 ++--- .../bulkdata/tests/test_mapping_parser.py | 4 +- .../tests/test_query_db_joins_lookups.yml | 43 +++++++------- cumulusci/tasks/bulkdata/utils.py | 6 +- .../GET_sobjects_Event_describe.yaml | 1 + datasets/mapping.yml | 2 - 13 files changed, 147 insertions(+), 87 deletions(-) diff --git a/cumulusci/tasks/bulkdata/extract.py b/cumulusci/tasks/bulkdata/extract.py index ea10317b39..adcaa0f4e1 100644 --- a/cumulusci/tasks/bulkdata/extract.py +++ b/cumulusci/tasks/bulkdata/extract.py @@ -2,10 +2,15 @@ import re from contextlib import contextmanager -from sqlalchemy import Column, Integer, MetaData, Table, Unicode, create_engine +from sqlalchemy import Column, MetaData, Table, Unicode, create_engine from sqlalchemy.orm import create_session, mapper -from cumulusci.core.exceptions import BulkDataException, TaskOptionsError, CumulusCIException, ConfigError +from cumulusci.core.exceptions import ( + BulkDataException, + ConfigError, + CumulusCIException, + TaskOptionsError, +) from cumulusci.core.utils import process_bool_arg from cumulusci.tasks.bulkdata.dates import adjust_relative_dates from cumulusci.tasks.bulkdata.mapping_parser import ( @@ -227,16 +232,18 @@ def strip_name_field(record): # into two separate streams and load into the main table and the sf_id_table values, ids = itertools.tee(record_iterator) + # Generate naming format (-) id_source_sobj, id_source_global = itertools.tee( self._id_generator_for_object(mapping.sf_object) ) + f_ids = ((row[0], next(id_source_global)) for row in ids) f_values = ([next(id_source_sobj)] + row[1:] for row in values) values_chunks = sql_bulk_insert_from_records_incremental( connection=conn, table=self.metadata.tables[mapping.table], - columns= ["id"] + columns[1:], + columns=["id"] + columns[1:], record_iterable=f_values, ) ids_chunks = sql_bulk_insert_from_records_incremental( @@ -261,6 +268,8 @@ def strip_name_field(record): self.session.commit() def _id_generator_for_object(self, sobject: str): + """Generates strings for local ids in format {sobject}-{counter} + (example: Account-2)""" if sobject not in self._id_generators: def _generate_ids(): @@ -288,15 +297,21 @@ def _map_autopks(self): def _get_mapping_for_table(self, table): """Return the first mapping for a table name""" + + # Get all mappings for lookups mappings = [ - mapping for mapping in self.mapping.values() - if (isinstance(table, str) and mapping["table"] == table) or - (isinstance(table, list) and mapping["table"] in table) + mapping + for mapping in self.mapping.values() + if (isinstance(table, str) and mapping["table"] == table) + or (isinstance(table, list) and mapping["table"] in table) ] + # For polymorphic lookups, raise exception if missing mappings if isinstance(table, list) and len(mappings) != len(table): missing_tables = set(table) - set(mapping["table"] for mapping in mappings) - raise CumulusCIException(f"The following tables are missing in the mapping file: {missing_tables}") + raise CumulusCIException( + f"The following tables are missing in the mapping file: {missing_tables}" + ) return mappings @@ -328,27 +343,35 @@ def throw(string): # pragma: no cover for lookup_mapping in lookup_mappings: lookup_model = self.models.get(lookup_mapping.get_sf_id_table()) try: - update_query = self.session.query(model).filter( - key_attr.isnot(None), key_attr == lookup_model.sf_id - ).update({key_attr: lookup_model.id}, synchronize_session=False) + update_query = ( + self.session.query(model) + .filter(key_attr.isnot(None), key_attr == lookup_model.sf_id) + .update({key_attr: lookup_model.id}, synchronize_session=False) + ) total_mapping_operations += update_query.rowcount except NotImplementedError: # Some databases such as sqlite don't support multitable update mappings = [] - for row, lookup_id in self.session.query(model, lookup_model.id).join( - lookup_model, key_attr == lookup_model.sf_id - ): + for row, lookup_id in self.session.query( + model, lookup_model.id + ).join(lookup_model, key_attr == lookup_model.sf_id): mappings.append({"id": row.id, key_field: lookup_id}) total_mapping_operations += len(mappings) self.session.bulk_update_mappings(model, mappings) # Count the total number of rows excluding those with no entry for that field - total_rows = self.session.query(model).filter( - key_attr.isnot(None), # Ensure key_attr is not None - key_attr.isnot('') # Ensure key_attr is not an empty string - ).count() + total_rows = ( + self.session.query(model) + .filter( + key_attr.isnot(None), # Ensure key_attr is not None + key_attr.isnot(""), # Ensure key_attr is not an empty string + ) + .count() + ) if total_mapping_operations != total_rows: - raise ConfigError(f"Total mapping operations ({total_mapping_operations}) do not match total non-empty rows ({total_rows}) for lookup_key: {lookup_key}") + raise ConfigError( + f"Total mapping operations ({total_mapping_operations}) do not match total non-empty rows ({total_rows}) for lookup_key: {lookup_key}. Mention all related tables for lookup: {lookup_key}" + ) self.session.commit() def _create_tables(self): diff --git a/cumulusci/tasks/bulkdata/mapping_parser.py b/cumulusci/tasks/bulkdata/mapping_parser.py index 0930b73fb2..a151c02681 100644 --- a/cumulusci/tasks/bulkdata/mapping_parser.py +++ b/cumulusci/tasks/bulkdata/mapping_parser.py @@ -1,11 +1,11 @@ import typing as T +from collections import OrderedDict from datetime import date from enum import Enum from functools import lru_cache from logging import getLogger from pathlib import Path from typing import IO, Any, Callable, Dict, List, Mapping, Optional, Tuple, Union -from collections import OrderedDict from pydantic import Field, ValidationError, root_validator, validator from requests.structures import CaseInsensitiveDict as RequestsCaseInsensitiveDict @@ -37,7 +37,7 @@ def __setitem__(self, key, value): class MappingLookup(CCIDictModel): "Lookup relationship between two tables." - table: Union[str, List[str]] + table: Union[str, List[str]] # Support for polymorphic lookups key_field: Optional[str] = None value_field: Optional[str] = None join_field: Optional[str] = None @@ -196,7 +196,7 @@ def get_load_field_list(self): return columns def get_extract_field_list(self): - """Build a flat list of Salesforce fields for the given mapping, including fields, lookups, and record types, + """Build a ordered list of Salesforce fields for the given mapping, including fields, lookups, and record types, for an extraction operation. The Id field is guaranteed to come first in the list.""" @@ -599,6 +599,9 @@ def parse_from_yaml(source: Union[str, Path, IO]) -> Dict: def _infer_and_validate_lookups(mapping: Dict, sf: Salesforce): + """Validate that all the lookup tables mentioned are valid references + to the lookup. Also verify that the mapping for the tables are mentioned + before they are mentioned in the lookups""" # store the table name and their sobject sf_objects = OrderedDict((m.table, m.sf_object) for m in mapping.values()) @@ -606,12 +609,7 @@ def _infer_and_validate_lookups(mapping: Dict, sf: Salesforce): for idx, m in enumerate(mapping.values()): describe = CaseInsensitiveDict( - { - f["name"]: f - for f in getattr(sf, m.sf_object).describe()[ - "fields" - ] - } + {f["name"]: f for f in getattr(sf, m.sf_object).describe()["fields"]} ) for lookup in m.lookups.values(): @@ -624,7 +622,9 @@ def _infer_and_validate_lookups(mapping: Dict, sf: Salesforce): reference_to_objects = field_describe.get("referenceTo", []) target_objects = [] - lookup_tables = [lookup.table] if isinstance(lookup.table, str) else lookup.table + lookup_tables = ( + [lookup.table] if isinstance(lookup.table, str) else lookup.table + ) for table in lookup_tables: try: @@ -633,16 +633,18 @@ def _infer_and_validate_lookups(mapping: Dict, sf: Salesforce): if sf_object in reference_to_objects: target_objects.append(sf_object) else: - logger.error(f"The lookup {sf_object} is not a valid lookup for {lookup.name} in sf_object: {m.sf_object}") + logger.error( + f"The lookup {sf_object} is not a valid lookup for {lookup.name} in sf_object: {m.sf_object}" + ) fail = True except KeyError: - logger.error(f"The table {table} does not exist in the mapping file") + logger.error( + f"The table {table} does not exist in the mapping file" + ) fail = True - + if fail: continue - - if len(target_objects) == 1: # This is a non-polymorphic lookup. @@ -661,7 +663,9 @@ def _infer_and_validate_lookups(mapping: Dict, sf: Salesforce): else: # This is a polymorphic lookup. # Make sure that any lookup targets present in the operation precede this step. - target_indices = [list(sf_objects.values()).index(t) for t in target_objects] + target_indices = [ + list(sf_objects.values()).index(t) for t in target_objects + ] if not all([target_index < idx for target_index in target_indices]): logger.error( f"All included target objects ({','.join(target_objects)}) for the field {m.sf_object}.{lookup.name} " @@ -675,6 +679,7 @@ def _infer_and_validate_lookups(mapping: Dict, sf: Salesforce): "One or more relationship errors blocked the operation." ) + def validate_and_inject_mapping( *, mapping: Dict, @@ -685,7 +690,9 @@ def validate_and_inject_mapping( drop_missing: bool, org_has_person_accounts_enabled: bool = False, ): + # Check if operation is load or extract is_load = True if data_operation == DataOperationType.INSERT else False + should_continue = [ m.validate_and_inject_namespace( sf, namespace, data_operation, inject_namespaces, drop_missing @@ -724,9 +731,9 @@ def validate_and_inject_mapping( f"{describe[field]['referenceTo']} was removed from the operation " "due to missing permissions." ) - + + # Infer/validate lookups if is_load: - # Synthesize `after` declarations and validate lookup references. _infer_and_validate_lookups(mapping, sf) # If the org has person accounts enable, add a field mapping to track "IsPersonAccount". diff --git a/cumulusci/tasks/bulkdata/query_transformers.py b/cumulusci/tasks/bulkdata/query_transformers.py index a4861917c7..84c1e66207 100644 --- a/cumulusci/tasks/bulkdata/query_transformers.py +++ b/cumulusci/tasks/bulkdata/query_transformers.py @@ -174,4 +174,4 @@ def filters_to_add(self): assert self.mapping.sf_object == "Contact" return [ func.lower(self.model.__table__.columns.get("IsPersonAccount")) == "false" - ] \ No newline at end of file + ] diff --git a/cumulusci/tasks/bulkdata/tests/mapping_poly.yml b/cumulusci/tasks/bulkdata/tests/mapping_poly.yml index 26454cc8d6..04af8a4e7f 100644 --- a/cumulusci/tasks/bulkdata/tests/mapping_poly.yml +++ b/cumulusci/tasks/bulkdata/tests/mapping_poly.yml @@ -1,3 +1,4 @@ +# Polymorphic Mapping File for extract Insert Households: api: Bulk sf_object: Account diff --git a/cumulusci/tasks/bulkdata/tests/mapping_poly_incomplete.yml b/cumulusci/tasks/bulkdata/tests/mapping_poly_incomplete.yml index 79fbb24144..f80fa33e99 100644 --- a/cumulusci/tasks/bulkdata/tests/mapping_poly_incomplete.yml +++ b/cumulusci/tasks/bulkdata/tests/mapping_poly_incomplete.yml @@ -1,3 +1,6 @@ +# Polymorphic mapping file for extract with +# some records of event referencing contact +# but contact not mentioned in lookup Insert Households: api: Bulk sf_object: Account diff --git a/cumulusci/tasks/bulkdata/tests/mapping_poly_wrong.yml b/cumulusci/tasks/bulkdata/tests/mapping_poly_wrong.yml index 343a989a61..675718d01a 100644 --- a/cumulusci/tasks/bulkdata/tests/mapping_poly_wrong.yml +++ b/cumulusci/tasks/bulkdata/tests/mapping_poly_wrong.yml @@ -1,3 +1,5 @@ +# Polymorphic Mapping file for extract +# with missing table contacts Insert Households: api: Bulk sf_object: Account diff --git a/cumulusci/tasks/bulkdata/tests/test_extract.py b/cumulusci/tasks/bulkdata/tests/test_extract.py index 7c76a8582c..e2fd9d65c7 100644 --- a/cumulusci/tasks/bulkdata/tests/test_extract.py +++ b/cumulusci/tasks/bulkdata/tests/test_extract.py @@ -3,13 +3,17 @@ from datetime import date, timedelta from tempfile import TemporaryDirectory from unittest import mock -from sqlalchemy.orm import Query import pytest import responses from sqlalchemy import create_engine -from cumulusci.core.exceptions import BulkDataException, TaskOptionsError, CumulusCIException, ConfigError +from cumulusci.core.exceptions import ( + BulkDataException, + ConfigError, + CumulusCIException, + TaskOptionsError, +) from cumulusci.tasks.bulkdata import ExtractData from cumulusci.tasks.bulkdata.mapping_parser import MappingLookup, MappingStep from cumulusci.tasks.bulkdata.step import ( @@ -342,10 +346,11 @@ def test_run__v2__person_accounts_enabled(self, query_op_mock): contact = next(conn.execute("select * from contacts")) assert contact.household_id == "Account-1" assert contact.IsPersonAccount == "true" - + @responses.activate @mock.patch("cumulusci.tasks.bulkdata.extract.get_query_operation") def test_run__poly__polymorphic_lookups(self, query_op_mock): + """Test for polymorphic lookups""" base_path = os.path.dirname(__file__) mapping_path = os.path.join(base_path, self.mapping_file_poly) mock_describe_calls() @@ -388,10 +393,14 @@ def test_run__poly__polymorphic_lookups(self, query_op_mock): ] mock_query_events.results = [ ["ijk789", "Last1", "abc123"], - ["lmn010", "Last2", "def456"] + ["lmn010", "Last2", "def456"], ] - query_op_mock.side_effect = [mock_query_households, mock_query_contacts, mock_query_events] + query_op_mock.side_effect = [ + mock_query_households, + mock_query_contacts, + mock_query_events, + ] task() with create_engine(task.options["database_url"]).connect() as conn: household = next(conn.execute("select * from households")) @@ -406,10 +415,12 @@ def test_run__poly__polymorphic_lookups(self, query_op_mock): events = conn.execute("select * from events").fetchall() assert events[0].who_id == "Account-1" assert events[1].who_id == "Contact-1" - + @responses.activate @mock.patch("cumulusci.tasks.bulkdata.extract.get_query_operation") def test_run__poly__wrong_mapping(self, query_op_mock): + """Test for polymorphic lookups with wrong mapping file + (missing table)""" base_path = os.path.dirname(__file__) mapping_path = os.path.join(base_path, self.mapping_file_poly_wrong) mock_describe_calls() @@ -452,18 +463,25 @@ def test_run__poly__wrong_mapping(self, query_op_mock): ] mock_query_events.results = [ ["ijk789", "Last1", "abc123"], - ["lmn010", "Last2", "def456"] + ["lmn010", "Last2", "def456"], ] - query_op_mock.side_effect = [mock_query_households, mock_query_contacts, mock_query_events] + query_op_mock.side_effect = [ + mock_query_households, + mock_query_contacts, + mock_query_events, + ] with pytest.raises(CumulusCIException) as e: task() - - assert "The following tables are missing in the mapping file:" in str(e.value) + + assert "The following tables are missing in the mapping file:" in str( + e.value + ) @responses.activate @mock.patch("cumulusci.tasks.bulkdata.extract.get_query_operation") def test_run__poly__incomplete_mapping(self, query_op_mock): + """Test for polymorphic lookups with incomplete mapping file""" base_path = os.path.dirname(__file__) mapping_path = os.path.join(base_path, self.mapping_file_poly_incomplete) mock_describe_calls() @@ -497,13 +515,13 @@ def test_run__poly__incomplete_mapping(self, query_op_mock): mock_query_households.results = [["abc123", "TestHousehold"]] mock_query_events.results = [ ["ijk789", "Last1", "abc123"], - ["lmn010", "Last2", "def456"] + ["lmn010", "Last2", "def456"], ] query_op_mock.side_effect = [mock_query_households, mock_query_events] with pytest.raises(ConfigError) as e: task() - + assert "Total mapping operations" in str(e.value) assert "do not match total non-empty rows" in str(e.value) @@ -565,7 +583,7 @@ def test_import_results__no_columns(self): # , query_op_mock): output_Opportunties = list( task.session.execute("select * from Opportunity") ) - assert output_Opportunties == [('Opportunity-1',), ('Opportunity-2',)] + assert output_Opportunties == [("Opportunity-1",), ("Opportunity-2",)] @responses.activate def test_import_results__relative_dates(self): @@ -739,7 +757,9 @@ def test_convert_lookups_to_id(self): } task.session.query.return_value.filter.return_value.count.return_value = 0 - task.session.query.return_value.filter.return_value.update.return_value.rowcount = 0 + task.session.query.return_value.filter.return_value.update.return_value.rowcount = ( + 0 + ) task._convert_lookups_to_id( MappingStep( sf_object="Opportunity", @@ -1185,8 +1205,8 @@ def test_import_results__autopk(self, create_task_fixture): task() with create_engine(task.options["database_url"]).connect() as conn: output_accounts = list(conn.execute("select * from Account")) - assert output_accounts[0][0:2] == ('Account-1', "Account1") - assert output_accounts[1][0:2] == ('Account-2', "Account2") + assert output_accounts[0][0:2] == ("Account-1", "Account1") + assert output_accounts[1][0:2] == ("Account-2", "Account2") assert len(output_accounts) == 2 output_opportunities = list(conn.execute("select * from Opportunity")) diff --git a/cumulusci/tasks/bulkdata/tests/test_load.py b/cumulusci/tasks/bulkdata/tests/test_load.py index 2c7e97bf00..ddd11f1b9a 100644 --- a/cumulusci/tasks/bulkdata/tests/test_load.py +++ b/cumulusci/tasks/bulkdata/tests/test_load.py @@ -878,9 +878,9 @@ def test_query_db__joins_self_lookups(self): mapping=Path(__file__).parent / "test_query_db__joins_self_lookups.yml", mapping_step_name="Update Accounts", expected="""SELECT accounts.id AS accounts_id, accounts."Name" AS "accounts_Name", cumulusci_id_table_1.sf_id AS cumulusci_id_table_1_sf_id FROM accounts LEFT OUTER JOIN cumulusci_id_table AS cumulusci_id_table_1 ON cumulusci_id_table_1.id = ? || accounts.parent_id ORDER BY accounts.parent_id""", - old_format= True + old_format=True, ) - + def test_query_db__joins_polymorphic_lookups(self): _validate_query_for_mapping_step( sql_path=Path(__file__).parent / "test_query_db_joins_lookups.sql", @@ -888,7 +888,7 @@ def test_query_db__joins_polymorphic_lookups(self): mapping_step_name="Update Event", expected="""SELECT events.id AS events_id, events."Subject" AS "events_Subject", cumulusci_id_table_1.sf_id AS cumulusci_id_table_1_sf_id FROM events LEFT OUTER JOIN cumulusci_id_table AS cumulusci_id_table_1 ON cumulusci_id_table_1.id = ? || events."WhoId" ORDER BY events."WhoId" """, ) - + @responses.activate def test_query_db__person_accounts_enabled__account_mapping(self): responses.add( @@ -1656,7 +1656,7 @@ def test_process_job_results__person_account_contact_ids__updated(self): task._generate_contact_id_map_for_person_accounts.assert_called_once_with( mapping, mapping.lookups["AccountId"], task.session.connection.return_value ) - assert task._old_format == True + assert task._old_format is True sql_bulk_insert_from_records.assert_called_with( connection=task.session.connection.return_value, table=task.metadata.tables[task._initialize_id_table.return_value], @@ -2483,7 +2483,7 @@ def test_generate_contact_id_map_for_person_accounts(self): account_sf_ids_table = mock.Mock() account_sf_ids_table.columns = {"id": mock.Mock(), "sf_id": mock.Mock()} - + contact_model.__table__ = mock.Mock() contact_model.__table__.primary_key.columns.keys.return_value = ["sf_id"] contact_model.__table__.columns = { @@ -2988,7 +2988,9 @@ def test_recreate_set_recent_bug( ] -def _validate_query_for_mapping_step(sql_path, mapping, mapping_step_name, expected,old_format= False): +def _validate_query_for_mapping_step( + sql_path, mapping, mapping_step_name, expected, old_format=False +): """Validate the text of a SQL query""" task = _make_task( LoadData, @@ -3004,7 +3006,7 @@ def _validate_query_for_mapping_step(sql_path, mapping, mapping_step_name, expec ), mock.patch.object(task, "sf", create=True): task._init_mapping() with task._init_db(): - task._old_format= mock.Mock(return_value= old_format) + task._old_format = mock.Mock(return_value=old_format) query = task._query_db(task.mapping[mapping_step_name]) def normalize(query): diff --git a/cumulusci/tasks/bulkdata/tests/test_mapping_parser.py b/cumulusci/tasks/bulkdata/tests/test_mapping_parser.py index 80a4e071fd..c4e7872f9f 100644 --- a/cumulusci/tasks/bulkdata/tests/test_mapping_parser.py +++ b/cumulusci/tasks/bulkdata/tests/test_mapping_parser.py @@ -1260,8 +1260,10 @@ def test_upsert_key_list(self): "FirstName", "LastName", ), mapping["Insert Accounts"]["update_key"] - + def test_get_extract_field_list(self): + """Test to ensure Id comes first, lookups come + last and order of mappings do not change""" m = MappingStep( sf_object="Account", fields=["Name", "RecordTypeId", "AccountSite"], diff --git a/cumulusci/tasks/bulkdata/tests/test_query_db_joins_lookups.yml b/cumulusci/tasks/bulkdata/tests/test_query_db_joins_lookups.yml index 314e8cf1ff..75d25a0088 100644 --- a/cumulusci/tasks/bulkdata/tests/test_query_db_joins_lookups.yml +++ b/cumulusci/tasks/bulkdata/tests/test_query_db_joins_lookups.yml @@ -1,27 +1,26 @@ Update Lead: - sf_object: Lead - table: leads - api: bulk - fields: - - LastName + sf_object: Lead + table: leads + api: bulk + fields: + - LastName Update Contact: - sf_object: Contact - table: contacts - api: bulk - fields: - - FirstName - - LastName + sf_object: Contact + table: contacts + api: bulk + fields: + - FirstName + - LastName Update Event: - sf_object: Event - table: events - api: rest - fields: - - Subject - lookups: - WhoId: - table: - - Contact - - Lead - + sf_object: Event + table: events + api: rest + fields: + - Subject + lookups: + WhoId: + table: + - Contact + - Lead diff --git a/cumulusci/tasks/bulkdata/utils.py b/cumulusci/tasks/bulkdata/utils.py index d77738b25f..ae3d651ed2 100644 --- a/cumulusci/tasks/bulkdata/utils.py +++ b/cumulusci/tasks/bulkdata/utils.py @@ -6,7 +6,7 @@ from pathlib import Path from simple_salesforce import Salesforce -from sqlalchemy import Boolean, Column, Integer, MetaData, Table, Unicode, inspect +from sqlalchemy import Boolean, Column, MetaData, Table, Unicode, inspect from sqlalchemy.engine.base import Connection from sqlalchemy.orm import Session, mapper @@ -15,6 +15,7 @@ ID_TABLE_NAME = "cumulusci_id_table" + class SqlAlchemyMixin: logger: logging.Logger metadata: MetaData @@ -81,8 +82,9 @@ def _handle_primary_key(mapping, fields): id_column = "id" if mapping.get_oid_as_pk(): + # Get Id column from mapping id_column = mapping.fields["Id"] - + fields.append(Column(id_column, Unicode(255), primary_key=True)) diff --git a/cumulusci/tests/shared_cassettes/GET_sobjects_Event_describe.yaml b/cumulusci/tests/shared_cassettes/GET_sobjects_Event_describe.yaml index 58623f0c3c..44fb97afac 100644 --- a/cumulusci/tests/shared_cassettes/GET_sobjects_Event_describe.yaml +++ b/cumulusci/tests/shared_cassettes/GET_sobjects_Event_describe.yaml @@ -1,3 +1,4 @@ +# Describe call for Event sobject (polymorphic purposes) request: method: GET uri: https://orgname.my.salesforce.com/services/data/vxx.0/sobjects/Event/describe diff --git a/datasets/mapping.yml b/datasets/mapping.yml index 34ee85e32a..ae7952b22c 100644 --- a/datasets/mapping.yml +++ b/datasets/mapping.yml @@ -3,7 +3,6 @@ Account: api: bulk fields: - Name - - LastName - Description - NumberOfEmployees - BillingStreet @@ -20,7 +19,6 @@ Account: - Fax - Website - AccountNumber - - RecordTypeId Contact: sf_object: Contact From 6a96cac2b349483cac385587d5a49d4ac8e0b50c Mon Sep 17 00:00:00 2001 From: lakshmi2506 Date: Fri, 19 Jan 2024 14:07:07 +0530 Subject: [PATCH 26/36] test_load.py covered --- cumulusci/tasks/bulkdata/load.py | 3 ++- cumulusci/tasks/bulkdata/tests/test_load.py | 28 ++++++++++++++++----- cumulusci/tasks/bulkdata/utils.py | 2 -- 3 files changed, 24 insertions(+), 9 deletions(-) diff --git a/cumulusci/tasks/bulkdata/load.py b/cumulusci/tasks/bulkdata/load.py index ecac47a04c..201153910b 100644 --- a/cumulusci/tasks/bulkdata/load.py +++ b/cumulusci/tasks/bulkdata/load.py @@ -22,6 +22,7 @@ validate_and_inject_mapping, ) from cumulusci.tasks.bulkdata.query_transformers import ( + ID_TABLE_NAME, AddLookupsToQuery, AddMappingFiltersToQuery, AddPersonAccountsToQuery, @@ -126,7 +127,7 @@ def _init_options(self, kwargs): ) self._id_generators = {} self._old_format = False - self.ID_TABLE_NAME = "cumulusci_id_table" + self.ID_TABLE_NAME = ID_TABLE_NAME def _init_dataset(self): """Find the dataset paths to use with the following sequence: diff --git a/cumulusci/tasks/bulkdata/tests/test_load.py b/cumulusci/tasks/bulkdata/tests/test_load.py index ddd11f1b9a..98775c8877 100644 --- a/cumulusci/tasks/bulkdata/tests/test_load.py +++ b/cumulusci/tasks/bulkdata/tests/test_load.py @@ -2444,8 +2444,8 @@ def test_db_has_person_accounts_column(self): # code to test too little code. We'll test Person accounts in context # instead. # Delete this comment whenever it is convenient. - - def test_generate_contact_id_map_for_person_accounts(self): + @pytest.mark.parametrize("old_format", [True, False]) + def test_generate_contact_id_map_for_person_accounts(self, old_format): mapping_file = "mapping-oid.yml" base_path = os.path.dirname(__file__) mapping_path = os.path.join(base_path, mapping_file) @@ -2470,7 +2470,7 @@ def test_generate_contact_id_map_for_person_accounts(self): task.session.query.filter.return_value = task.session.query task.session.query.outerjoin.return_value = task.session.query task.sf = mock.Mock() - # task._old_format= mock.Mock(return_value= True) + task._old_format = old_format # Set model mocks account_model.__table__ = mock.Mock() @@ -2490,7 +2490,7 @@ def test_generate_contact_id_map_for_person_accounts(self): "id": mock.Mock(), "sf_id": mock.Mock(), "IsPersonAccount": mock.MagicMock(), - "account_id": mock.Mock, + "account_id": "string", } account_id_lookup = MappingLookup( @@ -2505,6 +2505,12 @@ def test_generate_contact_id_map_for_person_accounts(self): account_id_column = getattr( contact_model, account_id_lookup.get_lookup_key_field(contact_model) ) + setattr( + contact_model, + account_id_lookup.get_lookup_key_field(contact_model), + "Contact.account_id", + ) + account_sf_ids_table = account_id_lookup["aliased_table"] account_sf_id_column = account_sf_ids_table.columns["sf_id"] @@ -2557,7 +2563,18 @@ def get_random_string(): query_result.fetchmany.expected_calls = [] task.sf.query_all.expected_calls = [] for chunk in chunks: - expected.extend([(record["id"], record["sf_id"]) for record in chunk]) + if task._old_format: + expected.extend( + [ + ( + str(contact_mapping.table) + "-" + record["id"], + record["sf_id"], + ) + for record in chunk + ] + ) + else: + expected.extend([(record["id"], record["sf_id"]) for record in chunk]) query_result.fetchmany.expected_calls.append(mock.call(200)) @@ -2620,7 +2637,6 @@ def query_all(query): ) actual = [value for value in generator] - assert expected == actual # Assert query executed diff --git a/cumulusci/tasks/bulkdata/utils.py b/cumulusci/tasks/bulkdata/utils.py index ae3d651ed2..082277fb16 100644 --- a/cumulusci/tasks/bulkdata/utils.py +++ b/cumulusci/tasks/bulkdata/utils.py @@ -13,8 +13,6 @@ from cumulusci.core.exceptions import BulkDataException from cumulusci.utils.iterators import iterate_in_chunks -ID_TABLE_NAME = "cumulusci_id_table" - class SqlAlchemyMixin: logger: logging.Logger From e5bd27f9b97ec6798adb3074524d1f6388922e9f Mon Sep 17 00:00:00 2001 From: lakshmi2506 Date: Fri, 19 Jan 2024 14:23:22 +0530 Subject: [PATCH 27/36] Update test_load.py --- cumulusci/tasks/bulkdata/tests/test_load.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/cumulusci/tasks/bulkdata/tests/test_load.py b/cumulusci/tasks/bulkdata/tests/test_load.py index 98775c8877..b8058d77b4 100644 --- a/cumulusci/tasks/bulkdata/tests/test_load.py +++ b/cumulusci/tasks/bulkdata/tests/test_load.py @@ -831,11 +831,9 @@ def test_stream_queried_data__adjusts_relative_dates(self): ] ) local_ids = io.StringIO() - print(local_ids) records = list( task._stream_queried_data(mapping, local_ids, task._query_db(mapping)) ) - print(records) assert [[(date.today() + timedelta(days=9)).isoformat()], [None]] == records def test_get_statics(self): From 5d1f1d92854a524aba2a751a6d658448d1e67f87 Mon Sep 17 00:00:00 2001 From: aditya-balachander Date: Fri, 19 Jan 2024 16:36:36 +0530 Subject: [PATCH 28/36] Fixed Tests --- cumulusci/tasks/bulkdata/mapping_parser.py | 10 +- .../TestUpsert.test_simple_upsert__rest.yaml | 6 + ...psert_complex_external_id_field__rest.yaml | 9 +- ...ternal_id_field_rest__duplicate_error.yaml | 6 + ...sert.test_upsert_complex_fields__bulk.yaml | 4 + ...tUpsert.test_upsert_external_id_field.yaml | 6 + .../bulkdata/tests/test_mapping_parser.py | 124 ++++++++++++++++++ 7 files changed, 153 insertions(+), 12 deletions(-) diff --git a/cumulusci/tasks/bulkdata/mapping_parser.py b/cumulusci/tasks/bulkdata/mapping_parser.py index a151c02681..2feb85cd62 100644 --- a/cumulusci/tasks/bulkdata/mapping_parser.py +++ b/cumulusci/tasks/bulkdata/mapping_parser.py @@ -648,15 +648,7 @@ def _infer_and_validate_lookups(mapping: Dict, sf: Salesforce): if len(target_objects) == 1: # This is a non-polymorphic lookup. - try: - target_index = list(sf_objects.values()).index(target_objects[0]) - except ValueError: - fail = True - logger.error( - f"The field {m.sf_object}.{lookup.name} looks up to {target_objects[0]}, which is not included in the operation" - ) - continue - + target_index = list(sf_objects.values()).index(target_objects[0]) if target_index > idx or target_index == idx: # This is a non-polymorphic after step. lookup.after = list(mapping.keys())[idx] diff --git a/cumulusci/tasks/bulkdata/tests/cassettes/TestUpsert.test_simple_upsert__rest.yaml b/cumulusci/tasks/bulkdata/tests/cassettes/TestUpsert.test_simple_upsert__rest.yaml index 89dbfbfd05..dc6e715a76 100644 --- a/cumulusci/tasks/bulkdata/tests/cassettes/TestUpsert.test_simple_upsert__rest.yaml +++ b/cumulusci/tasks/bulkdata/tests/cassettes/TestUpsert.test_simple_upsert__rest.yaml @@ -180,6 +180,12 @@ interactions: \ [ ],\n \"created\" : false\n}, {\n \"id\" : \"0035500001H9kaAAAR\",\n\ \ \"success\" : true,\n \"errors\" : [ ],\n \"created\" : true\n} ]" - *id007 + - *id002 + - *id003 + - *id007 + - *id002 + - *id003 + - *id007 - request: method: POST uri: https://orgname.my.salesforce.com/services/data/vxx.0/composite/sobjects diff --git a/cumulusci/tasks/bulkdata/tests/cassettes/TestUpsert.test_upsert_complex_external_id_field__rest.yaml b/cumulusci/tasks/bulkdata/tests/cassettes/TestUpsert.test_upsert_complex_external_id_field__rest.yaml index ccbe1961e5..f6e9cc623f 100644 --- a/cumulusci/tasks/bulkdata/tests/cassettes/TestUpsert.test_upsert_complex_external_id_field__rest.yaml +++ b/cumulusci/tasks/bulkdata/tests/cassettes/TestUpsert.test_upsert_complex_external_id_field__rest.yaml @@ -4,9 +4,6 @@ interactions: include_file: GET_sobjects_Global_describe.yaml - &id005 include_file: GET_sobjects_Account_describe.yaml - - &id016 - include_file: GET_sobjects_Account_describe.yaml - - *id016 - *id001 - &id007 include_file: GET_sobjects_Contact_describe.yaml @@ -276,6 +273,12 @@ interactions: \n },\n \"Id\" : \"0035500001H9dBaAAJ\",\n \"FirstName\" : \"Gabrielle\"\ ,\n \"LastName\" : \"Vargas\"\n } ]\n}" - *id007 + - *id005 + - *id007 + - *id008 + - *id005 + - *id007 + - *id008 - request: method: PATCH uri: https://orgname.my.salesforce.com/services/data/vxx.0/composite/sobjects/Contact/Id diff --git a/cumulusci/tasks/bulkdata/tests/cassettes/TestUpsert.test_upsert_complex_external_id_field_rest__duplicate_error.yaml b/cumulusci/tasks/bulkdata/tests/cassettes/TestUpsert.test_upsert_complex_external_id_field_rest__duplicate_error.yaml index afa069956e..49185cae47 100644 --- a/cumulusci/tasks/bulkdata/tests/cassettes/TestUpsert.test_upsert_complex_external_id_field_rest__duplicate_error.yaml +++ b/cumulusci/tasks/bulkdata/tests/cassettes/TestUpsert.test_upsert_complex_external_id_field_rest__duplicate_error.yaml @@ -196,3 +196,9 @@ interactions: \ \"attributes\" : {\n \"type\" : \"Account\",\n \"url\" : \"\ /services/data/vxx.0/sobjects/Account/0015500001QdQWfAAN\"\n },\n \"\ Id\" : \"0015500001QdQWfAAN\",\n \"AccountNumber\" : \"420\"\n } ]\n}" + - *id005 + - *id007 + - *id008 + - *id005 + - *id007 + - *id008 diff --git a/cumulusci/tasks/bulkdata/tests/cassettes/TestUpsert.test_upsert_complex_fields__bulk.yaml b/cumulusci/tasks/bulkdata/tests/cassettes/TestUpsert.test_upsert_complex_fields__bulk.yaml index 05927d98dc..e844cc923d 100644 --- a/cumulusci/tasks/bulkdata/tests/cassettes/TestUpsert.test_upsert_complex_fields__bulk.yaml +++ b/cumulusci/tasks/bulkdata/tests/cassettes/TestUpsert.test_upsert_complex_fields__bulk.yaml @@ -332,6 +332,10 @@ interactions: - *id001 - *id010 - *id011 + - *id009 + - *id010 + - *id009 + - *id010 - request: *id012 response: status: *id003 diff --git a/cumulusci/tasks/bulkdata/tests/cassettes/TestUpsert.test_upsert_external_id_field.yaml b/cumulusci/tasks/bulkdata/tests/cassettes/TestUpsert.test_upsert_external_id_field.yaml index 1205a9974c..78f61336ba 100644 --- a/cumulusci/tasks/bulkdata/tests/cassettes/TestUpsert.test_upsert_external_id_field.yaml +++ b/cumulusci/tasks/bulkdata/tests/cassettes/TestUpsert.test_upsert_external_id_field.yaml @@ -451,6 +451,12 @@ interactions: smts: "2022-10-20T16:45:52.000Z" qdbatches: "0" numbatchtotal: "0" + - *id011 + - *id011 + - *id012 + - *id012 + - *id013 + - *id013 - request: method: POST uri: https://orgname.my.salesforce.com/services/async/vxx.0/job/750P0000006HoSBIA0 diff --git a/cumulusci/tasks/bulkdata/tests/test_mapping_parser.py b/cumulusci/tasks/bulkdata/tests/test_mapping_parser.py index c4e7872f9f..acfae235b1 100644 --- a/cumulusci/tasks/bulkdata/tests/test_mapping_parser.py +++ b/cumulusci/tasks/bulkdata/tests/test_mapping_parser.py @@ -13,6 +13,7 @@ MappingLookup, MappingStep, ValidationError, + _infer_and_validate_lookups, parse_from_yaml, validate_and_inject_mapping, ) @@ -1277,3 +1278,126 @@ def test_get_extract_field_list(self): "AccountSite", "ParentId", ] + + @responses.activate + def test_infer_and_validate_lookups__table_doesnt_exist(self, caplog): + caplog.set_level(logging.ERROR) + mock_describe_calls() + mapping = parse_from_yaml( + StringIO( + ( + "Insert Contacts:\n sf_object: Contact\n table: Contact\n lookups:\n AccountId:\n table: Account" + ) + ) + ) + org_config = DummyOrgConfig( + {"instance_url": "https://example.com", "access_token": "abc123"}, "test" + ) + + expected_error_message = "The table Account does not exist in the mapping file" + + with pytest.raises(BulkDataException) as e: + _infer_and_validate_lookups( + mapping=mapping, sf=org_config.salesforce_client + ) + assert "One or more relationship errors blocked the operation" in str( + e.value + ) + error_logs = [ + record.message for record in caplog.records if record.levelname == "ERROR" + ] + assert any(expected_error_message in error_log for error_log in error_logs) + + @responses.activate + def test_infer_and_validate_lookups__incorrect_order(self, caplog): + caplog.set_level(logging.ERROR) + mock_describe_calls() + mapping = parse_from_yaml( + StringIO( + ( + "Insert Account:\n" + " sf_object: Account\n" + " table: Account\n" + " fields:\n" + " - Description__c\n" + "Insert Events:\n" + " sf_object: Event\n" + " table: Event\n" + " lookups:\n" + " WhatId:\n" + " table:\n" + " - Opportunity\n" + " - Account\n" + "Insert Opportunity:\n" + " sf_object: Opportunity\n" + " table: Opportunity\n" + " fields:\n" + " - Description__c\n" + ) + ) + ) + org_config = DummyOrgConfig( + {"instance_url": "https://example.com", "access_token": "abc123"}, "test" + ) + + expected_error_message = "All included target objects (Opportunity,Account) for the field Event.WhatId must precede Event in the mapping." + + with pytest.raises(BulkDataException) as e: + _infer_and_validate_lookups( + mapping=mapping, sf=org_config.salesforce_client + ) + assert "One or more relationship errors blocked the operation" in str( + e.value + ) + error_logs = [ + record.message for record in caplog.records if record.levelname == "ERROR" + ] + assert any(expected_error_message in error_log for error_log in error_logs) + + @responses.activate + def test_infer_and_validate_lookups__after(self): + mock_describe_calls() + mapping = parse_from_yaml( + StringIO( + ( + "Insert Accounts:\n sf_object: Account\n table: Account\n lookups:\n ParentId:\n table: Account\n" + "Insert Contacts:\n sf_object: Contact\n table: Contact\n lookups:\n AccountId:\n table: Account" + ) + ) + ) + org_config = DummyOrgConfig( + {"instance_url": "https://example.com", "access_token": "abc123"}, "test" + ) + _infer_and_validate_lookups(mapping=mapping, sf=org_config.salesforce_client) + assert mapping["Insert Accounts"].lookups["ParentId"].after == "Insert Accounts" + assert mapping["Insert Contacts"].lookups["AccountId"].after is None + + @responses.activate + def test_infer_and_validate_lookups__invalid_reference(self, caplog): + caplog.set_level(logging.ERROR) + mock_describe_calls() + mapping = parse_from_yaml( + StringIO( + ( + "Insert Events:\n sf_object: Event\n table: Event\n fields:\n - Description__c\n" + "Insert Contacts:\n sf_object: Contact\n table: Contact\n lookups:\n AccountId:\n table: Event" + ) + ) + ) + org_config = DummyOrgConfig( + {"instance_url": "https://example.com", "access_token": "abc123"}, "test" + ) + + expected_error_message = "The lookup Event is not a valid lookup" + + with pytest.raises(BulkDataException) as e: + _infer_and_validate_lookups( + mapping=mapping, sf=org_config.salesforce_client + ) + assert "One or more relationship errors blocked the operation" in str( + e.value + ) + error_logs = [ + record.message for record in caplog.records if record.levelname == "ERROR" + ] + assert any(expected_error_message in error_log for error_log in error_logs) From 6e7b2695c5965f3726e40a01e73eb394960deb7b Mon Sep 17 00:00:00 2001 From: aditya-balachander Date: Sun, 21 Jan 2024 20:25:02 +0530 Subject: [PATCH 29/36] Handling for is_person_type column not present --- cumulusci/tasks/bulkdata/query_transformers.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/cumulusci/tasks/bulkdata/query_transformers.py b/cumulusci/tasks/bulkdata/query_transformers.py index 84c1e66207..83b9d514c3 100644 --- a/cumulusci/tasks/bulkdata/query_transformers.py +++ b/cumulusci/tasks/bulkdata/query_transformers.py @@ -135,6 +135,17 @@ def outerjoins_to_add(self): rt_dest_table = self.metadata.tables[ self.mapping.get_destination_record_type_table() ] + + # Check if 'is_person_type' column exists in rt_source_table.columns + is_person_type_column = getattr( + rt_source_table.columns, "is_person_type", None + ) + is_person_type_condition = ( + is_person_type_column == rt_dest_table.columns.is_person_type + if is_person_type_column + else True + ) + return [ ( rt_source_table, @@ -147,8 +158,7 @@ def outerjoins_to_add(self): and_( rt_dest_table.columns.developer_name == rt_source_table.columns.developer_name, - rt_dest_table.columns.is_person_type - == rt_source_table.columns.is_person_type, + is_person_type_condition, ), ), ] From a71c22c95781265ff03e8e0ed7f239141219e390 Mon Sep 17 00:00:00 2001 From: aditya-balachander Date: Wed, 31 Jan 2024 09:24:18 +0530 Subject: [PATCH 30/36] Generation of mapping file with polymorphic support --- cumulusci/core/tests/test_datasets_e2e.py | 35 +++++++++- .../calculate_dependencies.py | 66 ++++++++++-------- cumulusci/tasks/bulkdata/generate_mapping.py | 49 ++++++------- .../generate_mapping_utils/dependency_map.py | 14 ++-- .../extract_mapping_file_generator.py | 2 +- .../generate_mapping_from_declarations.py | 17 ++--- .../mapping_generator_post_processes.py | 9 ++- ...erate_extract_mapping_from_declarations.py | 2 +- ...generate_load_mapping_from_declarations.py | 6 +- cumulusci/tasks/bulkdata/mapping_parser.py | 9 ++- .../tasks/bulkdata/query_transformers.py | 4 +- .../bulkdata/tests/test_generatemapping.py | 69 ++++++++++--------- .../GET_sobjects_Global_describe.yaml | 17 ++++- .../GET_sobjects_Lead_describe.yaml | 18 +++++ 14 files changed, 202 insertions(+), 115 deletions(-) create mode 100644 cumulusci/tests/shared_cassettes/GET_sobjects_Lead_describe.yaml diff --git a/cumulusci/core/tests/test_datasets_e2e.py b/cumulusci/core/tests/test_datasets_e2e.py index 5c9bdfb868..e7347bf8e6 100644 --- a/cumulusci/core/tests/test_datasets_e2e.py +++ b/cumulusci/core/tests/test_datasets_e2e.py @@ -225,11 +225,19 @@ def test_datasets_extract_standard_objects( def test_datasets_read_explicit_extract_declaration( self, sf, project_config, org_config, delete_data_from_org, ensure_accounts ): - object_counts = {"Account": 6, "Contact": 1, "Opportunity": 5} + object_counts = { + "Account": 6, + "Contact": 1, + "Opportunity": 5, + "Lead": 1, + "Event": 2, + } obj_describes = ( describe_for("Account"), describe_for("Contact"), describe_for("Opportunity"), + describe_for("Lead"), + describe_for("Event"), ) with patch.object(type(org_config), "is_person_accounts_enabled", False), patch( "cumulusci.core.datasets.get_org_schema", @@ -263,12 +271,16 @@ def write_yaml(filename: str, json: Any): "Contact": { "fields": ["FirstName", "LastName", "AccountId"] }, + "Event": {"fields": ["Subject", "WhoId"]}, } }, ) loading_rules = write_yaml( "loading_rules.load.yml", - [{"sf_object": "Account", "load_after": "Contact"}], + [ + {"sf_object": "Account", "load_after": "Contact"}, + {"sf_object": "Lead", "load_after": "Event"}, + ], ) # Don't actually extract data. @@ -286,17 +298,34 @@ def write_yaml(filename: str, json: Any): "fields": ["FirstName", "LastName"], "lookups": { "AccountId": { - "table": "Account", + "table": ["Account"], "key_field": "AccountId", "after": "Insert Account", } }, }, + "Insert Event": { + "sf_object": "Event", + "table": "Event", + "fields": ["Subject"], + "lookups": { + "WhoId": { + "table": ["Contact", "Lead"], + "key_field": "WhoId", + "after": "Insert Lead", + } + }, + }, "Insert Account": { "sf_object": "Account", "table": "Account", "fields": ["Name"], }, + "Insert Lead": { + "sf_object": "Lead", + "table": "Lead", + "fields": ["Company", "LastName"], + }, } assert tuple(actual.items()) == tuple(expected.items()), actual.items() diff --git a/cumulusci/tasks/bulkdata/extract_dataset_utils/calculate_dependencies.py b/cumulusci/tasks/bulkdata/extract_dataset_utils/calculate_dependencies.py index 19162e35b9..c5c446099b 100644 --- a/cumulusci/tasks/bulkdata/extract_dataset_utils/calculate_dependencies.py +++ b/cumulusci/tasks/bulkdata/extract_dataset_utils/calculate_dependencies.py @@ -11,7 +11,7 @@ class SObjDependency(T.NamedTuple): table_name_from: str - table_name_to: str + table_names_to: T.Union[str, T.Tuple] field_name: str priority: bool = False @@ -51,16 +51,25 @@ def _collect_dependencies_for_sobject( if not field_info.createable: # pragma: no cover continue references = field_info.referenceTo - if len(references) == 1 and not references[0] == "RecordType": - target = references[0] - - target_disallowed = target in NOT_EXTRACTABLE - field_disallowed = target_disallowed or not field_info.createable + if references: + # Remove RecordType from references + if "RecordType" in references: + references.remove("RecordType") + if not references: + continue + + targets = tuple( + target for target in references if target not in NOT_EXTRACTABLE + ) + field_disallowed = not targets or not field_info.createable field_allowed = not (only_required_fields or field_disallowed) if field_info.requiredOnCreate or field_allowed: dependencies.setdefault(source_sfobject, []).append( SObjDependency( - source_sfobject, target, field_name, field_info.requiredOnCreate + source_sfobject, + targets, + field_name, + field_info.requiredOnCreate, ) ) @@ -80,28 +89,29 @@ def extend_declarations_to_include_referenced_tables( assert isinstance(sf_object, str) my_dependencies = dependencies.get(sf_object, ()) for dep in my_dependencies: - target_table = dep.table_name_to - sobj = schema.get(target_table) - target_extractable = ( - target_table not in NOT_EXTRACTABLE and sobj and sobj.extractable - ) - if target_table not in decls and target_extractable: - required_fields = [ - field.name - for field in schema[target_table].fields.values() - if field.requiredOnCreate - ] - decls[target_table] = synthesize_declaration_for_sobject( - target_table, required_fields, schema[target_table].fields + target_tables = dep.table_names_to + for target_table in target_tables: + sobj = schema.get(target_table) + target_extractable = ( + target_table not in NOT_EXTRACTABLE and sobj and sobj.extractable ) + if target_table not in decls and target_extractable: + required_fields = [ + field.name + for field in schema[target_table].fields.values() + if field.requiredOnCreate + ] + decls[target_table] = synthesize_declaration_for_sobject( + target_table, required_fields, schema[target_table].fields + ) - new_dependencies = _collect_dependencies_for_sobject( - target_table, - decls[target_table].fields, - schema, - only_required_fields=True, - ) - dependencies.update(new_dependencies) - to_process.append(target_table) + new_dependencies = _collect_dependencies_for_sobject( + target_table, + decls[target_table].fields, + schema, + only_required_fields=True, + ) + dependencies.update(new_dependencies) + to_process.append(target_table) return list(decls.values()) diff --git a/cumulusci/tasks/bulkdata/generate_mapping.py b/cumulusci/tasks/bulkdata/generate_mapping.py index eb71b8f992..d5372a9ba1 100644 --- a/cumulusci/tasks/bulkdata/generate_mapping.py +++ b/cumulusci/tasks/bulkdata/generate_mapping.py @@ -227,36 +227,27 @@ def strip_namespace(element): "referenceTo" ] - if len(referenceTo) > 1: # Polymorphic lookup - self.logger.warning( - f"Field {orig_obj}.{orig_field} is a polymorphic lookup, which is not supported" - ) - else: - orig_reference = referenceTo[0] - - # Can we safely namespace-strip this reference? - stripped_reference = ( - strip_namespace(orig_reference) - if strip_namespace(orig_reference) not in stack - else orig_reference - ) + # Can we safely namespace-strip this reference? + stripped_references = [ + strip_namespace(orig_reference) + if strip_namespace(orig_reference) not in stack + else orig_reference + for orig_reference in referenceTo + ] - if orig_reference == orig_obj: # Self-lookup - self.mapping[key]["lookups"][stripped_field] = { - "table": stripped_reference, - "after": key, - } - elif stack.index(orig_reference) > stack.index( - orig_obj - ): # Dependent lookup - self.mapping[key]["lookups"][stripped_field] = { - "table": stripped_reference, - "after": f"Insert {stripped_reference}", - } - else: # Regular lookup - self.mapping[key]["lookups"][stripped_field] = { - "table": stripped_reference - } + # The maximum reference index + max_reference_index = max( + stack.index(orig_reference) for orig_reference in referenceTo + ) + if max_reference_index >= stack.index(orig_obj): # Dependent lookup + self.mapping[key]["lookups"][stripped_field] = { + "table": stripped_references, + "after": f"Insert {stripped_references[referenceTo.index(stack[max_reference_index])]}", + } + else: # Regular lookup + self.mapping[key]["lookups"][stripped_field] = { + "table": stripped_references + } def _split_dependencies(self, objs, dependencies): """Attempt to flatten the object network into a sequence of load operations.""" diff --git a/cumulusci/tasks/bulkdata/generate_mapping_utils/dependency_map.py b/cumulusci/tasks/bulkdata/generate_mapping_utils/dependency_map.py index ae9c530226..e103bde33a 100644 --- a/cumulusci/tasks/bulkdata/generate_mapping_utils/dependency_map.py +++ b/cumulusci/tasks/bulkdata/generate_mapping_utils/dependency_map.py @@ -45,9 +45,11 @@ def _map_references( table_deps.add(dep) self.reference_fields[ (dep.table_name_from, dep.field_name) - ] = dep.table_name_to + ] = dep.table_names_to - def target_table_for(self, tablename: str, fieldname: str) -> T.Optional[str]: + def target_table_for( + self, tablename: str, fieldname: str + ) -> T.Optional[T.Union[str, T.Tuple]]: return self.reference_fields.get((tablename, fieldname)) def get_dependency_order(self): @@ -135,9 +137,13 @@ def _table_is_free( """ tables_this_table_depends_upon = dependencies.get(table_name, OrderedSet()).copy() for dependency in sorted(tables_this_table_depends_upon): + table_names = ( + [dependency.table_names_to] + if isinstance(dependency.table_names_to, str) + else dependency.table_names_to + ) if ( - dependency.table_name_to in sorted_tables - or dependency.table_name_to == table_name + all(table in sorted_tables or table == table_name for table in table_names) or dependency.priority < priority ): tables_this_table_depends_upon.remove(dependency) diff --git a/cumulusci/tasks/bulkdata/generate_mapping_utils/extract_mapping_file_generator.py b/cumulusci/tasks/bulkdata/generate_mapping_utils/extract_mapping_file_generator.py index 0ff17f04bf..c74bb27c95 100644 --- a/cumulusci/tasks/bulkdata/generate_mapping_utils/extract_mapping_file_generator.py +++ b/cumulusci/tasks/bulkdata/generate_mapping_utils/extract_mapping_file_generator.py @@ -18,7 +18,7 @@ def _mapping_decl_for_extract_decl( decl: SimplifiedExtractDeclarationWithLookups, ): """Make a CCI extract mapping step from a SimplifiedExtractDeclarationWithLookups""" - lookups = {lookup: {"table": table} for lookup, table in decl.lookups.items()} + lookups = {lookup: {"table": tables} for lookup, tables in decl.lookups.items()} mapping_dict: dict[str, T.Any] = { "sf_object": decl.sf_object, } diff --git a/cumulusci/tasks/bulkdata/generate_mapping_utils/generate_mapping_from_declarations.py b/cumulusci/tasks/bulkdata/generate_mapping_utils/generate_mapping_from_declarations.py index 8912bb6e57..4b3b1784b4 100644 --- a/cumulusci/tasks/bulkdata/generate_mapping_utils/generate_mapping_from_declarations.py +++ b/cumulusci/tasks/bulkdata/generate_mapping_utils/generate_mapping_from_declarations.py @@ -18,7 +18,7 @@ class SimplifiedExtractDeclarationWithLookups(SimplifiedExtractDeclaration): - lookups: T.Dict[str, str] + lookups: T.Dict[str, T.Union[str, T.Tuple]] def create_load_mapping_file_from_extract_declarations( @@ -54,9 +54,9 @@ def _discover_dependendencies(simplified_decls: T.Sequence): intertable_dependencies = OrderedSet() for decl in simplified_decls: - for fieldname, tablename in decl.lookups.items(): + for fieldname, tablenames in decl.lookups.items(): intertable_dependencies.add( - SObjDependency(decl.sf_object, tablename, fieldname) + SObjDependency(decl.sf_object, tablenames, fieldname) ) return intertable_dependencies @@ -105,10 +105,7 @@ def is_lookup(field_name): simple_fields, lookups = partition(is_lookup, decl.fields) def target_table(field_info): - if len(field_info.referenceTo) == 1: - target = field_info.referenceTo[0] - else: # pragma: no cover # TODO: Cover - target = "Polymorphic lookups are not supported" + target = field_info.referenceTo return target lookups = list(lookups) @@ -117,8 +114,8 @@ def target_table(field_info): (lookup, target_table(sobject_schema_info.fields[lookup])) for lookup in lookups ) lookups_and_targets = ( - (lookup, table) - for lookup, table in lookups_and_targets - if table in referenceable_tables + (lookup, [table for table in tables if table in referenceable_tables]) + for lookup, tables in lookups_and_targets + if any(table in referenceable_tables for table in tables) ) return simple_fields, lookups_and_targets diff --git a/cumulusci/tasks/bulkdata/generate_mapping_utils/mapping_generator_post_processes.py b/cumulusci/tasks/bulkdata/generate_mapping_utils/mapping_generator_post_processes.py index aa2fe6213f..bb9eeab684 100644 --- a/cumulusci/tasks/bulkdata/generate_mapping_utils/mapping_generator_post_processes.py +++ b/cumulusci/tasks/bulkdata/generate_mapping_utils/mapping_generator_post_processes.py @@ -16,7 +16,14 @@ def add_after_statements(mappings: dict): "This code is not yet tested for use with Snowfakery and Person Accounts" ) # continue - target_mapping_index = indexed_by_sobject[target_table] + if isinstance(target_table, list): + target_mapping_index = max( + [indexed_by_sobject[table] for table in target_table], + key=lambda index: index.first_instance, + ) + else: + target_mapping_index = indexed_by_sobject[target_table] + if target_mapping_index.first_instance >= idx: if not lookup.get("after"): lookup["after"] = target_mapping_index.last_step_name diff --git a/cumulusci/tasks/bulkdata/generate_mapping_utils/tests/test_generate_extract_mapping_from_declarations.py b/cumulusci/tasks/bulkdata/generate_mapping_utils/tests/test_generate_extract_mapping_from_declarations.py index 430645514d..293fd0ad4b 100644 --- a/cumulusci/tasks/bulkdata/generate_mapping_utils/tests/test_generate_extract_mapping_from_declarations.py +++ b/cumulusci/tasks/bulkdata/generate_mapping_utils/tests/test_generate_extract_mapping_from_declarations.py @@ -69,7 +69,7 @@ def test_generate_mapping_from_declarations__lookups(self, org_config): "api": "smart", "sf_object": "Contact", "fields": ["LastName"], - "lookups": {"AccountId": {"table": "Account"}}, + "lookups": {"AccountId": {"table": ("Account",)}}, }, } diff --git a/cumulusci/tasks/bulkdata/generate_mapping_utils/tests/test_generate_load_mapping_from_declarations.py b/cumulusci/tasks/bulkdata/generate_mapping_utils/tests/test_generate_load_mapping_from_declarations.py index 9b163bfe3b..169797ba93 100644 --- a/cumulusci/tasks/bulkdata/generate_mapping_utils/tests/test_generate_load_mapping_from_declarations.py +++ b/cumulusci/tasks/bulkdata/generate_mapping_utils/tests/test_generate_load_mapping_from_declarations.py @@ -117,7 +117,7 @@ def test_generate_load_mapping_from_declarations__lookups(self, org_config): "table": "Contact", "fields": ["FirstName", "LastName"], "lookups": { - "AccountId": {"table": "Account", "key_field": "AccountId"} + "AccountId": {"table": ["Account"], "key_field": "AccountId"} }, }, } @@ -157,7 +157,7 @@ def test_generate_load_mapping_from_declarations__circular_lookups( "Primary_Contact__c": { "after": "Insert Contact", "key_field": "Primary_Contact__c", - "table": "Contact", + "table": ["Contact"], } }, "sf_object": "Account", @@ -168,7 +168,7 @@ def test_generate_load_mapping_from_declarations__circular_lookups( "table": "Contact", "fields": ["FirstName", "LastName"], "lookups": { - "AccountId": {"table": "Account", "key_field": "AccountId"} + "AccountId": {"table": ["Account"], "key_field": "AccountId"} }, }, }, mf diff --git a/cumulusci/tasks/bulkdata/mapping_parser.py b/cumulusci/tasks/bulkdata/mapping_parser.py index 2feb85cd62..67e72094b9 100644 --- a/cumulusci/tasks/bulkdata/mapping_parser.py +++ b/cumulusci/tasks/bulkdata/mapping_parser.py @@ -712,7 +712,14 @@ def validate_and_inject_mapping( for field in list(m.lookups.keys()): lookup = m.lookups[field] - if lookup.table not in [step.table for step in mapping.values()]: + if isinstance(lookup.table, list): + lookup_tables = lookup.table + else: + lookup_tables = [lookup.table] + if all( + table not in [step.table for step in mapping.values()] + for table in lookup_tables + ): del m.lookups[field] # Make sure this didn't cause the operation to be invalid diff --git a/cumulusci/tasks/bulkdata/query_transformers.py b/cumulusci/tasks/bulkdata/query_transformers.py index 83b9d514c3..a2a348a406 100644 --- a/cumulusci/tasks/bulkdata/query_transformers.py +++ b/cumulusci/tasks/bulkdata/query_transformers.py @@ -141,8 +141,8 @@ def outerjoins_to_add(self): rt_source_table.columns, "is_person_type", None ) is_person_type_condition = ( - is_person_type_column == rt_dest_table.columns.is_person_type - if is_person_type_column + rt_dest_table.columns.is_person_type == is_person_type_column + if is_person_type_column is not None else True ) diff --git a/cumulusci/tasks/bulkdata/tests/test_generatemapping.py b/cumulusci/tasks/bulkdata/tests/test_generatemapping.py index fa9a775a60..290bcf5b50 100644 --- a/cumulusci/tasks/bulkdata/tests/test_generatemapping.py +++ b/cumulusci/tasks/bulkdata/tests/test_generatemapping.py @@ -269,10 +269,9 @@ def test_run_task(self): assert ["Account__c"] == list( t.mapping["Insert Child__c"]["lookups"].keys() ) - assert ( - "Parent" - == t.mapping["Insert Child__c"]["lookups"]["Account__c"]["table"] - ) + assert ["Parent"] == t.mapping["Insert Child__c"]["lookups"]["Account__c"][ + "table" + ] @responses.activate def test_collect_objects__simple_custom_objects(self): @@ -582,20 +581,21 @@ def test_build_mapping(self, prompt): assert t.mapping["Insert Account"]["sf_object"] == "Account" assert ["Name"] == t.mapping["Insert Account"]["fields"] assert ["Dependent__c"] == list(t.mapping["Insert Account"]["lookups"].keys()) - assert ( - "Child__c" - == t.mapping["Insert Account"]["lookups"]["Dependent__c"]["table"] - ) + assert ["Child__c"] == t.mapping["Insert Account"]["lookups"]["Dependent__c"][ + "table" + ] assert t.mapping["Insert Child__c"]["sf_object"] == "Child__c" assert ["Name"] == t.mapping["Insert Child__c"]["fields"] assert ["Account__c", "Self__c"] == list( t.mapping["Insert Child__c"]["lookups"].keys() ) - assert ( - "Account" == t.mapping["Insert Child__c"]["lookups"]["Account__c"]["table"] - ) - assert t.mapping["Insert Child__c"]["lookups"]["Self__c"]["table"] == "Child__c" + assert ["Account"] == t.mapping["Insert Child__c"]["lookups"]["Account__c"][ + "table" + ] + assert t.mapping["Insert Child__c"]["lookups"]["Self__c"]["table"] == [ + "Child__c" + ] @mock.patch("click.prompt") def test_build_mapping__strip_namespace(self, prompt): @@ -642,20 +642,21 @@ def test_build_mapping__strip_namespace(self, prompt): assert t.mapping["Insert Parent__c"]["sf_object"] == "Parent__c" assert ["Name"] == t.mapping["Insert Parent__c"]["fields"] assert ["Dependent__c"] == list(t.mapping["Insert Parent__c"]["lookups"].keys()) - assert ( - "Child__c" - == t.mapping["Insert Parent__c"]["lookups"]["Dependent__c"]["table"] - ) + assert ["Child__c"] == t.mapping["Insert Parent__c"]["lookups"]["Dependent__c"][ + "table" + ] assert t.mapping["Insert Child__c"]["sf_object"] == "Child__c" assert ["Name"] == t.mapping["Insert Child__c"]["fields"] assert ["Parent__c", "Self__c"] == list( t.mapping["Insert Child__c"]["lookups"].keys() ) - assert ( - "Parent__c" == t.mapping["Insert Child__c"]["lookups"]["Parent__c"]["table"] - ) - assert t.mapping["Insert Child__c"]["lookups"]["Self__c"]["table"] == "Child__c" + assert ["Parent__c"] == t.mapping["Insert Child__c"]["lookups"]["Parent__c"][ + "table" + ] + assert t.mapping["Insert Child__c"]["lookups"]["Self__c"]["table"] == [ + "Child__c" + ] @mock.patch("click.prompt") def test_build_mapping__no_strip_namespace_if_dup_component(self, prompt): @@ -699,19 +700,17 @@ def test_build_mapping__no_strip_namespace_if_dup_component(self, prompt): assert set(["ns__Parent__c", "Parent__c"]) == set( t.mapping["Insert ns__Child__c"]["lookups"].keys() ) - assert ( + assert ["Parent__c"] == t.mapping["Insert ns__Child__c"]["lookups"][ + "ns__Parent__c" + ]["table"] + assert ["ns__Child__c"] == t.mapping["Insert ns__Child__c"]["lookups"][ "Parent__c" - == t.mapping["Insert ns__Child__c"]["lookups"]["ns__Parent__c"]["table"] - ) - assert ( - "ns__Child__c" - == t.mapping["Insert ns__Child__c"]["lookups"]["Parent__c"]["table"] - ) + ]["table"] assert t.mapping["Insert Child__c"]["sf_object"] == "Child__c" assert ["Name"] == t.mapping["Insert Child__c"]["fields"] - def test_build_mapping__warns_polymorphic_lookups(self): + def test_build_mapping__polymorphic_lookups(self): t = _make_task(GenerateMapping, {"options": {"path": "t"}}) t.mapping_objects = ["Account", "Contact", "Custom__c"] @@ -733,12 +732,20 @@ def test_build_mapping__warns_polymorphic_lookups(self): "Contact": {"PolyLookup__c": _Field("PolyLookup__c", {})}, } } - t.logger = mock.Mock() t._build_mapping() - t.logger.warning.assert_called_once_with( - "Field Custom__c.PolyLookup__c is a polymorphic lookup, which is not supported" + assert set(["Insert Account", "Insert Contact", "Insert Custom__c"]) == set( + t.mapping.keys() + ) + + assert t.mapping["Insert Custom__c"]["sf_object"] == "Custom__c" + assert ["Name"] == t.mapping["Insert Custom__c"]["fields"] + assert set(["PolyLookup__c"]) == set( + t.mapping["Insert Custom__c"]["lookups"].keys() ) + assert ["Account", "Contact"] == t.mapping["Insert Custom__c"]["lookups"][ + "PolyLookup__c" + ]["table"] def test_split_dependencies__no_cycles(self): t = _make_task(GenerateMapping, {"options": {"path": "t"}}) diff --git a/cumulusci/tests/shared_cassettes/GET_sobjects_Global_describe.yaml b/cumulusci/tests/shared_cassettes/GET_sobjects_Global_describe.yaml index 78440e4796..c643112251 100644 --- a/cumulusci/tests/shared_cassettes/GET_sobjects_Global_describe.yaml +++ b/cumulusci/tests/shared_cassettes/GET_sobjects_Global_describe.yaml @@ -1320,4 +1320,19 @@ response: "/services/data/v52.0/sobjects/Event/listviews", "describe": "/services/data/v52.0/sobjects/Event/describe", "quickActions": "/services/data/v52.0/sobjects/Event/quickActions", "layouts": "/services/data/v52.0/sobjects/Event/describe/layouts", "sobject": - "/services/data/v52.0/sobjects/Event"}}]}' + "/services/data/v52.0/sobjects/Event"}}, {"activateable": + false, "associateEntityType": null, "associateParentEntity": null, "createable": + true, "custom": false, "customSetting": false, "deepCloneable": false, + "deletable": true, "deprecatedAndHidden": false, "feedEnabled": true, + "hasSubtypes": false, "isInterface": false, "isSubtype": false, "keyPrefix": + null, "label": "Lead", "labelPlural": "Leads", "layoutable": true, + "mergeable": true, "mruEnabled": true, "name": "Lead", "queryable": + true, "replicateable": true, "retrieveable": true, "searchable": true, + "triggerable": true, "undeletable": true, "updateable": true, "urls": + {"compactLayouts": "/services/data/v52.0/sobjects/Lead/describe/compactLayouts", + "rowTemplate": "/services/data/v52.0/sobjects/Lead/{ID}", "approvalLayouts": + "/services/data/v52.0/sobjects/Lead/describe/approvalLayouts", "listviews": + "/services/data/v52.0/sobjects/Lead/listviews", "describe": "/services/data/v52.0/sobjects/Lead/describe", + "quickActions": "/services/data/v52.0/sobjects/Lead/quickActions", + "layouts": "/services/data/v52.0/sobjects/Lead/describe/layouts", "sobject": + "/services/data/v52.0/sobjects/Lead"}}]}' diff --git a/cumulusci/tests/shared_cassettes/GET_sobjects_Lead_describe.yaml b/cumulusci/tests/shared_cassettes/GET_sobjects_Lead_describe.yaml new file mode 100644 index 0000000000..368b000478 --- /dev/null +++ b/cumulusci/tests/shared_cassettes/GET_sobjects_Lead_describe.yaml @@ -0,0 +1,18 @@ +request: + method: GET + uri: https://orgname.my.salesforce.com/services/data/vxx.0/sobjects/Lead/describe + body: null + headers: + Request-Headers: + - Elided +response: + status: + code: 200 + message: OK + headers: + Content-Type: + - application/json;charset=UTF-8 + Others: Elided + body: + string: >- + {"actionOverrides":[],"activateable":false,"associateEntityType":null,"associateParentEntity":null,"childRelationships":[{"cascadeDelete":true,"childSObject":"AIInsightValue","deprecatedAndHidden":false,"field":"SobjectLookupValueId","junctionIdListNames":[],"junctionReferenceTo":[],"relationshipName":null,"restrictedDelete":false},{"cascadeDelete":true,"childSObject":"AIRecordInsight","deprecatedAndHidden":false,"field":"TargetId","junctionIdListNames":[],"junctionReferenceTo":[],"relationshipName":null,"restrictedDelete":false},{"cascadeDelete":false,"childSObject":"AcceptedEventRelation","deprecatedAndHidden":false,"field":"RelationId","junctionIdListNames":[],"junctionReferenceTo":[],"relationshipName":"AcceptedEventRelations","restrictedDelete":false},{"cascadeDelete":true,"childSObject":"ActivityHistory","deprecatedAndHidden":false,"field":"WhoId","junctionIdListNames":[],"junctionReferenceTo":[],"relationshipName":"ActivityHistories","restrictedDelete":false},{"cascadeDelete":true,"childSObject":"AttachedContentDocument","deprecatedAndHidden":false,"field":"LinkedEntityId","junctionIdListNames":[],"junctionReferenceTo":[],"relationshipName":"AttachedContentDocuments","restrictedDelete":false},{"cascadeDelete":true,"childSObject":"Attachment","deprecatedAndHidden":false,"field":"ParentId","junctionIdListNames":[],"junctionReferenceTo":[],"relationshipName":"Attachments","restrictedDelete":false},{"cascadeDelete":true,"childSObject":"CampaignMember","deprecatedAndHidden":false,"field":"LeadId","junctionIdListNames":[],"junctionReferenceTo":[],"relationshipName":"CampaignMembers","restrictedDelete":false},{"cascadeDelete":false,"childSObject":"CampaignMember","deprecatedAndHidden":false,"field":"LeadOrContactId","junctionIdListNames":[],"junctionReferenceTo":[],"relationshipName":null,"restrictedDelete":false},{"cascadeDelete":false,"childSObject":"CampaignMemberChangeEvent","deprecatedAndHidden":false,"field":"LeadId","junctionIdListNames":[],"junctionReferenceTo":[],"relationshipName":null,"restrictedDelete":false},{"cascadeDelete":true,"childSObject":"CollaborationGroupRecord","deprecatedAndHidden":false,"field":"RecordId","junctionIdListNames":[],"junctionReferenceTo":[],"relationshipName":"RecordAssociatedGroups","restrictedDelete":false},{"cascadeDelete":true,"childSObject":"CombinedAttachment","deprecatedAndHidden":false,"field":"ParentId","junctionIdListNames":[],"junctionReferenceTo":[],"relationshipName":"CombinedAttachments","restrictedDelete":false},{"cascadeDelete":false,"childSObject":"ContactRequest","deprecatedAndHidden":false,"field":"WhoId","junctionIdListNames":[],"junctionReferenceTo":[],"relationshipName":"ContactRequests","restrictedDelete":false},{"cascadeDelete":true,"childSObject":"ContentDistribution","deprecatedAndHidden":false,"field":"RelatedRecordId","junctionIdListNames":[],"junctionReferenceTo":[],"relationshipName":null,"restrictedDelete":false},{"cascadeDelete":true,"childSObject":"ContentDocumentLink","deprecatedAndHidden":false,"field":"LinkedEntityId","junctionIdListNames":[],"junctionReferenceTo":[],"relationshipName":"ContentDocumentLinks","restrictedDelete":false},{"cascadeDelete":false,"childSObject":"ContentDocumentLinkChangeEvent","deprecatedAndHidden":false,"field":"LinkedEntityId","junctionIdListNames":[],"junctionReferenceTo":[],"relationshipName":null,"restrictedDelete":false},{"cascadeDelete":false,"childSObject":"ContentVersion","deprecatedAndHidden":false,"field":"FirstPublishLocationId","junctionIdListNames":[],"junctionReferenceTo":[],"relationshipName":null,"restrictedDelete":false},{"cascadeDelete":false,"childSObject":"ContentVersionChangeEvent","deprecatedAndHidden":false,"field":"FirstPublishLocationId","junctionIdListNames":[],"junctionReferenceTo":[],"relationshipName":null,"restrictedDelete":false},{"cascadeDelete":false,"childSObject":"DeclinedEventRelation","deprecatedAndHidden":false,"field":"RelationId","junctionIdListNames":[],"junctionReferenceTo":[],"relationshipName":"DeclinedEventRelations","restrictedDelete":false},{"cascadeDelete":true,"childSObject":"DuplicateRecordItem","deprecatedAndHidden":false,"field":"RecordId","junctionIdListNames":[],"junctionReferenceTo":[],"relationshipName":"DuplicateRecordItems","restrictedDelete":false},{"cascadeDelete":true,"childSObject":"EmailMessageRelation","deprecatedAndHidden":false,"field":"RelationId","junctionIdListNames":[],"junctionReferenceTo":[],"relationshipName":"EmailMessageRelations","restrictedDelete":false},{"cascadeDelete":true,"childSObject":"EmailStatus","deprecatedAndHidden":false,"field":"WhoId","junctionIdListNames":[],"junctionReferenceTo":[],"relationshipName":"EmailStatuses","restrictedDelete":false},{"cascadeDelete":true,"childSObject":"EntitySubscription","deprecatedAndHidden":false,"field":"ParentId","junctionIdListNames":[],"junctionReferenceTo":[],"relationshipName":"FeedSubscriptionsForEntity","restrictedDelete":false},{"cascadeDelete":true,"childSObject":"Event","deprecatedAndHidden":false,"field":"WhoId","junctionIdListNames":[],"junctionReferenceTo":[],"relationshipName":"Events","restrictedDelete":false},{"cascadeDelete":false,"childSObject":"EventChangeEvent","deprecatedAndHidden":false,"field":"WhoId","junctionIdListNames":[],"junctionReferenceTo":[],"relationshipName":null,"restrictedDelete":false},{"cascadeDelete":true,"childSObject":"EventRelation","deprecatedAndHidden":false,"field":"RelationId","junctionIdListNames":[],"junctionReferenceTo":[],"relationshipName":"EventRelations","restrictedDelete":false},{"cascadeDelete":false,"childSObject":"EventRelationChangeEvent","deprecatedAndHidden":false,"field":"RelationId","junctionIdListNames":[],"junctionReferenceTo":[],"relationshipName":null,"restrictedDelete":false},{"cascadeDelete":false,"childSObject":"FeedComment","deprecatedAndHidden":false,"field":"ParentId","junctionIdListNames":[],"junctionReferenceTo":[],"relationshipName":null,"restrictedDelete":false},{"cascadeDelete":true,"childSObject":"FeedItem","deprecatedAndHidden":false,"field":"ParentId","junctionIdListNames":[],"junctionReferenceTo":[],"relationshipName":null,"restrictedDelete":false},{"cascadeDelete":false,"childSObject":"FlowExecutionErrorEvent","deprecatedAndHidden":false,"field":"ContextRecordId","junctionIdListNames":[],"junctionReferenceTo":[],"relationshipName":null,"restrictedDelete":false},{"cascadeDelete":false,"childSObject":"FlowOrchestrationWorkItem","deprecatedAndHidden":false,"field":"RelatedRecordId","junctionIdListNames":[],"junctionReferenceTo":[],"relationshipName":"FlowOrchestrationWorkItems","restrictedDelete":false},{"cascadeDelete":false,"childSObject":"FlowRecordRelation","deprecatedAndHidden":false,"field":"RelatedRecordId","junctionIdListNames":[],"junctionReferenceTo":[],"relationshipName":null,"restrictedDelete":false},{"cascadeDelete":true,"childSObject":"LeadCleanInfo","deprecatedAndHidden":false,"field":"LeadId","junctionIdListNames":[],"junctionReferenceTo":[],"relationshipName":"LeadCleanInfos","restrictedDelete":false},{"cascadeDelete":true,"childSObject":"LeadFeed","deprecatedAndHidden":false,"field":"ParentId","junctionIdListNames":[],"junctionReferenceTo":[],"relationshipName":"Feeds","restrictedDelete":false},{"cascadeDelete":true,"childSObject":"LeadHistory","deprecatedAndHidden":false,"field":"LeadId","junctionIdListNames":[],"junctionReferenceTo":[],"relationshipName":"Histories","restrictedDelete":false},{"cascadeDelete":true,"childSObject":"LeadShare","deprecatedAndHidden":false,"field":"LeadId","junctionIdListNames":[],"junctionReferenceTo":[],"relationshipName":"Shares","restrictedDelete":false},{"cascadeDelete":true,"childSObject":"ListEmailIndividualRecipient","deprecatedAndHidden":false,"field":"RecipientId","junctionIdListNames":[],"junctionReferenceTo":[],"relationshipName":"ListEmailIndividualRecipients","restrictedDelete":false},{"cascadeDelete":false,"childSObject":"MessagingEndUser","deprecatedAndHidden":false,"field":"LeadId","junctionIdListNames":[],"junctionReferenceTo":[],"relationshipName":"MessagingEndUsers","restrictedDelete":false},{"cascadeDelete":false,"childSObject":"MessagingSession","deprecatedAndHidden":false,"field":"LeadId","junctionIdListNames":[],"junctionReferenceTo":[],"relationshipName":"MessagingSessions","restrictedDelete":false},{"cascadeDelete":false,"childSObject":"NetworkActivityAudit","deprecatedAndHidden":false,"field":"ParentEntityId","junctionIdListNames":[],"junctionReferenceTo":[],"relationshipName":"ParentEntities","restrictedDelete":false},{"cascadeDelete":false,"childSObject":"NetworkFeedResponseMetric","deprecatedAndHidden":false,"field":"ParentRecordId","junctionIdListNames":[],"junctionReferenceTo":[],"relationshipName":null,"restrictedDelete":false},{"cascadeDelete":true,"childSObject":"Note","deprecatedAndHidden":false,"field":"ParentId","junctionIdListNames":[],"junctionReferenceTo":[],"relationshipName":"Notes","restrictedDelete":false},{"cascadeDelete":true,"childSObject":"NoteAndAttachment","deprecatedAndHidden":false,"field":"ParentId","junctionIdListNames":[],"junctionReferenceTo":[],"relationshipName":"NotesAndAttachments","restrictedDelete":false},{"cascadeDelete":true,"childSObject":"OpenActivity","deprecatedAndHidden":false,"field":"WhoId","junctionIdListNames":[],"junctionReferenceTo":[],"relationshipName":"OpenActivities","restrictedDelete":false},{"cascadeDelete":false,"childSObject":"OutgoingEmail","deprecatedAndHidden":false,"field":"WhoId","junctionIdListNames":[],"junctionReferenceTo":[],"relationshipName":null,"restrictedDelete":false},{"cascadeDelete":false,"childSObject":"OutgoingEmailRelation","deprecatedAndHidden":false,"field":"RelationId","junctionIdListNames":[],"junctionReferenceTo":[],"relationshipName":"OutgoingEmailRelations","restrictedDelete":false},{"cascadeDelete":true,"childSObject":"ProcessInstance","deprecatedAndHidden":false,"field":"TargetObjectId","junctionIdListNames":[],"junctionReferenceTo":[],"relationshipName":"ProcessInstances","restrictedDelete":false},{"cascadeDelete":false,"childSObject":"ProcessInstanceChangeEvent","deprecatedAndHidden":false,"field":"TargetObjectId","junctionIdListNames":[],"junctionReferenceTo":[],"relationshipName":null,"restrictedDelete":false},{"cascadeDelete":false,"childSObject":"ProcessInstanceHistory","deprecatedAndHidden":false,"field":"TargetObjectId","junctionIdListNames":[],"junctionReferenceTo":[],"relationshipName":"ProcessSteps","restrictedDelete":false},{"cascadeDelete":true,"childSObject":"RecordAction","deprecatedAndHidden":false,"field":"RecordId","junctionIdListNames":[],"junctionReferenceTo":[],"relationshipName":"RecordActions","restrictedDelete":false},{"cascadeDelete":false,"childSObject":"RecordActionHistory","deprecatedAndHidden":false,"field":"ParentRecordId","junctionIdListNames":[],"junctionReferenceTo":[],"relationshipName":"RecordActionHistories","restrictedDelete":false},{"cascadeDelete":true,"childSObject":"ServiceAppointment","deprecatedAndHidden":false,"field":"ParentRecordId","junctionIdListNames":[],"junctionReferenceTo":[],"relationshipName":"ServiceAppointments","restrictedDelete":false},{"cascadeDelete":false,"childSObject":"ServiceAppointmentChangeEvent","deprecatedAndHidden":false,"field":"ParentRecordId","junctionIdListNames":[],"junctionReferenceTo":[],"relationshipName":null,"restrictedDelete":false},{"cascadeDelete":true,"childSObject":"Task","deprecatedAndHidden":false,"field":"WhoId","junctionIdListNames":[],"junctionReferenceTo":[],"relationshipName":"Tasks","restrictedDelete":false},{"cascadeDelete":false,"childSObject":"TaskChangeEvent","deprecatedAndHidden":false,"field":"WhoId","junctionIdListNames":[],"junctionReferenceTo":[],"relationshipName":null,"restrictedDelete":false},{"cascadeDelete":true,"childSObject":"TopicAssignment","deprecatedAndHidden":false,"field":"EntityId","junctionIdListNames":[],"junctionReferenceTo":[],"relationshipName":"TopicAssignments","restrictedDelete":false},{"cascadeDelete":false,"childSObject":"UndecidedEventRelation","deprecatedAndHidden":false,"field":"RelationId","junctionIdListNames":[],"junctionReferenceTo":[],"relationshipName":"UndecidedEventRelations","restrictedDelete":false},{"cascadeDelete":true,"childSObject":"UserEmailPreferredPerson","deprecatedAndHidden":false,"field":"PersonRecordId","junctionIdListNames":[],"junctionReferenceTo":[],"relationshipName":"PersonRecord","restrictedDelete":false}],"compactLayoutable":true,"createable":true,"custom":false,"customSetting":false,"deepCloneable":false,"defaultImplementation":null,"deletable":true,"deprecatedAndHidden":false,"extendedBy":null,"extendsInterfaces":null,"feedEnabled":true,"fields":[{"aggregatable":true,"aiPredictionField":false,"autoNumber":false,"byteLength":18,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":false,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":true,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":true,"highScaleNumber":false,"htmlFormatted":false,"idLookup":true,"inlineHelpText":null,"label":"Lead ID","length":18,"mask":null,"maskType":null,"name":"Id","nameField":false,"namePointing":false,"nillable":false,"permissionable":false,"picklistValues":[],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"searchPrefilterable":false,"soapType":"tns:ID","sortable":true,"type":"id","unique":false,"updateable":false,"writeRequiresMasterRead":false},{"aggregatable":false,"aiPredictionField":false,"autoNumber":false,"byteLength":0,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":false,"custom":false,"defaultValue":false,"defaultValueFormula":null,"defaultedOnCreate":true,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":true,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Deleted","length":0,"mask":null,"maskType":null,"name":"IsDeleted","nameField":false,"namePointing":false,"nillable":false,"permissionable":false,"picklistValues":[],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"searchPrefilterable":false,"soapType":"xsd:boolean","sortable":true,"type":"boolean","unique":false,"updateable":false,"writeRequiresMasterRead":false},{"aggregatable":true,"aiPredictionField":false,"autoNumber":false,"byteLength":18,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":false,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":false,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":true,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Master Record ID","length":18,"mask":null,"maskType":null,"name":"MasterRecordId","nameField":false,"namePointing":false,"nillable":true,"permissionable":false,"picklistValues":[],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":["Lead"],"relationshipName":"MasterRecord","relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"searchPrefilterable":false,"soapType":"tns:ID","sortable":true,"type":"reference","unique":false,"updateable":false,"writeRequiresMasterRead":false},{"aggregatable":true,"aiPredictionField":false,"autoNumber":false,"byteLength":240,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":"Name","controllerName":null,"createable":true,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":false,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":"personname","filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":true,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Last Name","length":80,"mask":null,"maskType":null,"name":"LastName","nameField":false,"namePointing":false,"nillable":false,"permissionable":false,"picklistValues":[],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"searchPrefilterable":false,"soapType":"xsd:string","sortable":true,"type":"string","unique":false,"updateable":true,"writeRequiresMasterRead":false},{"aggregatable":true,"aiPredictionField":false,"autoNumber":false,"byteLength":120,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":"Name","controllerName":null,"createable":true,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":false,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":"personname","filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":true,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"First Name","length":40,"mask":null,"maskType":null,"name":"FirstName","nameField":false,"namePointing":false,"nillable":true,"permissionable":false,"picklistValues":[],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"searchPrefilterable":false,"soapType":"xsd:string","sortable":true,"type":"string","unique":false,"updateable":true,"writeRequiresMasterRead":false},{"aggregatable":true,"aiPredictionField":false,"autoNumber":false,"byteLength":120,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":"Name","controllerName":null,"createable":true,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":false,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":"personname","filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":true,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Salutation","length":40,"mask":null,"maskType":null,"name":"Salutation","nameField":false,"namePointing":false,"nillable":true,"permissionable":false,"picklistValues":[{"active":true,"defaultValue":false,"label":"Mr.","validFor":null,"value":"Mr."},{"active":true,"defaultValue":false,"label":"Ms.","validFor":null,"value":"Ms."},{"active":true,"defaultValue":false,"label":"Mrs.","validFor":null,"value":"Mrs."},{"active":true,"defaultValue":false,"label":"Dr.","validFor":null,"value":"Dr."},{"active":true,"defaultValue":false,"label":"Prof.","validFor":null,"value":"Prof."},{"active":true,"defaultValue":false,"label":"Mx.","validFor":null,"value":"Mx."}],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"searchPrefilterable":false,"soapType":"xsd:string","sortable":true,"type":"picklist","unique":false,"updateable":true,"writeRequiresMasterRead":false},{"aggregatable":true,"aiPredictionField":false,"autoNumber":false,"byteLength":363,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":false,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":false,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":"personname","filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":true,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Full Name","length":121,"mask":null,"maskType":null,"name":"Name","nameField":true,"namePointing":false,"nillable":false,"permissionable":false,"picklistValues":[],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"searchPrefilterable":false,"soapType":"xsd:string","sortable":true,"type":"string","unique":false,"updateable":false,"writeRequiresMasterRead":false},{"aggregatable":true,"aiPredictionField":false,"autoNumber":false,"byteLength":384,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":true,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":false,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":true,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Title","length":128,"mask":null,"maskType":null,"name":"Title","nameField":false,"namePointing":false,"nillable":true,"permissionable":true,"picklistValues":[],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"searchPrefilterable":false,"soapType":"xsd:string","sortable":true,"type":"string","unique":false,"updateable":true,"writeRequiresMasterRead":false},{"aggregatable":true,"aiPredictionField":false,"autoNumber":false,"byteLength":765,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":true,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":false,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":true,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Company","length":255,"mask":null,"maskType":null,"name":"Company","nameField":false,"namePointing":false,"nillable":false,"permissionable":false,"picklistValues":[],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"searchPrefilterable":false,"soapType":"xsd:string","sortable":true,"type":"string","unique":false,"updateable":true,"writeRequiresMasterRead":false},{"aggregatable":true,"aiPredictionField":false,"autoNumber":false,"byteLength":765,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":"Address","controllerName":null,"createable":true,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":false,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":"plaintextarea","filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":true,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Street","length":255,"mask":null,"maskType":null,"name":"Street","nameField":false,"namePointing":false,"nillable":true,"permissionable":true,"picklistValues":[],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"searchPrefilterable":false,"soapType":"xsd:string","sortable":true,"type":"textarea","unique":false,"updateable":true,"writeRequiresMasterRead":false},{"aggregatable":true,"aiPredictionField":false,"autoNumber":false,"byteLength":120,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":"Address","controllerName":null,"createable":true,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":false,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":true,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"City","length":40,"mask":null,"maskType":null,"name":"City","nameField":false,"namePointing":false,"nillable":true,"permissionable":true,"picklistValues":[],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"searchPrefilterable":false,"soapType":"xsd:string","sortable":true,"type":"string","unique":false,"updateable":true,"writeRequiresMasterRead":false},{"aggregatable":true,"aiPredictionField":false,"autoNumber":false,"byteLength":240,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":"Address","controllerName":null,"createable":true,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":false,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":true,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"State/Province","length":80,"mask":null,"maskType":null,"name":"State","nameField":false,"namePointing":false,"nillable":true,"permissionable":true,"picklistValues":[],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"searchPrefilterable":false,"soapType":"xsd:string","sortable":true,"type":"string","unique":false,"updateable":true,"writeRequiresMasterRead":false},{"aggregatable":true,"aiPredictionField":false,"autoNumber":false,"byteLength":60,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":"Address","controllerName":null,"createable":true,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":false,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":true,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Zip/Postal Code","length":20,"mask":null,"maskType":null,"name":"PostalCode","nameField":false,"namePointing":false,"nillable":true,"permissionable":true,"picklistValues":[],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"searchPrefilterable":false,"soapType":"xsd:string","sortable":true,"type":"string","unique":false,"updateable":true,"writeRequiresMasterRead":false},{"aggregatable":true,"aiPredictionField":false,"autoNumber":false,"byteLength":240,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":"Address","controllerName":null,"createable":true,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":false,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":true,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Country","length":80,"mask":null,"maskType":null,"name":"Country","nameField":false,"namePointing":false,"nillable":true,"permissionable":true,"picklistValues":[],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"searchPrefilterable":false,"soapType":"xsd:string","sortable":true,"type":"string","unique":false,"updateable":true,"writeRequiresMasterRead":false},{"aggregatable":true,"aiPredictionField":false,"autoNumber":false,"byteLength":0,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":"Address","controllerName":null,"createable":true,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":false,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":false,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Latitude","length":0,"mask":null,"maskType":null,"name":"Latitude","nameField":false,"namePointing":false,"nillable":true,"permissionable":true,"picklistValues":[],"polymorphicForeignKey":false,"precision":18,"queryByDistance":false,"referenceTargetField":null,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":15,"searchPrefilterable":false,"soapType":"xsd:double","sortable":true,"type":"double","unique":false,"updateable":true,"writeRequiresMasterRead":false},{"aggregatable":true,"aiPredictionField":false,"autoNumber":false,"byteLength":0,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":"Address","controllerName":null,"createable":true,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":false,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":false,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Longitude","length":0,"mask":null,"maskType":null,"name":"Longitude","nameField":false,"namePointing":false,"nillable":true,"permissionable":true,"picklistValues":[],"polymorphicForeignKey":false,"precision":18,"queryByDistance":false,"referenceTargetField":null,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":15,"searchPrefilterable":false,"soapType":"xsd:double","sortable":true,"type":"double","unique":false,"updateable":true,"writeRequiresMasterRead":false},{"aggregatable":true,"aiPredictionField":false,"autoNumber":false,"byteLength":120,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":"Address","controllerName":null,"createable":true,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":false,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":true,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Geocode Accuracy","length":40,"mask":null,"maskType":null,"name":"GeocodeAccuracy","nameField":false,"namePointing":false,"nillable":true,"permissionable":true,"picklistValues":[{"active":true,"defaultValue":false,"label":"Address","validFor":null,"value":"Address"},{"active":true,"defaultValue":false,"label":"NearAddress","validFor":null,"value":"NearAddress"},{"active":true,"defaultValue":false,"label":"Block","validFor":null,"value":"Block"},{"active":true,"defaultValue":false,"label":"Street","validFor":null,"value":"Street"},{"active":true,"defaultValue":false,"label":"ExtendedZip","validFor":null,"value":"ExtendedZip"},{"active":true,"defaultValue":false,"label":"Zip","validFor":null,"value":"Zip"},{"active":true,"defaultValue":false,"label":"Neighborhood","validFor":null,"value":"Neighborhood"},{"active":true,"defaultValue":false,"label":"City","validFor":null,"value":"City"},{"active":true,"defaultValue":false,"label":"County","validFor":null,"value":"County"},{"active":true,"defaultValue":false,"label":"State","validFor":null,"value":"State"},{"active":true,"defaultValue":false,"label":"Unknown","validFor":null,"value":"Unknown"}],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":true,"scale":0,"searchPrefilterable":false,"soapType":"xsd:string","sortable":true,"type":"picklist","unique":false,"updateable":true,"writeRequiresMasterRead":false},{"aggregatable":false,"aiPredictionField":false,"autoNumber":false,"byteLength":0,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":false,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":false,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":false,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Address","length":0,"mask":null,"maskType":null,"name":"Address","nameField":false,"namePointing":false,"nillable":true,"permissionable":true,"picklistValues":[],"polymorphicForeignKey":false,"precision":0,"queryByDistance":true,"referenceTargetField":null,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"searchPrefilterable":false,"soapType":"urn:address","sortable":false,"type":"address","unique":false,"updateable":false,"writeRequiresMasterRead":false},{"aggregatable":true,"aiPredictionField":false,"autoNumber":false,"byteLength":120,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":true,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":false,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":true,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Phone","length":40,"mask":null,"maskType":null,"name":"Phone","nameField":false,"namePointing":false,"nillable":true,"permissionable":true,"picklistValues":[],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"searchPrefilterable":false,"soapType":"xsd:string","sortable":true,"type":"phone","unique":false,"updateable":true,"writeRequiresMasterRead":false},{"aggregatable":true,"aiPredictionField":false,"autoNumber":false,"byteLength":120,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":true,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":false,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":true,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Mobile Phone","length":40,"mask":null,"maskType":null,"name":"MobilePhone","nameField":false,"namePointing":false,"nillable":true,"permissionable":true,"picklistValues":[],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"searchPrefilterable":false,"soapType":"xsd:string","sortable":true,"type":"phone","unique":false,"updateable":true,"writeRequiresMasterRead":false},{"aggregatable":true,"aiPredictionField":false,"autoNumber":false,"byteLength":120,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":true,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":false,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":true,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Fax","length":40,"mask":null,"maskType":null,"name":"Fax","nameField":false,"namePointing":false,"nillable":true,"permissionable":true,"picklistValues":[],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"searchPrefilterable":false,"soapType":"xsd:string","sortable":true,"type":"phone","unique":false,"updateable":true,"writeRequiresMasterRead":false},{"aggregatable":true,"aiPredictionField":false,"autoNumber":false,"byteLength":240,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":true,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":false,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":true,"highScaleNumber":false,"htmlFormatted":false,"idLookup":true,"inlineHelpText":null,"label":"Email","length":80,"mask":null,"maskType":null,"name":"Email","nameField":false,"namePointing":false,"nillable":true,"permissionable":true,"picklistValues":[],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"searchPrefilterable":false,"soapType":"xsd:string","sortable":true,"type":"email","unique":false,"updateable":true,"writeRequiresMasterRead":false},{"aggregatable":true,"aiPredictionField":false,"autoNumber":false,"byteLength":765,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":true,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":false,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":true,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Website","length":255,"mask":null,"maskType":null,"name":"Website","nameField":false,"namePointing":false,"nillable":true,"permissionable":true,"picklistValues":[],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"searchPrefilterable":false,"soapType":"xsd:string","sortable":true,"type":"url","unique":false,"updateable":true,"writeRequiresMasterRead":false},{"aggregatable":true,"aiPredictionField":false,"autoNumber":false,"byteLength":765,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":false,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":false,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":"imageurl","filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":true,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Photo URL","length":255,"mask":null,"maskType":null,"name":"PhotoUrl","nameField":false,"namePointing":false,"nillable":true,"permissionable":false,"picklistValues":[],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"searchPrefilterable":false,"soapType":"xsd:string","sortable":true,"type":"url","unique":false,"updateable":false,"writeRequiresMasterRead":false},{"aggregatable":false,"aiPredictionField":false,"autoNumber":false,"byteLength":96000,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":true,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":false,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":"plaintextarea","filterable":false,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":false,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Description","length":32000,"mask":null,"maskType":null,"name":"Description","nameField":false,"namePointing":false,"nillable":true,"permissionable":true,"picklistValues":[],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"searchPrefilterable":false,"soapType":"xsd:string","sortable":false,"type":"textarea","unique":false,"updateable":true,"writeRequiresMasterRead":false},{"aggregatable":true,"aiPredictionField":false,"autoNumber":false,"byteLength":765,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":true,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":false,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":true,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Lead Source","length":255,"mask":null,"maskType":null,"name":"LeadSource","nameField":false,"namePointing":false,"nillable":true,"permissionable":true,"picklistValues":[{"active":true,"defaultValue":false,"label":"Web","validFor":null,"value":"Web"},{"active":true,"defaultValue":false,"label":"Phone Inquiry","validFor":null,"value":"Phone Inquiry"},{"active":true,"defaultValue":false,"label":"Partner Referral","validFor":null,"value":"Partner Referral"},{"active":true,"defaultValue":false,"label":"Purchased List","validFor":null,"value":"Purchased List"},{"active":true,"defaultValue":false,"label":"Other","validFor":null,"value":"Other"}],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"searchPrefilterable":false,"soapType":"xsd:string","sortable":true,"type":"picklist","unique":false,"updateable":true,"writeRequiresMasterRead":false},{"aggregatable":true,"aiPredictionField":false,"autoNumber":false,"byteLength":765,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":true,"custom":false,"defaultValue":"Open - Not Contacted","defaultValueFormula":null,"defaultedOnCreate":true,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":true,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Status","length":255,"mask":null,"maskType":null,"name":"Status","nameField":false,"namePointing":false,"nillable":false,"permissionable":false,"picklistValues":[{"active":true,"defaultValue":true,"label":"Open - Not Contacted","validFor":null,"value":"Open - Not Contacted"},{"active":true,"defaultValue":false,"label":"Working - Contacted","validFor":null,"value":"Working - Contacted"},{"active":true,"defaultValue":false,"label":"Closed - Converted","validFor":null,"value":"Closed - Converted"},{"active":true,"defaultValue":false,"label":"Closed - Not Converted","validFor":null,"value":"Closed - Not Converted"}],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"searchPrefilterable":false,"soapType":"xsd:string","sortable":true,"type":"picklist","unique":false,"updateable":true,"writeRequiresMasterRead":false},{"aggregatable":true,"aiPredictionField":false,"autoNumber":false,"byteLength":765,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":true,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":false,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":true,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Industry","length":255,"mask":null,"maskType":null,"name":"Industry","nameField":false,"namePointing":false,"nillable":true,"permissionable":true,"picklistValues":[{"active":true,"defaultValue":false,"label":"Agriculture","validFor":null,"value":"Agriculture"},{"active":true,"defaultValue":false,"label":"Apparel","validFor":null,"value":"Apparel"},{"active":true,"defaultValue":false,"label":"Banking","validFor":null,"value":"Banking"},{"active":true,"defaultValue":false,"label":"Biotechnology","validFor":null,"value":"Biotechnology"},{"active":true,"defaultValue":false,"label":"Chemicals","validFor":null,"value":"Chemicals"},{"active":true,"defaultValue":false,"label":"Communications","validFor":null,"value":"Communications"},{"active":true,"defaultValue":false,"label":"Construction","validFor":null,"value":"Construction"},{"active":true,"defaultValue":false,"label":"Consulting","validFor":null,"value":"Consulting"},{"active":true,"defaultValue":false,"label":"Education","validFor":null,"value":"Education"},{"active":true,"defaultValue":false,"label":"Electronics","validFor":null,"value":"Electronics"},{"active":true,"defaultValue":false,"label":"Energy","validFor":null,"value":"Energy"},{"active":true,"defaultValue":false,"label":"Engineering","validFor":null,"value":"Engineering"},{"active":true,"defaultValue":false,"label":"Entertainment","validFor":null,"value":"Entertainment"},{"active":true,"defaultValue":false,"label":"Environmental","validFor":null,"value":"Environmental"},{"active":true,"defaultValue":false,"label":"Finance","validFor":null,"value":"Finance"},{"active":true,"defaultValue":false,"label":"Food & Beverage","validFor":null,"value":"Food & Beverage"},{"active":true,"defaultValue":false,"label":"Government","validFor":null,"value":"Government"},{"active":true,"defaultValue":false,"label":"Healthcare","validFor":null,"value":"Healthcare"},{"active":true,"defaultValue":false,"label":"Hospitality","validFor":null,"value":"Hospitality"},{"active":true,"defaultValue":false,"label":"Insurance","validFor":null,"value":"Insurance"},{"active":true,"defaultValue":false,"label":"Machinery","validFor":null,"value":"Machinery"},{"active":true,"defaultValue":false,"label":"Manufacturing","validFor":null,"value":"Manufacturing"},{"active":true,"defaultValue":false,"label":"Media","validFor":null,"value":"Media"},{"active":true,"defaultValue":false,"label":"Not For Profit","validFor":null,"value":"Not For Profit"},{"active":true,"defaultValue":false,"label":"Recreation","validFor":null,"value":"Recreation"},{"active":true,"defaultValue":false,"label":"Retail","validFor":null,"value":"Retail"},{"active":true,"defaultValue":false,"label":"Shipping","validFor":null,"value":"Shipping"},{"active":true,"defaultValue":false,"label":"Technology","validFor":null,"value":"Technology"},{"active":true,"defaultValue":false,"label":"Telecommunications","validFor":null,"value":"Telecommunications"},{"active":true,"defaultValue":false,"label":"Transportation","validFor":null,"value":"Transportation"},{"active":true,"defaultValue":false,"label":"Utilities","validFor":null,"value":"Utilities"},{"active":true,"defaultValue":false,"label":"Other","validFor":null,"value":"Other"}],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"searchPrefilterable":false,"soapType":"xsd:string","sortable":true,"type":"picklist","unique":false,"updateable":true,"writeRequiresMasterRead":false},{"aggregatable":true,"aiPredictionField":false,"autoNumber":false,"byteLength":765,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":true,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":false,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":true,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Rating","length":255,"mask":null,"maskType":null,"name":"Rating","nameField":false,"namePointing":false,"nillable":true,"permissionable":true,"picklistValues":[{"active":true,"defaultValue":false,"label":"Hot","validFor":null,"value":"Hot"},{"active":true,"defaultValue":false,"label":"Warm","validFor":null,"value":"Warm"},{"active":true,"defaultValue":false,"label":"Cold","validFor":null,"value":"Cold"}],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"searchPrefilterable":false,"soapType":"xsd:string","sortable":true,"type":"picklist","unique":false,"updateable":true,"writeRequiresMasterRead":false},{"aggregatable":true,"aiPredictionField":false,"autoNumber":false,"byteLength":0,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":true,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":false,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":false,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Annual Revenue","length":0,"mask":null,"maskType":null,"name":"AnnualRevenue","nameField":false,"namePointing":false,"nillable":true,"permissionable":true,"picklistValues":[],"polymorphicForeignKey":false,"precision":18,"queryByDistance":false,"referenceTargetField":null,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"searchPrefilterable":false,"soapType":"xsd:double","sortable":true,"type":"currency","unique":false,"updateable":true,"writeRequiresMasterRead":false},{"aggregatable":true,"aiPredictionField":false,"autoNumber":false,"byteLength":0,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":true,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":false,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":8,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":true,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Employees","length":0,"mask":null,"maskType":null,"name":"NumberOfEmployees","nameField":false,"namePointing":false,"nillable":true,"permissionable":true,"picklistValues":[],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"searchPrefilterable":false,"soapType":"xsd:int","sortable":true,"type":"int","unique":false,"updateable":true,"writeRequiresMasterRead":false},{"aggregatable":true,"aiPredictionField":false,"autoNumber":false,"byteLength":18,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":true,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":true,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":true,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Owner ID","length":18,"mask":null,"maskType":null,"name":"OwnerId","nameField":false,"namePointing":true,"nillable":false,"permissionable":false,"picklistValues":[],"polymorphicForeignKey":true,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":["Group","User"],"relationshipName":"Owner","relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"searchPrefilterable":false,"soapType":"tns:ID","sortable":true,"type":"reference","unique":false,"updateable":true,"writeRequiresMasterRead":false},{"aggregatable":false,"aiPredictionField":false,"autoNumber":false,"byteLength":0,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":true,"custom":false,"defaultValue":false,"defaultValueFormula":null,"defaultedOnCreate":true,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":true,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Converted","length":0,"mask":null,"maskType":null,"name":"IsConverted","nameField":false,"namePointing":false,"nillable":false,"permissionable":false,"picklistValues":[],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"searchPrefilterable":false,"soapType":"xsd:boolean","sortable":true,"type":"boolean","unique":false,"updateable":false,"writeRequiresMasterRead":false},{"aggregatable":true,"aiPredictionField":false,"autoNumber":false,"byteLength":0,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":false,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":false,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":true,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Converted Date","length":0,"mask":null,"maskType":null,"name":"ConvertedDate","nameField":false,"namePointing":false,"nillable":true,"permissionable":false,"picklistValues":[],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"searchPrefilterable":false,"soapType":"xsd:date","sortable":true,"type":"date","unique":false,"updateable":false,"writeRequiresMasterRead":false},{"aggregatable":true,"aiPredictionField":false,"autoNumber":false,"byteLength":18,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":false,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":false,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":true,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Converted Account ID","length":18,"mask":null,"maskType":null,"name":"ConvertedAccountId","nameField":false,"namePointing":false,"nillable":true,"permissionable":false,"picklistValues":[],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":["Account"],"relationshipName":"ConvertedAccount","relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"searchPrefilterable":true,"soapType":"tns:ID","sortable":true,"type":"reference","unique":false,"updateable":false,"writeRequiresMasterRead":false},{"aggregatable":true,"aiPredictionField":false,"autoNumber":false,"byteLength":18,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":false,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":false,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":true,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Converted Contact ID","length":18,"mask":null,"maskType":null,"name":"ConvertedContactId","nameField":false,"namePointing":false,"nillable":true,"permissionable":false,"picklistValues":[],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":["Contact"],"relationshipName":"ConvertedContact","relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"searchPrefilterable":true,"soapType":"tns:ID","sortable":true,"type":"reference","unique":false,"updateable":false,"writeRequiresMasterRead":false},{"aggregatable":true,"aiPredictionField":false,"autoNumber":false,"byteLength":18,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":false,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":false,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":true,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Converted Opportunity ID","length":18,"mask":null,"maskType":null,"name":"ConvertedOpportunityId","nameField":false,"namePointing":false,"nillable":true,"permissionable":false,"picklistValues":[],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":["Opportunity"],"relationshipName":"ConvertedOpportunity","relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"searchPrefilterable":true,"soapType":"tns:ID","sortable":true,"type":"reference","unique":false,"updateable":false,"writeRequiresMasterRead":false},{"aggregatable":false,"aiPredictionField":false,"autoNumber":false,"byteLength":0,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":true,"custom":false,"defaultValue":false,"defaultValueFormula":null,"defaultedOnCreate":true,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":true,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Unread By Owner","length":0,"mask":null,"maskType":null,"name":"IsUnreadByOwner","nameField":false,"namePointing":false,"nillable":false,"permissionable":false,"picklistValues":[],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"searchPrefilterable":false,"soapType":"xsd:boolean","sortable":true,"type":"boolean","unique":false,"updateable":true,"writeRequiresMasterRead":false},{"aggregatable":true,"aiPredictionField":false,"autoNumber":false,"byteLength":0,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":false,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":true,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":false,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Created Date","length":0,"mask":null,"maskType":null,"name":"CreatedDate","nameField":false,"namePointing":false,"nillable":false,"permissionable":false,"picklistValues":[],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"searchPrefilterable":false,"soapType":"xsd:dateTime","sortable":true,"type":"datetime","unique":false,"updateable":false,"writeRequiresMasterRead":false},{"aggregatable":true,"aiPredictionField":false,"autoNumber":false,"byteLength":18,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":false,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":true,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":true,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Created By ID","length":18,"mask":null,"maskType":null,"name":"CreatedById","nameField":false,"namePointing":false,"nillable":false,"permissionable":false,"picklistValues":[],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":["User"],"relationshipName":"CreatedBy","relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"searchPrefilterable":false,"soapType":"tns:ID","sortable":true,"type":"reference","unique":false,"updateable":false,"writeRequiresMasterRead":false},{"aggregatable":true,"aiPredictionField":false,"autoNumber":false,"byteLength":0,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":false,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":true,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":false,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Last Modified Date","length":0,"mask":null,"maskType":null,"name":"LastModifiedDate","nameField":false,"namePointing":false,"nillable":false,"permissionable":false,"picklistValues":[],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"searchPrefilterable":false,"soapType":"xsd:dateTime","sortable":true,"type":"datetime","unique":false,"updateable":false,"writeRequiresMasterRead":false},{"aggregatable":true,"aiPredictionField":false,"autoNumber":false,"byteLength":18,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":false,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":true,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":true,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Last Modified By ID","length":18,"mask":null,"maskType":null,"name":"LastModifiedById","nameField":false,"namePointing":false,"nillable":false,"permissionable":false,"picklistValues":[],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":["User"],"relationshipName":"LastModifiedBy","relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"searchPrefilterable":false,"soapType":"tns:ID","sortable":true,"type":"reference","unique":false,"updateable":false,"writeRequiresMasterRead":false},{"aggregatable":true,"aiPredictionField":false,"autoNumber":false,"byteLength":0,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":false,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":true,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":false,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"System Modstamp","length":0,"mask":null,"maskType":null,"name":"SystemModstamp","nameField":false,"namePointing":false,"nillable":false,"permissionable":false,"picklistValues":[],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"searchPrefilterable":false,"soapType":"xsd:dateTime","sortable":true,"type":"datetime","unique":false,"updateable":false,"writeRequiresMasterRead":false},{"aggregatable":true,"aiPredictionField":false,"autoNumber":false,"byteLength":0,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":false,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":false,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":true,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Last Activity","length":0,"mask":null,"maskType":null,"name":"LastActivityDate","nameField":false,"namePointing":false,"nillable":true,"permissionable":false,"picklistValues":[],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"searchPrefilterable":false,"soapType":"xsd:date","sortable":true,"type":"date","unique":false,"updateable":false,"writeRequiresMasterRead":false},{"aggregatable":true,"aiPredictionField":false,"autoNumber":false,"byteLength":0,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":false,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":false,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":false,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Last Viewed Date","length":0,"mask":null,"maskType":null,"name":"LastViewedDate","nameField":false,"namePointing":false,"nillable":true,"permissionable":false,"picklistValues":[],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"searchPrefilterable":false,"soapType":"xsd:dateTime","sortable":true,"type":"datetime","unique":false,"updateable":false,"writeRequiresMasterRead":false},{"aggregatable":true,"aiPredictionField":false,"autoNumber":false,"byteLength":0,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":false,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":false,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":false,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Last Referenced Date","length":0,"mask":null,"maskType":null,"name":"LastReferencedDate","nameField":false,"namePointing":false,"nillable":true,"permissionable":false,"picklistValues":[],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"searchPrefilterable":false,"soapType":"xsd:dateTime","sortable":true,"type":"datetime","unique":false,"updateable":false,"writeRequiresMasterRead":false},{"aggregatable":true,"aiPredictionField":false,"autoNumber":false,"byteLength":60,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":true,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":false,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":true,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Data.com Key","length":20,"mask":null,"maskType":null,"name":"Jigsaw","nameField":false,"namePointing":false,"nillable":true,"permissionable":true,"picklistValues":[],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"searchPrefilterable":false,"soapType":"xsd:string","sortable":true,"type":"string","unique":false,"updateable":true,"writeRequiresMasterRead":false},{"aggregatable":true,"aiPredictionField":false,"autoNumber":false,"byteLength":60,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":false,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":false,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":true,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Jigsaw Contact ID","length":20,"mask":null,"maskType":null,"name":"JigsawContactId","nameField":false,"namePointing":false,"nillable":true,"permissionable":false,"picklistValues":[],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":[],"relationshipName":"JigsawContact","relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"searchPrefilterable":false,"soapType":"xsd:string","sortable":true,"type":"string","unique":false,"updateable":false,"writeRequiresMasterRead":false},{"aggregatable":true,"aiPredictionField":false,"autoNumber":false,"byteLength":120,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":true,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":false,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":true,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Clean Status","length":40,"mask":null,"maskType":null,"name":"CleanStatus","nameField":false,"namePointing":false,"nillable":true,"permissionable":true,"picklistValues":[{"active":true,"defaultValue":false,"label":"In Sync","validFor":null,"value":"Matched"},{"active":true,"defaultValue":false,"label":"Different","validFor":null,"value":"Different"},{"active":true,"defaultValue":false,"label":"Reviewed","validFor":null,"value":"Acknowledged"},{"active":true,"defaultValue":false,"label":"Not Found","validFor":null,"value":"NotFound"},{"active":true,"defaultValue":false,"label":"Inactive","validFor":null,"value":"Inactive"},{"active":true,"defaultValue":false,"label":"Not Compared","validFor":null,"value":"Pending"},{"active":true,"defaultValue":false,"label":"Select Match","validFor":null,"value":"SelectMatch"},{"active":true,"defaultValue":false,"label":"Skipped","validFor":null,"value":"Skipped"}],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":true,"scale":0,"searchPrefilterable":false,"soapType":"xsd:string","sortable":true,"type":"picklist","unique":false,"updateable":true,"writeRequiresMasterRead":false},{"aggregatable":true,"aiPredictionField":false,"autoNumber":false,"byteLength":27,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":true,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":false,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":true,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Company D-U-N-S Number","length":9,"mask":null,"maskType":null,"name":"CompanyDunsNumber","nameField":false,"namePointing":false,"nillable":true,"permissionable":true,"picklistValues":[],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"searchPrefilterable":false,"soapType":"xsd:string","sortable":true,"type":"string","unique":false,"updateable":true,"writeRequiresMasterRead":false},{"aggregatable":true,"aiPredictionField":false,"autoNumber":false,"byteLength":18,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":true,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":false,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":true,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"D&B Company ID","length":18,"mask":null,"maskType":null,"name":"DandbCompanyId","nameField":false,"namePointing":false,"nillable":true,"permissionable":true,"picklistValues":[],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":["DandBCompany"],"relationshipName":"DandbCompany","relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"searchPrefilterable":true,"soapType":"tns:ID","sortable":true,"type":"reference","unique":false,"updateable":true,"writeRequiresMasterRead":false},{"aggregatable":true,"aiPredictionField":false,"autoNumber":false,"byteLength":765,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":false,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":false,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":true,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Email Bounced Reason","length":255,"mask":null,"maskType":null,"name":"EmailBouncedReason","nameField":false,"namePointing":false,"nillable":true,"permissionable":false,"picklistValues":[],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"searchPrefilterable":false,"soapType":"xsd:string","sortable":true,"type":"string","unique":false,"updateable":true,"writeRequiresMasterRead":false},{"aggregatable":true,"aiPredictionField":false,"autoNumber":false,"byteLength":0,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":false,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":false,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":false,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Email Bounced Date","length":0,"mask":null,"maskType":null,"name":"EmailBouncedDate","nameField":false,"namePointing":false,"nillable":true,"permissionable":false,"picklistValues":[],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"searchPrefilterable":false,"soapType":"xsd:dateTime","sortable":true,"type":"datetime","unique":false,"updateable":true,"writeRequiresMasterRead":false},{"aggregatable":true,"aiPredictionField":false,"autoNumber":false,"byteLength":18,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"compoundFieldName":null,"controllerName":null,"createable":true,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":false,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"encrypted":false,"externalId":false,"extraTypeInfo":null,"filterable":true,"filteredLookupInfo":null,"formulaTreatNullNumberAsZero":false,"groupable":true,"highScaleNumber":false,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Individual ID","length":18,"mask":null,"maskType":null,"name":"IndividualId","nameField":false,"namePointing":false,"nillable":true,"permissionable":false,"picklistValues":[],"polymorphicForeignKey":false,"precision":0,"queryByDistance":false,"referenceTargetField":null,"referenceTo":["Individual"],"relationshipName":"Individual","relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"searchPrefilterable":true,"soapType":"tns:ID","sortable":true,"type":"reference","unique":false,"updateable":true,"writeRequiresMasterRead":false}],"hasSubtypes":false,"implementedBy":null,"implementsInterfaces":null,"isInterface":false,"isSubtype":false,"keyPrefix":"00Q","label":"Lead","labelPlural":"Leads","layoutable":true,"listviewable":null,"lookupLayoutable":null,"mergeable":true,"mruEnabled":true,"name":"Lead","namedLayoutInfos":[],"networkScopeFieldName":null,"queryable":true,"recordTypeInfos":[{"active":true,"available":true,"defaultRecordTypeMapping":true,"developerName":"Master","master":true,"name":"Master","recordTypeId":"012000000000000AAA","urls":{"layout":"/services/data/v59.0/sobjects/Lead/describe/layouts/012000000000000AAA"}}],"replicateable":true,"retrieveable":true,"searchLayoutable":true,"searchable":true,"sobjectDescribeOption":"FULL","supportedScopes":[{"label":"All leads","name":"everything"},{"label":"My leads","name":"mine"},{"label":"Queue owned leads","name":"queue_owned"},{"label":"Filter by scope","name":"scopingRule"},{"label":"My team's leads","name":"team"},{"label":"User owned leads","name":"user_owned"}],"triggerable":true,"undeletable":true,"updateable":true,"urls":{"compactLayouts":"/services/data/v59.0/sobjects/Lead/describe/compactLayouts","rowTemplate":"/services/data/v59.0/sobjects/Lead/{ID}","approvalLayouts":"/services/data/v59.0/sobjects/Lead/describe/approvalLayouts","uiDetailTemplate":"https://broth-scorpio-4351-dev-ed.scratch.my.salesforce.com/{ID}","uiEditTemplate":"https://broth-scorpio-4351-dev-ed.scratch.my.salesforce.com/{ID}/e","listviews":"/services/data/v59.0/sobjects/Lead/listviews","describe":"/services/data/v59.0/sobjects/Lead/describe","uiNewRecord":"https://broth-scorpio-4351-dev-ed.scratch.my.salesforce.com/00Q/e","quickActions":"/services/data/v59.0/sobjects/Lead/quickActions","layouts":"/services/data/v59.0/sobjects/Lead/describe/layouts","sobject":"/services/data/v59.0/sobjects/Lead"}} From 59718b32b4840882e51ec72155b0b71222346250 Mon Sep 17 00:00:00 2001 From: aditya-balachander Date: Wed, 31 Jan 2024 12:40:08 +0530 Subject: [PATCH 31/36] Update query_transformers.py --- cumulusci/tasks/bulkdata/query_transformers.py | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/cumulusci/tasks/bulkdata/query_transformers.py b/cumulusci/tasks/bulkdata/query_transformers.py index a2a348a406..84c1e66207 100644 --- a/cumulusci/tasks/bulkdata/query_transformers.py +++ b/cumulusci/tasks/bulkdata/query_transformers.py @@ -135,17 +135,6 @@ def outerjoins_to_add(self): rt_dest_table = self.metadata.tables[ self.mapping.get_destination_record_type_table() ] - - # Check if 'is_person_type' column exists in rt_source_table.columns - is_person_type_column = getattr( - rt_source_table.columns, "is_person_type", None - ) - is_person_type_condition = ( - rt_dest_table.columns.is_person_type == is_person_type_column - if is_person_type_column is not None - else True - ) - return [ ( rt_source_table, @@ -158,7 +147,8 @@ def outerjoins_to_add(self): and_( rt_dest_table.columns.developer_name == rt_source_table.columns.developer_name, - is_person_type_condition, + rt_dest_table.columns.is_person_type + == rt_source_table.columns.is_person_type, ), ), ] From 3efb05aa030980d5ac6ec732911bd95af1cb8a90 Mon Sep 17 00:00:00 2001 From: aditya-balachander Date: Thu, 1 Feb 2024 17:40:28 +0530 Subject: [PATCH 32/36] Test for generation of mapping file --- .../test_synthesize_extract_declarations.py | 57 ++++++++++++++++ ...erate_extract_mapping_from_declarations.py | 57 ++++++++++++++++ ...generate_load_mapping_from_declarations.py | 58 +++++++++++++++++ .../test_mapping_generator_post_processes.py | 31 +++++++++ .../bulkdata/tests/test_mapping_parser.py | 65 +++++++++++++++++++ 5 files changed, 268 insertions(+) diff --git a/cumulusci/tasks/bulkdata/extract_dataset_utils/tests/test_synthesize_extract_declarations.py b/cumulusci/tasks/bulkdata/extract_dataset_utils/tests/test_synthesize_extract_declarations.py index 8fb4033cb6..b690ee96e0 100644 --- a/cumulusci/tasks/bulkdata/extract_dataset_utils/tests/test_synthesize_extract_declarations.py +++ b/cumulusci/tasks/bulkdata/extract_dataset_utils/tests/test_synthesize_extract_declarations.py @@ -315,6 +315,63 @@ def test_required_lookups__pulled_in(self, org_config): ) ) + def test_required_lookups__pulled_in__polymorphic_lookups(self, org_config): + """Bringing in the WhoId for sobject Event should force Contact + and Lead to come in. + + Including all Lead/Contact/Event required fields.""" + declarations = """ + extract: + Event: + fields: + WhoId + """ + object_counts = { + "Account": 3, + "Contact": 2, + "Custom__c": 5, + "Lead": 2, + "Event": 1, + } + obj_describes = ( + describe_for("Account"), + describe_for("Contact"), + describe_for("Custom__c"), + describe_for("Event"), + describe_for("Lead"), + ) + declarations = ExtractRulesFile.parse_extract(StringIO(declarations)) + with _fake_get_org_schema( + org_config, + obj_describes, + object_counts, + include_counts=True, + ) as schema: + decls = flatten_declarations(declarations.values(), schema) + + assert tuple(decl.dict() for decl in decls) == tuple( + ( + { + "where": mock.ANY, + "fields_": ["WhoId"], + "api": DataApi.SMART, + "sf_object": "Event", + }, + { + "where": mock.ANY, + "fields_": ["LastName"], + "api": DataApi.SMART, + "sf_object": "Contact", + }, + { + "where": mock.ANY, + "fields_": ["Company", "LastName"], + "api": DataApi.SMART, + "sf_object": "Lead", + }, + ) + ) + def test_parse_real_file(self, cumulusci_test_repo_root, org_config): declarations = ExtractRulesFile.parse_extract( cumulusci_test_repo_root / "datasets/test_minimal.extract.yml" diff --git a/cumulusci/tasks/bulkdata/generate_mapping_utils/tests/test_generate_extract_mapping_from_declarations.py b/cumulusci/tasks/bulkdata/generate_mapping_utils/tests/test_generate_extract_mapping_from_declarations.py index 293fd0ad4b..b1741c5592 100644 --- a/cumulusci/tasks/bulkdata/generate_mapping_utils/tests/test_generate_extract_mapping_from_declarations.py +++ b/cumulusci/tasks/bulkdata/generate_mapping_utils/tests/test_generate_extract_mapping_from_declarations.py @@ -73,5 +73,62 @@ def test_generate_mapping_from_declarations__lookups(self, org_config): }, } + def test_generate_mapping_from_declarations__polymorphic_lookups(self, org_config): + declarations = [ + ExtractDeclaration(sf_object="Account", fields=["Name", "Description"]), + ExtractDeclaration(sf_object="Contact", fields=["LastName", "AccountId"]), + ExtractDeclaration(sf_object="Event", fields=["Subject", "WhoId"]), + ExtractDeclaration(sf_object="Lead", fields=["LastName", "Company"]), + ] + object_counts = {"Account": 10, "Contact": 5, "Case": 5, "Event": 1, "Lead": 2} + obj_describes = ( + describe_for("Account"), + describe_for("Contact"), + describe_for("Case"), + describe_for("Lead"), + describe_for("Event"), + ) + with _fake_get_org_schema( + org_config, + obj_describes, + object_counts, + filters=[], + include_counts=True, + ) as schema: + mf = create_extract_mapping_file_from_declarations(declarations, schema, ()) + print(mf) + assert mf == { + "Extract Account": { + "api": "smart", + "sf_object": "Account", + "soql_filter": "Name != 'Sample Account for Entitlements'", + "fields": ["Name", "Description"], + }, + "Extract Contact": { + "api": "smart", + "sf_object": "Contact", + "fields": ["LastName"], + "lookups": {"AccountId": {"table": ("Account",)}}, + }, + "Extract Lead": { + "api": "smart", + "sf_object": "Lead", + "fields": ["LastName", "Company"], + }, + "Extract Event": { + "api": "smart", + "sf_object": "Event", + "fields": ["Subject"], + "lookups": { + "WhoId": { + "table": ( + "Contact", + "Lead", + ) + } + }, + }, + } + # TODO: Figure out where clauses diff --git a/cumulusci/tasks/bulkdata/generate_mapping_utils/tests/test_generate_load_mapping_from_declarations.py b/cumulusci/tasks/bulkdata/generate_mapping_utils/tests/test_generate_load_mapping_from_declarations.py index 169797ba93..deff57a81f 100644 --- a/cumulusci/tasks/bulkdata/generate_mapping_utils/tests/test_generate_load_mapping_from_declarations.py +++ b/cumulusci/tasks/bulkdata/generate_mapping_utils/tests/test_generate_load_mapping_from_declarations.py @@ -122,6 +122,64 @@ def test_generate_load_mapping_from_declarations__lookups(self, org_config): }, } + def test_generate_load_mapping_from_declarations__polymorphic_lookups( + self, org_config + ): + declarations = [ + ExtractDeclaration(sf_object="Account", fields=["Name", "Description"]), + ExtractDeclaration( + sf_object="Contact", fields=["FirstName", "LastName", "AccountId"] + ), + ExtractDeclaration(sf_object="Lead", fields=["LastName", "Company"]), + ExtractDeclaration(sf_object="Event", fields=["Subject", "WhoId"]), + ] + object_counts = {"Account": 10, "Contact": 5, "Case": 5, "Event": 2, "Lead": 1} + obj_describes = ( + describe_for("Account"), + describe_for("Contact"), + describe_for("Case"), + describe_for("Event"), + describe_for("Lead"), + ) + with _fake_get_org_schema( + org_config, + obj_describes, + object_counts, + filters=[], + include_counts=True, + ) as schema: + mf = create_load_mapping_file_from_extract_declarations( + declarations, schema + ) + assert mf == { + "Insert Account": { + "sf_object": "Account", + "table": "Account", + "fields": ["Name", "Description"], + }, + "Insert Contact": { + "sf_object": "Contact", + "table": "Contact", + "fields": ["FirstName", "LastName"], + "lookups": { + "AccountId": {"table": ["Account"], "key_field": "AccountId"} + }, + }, + "Insert Lead": { + "sf_object": "Lead", + "table": "Lead", + "fields": ["LastName", "Company"], + }, + "Insert Event": { + "sf_object": "Event", + "table": "Event", + "fields": ["Subject"], + "lookups": { + "WhoId": {"table": ["Contact", "Lead"], "key_field": "WhoId"} + }, + }, + } + def test_generate_load_mapping_from_declarations__circular_lookups( self, org_config ): diff --git a/cumulusci/tasks/bulkdata/generate_mapping_utils/tests/test_mapping_generator_post_processes.py b/cumulusci/tasks/bulkdata/generate_mapping_utils/tests/test_mapping_generator_post_processes.py index e01b86ce66..6b33e6a485 100644 --- a/cumulusci/tasks/bulkdata/generate_mapping_utils/tests/test_mapping_generator_post_processes.py +++ b/cumulusci/tasks/bulkdata/generate_mapping_utils/tests/test_mapping_generator_post_processes.py @@ -27,3 +27,34 @@ def test_add_after_statements(self): mappings["Insert Accounts"].lookups["PrimaryContact__c"].after == "Insert Contact" ) + + def test_add_after_statements__polymorphic_lookups(self): + """Test that the add_after_statements function will add an `after` statement to the correct mapping step + for polymorphic lookups""" + mappings = { + "Insert Accounts": MappingStep( + sf_object="Account", + lookups={"PrimaryContact__c": MappingLookup(table="Contact")}, + ), + "Update Account": MappingStep( + sf_object="Account", + action="Update", + ), + "Insert Contact": MappingStep( + sf_object="Contact", + lookups={"AccountId": MappingLookup(table="Account")}, + ), + "Insert Events": MappingStep( + sf_object="Event", + lookups={"WhoId": MappingLookup(table=["Lead", "Contact"])}, + ), + "Insert Lead": MappingStep( + sf_object="Lead", + ), + } + add_after_statements(mappings) + assert ( + mappings["Insert Accounts"].lookups["PrimaryContact__c"].after + == "Insert Contact" + ) + assert mappings["Insert Events"].lookups["WhoId"].after == "Insert Lead" diff --git a/cumulusci/tasks/bulkdata/tests/test_mapping_parser.py b/cumulusci/tasks/bulkdata/tests/test_mapping_parser.py index acfae235b1..bc13f398ae 100644 --- a/cumulusci/tasks/bulkdata/tests/test_mapping_parser.py +++ b/cumulusci/tasks/bulkdata/tests/test_mapping_parser.py @@ -951,6 +951,71 @@ def test_validate_and_inject_mapping_removes_lookups_with_drop_missing(self): assert "Insert Contacts" in mapping assert "AccountId" not in mapping["Insert Contacts"].lookups + @responses.activate + def test_validate_and_inject_mapping_removes_lookups_with_drop_missing__polymorphic_partial_present( + self, + ): + mock_describe_calls() + mapping = parse_from_yaml( + StringIO( + ( + "Insert Accounts:\n sf_object: NotAccount\n table: Account\n fields:\n - Nonsense__c\n" + "Insert Contacts:\n sf_object: Contact\n table: Contact\n lookups:\n AccountId:\n table: Account\n" + "Insert Events:\n sf_object: Event\n table: Event\n lookups:\n WhoId:\n table:\n - Contact\n - Lead" + ) + ) + ) + org_config = DummyOrgConfig( + {"instance_url": "https://example.com", "access_token": "abc123"}, "test" + ) + + validate_and_inject_mapping( + mapping=mapping, + sf=org_config.salesforce_client, + namespace=None, + data_operation=DataOperationType.QUERY, + inject_namespaces=False, + drop_missing=True, + ) + + assert "Insert Accounts" not in mapping + assert "Insert Contacts" in mapping + assert "Insert Events" in mapping + assert "AccountId" not in mapping["Insert Contacts"].lookups + assert "WhoId" in mapping["Insert Events"].lookups + + @responses.activate + def test_validate_and_inject_mapping_removes_lookups_with_drop_missing__polymorphic_none_present( + self, + ): + mock_describe_calls() + mapping = parse_from_yaml( + StringIO( + ( + "Insert Contacts:\n sf_object: NotContact\n table: NotContact\n fields:\n - LastName\n" + "Insert Leads:\n sf_object: NotLead\n table: NotLead\n fields:\n - LastName\n - Company\n" + "Insert Events:\n sf_object: Event\n table: Event\n lookups:\n WhoId:\n table:\n - Contact\n - Lead" + ) + ) + ) + org_config = DummyOrgConfig( + {"instance_url": "https://example.com", "access_token": "abc123"}, "test" + ) + + validate_and_inject_mapping( + mapping=mapping, + sf=org_config.salesforce_client, + namespace=None, + data_operation=DataOperationType.QUERY, + inject_namespaces=False, + drop_missing=True, + ) + + assert "Insert Contacts" not in mapping + assert "Insert Leads" not in mapping + assert "Insert Events" in mapping + assert "WhoId" not in mapping["Insert Events"].lookups + @responses.activate def test_validate_and_inject_mapping_throws_exception_required_lookup_dropped(self): mock_describe_calls() From d4f3e0bdef9d7ffd26758847eddd71897c54a2d0 Mon Sep 17 00:00:00 2001 From: aditya-balachander Date: Thu, 1 Feb 2024 19:19:21 +0530 Subject: [PATCH 33/36] Resolve pyright failure --- .../bulkdata/extract_dataset_utils/calculate_dependencies.py | 2 +- .../tasks/bulkdata/generate_mapping_utils/dependency_map.py | 2 +- .../generate_mapping_from_declarations.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cumulusci/tasks/bulkdata/extract_dataset_utils/calculate_dependencies.py b/cumulusci/tasks/bulkdata/extract_dataset_utils/calculate_dependencies.py index c5c446099b..5771aa524b 100644 --- a/cumulusci/tasks/bulkdata/extract_dataset_utils/calculate_dependencies.py +++ b/cumulusci/tasks/bulkdata/extract_dataset_utils/calculate_dependencies.py @@ -11,7 +11,7 @@ class SObjDependency(T.NamedTuple): table_name_from: str - table_names_to: T.Union[str, T.Tuple] + table_names_to: T.Union[str, T.Tuple[str, ...]] field_name: str priority: bool = False diff --git a/cumulusci/tasks/bulkdata/generate_mapping_utils/dependency_map.py b/cumulusci/tasks/bulkdata/generate_mapping_utils/dependency_map.py index e103bde33a..ebd5f6d2d4 100644 --- a/cumulusci/tasks/bulkdata/generate_mapping_utils/dependency_map.py +++ b/cumulusci/tasks/bulkdata/generate_mapping_utils/dependency_map.py @@ -49,7 +49,7 @@ def _map_references( def target_table_for( self, tablename: str, fieldname: str - ) -> T.Optional[T.Union[str, T.Tuple]]: + ) -> T.Optional[T.Union[str, T.Tuple[str, ...]]]: return self.reference_fields.get((tablename, fieldname)) def get_dependency_order(self): diff --git a/cumulusci/tasks/bulkdata/generate_mapping_utils/generate_mapping_from_declarations.py b/cumulusci/tasks/bulkdata/generate_mapping_utils/generate_mapping_from_declarations.py index 4b3b1784b4..043e9881bb 100644 --- a/cumulusci/tasks/bulkdata/generate_mapping_utils/generate_mapping_from_declarations.py +++ b/cumulusci/tasks/bulkdata/generate_mapping_utils/generate_mapping_from_declarations.py @@ -18,7 +18,7 @@ class SimplifiedExtractDeclarationWithLookups(SimplifiedExtractDeclaration): - lookups: T.Dict[str, T.Union[str, T.Tuple]] + lookups: T.Dict[str, T.Union[str, T.Tuple[str, ...]]] def create_load_mapping_file_from_extract_declarations( From 26a9484c3206d53f0ed06f731b2bc319f60fe33b Mon Sep 17 00:00:00 2001 From: aditya-balachander Date: Fri, 2 Feb 2024 12:35:19 +0530 Subject: [PATCH 34/36] Added comments --- cumulusci/tasks/bulkdata/generate_mapping.py | 3 ++- .../tests/test_generate_extract_mapping_from_declarations.py | 1 + .../tests/test_generate_load_mapping_from_declarations.py | 1 + cumulusci/tasks/bulkdata/load.py | 2 ++ 4 files changed, 6 insertions(+), 1 deletion(-) diff --git a/cumulusci/tasks/bulkdata/generate_mapping.py b/cumulusci/tasks/bulkdata/generate_mapping.py index d5372a9ba1..5aa35a5388 100644 --- a/cumulusci/tasks/bulkdata/generate_mapping.py +++ b/cumulusci/tasks/bulkdata/generate_mapping.py @@ -235,7 +235,8 @@ def strip_namespace(element): for orig_reference in referenceTo ] - # The maximum reference index + # The maximum reference index to set the after to the last + # sobject mentioned in the reference (polymorphic support) max_reference_index = max( stack.index(orig_reference) for orig_reference in referenceTo ) diff --git a/cumulusci/tasks/bulkdata/generate_mapping_utils/tests/test_generate_extract_mapping_from_declarations.py b/cumulusci/tasks/bulkdata/generate_mapping_utils/tests/test_generate_extract_mapping_from_declarations.py index b1741c5592..f51f94db39 100644 --- a/cumulusci/tasks/bulkdata/generate_mapping_utils/tests/test_generate_extract_mapping_from_declarations.py +++ b/cumulusci/tasks/bulkdata/generate_mapping_utils/tests/test_generate_extract_mapping_from_declarations.py @@ -74,6 +74,7 @@ def test_generate_mapping_from_declarations__lookups(self, org_config): } def test_generate_mapping_from_declarations__polymorphic_lookups(self, org_config): + """Generate correct mapping file for sobjects with polymorphic lookup fields""" declarations = [ ExtractDeclaration(sf_object="Account", fields=["Name", "Description"]), ExtractDeclaration(sf_object="Contact", fields=["LastName", "AccountId"]), diff --git a/cumulusci/tasks/bulkdata/generate_mapping_utils/tests/test_generate_load_mapping_from_declarations.py b/cumulusci/tasks/bulkdata/generate_mapping_utils/tests/test_generate_load_mapping_from_declarations.py index deff57a81f..7dbaefc740 100644 --- a/cumulusci/tasks/bulkdata/generate_mapping_utils/tests/test_generate_load_mapping_from_declarations.py +++ b/cumulusci/tasks/bulkdata/generate_mapping_utils/tests/test_generate_load_mapping_from_declarations.py @@ -125,6 +125,7 @@ def test_generate_load_mapping_from_declarations__lookups(self, org_config): def test_generate_load_mapping_from_declarations__polymorphic_lookups( self, org_config ): + """Generate correct mapping file for sobjects with polymorphic lookup fields""" declarations = [ ExtractDeclaration(sf_object="Account", fields=["Name", "Description"]), ExtractDeclaration( diff --git a/cumulusci/tasks/bulkdata/load.py b/cumulusci/tasks/bulkdata/load.py index 201153910b..dcbeae2635 100644 --- a/cumulusci/tasks/bulkdata/load.py +++ b/cumulusci/tasks/bulkdata/load.py @@ -491,8 +491,10 @@ def _process_job_results(self, mapping, step, local_ids): sf_id_results = self._generate_results_id_map(step, local_ids) for i in range(len(sf_id_results)): + # Check for old_format of load sql files if str(sf_id_results[i][0]).isnumeric(): self._old_format = True + # Set id column with new naming format ( - ) sf_id_results[i][0] = mapping.table + "-" + str(sf_id_results[i][0]) else: break From aea33677d64b818e9acd65b7c29c3aa7f24c0063 Mon Sep 17 00:00:00 2001 From: James Estevez Date: Thu, 7 Sep 2023 14:21:58 -0700 Subject: [PATCH 35/36] Add a real email for bot commits Fixes the :cla_missing: errors on bot-authored commits. --- .github/workflows/chores.yml | 4 ++-- .github/workflows/pre-release.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/chores.yml b/.github/workflows/chores.yml index 1221881940..f069df1417 100644 --- a/.github/workflows/chores.yml +++ b/.github/workflows/chores.yml @@ -48,8 +48,8 @@ jobs: ' cumulusci/cumulusci.yml - name: Commit changes run: | - git config user.name github-actions - git config user.email github-actions@github.com + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com git switch -c "update-sfdc-api-v$VERSION" git add cumulusci/cumulusci.yml git commit -m "Automated update to sfdc API version $VERSION" diff --git a/.github/workflows/pre-release.yml b/.github/workflows/pre-release.yml index 13bbdcf51f..931de3a290 100644 --- a/.github/workflows/pre-release.yml +++ b/.github/workflows/pre-release.yml @@ -58,8 +58,8 @@ jobs: npx prettier --write docs/history.md - name: Commit changes run: | - git config user.name github-actions - git config user.email github-actions@github.com + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com git switch -c "release-$(hatch version)" git add docs/history.md cumulusci/__about__.py git commit -m "Update changelog (automated)" From 63e925eed403a36c51eed2b1c7ff5a5e9dd36106 Mon Sep 17 00:00:00 2001 From: James Estevez Date: Wed, 7 Feb 2024 09:11:30 -0800 Subject: [PATCH 36/36] ADR: Migrate from SFDX to SF CLI (#3630) Salesforce has announced the deprecation of SFDX (v7) and is now focusing on the newer SF CLI (v2). They've said that SF CLI (v2) is smart enough to understand all SFDX commands, as well as new SF commands. However, our web applications and other integrations depend on SFDX. SFDX is end-of-life, so we need to cut over and make some improvements by using the new, better features. This ADR proposes that we migrate to the SF CLI. --- .../0003-switch-to-sf-cli-in-cumulusci.md | 67 +++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 docs/adrs/0003-switch-to-sf-cli-in-cumulusci.md diff --git a/docs/adrs/0003-switch-to-sf-cli-in-cumulusci.md b/docs/adrs/0003-switch-to-sf-cli-in-cumulusci.md new file mode 100644 index 0000000000..06b309d7ed --- /dev/null +++ b/docs/adrs/0003-switch-to-sf-cli-in-cumulusci.md @@ -0,0 +1,67 @@ +--- +date: 2023-12-04 +status: Accepted +author: "@jstvz" +--- + +# ADR 3: Switching to SF CLI in CumulusCI + +## Context and Problem Statement + +Salesforce has deprecated SFDX (v7) in favor of the newer SF CLI (v2). Our web applications and other integrations currently depend on SFDX. To leverage new features and ensure support, we need to transition to SF CLI. + +## Decision + +### Considered Options + +1. **Support Both SFDX and SF CLI:** + + - Good: Provides flexibility for users still on SFDX. + - Good: Gives us time to fully understand the new SF CLI. + - Bad: Increases code complexity and could lead to inconsistent behavior. + +2. **Immediately Drop SFDX Support:** + + - Good: Lets us focus on the new SF CLI and stop worrying about old code. + - Good: Speeds up our switch to the new CLI and helps us find and fix problems sooner. + - Good: Gives us the opportunity to improve integration with the SF CLI. + - Bad: It could disrupt users' work if they're still using SFDX. + - Bad: There could be problems with the new SF CLI we don't know about yet. + +3. **Delegate SFDX Command Handling to SF CLI:** + - Good: Offers a smoother transition. + - Bad: Could delay adoption of new SF CLI features. + +### Decision Outcome + +We will make a decisive switch to the SF CLI: + +- **Discontinue SFDX Command Use:** We will replace all SFDX commands in CumulusCI with their SF CLI equivalents. This shift ensures that we are using supported and up-to-date tools. + +- **Update Integration Tests:** Our tests will be updated to reflect this change. We will ensure that all tasks are compatible with the SF CLI. + +- **Update Web Applications and Heroku Buildpack:** Our Falcon web applications and the Heroku buildpack will be updated to depend solely on the SF CLI. We will ensure that all necessary JIT plugins are pre-installed. + +- **Direct Warning to SFDX Users:** A warning will be issued to users who are still on SFDX, informing them of the need to upgrade to the SF CLI. + +## Consequences + +- This approach ensures that we are aligned with the latest Salesforce development tools and standards. +- Immediate transition may require rapid adaptation but ensures future-proofing and access to the latest features. +- Users of SFDX will need to upgrade, which may require some adjustment. +- Our codebase will be simplified by focusing on a single CLI tool. +- Installing JIT plugins in the apps and buildpack will make builds more efficient and avoid problems during runtime. + +## References + +- [List of JIT plugins in SF CLI](https://github.com/salesforcecli/cli/blob/486a157c3d448d699c129f884bb3ab706523002a/package.json#L71-L81) +- [Salesforce CLI sf (v2) announcement blog post](https://developer.salesforce.com/blogs/2023/07/salesforce-cli-sf-v2-is-here) +- [CLI Deprecation Policy](https://developer.salesforce.com/docs/atlas.en-us.sfdx_setup.meta/sfdx_setup/sfdx_dev_cli_deprecation.htm) +- [Issue #3621](https://github.com/SFDO-Tooling/CumulusCI/issues/3621) + +