Skip to content

Commit

Permalink
review changes
Browse files Browse the repository at this point in the history
  • Loading branch information
@JYOTHINARAYANSETTY
JYOTHINARAYANSETTY committed Nov 9, 2023
1 parent 33b5f96 commit eeed6f3
Show file tree
Hide file tree
Showing 2 changed files with 30 additions and 11 deletions.
21 changes: 15 additions & 6 deletions tests/test_salesforce_switch_rep_method_ft_incrmntl.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ def name():

def expected_sync_streams(self):
streams = self.switchable_streams() - {'FlowDefinitionView','EntityDefinition'}
# Excluded the above two streams due to the bug TDL-24514
return self.partition_streams(streams)


Expand Down Expand Up @@ -62,11 +63,19 @@ def test_run(self):
self.assertGreater(incrmntl_sync_count, 0,
msg="We are not fully testing bookmarking for {}".format(stream))
# data from record messages
"""
If implementing in tap-tester framework the primary key implementation should account
for compound primary keys
"""
self.assertEqual(1, len(list(primary_keys[stream])),
msg="Compound primary keys require a change to test expectations")
primary_key = list(primary_keys[stream])[0]
fulltbl_sync_messages = [record['data'] for record in
fulltbl_sync_records.get(stream).get('messages')
if record.get('action') == 'upsert']
fulltbl_primary_keys = {message[primary_key] for message in fulltbl_sync_messages}
filtered_fulltbl_sync_messages = [message for message in fulltbl_sync_messages
if message[replication_key] >= self.start_date]
fulltbl_primary_keys = {message[primary_key] for message in filtered_fulltbl_sync_messages}
incrmntl_sync_messages = [record['data'] for record in
incrmntl_sync_records.get(stream).get('messages')
if record.get('action') == 'upsert']
Expand All @@ -75,22 +84,22 @@ def test_run(self):
#Verify all records are synced in the second sync
self.assertTrue(fulltbl_primary_keys.issubset(incrmntl_primary_keys))

#verify that the last message is not a activateversion message for incremental sync
#verify that the last message is not a activate version message for incremental sync
self.assertNotEqual('activate_version', incrmntl_sync_records[stream]['messages'][-1]['action'])

#verify that the table version incremented after every sync
self.assertGreater(
incrmntl_sync_records[stream]['table_version'], fulltbl_sync_records[stream]['table_version'])
self.assertGreater(incrmntl_sync_records[stream]['table_version'],
fulltbl_sync_records[stream]['table_version'],
msg = "Table version is not incremented after a successful sync")

# bookmarked states (top level objects)
incrmntl_bookmark_key_value = incrmntl_sync_bookmarks.get('bookmarks').get(stream)
incrmntl_bookmark_key_value = incrmntl_sync_bookmarks.get('bookmarks', {}).get(stream)

# bookmarked states (actual values)
incrmntl_bookmark_value = incrmntl_bookmark_key_value.get(replication_key)

# Verify the incremental sync sets a bookmark of the expected form
self.assertIsNotNone(incrmntl_bookmark_key_value)
self.assertIsNotNone(incrmntl_bookmark_key_value.get(replication_key))

#verify that bookmarks are present after switching to Incremental rep method
self.assertIsNotNone(incrmntl_bookmark_value)
20 changes: 15 additions & 5 deletions tests/test_salesforce_switch_rep_method_incrmntl_ft.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,10 +11,11 @@ def name():

def expected_sync_streams(self):
streams = self.switchable_streams() - {'FlowDefinitionView', 'EntityDefinition'}
# Excluded the above two streams due to the bug TDL-24514
return self.partition_streams(streams)

def test_run(self):
self.salesforce_api = 'BULK'
self.salesforce_api = 'REST'
replication_keys = self.expected_replication_keys()
primary_keys = self.expected_primary_keys()
# SYNC 1
Expand Down Expand Up @@ -59,6 +60,13 @@ def test_run(self):
msg="We are not fully testing bookmarking for {}".format(stream))

# data from record messages
"""
If implementing in tap-tester framework the primary key implementation should account
for compound primary keys
"""
self.assertEqual(1, len(list(primary_keys[stream])),
msg="Compound primary keys require a change to test expectations")

primary_key = list(primary_keys[stream])[0]
incrmntl_sync_messages = [record['data'] for record in
incrmntl_sync_records.get(stream).get('messages')
Expand All @@ -67,7 +75,9 @@ def test_run(self):
fulltbl_sync_messages = [record['data'] for record in
fulltbl_sync_records.get(stream).get('messages')
if record.get('action') == 'upsert']
fulltbl_primary_keys = {message[primary_key] for message in fulltbl_sync_messages}
filtered_fulltbl_sync_messages = [message for message in fulltbl_sync_messages
if message[replication_key] >= self.start_date]
fulltbl_primary_keys = {message[primary_key] for message in filtered_fulltbl_sync_messages}

#Verify all records are synced in the second sync
self.assertTrue(incrmntl_primary_keys.issubset(fulltbl_primary_keys))
Expand All @@ -84,11 +94,11 @@ def test_run(self):
self.assertIsNotNone(fulltbl_sync_records[stream]['table_version'])

#Verify that the table version is incremented after every sync
self.assertGreater(
fulltbl_sync_records[stream]['table_version'],incrmntl_sync_records[stream]['table_version'])
self.assertGreater(fulltbl_sync_records[stream]['table_version'],
incrmntl_sync_records[stream]['table_version'])

# bookmarked states (top level objects)
fulltbl_bookmark_key_value = fulltbl_sync_bookmarks.get('bookmarks').get(stream)
fulltbl_bookmark_key_value = fulltbl_sync_bookmarks.get('bookmarks', {}).get(stream)

# bookmarked states (actual values)
fulltbl_bookmark_value = fulltbl_bookmark_key_value.get(replication_key)
Expand Down

0 comments on commit eeed6f3

Please sign in to comment.