fix: skip last modified at timestamps that are the same (#1888)

## Test Plan [Describe test plan here] ## Deployment instructions [Add any special deployment instructions here]
supaglue-labs · Nov 10, 2023 · ef162f7 · ef162f7
1 parent 8b7e101
commit ef162f7
Showing 1 changed file with 4 additions and 2 deletions.
diff --git a/packages/core/destination_writers/postgres_impl.ts b/packages/core/destination_writers/postgres_impl.ts
@@ -198,7 +198,8 @@ DO UPDATE SET (${columnsToUpdateStr}) = (${excludedColumnsToUpdateStr})`,
               const dedupeKey = `${applicationId}__${providerName}__${customerId}__${unifiedData.id}`;
               const existingLastModifiedAt = dedupeMap.get(dedupeKey);
 
-              if (existingLastModifiedAt && existingLastModifiedAt > lastModifiedAt) {
+              // Some providers have the same lastModifiedAt for multiple records
+              if (existingLastModifiedAt && existingLastModifiedAt >= lastModifiedAt) {
                 // skip this record, it's older than the existing one
                 return callback(null);
               }
@@ -442,7 +443,8 @@ DO UPDATE SET (${columnsToUpdateStr}) = (${excludedColumnsToUpdateStr})`,
               const dedupeKey = `${applicationId}__${providerName}__${customerId}__${record.id}__${objectName}`;
               const existingLastModifiedAt = dedupeMap.get(dedupeKey);
 
-              if (existingLastModifiedAt && existingLastModifiedAt > lastModifiedAt) {
+              // Some providers have the same lastModifiedAt for multiple records
+              if (existingLastModifiedAt && existingLastModifiedAt >= lastModifiedAt) {
                 // skip this record, it's older than the existing one
                 return callback(null);
               }