diff --git a/python-threatexchange/threatexchange/exchanges/impl/ncmec_api.py b/python-threatexchange/threatexchange/exchanges/impl/ncmec_api.py index eedd737f5..ef78853c0 100644 --- a/python-threatexchange/threatexchange/exchanges/impl/ncmec_api.py +++ b/python-threatexchange/threatexchange/exchanges/impl/ncmec_api.py @@ -39,8 +39,8 @@ class NCMECCheckpoint( # The biggest value of "to", and the next "from" get_entries_max_ts: int - next_fetch: str - last_fetch_time: int + next_fetch: t.Optional[str] = "" + last_fetch_time: t.Optional[int] = 0 def get_progress_timestamp(self) -> t.Optional[int]: return self.get_entries_max_ts @@ -245,7 +245,7 @@ def fetch_iter( next_fetch = "" if checkpoint is not None: start_time = checkpoint.get_entries_max_ts - next_fetch = checkpoint.next_fetch + next_fetch = checkpoint.next_fetch or "" # Avoid being exactly at end time for updates showing up multiple # times in the fetch, since entries are not ordered by time end_time = int(time.time()) - 5 @@ -324,9 +324,8 @@ def log(event: str) -> None: ), ) current_next_fetch = entry.next - updates = [] - else: - updates.extend(entry.updates) + break + updates.extend(entry.updates) else: # AKA a successful fetch # If we're hovering near the single-fetch limit for a period # of time, we can likely safely expand our range. diff --git a/python-threatexchange/threatexchange/exchanges/tests/test_state_compatibility.py b/python-threatexchange/threatexchange/exchanges/tests/test_state_compatibility.py index 994d10a51..0a26f2746 100644 --- a/python-threatexchange/threatexchange/exchanges/tests/test_state_compatibility.py +++ b/python-threatexchange/threatexchange/exchanges/tests/test_state_compatibility.py @@ -147,8 +147,23 @@ def get_NCMECCheckpoint() -> t.Tuple[NCMECCheckpoint, t.Sequence[object]]: ## Current max_ts = 1197433091 + current = NCMECCheckpoint( + get_entries_max_ts=max_ts, next_fetch="", last_fetch_time=0 + ) + # 1.0.x - current = NCMECCheckpoint(get_entries_max_ts=max_ts, next="", last_fetch_time=0) + @dataclass + class NCMECCheckpointWithoutNext(FetchCheckpointBase): + """ + 0.99.x => 1.0.0 + + get_entries_max_ts: int => + get_entries_max_ts: int + next_fetch: str + last_fetch_time: int + """ + + get_entries_max_ts: int # 0.99.x @dataclass @@ -161,9 +176,10 @@ class NCMECCheckpointTsMoved(FetchCheckpointBase): max_timestamp: int + checkpoint_without_next = NCMECCheckpointWithoutNext(get_entries_max_ts=max_ts) ts_moved = NCMECCheckpointTsMoved(max_timestamp=max_ts) - return (current, [ts_moved]) + return (current, [checkpoint_without_next, ts_moved]) @pytest.mark.parametrize(