From f8495ddd8129501db6c9a27b7a991875996ab89a Mon Sep 17 00:00:00 2001 From: Joseph Long Date: Sat, 18 May 2024 04:44:49 -0400 Subject: [PATCH] python: use file inventory modification time to identify non-ingested telem --- python/magaox/db/ingest.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/python/magaox/db/ingest.py b/python/magaox/db/ingest.py index 211d1b39c..fdea85c5b 100644 --- a/python/magaox/db/ingest.py +++ b/python/magaox/db/ingest.py @@ -82,8 +82,16 @@ def identify_non_ingested_telem(cur: psycopg.Cursor, host: str) -> list[str]: LEFT JOIN file_ingest_times fit ON fi.origin_host = fit.origin_host AND fi.origin_path = fit.origin_path -WHERE fit.origin_host IS NULL AND - fit.origin_path IS NULL AND +WHERE + ( + ( + fit.origin_host IS NULL AND + fit.origin_path IS NULL + ) OR ( + fit.ingested_at < fi.modification_time + ) + ) + AND fi.origin_host = %s AND fi.origin_path LIKE '%%.bintel' ;