Fix for NCBI downloads to avoid timing out when requesting data too c…

…lose to today's date

workflow_flu_genbank_ingest/scripts/download_metadata.py

@@ -126,6 +126,7 @@
 """
 
 import argparse
+import datetime
 import requests
 
 RETRY_ATTEMPTS = 5
@@ -147,6 +148,12 @@ def main():
     )
     args = parser.parse_args()
 
+    # NCBI doesn't like an end date even close to today
+    # so set the maximum end date to today minus 14 days
+    max_date = (datetime.datetime.now() - datetime.timedelta(days=14)).isoformat()
+    if args.end_time > max_date:
+        args.end_time = max_date
+
     endpoint = "https://www.ncbi.nlm.nih.gov/genomes/VirusVariation/vvsearch2/"
     params = {
         # Search criteria

workflow_flu_genbank_ingest/scripts/download_sequences.py

@@ -34,6 +34,7 @@
 
 
 import argparse
+import datetime
 import requests
 
 RETRY_ATTEMPTS = 5
@@ -55,6 +56,12 @@ def main():
     )
     args = parser.parse_args()
 
+    # NCBI doesn't like an end date even close to today
+    # so set the maximum end date to today minus 14 days
+    max_date = (datetime.datetime.now() - datetime.timedelta(days=14)).isoformat()
+    if args.end_time > max_date:
+        args.end_time = max_date
+
     endpoint = "https://www.ncbi.nlm.nih.gov/genomes/VirusVariation/vvsearch2/"
     params = {
         # Search criteria

workflow_rsv_genbank_ingest/scripts/download_metadata.py

@@ -25,6 +25,7 @@
 
 
 import argparse
+import datetime
 import requests
 
 RETRY_ATTEMPTS = 5
@@ -46,6 +47,12 @@ def main():
     )
     args = parser.parse_args()
 
+    # NCBI doesn't like an end date even close to today
+    # so set the maximum end date to today minus 14 days
+    max_date = (datetime.datetime.now() - datetime.timedelta(days=14)).isoformat()
+    if args.end_time > max_date:
+        args.end_time = max_date
+
     endpoint = "https://www.ncbi.nlm.nih.gov/genomes/VirusVariation/vvsearch2/"
     params = {
         # Search criteria

workflow_rsv_genbank_ingest/scripts/download_sequences.py

@@ -34,6 +34,7 @@
 
 
 import argparse
+import datetime
 import requests
 
 RETRY_ATTEMPTS = 5
@@ -55,6 +56,12 @@ def main():
     )
     args = parser.parse_args()
 
+    # NCBI doesn't like an end date even close to today
+    # so set the maximum end date to today minus 14 days
+    max_date = (datetime.datetime.now() - datetime.timedelta(days=14)).isoformat()
+    if args.end_time > max_date:
+        args.end_time = max_date
+
     endpoint = "https://www.ncbi.nlm.nih.gov/genomes/VirusVariation/vvsearch2/"
     params = {
         # Search criteria

workflow_sars2_genbank_ingest/scripts/download_metadata.py

@@ -24,6 +24,7 @@
 """
 
 import argparse
+import datetime
 import requests
 
 RETRY_ATTEMPTS = 5
@@ -45,6 +46,12 @@ def main():
     )
     args = parser.parse_args()
 
+    # NCBI doesn't like an end date even close to today
+    # so set the maximum end date to today minus 14 days
+    max_date = (datetime.datetime.now() - datetime.timedelta(days=14)).isoformat()
+    if args.end_time > max_date:
+        args.end_time = max_date
+
     endpoint = "https://www.ncbi.nlm.nih.gov/genomes/VirusVariation/vvsearch2/"
     params = {
         # Search criteria

workflow_sars2_genbank_ingest/scripts/download_sequences.py

@@ -34,6 +34,7 @@
 
 
 import argparse
+import datetime
 import requests
 
 RETRY_ATTEMPTS = 5
@@ -55,6 +56,12 @@ def main():
     )
     args = parser.parse_args()
 
+    # NCBI doesn't like an end date even close to today
+    # so set the maximum end date to today minus 14 days
+    max_date = (datetime.datetime.now() - datetime.timedelta(days=14)).isoformat()
+    if args.end_time > max_date:
+        args.end_time = max_date
+
     endpoint = "https://www.ncbi.nlm.nih.gov/genomes/VirusVariation/vvsearch2/"
     params = {
         # Search criteria