Handling of lags (delayed data)

mseed/ooipypull.py

@@ -6,14 +6,13 @@
 import logging.handlers
 import sys
 
-
 LOGLEVEL = logging.DEBUG
+PREFIX = os.environ["TIME_PREFIX"]
+DELAY = os.environ["DELAY_SEGMENT"]
 NODE = os.environ["NODE_NAME"]
 BASEPATH = os.path.join('/tmp', NODE)
 PATH = os.path.join(BASEPATH, 'hls')
 
-ffmpeg_cmd = "ffmpeg -i files.txt -flush_packets 0 -f segment -segment_list '/tmp/$NODE_NAME/hls/$timestamp/live.m3u8' -segment_list_flags -segment_time 10 -segment_format mpegts -ar 64000 -ac 1 -acodec aac '/tmp/$NODE_NAME/hls/$timestamp/live%03d.ts'"
-
 log = logging.getLogger(__name__)
 
 log.setLevel(LOGLEVEL)
@@ -30,6 +29,16 @@ def fetchData(start_time, segment_length, end_time, node):
     os.makedirs(PATH, exist_ok=True)
     while start_time < end_time:
         segment_end = min(start_time + segment_length, end_time)
+        #paths and filenames
+        datestr = start_time.strftime("%Y-%m-%dT%H-%M-%S-%f")[:-3]
+        sub_directory = start_time.strftime("%Y-%m-%d")
+        file_path = os.path.join(PATH, sub_directory)
+        wav_name = "{date}.wav".format(date=datestr)
+        ts_name = "{prefix}{date}.ts".format(prefix=PREFIX, date=datestr)     
+        #fetch if file doesn't already exist
+        if(os.path.exists(os.path.join(file_path, ts_name))):
+            print("EXISTS")
+            continue
         hydrophone_data = ooipy.request.hydrophone_request.get_acoustic_data(
             start_time, segment_end, node, verbose=True, data_gap_mode=2
         )
@@ -38,14 +47,7 @@ def fetchData(start_time, segment_length, end_time, node):
             start_time = segment_end
             continue
         print(f"data: {hydrophone_data}")
-        datestr = start_time.strftime("%Y-%m-%dT%H-%M-%S-%f")[:-3]
-        sub_directory = start_time.strftime("%Y-%m-%d")
-        file_path = os.path.join(PATH, sub_directory)
-        wav_name = f"{datestr}.wav"
         hydrophone_data.wav_write(wav_name)   
-        ts_name = f"{datestr}.ts"
-        #os.system('ffmpeg -i {wavfile} -f mpegts -ar 64000 -acodec aac -ac 1 {tsfile}'.format(wavfile=wav_name, tsfile=ts_name))
-        #os.system("ffmpeg -f concat -safe 0 -i files.txt -flush_packets 0 -f segment -segment_list '/tmp/$NODE/hls/$sub_directory/live.m3u8'")
         os.system("ffmpeg -i {wavfile} -f segment -segment_list './live.m3u8' -strftime 1 -segment_time 10 -segment_format mpegts -ac 1 -acodec aac {tsfile}".format(wavfile=wav_name, tsfile=ts_name))
         if not os.path.exists(file_path):
            os.makedirs(file_path)
@@ -58,13 +60,27 @@ def fetchData(start_time, segment_length, end_time, node):
         f.write(sub_directory)
     shutil.copy('/root/latest.txt', BASEPATH) 
 
+
+
 def _main():
 
-    start_time = datetime.datetime(2021, 4, 27)
-    end_time = datetime.datetime(2022, 4, 30)
     segment_length = datetime.timedelta(seconds = 10)
+    fixed_delay = datetime.timedelta(hours=8)
+
+    while True:
+        end_time = datetime.datetime.utcnow()
+        start_time = end_time - datetime.timedelta(hours=8)
+
+        #near live fetch
+        fetchData(start_time, segment_length, end_time, 'PC01A')
+
+        #delayed fetch
+        fetchData(end_time-datetime.timedelta(hours=24), segment_length, end_time, 'PC01A')
+
+        start_time, end_time = end_time, datetime.datetime.utcnow()
+
+
 
-    fetchData(start_time, segment_length, end_time, 'PC01A')
 
 
 _main()