Add lessons learned from 2023, made alert colors clear, and added an

alternate algorithm for tracking takeoff/landing

adsb_pusher.py

@@ -2,6 +2,8 @@
 """
 Accept traffic over a port from readsb and push relevant flights to
 an appsheet app.
+
+TODO: write large tests, maybe using debug_stats, combine adsb_alerter.py functionality?
 """
 
 from collections import defaultdict
@@ -48,12 +50,63 @@ def lookup_or_create_aircraft(flight):
 
     return flight.external_id
 
-def bbox_start_change_cb(flight, flight_str):
+def bbox_start_change_cb(flight, flight_str, prev_flight_str):
     dbg("*** bbox_start_change_cb "+flight_str)
-    t = threading.Thread(target=bbox_change_cb, args=[flight, flight_str])
+    t = threading.Thread(target=alt_bbox_change_cb, args=[flight, flight_str, 
+                                                          prev_flight_str])
     t.start()
 
-def bbox_change_cb(flight, flight_str):
+def alt_bbox_change_cb(flight, current_str, old_str):
+    """
+    Different approach: try to catch the transition from ground to air and vice versa.
+    This addresses the problem of aircraft that aren't reliably received, especially when 
+    they're low to the ground.
+    This also allows us to catch "popups" that suddenly appear in flight near the airport.
+
+    Bounding boxes: larger air box, small ground box. ground first in list so takes priority
+
+    Empirically this seems to work pretty well, however I do see a few a/c that
+    have a single ground ping but then are classified as a popup.  manually checked
+    about 20 popups, 18 looked good
+
+    Scenics: check if we saw a local takeoff.  This will also count medevac goarounds tho
+    """
+    SAW_TAKEOFF = 'saw_takeoff'  # tracks scenic flights
+
+    flight_id = flight.tail
+    flight_name = flight.flight_id.strip()
+    if not flight_id:
+        flight_id = flight_name
+
+    op = None
+    note_string = ''
+
+    if "Ground" in old_str and "Air" in current_str:
+        op = 'Takeoff'
+        flight.flags[SAW_TAKEOFF] = True
+
+    if "Ground" in current_str and "Air" in old_str:
+        op = 'Landing'
+        if SAW_TAKEOFF in flight.flags:
+            note_string += ' Scenic'
+
+    if "Air" in current_str and not "Vicinity" in old_str and not 'Ground' in old_str:
+        op = 'Takeoff'
+        note_string += " Popup"
+        flight.flags[SAW_TAKEOFF] = True
+        # XXX more handling for a/c that go silent for a while? > 60s expire?  saw a few
+
+    if op:
+        flighttime = datetime.datetime.fromtimestamp(flight.lastloc.now +  7*60*60)
+        print(f"Got op {op} {flight_name} at {flighttime.strftime('%H:%M %d')}{note_string}")
+        debug_stats[op] += 1
+        aircraft_internal_id = lookup_or_create_aircraft(flight)
+
+        as_instance.add_op(aircraft_internal_id, flight.lastloc.now + TZ_CONVERT*60*60,
+            SAW_TAKEOFF in flight.flags, op, flight_name)
+
+
+def bbox_change_cb(flight, flight_str, old_bboxes):
     """
     Called on all bbox changes, but only log to appsheet when LOGGED_BBOXES are entered.
     Also take note and log it later if NOTED_BBOX is seen.
@@ -64,7 +117,7 @@ def bbox_change_cb(flight, flight_str):
     FINAL_BBOX = 'Landing' # Must be in LOGGED_BBOXES.  When seen clears note about NOTED_BBOX.
 
     local_time = datetime.datetime.fromtimestamp(flight.lastloc.now)
-    log(f"*** bbox_change_cb at {local_time}: {flight_str}")
+    dbg(f"*** bbox_change_cb at {local_time}: {flight_str}")
     debug_stats["bbox_change"] += 1
 
     logged_bbox = next((b for b in LOGGED_BBOXES if b in flight_str), None)
@@ -80,11 +133,11 @@ def bbox_change_cb(flight, flight_str):
     if logged_bbox:
         debug_stats[logged_bbox] += 1
         noted = NOTED_BBOX in flight.flags
-        dbg("adsb_pusher adding " + logged_bbox + " with note " + str(noted))
+        log(f"***** {flight_id},{logged_bbox},{str(noted)}")
 
         aircraft_internal_id = lookup_or_create_aircraft(flight)
 
-        as_instance.add_op(aircraft_internal_id, flight.lastloc.now + TZ_CONVERT*60*60,
+        as_instance.add_op(aircraft_internal_id, flight.lastloc.now + 7*60*60, # XXX TZ
                             noted, logged_bbox, flight_name)
 
     if logged_bbox is FINAL_BBOX:
@@ -164,7 +217,7 @@ def cpe_cb(flight1, flight2, latdist, altdist):
         flight2_internal_id = lookup_or_create_aircraft(cpe.flight2)
 
         cpe.id = as_instance.add_cpe(flight1_internal_id, flight2_internal_id,
-            latdist, altdist, now)
+            latdist, altdist, now, flight1.lastloc.lat, flight1.lastloc.lon)
 
 def gc_loop():
     while True:
@@ -197,6 +250,7 @@ def cpe_gc():
 
 
 def test_cb():
+    print("starting thread")
     t = threading.Thread(target=test_cb_body)
     t.start()
 
@@ -234,7 +288,7 @@ def print_stats():
     for f in args.file:
         bboxes_list.append(Bboxes(f))
 
-    listen = adsb_receiver.setup(args.ipaddr, args.port)
+    listen = adsb_receiver.setup(args.ipaddr, args.port, retry_conn=False, exit_cb=print_stats)
 
     adsb_receiver.flight_read_loop(listen, bboxes_list, None, None,
         cpe_start_cb, bbox_start_change_cb, test_cb=test_cb)

adsb_receiver.py

@@ -4,6 +4,7 @@
 import signal
 import datetime
 import sys
+import time
 from typing import Dict
 
 from test import test_insert, tests_enable, run_test
@@ -15,7 +16,7 @@ class Flights:
     """all Flight objects in the system, indexed by flight_id"""
     flight_dict: Dict[str, Flight] = {}
     lock: threading.Lock = threading.Lock()
-    EXPIRE_SECS: int = 60
+    EXPIRE_SECS: int = 180  # 3 minutes emperically needed to debounce poor-signal airplanes
 
     def __init__(self, bboxes):
         self.bboxes = bboxes
@@ -104,10 +105,12 @@ def check_distance(self, annotate_cb, last_read_time):
 
 
 class TCPConnection:
-    def __init__(self, host, port):
+    def __init__(self, host, port, retry, exit_cb):
         self.host = host
         self.port = port
         self.sock = None
+        self.retry = retry
+        self.exit_cb = exit_cb
         self.f = None
 
     def connect(self):
@@ -116,10 +119,8 @@ def connect(self):
             self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
             self.sock.connect((self.host, self.port))
             print('Successful Connection')
-        except Exception:
-            print('Connection Failed')
-            sys.exit(1)
-            raise
+        except Exception as e:
+            print('Connection Failed: '+str(e))
 
         self.f = self.sock.makefile()
 
@@ -129,11 +130,11 @@ def readline(self):
 def sigint_handler(signum, frame):
     sys.exit(1)
 
-def setup(ipaddr, port):
+def setup(ipaddr, port, retry_conn=True, exit_cb=None):
     print("Connecting to %s:%d" % (ipaddr, int(port)))
 
     signal.signal(signal.SIGINT, sigint_handler)
-    conn = TCPConnection(ipaddr, int(port))
+    conn = TCPConnection(ipaddr, int(port), retry_conn, exit_cb)
     conn.connect()
 
     dbg("Setup done")
@@ -144,16 +145,24 @@ def flight_update_read(flights, listen, update_cb, bbox_change_cb):
         line = listen.readline()
         jsondict = json.loads(line)
     except Exception:
-        print("Socket input/parse error, attempting to reconnect...")
-        listen.connect()
+        print(f"Socket input/parse error, reconnect plan = {listen.retry}")
+        if listen.retry:
+            time.sleep(2)
+            listen.connect()
+        else:
+            if listen.exit_cb:
+                listen.exit_cb()
+            sys.exit(1)
         return
     #ppdbg(jsondict)
 
     loc_update = Location.from_dict(jsondict)
     last_ts = flights.add_location(loc_update, update_cb, update_cb, bbox_change_cb)
     return last_ts
 
-def flight_read_loop(listen, bbox_list, update_cb, expire_cb, annotate_cb, bbox_change_cb, test_cb=None):
+def flight_read_loop(listen, bbox_list, update_cb, expire_cb, annotate_cb, bbox_change_cb, 
+                     test_cb=None):
+
     CHECKPOINT_INTERVAL = 10 # seconds
     last_checkpoint = 0
     TEST_INTERVAL = 60*60 # run test every this many seconds

appsheet_api.py

@@ -20,9 +20,10 @@
 }
 
 DELAYTEST = False            # for threading testing
-SEND_AIRCRAFT = True
-SEND_OPS = True
-SEND_CPES = True
+SEND_AIRCRAFT = False
+LOOKUP_AIRCRAFT = False
+SEND_OPS = False
+SEND_CPES = False
 FAKE_KEY = "XXXfake keyXXX"  # for testing purposes
 
 class Appsheet:
@@ -33,14 +34,14 @@ def __init__(self):
 
     def aircraft_lookup(self, tail, wholeobj=False):
         """return appsheet internal ID for this tail number """
-        log("aircraft_lookup %s" % (tail))
+        dbg("aircraft_lookup %s" % (tail))
 
         body = copy.deepcopy(BODY)
         body["Action"] = "Find"
         body["Properties"]["Selector"] = "Select(Aircraft[Row ID], [Regno] = \"%s\")" % tail
-        ppd(body)
+        # ppd(body)
         try:
-            if SEND_AIRCRAFT:
+            if LOOKUP_AIRCRAFT:
                 ret = self.sendop(self.config.private_vars["appsheet"]["aircraft_url"], body)
                 if ret:
                     dbg("lookup for tail " + tail + " lookup returning "+ ret[0]["Row ID"])
@@ -58,7 +59,7 @@ def aircraft_lookup(self, tail, wholeobj=False):
 
     def add_aircraft(self, regno, test=False, description=""):
         """Create aircraft in appsheet"""
-        log("add_aircraft %s" % (regno))
+        dbg("add_aircraft %s" % (regno))
 
         body = copy.deepcopy(BODY)
         body["Action"] = "Add"
@@ -81,7 +82,7 @@ def add_aircraft(self, regno, test=False, description=""):
         return None
 
     def get_all_entries(self, table):
-        log("get_all_entries " + table)
+        dbg("get_all_entries " + table)
 
         body = copy.deepcopy(BODY)
         body["Action"] = "Find"
@@ -135,7 +136,7 @@ def add_aircraft_from_file(self, fn):
                     self.add_aircraft(line)
 
     def add_op(self, aircraft, time, scenic, optype, flight_name):
-        log("add_op %s %s" % (aircraft, optype))
+        dbg("add_op %s %s" % (aircraft, optype))
         optime = datetime.datetime.fromtimestamp(time)
 
         body = copy.deepcopy(BODY)
@@ -159,7 +160,8 @@ def add_op(self, aircraft, time, scenic, optype, flight_name):
 
         return None
 
-    def add_cpe(self, flight1, flight2, latdist, altdist, time):
+    def add_cpe(self, flight1, flight2, latdist, altdist, time, lat, long):
+        # XXX needs test w/ lat /long addition
         log("add_cpe %s %s" % (flight1, flight2))
         optime = datetime.datetime.fromtimestamp(time)
 
@@ -170,7 +172,9 @@ def add_cpe(self, flight1, flight2, latdist, altdist, time):
             "Aircraft2": flight2,
             "Time": optime.strftime("%m/%d/%Y %H:%M:%S"),
             "Min alt sep": altdist,
-            "Min lat sep": latdist*6076
+            "Min lat sep": latdist*6076,
+            "lat": lat,
+            "long": long
         }]
         #ppd(body)
         try:

bboxes.py

@@ -11,6 +11,7 @@
 
 @dataclass
 class Bbox:
+    """A single bounding box defined by a polygon, altitude range, and heading range."""
     polygon: Polygon
     minalt: int
     maxalt: int
@@ -19,6 +20,13 @@ class Bbox:
     name: str
 
 class Bboxes:
+    """
+    A collection of Bbox objects, defined by a KML file with polygons inside.
+    Each polygon should have a name formatted like this in the KML: 
+        name: minalt-maxalt minhdg-maxhdg
+    For example:
+        RHV apporach: 500-1500 280-320
+    """
     def __init__(self, fn):
         self.boxes = []    # list of Bbox objects
 
@@ -68,6 +76,6 @@ def contains(self, lat, long, hdg, alt):
         for i, box in enumerate(self.boxes):
             if (box.polygon.contains(Point(long,lat)) and
                 self.hdg_contains(hdg, box.starthdg, box.endhdg)):
-                if (alt > box.minalt and alt < box.maxalt):
+                if (alt >= box.minalt and alt <= box.maxalt):
                     return i
         return -1

controller.py

@@ -138,7 +138,6 @@ def do_server_update(self, flight):
                     try:
                         if int(arr) > 2:
                             self.note_string += "* >2 arrivals "
-                            self.bg_color_warn = True
                     except Exception:
                         pass
 

flight.py

@@ -82,6 +82,9 @@ def __post_init__(self):
         self.inside_bboxes = [-1] * len(self.bboxes_list)
 
     def to_str(self):
+        """
+        String representation includes lat/long and bbox list
+        """
         string = self.lastloc.to_str()
         bbox_name_list = []
         for i, bboxes in enumerate(self.bboxes_list):
@@ -130,7 +133,11 @@ def update_loc(self, loc):
         self.lastloc = loc
 
     def update_inside_bboxes(self, bbox_list, loc, change_cb):
+        """
+        Array indices in here are all per kml file.
+        """
         changes = False
+        old_str = self.to_str()
         for i, bbox in enumerate(bbox_list):
             new_bbox = bbox_list[i].contains(loc.lat, loc.lon, loc.track, loc.alt_baro)
             if self.inside_bboxes[i] != new_bbox:
@@ -143,7 +150,7 @@ def update_inside_bboxes(self, bbox_list, loc, change_cb):
             tail = self.tail if self.tail else "(unk)"
             log(tail + " Flight bbox change at " + flighttime.strftime("%H:%M") +
                 ": " + self.to_str())
-            if change_cb: change_cb(self, self.to_str())
+            if change_cb: change_cb(self, self.to_str(), old_str)
 
     def get_bbox_at_level(self, level, bboxes_list):
         inside_n = self.inside_bboxes[level]