Merge pull request #106 from SNEWS2/sybenzvi/csv-to-sql

Sybenzvi/csv to sql

pyproject.toml

@@ -1,58 +1,60 @@
-[tool.poetry]
-name = "snews-cs"
-version = "2.2.0"
-description = "An alert application for observing supernovas."
-authors = ["Sebastian Torres-Lara <[email protected]>", "Melih Kara <[email protected]>"]
-license = "BSD 3-Clause"
-readme = "README.md"
-packages = [{include = "snews_cs"}]
-homepage = "https://github.com/SNEWS2/SNEWS_Coincidence_System.git"
-repository = "https://github.com/SNEWS2/SNEWS_Coincidence_System.git"
-documentation = "https://snews-coincidence-system.readthedocs.io/"
-classifiers = [
-    "Development Status :: 2 - Pre-Alpha",
-    "Programming Language :: Python",
-    "Programming Language :: Python :: 3",
-    "Intended Audience :: Science/Research",
-    "Intended Audience :: Developers",
-    "Natural Language :: English",
-    "Topic :: Scientific/Engineering",
-    "Topic :: Scientific/Engineering :: Astronomy",
-    "Topic :: Scientific/Engineering :: Physics",
-    "Operating System :: POSIX",
-    "Operating System :: Unix",
-    "Operating System :: MacOS",
-    "License :: OSI Approved :: BSD License",
-]
-
-[tool.poetry.scripts]
-snews_cs = "snews_cs.__main__:main"
-
-[tool.poetry.dependencies]
-python = ">=3.9, <3.13"
-click = "^8.1.7"
-hop-client = "^0.8.0"
-ipython = "<7.33.0"  # Constrained by snews_pt
-numpy = "^1.26.0"
-pandas = "^2.1.1"
-python-dotenv = "0.19.2"  # Constrained by snews_pt
-pymongo = "^4.5.0"
-slack-sdk = "^3.22.0"
-snews-pt = "^1.3.3"
-matplotlib = "^3.8.0"
-scipy = "^1.11.3"
-
-[tool.poetry.group.dev.dependencies]
-autopep8 = "^2.0.4"
-flake8 = "^6.1.0"
-mongomock = "^4.1.2"
-pytest = "<6.3.0"  # Constrained by snews_pt
-
-[tool.poetry.group.docs.dependencies]
-sphinx = "<5"  # Constrained by snews_pt
-sphinx-pdj-theme = "^0.4.0"
-
-[build-system]
-requires = ["poetry-core"]
-build-backend = "poetry.core.masonry.api"
-
+[tool.poetry]
+name = "snews-cs"
+version = "2.2.0"
+description = "An alert application for observing supernovas."
+authors = ["Sebastian Torres-Lara <[email protected]>", "Melih Kara <[email protected]>"]
+license = "BSD 3-Clause"
+readme = "README.md"
+packages = [{include = "snews_cs"}]
+homepage = "https://github.com/SNEWS2/SNEWS_Coincidence_System.git"
+repository = "https://github.com/SNEWS2/SNEWS_Coincidence_System.git"
+documentation = "https://snews-coincidence-system.readthedocs.io/"
+classifiers = [
+    "Development Status :: 2 - Pre-Alpha",
+    "Programming Language :: Python",
+    "Programming Language :: Python :: 3",
+    "Intended Audience :: Science/Research",
+    "Intended Audience :: Developers",
+    "Natural Language :: English",
+    "Topic :: Scientific/Engineering",
+    "Topic :: Scientific/Engineering :: Astronomy",
+    "Topic :: Scientific/Engineering :: Physics",
+    "Operating System :: POSIX",
+    "Operating System :: Unix",
+    "Operating System :: MacOS",
+    "License :: OSI Approved :: BSD License",
+]
+
+[tool.poetry.scripts]
+snews_cs = "snews_cs.__main__:main"
+
+[tool.poetry.dependencies]
+python = ">=3.9, <3.13"
+click = "^8.1.7"
+hop-client = "^0.8.0"
+ipython = "<7.33.0"  # Constrained by snews_pt
+numpy = "^1.26.0"
+pandas = "^2.1.1"
+python-dotenv = "0.19.2"  # Constrained by snews_pt
+pymongo = "^4.5.0"
+slack-sdk = "^3.22.0"
+snews-pt = "^1.3.3"
+matplotlib = "^3.8.0"
+scipy = "^1.11.3"
+sqlalchemy = "^2.0.32"
+tabulate = "^0.9.0"
+
+[tool.poetry.group.dev.dependencies]
+autopep8 = "^2.0.4"
+flake8 = "^6.1.0"
+mongomock = "^4.1.2"
+pytest = "<6.3.0"  # Constrained by snews_pt
+
+[tool.poetry.group.docs.dependencies]
+sphinx = "<5"  # Constrained by snews_pt
+sphinx-pdj-theme = "^0.4.0"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"
+

snews_cs/_version.py

@@ -1 +1 @@
-version = '2.2.0'
+version = '2.2.1'

snews_cs/cs_stats.py

@@ -3,6 +3,10 @@
 
 def ncr(n, r):
     f = math.factorial
+    # if the HB cache is empty, n-r returns negative number for which factorial is not defined
+    # as a temporary fix, let's return 0
+    if n-r < 0:
+        return 0
     return int(f(n) / f(r) / f(n - r))
 
 def cache_false_alarm_rate(cache_sub_list, hb_cache):

snews_cs/heartbeat_feedbacks.py

@@ -7,7 +7,8 @@
 from time import sleep
 from .core.logging import getLogger
 from .cs_email import send_warning_mail, send_feedback_mail
-from .snews_hb import beats_path, mirror_csv
+from .snews_hb import beats_path
+from sqlalchemy import create_engine
 
 log = getLogger(__name__)
 
@@ -24,6 +25,13 @@
 # verbose print. Prints only if verbose=True
 vprint = lambda inp, _bool: print(inp) if _bool else None
 
+# SQLite database for the cached heartbeats
+cache_db = os.path.abspath(os.path.join(beats_path, 'cached_heartbeats_mirror.db'))
+cache_engine = create_engine(f'sqlite:///{cache_db}')
+cache_df = None
+
+
+
 class FeedBack:
     """ Once every minute, check the HB of each detector
         If the last heartbeat is from longer than usual, send an email
@@ -37,6 +45,7 @@ def __init__(self, verbose=False):
             self.last_feedback_time[k] = np.datetime64("2022-01-01")
         self.day_in_min = 1440
         self.running_min = 0
+        self.db_found = False
         self.verbose = verbose
         log.info(f"\t> Heartbeat tracking initiated.")
 
@@ -48,15 +57,9 @@ def __call__(self):
         while True:
             # run every minute
             sleep(60)
-            try:
-                df = pd.read_csv(mirror_csv, parse_dates=['Received Times'], )
-            except FileNotFoundError:
-                log.error(f"{mirror_csv} does not exist yet! Maybe `snews_cs run-coincidence` is not invoked?")
-                while not os.path.isfile(mirror_csv):
-                    sleep(60)
-                df = pd.read_csv(mirror_csv, parse_dates=['Received Times'], )
-                log.debug(f"OK {mirror_csv} found! Moving on")
-
+            # The database is continuosly updated, read it every minute
+            # it will wait until it finds a database
+            df = self.read_database() # Received times are already parsed
             self.control(df) # check if a detector is taking longer than usual (mean+3*sigma>)
             self.running_min += 1
             vprint(f"[DEBUG] >>>>> Running minute: {self.running_min}", self.verbose)
@@ -66,6 +69,28 @@ def __call__(self):
                 self.running_min = 0  # reset the counter
                 delete_old_figures()
 
+    def read_database(self):
+        """ Try to read the database, if it is empty (or does not exist) wait
+        """
+        try:
+            # - Try reading cached data from SQL DB
+            cache_df = pd.read_sql_table('cached_heartbeats', cache_engine,
+                                         parse_dates=['Received Times', 'Stamped Times'])
+            if len(cache_df) < 1:
+                raise FileNotFoundError
+            log.debug(f"OK DataBase found! Moving on")
+        except FileNotFoundError:
+            log.error(f"DataBase does not exist yet! Maybe `snews_cs run-coincidence` is not invoked?\n"
+                      f"Sleeping until I find something")
+            sleep(60)
+            self.running_min += 1
+            return self.read_database()
+        # if it passes, that means it found a non-empty database
+        if not self.db_found:
+            log.debug(f"OK DataBase found! Moving on")
+            # log it once, then set db found true
+        self.db_found=True
+        return cache_df
 
     def control(self, df):
         """ Check the current cache, check if any detector
@@ -163,7 +188,8 @@ def plot_beats(df, detector, figname):
     from mpl_toolkits.axes_grid1 import make_axes_locatable
     from matplotlib.colors import LinearSegmentedColormap, Normalize
 
-    latency = pd.to_timedelta(df['Latency'].values).total_seconds()
+    # latency = pd.to_timedelta(df['Latency'].values).total_seconds()
+    latency = df['Latency'].values
     received_times = df['Received Times']  # should be numpy datetime object
     try:
         unique_days_np = np.unique(received_times.astype('datetime64[D]'))