Now supporting writing data to Postgres DB, created requirements.txt …

…file, filled in README.md, updated .gitignore

.gitignore

@@ -1,2 +1,3 @@
 .DS_Store
+__pycache__
 *.csv

README.md

@@ -1,4 +1,120 @@
-Under development. README to be populated later.
+This library makes use of [pynmea2](https://github.com/Knio/pynmea2) to parse through input NMEA data, organize it, and output it to CSV files or to a PostgreSQL database.
 
-Reference materials:
-https://www.trimble.com/OEM_ReceiverHelp/V4.44/en/NMEA-0183messages_MessageOverview.html
+## Setup
+
+Your input file should have a format similiar to those under `test_data`. To have your data datetime stamped, it must be in a format like that of `test_data/test_data_all.nmea`, with RMC sentences containing date and time stamps proceed other sentences in the same cycle.
+
+If working with a database, the database access information/credentials must be setup in `db_creds.py`.
+
+
+## Usage
+```
+$ cd ~/Downloads/nmea_parser/
+$ pip install -r requirements.txt
+...
+$ python nmea_parser.py --help
+usage: nmea_parser.py [-h] [--drop_previous_db_tables] filepath {csv,db,both}
+
+positional arguments:
+  filepath              file system path to file containing NMEA data
+  {csv,db,both}         where to output data: CSV files, database, or both
+
+optional arguments:
+  -h, --help            show this help message and exit
+  --drop_previous_db_tables
+                        drop previous DB tables before importing new data;
+                        only applies when output_method is 'db' or 'both'
+```
+## Examples
+### Example 1
+```
+$ ls -l *.csv
+ls: *.csv: No such file or directory
+$ python nmea_parser.py test_data/test_data_all.nmea csv
+
+Reading in data... done.
+
+Processing data... done.
+
+Writing data to CSVs... data from logfile 'test_data/test_data_all.nmea' written to:
+  test_data_all_GNRMC.csv
+  test_data_all_GNVTG.csv
+  test_data_all_GNGGA.csv
+  test_data_all_GNGSA.csv
+  test_data_all_GPGSV.csv
+  test_data_all_GLGSV.csv
+  test_data_all_GNGLL.csv
+done.
+
+All done. Exiting.
+
+
+$ ls -l *.csv
+-rw-r--r--  1 Thomas  staff  14310 Dec 30 18:19 test_data_all_GLGSV.csv
+-rw-r--r--  1 Thomas  staff   9502 Dec 30 18:19 test_data_all_GNGGA.csv
+-rw-r--r--  1 Thomas  staff   6852 Dec 30 18:19 test_data_all_GNGLL.csv
+-rw-r--r--  1 Thomas  staff  18472 Dec 30 18:19 test_data_all_GNGSA.csv
+-rw-r--r--  1 Thomas  staff   8672 Dec 30 18:19 test_data_all_GNRMC.csv
+-rw-r--r--  1 Thomas  staff   5779 Dec 30 18:19 test_data_all_GNVTG.csv
+-rw-r--r--  1 Thomas  staff  40263 Dec 30 18:19 test_data_all_GPGSV.csv
+```
+
+### Example 2
+```
+$ python nmea_parser.py test_data/test_data_all.nmea db
+
+Reading in data... done.
+
+Processing data... done.
+
+Writing data to database... data from logfile 'test_data/test_data_all.nmea' written to:
+  'nmea_gn_rmc' table in 'nmea_data' database
+  'nmea_gn_vtg' table in 'nmea_data' database
+  'nmea_gn_gga' table in 'nmea_data' database
+  'nmea_gn_gsa' table in 'nmea_data' database
+  'nmea_gp_gsv' table in 'nmea_data' database
+  'nmea_gl_gsv' table in 'nmea_data' database
+  'nmea_gn_gll' table in 'nmea_data' database
+done.
+
+All done. Exiting.
+```
+
+### Example 3
+```
+$ python nmea_parser.py test_data/test_data_all.nmea both
+
+Reading in data... done.
+
+Processing data... done.
+
+Writing data to CSVs... data from logfile 'test_data/test_data_all.nmea' written to:
+  test_data_all_GNRMC.csv
+  test_data_all_GNVTG.csv
+  test_data_all_GNGGA.csv
+  test_data_all_GNGSA.csv
+  test_data_all_GPGSV.csv
+  test_data_all_GLGSV.csv
+  test_data_all_GNGLL.csv
+done.
+
+Writing data to database... data from logfile 'test_data/test_data_all.nmea' written to:
+  'nmea_gn_rmc' table in 'nmea_data' database
+  'nmea_gn_vtg' table in 'nmea_data' database
+  'nmea_gn_gga' table in 'nmea_data' database
+  'nmea_gn_gsa' table in 'nmea_data' database
+  'nmea_gp_gsv' table in 'nmea_data' database
+  'nmea_gl_gsv' table in 'nmea_data' database
+  'nmea_gn_gll' table in 'nmea_data' database
+done.
+
+All done. Exiting.
+```
+
+
+## References Used in Development
+https://github.com/Knio/pynmea2/blob/master/README.md
+
+https://www.trimble.com/OEM_ReceiverHelp/V4.44/en/NMEA-0183messages_MessageOverview.html
+
+https://www.u-blox.com/sites/default/files/products/documents/u-blox8-M8_ReceiverDescrProtSpec_%28UBX-13003221%29.pdf (section 31 'NMEA Protocol')

db_creds.py

@@ -0,0 +1,5 @@
+DB_USER     = "postgres"
+DB_PASSWORD = "postgres"    
+DB_HOST     = "localhost"
+DB_PORT     = "5432"
+DB_NAME     = "nmea_data"

db_data_import.py

@@ -0,0 +1,36 @@
+IF_EXISTS_OPT = 'append'  # 'fail', 'replace', or 'append', see https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.to_sql.html
+
+
+import os
+import sqlalchemy #import create_engine
+
+# Local modules/libary files:
+import db_creds
+
+
+def send_data_to_db(log_file_path, dfs, table_name_base, table_name_suffixes=None):
+
+    log_file_name = os.path.basename(log_file_path)
+
+    db_access_str = f'postgresql://{db_creds.DB_USER}:{db_creds.DB_PASSWORD}@{db_creds.DB_HOST}:{db_creds.DB_PORT}/{db_creds.DB_NAME}'
+    engine = sqlalchemy.create_engine(db_access_str)
+
+    table_names = []
+
+    # Put data in database
+    for df_idx, df in enumerate(dfs):
+
+        if_exists_opt_loc = IF_EXISTS_OPT
+
+        table_name = table_name_base
+        if table_name_suffixes:
+            table_name = table_name + '_' + table_name_suffixes[df_idx]
+
+        df.to_sql(table_name, engine, method='multi', if_exists=if_exists_opt_loc)
+
+        table_names.append(table_name)
+
+    return table_names
+
+# TODO: Create separate table for log file IDs and names. Check what the current larged ID is, then append a column to
+# the dfs with that ID + 1, and a row to the log file table with that ID and the log file name, or something like that

db_table_lists.py

@@ -0,0 +1,9 @@
+nmea_tables = [
+  'nmea_gl_gsv',
+  'nmea_gn_gga',
+  'nmea_gn_gll',
+  'nmea_gn_gsa',
+  'nmea_gn_rmc',
+  'nmea_gn_vtg',
+  'nmea_gp_gsv',
+]

db_utils.py

@@ -0,0 +1,78 @@
+import psycopg2
+from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT
+import functools
+print = functools.partial(print, flush=True)  # Prevent print statements from buffering till end of execution
+
+# Local modules/libary files:
+import db_creds
+import db_table_lists
+
+
+def drop_db_tables(tables_to_drop, verbose=False):
+
+    [psqlCon, psqlCursor] = setup_db_connection()
+
+    # Drop tables
+    tableList = ""
+    for idx, tableName in enumerate(tables_to_drop):
+        tableList = tableList + tableName
+        if idx < len(tables_to_drop)-1:  # Don't append comma after last table name
+            tableList = tableList + ", " 
+        if verbose:
+            print(f"Dropping database table {tableName} (and any dependent objects) if it exists.")
+
+        dropTableStmt = f"DROP TABLE IF EXISTS \"{tableName}\" CASCADE;"  # Quotes arouund table names are required for case sensitivity
+        psqlCursor.execute(dropTableStmt);
+
+    free_db_connection(psqlCon, psqlCursor)
+
+
+def create_table(table_name, columns=None):
+
+    db_command = f"""
+        CREATE TABLE IF NOT EXISTS "{table_name}" (
+    """
+
+    if columns:
+        for idx, column in enumerate(columns):
+            db_command = db_command + '"' + column['name'] + '" ' + column['datatype']
+            if idx < len(columns)-1:  # Don't append a comman after the last column declaration
+                db_command = db_command + ','
+
+    db_command = db_command + ')'
+
+    run_db_command(db_command)
+
+
+def run_db_command(db_command):
+
+    [psqlCon, psqlCursor] = setup_db_connection()
+
+    # Run command on database    
+    psqlCursor.execute(db_command);
+
+    # print(psqlCon.notices)
+    # print(psqlCon.notifies)
+
+    free_db_connection(psqlCon, psqlCursor)
+
+
+def setup_db_connection():
+
+    db_access_str = f'postgresql://{db_creds.DB_USER}:{db_creds.DB_PASSWORD}@{db_creds.DB_HOST}:{db_creds.DB_PORT}/{db_creds.DB_NAME}'
+
+    # Start a PostgreSQL database session   
+    psqlCon = psycopg2.connect(db_access_str);
+    psqlCon.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT);
+
+    # Open a database cursor
+    psqlCursor = psqlCon.cursor();
+
+    return [psqlCon, psqlCursor]
+
+
+def free_db_connection(psqlCon, psqlCursor):
+
+    # Free the resources
+    psqlCursor.close();
+    psqlCon.close();

nmea_parser.py

@@ -7,12 +7,27 @@
 from collections import namedtuple
 import re
 import functools
-print = functools.partial(print, flush=True)
+print = functools.partial(print, flush=True)  # Prevent print statements from buffering till end of execution
+
+# Local modules/libary files:
+import db_data_import
+import db_creds
+import db_utils
+import db_table_lists
+
 
 def parse_and_validate_args():
 
     parser = argparse.ArgumentParser()
-    parser.add_argument("filepath", help="File system path to file containing NMEA data")
+    parser.add_argument("filepath",
+                            help="file system path to file containing NMEA data")
+    parser.add_argument("output_method",
+                            choices=['csv', 'db', 'both'],
+                            help="where to output data: CSV files, database, or both")
+    parser.add_argument("--drop_previous_db_tables",
+                            action="store_true",
+                            help="drop previous DB tables before importing new data; only applies when output_method is 'db' or 'both'")
+
     args = parser.parse_args()
 
     if os.path.isfile(args.filepath):
@@ -203,9 +218,24 @@ def dfs_to_csv(sentence_dfs, input_file_path, verbose=False):
 
         if verbose:
             if df_idx is 0:  # If this is the first df
-                print("data written to:")
+                print(f"data from logfile '{input_file_path}' written to:")
             print("  " + filename)
 
+
+def dfs_to_db(sentence_dfs, input_file_path, verbose=False):
+
+    table_name_base = 'nmea'
+    # Pass lowercase 'talker_sentencetype' as table name suffixes
+    table_name_suffixes = [f"{df['talker'][0]}_{df['sentence_type'][0]}".lower() for df in sentence_dfs]
+
+    table_names = db_data_import.send_data_to_db(input_file_path, sentence_dfs, table_name_base, table_name_suffixes)
+
+    if verbose:
+        print(f"data from logfile '{input_file_path}' written to:")
+        for table_name in table_names:
+            print(f"  '{table_name}' table in '{db_creds.DB_NAME}' database")
+
+
 def get_sentence_type(sentence):
 
     return sentence.talker + sentence.sentence_type
@@ -264,8 +294,9 @@ def expand_GSV_fields(fields):
 
 def main():
 
-    print("\nReading in data... ", end="")
     args = parse_and_validate_args()
+
+    print("\nReading in data... ", end="")
     file = open_file(args.filepath)
     sentences = read_file(file)
     print("done.")
@@ -279,9 +310,20 @@ def main():
     sentence_dfs  = sentences_to_dataframes(sentence_sets)
     print("done.")
 
-    print("\nWriting data to CSVs... ", end="")
-    dfs_to_csv(sentence_dfs, args.filepath, verbose=True)
-    print("done.")
+    if (args.output_method == 'csv' or args.output_method == 'both'):
+        print("\nWriting data to CSVs... ", end="")
+        dfs_to_csv(sentence_dfs, args.filepath, verbose=True)
+        print("done.")
+
+    if (args.output_method == 'db'  or args.output_method == 'both'):
+
+        if args.drop_previous_db_tables:
+            print()
+            db_utils.drop_db_tables(db_table_lists.nmea_tables, verbose=True)
+
+        print("\nWriting data to database... ", end="")
+        dfs_to_db(sentence_dfs, args.filepath, verbose=True)
+        print("done.")
 
     print("\nAll done. Exiting.\n\n")
 

requirements.txt

@@ -0,0 +1,11 @@
+
+# nmea_parser/data_to_db.py: 5
+SQLAlchemy == 1.3.22
+
+# nmea_parser/nmea_parser.py: 6
+pandas == 1.2.0
+
+# nmea_parser/nmea_parser.py: 1
+pynmea2 == 1.15.0
+
+psycopg2-binary >= 2.8.6