diff --git a/.gitignore b/.gitignore index 3323363..35c26d4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ .DS_Store +__pycache__ *.csv diff --git a/README.md b/README.md index d3f4ee1..2ccf8b3 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,120 @@ -Under development. README to be populated later. +This library makes use of [pynmea2](https://github.com/Knio/pynmea2) to parse through input NMEA data, organize it, and output it to CSV files or to a PostgreSQL database. -Reference materials: -https://www.trimble.com/OEM_ReceiverHelp/V4.44/en/NMEA-0183messages_MessageOverview.html \ No newline at end of file +## Setup + +Your input file should have a format similiar to those under `test_data`. To have your data datetime stamped, it must be in a format like that of `test_data/test_data_all.nmea`, with RMC sentences containing date and time stamps proceed other sentences in the same cycle. + +If working with a database, the database access information/credentials must be setup in `db_creds.py`. + + +## Usage +``` +$ cd ~/Downloads/nmea_parser/ +$ pip install -r requirements.txt +... +$ python nmea_parser.py --help +usage: nmea_parser.py [-h] [--drop_previous_db_tables] filepath {csv,db,both} + +positional arguments: + filepath file system path to file containing NMEA data + {csv,db,both} where to output data: CSV files, database, or both + +optional arguments: + -h, --help show this help message and exit + --drop_previous_db_tables + drop previous DB tables before importing new data; + only applies when output_method is 'db' or 'both' +``` +## Examples +### Example 1 +``` +$ ls -l *.csv +ls: *.csv: No such file or directory +$ python nmea_parser.py test_data/test_data_all.nmea csv + +Reading in data... done. + +Processing data... done. + +Writing data to CSVs... data from logfile 'test_data/test_data_all.nmea' written to: + test_data_all_GNRMC.csv + test_data_all_GNVTG.csv + test_data_all_GNGGA.csv + test_data_all_GNGSA.csv + test_data_all_GPGSV.csv + test_data_all_GLGSV.csv + test_data_all_GNGLL.csv +done. + +All done. Exiting. + + +$ ls -l *.csv +-rw-r--r-- 1 Thomas staff 14310 Dec 30 18:19 test_data_all_GLGSV.csv +-rw-r--r-- 1 Thomas staff 9502 Dec 30 18:19 test_data_all_GNGGA.csv +-rw-r--r-- 1 Thomas staff 6852 Dec 30 18:19 test_data_all_GNGLL.csv +-rw-r--r-- 1 Thomas staff 18472 Dec 30 18:19 test_data_all_GNGSA.csv +-rw-r--r-- 1 Thomas staff 8672 Dec 30 18:19 test_data_all_GNRMC.csv +-rw-r--r-- 1 Thomas staff 5779 Dec 30 18:19 test_data_all_GNVTG.csv +-rw-r--r-- 1 Thomas staff 40263 Dec 30 18:19 test_data_all_GPGSV.csv +``` + +### Example 2 +``` +$ python nmea_parser.py test_data/test_data_all.nmea db + +Reading in data... done. + +Processing data... done. + +Writing data to database... data from logfile 'test_data/test_data_all.nmea' written to: + 'nmea_gn_rmc' table in 'nmea_data' database + 'nmea_gn_vtg' table in 'nmea_data' database + 'nmea_gn_gga' table in 'nmea_data' database + 'nmea_gn_gsa' table in 'nmea_data' database + 'nmea_gp_gsv' table in 'nmea_data' database + 'nmea_gl_gsv' table in 'nmea_data' database + 'nmea_gn_gll' table in 'nmea_data' database +done. + +All done. Exiting. +``` + +### Example 3 +``` +$ python nmea_parser.py test_data/test_data_all.nmea both + +Reading in data... done. + +Processing data... done. + +Writing data to CSVs... data from logfile 'test_data/test_data_all.nmea' written to: + test_data_all_GNRMC.csv + test_data_all_GNVTG.csv + test_data_all_GNGGA.csv + test_data_all_GNGSA.csv + test_data_all_GPGSV.csv + test_data_all_GLGSV.csv + test_data_all_GNGLL.csv +done. + +Writing data to database... data from logfile 'test_data/test_data_all.nmea' written to: + 'nmea_gn_rmc' table in 'nmea_data' database + 'nmea_gn_vtg' table in 'nmea_data' database + 'nmea_gn_gga' table in 'nmea_data' database + 'nmea_gn_gsa' table in 'nmea_data' database + 'nmea_gp_gsv' table in 'nmea_data' database + 'nmea_gl_gsv' table in 'nmea_data' database + 'nmea_gn_gll' table in 'nmea_data' database +done. + +All done. Exiting. +``` + + +## References Used in Development +https://github.com/Knio/pynmea2/blob/master/README.md + +https://www.trimble.com/OEM_ReceiverHelp/V4.44/en/NMEA-0183messages_MessageOverview.html + +https://www.u-blox.com/sites/default/files/products/documents/u-blox8-M8_ReceiverDescrProtSpec_%28UBX-13003221%29.pdf (section 31 'NMEA Protocol') \ No newline at end of file diff --git a/db_creds.py b/db_creds.py new file mode 100644 index 0000000..c49448d --- /dev/null +++ b/db_creds.py @@ -0,0 +1,5 @@ +DB_USER = "postgres" +DB_PASSWORD = "postgres" +DB_HOST = "localhost" +DB_PORT = "5432" +DB_NAME = "nmea_data" \ No newline at end of file diff --git a/db_data_import.py b/db_data_import.py new file mode 100644 index 0000000..6396b4b --- /dev/null +++ b/db_data_import.py @@ -0,0 +1,36 @@ +IF_EXISTS_OPT = 'append' # 'fail', 'replace', or 'append', see https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.to_sql.html + + +import os +import sqlalchemy #import create_engine + +# Local modules/libary files: +import db_creds + + +def send_data_to_db(log_file_path, dfs, table_name_base, table_name_suffixes=None): + + log_file_name = os.path.basename(log_file_path) + + db_access_str = f'postgresql://{db_creds.DB_USER}:{db_creds.DB_PASSWORD}@{db_creds.DB_HOST}:{db_creds.DB_PORT}/{db_creds.DB_NAME}' + engine = sqlalchemy.create_engine(db_access_str) + + table_names = [] + + # Put data in database + for df_idx, df in enumerate(dfs): + + if_exists_opt_loc = IF_EXISTS_OPT + + table_name = table_name_base + if table_name_suffixes: + table_name = table_name + '_' + table_name_suffixes[df_idx] + + df.to_sql(table_name, engine, method='multi', if_exists=if_exists_opt_loc) + + table_names.append(table_name) + + return table_names + +# TODO: Create separate table for log file IDs and names. Check what the current larged ID is, then append a column to +# the dfs with that ID + 1, and a row to the log file table with that ID and the log file name, or something like that \ No newline at end of file diff --git a/db_table_lists.py b/db_table_lists.py new file mode 100644 index 0000000..fe474ae --- /dev/null +++ b/db_table_lists.py @@ -0,0 +1,9 @@ +nmea_tables = [ + 'nmea_gl_gsv', + 'nmea_gn_gga', + 'nmea_gn_gll', + 'nmea_gn_gsa', + 'nmea_gn_rmc', + 'nmea_gn_vtg', + 'nmea_gp_gsv', +] \ No newline at end of file diff --git a/db_utils.py b/db_utils.py new file mode 100644 index 0000000..e426883 --- /dev/null +++ b/db_utils.py @@ -0,0 +1,78 @@ +import psycopg2 +from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT +import functools +print = functools.partial(print, flush=True) # Prevent print statements from buffering till end of execution + +# Local modules/libary files: +import db_creds +import db_table_lists + + +def drop_db_tables(tables_to_drop, verbose=False): + + [psqlCon, psqlCursor] = setup_db_connection() + + # Drop tables + tableList = "" + for idx, tableName in enumerate(tables_to_drop): + tableList = tableList + tableName + if idx < len(tables_to_drop)-1: # Don't append comma after last table name + tableList = tableList + ", " + if verbose: + print(f"Dropping database table {tableName} (and any dependent objects) if it exists.") + + dropTableStmt = f"DROP TABLE IF EXISTS \"{tableName}\" CASCADE;" # Quotes arouund table names are required for case sensitivity + psqlCursor.execute(dropTableStmt); + + free_db_connection(psqlCon, psqlCursor) + + +def create_table(table_name, columns=None): + + db_command = f""" + CREATE TABLE IF NOT EXISTS "{table_name}" ( + """ + + if columns: + for idx, column in enumerate(columns): + db_command = db_command + '"' + column['name'] + '" ' + column['datatype'] + if idx < len(columns)-1: # Don't append a comman after the last column declaration + db_command = db_command + ',' + + db_command = db_command + ')' + + run_db_command(db_command) + + +def run_db_command(db_command): + + [psqlCon, psqlCursor] = setup_db_connection() + + # Run command on database + psqlCursor.execute(db_command); + + # print(psqlCon.notices) + # print(psqlCon.notifies) + + free_db_connection(psqlCon, psqlCursor) + + +def setup_db_connection(): + + db_access_str = f'postgresql://{db_creds.DB_USER}:{db_creds.DB_PASSWORD}@{db_creds.DB_HOST}:{db_creds.DB_PORT}/{db_creds.DB_NAME}' + + # Start a PostgreSQL database session + psqlCon = psycopg2.connect(db_access_str); + psqlCon.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT); + + # Open a database cursor + psqlCursor = psqlCon.cursor(); + + return [psqlCon, psqlCursor] + + +def free_db_connection(psqlCon, psqlCursor): + + # Free the resources + psqlCursor.close(); + psqlCon.close(); \ No newline at end of file diff --git a/nmea_parser.py b/nmea_parser.py index ac9f540..18179c4 100644 --- a/nmea_parser.py +++ b/nmea_parser.py @@ -7,12 +7,27 @@ from collections import namedtuple import re import functools -print = functools.partial(print, flush=True) +print = functools.partial(print, flush=True) # Prevent print statements from buffering till end of execution + +# Local modules/libary files: +import db_data_import +import db_creds +import db_utils +import db_table_lists + def parse_and_validate_args(): parser = argparse.ArgumentParser() - parser.add_argument("filepath", help="File system path to file containing NMEA data") + parser.add_argument("filepath", + help="file system path to file containing NMEA data") + parser.add_argument("output_method", + choices=['csv', 'db', 'both'], + help="where to output data: CSV files, database, or both") + parser.add_argument("--drop_previous_db_tables", + action="store_true", + help="drop previous DB tables before importing new data; only applies when output_method is 'db' or 'both'") + args = parser.parse_args() if os.path.isfile(args.filepath): @@ -203,9 +218,24 @@ def dfs_to_csv(sentence_dfs, input_file_path, verbose=False): if verbose: if df_idx is 0: # If this is the first df - print("data written to:") + print(f"data from logfile '{input_file_path}' written to:") print(" " + filename) + +def dfs_to_db(sentence_dfs, input_file_path, verbose=False): + + table_name_base = 'nmea' + # Pass lowercase 'talker_sentencetype' as table name suffixes + table_name_suffixes = [f"{df['talker'][0]}_{df['sentence_type'][0]}".lower() for df in sentence_dfs] + + table_names = db_data_import.send_data_to_db(input_file_path, sentence_dfs, table_name_base, table_name_suffixes) + + if verbose: + print(f"data from logfile '{input_file_path}' written to:") + for table_name in table_names: + print(f" '{table_name}' table in '{db_creds.DB_NAME}' database") + + def get_sentence_type(sentence): return sentence.talker + sentence.sentence_type @@ -264,8 +294,9 @@ def expand_GSV_fields(fields): def main(): - print("\nReading in data... ", end="") args = parse_and_validate_args() + + print("\nReading in data... ", end="") file = open_file(args.filepath) sentences = read_file(file) print("done.") @@ -279,9 +310,20 @@ def main(): sentence_dfs = sentences_to_dataframes(sentence_sets) print("done.") - print("\nWriting data to CSVs... ", end="") - dfs_to_csv(sentence_dfs, args.filepath, verbose=True) - print("done.") + if (args.output_method == 'csv' or args.output_method == 'both'): + print("\nWriting data to CSVs... ", end="") + dfs_to_csv(sentence_dfs, args.filepath, verbose=True) + print("done.") + + if (args.output_method == 'db' or args.output_method == 'both'): + + if args.drop_previous_db_tables: + print() + db_utils.drop_db_tables(db_table_lists.nmea_tables, verbose=True) + + print("\nWriting data to database... ", end="") + dfs_to_db(sentence_dfs, args.filepath, verbose=True) + print("done.") print("\nAll done. Exiting.\n\n") diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..129e0cf --- /dev/null +++ b/requirements.txt @@ -0,0 +1,11 @@ + +# nmea_parser/data_to_db.py: 5 +SQLAlchemy == 1.3.22 + +# nmea_parser/nmea_parser.py: 6 +pandas == 1.2.0 + +# nmea_parser/nmea_parser.py: 1 +pynmea2 == 1.15.0 + +psycopg2-binary >= 2.8.6 \ No newline at end of file