Fleshed out worker.py and tests

Author: 4OH4

README.md

@@ -26,6 +26,6 @@ Doctests are executed for a single module at a time. The command runs the tests
 The `-v` argument requests verbose output - otherwise it only reports test failures.
 
 ### Pytest
-To run all test cases (in the `tests/` folder):
+To run all test cases:
 
-    python -m pytest tests
+    pytest

data/salary_data.csv

@@ -0,0 +1,4 @@
+date,job_title,company_name,salary
+11/04/19,Manager,Big Red Jumbo Fries,10000
+04/05/19,Python coder,Convex Code,13000
+11/06/19,Cook,Tasty Food Shack,14000

tests/requirements.txt requirements.txt

@@ -0,0 +1,69 @@
+# -*- coding: utf-8 -*-
+"""
+Pytesting/run_tests.py
+
+Test runner for demo app
+
+@author: Rupert.Thomas
+Created 16/11/2019
+
+Adapted from: https://github.com/cambridgespark/pydata-testing-for-data-science
+Raoul-Gabriel Urma
+
+Run tests (from the root folder using):
+python -m pytest test/
+
+"""
+import argparse
+import os
+# import urllib.request
+import sys
+
+import pytest
+
+def run_unittests():
+    print("Unittesting model...")
+    pytest.main(['-v', 'tests'])
+
+def run_coverage():
+    print("Running coverage report...")
+    pytest.main(['--cov-report', 'term-missing', '--cov=src/', 'tests/'])
+
+def run_generative():
+    print("Running generative testing...")
+    pytest.main(['-v', '--hypothesis-show-statistics', 'tests/test_transformers_hypothesis.py'])
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="A command line-tool to manage the project.")
+    parser.add_argument(
+        'stage',
+        metavar='stage',
+        type=str,
+        choices=['unittest', 'generative', 'coverage'],
+        help="Stage to run. Either unittest, generative, coverage")
+
+    if len(sys.argv[1:]) == 0:
+        # parser.print_help()
+        # parser.exit()
+        print("Running all...")
+        run_unittests()
+        run_generative()
+        run_coverage()
+        return
+
+    stage = parser.parse_args().stage
+
+    if stage == "unittest":
+        run_unittests()
+
+    elif stage == "coverage":
+        run_coverage()
+
+    elif stage == "generative":
+        run_generative()
+
+
+if __name__ == "__main__":
+    main()

run_tests.py

@@ -28,10 +28,10 @@ class DAO(object):
         >>> dao = DAO('example.db')
         Database connection initialised
         >>> dao.create_jobs_table()
-        >>> dao.insert_job('Cook', 1, 'Tasty Food Shack', 1000)
+        >>> dao.insert_job('Cook','Tasty Food Shack',11000,'11/06/2019')
         1
         >>> dao.query_job(1)
-        ('Cook', 1, 'Tasty Food Shack', 1000)
+        ('Cook', 1, 'Tasty Food Shack', 11000, '11/06/2019')
         >>> dao.close()
         >>> dao.delete_db()
     """
@@ -56,18 +56,19 @@ def create_jobs_table(self):
         self.cur.execute('''CREATE TABLE IF NOT EXISTS jobs(job_title text,
                                                             job_id integer PRIMARY KEY, 
                                                             company_name text,
-                                                            salary integer)''')
-    def insert_job(self, job_title, job_id, company_name, salary):
+                                                            salary integer,
+                                                            date text)''')
+    def insert_job(self, job_title, company_name, salary, date):
         sql = """
-            INSERT INTO jobs (job_title, job_id, company_name, salary)
+            INSERT INTO jobs (job_title, company_name, salary, date)
             VALUES (?, ?, ?, ?)"""
-        self.cur.execute(sql, (job_title, job_id, company_name, salary))
+        self.cur.execute(sql, (job_title, company_name, salary, date))
         self.commit()
         return self.cur.lastrowid
 
     def query_job(self, job_id):
         sql = """
-            SELECT job_title, job_id, company_name, salary FROM jobs WHERE job_id = ?
+            SELECT job_title, job_id, company_name, salary, date FROM jobs WHERE job_id = ?
             """
         self.cur.execute(sql, str(job_id))
         result = self.cur.fetchone()

src/database.py

@@ -8,21 +8,38 @@
 Created 15/11/2019
 """
 
-from worker import Worker
-from database import DAO
+from src.worker import Worker
+from src.database import DAO
 
 class Application:
     def __init__(self):
+        """ Constructor """
         super().__init__()
 
         # Setup database access
         dao = DAO()
         dao.create_jobs_table()
         self.dao = dao
+        self.last_DB_row = None
 
+        # Setup processing worker
+        self.worker = Worker()
+
+    def loadData(self, data_filepath):
+
+        parsed_data = self.worker.readData(data_filepath)
+
+        for row in parsed_data:
+            self.last_DB_row = self.dao.insert_job(row['job_title'], 
+                                row['company_name'], 
+                                row['salary'], 
+                                row['date'])
+        
+        print(f'Last row in database: {self.last_DB_row}')
 
 
 if __name__ == '__main__':
 
     # if being exectuted, create the app object
     app = Application()
+    app.loadData('data/salary_data.csv')

src/main.py

@@ -8,11 +8,109 @@
 Created 15/11/2019
 """
 
+# Conversion map for three-letter months
+short_months = {'Jan': 1, 'Feb': 1, 'Mar': 3, 'Apr': 4, 'May': 5, 'June': 6,
+                'Jul': 7, 'Aug': 8, 'Sep': 9, 'Oct': 10, 'Nov': 11, 'Dec': 12}
+
+# Data file format (columns)
+# (date, job_title, company_name, salary)
+
 class Worker:
 
-    def extractDates(self, input_string):
+    def readData(self, data_filepath):
+        """
+        Read comma-separated data from a data file, and parse.
+
+        N.B. this function is deliberately lacking in robust validation and checking of the input data
+
+        :param data_filepath: (string) path to data file
+        :returns outputdata: (list) of dictionaries, one per row of imported data
+        """
+        
+        output_data = []
+
+        in_file = open(data_filepath, 'r')
+        in_file.readline()  # skip the header line
+
+        for temp_line in in_file.readlines():
+
+            parsed = self.parseLine(temp_line)
+            if parsed is not None:
+                output_data.append(parsed)
+            
+            else:
+                # data incomplete
+                continue
+
+        return output_data
+
+
+    def parseLine(self, input_string):
+        """
+        Parse a single row of input data, and run rudimentary checks
+
+        N.B. this function is deliberately lacking in robust validation and checking of the input data
+
+        :param input_string: (string) of data, hopefully comma-separated
+        :returns result: (dict) parsed data
+        """
+
+        result = None
+
+        temp_data = input_string.split(',')
+        if len(temp_data) == 4:
+            date_string = self.parseDate(temp_data[0])
+            
+            if date_string is not None:
+                result = {
+                    'date': date_string,
+                    'job_title': temp_data[1],
+                    'company_name': temp_data[2],
+                    'salary': int(temp_data[3])
+                }
+
+        return result
+
+                
+                
+    def parseDate(self, input_string):
         """ 
-            Parses an input string, and returns dates
+        Parses an input string, and extracts integer values for day, month and year
+
+        N.B. this function is deliberately lacking in robust validation and checking of the input data
+
+        :param input_string: (string) hopefully containing a date in one of the two recognised formats
+        :returns result: (string) parsed date in dd/mm/YYYY format
+        """
+        result_string = None
+        # Date format 1: dd/mm/yy     e.g. 01/09/19
+        split_parts = input_string.split('/')
+        if len(split_parts) == 3:
+            dd = int(split_parts[0])
+            mm = int(split_parts[1])
+            YYYY = 2000 + int(split_parts[2])
+            
+            # Test and assemble output
+            if (dd>0) and (dd<=31) and \
+                (mm>0) and (mm<=12) and \
+                (YYYY>2000) and (YYYY<2100):
+                    result_string = f'{dd:02}/{mm:02}/{YYYY}'
 
+        # Date format 2: ddmmmYYYY    09Jan2019
+        elif any(month in input_string for month in short_months.keys()):
+            # One of the three letter month strings is in the input
+            match = next(month for month in short_months.keys() if month in input_string)
+            split_parts = input_string.split(match)
+            if len(split_parts) == 2:
+                dd = int(split_parts[0])
+                mm = short_months[match]
+                YYYY = int(split_parts[1])
+                
+                # Test and assemble output
+                if (dd>0) and (dd<=31) and \
+                    (YYYY>2000) and (YYYY<2100):
+                    result_string = f'{dd:02}/{mm:02}/{YYYY}'
+                
+#        else: # Unknown date format
 
-        return True
+        return result_string

src/worker.py

@@ -11,26 +11,60 @@
 python -m pytest test/
 
 """
+import sqlite3
 
-import os
-import mock
+from truth.truth import AssertThat
 
 # Module under test
 from src.database import DAO
 
 
-def test_delete_db(mocker):
+def test_init(mocker):
     """
-        Test deleting the database file
+        Test database initialisation
     """
 
-    mocker.patch('os.remove')
+    # given: setup test framework
+    mock_SQLite3 = mocker.patch('src.database.sqlite3')
+    mock_ConnectionObj = mocker.MagicMock(sqlite3.Connection)  # mock Connection object
+    mock_CursorObj = mocker.MagicMock(sqlite3.Cursor)  # mock Cursor object
+    mock_ConnectionObj.cursor.return_value = mock_CursorObj  # setup method calls
+    mock_SQLite3.connect.return_value = mock_ConnectionObj
+    
+    # when: database is initialised
+    dao = DAO()
+    
+    # then: expect connection and request for cursor
+    AssertThat(mock_SQLite3.connect).WasCalled().Once()
+    mock_SQLite3.connect.assert_called_once()
+    mock_ConnectionObj.cursor.assert_called_once()
+
 
-    # /given
+def test_delete_db(mocker):
+    """
+        Test deleting the database file
+    """
+    
+    # given: setup test framework
+    mock_SQLite3 = mocker.patch('src.database.sqlite3')  # patch database createion in DAO.__init__
     dao = DAO()
+    mock_os_remove = mocker.patch('src.database.os.remove')  # patch database deletion command
 
     # /when
     dao.delete_db()
 
     # /then
-    os.remove.assert_called_once()
+    mock_os_remove.assert_called_once()
+
+
+def test_close_db(mocker):
+    pass
+
+def test_create_jobs_table(mocker):
+    pass
+
+def test_insert_job(mocker):
+    pass
+
+def test_query_job(mocker):
+    pass