BritishGeologicalSurvey · KoalaGeo · Mar 7, 2024 · Mar 6, 2024 · Mar 7, 2024 · Mar 7, 2024
diff --git a/app/checkers.py b/app/checkers.py
@@ -42,10 +42,6 @@ def check_ags(filename: Path, standard_AGS4_dictionary: Optional[str] = None) ->
         description = f"UnicodeDecodeError: {err.reason}"
         errors = {'File read error': [{'line': line_no, 'group': '', 'desc': description}]}
         dictionary = ''
-    except AGS4.AGS4Error as err:
-        description = f"AGS4Error: {err}"
-        errors = {'File read error': [{'line': '-', 'group': '', 'desc': description}]}
-        dictionary = ''
 
     # Discard unecessary summary from errors dictionary
     errors.pop('Summary of data', None)

diff --git a/app/schemas.py b/app/schemas.py
@@ -8,13 +8,13 @@
 VALID_KEYS = [
     # AGS schema rules
     'AGS Format Rule 1', 'AGS Format Rule 2', 'AGS Format Rule 2a', 'AGS Format Rule 2b',
-    'AGS Format Rule 2c', 'AGS Format Rule 3', 'AGS Format Rule 4a', 'AGS Format Rule 4b',
+    'AGS Format Rule 2c', 'AGS Format Rule 3', 'AGS Format Rule 4', 'AGS Format Rule 4a', 'AGS Format Rule 4b',
     'AGS Format Rule 5', 'AGS Format Rule 6', 'AGS Format Rule 7', 'AGS Format Rule 8',
     'AGS Format Rule 9', 'AGS Format Rule 10a', 'AGS Format Rule 10b', 'AGS Format Rule 10c',
     'AGS Format Rule 11a', 'AGS Format Rule 11b', 'AGS Format Rule 11c', 'AGS Format Rule 12',
     'AGS Format Rule 13', 'AGS Format Rule 14', 'AGS Format Rule 15', 'AGS Format Rule 16',
     'AGS Format Rule 17', 'AGS Format Rule 18', 'AGS Format Rule 19', 'AGS Format Rule 19a',
-    'AGS Format Rule 19b', 'AGS Format Rule 20', 'General',
+    'AGS Format Rule 19b', 'AGS Format Rule 20', 'AGS Format Rule ?', 'General',
     # Warnings and FYIs
     'Warning (Related to Rule 16)', 'FYI (Related to Rule 1)',
     # Errors

diff --git a/test/files/example_broken_ags.ags b/test/files/example_broken_ags.ags
@@ -0,0 +1,68 @@
+"GROUP","PROJ"
+"HEADING","PROJ_ID","PROJ_NAME","PROJ_LOC","PROJ_CLNT","PROJ_CONT","PROJ_ENG","PROJ_MEMO","FILE_FSET"
+"UNIT","","","","","","","",""
+"TYPE","X","X","X","X","X","X","X","X"
+"DATA","121415","ACME Gas Works Redevelopment","Anytown","ACME Enterprises","ACME Drilling Ltd","","",""
+
+"GROUP","ABBR"
+"HEADING","ABBR_HDNG","ABBR_CODE","ABBR_DESC","ABBR_LIST","ABBR_REM","FILE_FSET"
+"UNIT","","","","","",""
+"TYPE","X","X","X","X","X","X"
+"DATA","DICT_TYPE","GROUP","Flag to indicate definition is a GROUP","","",""
+"DATA","DICT_TYPE","HEADING","Flag to indicate definition is a HEADING","","",""
+"DATA","DICT_STAT","OTHER","Other Field","","",""
+"DATA","DICT_STAT","KEY","Key Field","","",""
+"DATA","SAMP_TYPE","U","Undisturbed sample - open drive","","",""
+
+"GROUP","TRAN"
+"HEADING","TRAN_ISNO","TRAN_DATE","TRAN_PROD","TRAN_STAT","TRAN_DESC","TRAN_AGS","TRAN_RECV","TRAN_DLIM","TRAN_RCON","TRAN_REM","FILE_FSET"
+"UNIT","","yyyy-mm-dd","","","","","","","","",""
+"TYPE","X","DT","X","X","X","X","X","X","X","X","X"
+"DATA","1","2021-01-18","ACME Drilling Ltd","Preliminary","Draft Logs only","4.1","ACME Consulting","|","+","",""
+
+"GROUP","TYPE"
+"HEADING","TYPE_TYPE","TYPE_DESC","FILE_FSET"
+"UNIT","","",""
+"TYPE","X","X","X"
+"DATA","U","Undefined",""
+"DATA","X","Text",""
+"DATA","ID","Unique identifier",""
+"DATA","PA","ABBR pick list",""
+"DATA","2DP","Value; required 
+number of decimal places, 2",""
+"DATA","DT","Date Time (ISO 8601:2004)",""
+"DATA","0DP","Value; required 
+number of decimal places, 0",""
+"DATA","1DP","Value; required 
+number of decimal places, 1",""
+"DATA","PT","TYPE pick list",""
+"DATA","PU","UNIT pick list",""
+"DATA","DMS","Degrees:Minutes:Seconds",""
+"DATA","T","Elapsed time",""
+"DATA","RL","Record link",""
+
+"GROUP","UNIT"
+"HEADING","UNIT_UNIT","UNIT_DESC","UNIT_REM","FILE_FSET"
+"UNIT","","","",""
+"TYPE","X","X","X","X"
+"DATA","-","No unit","",""
+"DATA","m","metre","",""
+"DATA","yyyy-mm-dd","Date (ISO8601)","",""
+"DATA","yyyy-mm-ddThh:mm:ss.sss","Date Time (ISO8601)","",""
+"DATA","mm","millimetre","",""
+"DATA","%","percentage","",""
+"DATA","l/min","litres per minute","",""
+"DATA","hh:mm:ss","hh:mm:ss","",""
+
+"GROUP","LOCA"
+"HEADING","LOCA_ID","LOCA_TYPE","LOCA_STAT","LOCA_NATE","LOCA_NATN","LOCA_GREF","LOCA_GL","LOCA_REM","LOCA_FDEP","LOCA_STAR","LOCA_PURP","LOCA_TERM","LOCA_ENDD","LOCA_LETT","LOCA_LOCX","LOCA_LOCY","LOCA_LOCZ","LOCA_LREF","LOCA_DATM","LOCA_ETRV","LOCA_NTRV","LOCA_LTRV","LOCA_XTRL","LOCA_YTRL","LOCA_ZTRL","LOCA_LAT","LOCA_LON","LOCA_ELAT","LOCA_ELON","LOCA_LLZ","LOCA_LOCM","LOCA_LOCA","LOCA_CLST","LOCA_ALID","LOCA_OFFS","LOCA_CNGE","LOCA_TRAN","FILE_FSET"
+"UNIT","","","","m","m","","m","","m","yyyy-mm-dd","","","yyyy-mm-dd","","m","m","m","","","m","m","m","m","m","m","","","","","","","","","","","","",""
+"TYPE","ID","PA","PA","2DP","2DP","PA","2DP","X","2DP","DT","X","X","DT","X","2DP","2DP","2DP","X","X","2DP","2DP","2DP","2DP","2DP","2DP","DMS","DMS","DMS","DMS","X","X","X","X","X","2DP","X","X","X"
+"DATA","327-16A","","","523145.00","178456.12","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","",""
+
+"GROUP","SAMP"
+"HEADING","LOCA_ID","SAMP_TOP","SAMP_REF","SAMP_TYPE","SAMP_ID","SAMP_BASE","SAMP_DTIM","SAMP_UBLO","SAMP_CONT","SAMP_PREP","SAMP_SDIA","SAMP_WDEP","SAMP_RECV","SAMP_TECH","SAMP_MATX","SAMP_TYPC","SAMP_WHO","SAMP_WHY","SAMP_REM","SAMP_DESC","SAMP_DESD","SAMP_LOG","SAMP_COND","SAMP_CLSS","SAMP_BAR","SAMP_TEMP","SAMP_PRES","SAMP_FLOW","SAMP_ETIM","SAMP_DURN","SAMP_CAPT","SAMP_LINK","GEOL_STAT","FILE_FSET"
+"UNIT","","m","","","","m","yyyy-mm-ddThh:mm:ss.sss","","","","mm","m","%","","","","","","","","yyyy-mm-dd","","","","","","","l/min","yyyy-mm-ddThh:mm:ss.sss","hh:mm:ss","","","",""
+"TYPE","ID","2DP","X","PA","ID","2DP","DT","0DP","X","X","0DP","X","0DP","X","X","X","X","X","X","X","DT","X","X","X","1DP","0DP","1DP","1DP","DT","T","X","X","X","X"
+"DATA","327-16A","24.55","24","U","ABC121415010","25.00","","","","","","","","","","","","","","","","","","","","","","","","","","","",""
+
diff --git a/test/fixtures_json.py b/test/fixtures_json.py
@@ -12,6 +12,91 @@
         'valid': True,
         'additional_metadata': {}
     },
+    'example_broken_ags.ags': {
+        "filename": "example_broken_ags.ags",
+        "filesize": 4111,
+        "checkers": ["python_ags4 v0.5.0"],
+        'dictionary': 'Standard_dictionary_v4_1_1.ags',
+        'time': dt.datetime(2021, 8, 23, 14, 25, 43, tzinfo=dt.timezone.utc),
+        "message": "13 error(s) found in file!",
+        "errors": {
+            "AGS Format Rule 4": [
+                {
+                    "line": 31,
+                    "group": "TYPE",
+                    "desc": "Number of fields does not match the HEADING row."
+                },
+                {
+                    "line": 34,
+                    "group": "TYPE",
+                    "desc": "Number of fields does not match the HEADING row."
+                },
+                {
+                    "line": 36,
+                    "group": "TYPE",
+                    "desc": "Number of fields does not match the HEADING row."
+                }
+            ],
+            "AGS Format Rule 5": [
+                {
+                    "line": 31,
+                    "group": "",
+                    "desc": "Contains fields that are not enclosed in double quotes."
+                },
+                {
+                    "line": 32,
+                    "group": "",
+                    "desc": "Contains fields that are not enclosed in double quotes."
+                },
+                {
+                    "line": 34,
+                    "group": "",
+                    "desc": "Contains fields that are not enclosed in double quotes."
+                },
+                {
+                    "line": 35,
+                    "group": "",
+                    "desc": "Contains fields that are not enclosed in double quotes."
+                },
+                {
+                    "line": 36,
+                    "group": "",
+                    "desc": "Contains fields that are not enclosed in double quotes."
+                },
+                {
+                    "line": 37,
+                    "group": "",
+                    "desc": "Contains fields that are not enclosed in double quotes."
+                }
+            ],
+            "AGS Format Rule 3": [
+                {
+                    "line": 32,
+                    "group": "",
+                    "desc": "Does not start with a valid data descriptor."
+                },
+                {
+                    "line": 35,
+                    "group": "",
+                    "desc": "Does not start with a valid data descriptor."
+                },
+                {
+                    "line": 37,
+                    "group": "",
+                    "desc": "Does not start with a valid data descriptor."
+                }
+            ],
+            "AGS Format Rule ?": [
+                {
+                    "line": "-",
+                    "group": "",
+                    "desc": "Line 31 does not have the same number of entries as the HEADING row in TYPE."
+                }
+            ],
+        },
+        "valid": False,
+        'additional_metadata': {}
+    },
     'nonsense.AGS': {
         'filename': 'nonsense.AGS',
         'filesize': 9,

diff --git a/test/fixtures_plain_text.py b/test/fixtures_plain_text.py
@@ -12,6 +12,47 @@
 Time: 2021-08-23 14:25:43+00:00
 
 
+================================================================================
+""",
+    'example_broken_ags.ags': """
+================================================================================
+example_broken_ags.ags: 13 error(s) found in file!
+
+# Metadata
+
+File size: 4111 bytes
+Checkers: ['python_ags4 v0.5.0']
+Dictionary: Standard_dictionary_v4_1_1.ags
+Time: 2021-08-23 14:25:43+00:00
+
+
+# Errors
+
+## AGS Format Rule 4
+
+Line: 31 - Number of fields does not match the HEADING row.
+Line: 34 - Number of fields does not match the HEADING row.
+Line: 36 - Number of fields does not match the HEADING row.
+
+## AGS Format Rule 5
+
+Line: 31 - Contains fields that are not enclosed in double quotes.
+Line: 32 - Contains fields that are not enclosed in double quotes.
+Line: 34 - Contains fields that are not enclosed in double quotes.
+Line: 35 - Contains fields that are not enclosed in double quotes.
+Line: 36 - Contains fields that are not enclosed in double quotes.
+Line: 37 - Contains fields that are not enclosed in double quotes.
+
+## AGS Format Rule 3
+
+Line: 32 - Does not start with a valid data descriptor.
+Line: 35 - Does not start with a valid data descriptor.
+Line: 37 - Does not start with a valid data descriptor.
+
+## AGS Format Rule ?
+
+Group:  - Line 31 does not have the same number of entries as the HEADING row in TYPE.
+
 ================================================================================
 """,
     'nonsense.AGS': """

diff --git a/test/unit/test_validation.py b/test/unit/test_validation.py
@@ -164,8 +164,8 @@ def test_validate_custom_dictionary_bad_file():
 
 
 @pytest.mark.parametrize('filename', [
-    'example_ags.ags', 'nonsense.AGS', 'random_binary.ags',
-    'real/Blackburn Southern Bypass.ags'])
+    'example_ags.ags', 'example_broken_ags.ags', 'nonsense.AGS',
+    'random_binary.ags', 'real/Blackburn Southern Bypass.ags'])
 def test_to_plain_text(filename):
     # Arrange
     response = JSON_RESPONSES[filename]