From 354e7ac9b76cb66541947ae1867b5543a116d20d Mon Sep 17 00:00:00 2001
From: James <james.baster@opendataservices.coop>
Date: Thu, 25 Jun 2020 17:23:15 +0100
Subject: [PATCH] spider: Write errors to database

https://github.com/openactive/conformance-services/issues/3
---
 .../008-spider-data-catalog-errors.sql        |  6 ++++++
 src/lib/database.js                           |  1 +
 src/lib/spider.js                             | 19 +++++++++++++++++++
 3 files changed, 26 insertions(+)
 create mode 100644 src/lib/database-migrations/008-spider-data-catalog-errors.sql

diff --git a/src/lib/database-migrations/008-spider-data-catalog-errors.sql b/src/lib/database-migrations/008-spider-data-catalog-errors.sql
new file mode 100644
index 0000000..b8f6f88
--- /dev/null
+++ b/src/lib/database-migrations/008-spider-data-catalog-errors.sql
@@ -0,0 +1,6 @@
+CREATE TABLE spider_data_catalog_error (
+    url TEXT NOT NULL,
+    error TEXT NOT NULL,
+    error_at TIMESTAMP WITHOUT TIME ZONE NOT NULL default (now() at time zone 'utc'),
+    found_via JSON NOT NULL
+);
diff --git a/src/lib/database.js b/src/lib/database.js
index 80ae92e..6a76c2e 100644
--- a/src/lib/database.js
+++ b/src/lib/database.js
@@ -22,6 +22,7 @@ async function delete_database() {
     var client;
     try {
         client = await database_pool.connect();
+        await client.query('DROP TABLE IF EXISTS spider_data_catalog_error CASCADE');
         await client.query('DROP TABLE IF EXISTS normalised_data CASCADE');
         await client.query('DROP TABLE IF EXISTS raw_data CASCADE');
         await client.query('DROP TABLE IF EXISTS publisher_feed CASCADE');
diff --git a/src/lib/spider.js b/src/lib/spider.js
index d81c996..db8bbea 100644
--- a/src/lib/spider.js
+++ b/src/lib/spider.js
@@ -47,6 +47,7 @@ async function spider_data_catalog(url, url_history) {
         console.error(url_history);
         console.error(url);
         console.error(error);
+        spider_data_catalog_error(url, error, url_history)
     }
 
 }
@@ -82,6 +83,7 @@ async function spider_data_set(url, url_history) {
         console.error(url_history);
         console.error(url);
         console.error(error);
+        spider_data_catalog_error(url, error, url_history)
     }
 }
 
@@ -132,6 +134,23 @@ async function write_publisher(data) {
 
 };
 
+async function spider_data_catalog_error(url, error, found_via) {
+    const client = await database_pool.connect();
+    try {
+        await client.query(
+            'INSERT INTO  spider_data_catalog_error (url, error, found_via) VALUES ($1, $2, $3)',
+            [url, error, {"trail":found_via}]
+        );
+    } catch(error) {
+        console.error("ERROR spider_data_catalog_error");
+        console.error(error);
+    } finally {
+        // Make sure to release the client before any error handling,
+        // just in case the error handling itself throws an error.
+        client.release()
+    }
+}
+
 export {
   spider,
   spider_data_catalog,