spider: Write errors to database

#3

src/lib/database-migrations/008-spider-data-catalog-errors.sql

@@ -0,0 +1,6 @@
+CREATE TABLE spider_data_catalog_error (
+    url TEXT NOT NULL,
+    error TEXT NOT NULL,
+    error_at TIMESTAMP WITHOUT TIME ZONE NOT NULL default (now() at time zone 'utc'),
+    found_via JSON NOT NULL
+);

src/lib/database.js

@@ -22,6 +22,7 @@ async function delete_database() {
     var client;
     try {
         client = await database_pool.connect();
+        await client.query('DROP TABLE IF EXISTS spider_data_catalog_error CASCADE');
         await client.query('DROP TABLE IF EXISTS normalised_data CASCADE');
         await client.query('DROP TABLE IF EXISTS raw_data CASCADE');
         await client.query('DROP TABLE IF EXISTS publisher_feed CASCADE');

src/lib/spider.js

@@ -47,6 +47,7 @@ async function spider_data_catalog(url, url_history) {
         console.error(url_history);
         console.error(url);
         console.error(error);
+        spider_data_catalog_error(url, error, url_history)
     }
 
 }
@@ -82,6 +83,7 @@ async function spider_data_set(url, url_history) {
         console.error(url_history);
         console.error(url);
         console.error(error);
+        spider_data_catalog_error(url, error, url_history)
     }
 }
 
@@ -132,6 +134,23 @@ async function write_publisher(data) {
 
 };
 
+async function spider_data_catalog_error(url, error, found_via) {
+    const client = await database_pool.connect();
+    try {
+        await client.query(
+            'INSERT INTO  spider_data_catalog_error (url, error, found_via) VALUES ($1, $2, $3)',
+            [url, error, {"trail":found_via}]
+        );
+    } catch(error) {
+        console.error("ERROR spider_data_catalog_error");
+        console.error(error);
+    } finally {
+        // Make sure to release the client before any error handling,
+        // just in case the error handling itself throws an error.
+        client.release()
+    }
+}
+
 export {
   spider,
   spider_data_catalog,