download-raw: Store errors

docs/stage/download-raw-data.md

@@ -9,3 +9,13 @@ Deletes are soft deletes, marked by the `data_deleted` column in the table.
 To run this:
 
 `$ node ./src/bin/download-raw.js`
+
+
+## Errors
+
+Any errors encountered during this stage will be stored in the `download_raw_errors` table.
+
+* `url` - Where the error occurred
+* `error` - What the error was
+* `publisher_feed_id` - The publisher feed we were fetching when we had an error
+* `error_at` - What date and time the error occurred

src/lib/database-migrations/009-download-raw-errors.sql

@@ -0,0 +1,7 @@
+CREATE TABLE download_raw_errors (
+    url TEXT NOT NULL,
+    error TEXT NOT NULL,
+    error_at TIMESTAMP WITHOUT TIME ZONE NOT NULL default (now() at time zone 'utc'),
+    publisher_feed_id INTEGER NOT NULL,
+    CONSTRAINT download_raw_errors_publisher_feed_id FOREIGN KEY (publisher_feed_id) REFERENCES publisher_feed(id)
+);

src/lib/database.js

@@ -22,6 +22,7 @@ async function delete_database() {
     var client;
     try {
         client = await database_pool.connect();
+        await client.query('DROP TABLE IF EXISTS download_raw_errors CASCADE');
         await client.query('DROP TABLE IF EXISTS spider_data_catalog_error CASCADE');
         await client.query('DROP TABLE IF EXISTS normalised_data CASCADE');
         await client.query('DROP TABLE IF EXISTS raw_data CASCADE');

src/lib/download-raw.js

@@ -114,6 +114,7 @@ async function download_raw_publisher_feed(publisher_feed, callback) {
 
         } catch (er) {
           console.log(`Issue with publisher feed ${publisher_feed.id} - ${er}`);
+          download_raw_error(nextURL, er, publisher_feed.id);
           break;
         }
 
@@ -122,6 +123,23 @@ async function download_raw_publisher_feed(publisher_feed, callback) {
 }
 
 
+async function download_raw_error(url, error, publisher_feed_id) {
+    const client = await database_pool.connect();
+    try {
+        await client.query(
+            'INSERT INTO  download_raw_errors (url, error, publisher_feed_id) VALUES ($1, $2, $3)',
+            [url, error, publisher_feed_id]
+        );
+    } catch(error) {
+        console.error("ERROR download_raw_error");
+        console.error(error);
+    } finally {
+        // Make sure to release the client before any error handling,
+        // just in case the error handling itself throws an error.
+        client.release()
+    }
+};
+
 
 export {
   download_raw_all_publisher_feeds,