diff --git a/src/lib/database-migrations/008-spider-data-catalog-errors.sql b/src/lib/database-migrations/008-spider-data-catalog-errors.sql new file mode 100644 index 0000000..b8f6f88 --- /dev/null +++ b/src/lib/database-migrations/008-spider-data-catalog-errors.sql @@ -0,0 +1,6 @@ +CREATE TABLE spider_data_catalog_error ( + url TEXT NOT NULL, + error TEXT NOT NULL, + error_at TIMESTAMP WITHOUT TIME ZONE NOT NULL default (now() at time zone 'utc'), + found_via JSON NOT NULL +); diff --git a/src/lib/spider.js b/src/lib/spider.js index d81c996..db8bbea 100644 --- a/src/lib/spider.js +++ b/src/lib/spider.js @@ -47,6 +47,7 @@ async function spider_data_catalog(url, url_history) { console.error(url_history); console.error(url); console.error(error); + spider_data_catalog_error(url, error, url_history) } } @@ -82,6 +83,7 @@ async function spider_data_set(url, url_history) { console.error(url_history); console.error(url); console.error(error); + spider_data_catalog_error(url, error, url_history) } } @@ -132,6 +134,23 @@ async function write_publisher(data) { }; +async function spider_data_catalog_error(url, error, found_via) { + const client = await database_pool.connect(); + try { + await client.query( + 'INSERT INTO spider_data_catalog_error (url, error, found_via) VALUES ($1, $2, $3)', + [url, error, {"trail":found_via}] + ); + } catch(error) { + console.error("ERROR spider_data_catalog_error"); + console.error(error); + } finally { + // Make sure to release the client before any error handling, + // just in case the error handling itself throws an error. + client.release() + } +} + export { spider, spider_data_catalog,