diff --git a/app.js b/app.js index 138b98a..b39839d 100644 --- a/app.js +++ b/app.js @@ -3,14 +3,14 @@ const app = mu.app; const bodyParser = require('body-parser'); const cors = require('cors'); import {handleDelta} from './handle-deltas'; -import {directQuery, configurableQuery} from './repository/helpers'; +import { cleanup, directQuery, configurableQuery} from './repository/helpers'; const fillInterneOverheid = require('./repository/fill-intern-overheid'); const fillInterneRegering = require('./repository/fill-intern-regering'); const fillPublic = require('./repository/fill-public'); if (!process.env.DIRECT_ENDPOINT) { - throw new Error("DIRECT_ENDPOINT not set!") + throw new Error("DIRECT_ENDPOINT not set!"); } app.use(bodyParser.json({ type: 'application/json' , limit: '50mb' })); @@ -59,7 +59,7 @@ const builders = { } }; -const initialLoad = function(){ +async function initialLoad(){ let toFillUp = ''; if(process.env.RELOAD_ALL_DATA_ON_INIT == "true") { toFillUp = 'public,intern-overheid,intern-regering,minister'; @@ -88,10 +88,15 @@ const initialLoad = function(){ } } }; - fillAll(); + await fillAll(); }; -initialLoad(); +async function startup() { + await cleanup(); + await initialLoad(); +} + +startup(); const deltaBuilders = Object.assign({}, builders); delete deltaBuilders.public; diff --git a/repository/fill-intern-overheid.js b/repository/fill-intern-overheid.js index 064e631..ded9ee2 100644 --- a/repository/fill-intern-overheid.js +++ b/repository/fill-intern-overheid.js @@ -91,15 +91,13 @@ export const fillUp = async (queryEnv, agendas) => { await runStage('copy temp to target', queryEnv, () => { return copyTempToTarget(queryEnv); }); - await runStage('done filling overheid', queryEnv, () => { - return cleanup(queryEnv); - }); + await runStage('done filling overheid', queryEnv, cleanup); const end = moment().utc(); logStage(start, `fill overheid ended at: ${end.format()}`, targetGraph); } catch (e) { logStage(moment(), `${e}\n${e.stack}`, queryEnv.targetGraph); try { - cleanup(queryEnv); + await cleanup(queryEnv); } catch (e2) { console.log(e2); } diff --git a/repository/fill-intern-regering.js b/repository/fill-intern-regering.js index 84c97dd..813f146 100644 --- a/repository/fill-intern-regering.js +++ b/repository/fill-intern-regering.js @@ -82,17 +82,15 @@ export const fillUp = async (queryEnv, agendas) => { await runStage('copy temp to target', queryEnv, () => { return copyTempToTarget(queryEnv); }); - await runStage('cleaned up', queryEnv, () => { - return cleanup(queryEnv); - }); + await runStage('cleaned up', queryEnv, cleanup); const end = moment().utc(); logStage(start, `fill regering ended at: ${end.format()}`, targetGraph); }catch (e) { logStage(moment(), `${e}\n${e.stack}`, queryEnv.targetGraph); try { - cleanup(queryEnv); + await cleanup(queryEnv); }catch (e2) { console.log(e2); } } -}; \ No newline at end of file +}; diff --git a/repository/helpers.js b/repository/helpers.js index 61c3a7f..a502f35 100644 --- a/repository/helpers.js +++ b/repository/helpers.js @@ -129,22 +129,16 @@ const addRelatedFiles = (queryEnv, extraFilters) => { return queryEnv.run(query, true); }; -const cleanup = (queryEnv) => { - // TODO should we not batch this delete? - const query = ` - PREFIX ext: - DELETE { - GRAPH ?g { - ?s ?p ?o. - } - } WHERE { - GRAPH ?g { - ?g a ext:TempGraph . - ?s ?p ?o. - } - }`; - return queryEnv.run(query, true); -}; +async function cleanup() { + const result = JSON.parse(await directQuery("PREFIX ext: SELECT ?g WHERE { GRAPH ?g { ?g a ext:TempGraph }}")); + if (result.results && result.results.bindings) { + console.log(`found ${result.results.bindings.length} old temporary graphs, removing before going further`); + for (let binding of result.results.bindings) { + console.log(`dropping graph ${binding.g.value}`); + await directQuery(`DROP SILENT GRAPH <${binding.g.value}>`); + } + } +} const fillOutDetailsOnVisibleItemsLeft = async (queryEnv) => { const result = await queryEnv.run(` @@ -892,7 +886,7 @@ const configurableQuery = function(queryString, direct){ }, queryString); }; -const directQuery = function(queryString){ +function directQuery(queryString){ return configurableQuery(queryString, true); };