From 94fba5145364558ad53a5fd35cb4afde2e580326 Mon Sep 17 00:00:00 2001
From: Niels Vandekeybus <progster@gmail.com>
Date: Sat, 11 Jan 2020 13:50:54 +0100
Subject: [PATCH 1/2] on startup clean up temp graphs

this commit also optimizes the cleanup, by assuming the endpoint supports GRAPH operations
---
 app.js                | 15 ++++++++++-----
 repository/helpers.js | 28 +++++++++++-----------------
 2 files changed, 21 insertions(+), 22 deletions(-)

diff --git a/app.js b/app.js
index 138b98a..b39839d 100644
--- a/app.js
+++ b/app.js
@@ -3,14 +3,14 @@ const app = mu.app;
 const bodyParser = require('body-parser');
 const cors = require('cors');
 import {handleDelta} from './handle-deltas';
-import {directQuery, configurableQuery} from './repository/helpers';
+import { cleanup, directQuery, configurableQuery} from './repository/helpers';
 
 const fillInterneOverheid = require('./repository/fill-intern-overheid');
 const fillInterneRegering = require('./repository/fill-intern-regering');
 const fillPublic = require('./repository/fill-public');
 
 if (!process.env.DIRECT_ENDPOINT) {
-  throw new Error("DIRECT_ENDPOINT not set!")
+  throw new Error("DIRECT_ENDPOINT not set!");
 }
 
 app.use(bodyParser.json({ type: 'application/json' , limit: '50mb' }));
@@ -59,7 +59,7 @@ const builders = {
   }
 };
 
-const initialLoad = function(){
+async function initialLoad(){
   let toFillUp = '';
   if(process.env.RELOAD_ALL_DATA_ON_INIT == "true") {
     toFillUp = 'public,intern-overheid,intern-regering,minister';
@@ -88,10 +88,15 @@ const initialLoad = function(){
       }
     }
   };
-  fillAll();
+  await fillAll();
 };
 
-initialLoad();
+async function startup() {
+    await cleanup();
+    await initialLoad();
+}
+
+startup();
 
 const deltaBuilders = Object.assign({}, builders);
 delete deltaBuilders.public;
diff --git a/repository/helpers.js b/repository/helpers.js
index e2722c1..f0d5bc2 100644
--- a/repository/helpers.js
+++ b/repository/helpers.js
@@ -141,22 +141,16 @@ const addRelatedFiles = (queryEnv, extraFilters) => {
   return queryEnv.run(query, true);
 };
 
-const cleanup = (queryEnv) => {
-  // TODO should we not batch this delete?
-  const query = `
-  PREFIX ext: <http://mu.semte.ch/vocabularies/ext/>
-  DELETE {
-    GRAPH ?g {
-      ?s ?p ?o.
-    }
-  } WHERE {
-    GRAPH ?g {
-      ?g a ext:TempGraph .
-      ?s ?p ?o.
-    }
-  }`;
-  return queryEnv.run(query, true);
-};
+async function cleanup() {
+  const result = JSON.parse(await directQuery("PREFIX ext: <http://mu.semte.ch/vocabularies/ext/> SELECT ?g WHERE { GRAPH ?g { ?g a ext:TempGraph }}"));
+  if (result.results && result.results.bindings) {
+	  console.log(`found ${result.results.bindings.length} old temporary graphs, removing before going further`);
+	  for (let binding of result.results.bindings) {
+	    console.log(`dropping graph ${binding.g.value}`);
+	    await directQuery(`DROP SILENT GRAPH <${binding.g.value}>`);
+	  }
+  }
+}
 
 const fillOutDetailsOnVisibleItemsLeft = async (queryEnv) => {
   const result = await queryEnv.run(`
@@ -916,7 +910,7 @@ const configurableQuery = function(queryString, direct){
   }, queryString);
 };
 
-const directQuery = function(queryString){
+function directQuery(queryString){
   return configurableQuery(queryString, true);
 };
 

From e5ad8b34ece8f233cf3d52e02b6d8674cc634418 Mon Sep 17 00:00:00 2001
From: Niels Vandekeybus <progster@gmail.com>
Date: Sat, 11 Jan 2020 14:16:16 +0100
Subject: [PATCH 2/2] make sure to await cleanup (no longer returns a promise)

---
 repository/fill-intern-overheid.js | 6 ++----
 repository/fill-intern-regering.js | 8 +++-----
 2 files changed, 5 insertions(+), 9 deletions(-)

diff --git a/repository/fill-intern-overheid.js b/repository/fill-intern-overheid.js
index 2afe09e..cf3454a 100644
--- a/repository/fill-intern-overheid.js
+++ b/repository/fill-intern-overheid.js
@@ -92,15 +92,13 @@ export const fillUp = async (queryEnv, agendas) => {
     await runStage('copy temp to target', queryEnv, () => {
       return copyTempToTarget(queryEnv);
     });
-    await runStage('done filling overheid', queryEnv, () => {
-      return cleanup(queryEnv);
-    });
+    await runStage('done filling overheid', queryEnv, cleanup);
     const end = moment().utc();
     logStage(start,`fill overheid ended at: ${end.format()}`, targetGraph);
   } catch (e) {
     logStage(moment(), `${e}\n${e.stack}`, queryEnv.targetGraph);
     try {
-      cleanup(queryEnv);
+      await cleanup(queryEnv);
     }catch (e2) {
       console.log(e2);
     }
diff --git a/repository/fill-intern-regering.js b/repository/fill-intern-regering.js
index 84c97dd..813f146 100644
--- a/repository/fill-intern-regering.js
+++ b/repository/fill-intern-regering.js
@@ -82,17 +82,15 @@ export const fillUp = async (queryEnv, agendas) => {
     await runStage('copy temp to target', queryEnv, () => {
       return copyTempToTarget(queryEnv);
     });
-    await runStage('cleaned up', queryEnv, () => {
-      return cleanup(queryEnv);
-    });
+    await runStage('cleaned up', queryEnv, cleanup);
     const end = moment().utc();
     logStage(start, `fill regering ended at: ${end.format()}`, targetGraph);
   }catch (e) {
     logStage(moment(), `${e}\n${e.stack}`, queryEnv.targetGraph);
     try {
-      cleanup(queryEnv);
+      await cleanup(queryEnv);
     }catch (e2) {
       console.log(e2);
     }
   }
-};
\ No newline at end of file
+};