From 67aa397755b346f240831c88312576b53e2d3267 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Parmentier?= Date: Thu, 10 Oct 2024 16:59:11 +0200 Subject: [PATCH 1/9] chore: Ignore node_modules and bun.lockb --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 321c05d..db99706 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ */**/.venv */**/.env .idea +*/**/node_modules/ +*/**/bun.lockb From 1b93b2653c1c9fe89247e9aa16fd01a789f0e4c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Parmentier?= Date: Thu, 10 Oct 2024 16:59:49 +0200 Subject: [PATCH 2/9] chore: Spelling --- .vscode/settings.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 9a92850..d6c18dc 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -4,8 +4,10 @@ "editor.defaultFormatter": "ms-python.autopep8" }, "cSpell.words": [ + "loterre", "Loterre", + "skos", "venv", "webdav" ] -} \ No newline at end of file +} From 3bb8b707fbcc8f03c181c9c426773fa7b2eec627 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Parmentier?= Date: Thu, 10 Oct 2024 17:00:53 +0200 Subject: [PATCH 3/9] feat(loterre-resolvers): Add create-databases Create concepts, indexes databases for a vocabulary. --- loterre-resolvers/combine.ini | 62 ++++++++++++++++++++++++++ loterre-resolvers/compile.ini | 46 +++++++++++++++++++ loterre-resolvers/create-databases.ini | 38 ++++++++++++++++ loterre-resolvers/download.ini | 38 ++++++++++++++++ loterre-resolvers/package.json | 1 + 5 files changed, 185 insertions(+) create mode 100644 loterre-resolvers/combine.ini create mode 100644 loterre-resolvers/compile.ini create mode 100644 loterre-resolvers/create-databases.ini create mode 100644 loterre-resolvers/download.ini create mode 100644 loterre-resolvers/package.json diff --git a/loterre-resolvers/combine.ini b/loterre-resolvers/combine.ini new file mode 100644 index 0000000..50d0d6f --- /dev/null +++ b/loterre-resolvers/combine.ini @@ -0,0 +1,62 @@ +# Configuration du chemin de stockage des bases des données locales +[env] +path = location +value = fix(`${env('TMPDIR', '/tmp')}/databases/${env('weekNumber')}/${env('loterreID', 'noid')}`) + +# STEP 0 : On normalise la valeur à rechercher (de la même manière que l'index a été créé) +[assign] +path = value +value = fix(_.deburr(String(self.value).replace(/[`~!@#$%^&*()_|+\-=?;:'",.<>\{\}\[\]\\\/\s]/g, '').toLowerCase())) + +path = loterreID +value = env('loterreID') + +# primer contient l'URL à Télécharger (voir download.ini) +path = primer +value = env('primer') + +# STEP 1 : On vérifie si le vocabulaire est présent localement +[load] +location = env('location') +domain = fix('loterre-vocabulaires').append('-loaded') +path = loterreID +target = isAlreadyDownload + +# STEP 1.1 : s'il n'est pas présent localement +[swing] +test = get('isAlreadyDownload').isEmpty() + +# STEP 1.2 : on mémorise le fait que l'on va en faire une copie locale +[swing/save] +location = env('location') +domain = fix('loterre-vocabulaires').append('-loaded') +path = loterreID + +# STEP 1.3 : On télécharge la version distante en XML, on la convertit en JSON et on sauvegarde localement +[swing/singleton] +file = ./v1/download.cfg + +# STEP 1.4 : Avec la version JSON fraîchement générée on construit un index inversé +[swing/singleton] +file = ./v1/compile.cfg + +[swing/exchange] +value = self().pick(['id', 'value', 'state']) + +# STEP 2 : On cherche une correspondance dans l'index inversé +[load] +location = env('location') +domain = env('loterreID').prepend('loterre-').append('indexes') +path = value +target = result + +[replace] +path = id +value = get('id') +path = value +value = get('result.value', 'n/a') + +# STEP 2 : si l'index inversé nous donne une URI, avec on va chercher les informations associées +[expand] +path = value +file = ./v1/transcribe.cfg diff --git a/loterre-resolvers/compile.ini b/loterre-resolvers/compile.ini new file mode 100644 index 0000000..4c63c61 --- /dev/null +++ b/loterre-resolvers/compile.ini @@ -0,0 +1,46 @@ +[use] +plugin = basics +plugin = analytics +plugin = storage + +# build the reverse index +[cast] +location = env('location') +domain = env('loterreID').prepend('loterre-').append('concepts') + +[replace] +path = id.about +value = get('rdf$about') +# select all keys and remove duplicates to avoid ambiguities +path = value +value = get('indexKeys') + +path = id.score +value = get('score') + +# create an entry for all keys +[exploding] + +# invert id & value because of [exploding] which works only by value (not by id) +[replace] +path = id +value = get('value') +path = value +value = get('id.about') +path = score +value = get('id.score') + +[debug] +text = INDEX + +[save] +location = env('location') +domain = env('loterreID').prepend('loterre-').append('indexes') +path = id +score = get('score') + +[pop] + +[replace] +path = state +value = compile diff --git a/loterre-resolvers/create-databases.ini b/loterre-resolvers/create-databases.ini new file mode 100644 index 0000000..27227c3 --- /dev/null +++ b/loterre-resolvers/create-databases.ini @@ -0,0 +1,38 @@ +#!/usr/bin/env npx ezs + +# Input: [{ "loterreID": "216" }] + +# Usage: loterreID=2XK bunx ezs create-databases.ini <<< '[{"loterreId": "2XK"}]' + +[use] +# JSONParse +plugin = basics +# save +plugin = storage + +[env] +path = location +value = fix(`${env('PWD', '.')}/databases/${env('loterreID', 'noid')}`) + +[JSONParse] + +[replace] +path = loterreID +value = env('loterreID', _.get("loterreID")) + +[debug] +text = Avant save + +[save] +location = env('location') +domain = fix('loterre-vocabulaires').append('-loaded') +path = loterreID + +[delegate] +file = ./download.ini + +[delegate] +file = ./compile.ini + +[dump] +indent = true diff --git a/loterre-resolvers/download.ini b/loterre-resolvers/download.ini new file mode 100644 index 0000000..0cfb70d --- /dev/null +++ b/loterre-resolvers/download.ini @@ -0,0 +1,38 @@ +[use] +plugin = basics +plugin = analytics +plugin = storage + + +# Download the skos, parse it and save it +[exchange] +value = fix([`./data/${env('loterreID', 'noid')}.skos`]) + +[FILELoad] +[FILEMerge] + +[XMLParse] +separator = /rdf:RDF/skos:Concept + +# build indexKeys (remove duplicates to avoid ambiguities) +[assign] +path = indexKeys +value = fix(_.get(self, 'skos$prefLabel'), _.get(self, 'skos$altLabel'), _.get(self, 'skos$hiddenLabel')). \ + flatten().\ + filter(o => _.has(o, '$t')).\ + map(o => _.deburr(String(o.$t).replace(/[`~!@#$%^&*()_|+\-=?;:'",.<>\{\}\[\]\\\/\s]/g, '').toLowerCase())).\ + uniq() + +path = score +value = get('owl$deprecated.$t', 3).thru(x => Number(String(x).toLowerCase() === 'true' ? 1 : 2)) + +[save] +location = env('location') +domain = env('loterreID').prepend('loterre-').append('concepts') +path = rdf$about + +[pop] + +[replace] +path = state +value = download diff --git a/loterre-resolvers/package.json b/loterre-resolvers/package.json new file mode 100644 index 0000000..eedb66f --- /dev/null +++ b/loterre-resolvers/package.json @@ -0,0 +1 @@ +{ "dependencies": { "@ezs/analytics": "^2.3.2", "@ezs/basics": "^2.7.2", "@ezs/core": "^3.10.4", "@ezs/storage": "^3.2.3" } } \ No newline at end of file From c46b955e34602e192e0fb2d1207e3ce420dada60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Parmentier?= Date: Fri, 11 Oct 2024 14:38:55 +0200 Subject: [PATCH 4/9] chore: Spelling --- .vscode/settings.json | 2 ++ loterre-resolvers/databases/.gitignore | 2 ++ loterre-resolvers/dvc.lock | 43 ++++++++++++++++++++++++++ 3 files changed, 47 insertions(+) create mode 100644 loterre-resolvers/databases/.gitignore create mode 100644 loterre-resolvers/dvc.lock diff --git a/.vscode/settings.json b/.vscode/settings.json index d6c18dc..633c14e 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -4,8 +4,10 @@ "editor.defaultFormatter": "ms-python.autopep8" }, "cSpell.words": [ + "bunx", "loterre", "Loterre", + "NETSCITY", "skos", "venv", "webdav" diff --git a/loterre-resolvers/databases/.gitignore b/loterre-resolvers/databases/.gitignore new file mode 100644 index 0000000..c142ea2 --- /dev/null +++ b/loterre-resolvers/databases/.gitignore @@ -0,0 +1,2 @@ +/2XK +/2XK.tgz diff --git a/loterre-resolvers/dvc.lock b/loterre-resolvers/dvc.lock new file mode 100644 index 0000000..1708377 --- /dev/null +++ b/loterre-resolvers/dvc.lock @@ -0,0 +1,43 @@ +schema: '2.0' +stages: + db-2XK: + cmd: loterreID=2XK bunx ezs create-databases.ini <<< '[{}]' + deps: + - path: create-databases.ini + hash: md5 + md5: d24e36d0f1e6c881b412528ecbd78b4d + size: 604 + - path: data/2XK.skos + hash: md5 + md5: b9c11f0d64f73cbd1109b8f80a0850e7 + size: 28170863 + outs: + - path: databases/2XK + hash: md5 + md5: dbcded22fe5e3d87fd9f489c9d7799d0.dir + size: 36721558 + nfiles: 91907 + tgz-2XK: + cmd: loterreID=2XK bunx ezs create-databases.ini <<< '[{}]' && tar -czf databases/2XK.tgz + databases/2XK + deps: + - path: create-databases.ini + hash: md5 + md5: d24e36d0f1e6c881b412528ecbd78b4d + size: 604 + - path: data/2XK.skos + hash: md5 + md5: b9c11f0d64f73cbd1109b8f80a0850e7 + size: 28170863 + outs: + - path: databases/2XK.tgz + hash: md5 + md5: b75e4538339c861d7a1c5f9b9b4daf3d + size: 20186840 + all: + cmd: echo "all" + deps: + - path: databases/2XK.tgz + hash: md5 + md5: b75e4538339c861d7a1c5f9b9b4daf3d + size: 20186840 From b6ad29ab97bab8b02d48d1b8a88b484294725718 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Parmentier?= Date: Fri, 11 Oct 2024 14:40:22 +0200 Subject: [PATCH 5/9] feat(loterre-resolvers): Add dvc.yaml --- loterre-resolvers/dvc.yaml | 108 +++++++++++++++++++++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 loterre-resolvers/dvc.yaml diff --git a/loterre-resolvers/dvc.yaml b/loterre-resolvers/dvc.yaml new file mode 100644 index 0000000..5bfb8f0 --- /dev/null +++ b/loterre-resolvers/dvc.yaml @@ -0,0 +1,108 @@ +stages: + tgz-2XK: # Laboratoires + cmd: loterreID=2XK bunx ezs create-databases.ini <<< '[{}]' && tar -czf databases/2XK.tgz databases/2XK + deps: + - data/2XK.skos + - create-databases.ini + outs: + - databases/2XK.tgz + tgz-216: # Éducation + cmd: loterreID=216 bunx ezs create-databases.ini <<< '[{}]' && tar -czf databases/216.tgz databases/216 + deps: + - data/216.skos + - create-databases.ini + outs: + - databases/216.tgz + tgz-3JP: # Sociologie + cmd: loterreID=3JP bunx ezs create-databases.ini <<< '[{}]' && tar -czf databases/3JP.tgz databases/3JP + deps: + - data/3JP.skos + - create-databases.ini + outs: + - databases/3JP.tgz + tgz-73G: # Philosophie + cmd: loterreID=73G bunx ezs create-databases.ini <<< '[{}]' && tar -czf databases/73G.tgz databases/73G + deps: + - data/73G.skos + - create-databases.ini + outs: + - databases/73G.tgz + tgz-9SD: # Pays et subdivisions + cmd: loterreID=9SD bunx ezs create-databases.ini <<< '[{}]' && tar -czf databases/9SD.tgz databases/9SD + deps: + - data/9SD.skos + - create-databases.ini + outs: + - databases/9SD.tgz + tgz-BVM: # NETSCITY + cmd: loterreID=BVM bunx ezs create-databases.ini <<< '[{}]' && tar -czf databases/BVM.tgz databases/BVM + deps: + - data/BVM.skos + - create-databases.ini + outs: + - databases/BVM.tgz + tgz-D63: # Communes (France) + cmd: loterreID=D63 bunx ezs create-databases.ini <<< '[{}]' && tar -czf databases/D63.tgz databases/D63 + deps: + - data/D63.skos + - create-databases.ini + outs: + - databases/D63.tgz + tgz-JVR: # MESH + cmd: loterreID=JVR bunx ezs create-databases.ini <<< '[{}]' && tar -czf databases/JVR.tgz databases/JVR + deps: + - data/JVR.skos + - create-databases.ini + outs: + - databases/JVR.tgz + tgz-MDL: # Astronomie + cmd: loterreID=MDL bunx ezs create-databases.ini <<< '[{}]' && tar -czf databases/MDL.tgz databases/MDL + deps: + - data/MDL.skos + - create-databases.ini + outs: + - databases/MDL.tgz + tgz-N9J: # SAGE + cmd: loterreID=N9J bunx ezs create-databases.ini <<< '[{}]' && tar -czf databases/N9J.tgz databases/N9J + deps: + - data/N9J.skos + - create-databases.ini + outs: + - databases/N9J.tgz + tgz-P21: # Littérature + cmd: loterreID=P21 bunx ezs create-databases.ini <<< '[{}]' && tar -czf databases/P21.tgz databases/P21 + deps: + - data/P21.skos + - create-databases.ini + outs: + - databases/P21.tgz + tgz-P66: # Mémoire + cmd: loterreID=P66 bunx ezs create-databases.ini <<< '[{}]' && tar -czf databases/P66.tgz databases/P66 + deps: + - data/P66.skos + - create-databases.ini + outs: + - databases/P66.tgz + tgz-QX8: # Paléoclimatologie + cmd: loterreID=QX8 bunx ezs create-databases.ini <<< '[{}]' && tar -czf databases/QX8.tgz databases/QX8 + deps: + - data/QX8.skos + - create-databases.ini + outs: + - databases/QX8.tgz + all: + cmd: echo "all" + deps: + - databases/216.tgz + - databases/QX8.tgz + - databases/3JP.tgz + - databases/73G.tgz + - databases/9SD.tgz + - databases/P66.tgz + - databases/MDL.tgz + - databases/P21.tgz + - databases/BVM.tgz + - databases/2XK.tgz + - databases/N9J.tgz + - databases/D63.tgz + - databases/JVR.tgz From 7d9a25f9790e51ecf83ed1e6a29b82e3007cc621 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Parmentier?= Date: Fri, 11 Oct 2024 15:05:04 +0200 Subject: [PATCH 6/9] chore(loterre-resolvers): Ignore databases files As they are generated, and managed by dvc --- loterre-resolvers/.gitignore | 1 + loterre-resolvers/databases/.gitignore | 2 -- loterre-resolvers/dvc.lock | 17 +++++++++++++++++ 3 files changed, 18 insertions(+), 2 deletions(-) create mode 100644 loterre-resolvers/.gitignore delete mode 100644 loterre-resolvers/databases/.gitignore diff --git a/loterre-resolvers/.gitignore b/loterre-resolvers/.gitignore new file mode 100644 index 0000000..b8f17d9 --- /dev/null +++ b/loterre-resolvers/.gitignore @@ -0,0 +1 @@ +/databases diff --git a/loterre-resolvers/databases/.gitignore b/loterre-resolvers/databases/.gitignore deleted file mode 100644 index c142ea2..0000000 --- a/loterre-resolvers/databases/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -/2XK -/2XK.tgz diff --git a/loterre-resolvers/dvc.lock b/loterre-resolvers/dvc.lock index 1708377..fcc85d3 100644 --- a/loterre-resolvers/dvc.lock +++ b/loterre-resolvers/dvc.lock @@ -41,3 +41,20 @@ stages: hash: md5 md5: b75e4538339c861d7a1c5f9b9b4daf3d size: 20186840 + tgz-216: + cmd: loterreID=216 bunx ezs create-databases.ini <<< '[{}]' && tar -czf databases/216.tgz + databases/216 + deps: + - path: create-databases.ini + hash: md5 + md5: d24e36d0f1e6c881b412528ecbd78b4d + size: 604 + - path: data/216.skos + hash: md5 + md5: 3adb88d03412ed4e4569b71b14354d0c + size: 1936035 + outs: + - path: databases/216.tgz + hash: md5 + md5: b323829722c7d93f25d0199a3f013e36 + size: 3031833 From 1f2c3adc239aaf7b5de6904408657121ff23b2f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Parmentier?= Date: Fri, 11 Oct 2024 16:00:23 +0200 Subject: [PATCH 7/9] docs(loterre-resolvers): Update README --- loterre-resolvers/README.md | 4 +- loterre-resolvers/dvc.lock | 170 ++++++++++++++++++++++++++++++++++++ 2 files changed, 172 insertions(+), 2 deletions(-) diff --git a/loterre-resolvers/README.md b/loterre-resolvers/README.md index 69f870d..6b9f6ac 100644 --- a/loterre-resolvers/README.md +++ b/loterre-resolvers/README.md @@ -49,5 +49,5 @@ dvc config core.autostage true 1. se mettre dans ce répertoire: `cd loterre-resolvers` 2. activer l'environnement virtuel: `source .venv/bin/activate` 3. lancer le script: `./bin/get-files.sh` -4. ajouter les fichiers à DVC: `dvc add ./data/*.skos` -5. pousser les fichiers sur le remote: `dvc push` +4. lancer la création des bases à mettre à jour: `dvc repro` +5. pousser les fichiers: `dvc push` diff --git a/loterre-resolvers/dvc.lock b/loterre-resolvers/dvc.lock index fcc85d3..8e12238 100644 --- a/loterre-resolvers/dvc.lock +++ b/loterre-resolvers/dvc.lock @@ -58,3 +58,173 @@ stages: hash: md5 md5: b323829722c7d93f25d0199a3f013e36 size: 3031833 + tgz-QX8: + cmd: loterreID=QX8 bunx ezs create-databases.ini <<< '[{}]' && tar -czf databases/QX8.tgz + databases/QX8 + deps: + - path: create-databases.ini + hash: md5 + md5: d24e36d0f1e6c881b412528ecbd78b4d + size: 604 + - path: data/QX8.skos + hash: md5 + md5: 0e0210f8098cfa0d19918469480f60a0 + size: 2027326 + outs: + - path: databases/QX8.tgz + hash: md5 + md5: fb4a666fe1cb78e0ce556dd3d1e7da0b + size: 3014673 + tgz-3JP: + cmd: loterreID=3JP bunx ezs create-databases.ini <<< '[{}]' && tar -czf databases/3JP.tgz + databases/3JP + deps: + - path: create-databases.ini + hash: md5 + md5: d24e36d0f1e6c881b412528ecbd78b4d + size: 604 + - path: data/3JP.skos + hash: md5 + md5: 8a20304e8bf72fd72c82879d922dc173 + size: 2308578 + outs: + - path: databases/3JP.tgz + hash: md5 + md5: d2cb87a42a66d252b8eeb947307681e5 + size: 5574459 + tgz-73G: + cmd: loterreID=73G bunx ezs create-databases.ini <<< '[{}]' && tar -czf databases/73G.tgz + databases/73G + deps: + - path: create-databases.ini + hash: md5 + md5: d24e36d0f1e6c881b412528ecbd78b4d + size: 604 + - path: data/73G.skos + hash: md5 + md5: f77a6ade78f20824c4ac32b5a5a290be + size: 2496320 + outs: + - path: databases/73G.tgz + hash: md5 + md5: febefac830ee4398863627ee846f9dbd + size: 4684383 + tgz-9SD: + cmd: loterreID=9SD bunx ezs create-databases.ini <<< '[{}]' && tar -czf databases/9SD.tgz + databases/9SD + deps: + - path: create-databases.ini + hash: md5 + md5: d24e36d0f1e6c881b412528ecbd78b4d + size: 604 + - path: data/9SD.skos + hash: md5 + md5: 7a7ecf5ef02e07bfb0ca26af228a0ea9 + size: 3078291 + outs: + - path: databases/9SD.tgz + hash: md5 + md5: 1e9ab7831c0f9e9b959173a73acbd993 + size: 2399224 + tgz-P66: + cmd: loterreID=P66 bunx ezs create-databases.ini <<< '[{}]' && tar -czf databases/P66.tgz + databases/P66 + deps: + - path: create-databases.ini + hash: md5 + md5: d24e36d0f1e6c881b412528ecbd78b4d + size: 604 + - path: data/P66.skos + hash: md5 + md5: 0c9cfd915beba169a839ebfa90c20c76 + size: 3917649 + outs: + - path: databases/P66.tgz + hash: md5 + md5: 916dc97d57674d90c3d7eb3d269d1c6f + size: 3547544 + tgz-MDL: + cmd: loterreID=MDL bunx ezs create-databases.ini <<< '[{}]' && tar -czf databases/MDL.tgz + databases/MDL + deps: + - path: create-databases.ini + hash: md5 + md5: d24e36d0f1e6c881b412528ecbd78b4d + size: 604 + - path: data/MDL.skos + hash: md5 + md5: 03fa07edb29af5a50801302722c8fb16 + size: 5657134 + outs: + - path: databases/MDL.tgz + hash: md5 + md5: 8040c6e7287827d5591e590c89e0c302 + size: 7564247 + tgz-P21: + cmd: loterreID=P21 bunx ezs create-databases.ini <<< '[{}]' && tar -czf databases/P21.tgz + databases/P21 + deps: + - path: create-databases.ini + hash: md5 + md5: d24e36d0f1e6c881b412528ecbd78b4d + size: 604 + - path: data/P21.skos + hash: md5 + md5: 3d44f296cb995aedae522a50a33b2762 + size: 8495524 + outs: + - path: databases/P21.tgz + hash: md5 + md5: 04079323aa27e49150946bb5d08ebe04 + size: 12397953 + tgz-BVM: + cmd: loterreID=BVM bunx ezs create-databases.ini <<< '[{}]' && tar -czf databases/BVM.tgz + databases/BVM + deps: + - path: create-databases.ini + hash: md5 + md5: d24e36d0f1e6c881b412528ecbd78b4d + size: 604 + - path: data/BVM.skos + hash: md5 + md5: 08e1013cdf5b8d13f9da6c318a9e90c4 + size: 9635826 + outs: + - path: databases/BVM.tgz + hash: md5 + md5: fbcb94e65d6a01115c815595e3379077 + size: 4947959 + tgz-N9J: + cmd: loterreID=N9J bunx ezs create-databases.ini <<< '[{}]' && tar -czf databases/N9J.tgz + databases/N9J + deps: + - path: create-databases.ini + hash: md5 + md5: d24e36d0f1e6c881b412528ecbd78b4d + size: 604 + - path: data/N9J.skos + hash: md5 + md5: 4cbae496c05d3d2a534f52c059153d90 + size: 58246718 + outs: + - path: databases/N9J.tgz + hash: md5 + md5: dd15bb423c42e47cd1c2c63e0f0b1511 + size: 53493886 + tgz-D63: + cmd: loterreID=D63 bunx ezs create-databases.ini <<< '[{}]' && tar -czf databases/D63.tgz + databases/D63 + deps: + - path: create-databases.ini + hash: md5 + md5: d24e36d0f1e6c881b412528ecbd78b4d + size: 604 + - path: data/D63.skos + hash: md5 + md5: 990c26cb46fc302441f84ca46b6d0a28 + size: 92156390 + outs: + - path: databases/D63.tgz + hash: md5 + md5: c250a67ee3ea5e89d25159d3ebb8e27c + size: 38554176 From e69168fbf6fde7f0600a28de59787991e391052f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Parmentier?= Date: Mon, 14 Oct 2024 11:35:11 +0200 Subject: [PATCH 8/9] fix(loterre-resolvers): Biggest vocabulary didn't work --- loterre-resolvers/dvc.lock | 65 ++++++++++++++++++++++++++++++++++++++ loterre-resolvers/dvc.yaml | 16 +++++----- 2 files changed, 73 insertions(+), 8 deletions(-) diff --git a/loterre-resolvers/dvc.lock b/loterre-resolvers/dvc.lock index 8e12238..c0e687c 100644 --- a/loterre-resolvers/dvc.lock +++ b/loterre-resolvers/dvc.lock @@ -37,10 +37,58 @@ stages: all: cmd: echo "all" deps: + - path: databases/216.tgz + hash: md5 + md5: b323829722c7d93f25d0199a3f013e36 + size: 3031833 - path: databases/2XK.tgz hash: md5 md5: b75e4538339c861d7a1c5f9b9b4daf3d size: 20186840 + - path: databases/3JP.tgz + hash: md5 + md5: d2cb87a42a66d252b8eeb947307681e5 + size: 5574459 + - path: databases/73G.tgz + hash: md5 + md5: febefac830ee4398863627ee846f9dbd + size: 4684383 + - path: databases/9SD.tgz + hash: md5 + md5: 1e9ab7831c0f9e9b959173a73acbd993 + size: 2399224 + - path: databases/BVM.tgz + hash: md5 + md5: fbcb94e65d6a01115c815595e3379077 + size: 4947959 + - path: databases/D63.tgz + hash: md5 + md5: c250a67ee3ea5e89d25159d3ebb8e27c + size: 38554176 + - path: databases/JVR.tgz + hash: md5 + md5: 524e06061bbfba82bf08e4840a5aee65 + size: 111958290 + - path: databases/MDL.tgz + hash: md5 + md5: 8040c6e7287827d5591e590c89e0c302 + size: 7564247 + - path: databases/N9J.tgz + hash: md5 + md5: dd15bb423c42e47cd1c2c63e0f0b1511 + size: 53493886 + - path: databases/P21.tgz + hash: md5 + md5: 04079323aa27e49150946bb5d08ebe04 + size: 12397953 + - path: databases/P66.tgz + hash: md5 + md5: 916dc97d57674d90c3d7eb3d269d1c6f + size: 3547544 + - path: databases/QX8.tgz + hash: md5 + md5: fb4a666fe1cb78e0ce556dd3d1e7da0b + size: 3014673 tgz-216: cmd: loterreID=216 bunx ezs create-databases.ini <<< '[{}]' && tar -czf databases/216.tgz databases/216 @@ -228,3 +276,20 @@ stages: hash: md5 md5: c250a67ee3ea5e89d25159d3ebb8e27c size: 38554176 + tgz-JVR: + cmd: EZS_CONTINUE_DELAY=60 EZS_PIPELINE_DELAY=1200 loterreID=JVR bunx ezs create-databases.ini + <<< '[{}]' && tar -czf databases/JVR.tgz databases/JVR + deps: + - path: create-databases.ini + hash: md5 + md5: d24e36d0f1e6c881b412528ecbd78b4d + size: 604 + - path: data/JVR.skos + hash: md5 + md5: 4ef3370991e26d92054eeb66c5f75b42 + size: 117642710 + outs: + - path: databases/JVR.tgz + hash: md5 + md5: 524e06061bbfba82bf08e4840a5aee65 + size: 111958290 diff --git a/loterre-resolvers/dvc.yaml b/loterre-resolvers/dvc.yaml index 5bfb8f0..ce351bc 100644 --- a/loterre-resolvers/dvc.yaml +++ b/loterre-resolvers/dvc.yaml @@ -49,7 +49,7 @@ stages: outs: - databases/D63.tgz tgz-JVR: # MESH - cmd: loterreID=JVR bunx ezs create-databases.ini <<< '[{}]' && tar -czf databases/JVR.tgz databases/JVR + cmd: EZS_CONTINUE_DELAY=60 EZS_PIPELINE_DELAY=1200 loterreID=JVR bunx ezs create-databases.ini <<< '[{}]' && tar -czf databases/JVR.tgz databases/JVR deps: - data/JVR.skos - create-databases.ini @@ -76,13 +76,6 @@ stages: - create-databases.ini outs: - databases/P21.tgz - tgz-P66: # Mémoire - cmd: loterreID=P66 bunx ezs create-databases.ini <<< '[{}]' && tar -czf databases/P66.tgz databases/P66 - deps: - - data/P66.skos - - create-databases.ini - outs: - - databases/P66.tgz tgz-QX8: # Paléoclimatologie cmd: loterreID=QX8 bunx ezs create-databases.ini <<< '[{}]' && tar -czf databases/QX8.tgz databases/QX8 deps: @@ -90,6 +83,13 @@ stages: - create-databases.ini outs: - databases/QX8.tgz + tgz-P66: # Mémoire + cmd: loterreID=P66 bunx ezs create-databases.ini <<< '[{}]' && tar -czf databases/P66.tgz databases/P66 + deps: + - data/P66.skos + - create-databases.ini + outs: + - databases/P66.tgz all: cmd: echo "all" deps: From b484f9e548fb560c6f44d8110216487983480ebe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Parmentier?= Date: Mon, 14 Oct 2024 16:00:00 +0200 Subject: [PATCH 9/9] refactor(loterre-resolvers): Better dvc.yaml --- loterre-resolvers/README.md | 4 + loterre-resolvers/dvc.lock | 239 ++++++++++++++---------------------- loterre-resolvers/dvc.yaml | 131 ++++---------------- 3 files changed, 118 insertions(+), 256 deletions(-) diff --git a/loterre-resolvers/README.md b/loterre-resolvers/README.md index 6b9f6ac..8d41508 100644 --- a/loterre-resolvers/README.md +++ b/loterre-resolvers/README.md @@ -51,3 +51,7 @@ dvc config core.autostage true 3. lancer le script: `./bin/get-files.sh` 4. lancer la création des bases à mettre à jour: `dvc repro` 5. pousser les fichiers: `dvc push` + +> [!NOTE] +> Pour lancer la génération d'un vocabulaire en particulier (ex: `216`), +> utiliser `dvc repro tgz@216`. diff --git a/loterre-resolvers/dvc.lock b/loterre-resolvers/dvc.lock index c0e687c..d98dafb 100644 --- a/loterre-resolvers/dvc.lock +++ b/loterre-resolvers/dvc.lock @@ -1,97 +1,9 @@ schema: '2.0' stages: - db-2XK: - cmd: loterreID=2XK bunx ezs create-databases.ini <<< '[{}]' - deps: - - path: create-databases.ini - hash: md5 - md5: d24e36d0f1e6c881b412528ecbd78b4d - size: 604 - - path: data/2XK.skos - hash: md5 - md5: b9c11f0d64f73cbd1109b8f80a0850e7 - size: 28170863 - outs: - - path: databases/2XK - hash: md5 - md5: dbcded22fe5e3d87fd9f489c9d7799d0.dir - size: 36721558 - nfiles: 91907 - tgz-2XK: - cmd: loterreID=2XK bunx ezs create-databases.ini <<< '[{}]' && tar -czf databases/2XK.tgz - databases/2XK - deps: - - path: create-databases.ini - hash: md5 - md5: d24e36d0f1e6c881b412528ecbd78b4d - size: 604 - - path: data/2XK.skos - hash: md5 - md5: b9c11f0d64f73cbd1109b8f80a0850e7 - size: 28170863 - outs: - - path: databases/2XK.tgz - hash: md5 - md5: b75e4538339c861d7a1c5f9b9b4daf3d - size: 20186840 - all: - cmd: echo "all" - deps: - - path: databases/216.tgz - hash: md5 - md5: b323829722c7d93f25d0199a3f013e36 - size: 3031833 - - path: databases/2XK.tgz - hash: md5 - md5: b75e4538339c861d7a1c5f9b9b4daf3d - size: 20186840 - - path: databases/3JP.tgz - hash: md5 - md5: d2cb87a42a66d252b8eeb947307681e5 - size: 5574459 - - path: databases/73G.tgz - hash: md5 - md5: febefac830ee4398863627ee846f9dbd - size: 4684383 - - path: databases/9SD.tgz - hash: md5 - md5: 1e9ab7831c0f9e9b959173a73acbd993 - size: 2399224 - - path: databases/BVM.tgz - hash: md5 - md5: fbcb94e65d6a01115c815595e3379077 - size: 4947959 - - path: databases/D63.tgz - hash: md5 - md5: c250a67ee3ea5e89d25159d3ebb8e27c - size: 38554176 - - path: databases/JVR.tgz - hash: md5 - md5: 524e06061bbfba82bf08e4840a5aee65 - size: 111958290 - - path: databases/MDL.tgz - hash: md5 - md5: 8040c6e7287827d5591e590c89e0c302 - size: 7564247 - - path: databases/N9J.tgz - hash: md5 - md5: dd15bb423c42e47cd1c2c63e0f0b1511 - size: 53493886 - - path: databases/P21.tgz - hash: md5 - md5: 04079323aa27e49150946bb5d08ebe04 - size: 12397953 - - path: databases/P66.tgz - hash: md5 - md5: 916dc97d57674d90c3d7eb3d269d1c6f - size: 3547544 - - path: databases/QX8.tgz - hash: md5 - md5: fb4a666fe1cb78e0ce556dd3d1e7da0b - size: 3014673 - tgz-216: - cmd: loterreID=216 bunx ezs create-databases.ini <<< '[{}]' && tar -czf databases/216.tgz - databases/216 + tgz@216: + cmd: + - EZS_PIPELINE_DELAY=1200 loterreID=216 bunx ezs create-databases.ini <<< '[{}]' + - tar -czf databases/216.tgz databases/216 deps: - path: create-databases.ini hash: md5 @@ -104,11 +16,12 @@ stages: outs: - path: databases/216.tgz hash: md5 - md5: b323829722c7d93f25d0199a3f013e36 - size: 3031833 - tgz-QX8: - cmd: loterreID=QX8 bunx ezs create-databases.ini <<< '[{}]' && tar -czf databases/QX8.tgz - databases/QX8 + md5: 28ef3059d93b91020f524680d54d6b2f + size: 3343752 + tgz@QX8: + cmd: + - EZS_PIPELINE_DELAY=1200 loterreID=QX8 bunx ezs create-databases.ini <<< '[{}]' + - tar -czf databases/QX8.tgz databases/QX8 deps: - path: create-databases.ini hash: md5 @@ -121,11 +34,12 @@ stages: outs: - path: databases/QX8.tgz hash: md5 - md5: fb4a666fe1cb78e0ce556dd3d1e7da0b - size: 3014673 - tgz-3JP: - cmd: loterreID=3JP bunx ezs create-databases.ini <<< '[{}]' && tar -czf databases/3JP.tgz - databases/3JP + md5: d9a42f34fb27d39ad02c1811aedcd3e1 + size: 3095298 + tgz@3JP: + cmd: + - EZS_PIPELINE_DELAY=1200 loterreID=3JP bunx ezs create-databases.ini <<< '[{}]' + - tar -czf databases/3JP.tgz databases/3JP deps: - path: create-databases.ini hash: md5 @@ -138,11 +52,12 @@ stages: outs: - path: databases/3JP.tgz hash: md5 - md5: d2cb87a42a66d252b8eeb947307681e5 - size: 5574459 - tgz-73G: - cmd: loterreID=73G bunx ezs create-databases.ini <<< '[{}]' && tar -czf databases/73G.tgz - databases/73G + md5: 8dede3f3882c5238fe9fd19743f6de23 + size: 5786310 + tgz@73G: + cmd: + - EZS_PIPELINE_DELAY=1200 loterreID=73G bunx ezs create-databases.ini <<< '[{}]' + - tar -czf databases/73G.tgz databases/73G deps: - path: create-databases.ini hash: md5 @@ -155,11 +70,12 @@ stages: outs: - path: databases/73G.tgz hash: md5 - md5: febefac830ee4398863627ee846f9dbd - size: 4684383 - tgz-9SD: - cmd: loterreID=9SD bunx ezs create-databases.ini <<< '[{}]' && tar -czf databases/9SD.tgz - databases/9SD + md5: aa9291d2e8edabea9c4d87e1cad22bde + size: 4861450 + tgz@9SD: + cmd: + - EZS_PIPELINE_DELAY=1200 loterreID=9SD bunx ezs create-databases.ini <<< '[{}]' + - tar -czf databases/9SD.tgz databases/9SD deps: - path: create-databases.ini hash: md5 @@ -172,11 +88,12 @@ stages: outs: - path: databases/9SD.tgz hash: md5 - md5: 1e9ab7831c0f9e9b959173a73acbd993 - size: 2399224 - tgz-P66: - cmd: loterreID=P66 bunx ezs create-databases.ini <<< '[{}]' && tar -czf databases/P66.tgz - databases/P66 + md5: 2e8c5de44935cbb1ed3905d5334554f9 + size: 2447909 + tgz@P66: + cmd: + - EZS_PIPELINE_DELAY=1200 loterreID=P66 bunx ezs create-databases.ini <<< '[{}]' + - tar -czf databases/P66.tgz databases/P66 deps: - path: create-databases.ini hash: md5 @@ -189,11 +106,12 @@ stages: outs: - path: databases/P66.tgz hash: md5 - md5: 916dc97d57674d90c3d7eb3d269d1c6f - size: 3547544 - tgz-MDL: - cmd: loterreID=MDL bunx ezs create-databases.ini <<< '[{}]' && tar -czf databases/MDL.tgz - databases/MDL + md5: ec4329197f7cde2ce323cad10a06d9fb + size: 3599562 + tgz@MDL: + cmd: + - EZS_PIPELINE_DELAY=1200 loterreID=MDL bunx ezs create-databases.ini <<< '[{}]' + - tar -czf databases/MDL.tgz databases/MDL deps: - path: create-databases.ini hash: md5 @@ -206,11 +124,12 @@ stages: outs: - path: databases/MDL.tgz hash: md5 - md5: 8040c6e7287827d5591e590c89e0c302 - size: 7564247 - tgz-P21: - cmd: loterreID=P21 bunx ezs create-databases.ini <<< '[{}]' && tar -czf databases/P21.tgz - databases/P21 + md5: 3db94dbb55139947b21a466f5f1dbf8b + size: 7664643 + tgz@P21: + cmd: + - EZS_PIPELINE_DELAY=1200 loterreID=P21 bunx ezs create-databases.ini <<< '[{}]' + - tar -czf databases/P21.tgz databases/P21 deps: - path: create-databases.ini hash: md5 @@ -223,11 +142,12 @@ stages: outs: - path: databases/P21.tgz hash: md5 - md5: 04079323aa27e49150946bb5d08ebe04 - size: 12397953 - tgz-BVM: - cmd: loterreID=BVM bunx ezs create-databases.ini <<< '[{}]' && tar -czf databases/BVM.tgz - databases/BVM + md5: ed22963ea4f4470fb4d293bc5f312a49 + size: 12846318 + tgz@BVM: + cmd: + - EZS_PIPELINE_DELAY=1200 loterreID=BVM bunx ezs create-databases.ini <<< '[{}]' + - tar -czf databases/BVM.tgz databases/BVM deps: - path: create-databases.ini hash: md5 @@ -240,11 +160,30 @@ stages: outs: - path: databases/BVM.tgz hash: md5 - md5: fbcb94e65d6a01115c815595e3379077 - size: 4947959 - tgz-N9J: - cmd: loterreID=N9J bunx ezs create-databases.ini <<< '[{}]' && tar -czf databases/N9J.tgz - databases/N9J + md5: 3f8a4c3ceba8f3c45dc91be3d9ba948f + size: 5085076 + tgz@2XK: + cmd: + - EZS_PIPELINE_DELAY=1200 loterreID=2XK bunx ezs create-databases.ini <<< '[{}]' + - tar -czf databases/2XK.tgz databases/2XK + deps: + - path: create-databases.ini + hash: md5 + md5: d24e36d0f1e6c881b412528ecbd78b4d + size: 604 + - path: data/2XK.skos + hash: md5 + md5: b9c11f0d64f73cbd1109b8f80a0850e7 + size: 28170863 + outs: + - path: databases/2XK.tgz + hash: md5 + md5: ebf4d581d27c9db024e1debf9ec849a1 + size: 21391769 + tgz@N9J: + cmd: + - EZS_PIPELINE_DELAY=1200 loterreID=N9J bunx ezs create-databases.ini <<< '[{}]' + - tar -czf databases/N9J.tgz databases/N9J deps: - path: create-databases.ini hash: md5 @@ -257,11 +196,12 @@ stages: outs: - path: databases/N9J.tgz hash: md5 - md5: dd15bb423c42e47cd1c2c63e0f0b1511 - size: 53493886 - tgz-D63: - cmd: loterreID=D63 bunx ezs create-databases.ini <<< '[{}]' && tar -czf databases/D63.tgz - databases/D63 + md5: 2b78b493c8a108597296503e857e3c93 + size: 56017597 + tgz@D63: + cmd: + - EZS_PIPELINE_DELAY=1200 loterreID=D63 bunx ezs create-databases.ini <<< '[{}]' + - tar -czf databases/D63.tgz databases/D63 deps: - path: create-databases.ini hash: md5 @@ -274,11 +214,12 @@ stages: outs: - path: databases/D63.tgz hash: md5 - md5: c250a67ee3ea5e89d25159d3ebb8e27c - size: 38554176 - tgz-JVR: - cmd: EZS_CONTINUE_DELAY=60 EZS_PIPELINE_DELAY=1200 loterreID=JVR bunx ezs create-databases.ini - <<< '[{}]' && tar -czf databases/JVR.tgz databases/JVR + md5: bac27a547b6bcacb5b7c0eeb60a25825 + size: 40158038 + tgz@JVR: + cmd: + - EZS_PIPELINE_DELAY=1200 loterreID=JVR bunx ezs create-databases.ini <<< '[{}]' + - tar -czf databases/JVR.tgz databases/JVR deps: - path: create-databases.ini hash: md5 @@ -291,5 +232,5 @@ stages: outs: - path: databases/JVR.tgz hash: md5 - md5: 524e06061bbfba82bf08e4840a5aee65 - size: 111958290 + md5: 3df5c6acfb417d2549721c61af666edf + size: 113042022 diff --git a/loterre-resolvers/dvc.yaml b/loterre-resolvers/dvc.yaml index ce351bc..f71d731 100644 --- a/loterre-resolvers/dvc.yaml +++ b/loterre-resolvers/dvc.yaml @@ -1,108 +1,25 @@ stages: - tgz-2XK: # Laboratoires - cmd: loterreID=2XK bunx ezs create-databases.ini <<< '[{}]' && tar -czf databases/2XK.tgz databases/2XK - deps: - - data/2XK.skos - - create-databases.ini - outs: - - databases/2XK.tgz - tgz-216: # Éducation - cmd: loterreID=216 bunx ezs create-databases.ini <<< '[{}]' && tar -czf databases/216.tgz databases/216 - deps: - - data/216.skos - - create-databases.ini - outs: - - databases/216.tgz - tgz-3JP: # Sociologie - cmd: loterreID=3JP bunx ezs create-databases.ini <<< '[{}]' && tar -czf databases/3JP.tgz databases/3JP - deps: - - data/3JP.skos - - create-databases.ini - outs: - - databases/3JP.tgz - tgz-73G: # Philosophie - cmd: loterreID=73G bunx ezs create-databases.ini <<< '[{}]' && tar -czf databases/73G.tgz databases/73G - deps: - - data/73G.skos - - create-databases.ini - outs: - - databases/73G.tgz - tgz-9SD: # Pays et subdivisions - cmd: loterreID=9SD bunx ezs create-databases.ini <<< '[{}]' && tar -czf databases/9SD.tgz databases/9SD - deps: - - data/9SD.skos - - create-databases.ini - outs: - - databases/9SD.tgz - tgz-BVM: # NETSCITY - cmd: loterreID=BVM bunx ezs create-databases.ini <<< '[{}]' && tar -czf databases/BVM.tgz databases/BVM - deps: - - data/BVM.skos - - create-databases.ini - outs: - - databases/BVM.tgz - tgz-D63: # Communes (France) - cmd: loterreID=D63 bunx ezs create-databases.ini <<< '[{}]' && tar -czf databases/D63.tgz databases/D63 - deps: - - data/D63.skos - - create-databases.ini - outs: - - databases/D63.tgz - tgz-JVR: # MESH - cmd: EZS_CONTINUE_DELAY=60 EZS_PIPELINE_DELAY=1200 loterreID=JVR bunx ezs create-databases.ini <<< '[{}]' && tar -czf databases/JVR.tgz databases/JVR - deps: - - data/JVR.skos - - create-databases.ini - outs: - - databases/JVR.tgz - tgz-MDL: # Astronomie - cmd: loterreID=MDL bunx ezs create-databases.ini <<< '[{}]' && tar -czf databases/MDL.tgz databases/MDL - deps: - - data/MDL.skos - - create-databases.ini - outs: - - databases/MDL.tgz - tgz-N9J: # SAGE - cmd: loterreID=N9J bunx ezs create-databases.ini <<< '[{}]' && tar -czf databases/N9J.tgz databases/N9J - deps: - - data/N9J.skos - - create-databases.ini - outs: - - databases/N9J.tgz - tgz-P21: # Littérature - cmd: loterreID=P21 bunx ezs create-databases.ini <<< '[{}]' && tar -czf databases/P21.tgz databases/P21 - deps: - - data/P21.skos - - create-databases.ini - outs: - - databases/P21.tgz - tgz-QX8: # Paléoclimatologie - cmd: loterreID=QX8 bunx ezs create-databases.ini <<< '[{}]' && tar -czf databases/QX8.tgz databases/QX8 - deps: - - data/QX8.skos - - create-databases.ini - outs: - - databases/QX8.tgz - tgz-P66: # Mémoire - cmd: loterreID=P66 bunx ezs create-databases.ini <<< '[{}]' && tar -czf databases/P66.tgz databases/P66 - deps: - - data/P66.skos - - create-databases.ini - outs: - - databases/P66.tgz - all: - cmd: echo "all" - deps: - - databases/216.tgz - - databases/QX8.tgz - - databases/3JP.tgz - - databases/73G.tgz - - databases/9SD.tgz - - databases/P66.tgz - - databases/MDL.tgz - - databases/P21.tgz - - databases/BVM.tgz - - databases/2XK.tgz - - databases/N9J.tgz - - databases/D63.tgz - - databases/JVR.tgz + tgz: + foreach: + - 216 # Éducation + - QX8 # Paléoclimatologie + - 3JP # Sociologie + - 73G # Philosophie + - 9SD # Pays et subdivisions + - P66 # Mémoire + - MDL # Astronomie + - P21 # Littérature + - BVM # NETSCITY + - 2XK # Laboratoires + - N9J # SAGE + - D63 # Communes (France) + - JVR # MESH + do: + cmd: + - EZS_PIPELINE_DELAY=1200 loterreID=${item} bunx ezs create-databases.ini <<< '[{}]' + - tar -czf databases/${item}.tgz databases/${item} + deps: + - data/${item}.skos + - create-databases.ini + outs: + - databases/${item}.tgz