From 624e6e4c2ca5f4a91af6be4e32bac6040ce436a9 Mon Sep 17 00:00:00 2001 From: theresa Date: Mon, 29 Jul 2024 11:49:44 +0200 Subject: [PATCH 01/33] #3303 added pid field to variable class --- .../configuration/translations-de.js | 4 ++ .../configuration/translations-en.js | 4 ++ .../common/domain/util/Patterns.java | 1 + .../variablemanagement/domain/Variable.java | 45 ++++++++++++------- 4 files changed, 38 insertions(+), 16 deletions(-) diff --git a/mdm-frontend/src/app/legacy/variablemanagement/configuration/translations-de.js b/mdm-frontend/src/app/legacy/variablemanagement/configuration/translations-de.js index adc1fac40b4..4e06cbb0218 100644 --- a/mdm-frontend/src/app/legacy/variablemanagement/configuration/translations-de.js +++ b/mdm-frontend/src/app/legacy/variablemanagement/configuration/translations-de.js @@ -308,6 +308,10 @@ angular.module('metadatamanagementApp').config([ 'variable-has-invalid-data-set-id': 'Die Variable {{id}} referenziert auf einen unbekannten Datensatz ({{toBereferenzedId}}).', 'variable-has-invalid-question-id': 'Die Variable {{id}} referenziert auf einen unbekannte Frage ({{toBereferenzedId}}).', 'variable-survey-ids-are-not-consistent-with-data-set': 'Die Variable {{id}} referenziert auf andere Erhebungen als ihr Datensatz {{toBereferenzedId}}.' + }, + "pid": { + "pattern": "Der angegebene Wert stimmt nicht mit dem benötigten Format der PID überein.", + "size": "Der angegebene Wert darf nicht länger als 512 Zeichen sein." } }, 'edit': { diff --git a/mdm-frontend/src/app/legacy/variablemanagement/configuration/translations-en.js b/mdm-frontend/src/app/legacy/variablemanagement/configuration/translations-en.js index 2fa29936859..ebbff235cb2 100644 --- a/mdm-frontend/src/app/legacy/variablemanagement/configuration/translations-en.js +++ b/mdm-frontend/src/app/legacy/variablemanagement/configuration/translations-en.js @@ -308,6 +308,10 @@ angular.module('metadatamanagementApp').config([ 'variable-has-invalid-data-set-id': 'The Variable {{id}} references to an unknown Data Set ({{toBereferenzedId}}).', 'variable-has-invalid-question-id': 'The Variable {{id}} references to an unknown Question ({{toBereferenzedId}}).', 'variable-survey-ids-are-not-consistent-with-data-set': 'The Variable {{id}} references different surveys than its Data Set {{toBereferenzedId}}.' + }, + "pid": { + "pattern": "The provided value does not match the PID pattern.", + "size": "The provided value must not contain more than 512 characters." } }, 'edit': { diff --git a/src/main/java/eu/dzhw/fdz/metadatamanagement/common/domain/util/Patterns.java b/src/main/java/eu/dzhw/fdz/metadatamanagement/common/domain/util/Patterns.java index 79f9b27b060..ffd3b7162e3 100644 --- a/src/main/java/eu/dzhw/fdz/metadatamanagement/common/domain/util/Patterns.java +++ b/src/main/java/eu/dzhw/fdz/metadatamanagement/common/domain/util/Patterns.java @@ -15,6 +15,7 @@ public interface Patterns { String GERMAN_ALPHANUMERIC_WITH_UNDERSCORE_AND_MINUS_AND_DOT = "^[_A-Za-z0-9äöüÄÖÜß\\-\\.]*$"; String DOI = "^https:\\/\\/doi.org\\/([_A-Za-z0-9äöüÄÖÜß\\-\\/\\:.]{1,}$)"; + String PID = "^21\\.T11998\\/dzhw:([_A-Za-z0-9äöüÄÖÜß\\-\\/\\:.]{1,}$)"; String GERMAN_ALPHANUMERIC_WITH_UNDERSCORE_AND_MINUS_AND_DOT_AND_DOLLAR = "^[_A-Za-z0-9äöüÄÖÜß\\-\\.\\$]*$"; String NO_WHITESPACE = "^[^\\s]*$"; diff --git a/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/domain/Variable.java b/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/domain/Variable.java index b1fde177665..7e63b8411ee 100644 --- a/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/domain/Variable.java +++ b/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/domain/Variable.java @@ -117,7 +117,7 @@ public class Variable extends AbstractShadowableRdcDomainObject { /** * The id of the variable which uniquely identifies the variable in this application. - * + * * The id must not be empty and must be of the form * var-{{dataAcquisitionProjectId}}-ds{{dataSetNumber}}-{{name}}$. The id must not contain more * than 512 characters. @@ -134,9 +134,22 @@ public class Variable extends AbstractShadowableRdcDomainObject { @Indexed private String masterId; + /** + * The PID of the variable. + * + * Must not contain more than 512 characters. + * + * Must match the pattern of a PID "21.T11998/dzhw:{DataPackageID}_{VariableID}:{version} + */ + @Size(max = StringLengths.MEDIUM, message = "variable-management.error.pid.size") + @Pattern( + message = "variable-management.error.pid.pattern", + regexp = Patterns.PID) + private String pid; + /** * The id of the {@link DataAcquisitionProject} to which this variable belongs. - * + * * The dataAcquisitionProjectId must not be empty. */ @Indexed @@ -145,7 +158,7 @@ public class Variable extends AbstractShadowableRdcDomainObject { /** * The name of the variable as it is used in the {@link DataSet}. - * + * * It must not be empty and must be unique in the {@link DataSet}. It must contain only * alphanumeric (english) characters and "_". The first character must not be a number. It must * not contain more than 32 characters. @@ -158,7 +171,7 @@ public class Variable extends AbstractShadowableRdcDomainObject { /** * The label of the variable should describe its content. - * + * * It must be specified in at least one language and it must not contain more than 512 characters. */ @NotNull(message = "variable-management.error.variable.label.not-null") @@ -169,7 +182,7 @@ public class Variable extends AbstractShadowableRdcDomainObject { /** * Arbitrary additional text for this variable. Markdown is supported. - * + * * Must not contain more than 2048 characters. */ @I18nStringSize(max = StringLengths.LARGE, @@ -178,7 +191,7 @@ public class Variable extends AbstractShadowableRdcDomainObject { /** * The id of the {@link DataSet} to which this variable belongs. - * + * * Must not be empty. */ @Indexed @@ -187,7 +200,7 @@ public class Variable extends AbstractShadowableRdcDomainObject { /** * The number of the {@link DataSet} to which this variable belongs. - * + * * Must not be empty. */ @NotNull(message = "variable-management.error.variable.data-set-number-not-null") @@ -195,7 +208,7 @@ public class Variable extends AbstractShadowableRdcDomainObject { /** * The technical type which the {@link ValidResponse}s have. - * + * * Must be one of {@link DataTypes} and must not be empty. */ @NotNull(message = "variable-management.error.variable.data-type.not-null") @@ -205,7 +218,7 @@ public class Variable extends AbstractShadowableRdcDomainObject { /** * Associated with each data type is a storage type. For instance numerics can be stored as * integer or double. - * + * * Must be one of {@link StorageTypes} and must not be empty. */ @NotNull(message = "variable-management.error.variable.storage-type.not-null") @@ -216,7 +229,7 @@ public class Variable extends AbstractShadowableRdcDomainObject { * The scale level (or level of measurement) classifies the nature of information within the * values assigned to this variable ({@link ValidResponse}s). It determines which mathematical * operations can be performed with the values. - * + * * It must be one of {@link ScaleLevels} and must not be empty. If the data type of this variable * is {@link DataTypes#DATE} then the ScaleLevel must be {@link ScaleLevels#ORDINAL}, * {@link ScaleLevels#INTERVAL} or {@link ScaleLevels#NOMINAL}. @@ -228,7 +241,7 @@ public class Variable extends AbstractShadowableRdcDomainObject { /** * The access way of this variable. Depends on the sensitivity of the data and describes how the * data user will be able to work with the data. - * + * * Must not be empty and be one of {@link AccessWays}. */ // checks for min size too. @@ -245,7 +258,7 @@ public class Variable extends AbstractShadowableRdcDomainObject { /** * The index in the {@link DataSet} of this variable. Used for sorting the variables of this * {@link DataSet} and for displaying successors and predecessors of this variable. - * + * * Must not be empty and the successor of this variable must have indexInDataSet incremented by * one. */ @@ -254,7 +267,7 @@ public class Variable extends AbstractShadowableRdcDomainObject { /** * List of numbers of {@link Survey}s which have been conducted to create this variable. - * + * * Must not be empty. */ @NotEmpty(message = "variable-management.error.variable.survey-numbers-not-empty") @@ -263,7 +276,7 @@ public class Variable extends AbstractShadowableRdcDomainObject { /** * Identifier used to group variables within this {@link DataSet} which measure the same across * multiple waves. - * + * * Must be of the form {{dataAcquisitionProjectId}}-ds{{dataSetNumber}}-{{string}}$. Must not * contain more than 512 characters and must contain only (german) alphanumeric characters and "_" * and "-". @@ -277,7 +290,7 @@ public class Variable extends AbstractShadowableRdcDomainObject { /** * Identifier used to group variables within this {@link DataSet} which have been derived from * each other. For instance one variable might be an aggregated version of the other. - * + * * Must be of the form {{dataAcquisitionProjectId}}-ds{{dataSetNumber}}-{{string}}$. Must not * contain more than 512 characters and must contain only (german) alphanumeric characters and "_" * and "-". @@ -333,7 +346,7 @@ public class Variable extends AbstractShadowableRdcDomainObject { /** * List of ids of {@link Survey}s which have been conducted to create this variable. - * + * * Must not be empty. */ @Indexed From e326ea3991f724283d614875e0069abb51afb1e5 Mon Sep 17 00:00:00 2001 From: theresa Date: Mon, 29 Jul 2024 12:10:24 +0200 Subject: [PATCH 02/33] #3303 switched quotation marks --- .../variablemanagement/configuration/translations-de.js | 6 +++--- .../variablemanagement/configuration/translations-en.js | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/mdm-frontend/src/app/legacy/variablemanagement/configuration/translations-de.js b/mdm-frontend/src/app/legacy/variablemanagement/configuration/translations-de.js index 4e06cbb0218..f124f9f90b1 100644 --- a/mdm-frontend/src/app/legacy/variablemanagement/configuration/translations-de.js +++ b/mdm-frontend/src/app/legacy/variablemanagement/configuration/translations-de.js @@ -309,9 +309,9 @@ angular.module('metadatamanagementApp').config([ 'variable-has-invalid-question-id': 'Die Variable {{id}} referenziert auf einen unbekannte Frage ({{toBereferenzedId}}).', 'variable-survey-ids-are-not-consistent-with-data-set': 'Die Variable {{id}} referenziert auf andere Erhebungen als ihr Datensatz {{toBereferenzedId}}.' }, - "pid": { - "pattern": "Der angegebene Wert stimmt nicht mit dem benötigten Format der PID überein.", - "size": "Der angegebene Wert darf nicht länger als 512 Zeichen sein." + 'pid': { + 'pattern': 'Der angegebene Wert stimmt nicht mit dem benötigten Format der PID überein.', + 'size': 'Der angegebene Wert darf nicht länger als 512 Zeichen sein.' } }, 'edit': { diff --git a/mdm-frontend/src/app/legacy/variablemanagement/configuration/translations-en.js b/mdm-frontend/src/app/legacy/variablemanagement/configuration/translations-en.js index ebbff235cb2..a912336b482 100644 --- a/mdm-frontend/src/app/legacy/variablemanagement/configuration/translations-en.js +++ b/mdm-frontend/src/app/legacy/variablemanagement/configuration/translations-en.js @@ -309,9 +309,9 @@ angular.module('metadatamanagementApp').config([ 'variable-has-invalid-question-id': 'The Variable {{id}} references to an unknown Question ({{toBereferenzedId}}).', 'variable-survey-ids-are-not-consistent-with-data-set': 'The Variable {{id}} references different surveys than its Data Set {{toBereferenzedId}}.' }, - "pid": { - "pattern": "The provided value does not match the PID pattern.", - "size": "The provided value must not contain more than 512 characters." + 'pid': { + 'pattern': 'The provided value does not match the PID pattern.', + 'size': 'The provided value must not contain more than 512 characters.' } }, 'edit': { From c217c9efcfd6e85e2d09cf365731545a58c80795 Mon Sep 17 00:00:00 2001 From: theresa Date: Mon, 29 Jul 2024 14:48:29 +0200 Subject: [PATCH 03/33] #3303 added REST endpoint for JSON export --- .../ExportAllVariablesResourceController.java | 49 ++++++++++++++++ .../service/VariableManagementService.java | 57 +++++++++++++++---- 2 files changed, 96 insertions(+), 10 deletions(-) create mode 100644 src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/rest/ExportAllVariablesResourceController.java diff --git a/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/rest/ExportAllVariablesResourceController.java b/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/rest/ExportAllVariablesResourceController.java new file mode 100644 index 00000000000..294423242fb --- /dev/null +++ b/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/rest/ExportAllVariablesResourceController.java @@ -0,0 +1,49 @@ +package eu.dzhw.fdz.metadatamanagement.variablemanagement.rest; + +import java.io.IOException; +import java.rmi.server.ExportException; + +import javax.persistence.PersistenceException; + +import eu.dzhw.fdz.metadatamanagement.projectmanagement.service.DataAcquisitionProjectManagementService; +import eu.dzhw.fdz.metadatamanagement.usermanagement.security.AuthoritiesConstants; +import eu.dzhw.fdz.metadatamanagement.variablemanagement.service.VariableManagementService; +import lombok.RequiredArgsConstructor; + +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.security.access.annotation.Secured; +import org.springframework.stereotype.Controller; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.ResponseBody; + +@Controller +@RequestMapping("/api") +@RequiredArgsConstructor +public class ExportAllVariablesResourceController { + + private final VariableManagementService variablesManagementService; + + /** + * export all variables for PID registration. + * + * @return a JSON file + */ + @PostMapping(value = "/variables/exportAll", produces = MediaType.APPLICATION_JSON_VALUE) + @ResponseBody + @Secured(value = {AuthoritiesConstants.ADMIN}) + public ResponseEntity exportAllVariables() { + ResponseEntity response = new ResponseEntity<>(null, null, HttpStatus.NOT_FOUND); + try { + response = this.variablesManagementService.exportVariablesAsJSON(); + } catch (PersistenceException | IOException ex) { + return response; + } + + return response; + } + + +} diff --git a/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/service/VariableManagementService.java b/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/service/VariableManagementService.java index ad331fbf77a..8972a92fc7f 100644 --- a/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/service/VariableManagementService.java +++ b/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/service/VariableManagementService.java @@ -1,17 +1,30 @@ package eu.dzhw.fdz.metadatamanagement.variablemanagement.service; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.rmi.server.ExportException; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; +import org.springframework.core.io.ByteArrayResource; import org.springframework.data.rest.core.annotation.HandleAfterCreate; import org.springframework.data.rest.core.annotation.HandleAfterDelete; import org.springframework.data.rest.core.annotation.HandleAfterSave; import org.springframework.data.rest.core.annotation.RepositoryEventHandler; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; import org.springframework.security.access.annotation.Secured; import org.springframework.stereotype.Service; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import eu.dzhw.fdz.metadatamanagement.common.service.CrudService; import eu.dzhw.fdz.metadatamanagement.conceptmanagement.domain.Concept; import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.DataPackage; @@ -31,7 +44,7 @@ /** * Service for managing the domain object/aggregate {@link Variable}. - * + * * @author René Reitmann */ @Service @@ -49,7 +62,7 @@ public class VariableManagementService implements CrudService { /** * Delete all variables when the dataAcquisitionProject was deleted. - * + * * @param dataAcquisitionProject the dataAcquisitionProject which has been deleted. */ @HandleAfterDelete @@ -59,7 +72,7 @@ public void onDataAcquisitionProjectDeleted(DataAcquisitionProject dataAcquisiti /** * Update all variables of the project, when the project is released. - * + * * @param dataAcquisitionProject the changed project */ @HandleAfterSave @@ -72,7 +85,7 @@ public void onDataAcquisitionProjectUpdated(DataAcquisitionProject dataAcquisiti /** * A service method for deletion of variables within a data acquisition project. - * + * * @param dataAcquisitionProjectId the id for to the data acquisition project. */ @Secured(value = {AuthoritiesConstants.PUBLISHER, AuthoritiesConstants.DATA_PROVIDER}) @@ -88,7 +101,7 @@ public void deleteAllVariablesByProjectId(String dataAcquisitionProjectId) { /** * Enqueue update of variable search documents when the data set is changed. - * + * * @param dataSet the updated, created or deleted data set. */ @HandleAfterCreate @@ -102,7 +115,7 @@ public void onDataSetChanged(DataSet dataSet) { /** * Enqueue update of variable search documents when the dataPackage is changed. - * + * * @param dataPackage the updated, created or deleted dataPackage. */ @HandleAfterCreate @@ -116,7 +129,7 @@ public void onDataPackageChanged(DataPackage dataPackage) { /** * Enqueue update of variable search documents when the instrument is changed. - * + * * @param instrument the updated, created or deleted instrument. */ @HandleAfterCreate @@ -130,7 +143,7 @@ public void onInstrumentChanged(Instrument instrument) { /** * Enqueue update of variable search documents when the question is changed. - * + * * @param question the updated, created or deleted question. */ @HandleAfterCreate @@ -144,7 +157,7 @@ public void onQuestionChanged(Question question) { /** * Enqueue update of variable search documents when the survey is updated. - * + * * @param survey the updated, created or deleted survey. */ @HandleAfterCreate @@ -158,7 +171,7 @@ public void onSurveyChanged(Survey survey) { /** * Enqueue update of variable search documents when the concept is changed. - * + * * @param concept the updated, created or deleted concept. */ @HandleAfterCreate @@ -202,4 +215,28 @@ public Optional read(String id) { public Optional readSearchDocument(String id) { return crudHelper.readSearchDocument(id); } + + /** + * Writes all variable data according to the PID metadata schema to a JSON file. + * @return the JSON file + * @throws IOException + */ + public ResponseEntity exportVariablesAsJSON() throws IOException { + ObjectMapper objectMapper = new ObjectMapper(); + ArrayNode jsonNode = objectMapper.createArrayNode(); + File tempFile = new File("tempfile.json"); + tempFile.deleteOnExit(); + objectMapper.writeValue(tempFile, jsonNode); + Path path = Paths.get(tempFile.getAbsolutePath()); + try { + ByteArrayResource resource = new ByteArrayResource(Files.readAllBytes(path)); + HttpHeaders headers = new HttpHeaders(); + headers.add("Content-Disposition", "attachment; filename=Variables_PID_MDM_Export.json"); + return ResponseEntity.ok() + .headers(headers) + .body(resource); + } catch (IOException ex) { + return new ResponseEntity<>(null, null, HttpStatus.NOT_FOUND); + } + } } From c3a59fd89524115d274907d46bca621eda298d32 Mon Sep 17 00:00:00 2001 From: theresa Date: Tue, 30 Jul 2024 08:49:57 +0200 Subject: [PATCH 04/33] #3303 added resource to frontend, added download button in administration panel --- .../configuration/translations-de.js | 6 ++++- .../configuration/translations-en.js | 6 ++++- .../health/health.controller.js | 22 ++++++++++++++++++- .../administration/health/health.html.tmpl | 12 ++++++++++ .../resources/variable.resource.js | 18 +++++++++++++++ 5 files changed, 61 insertions(+), 3 deletions(-) diff --git a/mdm-frontend/src/app/legacy/administration/configuration/translations-de.js b/mdm-frontend/src/app/legacy/administration/configuration/translations-de.js index 461cef8a26a..a8ce48ba5d4 100644 --- a/mdm-frontend/src/app/legacy/administration/configuration/translations-de.js +++ b/mdm-frontend/src/app/legacy/administration/configuration/translations-de.js @@ -26,7 +26,8 @@ angular.module('metadatamanagementApp').config([ 'dara': 'Dara', 'messageBroker': 'Message Broker (für Websockets)', 'rabbit': 'RabbitMQ', - 'seo4Ajax': 'Seo4Ajax (Prerender as a Service)' + 'seo4Ajax': 'Seo4Ajax (Prerender as a Service)', + 'pid': 'PID' }, 'table': { 'service': 'Servicename', @@ -45,6 +46,9 @@ angular.module('metadatamanagementApp').config([ 'update': 'Aktualisieren', 'update-success': 'DARA erfolgreich aktualisiert', 'update-failure': 'Es sind Fehler beim Aktualisieren von DARA aufgetreten! (Siehe Details)' + }, + 'pid': { + 'download': 'Daten herunterladen' } }, 'logs': { diff --git a/mdm-frontend/src/app/legacy/administration/configuration/translations-en.js b/mdm-frontend/src/app/legacy/administration/configuration/translations-en.js index ba947595330..40321dd4978 100644 --- a/mdm-frontend/src/app/legacy/administration/configuration/translations-en.js +++ b/mdm-frontend/src/app/legacy/administration/configuration/translations-en.js @@ -26,7 +26,8 @@ angular.module('metadatamanagementApp').config([ 'dara': 'Dara', 'messageBroker': 'Message Broker (for Websockets)', 'rabbit': 'RabbitMQ', - 'seo4Ajax': 'Seo4Ajax (Prerender as a Service)' + 'seo4Ajax': 'Seo4Ajax (Prerender as a Service)', + 'pid': 'PID' }, 'table': { 'service': 'Service name', @@ -45,6 +46,9 @@ angular.module('metadatamanagementApp').config([ 'update': 'Update', 'update-success': 'DARA update successful', 'update-failure': 'Errors occurred during DARA update! (see details)' + }, + 'pid': { + 'download': 'Download data' } }, 'logs': { diff --git a/mdm-frontend/src/app/legacy/administration/health/health.controller.js b/mdm-frontend/src/app/legacy/administration/health/health.controller.js index 0644a448377..298e1dbee5f 100644 --- a/mdm-frontend/src/app/legacy/administration/health/health.controller.js +++ b/mdm-frontend/src/app/legacy/administration/health/health.controller.js @@ -10,8 +10,9 @@ angular.module('metadatamanagementApp').controller('HealthController', [ 'BreadcrumbService', 'SimpleMessageToastService', 'DaraReleaseCustomResource', + 'ExportAllVariablesResource', function($scope, MonitoringService, $uibModal, ElasticSearchAdminService, - PageMetadataService, $state, BreadcrumbService, SimpleMessageToastService, DaraReleaseCustomResource) { + PageMetadataService, $state, BreadcrumbService, SimpleMessageToastService, DaraReleaseCustomResource, ExportAllVariablesResource) { PageMetadataService.setPageTitle('administration.health.title'); $scope.isRecreatingIndices = false; $scope.updatingHealth = true; @@ -70,6 +71,25 @@ angular.module('metadatamanagementApp').controller('HealthController', [ }); } + /** + * Method to download all variable metadata as needed for PID registration. + * Data will be downloaded as a JSON file. + */ + $scope.downloadPidData = function() { + $scope.isDownloadingData = true; + ExportAllVariablesResource.exportAll().then(function(res) { + var blob = new Blob([angular.toJson(res, true)],{ + type: "application/json;charset=utf-8;" + }); + var downloadLink = document.createElement('a'); + downloadLink.setAttribute('download', 'variable_pid_mdm_export.json'); + downloadLink.setAttribute('href', window.URL.createObjectURL(blob)); + downloadLink.click(); + console.log("Download", res); + $scope.isDownloadingData = false; + }) + } + $scope.refresh(); $scope.getLabelClass = function(statusState) { diff --git a/mdm-frontend/src/app/legacy/administration/health/health.html.tmpl b/mdm-frontend/src/app/legacy/administration/health/health.html.tmpl index c78eb1f9496..99e102c6bd2 100644 --- a/mdm-frontend/src/app/legacy/administration/health/health.html.tmpl +++ b/mdm-frontend/src/app/legacy/administration/health/health.html.tmpl @@ -44,6 +44,18 @@ + + {{'administration.health.indicator.pid' | translate}} + + + + + + + + diff --git a/mdm-frontend/src/app/legacy/variablemanagement/resources/variable.resource.js b/mdm-frontend/src/app/legacy/variablemanagement/resources/variable.resource.js index cb060358881..f92cff56e9c 100644 --- a/mdm-frontend/src/app/legacy/variablemanagement/resources/variable.resource.js +++ b/mdm-frontend/src/app/legacy/variablemanagement/resources/variable.resource.js @@ -17,3 +17,21 @@ angular.module('metadatamanagementApp') }); }]); + /** + * Method to request the endpoint that downloads all variables as a PID-formatted json file. + * The response is a list (List) of variable metadata. + * We could not put this into the "VariableResource"-schema (above), because the URLs would not match. + */ + angular.module('metadatamanagementApp').factory('ExportAllVariablesResource', ['$http', + function($http) { + return { + exportAll: function() { + return $http.post('/api/variables/exportAll').then(function(response) { + return response.data; + var blob = new Blob([ response.data ], { type : 'application/json' }); + return blob; //{ blob: response.data, filename: "filename.json" }; + }); + } + } + }]); + From 550817cde7f791c8ecb1b62f9e5dbbb790fc5cf1 Mon Sep 17 00:00:00 2001 From: theresa Date: Wed, 31 Jul 2024 11:53:46 +0200 Subject: [PATCH 05/33] #3303 implemented data collection for PID metadata compilation --- .../ExportAllVariablesResourceController.java | 2 - .../service/VariableManagementService.java | 133 +++++++++++++++++- 2 files changed, 131 insertions(+), 4 deletions(-) diff --git a/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/rest/ExportAllVariablesResourceController.java b/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/rest/ExportAllVariablesResourceController.java index 294423242fb..ad186805094 100644 --- a/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/rest/ExportAllVariablesResourceController.java +++ b/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/rest/ExportAllVariablesResourceController.java @@ -1,11 +1,9 @@ package eu.dzhw.fdz.metadatamanagement.variablemanagement.rest; import java.io.IOException; -import java.rmi.server.ExportException; import javax.persistence.PersistenceException; -import eu.dzhw.fdz.metadatamanagement.projectmanagement.service.DataAcquisitionProjectManagementService; import eu.dzhw.fdz.metadatamanagement.usermanagement.security.AuthoritiesConstants; import eu.dzhw.fdz.metadatamanagement.variablemanagement.service.VariableManagementService; import lombok.RequiredArgsConstructor; diff --git a/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/service/VariableManagementService.java b/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/service/VariableManagementService.java index 8972a92fc7f..a413f1cc302 100644 --- a/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/service/VariableManagementService.java +++ b/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/service/VariableManagementService.java @@ -6,6 +6,10 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.rmi.server.ExportException; +import java.time.format.DateTimeFormatter; +import java.util.Arrays; +import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -22,17 +26,28 @@ import org.springframework.security.access.annotation.Secured; import org.springframework.stereotype.Service; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.gson.Gson; +import eu.dzhw.fdz.metadatamanagement.common.domain.projections.IdAndVersionProjection; import eu.dzhw.fdz.metadatamanagement.common.service.CrudService; import eu.dzhw.fdz.metadatamanagement.conceptmanagement.domain.Concept; import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.DataPackage; +import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.repository.DataPackageRepository; import eu.dzhw.fdz.metadatamanagement.datasetmanagement.domain.DataSet; import eu.dzhw.fdz.metadatamanagement.instrumentmanagement.domain.Instrument; import eu.dzhw.fdz.metadatamanagement.projectmanagement.domain.DataAcquisitionProject; +import eu.dzhw.fdz.metadatamanagement.projectmanagement.domain.Release; +import eu.dzhw.fdz.metadatamanagement.projectmanagement.repository.DataAcquisitionProjectRepository; import eu.dzhw.fdz.metadatamanagement.questionmanagement.domain.Question; import eu.dzhw.fdz.metadatamanagement.questionmanagement.repository.QuestionRepository; +import eu.dzhw.fdz.metadatamanagement.searchmanagement.dao.exception.ElasticsearchIoException; +import eu.dzhw.fdz.metadatamanagement.searchmanagement.documents.DataAcquisitionProjectSearchDocument; +import eu.dzhw.fdz.metadatamanagement.searchmanagement.documents.DataPackageSearchDocument; +import eu.dzhw.fdz.metadatamanagement.searchmanagement.documents.DataPackageSubDocument; +import eu.dzhw.fdz.metadatamanagement.searchmanagement.documents.VariableSearchDocument; import eu.dzhw.fdz.metadatamanagement.searchmanagement.service.ElasticsearchType; import eu.dzhw.fdz.metadatamanagement.searchmanagement.service.ElasticsearchUpdateQueueService; import eu.dzhw.fdz.metadatamanagement.surveymanagement.domain.Survey; @@ -41,6 +56,21 @@ import eu.dzhw.fdz.metadatamanagement.variablemanagement.repository.VariableRepository; import eu.dzhw.fdz.metadatamanagement.variablemanagement.service.helper.VariableCrudHelper; import lombok.RequiredArgsConstructor; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.client.core.CountRequest; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.ExistsQueryBuilder; +import org.elasticsearch.index.query.MatchQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.query.TermQueryBuilder; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.SearchHits; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.objectweb.asm.TypeReference; /** * Service for managing the domain object/aggregate {@link Variable}. @@ -58,8 +88,15 @@ public class VariableManagementService implements CrudService { private final VariableRepository variableRepository; + private final DataAcquisitionProjectRepository dataAcquisitionProjectRepository; + private final DataPackageRepository dataPackageRepository; + private final VariableCrudHelper crudHelper; + private final ObjectMapper objectMapper = new ObjectMapper(); + private final RestHighLevelClient client; + private final Gson gson; + /** * Delete all variables when the dataAcquisitionProject was deleted. * @@ -222,8 +259,7 @@ public Optional readSearchDocument(String id) { * @throws IOException */ public ResponseEntity exportVariablesAsJSON() throws IOException { - ObjectMapper objectMapper = new ObjectMapper(); - ArrayNode jsonNode = objectMapper.createArrayNode(); + ArrayNode jsonNode = this.getPidVariablesMetadata(); File tempFile = new File("tempfile.json"); tempFile.deleteOnExit(); objectMapper.writeValue(tempFile, jsonNode); @@ -239,4 +275,97 @@ public ResponseEntity exportVariablesAsJSON() throws IOException { return new ResponseEntity<>(null, null, HttpStatus.NOT_FOUND); } } + + /** + * Mapping variable metadata to metadata schema needed for PID registration + * @return + */ + private ArrayNode getPidVariablesMetadata() { + ArrayNode jsonNode = objectMapper.createArrayNode(); + // 1. ES-Query alle DatAaAcquisitionProjects shadow=false, release != null, isPreRelease == false + SearchRequest projectsRequest = new SearchRequest(); + SearchSourceBuilder builderProjects = new SearchSourceBuilder(); + builderProjects.query(QueryBuilders.boolQuery() + .must(new TermQueryBuilder("shadow", true)) + .must(new ExistsQueryBuilder("release")) + .must(new TermQueryBuilder("release.isPreRelease", false))) + .size(10000); + projectsRequest.source(builderProjects); + projectsRequest.indices("data_acquisition_projects"); + try { + SearchResponse response = client.search(projectsRequest, RequestOptions.DEFAULT); + List hits = Arrays.asList(response.getHits().getHits()); + for (var hit : hits) { + DataAcquisitionProjectSearchDocument project = gson.fromJson(hit.getSourceAsString(), DataAcquisitionProjectSearchDocument.class); + // für jedes Ergebnis: 2. ES-Query DataPackages shadow.=true, dataAcquisitionProjectId=project.id-project.version + SearchRequest dataPackagesRequest = new SearchRequest(); + SearchSourceBuilder builderDataPackages = new SearchSourceBuilder(); + builderDataPackages.query(QueryBuilders.boolQuery() + .must(new TermQueryBuilder("shadow", true)) + .must(new TermQueryBuilder("dataAcquisitionProjectId", project.getId()))) + .size(10000); + dataPackagesRequest.source(builderDataPackages); + dataPackagesRequest.indices("data_packages"); + SearchResponse responseDataPackage = client.search(dataPackagesRequest, RequestOptions.DEFAULT); + List hitsDataPackages = Arrays.asList(responseDataPackage.getHits().getHits()); + assert hitsDataPackages.size() == 1; + + // 3. ES-Query Variables shadow=true, dataAcquisitionProjectId=project.id-project.version + SearchRequest variablesRequest = new SearchRequest(); + SearchSourceBuilder builderVariables = new SearchSourceBuilder(); + builderVariables.query(QueryBuilders.boolQuery() + .must(new TermQueryBuilder("shadow", true)) + .must(new TermQueryBuilder("dataAcquisitionProjectId", project.getId()))) + .size(10000); + variablesRequest.source(builderVariables); + variablesRequest.indices("variables"); + SearchResponse responseVariables = client.search(variablesRequest, RequestOptions.DEFAULT); + List hitsVariables = Arrays.asList(responseVariables.getHits().getHits()); + for (var variable : hitsVariables) { + VariableSearchDocument variableObj = gson.fromJson(variable.getSourceAsString(), VariableSearchDocument.class); + DataPackageSearchDocument dataPackage = gson.fromJson(hitsDataPackages.get(0).getSourceAsString(), DataPackageSearchDocument.class); + ObjectNode obj = this.getPidMetadataOfVariable(variableObj, project, dataPackage); + jsonNode.add(obj); + System.out.println("Adding variable to json " + variableObj.getId()); + } + } + return jsonNode; + } catch (IOException e) { + throw new ElasticsearchIoException(e); + } + } + + /** + * Collects PID metadata of variable + * @param variable the variable metadata is collected for + * @param project the project this variable belongs to + * @param dataPackage the data package this variable belongs to + * @return the metadata as a json node + */ + private ObjectNode getPidMetadataOfVariable( + VariableSearchDocument variable, + DataAcquisitionProject project, + DataPackageSearchDocument dataPackage) { + ObjectNode variableObj = objectMapper.createObjectNode(); + variableObj.put("studyDOI", dataPackage.getDoi()); + variableObj.put("variableName", variable.getName()); + variableObj.put("variableLabel", variable.getLabel().getEn()); // todo: Deutsch oder Englisch? + variableObj.put("pidProposal", "21.T11998/dzhw:" + project.getId() + "_" + variable.getId() + ":" + project.getRelease().getVersion()); + // todo base URL as Constant, Deutsch oder Englisch? + variableObj.put("landingPage", "https://metadata.fdz.dzhw.eu/en/variables/" + variable.getId().replace("$", "") + "?version=" + project.getRelease().getVersion()); + variableObj.put("resourceType", "variable"); + variableObj.put("title", variable.getName() + ": " + variable.getLabel().getEn()); // todo: Deutsch oder Englisch? + // todo creators zusammenstellen + //variableObj.put("creators", ) + variableObj.put("publisher", "FDZ-DZHW"); + // Create DateTimeFormatter instance with specified format + DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm"); + // Format LocalDateTime to String + String formattedDateTime = project.getRelease().getLastDate().format(dateTimeFormatter); + variableObj.put("publicationDate", formattedDateTime); + //todo access ways verarbeiten + String accessWays = variable.getAccessWays().toString(); + variableObj.put("availability", accessWays); + return variableObj; + } } From 7075815714a966007ecc4c4200a1fed94e82aef0 Mon Sep 17 00:00:00 2001 From: theresa Date: Wed, 31 Jul 2024 14:23:52 +0200 Subject: [PATCH 06/33] #3303 implemented metadata compilation for PID registration --- .../service/VariableManagementService.java | 78 ++++++++++++++----- 1 file changed, 58 insertions(+), 20 deletions(-) diff --git a/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/service/VariableManagementService.java b/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/service/VariableManagementService.java index a413f1cc302..e0db8cf10fe 100644 --- a/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/service/VariableManagementService.java +++ b/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/service/VariableManagementService.java @@ -7,6 +7,7 @@ import java.nio.file.Paths; import java.rmi.server.ExportException; import java.time.format.DateTimeFormatter; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Map; @@ -31,6 +32,7 @@ import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.gson.Gson; +import eu.dzhw.fdz.metadatamanagement.common.domain.Person; import eu.dzhw.fdz.metadatamanagement.common.domain.projections.IdAndVersionProjection; import eu.dzhw.fdz.metadatamanagement.common.service.CrudService; import eu.dzhw.fdz.metadatamanagement.conceptmanagement.domain.Concept; @@ -52,6 +54,7 @@ import eu.dzhw.fdz.metadatamanagement.searchmanagement.service.ElasticsearchUpdateQueueService; import eu.dzhw.fdz.metadatamanagement.surveymanagement.domain.Survey; import eu.dzhw.fdz.metadatamanagement.usermanagement.security.AuthoritiesConstants; +import eu.dzhw.fdz.metadatamanagement.variablemanagement.domain.AccessWays; import eu.dzhw.fdz.metadatamanagement.variablemanagement.domain.Variable; import eu.dzhw.fdz.metadatamanagement.variablemanagement.repository.VariableRepository; import eu.dzhw.fdz.metadatamanagement.variablemanagement.service.helper.VariableCrudHelper; @@ -88,15 +91,19 @@ public class VariableManagementService implements CrudService { private final VariableRepository variableRepository; - private final DataAcquisitionProjectRepository dataAcquisitionProjectRepository; - private final DataPackageRepository dataPackageRepository; - private final VariableCrudHelper crudHelper; private final ObjectMapper objectMapper = new ObjectMapper(); + private final RestHighLevelClient client; + private final Gson gson; + private final String LANDING_PAGE_BASE_URL = "https://metadata.fdz.dzhw.eu/en/variables/"; + private final String PID_PREFIX = "21.T11998/dzhw:"; + + + /** * Delete all variables when the dataAcquisitionProject was deleted. * @@ -259,10 +266,12 @@ public Optional readSearchDocument(String id) { * @throws IOException */ public ResponseEntity exportVariablesAsJSON() throws IOException { - ArrayNode jsonNode = this.getPidVariablesMetadata(); + ArrayNode variableMetadata = this.getPidVariablesMetadata(); + ObjectNode node = objectMapper.createObjectNode(); + node.set("variables", variableMetadata); File tempFile = new File("tempfile.json"); tempFile.deleteOnExit(); - objectMapper.writeValue(tempFile, jsonNode); + objectMapper.writeValue(tempFile, node); Path path = Paths.get(tempFile.getAbsolutePath()); try { ByteArrayResource resource = new ByteArrayResource(Files.readAllBytes(path)); @@ -289,7 +298,7 @@ private ArrayNode getPidVariablesMetadata() { .must(new TermQueryBuilder("shadow", true)) .must(new ExistsQueryBuilder("release")) .must(new TermQueryBuilder("release.isPreRelease", false))) - .size(10000); + .size(10); projectsRequest.source(builderProjects); projectsRequest.indices("data_acquisition_projects"); try { @@ -303,7 +312,7 @@ private ArrayNode getPidVariablesMetadata() { builderDataPackages.query(QueryBuilders.boolQuery() .must(new TermQueryBuilder("shadow", true)) .must(new TermQueryBuilder("dataAcquisitionProjectId", project.getId()))) - .size(10000); + .size(10); dataPackagesRequest.source(builderDataPackages); dataPackagesRequest.indices("data_packages"); SearchResponse responseDataPackage = client.search(dataPackagesRequest, RequestOptions.DEFAULT); @@ -316,7 +325,7 @@ private ArrayNode getPidVariablesMetadata() { builderVariables.query(QueryBuilders.boolQuery() .must(new TermQueryBuilder("shadow", true)) .must(new TermQueryBuilder("dataAcquisitionProjectId", project.getId()))) - .size(10000); + .size(10); variablesRequest.source(builderVariables); variablesRequest.indices("variables"); SearchResponse responseVariables = client.search(variablesRequest, RequestOptions.DEFAULT); @@ -350,22 +359,51 @@ private ObjectNode getPidMetadataOfVariable( variableObj.put("studyDOI", dataPackage.getDoi()); variableObj.put("variableName", variable.getName()); variableObj.put("variableLabel", variable.getLabel().getEn()); // todo: Deutsch oder Englisch? - variableObj.put("pidProposal", "21.T11998/dzhw:" + project.getId() + "_" + variable.getId() + ":" + project.getRelease().getVersion()); - // todo base URL as Constant, Deutsch oder Englisch? - variableObj.put("landingPage", "https://metadata.fdz.dzhw.eu/en/variables/" + variable.getId().replace("$", "") + "?version=" + project.getRelease().getVersion()); - variableObj.put("resourceType", "variable"); + variableObj.put("pidProposal", PID_PREFIX + project.getId() + "_" + variable.getName() + ":" + project.getRelease().getVersion()); + // todo: Deutsch oder Englisch? + variableObj.put("landingPage", LANDING_PAGE_BASE_URL + + variable.getMasterId().replace("$", "") + + "?version=" + project.getRelease().getVersion()); + variableObj.put("resourceType", "Variable"); variableObj.put("title", variable.getName() + ": " + variable.getLabel().getEn()); // todo: Deutsch oder Englisch? - // todo creators zusammenstellen - //variableObj.put("creators", ) + variableObj.set("creators", this.compileCreators(dataPackage.getProjectContributors())); variableObj.put("publisher", "FDZ-DZHW"); - // Create DateTimeFormatter instance with specified format - DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm"); - // Format LocalDateTime to String + DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); String formattedDateTime = project.getRelease().getLastDate().format(dateTimeFormatter); variableObj.put("publicationDate", formattedDateTime); - //todo access ways verarbeiten - String accessWays = variable.getAccessWays().toString(); - variableObj.put("availability", accessWays); + variableObj.put("availability", this.compileAccessWays(variable.getAccessWays())); return variableObj; } + + /** + * Compiles the given list of project contributor entries into a PID metadata list. + * @param projectContributors the list of contributors of this project + * @return a list of contributors as needed for PID registration + */ + private ArrayNode compileCreators(List projectContributors) { + ArrayNode contributorList = objectMapper.createArrayNode(); + for (var person : projectContributors) { + ObjectNode p = objectMapper.createObjectNode(); + p.put("firstName", person.getFirstName()); + if (person.getMiddleName() != null && !person.getMiddleName().isEmpty()) { + p.put("middleName", person.getMiddleName()); + } + p.put("lastName", person.getLastName()); + contributorList.add(p); + } + return contributorList; + } + + /** + * Compiles the given list of accessway entries into a readable string. + * @param accessWays list of access ways + * @return a readable string of access ways + */ + private String compileAccessWays(List accessWays) { + List strList = new ArrayList<>(); + for (var access : accessWays) { + strList.add(AccessWays.displayAccessWay(access)); + } + return String.join(", ", strList); + } } From 37a76899af32ac11dc767ea98ed0c4bcb889dc0c Mon Sep 17 00:00:00 2001 From: theresa Date: Fri, 2 Aug 2024 10:55:58 +0200 Subject: [PATCH 07/33] #3303: improved queries --- .../service/VariableManagementService.java | 68 ++++++++++++++++--- 1 file changed, 58 insertions(+), 10 deletions(-) diff --git a/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/service/VariableManagementService.java b/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/service/VariableManagementService.java index e0db8cf10fe..75c8a34409f 100644 --- a/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/service/VariableManagementService.java +++ b/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/service/VariableManagementService.java @@ -59,6 +59,7 @@ import eu.dzhw.fdz.metadatamanagement.variablemanagement.repository.VariableRepository; import eu.dzhw.fdz.metadatamanagement.variablemanagement.service.helper.VariableCrudHelper; import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.RequestOptions; @@ -83,6 +84,7 @@ @Service @RepositoryEventHandler @RequiredArgsConstructor +@Slf4j public class VariableManagementService implements CrudService { private final QuestionRepository questionRepository; @@ -277,6 +279,7 @@ public ResponseEntity exportVariablesAsJSON() throws IOException { ByteArrayResource resource = new ByteArrayResource(Files.readAllBytes(path)); HttpHeaders headers = new HttpHeaders(); headers.add("Content-Disposition", "attachment; filename=Variables_PID_MDM_Export.json"); + log.info("Finished variable export with " + variableMetadata.size() + " variables."); return ResponseEntity.ok() .headers(headers) .body(resource); @@ -287,7 +290,7 @@ public ResponseEntity exportVariablesAsJSON() throws IOException { /** * Mapping variable metadata to metadata schema needed for PID registration - * @return + * @return collected metadata as JsonNode */ private ArrayNode getPidVariablesMetadata() { ArrayNode jsonNode = objectMapper.createArrayNode(); @@ -295,15 +298,17 @@ private ArrayNode getPidVariablesMetadata() { SearchRequest projectsRequest = new SearchRequest(); SearchSourceBuilder builderProjects = new SearchSourceBuilder(); builderProjects.query(QueryBuilders.boolQuery() - .must(new TermQueryBuilder("shadow", true)) + .must(new TermQueryBuilder("shadow", false)) + .must(new TermQueryBuilder("hidden", false)) .must(new ExistsQueryBuilder("release")) .must(new TermQueryBuilder("release.isPreRelease", false))) - .size(10); + .size(10000); projectsRequest.source(builderProjects); projectsRequest.indices("data_acquisition_projects"); try { SearchResponse response = client.search(projectsRequest, RequestOptions.DEFAULT); List hits = Arrays.asList(response.getHits().getHits()); + log.info("Exporting variables for " + hits.size() + " projects"); for (var hit : hits) { DataAcquisitionProjectSearchDocument project = gson.fromJson(hit.getSourceAsString(), DataAcquisitionProjectSearchDocument.class); // für jedes Ergebnis: 2. ES-Query DataPackages shadow.=true, dataAcquisitionProjectId=project.id-project.version @@ -311,7 +316,7 @@ private ArrayNode getPidVariablesMetadata() { SearchSourceBuilder builderDataPackages = new SearchSourceBuilder(); builderDataPackages.query(QueryBuilders.boolQuery() .must(new TermQueryBuilder("shadow", true)) - .must(new TermQueryBuilder("dataAcquisitionProjectId", project.getId()))) + .must(new TermQueryBuilder("dataAcquisitionProjectId", project.getId() + "-" + project.getRelease().getVersion()))) .size(10); dataPackagesRequest.source(builderDataPackages); dataPackagesRequest.indices("data_packages"); @@ -324,8 +329,8 @@ private ArrayNode getPidVariablesMetadata() { SearchSourceBuilder builderVariables = new SearchSourceBuilder(); builderVariables.query(QueryBuilders.boolQuery() .must(new TermQueryBuilder("shadow", true)) - .must(new TermQueryBuilder("dataAcquisitionProjectId", project.getId()))) - .size(10); + .must(new TermQueryBuilder("dataAcquisitionProjectId", project.getId() + "-" + project.getRelease().getVersion()))) + .size(10000); variablesRequest.source(builderVariables); variablesRequest.indices("variables"); SearchResponse responseVariables = client.search(variablesRequest, RequestOptions.DEFAULT); @@ -334,8 +339,9 @@ private ArrayNode getPidVariablesMetadata() { VariableSearchDocument variableObj = gson.fromJson(variable.getSourceAsString(), VariableSearchDocument.class); DataPackageSearchDocument dataPackage = gson.fromJson(hitsDataPackages.get(0).getSourceAsString(), DataPackageSearchDocument.class); ObjectNode obj = this.getPidMetadataOfVariable(variableObj, project, dataPackage); - jsonNode.add(obj); - System.out.println("Adding variable to json " + variableObj.getId()); + if (obj != null) { + jsonNode.add(obj); + } } } return jsonNode; @@ -345,7 +351,7 @@ private ArrayNode getPidVariablesMetadata() { } /** - * Collects PID metadata of variable + * Collects PID metadata of a variable * @param variable the variable metadata is collected for * @param project the project this variable belongs to * @param dataPackage the data package this variable belongs to @@ -355,10 +361,13 @@ private ObjectNode getPidMetadataOfVariable( VariableSearchDocument variable, DataAcquisitionProject project, DataPackageSearchDocument dataPackage) { + if (this.hasMissingFields(variable, project, dataPackage)) { + return null; + } ObjectNode variableObj = objectMapper.createObjectNode(); variableObj.put("studyDOI", dataPackage.getDoi()); variableObj.put("variableName", variable.getName()); - variableObj.put("variableLabel", variable.getLabel().getEn()); // todo: Deutsch oder Englisch? + variableObj.put("variableLabel", variable.getLabel().getEn() != null ? variable.getLabel().getEn() : variable.getLabel().getDe()); // todo: Deutsch oder Englisch? variableObj.put("pidProposal", PID_PREFIX + project.getId() + "_" + variable.getName() + ":" + project.getRelease().getVersion()); // todo: Deutsch oder Englisch? variableObj.put("landingPage", LANDING_PAGE_BASE_URL @@ -375,6 +384,45 @@ private ObjectNode getPidMetadataOfVariable( return variableObj; } + /** + * Checks the required fields for PID registration. Variables with missing required fields will not be exported. + * @param variable the variable for which a PID should be registered + * @param project the project this variable belongs to + * @param dataPackage the data package this variable belongs to + * @return true if required fields are missing, else false + */ + private boolean hasMissingFields(VariableSearchDocument variable, DataAcquisitionProject project, DataPackageSearchDocument dataPackage) { + if (variable.getName() == null || variable.getName().isEmpty()) { + log.info(String.format("Unable to export variable with id '%s'. Required variable name is missing.", variable.getId())); + return true; + } + if (variable.getLabel() == null) { + log.info(String.format("Unable to export variable with id '%s'. Required variable label is missing.", variable.getId())); + return true; + } else if ((variable.getLabel().getDe() == null || variable.getLabel().getDe().trim().isEmpty()) + && (variable.getLabel().getEn() == null || variable.getLabel().getEn().trim().isEmpty())) { + log.info(String.format("Unable to export variable with id '%s'. Required variable label is missing.", variable.getId())); + return true; + } + if (dataPackage.getDoi() == null || dataPackage.getDoi().trim().isEmpty()) { + log.info(String.format("Unable to export variable with id '%s'. Required datapackage DOIl is missing.", variable.getId())); + return true; + } + if (dataPackage.getProjectContributors() == null || dataPackage.getProjectContributors().isEmpty()) { + log.info(String.format("Unable to export variable with id '%s'. Datapackage has no contributors.", variable.getId())); + return true; + } + if (project.getId() == null + || project.getRelease() == null + || project.getRelease().getVersion() == null + || project.getRelease().getVersion().trim().isEmpty() + || project.getRelease().getLastDate() == null){ + log.info(String.format("Unable to export variable with id '%s'. Required project fields are missing.", variable.getId())); + return true; + } + return false; + } + /** * Compiles the given list of project contributor entries into a PID metadata list. * @param projectContributors the list of contributors of this project From 4fac024350085ecb9e2f96520f4fdacdffb58a44 Mon Sep 17 00:00:00 2001 From: theresa Date: Fri, 2 Aug 2024 13:03:45 +0200 Subject: [PATCH 08/33] #3303 fixed code style --- .../variablemanagement/domain/Variable.java | 4 +- .../ExportAllVariablesResourceController.java | 5 +- .../service/VariableManagementService.java | 92 +++++++++++-------- 3 files changed, 59 insertions(+), 42 deletions(-) diff --git a/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/domain/Variable.java b/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/domain/Variable.java index 7e63b8411ee..5330a8f51a2 100644 --- a/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/domain/Variable.java +++ b/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/domain/Variable.java @@ -143,8 +143,8 @@ public class Variable extends AbstractShadowableRdcDomainObject { */ @Size(max = StringLengths.MEDIUM, message = "variable-management.error.pid.size") @Pattern( - message = "variable-management.error.pid.pattern", - regexp = Patterns.PID) + message = "variable-management.error.pid.pattern", + regexp = Patterns.PID) private String pid; /** diff --git a/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/rest/ExportAllVariablesResourceController.java b/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/rest/ExportAllVariablesResourceController.java index ad186805094..625360fd99c 100644 --- a/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/rest/ExportAllVariablesResourceController.java +++ b/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/rest/ExportAllVariablesResourceController.java @@ -17,6 +17,9 @@ import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.ResponseBody; +/** + * Controller for exporting all variables for PID registration. + */ @Controller @RequestMapping("/api") @RequiredArgsConstructor @@ -35,7 +38,7 @@ public class ExportAllVariablesResourceController { public ResponseEntity exportAllVariables() { ResponseEntity response = new ResponseEntity<>(null, null, HttpStatus.NOT_FOUND); try { - response = this.variablesManagementService.exportVariablesAsJSON(); + response = this.variablesManagementService.exportVariablesAsJson(); } catch (PersistenceException | IOException ex) { return response; } diff --git a/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/service/VariableManagementService.java b/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/service/VariableManagementService.java index 75c8a34409f..687e1870642 100644 --- a/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/service/VariableManagementService.java +++ b/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/service/VariableManagementService.java @@ -101,8 +101,8 @@ public class VariableManagementService implements CrudService { private final Gson gson; - private final String LANDING_PAGE_BASE_URL = "https://metadata.fdz.dzhw.eu/en/variables/"; - private final String PID_PREFIX = "21.T11998/dzhw:"; + private final String landingPageBaseUrl = "https://metadata.fdz.dzhw.eu/en/variables/"; + private final String pidPrefix = "21.T11998/dzhw:"; @@ -265,9 +265,9 @@ public Optional readSearchDocument(String id) { /** * Writes all variable data according to the PID metadata schema to a JSON file. * @return the JSON file - * @throws IOException + * @throws IOException if elasticsearch fails */ - public ResponseEntity exportVariablesAsJSON() throws IOException { + public ResponseEntity exportVariablesAsJson() throws IOException { ArrayNode variableMetadata = this.getPidVariablesMetadata(); ObjectNode node = objectMapper.createObjectNode(); node.set("variables", variableMetadata); @@ -289,12 +289,11 @@ public ResponseEntity exportVariablesAsJSON() throws IOException { } /** - * Mapping variable metadata to metadata schema needed for PID registration + * Mapping variable metadata to metadata schema needed for PID registration. * @return collected metadata as JsonNode */ private ArrayNode getPidVariablesMetadata() { ArrayNode jsonNode = objectMapper.createArrayNode(); - // 1. ES-Query alle DatAaAcquisitionProjects shadow=false, release != null, isPreRelease == false SearchRequest projectsRequest = new SearchRequest(); SearchSourceBuilder builderProjects = new SearchSourceBuilder(); builderProjects.query(QueryBuilders.boolQuery() @@ -310,13 +309,14 @@ private ArrayNode getPidVariablesMetadata() { List hits = Arrays.asList(response.getHits().getHits()); log.info("Exporting variables for " + hits.size() + " projects"); for (var hit : hits) { - DataAcquisitionProjectSearchDocument project = gson.fromJson(hit.getSourceAsString(), DataAcquisitionProjectSearchDocument.class); - // für jedes Ergebnis: 2. ES-Query DataPackages shadow.=true, dataAcquisitionProjectId=project.id-project.version + DataAcquisitionProjectSearchDocument project = gson.fromJson( + hit.getSourceAsString(), DataAcquisitionProjectSearchDocument.class); SearchRequest dataPackagesRequest = new SearchRequest(); SearchSourceBuilder builderDataPackages = new SearchSourceBuilder(); builderDataPackages.query(QueryBuilders.boolQuery() .must(new TermQueryBuilder("shadow", true)) - .must(new TermQueryBuilder("dataAcquisitionProjectId", project.getId() + "-" + project.getRelease().getVersion()))) + .must(new TermQueryBuilder("dataAcquisitionProjectId", project.getId() + + "-" + project.getRelease().getVersion()))) .size(10); dataPackagesRequest.source(builderDataPackages); dataPackagesRequest.indices("data_packages"); @@ -329,15 +329,18 @@ private ArrayNode getPidVariablesMetadata() { SearchSourceBuilder builderVariables = new SearchSourceBuilder(); builderVariables.query(QueryBuilders.boolQuery() .must(new TermQueryBuilder("shadow", true)) - .must(new TermQueryBuilder("dataAcquisitionProjectId", project.getId() + "-" + project.getRelease().getVersion()))) + .must(new TermQueryBuilder("dataAcquisitionProjectId", project.getId() + + "-" + project.getRelease().getVersion()))) .size(10000); variablesRequest.source(builderVariables); variablesRequest.indices("variables"); SearchResponse responseVariables = client.search(variablesRequest, RequestOptions.DEFAULT); List hitsVariables = Arrays.asList(responseVariables.getHits().getHits()); for (var variable : hitsVariables) { - VariableSearchDocument variableObj = gson.fromJson(variable.getSourceAsString(), VariableSearchDocument.class); - DataPackageSearchDocument dataPackage = gson.fromJson(hitsDataPackages.get(0).getSourceAsString(), DataPackageSearchDocument.class); + VariableSearchDocument variableObj = gson.fromJson( + variable.getSourceAsString(), VariableSearchDocument.class); + DataPackageSearchDocument dataPackage = gson.fromJson( + hitsDataPackages.get(0).getSourceAsString(), DataPackageSearchDocument.class); ObjectNode obj = this.getPidMetadataOfVariable(variableObj, project, dataPackage); if (obj != null) { jsonNode.add(obj); @@ -351,30 +354,35 @@ private ArrayNode getPidVariablesMetadata() { } /** - * Collects PID metadata of a variable + * Collects PID metadata of a variable. * @param variable the variable metadata is collected for * @param project the project this variable belongs to * @param dataPackage the data package this variable belongs to * @return the metadata as a json node */ private ObjectNode getPidMetadataOfVariable( - VariableSearchDocument variable, - DataAcquisitionProject project, - DataPackageSearchDocument dataPackage) { + VariableSearchDocument variable, + DataAcquisitionProject project, + DataPackageSearchDocument dataPackage) { if (this.hasMissingFields(variable, project, dataPackage)) { return null; } ObjectNode variableObj = objectMapper.createObjectNode(); variableObj.put("studyDOI", dataPackage.getDoi()); variableObj.put("variableName", variable.getName()); - variableObj.put("variableLabel", variable.getLabel().getEn() != null ? variable.getLabel().getEn() : variable.getLabel().getDe()); // todo: Deutsch oder Englisch? - variableObj.put("pidProposal", PID_PREFIX + project.getId() + "_" + variable.getName() + ":" + project.getRelease().getVersion()); - // todo: Deutsch oder Englisch? - variableObj.put("landingPage", LANDING_PAGE_BASE_URL - + variable.getMasterId().replace("$", "") - + "?version=" + project.getRelease().getVersion()); + // todo: Clarify which language to use + variableObj.put("variableLabel", variable.getLabel().getEn() != null ? + variable.getLabel().getEn() : + variable.getLabel().getDe()); + variableObj.put("pidProposal", pidPrefix + project.getId() + "_" + variable.getName() + ":" + + project.getRelease().getVersion()); + // todo: Clarify which language to use + variableObj.put("landingPage", landingPageBaseUrl + + variable.getMasterId().replace("$", "") + + "?version=" + project.getRelease().getVersion()); variableObj.put("resourceType", "Variable"); - variableObj.put("title", variable.getName() + ": " + variable.getLabel().getEn()); // todo: Deutsch oder Englisch? + // todo: Clarify which language to use + variableObj.put("title", variable.getName() + ": " + variable.getLabel().getEn()); variableObj.set("creators", this.compileCreators(dataPackage.getProjectContributors())); variableObj.put("publisher", "FDZ-DZHW"); DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); @@ -393,31 +401,37 @@ private ObjectNode getPidMetadataOfVariable( */ private boolean hasMissingFields(VariableSearchDocument variable, DataAcquisitionProject project, DataPackageSearchDocument dataPackage) { if (variable.getName() == null || variable.getName().isEmpty()) { - log.info(String.format("Unable to export variable with id '%s'. Required variable name is missing.", variable.getId())); + log.info(String.format("Unable to export variable with id '%s'. Required variable name is missing.", + variable.getId())); return true; } if (variable.getLabel() == null) { - log.info(String.format("Unable to export variable with id '%s'. Required variable label is missing.", variable.getId())); + log.info(String.format("Unable to export variable with id '%s'. Required variable label is missing.", + variable.getId())); return true; } else if ((variable.getLabel().getDe() == null || variable.getLabel().getDe().trim().isEmpty()) - && (variable.getLabel().getEn() == null || variable.getLabel().getEn().trim().isEmpty())) { - log.info(String.format("Unable to export variable with id '%s'. Required variable label is missing.", variable.getId())); + && (variable.getLabel().getEn() == null || variable.getLabel().getEn().trim().isEmpty())) { + log.info(String.format("Unable to export variable with id '%s'. Required variable label is missing.", + variable.getId())); return true; } if (dataPackage.getDoi() == null || dataPackage.getDoi().trim().isEmpty()) { - log.info(String.format("Unable to export variable with id '%s'. Required datapackage DOIl is missing.", variable.getId())); + log.info(String.format("Unable to export variable with id '%s'. Required datapackage DOIl is missing.", + variable.getId())); return true; } if (dataPackage.getProjectContributors() == null || dataPackage.getProjectContributors().isEmpty()) { - log.info(String.format("Unable to export variable with id '%s'. Datapackage has no contributors.", variable.getId())); + log.info(String.format("Unable to export variable with id '%s'. Datapackage has no contributors.", + variable.getId())); return true; } if (project.getId() == null - || project.getRelease() == null - || project.getRelease().getVersion() == null - || project.getRelease().getVersion().trim().isEmpty() - || project.getRelease().getLastDate() == null){ - log.info(String.format("Unable to export variable with id '%s'. Required project fields are missing.", variable.getId())); + || project.getRelease() == null + || project.getRelease().getVersion() == null + || project.getRelease().getVersion().trim().isEmpty() + || project.getRelease().getLastDate() == null) { + log.info(String.format("Unable to export variable with id '%s'. Required project fields are missing.", + variable.getId())); return true; } return false; @@ -431,13 +445,13 @@ private boolean hasMissingFields(VariableSearchDocument variable, DataAcquisitio private ArrayNode compileCreators(List projectContributors) { ArrayNode contributorList = objectMapper.createArrayNode(); for (var person : projectContributors) { - ObjectNode p = objectMapper.createObjectNode(); - p.put("firstName", person.getFirstName()); + ObjectNode personNode = objectMapper.createObjectNode(); + personNode.put("firstName", person.getFirstName()); if (person.getMiddleName() != null && !person.getMiddleName().isEmpty()) { - p.put("middleName", person.getMiddleName()); + personNode.put("middleName", person.getMiddleName()); } - p.put("lastName", person.getLastName()); - contributorList.add(p); + personNode.put("lastName", person.getLastName()); + contributorList.add(personNode); } return contributorList; } From 09fa0af47c57b4555484e816d18c9725b59df2ec Mon Sep 17 00:00:00 2001 From: theresa Date: Fri, 2 Aug 2024 13:08:28 +0200 Subject: [PATCH 09/33] #3303 code style fix again --- .../service/VariableManagementService.java | 28 ++++++++++--------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/service/VariableManagementService.java b/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/service/VariableManagementService.java index 687e1870642..0f671645ea1 100644 --- a/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/service/VariableManagementService.java +++ b/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/service/VariableManagementService.java @@ -310,7 +310,7 @@ private ArrayNode getPidVariablesMetadata() { log.info("Exporting variables for " + hits.size() + " projects"); for (var hit : hits) { DataAcquisitionProjectSearchDocument project = gson.fromJson( - hit.getSourceAsString(), DataAcquisitionProjectSearchDocument.class); + hit.getSourceAsString(), DataAcquisitionProjectSearchDocument.class); SearchRequest dataPackagesRequest = new SearchRequest(); SearchSourceBuilder builderDataPackages = new SearchSourceBuilder(); builderDataPackages.query(QueryBuilders.boolQuery() @@ -338,9 +338,9 @@ private ArrayNode getPidVariablesMetadata() { List hitsVariables = Arrays.asList(responseVariables.getHits().getHits()); for (var variable : hitsVariables) { VariableSearchDocument variableObj = gson.fromJson( - variable.getSourceAsString(), VariableSearchDocument.class); + variable.getSourceAsString(), VariableSearchDocument.class); DataPackageSearchDocument dataPackage = gson.fromJson( - hitsDataPackages.get(0).getSourceAsString(), DataPackageSearchDocument.class); + hitsDataPackages.get(0).getSourceAsString(), DataPackageSearchDocument.class); ObjectNode obj = this.getPidMetadataOfVariable(variableObj, project, dataPackage); if (obj != null) { jsonNode.add(obj); @@ -371,9 +371,9 @@ private ObjectNode getPidMetadataOfVariable( variableObj.put("studyDOI", dataPackage.getDoi()); variableObj.put("variableName", variable.getName()); // todo: Clarify which language to use - variableObj.put("variableLabel", variable.getLabel().getEn() != null ? - variable.getLabel().getEn() : - variable.getLabel().getDe()); + variableObj.put("variableLabel", variable.getLabel().getEn() != null + ? variable.getLabel().getEn() : + variable.getLabel().getDe()); variableObj.put("pidProposal", pidPrefix + project.getId() + "_" + variable.getName() + ":" + project.getRelease().getVersion()); // todo: Clarify which language to use @@ -399,30 +399,32 @@ private ObjectNode getPidMetadataOfVariable( * @param dataPackage the data package this variable belongs to * @return true if required fields are missing, else false */ - private boolean hasMissingFields(VariableSearchDocument variable, DataAcquisitionProject project, DataPackageSearchDocument dataPackage) { + private boolean hasMissingFields(VariableSearchDocument variable, + DataAcquisitionProject project, + DataPackageSearchDocument dataPackage) { if (variable.getName() == null || variable.getName().isEmpty()) { log.info(String.format("Unable to export variable with id '%s'. Required variable name is missing.", - variable.getId())); + variable.getId())); return true; } if (variable.getLabel() == null) { log.info(String.format("Unable to export variable with id '%s'. Required variable label is missing.", - variable.getId())); + variable.getId())); return true; } else if ((variable.getLabel().getDe() == null || variable.getLabel().getDe().trim().isEmpty()) && (variable.getLabel().getEn() == null || variable.getLabel().getEn().trim().isEmpty())) { log.info(String.format("Unable to export variable with id '%s'. Required variable label is missing.", - variable.getId())); + variable.getId())); return true; } if (dataPackage.getDoi() == null || dataPackage.getDoi().trim().isEmpty()) { log.info(String.format("Unable to export variable with id '%s'. Required datapackage DOIl is missing.", - variable.getId())); + variable.getId())); return true; } if (dataPackage.getProjectContributors() == null || dataPackage.getProjectContributors().isEmpty()) { log.info(String.format("Unable to export variable with id '%s'. Datapackage has no contributors.", - variable.getId())); + variable.getId())); return true; } if (project.getId() == null @@ -431,7 +433,7 @@ private boolean hasMissingFields(VariableSearchDocument variable, DataAcquisitio || project.getRelease().getVersion().trim().isEmpty() || project.getRelease().getLastDate() == null) { log.info(String.format("Unable to export variable with id '%s'. Required project fields are missing.", - variable.getId())); + variable.getId())); return true; } return false; From 2079bbfde0949073185bcfec3ecf1147285bb004 Mon Sep 17 00:00:00 2001 From: theresa Date: Mon, 5 Aug 2024 13:31:10 +0200 Subject: [PATCH 10/33] #3303 optimized variable export without temp file --- .../service/VariableManagementService.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/service/VariableManagementService.java b/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/service/VariableManagementService.java index 0f671645ea1..6803868b7ff 100644 --- a/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/service/VariableManagementService.java +++ b/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/service/VariableManagementService.java @@ -271,12 +271,8 @@ public ResponseEntity exportVariablesAsJson() throws IOException { ArrayNode variableMetadata = this.getPidVariablesMetadata(); ObjectNode node = objectMapper.createObjectNode(); node.set("variables", variableMetadata); - File tempFile = new File("tempfile.json"); - tempFile.deleteOnExit(); - objectMapper.writeValue(tempFile, node); - Path path = Paths.get(tempFile.getAbsolutePath()); try { - ByteArrayResource resource = new ByteArrayResource(Files.readAllBytes(path)); + ByteArrayResource resource = new ByteArrayResource(objectMapper.writeValueAsBytes(node)); HttpHeaders headers = new HttpHeaders(); headers.add("Content-Disposition", "attachment; filename=Variables_PID_MDM_Export.json"); log.info("Finished variable export with " + variableMetadata.size() + " variables."); From 287f633539a69dd5a0cc114cb2072b15535830b1 Mon Sep 17 00:00:00 2001 From: theresa Date: Wed, 7 Aug 2024 10:20:41 +0200 Subject: [PATCH 11/33] #3304 prepared endpoint for DDI codebook export of study variables --- .../views/survey-detail.controller.js | 23 +++++++- .../views/survey-detail.html.tmpl | 8 +++ .../resources/variable.resource.js | 15 +++++ pom.xml | 4 ++ .../variablemanagement/domain/CodeBook.java | 8 +++ .../ExportDdiVariablesResourceController.java | 51 ++++++++++++++++ .../service/VariableManagementService.java | 59 +++++++++++++++---- 7 files changed, 157 insertions(+), 11 deletions(-) create mode 100644 src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/domain/CodeBook.java create mode 100644 src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/rest/ExportDdiVariablesResourceController.java diff --git a/mdm-frontend/src/app/legacy/surveymanagement/views/survey-detail.controller.js b/mdm-frontend/src/app/legacy/surveymanagement/views/survey-detail.controller.js index ef66dc214c1..41e92391d88 100644 --- a/mdm-frontend/src/app/legacy/surveymanagement/views/survey-detail.controller.js +++ b/mdm-frontend/src/app/legacy/surveymanagement/views/survey-detail.controller.js @@ -24,6 +24,7 @@ angular.module('metadatamanagementApp') '$stateParams', 'blockUI', '$mdSidenav', + 'ExportDdiVariablesResource', function(entity, LanguageService, CleanJSObjectService, PageMetadataService, $state, BreadcrumbService, MessageBus, SurveySearchService, SurveyAttachmentResource, Principal, @@ -31,7 +32,7 @@ angular.module('metadatamanagementApp') SurveyResponseRateImageUploadService, OutdatedVersionNotifier, DataAcquisitionProjectResource, $mdDialog, ProjectUpdateAccessService, CountryCodesResource, $stateParams, - blockUI, $mdSidenav) { + blockUI, $mdSidenav, ExportDdiVariablesResource) { blockUI.start(); SearchResultNavigatorService .setSearchIndex($stateParams['search-result-index']); @@ -53,6 +54,8 @@ angular.module('metadatamanagementApp') ctrl.responseRateImage = null; ctrl.enableJsonView = Principal .hasAnyAuthority(['ROLE_PUBLISHER', 'ROLE_ADMIN']); + ctrl.enableVariableExport = Principal + .hasAnyAuthority(['ROLE_PUBLISHER', 'ROLE_ADMIN']); entity.promise.then(function(survey) { var fetchFn = SurveySearchService.findShadowByIdAndVersion @@ -174,5 +177,23 @@ angular.module('metadatamanagementApp') ctrl.toggleSidenav = function() { $mdSidenav('SideNavBar').toggle(); }; + + /** + * Exports Variables metadata as DDI Codebook XML. + */ + ctrl.exportVariables = function() { + ExportDdiVariablesResource.exportVariablesAsXml({ + studyId: "foo" + }).then(function(res) { + var blob = new Blob([res],{ + type: "application/xml;charset=utf-8;" + }); + var downloadLink = document.createElement('a'); + downloadLink.setAttribute('download', 'variable_ddi_mdm_export.xml'); + downloadLink.setAttribute('href', window.URL.createObjectURL(blob)); + downloadLink.click(); + $scope.isDownloadingData = false; + }) + } }]); diff --git a/mdm-frontend/src/app/legacy/surveymanagement/views/survey-detail.html.tmpl b/mdm-frontend/src/app/legacy/surveymanagement/views/survey-detail.html.tmpl index bbae6877755..896dc20adc3 100644 --- a/mdm-frontend/src/app/legacy/surveymanagement/views/survey-detail.html.tmpl +++ b/mdm-frontend/src/app/legacy/surveymanagement/views/survey-detail.html.tmpl @@ -10,6 +10,14 @@

+ + + Daten als XML exportieren + + DDI + diff --git a/mdm-frontend/src/app/legacy/variablemanagement/resources/variable.resource.js b/mdm-frontend/src/app/legacy/variablemanagement/resources/variable.resource.js index cb060358881..9704ceea70c 100644 --- a/mdm-frontend/src/app/legacy/variablemanagement/resources/variable.resource.js +++ b/mdm-frontend/src/app/legacy/variablemanagement/resources/variable.resource.js @@ -17,3 +17,18 @@ angular.module('metadatamanagementApp') }); }]); + /** + * Method to request the endpoint that downloads all variables of the study as a DDI-formatted XML file. + * We could not put this into the "VariableResource"-schema (above), because the URLs would not match. + */ + angular.module('metadatamanagementApp').factory('ExportDdiVariablesResource', ['$http', + function($http) { + return { + exportVariablesAsXml: function() { + return $http.get('api/variables/exportDDI/xml/:studyId').then(function(response) { + return response.data; + }); + } + } + }]); + diff --git a/pom.xml b/pom.xml index 7632a09471b..32a6413cbc2 100644 --- a/pom.xml +++ b/pom.xml @@ -66,6 +66,10 @@ com.fasterxml.jackson.core jackson-databind + + com.fasterxml.jackson.dataformat + jackson-dataformat-xml + com.github.cloudyrock.mongock mongock-spring-v5 diff --git a/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/domain/CodeBook.java b/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/domain/CodeBook.java new file mode 100644 index 00000000000..458c10b8efd --- /dev/null +++ b/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/domain/CodeBook.java @@ -0,0 +1,8 @@ +package eu.dzhw.fdz.metadatamanagement.variablemanagement.domain; + +public record CodeBook(String name, String address) { + public CodeBook(String name, String address) { + this.name = name; + this.address = address; + } +} diff --git a/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/rest/ExportDdiVariablesResourceController.java b/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/rest/ExportDdiVariablesResourceController.java new file mode 100644 index 00000000000..3adf8f5bb5e --- /dev/null +++ b/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/rest/ExportDdiVariablesResourceController.java @@ -0,0 +1,51 @@ +package eu.dzhw.fdz.metadatamanagement.variablemanagement.rest; + +import java.io.IOException; + +import javax.persistence.PersistenceException; + +import eu.dzhw.fdz.metadatamanagement.usermanagement.security.AuthoritiesConstants; +import eu.dzhw.fdz.metadatamanagement.variablemanagement.service.VariableManagementService; +import lombok.RequiredArgsConstructor; + +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.security.access.annotation.Secured; +import org.springframework.stereotype.Controller; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.ResponseBody; + +/** + * Controller for exporting variable metadata as DDI codebook. + */ +@Controller +@RequestMapping("/api") +@RequiredArgsConstructor +public class ExportDdiVariablesResourceController { + + private final VariableManagementService variablesManagementService; + + /** + * Exports all variables metadata belonging to the given survey ID as DDI Codebook. + * + * @return an XML file + */ + @GetMapping(value = "/variables/exportDDI/xml/{studyId:.+}", produces = MediaType.APPLICATION_XML_VALUE) + @ResponseBody + @Secured(value = {AuthoritiesConstants.PUBLISHER, AuthoritiesConstants.DATA_PROVIDER}) + public ResponseEntity exportVariablesAsXml(@PathVariable String studyId) { + ResponseEntity response = new ResponseEntity<>(null, null, HttpStatus.NOT_FOUND); + try { + response = this.variablesManagementService.exportDdiVariablesAsXML(studyId); + } catch (PersistenceException ex) { + return response; + } + + return response; + } + +} diff --git a/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/service/VariableManagementService.java b/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/service/VariableManagementService.java index ad331fbf77a..0829d50bcb3 100644 --- a/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/service/VariableManagementService.java +++ b/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/service/VariableManagementService.java @@ -1,17 +1,25 @@ package eu.dzhw.fdz.metadatamanagement.variablemanagement.service; +import java.io.IOException; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; +import org.springframework.core.io.ByteArrayResource; import org.springframework.data.rest.core.annotation.HandleAfterCreate; import org.springframework.data.rest.core.annotation.HandleAfterDelete; import org.springframework.data.rest.core.annotation.HandleAfterSave; import org.springframework.data.rest.core.annotation.RepositoryEventHandler; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; import org.springframework.security.access.annotation.Secured; import org.springframework.stereotype.Service; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.xml.XmlMapper; import eu.dzhw.fdz.metadatamanagement.common.service.CrudService; import eu.dzhw.fdz.metadatamanagement.conceptmanagement.domain.Concept; import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.DataPackage; @@ -24,19 +32,22 @@ import eu.dzhw.fdz.metadatamanagement.searchmanagement.service.ElasticsearchUpdateQueueService; import eu.dzhw.fdz.metadatamanagement.surveymanagement.domain.Survey; import eu.dzhw.fdz.metadatamanagement.usermanagement.security.AuthoritiesConstants; +import eu.dzhw.fdz.metadatamanagement.variablemanagement.domain.CodeBook; import eu.dzhw.fdz.metadatamanagement.variablemanagement.domain.Variable; import eu.dzhw.fdz.metadatamanagement.variablemanagement.repository.VariableRepository; import eu.dzhw.fdz.metadatamanagement.variablemanagement.service.helper.VariableCrudHelper; import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; /** * Service for managing the domain object/aggregate {@link Variable}. - * + * * @author René Reitmann */ @Service @RepositoryEventHandler @RequiredArgsConstructor +@Slf4j public class VariableManagementService implements CrudService { private final QuestionRepository questionRepository; @@ -49,7 +60,7 @@ public class VariableManagementService implements CrudService { /** * Delete all variables when the dataAcquisitionProject was deleted. - * + * * @param dataAcquisitionProject the dataAcquisitionProject which has been deleted. */ @HandleAfterDelete @@ -59,7 +70,7 @@ public void onDataAcquisitionProjectDeleted(DataAcquisitionProject dataAcquisiti /** * Update all variables of the project, when the project is released. - * + * * @param dataAcquisitionProject the changed project */ @HandleAfterSave @@ -72,7 +83,7 @@ public void onDataAcquisitionProjectUpdated(DataAcquisitionProject dataAcquisiti /** * A service method for deletion of variables within a data acquisition project. - * + * * @param dataAcquisitionProjectId the id for to the data acquisition project. */ @Secured(value = {AuthoritiesConstants.PUBLISHER, AuthoritiesConstants.DATA_PROVIDER}) @@ -88,7 +99,7 @@ public void deleteAllVariablesByProjectId(String dataAcquisitionProjectId) { /** * Enqueue update of variable search documents when the data set is changed. - * + * * @param dataSet the updated, created or deleted data set. */ @HandleAfterCreate @@ -102,7 +113,7 @@ public void onDataSetChanged(DataSet dataSet) { /** * Enqueue update of variable search documents when the dataPackage is changed. - * + * * @param dataPackage the updated, created or deleted dataPackage. */ @HandleAfterCreate @@ -116,7 +127,7 @@ public void onDataPackageChanged(DataPackage dataPackage) { /** * Enqueue update of variable search documents when the instrument is changed. - * + * * @param instrument the updated, created or deleted instrument. */ @HandleAfterCreate @@ -130,7 +141,7 @@ public void onInstrumentChanged(Instrument instrument) { /** * Enqueue update of variable search documents when the question is changed. - * + * * @param question the updated, created or deleted question. */ @HandleAfterCreate @@ -144,7 +155,7 @@ public void onQuestionChanged(Question question) { /** * Enqueue update of variable search documents when the survey is updated. - * + * * @param survey the updated, created or deleted survey. */ @HandleAfterCreate @@ -158,7 +169,7 @@ public void onSurveyChanged(Survey survey) { /** * Enqueue update of variable search documents when the concept is changed. - * + * * @param concept the updated, created or deleted concept. */ @HandleAfterCreate @@ -202,4 +213,32 @@ public Optional read(String id) { public Optional readSearchDocument(String id) { return crudHelper.readSearchDocument(id); } + + /** + * Exports all variables belonging to a given study according to the DDI Codebook standard. + * @param studyId the ID of the study + * @return DDI metadata as XML + */ + public ResponseEntity exportDdiVariablesAsXML(String studyId) { + try { + CodeBook variableMetadata = this.getDdiVariablesMetadata(); + XmlMapper mapper = new XmlMapper(); + ByteArrayResource resource = new ByteArrayResource(mapper.writeValueAsBytes(variableMetadata)); + HttpHeaders headers = new HttpHeaders(); + headers.add("Content-Disposition", "attachment; filename=Variables_PID_MDM_Export.xml"); + return ResponseEntity.ok() + .headers(headers) + .body(resource); + } catch (IOException ex) { + return new ResponseEntity<>(null, null, HttpStatus.NOT_FOUND); + } + } + + /** + * Collects metadata according to DDI Codebook standard. + * @return the metadata + */ + private CodeBook getDdiVariablesMetadata() throws JsonProcessingException { + return new CodeBook("Foo", "bar"); + } } From 2b3ad054670017750fe22e0d5d5f9d5f1b913f9f Mon Sep 17 00:00:00 2001 From: theresa Date: Fri, 9 Aug 2024 15:14:28 +0200 Subject: [PATCH 12/33] #3304 implemented metadata class and basic export functionality --- .../views/survey-detail.controller.js | 9 ++- .../resources/variable.resource.js | 4 +- .../variablemanagement/domain/CodeBook.java | 66 +++++++++++++++- .../ExportDdiVariablesResourceController.java | 6 +- .../service/VariableManagementService.java | 75 +++++++++++++++++-- 5 files changed, 142 insertions(+), 18 deletions(-) diff --git a/mdm-frontend/src/app/legacy/surveymanagement/views/survey-detail.controller.js b/mdm-frontend/src/app/legacy/surveymanagement/views/survey-detail.controller.js index 41e92391d88..9b2f1656c81 100644 --- a/mdm-frontend/src/app/legacy/surveymanagement/views/survey-detail.controller.js +++ b/mdm-frontend/src/app/legacy/surveymanagement/views/survey-detail.controller.js @@ -56,12 +56,17 @@ angular.module('metadatamanagementApp') .hasAnyAuthority(['ROLE_PUBLISHER', 'ROLE_ADMIN']); ctrl.enableVariableExport = Principal .hasAnyAuthority(['ROLE_PUBLISHER', 'ROLE_ADMIN']); + ctrl.surveyId = null; entity.promise.then(function(survey) { var fetchFn = SurveySearchService.findShadowByIdAndVersion .bind(null, survey.masterId, null, ['nested*','variables','questions', 'instruments', 'dataSets', 'relatedPublications','concepts']); OutdatedVersionNotifier.checkVersionAndNotify(survey, fetchFn); +; + if (survey.release) { + ctrl.surveyId = survey.id + '-' + survey.release.version; + } if (Principal .hasAnyAuthority(['ROLE_PUBLISHER', 'ROLE_DATA_PROVIDER'])) { @@ -182,9 +187,7 @@ angular.module('metadatamanagementApp') * Exports Variables metadata as DDI Codebook XML. */ ctrl.exportVariables = function() { - ExportDdiVariablesResource.exportVariablesAsXml({ - studyId: "foo" - }).then(function(res) { + ExportDdiVariablesResource.exportVariablesAsXml(ctrl.surveyId).then(function(res) { var blob = new Blob([res],{ type: "application/xml;charset=utf-8;" }); diff --git a/mdm-frontend/src/app/legacy/variablemanagement/resources/variable.resource.js b/mdm-frontend/src/app/legacy/variablemanagement/resources/variable.resource.js index 9704ceea70c..4cf7ef21b40 100644 --- a/mdm-frontend/src/app/legacy/variablemanagement/resources/variable.resource.js +++ b/mdm-frontend/src/app/legacy/variablemanagement/resources/variable.resource.js @@ -24,8 +24,8 @@ angular.module('metadatamanagementApp') angular.module('metadatamanagementApp').factory('ExportDdiVariablesResource', ['$http', function($http) { return { - exportVariablesAsXml: function() { - return $http.get('api/variables/exportDDI/xml/:studyId').then(function(response) { + exportVariablesAsXml: function(surveyId) { + return $http.get('api/variables/exportDDI/xml/' + surveyId).then(function(response) { return response.data; }); } diff --git a/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/domain/CodeBook.java b/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/domain/CodeBook.java index 458c10b8efd..8e2f86e6116 100644 --- a/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/domain/CodeBook.java +++ b/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/domain/CodeBook.java @@ -1,8 +1,66 @@ package eu.dzhw.fdz.metadatamanagement.variablemanagement.domain; -public record CodeBook(String name, String address) { - public CodeBook(String name, String address) { - this.name = name; - this.address = address; +import java.util.List; + +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement; + +/** + * Record defining the mapping of the DDI Codebook standard. + */ +@JacksonXmlRootElement(namespace = "ddi:codebook:2_5", localName = "codeBook") +public class CodeBook { + public CodeBook(StdyDscr stdyDscr, DataDscr dataDscr) { + this.stdyDscr = stdyDscr; + this.dataDscr = dataDscr; + } + + @JacksonXmlProperty(localName = "stdyDscr") + private StdyDscr stdyDscr; + @JacksonXmlProperty(localName = "dataDscr") + private DataDscr dataDscr; + + public static class StdyDscr { + public StdyDscr(Citation citation) { + this.citation = citation; + } + + @JacksonXmlProperty(localName = "citation") + Citation citation; + + public static class Citation { + public Citation(TitlStmt titlStmt) { + this.titlStmt = titlStmt; + } + + @JacksonXmlProperty(localName = "titlStmt") + private TitlStmt titlStmt; + + public static class TitlStmt { + public TitlStmt(String titl) { + this.titl = titl; + } + + @JacksonXmlProperty(localName = "titl") + private String titl; + } + } + } + + public static class DataDscr { + public DataDscr(List var) { + this.var = var; + } + + @JacksonXmlProperty(localName = "var") + List var; + + public static class Var { + public Var(String name) { + this.name = name; + } + @JacksonXmlProperty(localName = "name", isAttribute = true) + String name; } + } } diff --git a/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/rest/ExportDdiVariablesResourceController.java b/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/rest/ExportDdiVariablesResourceController.java index 3adf8f5bb5e..6ecf9bf1814 100644 --- a/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/rest/ExportDdiVariablesResourceController.java +++ b/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/rest/ExportDdiVariablesResourceController.java @@ -34,13 +34,13 @@ public class ExportDdiVariablesResourceController { * * @return an XML file */ - @GetMapping(value = "/variables/exportDDI/xml/{studyId:.+}", produces = MediaType.APPLICATION_XML_VALUE) + @GetMapping(value = "/variables/exportDDI/xml/{surveyId:.+}", produces = MediaType.APPLICATION_XML_VALUE) @ResponseBody @Secured(value = {AuthoritiesConstants.PUBLISHER, AuthoritiesConstants.DATA_PROVIDER}) - public ResponseEntity exportVariablesAsXml(@PathVariable String studyId) { + public ResponseEntity exportVariablesAsXml(@PathVariable String surveyId) { ResponseEntity response = new ResponseEntity<>(null, null, HttpStatus.NOT_FOUND); try { - response = this.variablesManagementService.exportDdiVariablesAsXML(studyId); + response = this.variablesManagementService.exportDdiVariablesAsXML(surveyId); } catch (PersistenceException ex) { return response; } diff --git a/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/service/VariableManagementService.java b/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/service/VariableManagementService.java index 0829d50bcb3..51ad5cd72ce 100644 --- a/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/service/VariableManagementService.java +++ b/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/service/VariableManagementService.java @@ -1,6 +1,9 @@ package eu.dzhw.fdz.metadatamanagement.variablemanagement.service; import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -20,6 +23,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.dataformat.xml.XmlMapper; +import com.google.gson.Gson; import eu.dzhw.fdz.metadatamanagement.common.service.CrudService; import eu.dzhw.fdz.metadatamanagement.conceptmanagement.domain.Concept; import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.DataPackage; @@ -28,16 +32,28 @@ import eu.dzhw.fdz.metadatamanagement.projectmanagement.domain.DataAcquisitionProject; import eu.dzhw.fdz.metadatamanagement.questionmanagement.domain.Question; import eu.dzhw.fdz.metadatamanagement.questionmanagement.repository.QuestionRepository; +import eu.dzhw.fdz.metadatamanagement.searchmanagement.dao.exception.ElasticsearchIoException; +import eu.dzhw.fdz.metadatamanagement.searchmanagement.documents.SurveySearchDocument; +import eu.dzhw.fdz.metadatamanagement.searchmanagement.documents.VariableSearchDocument; import eu.dzhw.fdz.metadatamanagement.searchmanagement.service.ElasticsearchType; import eu.dzhw.fdz.metadatamanagement.searchmanagement.service.ElasticsearchUpdateQueueService; import eu.dzhw.fdz.metadatamanagement.surveymanagement.domain.Survey; import eu.dzhw.fdz.metadatamanagement.usermanagement.security.AuthoritiesConstants; -import eu.dzhw.fdz.metadatamanagement.variablemanagement.domain.CodeBook; import eu.dzhw.fdz.metadatamanagement.variablemanagement.domain.Variable; +import eu.dzhw.fdz.metadatamanagement.variablemanagement.domain.CodeBook; import eu.dzhw.fdz.metadatamanagement.variablemanagement.repository.VariableRepository; import eu.dzhw.fdz.metadatamanagement.variablemanagement.service.helper.VariableCrudHelper; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.bson.types.Code; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.builder.SearchSourceBuilder; /** * Service for managing the domain object/aggregate {@link Variable}. @@ -58,6 +74,10 @@ public class VariableManagementService implements CrudService { private final VariableCrudHelper crudHelper; + private final RestHighLevelClient client; + + private final Gson gson; + /** * Delete all variables when the dataAcquisitionProject was deleted. * @@ -216,12 +236,12 @@ public Optional readSearchDocument(String id) { /** * Exports all variables belonging to a given study according to the DDI Codebook standard. - * @param studyId the ID of the study + * @param surveyId the ID of the study * @return DDI metadata as XML */ - public ResponseEntity exportDdiVariablesAsXML(String studyId) { + public ResponseEntity exportDdiVariablesAsXML(String surveyId) { try { - CodeBook variableMetadata = this.getDdiVariablesMetadata(); + CodeBook variableMetadata = this.getDdiVariablesMetadata(surveyId); XmlMapper mapper = new XmlMapper(); ByteArrayResource resource = new ByteArrayResource(mapper.writeValueAsBytes(variableMetadata)); HttpHeaders headers = new HttpHeaders(); @@ -238,7 +258,50 @@ public ResponseEntity exportDdiVariablesAsXML(String studyId) { * Collects metadata according to DDI Codebook standard. * @return the metadata */ - private CodeBook getDdiVariablesMetadata() throws JsonProcessingException { - return new CodeBook("Foo", "bar"); + private CodeBook getDdiVariablesMetadata(String surveyId) throws JsonProcessingException { + SearchRequest surveyRequest = new SearchRequest(); + SearchSourceBuilder builderSurveys = new SearchSourceBuilder(); + builderSurveys.query(QueryBuilders.termsQuery("id", surveyId)); + surveyRequest.source(builderSurveys); + surveyRequest.indices("surveys"); + + SearchRequest variablesRequest = new SearchRequest(); + SearchSourceBuilder builderVariables = new SearchSourceBuilder(); + builderVariables.query(QueryBuilders.termsQuery("surveyIds", surveyId)) + .size(10000); + variablesRequest.source(builderVariables); + variablesRequest.indices("variables"); + try { + SearchResponse surveyResponse = client.search(surveyRequest, RequestOptions.DEFAULT); + List surveyHits = Arrays.asList(surveyResponse.getHits().getHits()); + if (surveyHits.size() == 0) { + throw new ElasticsearchException(String.format("Could not find survey with id '%s'", surveyId)); + } + SurveySearchDocument surveyDoc = gson.fromJson( + surveyHits.get(0).getSourceAsString(), SurveySearchDocument.class); + // toDo: Decide on language usage + CodeBook.StdyDscr.Citation.TitlStmt titlStmt = new CodeBook.StdyDscr.Citation.TitlStmt(surveyDoc.getTitle().getEn()); + CodeBook.StdyDscr.Citation citation = new CodeBook.StdyDscr.Citation(titlStmt); + CodeBook.StdyDscr study = new CodeBook.StdyDscr(citation); + SearchResponse variableResponse = client.search(variablesRequest, RequestOptions.DEFAULT); + List hits = Arrays.asList(variableResponse.getHits().getHits()); + if (hits.size() > 0) { + log.info(String.format("Found %d variables for survey '%s'", hits.size(),surveyId)); + List variableList = new ArrayList<>(); + for (var variable : hits) { + VariableSearchDocument variableDoc = gson.fromJson( + variable.getSourceAsString(), VariableSearchDocument.class); + CodeBook.DataDscr.Var varMetadata = new CodeBook.DataDscr.Var(variableDoc.getName()); + variableList.add(varMetadata); + } + CodeBook.DataDscr dataDscr = new CodeBook.DataDscr(variableList); + return new CodeBook(study, dataDscr); + } else { + log.info(String.format("No variables found for studyId '%s'", surveyId)); + } + } catch (IOException e) { + throw new ElasticsearchIoException(e); + } + return null; } } From f2d02c9a16535aea61a20ee95a462ab5280a5c00 Mon Sep 17 00:00:00 2001 From: theresa Date: Mon, 16 Sep 2024 12:30:07 +0200 Subject: [PATCH 13/33] #3304 implemented classes for DDI codebook mapping --- .../domain/ddiCodebook/Catgry.java | 17 +++++ .../domain/ddiCodebook/Citation.java | 12 ++++ .../domain/ddiCodebook/CodeBook.java | 25 +++++++ .../domain/ddiCodebook/DataDscr.java | 14 ++++ .../domain/ddiCodebook/FileDscr.java | 14 ++++ .../domain/ddiCodebook/FileTxt.java | 15 +++++ .../domain/ddiCodebook/LanguageEnum.java | 19 ++++++ .../domain/ddiCodebook/StdyDscr.java | 12 ++++ .../domain/ddiCodebook/TextElement.java | 28 ++++++++ .../domain/ddiCodebook/TitlStmt.java | 15 +++++ .../domain/ddiCodebook/Var.java | 26 ++++++++ .../variablemanagement/domain/CodeBook.java | 66 ------------------- .../ExportDdiVariablesResourceController.java | 51 -------------- 13 files changed, 197 insertions(+), 117 deletions(-) create mode 100644 src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/Catgry.java create mode 100644 src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/Citation.java create mode 100644 src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/CodeBook.java create mode 100644 src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/DataDscr.java create mode 100644 src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/FileDscr.java create mode 100644 src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/FileTxt.java create mode 100644 src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/LanguageEnum.java create mode 100644 src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/StdyDscr.java create mode 100644 src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/TextElement.java create mode 100644 src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/TitlStmt.java create mode 100644 src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/Var.java delete mode 100644 src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/domain/CodeBook.java delete mode 100644 src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/rest/ExportDdiVariablesResourceController.java diff --git a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/Catgry.java b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/Catgry.java new file mode 100644 index 00000000000..e6acdbcb9b3 --- /dev/null +++ b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/Catgry.java @@ -0,0 +1,17 @@ +package eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddiCodebook; + +import java.util.List; + +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; +import lombok.AllArgsConstructor; + +@AllArgsConstructor +public class Catgry { + + @JacksonXmlProperty(localName = "catValu") + String catValu; + + @JacksonXmlProperty(localName = "labl") + List labl; + +} diff --git a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/Citation.java b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/Citation.java new file mode 100644 index 00000000000..716c886175e --- /dev/null +++ b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/Citation.java @@ -0,0 +1,12 @@ +package eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddiCodebook; + +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; +import lombok.AllArgsConstructor; + +@AllArgsConstructor +public class Citation { + + @JacksonXmlProperty(localName = "titlStmt") + private TitlStmt titlStmt; + +} diff --git a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/CodeBook.java b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/CodeBook.java new file mode 100644 index 00000000000..dff4a1cafd5 --- /dev/null +++ b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/CodeBook.java @@ -0,0 +1,25 @@ +package eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddiCodebook; + +import java.util.List; + +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement; +import lombok.AllArgsConstructor; + +/** + * Record defining the mapping of the DDI Codebook standard. + */ +@AllArgsConstructor +@JacksonXmlRootElement(namespace = "ddi:codebook:2_5", localName = "codeBook") +public class CodeBook { + + @JacksonXmlProperty(localName = "stdyDscr") + private StdyDscr stdyDscr; + + @JacksonXmlProperty(localName = "fileDscr") + private List fileDscr; + + @JacksonXmlProperty(localName = "dataDscr") + private DataDscr dataDscr; + +} diff --git a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/DataDscr.java b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/DataDscr.java new file mode 100644 index 00000000000..9a4e0a57e21 --- /dev/null +++ b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/DataDscr.java @@ -0,0 +1,14 @@ +package eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddiCodebook; + +import java.util.List; + +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; +import lombok.AllArgsConstructor; + +@AllArgsConstructor +public class DataDscr { + + @JacksonXmlProperty(localName = "var") + List var; + +} diff --git a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/FileDscr.java b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/FileDscr.java new file mode 100644 index 00000000000..c4ede7e849a --- /dev/null +++ b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/FileDscr.java @@ -0,0 +1,14 @@ +package eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddiCodebook; + +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; +import lombok.AllArgsConstructor; + +@AllArgsConstructor +public class FileDscr { + + @JacksonXmlProperty(localName = "ID", isAttribute = true) + String id; + + @JacksonXmlProperty(localName = "fileTxt") + FileTxt fileTxt; +} diff --git a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/FileTxt.java b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/FileTxt.java new file mode 100644 index 00000000000..5f9e18db2ba --- /dev/null +++ b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/FileTxt.java @@ -0,0 +1,15 @@ +package eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddiCodebook; + +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; +import lombok.AllArgsConstructor; + +@AllArgsConstructor +public class FileTxt { + + @JacksonXmlProperty(localName = "fileName") + String fileName; + + @JacksonXmlProperty(localName = "fileCont") + TextElement fileCont; + +} diff --git a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/LanguageEnum.java b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/LanguageEnum.java new file mode 100644 index 00000000000..c3c914ab36c --- /dev/null +++ b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/LanguageEnum.java @@ -0,0 +1,19 @@ +package eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddiCodebook; + +public enum LanguageEnum { + de("de"), en("en"); + + /** + * The language string + */ + public final String languageString; + + /** + * Construct the enum. + * + * @param languageString The language string. + */ + LanguageEnum(String languageString) { + this.languageString = languageString; + } +} diff --git a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/StdyDscr.java b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/StdyDscr.java new file mode 100644 index 00000000000..08bf01808e0 --- /dev/null +++ b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/StdyDscr.java @@ -0,0 +1,12 @@ +package eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddiCodebook; + +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; +import lombok.AllArgsConstructor; + +@AllArgsConstructor +public class StdyDscr { + + @JacksonXmlProperty(localName = "citation") + Citation citation; + +} diff --git a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/TextElement.java b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/TextElement.java new file mode 100644 index 00000000000..e38c29434b7 --- /dev/null +++ b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/TextElement.java @@ -0,0 +1,28 @@ +package eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddiCodebook; + +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlText; +import lombok.AllArgsConstructor; + +/** + * General element for a language annotated XML element containing a string, + * e.g. labels (labl in DDI standard). + */ +@AllArgsConstructor +public class TextElement { + + @JacksonXmlProperty(localName = "xml:lang", isAttribute = true) + private LanguageEnum lang; + + @JacksonXmlText + private String value; + + public String getValue() { + return value; + } + + public void setValue(String value) { + this.value = value; + } + +} diff --git a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/TitlStmt.java b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/TitlStmt.java new file mode 100644 index 00000000000..d5172cfd010 --- /dev/null +++ b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/TitlStmt.java @@ -0,0 +1,15 @@ +package eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddiCodebook; + +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; +import lombok.AllArgsConstructor; + +@AllArgsConstructor +public class TitlStmt { + + @JacksonXmlProperty(localName = "titl") + private TextElement titl; + + @JacksonXmlProperty(localName = "parTitl") + private TextElement parTitle; + +} diff --git a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/Var.java b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/Var.java new file mode 100644 index 00000000000..27bc7970bbe --- /dev/null +++ b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/Var.java @@ -0,0 +1,26 @@ +package eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddiCodebook; + +import java.util.List; + +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; +import lombok.AllArgsConstructor; + +@AllArgsConstructor +public class Var { + + @JacksonXmlProperty(localName = "name", isAttribute = true) + String name; + + @JacksonXmlProperty(localName = "files", isAttribute = true) + String files; + + @JacksonXmlProperty(localName = "labl") + List labl; + + @JacksonXmlProperty(localName = "qstn") + List qstn; + + @JacksonXmlProperty(localName = "catgry") + List catgry; + +} diff --git a/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/domain/CodeBook.java b/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/domain/CodeBook.java deleted file mode 100644 index 8e2f86e6116..00000000000 --- a/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/domain/CodeBook.java +++ /dev/null @@ -1,66 +0,0 @@ -package eu.dzhw.fdz.metadatamanagement.variablemanagement.domain; - -import java.util.List; - -import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; -import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement; - -/** - * Record defining the mapping of the DDI Codebook standard. - */ -@JacksonXmlRootElement(namespace = "ddi:codebook:2_5", localName = "codeBook") -public class CodeBook { - public CodeBook(StdyDscr stdyDscr, DataDscr dataDscr) { - this.stdyDscr = stdyDscr; - this.dataDscr = dataDscr; - } - - @JacksonXmlProperty(localName = "stdyDscr") - private StdyDscr stdyDscr; - @JacksonXmlProperty(localName = "dataDscr") - private DataDscr dataDscr; - - public static class StdyDscr { - public StdyDscr(Citation citation) { - this.citation = citation; - } - - @JacksonXmlProperty(localName = "citation") - Citation citation; - - public static class Citation { - public Citation(TitlStmt titlStmt) { - this.titlStmt = titlStmt; - } - - @JacksonXmlProperty(localName = "titlStmt") - private TitlStmt titlStmt; - - public static class TitlStmt { - public TitlStmt(String titl) { - this.titl = titl; - } - - @JacksonXmlProperty(localName = "titl") - private String titl; - } - } - } - - public static class DataDscr { - public DataDscr(List var) { - this.var = var; - } - - @JacksonXmlProperty(localName = "var") - List var; - - public static class Var { - public Var(String name) { - this.name = name; - } - @JacksonXmlProperty(localName = "name", isAttribute = true) - String name; - } - } -} diff --git a/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/rest/ExportDdiVariablesResourceController.java b/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/rest/ExportDdiVariablesResourceController.java deleted file mode 100644 index 6ecf9bf1814..00000000000 --- a/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/rest/ExportDdiVariablesResourceController.java +++ /dev/null @@ -1,51 +0,0 @@ -package eu.dzhw.fdz.metadatamanagement.variablemanagement.rest; - -import java.io.IOException; - -import javax.persistence.PersistenceException; - -import eu.dzhw.fdz.metadatamanagement.usermanagement.security.AuthoritiesConstants; -import eu.dzhw.fdz.metadatamanagement.variablemanagement.service.VariableManagementService; -import lombok.RequiredArgsConstructor; - -import org.springframework.http.HttpStatus; -import org.springframework.http.MediaType; -import org.springframework.http.ResponseEntity; -import org.springframework.security.access.annotation.Secured; -import org.springframework.stereotype.Controller; -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.PathVariable; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.ResponseBody; - -/** - * Controller for exporting variable metadata as DDI codebook. - */ -@Controller -@RequestMapping("/api") -@RequiredArgsConstructor -public class ExportDdiVariablesResourceController { - - private final VariableManagementService variablesManagementService; - - /** - * Exports all variables metadata belonging to the given survey ID as DDI Codebook. - * - * @return an XML file - */ - @GetMapping(value = "/variables/exportDDI/xml/{surveyId:.+}", produces = MediaType.APPLICATION_XML_VALUE) - @ResponseBody - @Secured(value = {AuthoritiesConstants.PUBLISHER, AuthoritiesConstants.DATA_PROVIDER}) - public ResponseEntity exportVariablesAsXml(@PathVariable String surveyId) { - ResponseEntity response = new ResponseEntity<>(null, null, HttpStatus.NOT_FOUND); - try { - response = this.variablesManagementService.exportDdiVariablesAsXML(surveyId); - } catch (PersistenceException ex) { - return response; - } - - return response; - } - -} From 0bdededa8dabddb83f0a22f42ed2a0635f3f25bf Mon Sep 17 00:00:00 2001 From: theresa Date: Mon, 16 Sep 2024 12:31:28 +0200 Subject: [PATCH 14/33] #3304 refactored ddi codebook export --- .../ExportDdiVariablesResourceController.java | 49 +++++ .../service/DataPackageDdiService.java | 188 ++++++++++++++++++ .../service/DataPackageManagementService.java | 23 +++ .../service/VariableManagementService.java | 109 +--------- 4 files changed, 267 insertions(+), 102 deletions(-) create mode 100644 src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/rest/ExportDdiVariablesResourceController.java create mode 100644 src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/service/DataPackageDdiService.java diff --git a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/rest/ExportDdiVariablesResourceController.java b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/rest/ExportDdiVariablesResourceController.java new file mode 100644 index 00000000000..c865ffe82f8 --- /dev/null +++ b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/rest/ExportDdiVariablesResourceController.java @@ -0,0 +1,49 @@ +package eu.dzhw.fdz.metadatamanagement.datapackagemanagement.rest; + +import javax.persistence.PersistenceException; + +import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.service.DataPackageDdiService; +import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.service.DataPackageManagementService; +import eu.dzhw.fdz.metadatamanagement.usermanagement.security.AuthoritiesConstants; +import lombok.RequiredArgsConstructor; + +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.security.access.annotation.Secured; +import org.springframework.stereotype.Controller; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.ResponseBody; + +/** + * Controller for exporting variable metadata as DDI codebook. + */ +@Controller +@RequestMapping("/api") +@RequiredArgsConstructor +public class ExportDdiVariablesResourceController { + + private final DataPackageDdiService dataPackageDdiService; + + /** + * Exports all variables metadata belonging to the given dataPackage ID as DDI Codebook XML file. + * + * @return an XML file + */ + @GetMapping(value = "/data-packages/exportDDI/xml/{dataPackageId:.+}", produces = MediaType.APPLICATION_XML_VALUE) + @ResponseBody + @Secured(value = {AuthoritiesConstants.PUBLISHER, AuthoritiesConstants.DATA_PROVIDER}) + public ResponseEntity exportVariablesAsXml(@PathVariable String dataPackageId) { + ResponseEntity response = new ResponseEntity<>(null, null, HttpStatus.NOT_FOUND); + try { + response = this.dataPackageDdiService.exportDdiVariablesAsXML(dataPackageId); + } catch (PersistenceException ex) { + return response; + } + + return response; + } + +} diff --git a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/service/DataPackageDdiService.java b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/service/DataPackageDdiService.java new file mode 100644 index 00000000000..bfff6659a15 --- /dev/null +++ b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/service/DataPackageDdiService.java @@ -0,0 +1,188 @@ +package eu.dzhw.fdz.metadatamanagement.datapackagemanagement.service; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.dataformat.xml.XmlMapper; +import com.google.gson.Gson; +import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.DataPackage; +import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddiCodebook.Catgry; +import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddiCodebook.Citation; +import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddiCodebook.CodeBook; +import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddiCodebook.DataDscr; +import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddiCodebook.FileDscr; +import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddiCodebook.FileTxt; +import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddiCodebook.LanguageEnum; +import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddiCodebook.StdyDscr; +import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddiCodebook.TextElement; +import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddiCodebook.TitlStmt; +import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddiCodebook.Var; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.elasticsearch.client.RestHighLevelClient; + +import org.springframework.core.io.ByteArrayResource; +import org.springframework.data.rest.core.annotation.RepositoryEventHandler; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.stereotype.Service; + +/** + * Service for construction DDI codebook metadata for all veriables of a {@link DataPackage}. + * @since Sep 2024 + * @author Theresa Möller + */ +@Service +@RepositoryEventHandler +@RequiredArgsConstructor +@Slf4j +public class DataPackageDdiService { + + private final RestHighLevelClient client; + + private final Gson gson; + + /** + * Exports all variables belonging to a given datapackage according to the DDI Codebook standard. + * @param dataPackageId the ID of the study + * @return DDI metadata as XML + */ + public ResponseEntity exportDdiVariablesAsXML(String dataPackageId) { + try { + CodeBook variableMetadata = this.getDdiVariablesMetadata(dataPackageId); + XmlMapper mapper = new XmlMapper(); + ByteArrayResource resource = new ByteArrayResource(mapper.writeValueAsBytes(variableMetadata)); + HttpHeaders headers = new HttpHeaders(); + headers.add("Content-Disposition", "attachment; filename=Variables_PID_MDM_Export.xml"); + return ResponseEntity.ok() + .headers(headers) + .body(resource); + } catch (IOException ex) { + return new ResponseEntity<>(null, null, HttpStatus.NOT_FOUND); + } + } + + /** + * Collects variables metadata according to DDI Codebook standard. + * @return the metadata + */ + private CodeBook getDdiVariablesMetadata(String dataPackageId) throws JsonProcessingException { + StdyDscr stdyDscr = this.getDdiStdyDscr(); + + List fileDscrList = new ArrayList<>(); + // for each dataset of the data package + for (int i = 0; i < 3; i++) { + //todo make a list of all added datasets and do add there variables later on + fileDscrList.add(this.getDdiFileDsrc()); + } + + List varList = new ArrayList<>(); + // for each variable of the data package + for (int j = 0; j < 10; j++) { + varList.add(this.getDdiVar()); + } + DataDscr dataDscr = new DataDscr(varList); + CodeBook codeBook = new CodeBook(stdyDscr, fileDscrList, dataDscr); + + return codeBook; + + +// SearchRequest surveyRequest = new SearchRequest(); +// SearchSourceBuilder builderSurveys = new SearchSourceBuilder(); +// builderSurveys.query(QueryBuilders.termsQuery("id", surveyId)); +// surveyRequest.source(builderSurveys); +// surveyRequest.indices("surveys"); +// +// SearchRequest variablesRequest = new SearchRequest(); +// SearchSourceBuilder builderVariables = new SearchSourceBuilder(); +// builderVariables.query(QueryBuilders.termsQuery("surveyIds", surveyId)) +// .size(10000); +// variablesRequest.source(builderVariables); +// variablesRequest.indices("variables"); +// try { +// SearchResponse surveyResponse = client.search(surveyRequest, RequestOptions.DEFAULT); +// List surveyHits = Arrays.asList(surveyResponse.getHits().getHits()); +// if (surveyHits.size() == 0) { +// throw new ElasticsearchException(String.format("Could not find survey with id '%s'", surveyId)); +// } +// SurveySearchDocument surveyDoc = gson.fromJson( +// surveyHits.get(0).getSourceAsString(), SurveySearchDocument.class); +// // toDo: Decide on language usage +// CodeBook.StdyDscr.Citation.TitlStmt titlStmt = new CodeBook.StdyDscr.Citation.TitlStmt(surveyDoc.getTitle().getEn()); +// CodeBook.StdyDscr.Citation citation = new CodeBook.StdyDscr.Citation(titlStmt); +// CodeBook.StdyDscr study = new CodeBook.StdyDscr(citation); +// SearchResponse variableResponse = client.search(variablesRequest, RequestOptions.DEFAULT); +// List hits = Arrays.asList(variableResponse.getHits().getHits()); +// if (hits.size() > 0) { +// log.info(String.format("Found %d variables for survey '%s'", hits.size(),surveyId)); +// List variableList = new ArrayList<>(); +// for (var variable : hits) { +// VariableSearchDocument variableDoc = gson.fromJson( +// variable.getSourceAsString(), VariableSearchDocument.class); +// CodeBook.DataDscr.Var varMetadata = new CodeBook.DataDscr.Var(variableDoc.getName()); +// variableList.add(varMetadata); +// } +// CodeBook.DataDscr dataDscr = new CodeBook.DataDscr(variableList); +// return new CodeBook(study, dataDscr); +// } else { +// log.info(String.format("No variables found for studyId '%s'", surveyId)); +// } +// } catch (IOException e) { +// throw new ElasticsearchIoException(e); +// } + //return null; + } + + /** + * + * @return + */ + private Var getDdiVar() { + String name = "Variable ID"; + String files = "Dataset ID"; + List varLablList = new ArrayList<>(); + varLablList.add(new TextElement(LanguageEnum.de, "Variable Label in German")); + varLablList.add(new TextElement(LanguageEnum.en, "Variable Label in English")); + List qstnList = new ArrayList<>(); + // possibly empty + qstnList.add(new TextElement(LanguageEnum.de, "Related Question label in German")); + qstnList.add(new TextElement(LanguageEnum.en, "Related Question label in English")); + List catgryList = new ArrayList<>(); + for (int k = 0; k < 5; k++) { + // only for nominal/ordinal + String catValu = "Value"; + List catLablList = new ArrayList<>(); + catLablList.add(new TextElement(LanguageEnum.de, "Value Label in German")); + catLablList.add(new TextElement(LanguageEnum.en, "Value Label in English")); + catgryList.add(new Catgry(catValu, catLablList)); + } + return new Var(name, files, varLablList, qstnList, catgryList); + } + + /** + * + * @return + */ + private FileDscr getDdiFileDsrc() { + String id = "Dataset ID"; + String fileName = "Dataset ID"; + TextElement fileCont = new TextElement(LanguageEnum.de, "Dataset Title in German"); + FileTxt fileTxt = new FileTxt(fileName, fileCont); + return new FileDscr(id, fileTxt); + } + + /** + * + * @return + */ + private StdyDscr getDdiStdyDscr() { + TextElement titl = new TextElement(LanguageEnum.de, "Data Package Title in German"); + TextElement parTitl = new TextElement(LanguageEnum.en, "Data Package Title in English"); + Citation citation = new Citation(new TitlStmt(titl, parTitl)); + return new StdyDscr(citation); + } + +} diff --git a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/service/DataPackageManagementService.java b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/service/DataPackageManagementService.java index aacef3e7ae5..1e2b7f8959f 100644 --- a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/service/DataPackageManagementService.java +++ b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/service/DataPackageManagementService.java @@ -1,22 +1,41 @@ package eu.dzhw.fdz.metadatamanagement.datapackagemanagement.service; +import java.io.IOException; +import java.util.ArrayList; import java.util.List; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; +import org.springframework.core.io.ByteArrayResource; import org.springframework.data.rest.core.annotation.HandleAfterCreate; import org.springframework.data.rest.core.annotation.HandleAfterDelete; import org.springframework.data.rest.core.annotation.HandleAfterSave; import org.springframework.data.rest.core.annotation.RepositoryEventHandler; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; import org.springframework.security.access.annotation.Secured; import org.springframework.stereotype.Service; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.dataformat.xml.XmlMapper; +import com.google.gson.Gson; import eu.dzhw.fdz.metadatamanagement.analysispackagemanagement.domain.AnalysisPackage; import eu.dzhw.fdz.metadatamanagement.common.service.CrudService; import eu.dzhw.fdz.metadatamanagement.conceptmanagement.domain.Concept; import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.DataPackage; +import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddiCodebook.Catgry; +import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddiCodebook.Citation; +import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddiCodebook.DataDscr; +import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddiCodebook.FileDscr; +import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddiCodebook.FileTxt; +import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddiCodebook.LanguageEnum; +import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddiCodebook.StdyDscr; +import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddiCodebook.TextElement; +import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddiCodebook.TitlStmt; +import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddiCodebook.Var; import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.repository.DataPackageRepository; import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.service.helper.DataPackageCrudHelper; import eu.dzhw.fdz.metadatamanagement.datasetmanagement.domain.DataSet; @@ -34,8 +53,11 @@ import eu.dzhw.fdz.metadatamanagement.searchmanagement.service.ElasticsearchUpdateQueueService; import eu.dzhw.fdz.metadatamanagement.surveymanagement.domain.Survey; import eu.dzhw.fdz.metadatamanagement.usermanagement.security.AuthoritiesConstants; +import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddiCodebook.CodeBook; import eu.dzhw.fdz.metadatamanagement.variablemanagement.domain.Variable; import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.elasticsearch.client.RestHighLevelClient; /** * Service for managing the domain object/aggregate {@link DataPackage}. @@ -45,6 +67,7 @@ @Service @RepositoryEventHandler @RequiredArgsConstructor +@Slf4j public class DataPackageManagementService implements CrudService { private final DataPackageRepository dataPackageRepository; diff --git a/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/service/VariableManagementService.java b/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/service/VariableManagementService.java index 51ad5cd72ce..d6017438d6f 100644 --- a/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/service/VariableManagementService.java +++ b/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/service/VariableManagementService.java @@ -1,28 +1,10 @@ package eu.dzhw.fdz.metadatamanagement.variablemanagement.service; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; -import org.springframework.core.io.ByteArrayResource; -import org.springframework.data.rest.core.annotation.HandleAfterCreate; -import org.springframework.data.rest.core.annotation.HandleAfterDelete; -import org.springframework.data.rest.core.annotation.HandleAfterSave; -import org.springframework.data.rest.core.annotation.RepositoryEventHandler; -import org.springframework.http.HttpHeaders; -import org.springframework.http.HttpStatus; -import org.springframework.http.ResponseEntity; -import org.springframework.security.access.annotation.Secured; -import org.springframework.stereotype.Service; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.dataformat.xml.XmlMapper; import com.google.gson.Gson; import eu.dzhw.fdz.metadatamanagement.common.service.CrudService; import eu.dzhw.fdz.metadatamanagement.conceptmanagement.domain.Concept; @@ -32,28 +14,23 @@ import eu.dzhw.fdz.metadatamanagement.projectmanagement.domain.DataAcquisitionProject; import eu.dzhw.fdz.metadatamanagement.questionmanagement.domain.Question; import eu.dzhw.fdz.metadatamanagement.questionmanagement.repository.QuestionRepository; -import eu.dzhw.fdz.metadatamanagement.searchmanagement.dao.exception.ElasticsearchIoException; -import eu.dzhw.fdz.metadatamanagement.searchmanagement.documents.SurveySearchDocument; -import eu.dzhw.fdz.metadatamanagement.searchmanagement.documents.VariableSearchDocument; import eu.dzhw.fdz.metadatamanagement.searchmanagement.service.ElasticsearchType; import eu.dzhw.fdz.metadatamanagement.searchmanagement.service.ElasticsearchUpdateQueueService; import eu.dzhw.fdz.metadatamanagement.surveymanagement.domain.Survey; import eu.dzhw.fdz.metadatamanagement.usermanagement.security.AuthoritiesConstants; import eu.dzhw.fdz.metadatamanagement.variablemanagement.domain.Variable; -import eu.dzhw.fdz.metadatamanagement.variablemanagement.domain.CodeBook; import eu.dzhw.fdz.metadatamanagement.variablemanagement.repository.VariableRepository; import eu.dzhw.fdz.metadatamanagement.variablemanagement.service.helper.VariableCrudHelper; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.bson.types.Code; -import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.RestHighLevelClient; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.search.SearchHit; -import org.elasticsearch.search.builder.SearchSourceBuilder; + +import org.springframework.data.rest.core.annotation.HandleAfterCreate; +import org.springframework.data.rest.core.annotation.HandleAfterDelete; +import org.springframework.data.rest.core.annotation.HandleAfterSave; +import org.springframework.data.rest.core.annotation.RepositoryEventHandler; +import org.springframework.security.access.annotation.Secured; +import org.springframework.stereotype.Service; /** * Service for managing the domain object/aggregate {@link Variable}. @@ -63,7 +40,6 @@ @Service @RepositoryEventHandler @RequiredArgsConstructor -@Slf4j public class VariableManagementService implements CrudService { private final QuestionRepository questionRepository; @@ -233,75 +209,4 @@ public Optional read(String id) { public Optional readSearchDocument(String id) { return crudHelper.readSearchDocument(id); } - - /** - * Exports all variables belonging to a given study according to the DDI Codebook standard. - * @param surveyId the ID of the study - * @return DDI metadata as XML - */ - public ResponseEntity exportDdiVariablesAsXML(String surveyId) { - try { - CodeBook variableMetadata = this.getDdiVariablesMetadata(surveyId); - XmlMapper mapper = new XmlMapper(); - ByteArrayResource resource = new ByteArrayResource(mapper.writeValueAsBytes(variableMetadata)); - HttpHeaders headers = new HttpHeaders(); - headers.add("Content-Disposition", "attachment; filename=Variables_PID_MDM_Export.xml"); - return ResponseEntity.ok() - .headers(headers) - .body(resource); - } catch (IOException ex) { - return new ResponseEntity<>(null, null, HttpStatus.NOT_FOUND); - } - } - - /** - * Collects metadata according to DDI Codebook standard. - * @return the metadata - */ - private CodeBook getDdiVariablesMetadata(String surveyId) throws JsonProcessingException { - SearchRequest surveyRequest = new SearchRequest(); - SearchSourceBuilder builderSurveys = new SearchSourceBuilder(); - builderSurveys.query(QueryBuilders.termsQuery("id", surveyId)); - surveyRequest.source(builderSurveys); - surveyRequest.indices("surveys"); - - SearchRequest variablesRequest = new SearchRequest(); - SearchSourceBuilder builderVariables = new SearchSourceBuilder(); - builderVariables.query(QueryBuilders.termsQuery("surveyIds", surveyId)) - .size(10000); - variablesRequest.source(builderVariables); - variablesRequest.indices("variables"); - try { - SearchResponse surveyResponse = client.search(surveyRequest, RequestOptions.DEFAULT); - List surveyHits = Arrays.asList(surveyResponse.getHits().getHits()); - if (surveyHits.size() == 0) { - throw new ElasticsearchException(String.format("Could not find survey with id '%s'", surveyId)); - } - SurveySearchDocument surveyDoc = gson.fromJson( - surveyHits.get(0).getSourceAsString(), SurveySearchDocument.class); - // toDo: Decide on language usage - CodeBook.StdyDscr.Citation.TitlStmt titlStmt = new CodeBook.StdyDscr.Citation.TitlStmt(surveyDoc.getTitle().getEn()); - CodeBook.StdyDscr.Citation citation = new CodeBook.StdyDscr.Citation(titlStmt); - CodeBook.StdyDscr study = new CodeBook.StdyDscr(citation); - SearchResponse variableResponse = client.search(variablesRequest, RequestOptions.DEFAULT); - List hits = Arrays.asList(variableResponse.getHits().getHits()); - if (hits.size() > 0) { - log.info(String.format("Found %d variables for survey '%s'", hits.size(),surveyId)); - List variableList = new ArrayList<>(); - for (var variable : hits) { - VariableSearchDocument variableDoc = gson.fromJson( - variable.getSourceAsString(), VariableSearchDocument.class); - CodeBook.DataDscr.Var varMetadata = new CodeBook.DataDscr.Var(variableDoc.getName()); - variableList.add(varMetadata); - } - CodeBook.DataDscr dataDscr = new CodeBook.DataDscr(variableList); - return new CodeBook(study, dataDscr); - } else { - log.info(String.format("No variables found for studyId '%s'", surveyId)); - } - } catch (IOException e) { - throw new ElasticsearchIoException(e); - } - return null; - } } From 428e589055aadb2025524817cbb2e2e7584afa62 Mon Sep 17 00:00:00 2001 From: theresa Date: Mon, 16 Sep 2024 12:32:29 +0200 Subject: [PATCH 15/33] #3304 refactored frontend ddi codebook export --- .../resources/dataPackage.resource.js | 16 +++++++++++++ .../views/data-package-detail.controller.js | 24 ++++++++++++++++++- .../views/data-package-detail.html.tmpl | 16 ++++++++++--- .../views/survey-detail.controller.js | 24 ++++--------------- .../resources/variable.resource.js | 15 ------------ 5 files changed, 56 insertions(+), 39 deletions(-) diff --git a/mdm-frontend/src/app/legacy/datapackagemanagement/resources/dataPackage.resource.js b/mdm-frontend/src/app/legacy/datapackagemanagement/resources/dataPackage.resource.js index 3f55ec07c7f..386864e4ca8 100644 --- a/mdm-frontend/src/app/legacy/datapackagemanagement/resources/dataPackage.resource.js +++ b/mdm-frontend/src/app/legacy/datapackagemanagement/resources/dataPackage.resource.js @@ -25,3 +25,19 @@ angular.module('metadatamanagementApp') }); }]); + + /** + * Method to request the endpoint that downloads all variables of the data package as a DDI-formatted XML file. + * We could not put this into the "DataPackageResource"-schema (above), because the URLs would not match. + */ + angular.module('metadatamanagementApp').factory('ExportDdiVariablesResource', ['$http', + function($http) { + return { + exportVariablesAsXml: function(dataPackageId) { + return $http.get('api/data-packages/exportDDI/xml/' + dataPackageId).then(function(response) { + return response.data; + }); + } + } + }]); + diff --git a/mdm-frontend/src/app/legacy/datapackagemanagement/views/data-package-detail.controller.js b/mdm-frontend/src/app/legacy/datapackagemanagement/views/data-package-detail.controller.js index 5ae229afc46..bce54e94843 100644 --- a/mdm-frontend/src/app/legacy/datapackagemanagement/views/data-package-detail.controller.js +++ b/mdm-frontend/src/app/legacy/datapackagemanagement/views/data-package-detail.controller.js @@ -35,6 +35,7 @@ angular.module('metadatamanagementApp') 'blockUI', '$mdSidenav', 'ContainsOnlyQualitativeDataChecker', + 'ExportDdiVariablesResource', function(entity, MessageBus, PageMetadataService, @@ -48,7 +49,8 @@ angular.module('metadatamanagementApp') ProjectUpdateAccessService, $scope, $timeout, $document, DataPackageOverviewResource, OutdatedVersionNotifier, DataPackageSearchService, $log, - blockUI, $mdSidenav, ContainsOnlyQualitativeDataChecker) { + blockUI, $mdSidenav, ContainsOnlyQualitativeDataChecker, + ExportDdiVariablesResource) { blockUI.start(); SearchResultNavigatorService .setSearchIndex($stateParams['search-result-index']); @@ -89,6 +91,8 @@ angular.module('metadatamanagementApp') .hasAnyAuthority(['ROLE_PUBLISHER', 'ROLE_ADMIN']); ctrl.showRemarks = Principal .hasAnyAuthority(['ROLE_PUBLISHER']); + ctrl.enableVariableExport = Principal + .hasAnyAuthority(['ROLE_PUBLISHER', 'ROLE_ADMIN']); var bowser = $rootScope.bowser; @@ -306,5 +310,23 @@ angular.module('metadatamanagementApp') } return true; } + + /** + * Exports Variables metadata as DDI Codebook XML. + */ + ctrl.exportVariables = function() { + console.log(ctrl.dataPackage) + ExportDdiVariablesResource.exportVariablesAsXml(ctrl.dataPackage.id).then(function(res) { + var blob = new Blob([res],{ + type: "application/xml;charset=utf-8;" + }); + var downloadLink = document.createElement('a'); + const fileName = 'mdm_export_ddi_variables_' + ctrl.dataPackage.id + '.xml' + downloadLink.setAttribute('download', fileName); + downloadLink.setAttribute('href', window.URL.createObjectURL(blob)); + downloadLink.click(); + $scope.isDownloadingData = false; + }) + } }]); diff --git a/mdm-frontend/src/app/legacy/datapackagemanagement/views/data-package-detail.html.tmpl b/mdm-frontend/src/app/legacy/datapackagemanagement/views/data-package-detail.html.tmpl index 30049515cb4..d1a447454c1 100644 --- a/mdm-frontend/src/app/legacy/datapackagemanagement/views/data-package-detail.html.tmpl +++ b/mdm-frontend/src/app/legacy/datapackagemanagement/views/data-package-detail.html.tmpl @@ -6,10 +6,20 @@
-
+

- +
+ + + + Variablen als XML nach DDI Standard exportieren + + DDI + +
{{'data-package-management.detail.label.tags' | translate}}:
diff --git a/mdm-frontend/src/app/legacy/surveymanagement/views/survey-detail.controller.js b/mdm-frontend/src/app/legacy/surveymanagement/views/survey-detail.controller.js index 9b2f1656c81..d8bfe997cbc 100644 --- a/mdm-frontend/src/app/legacy/surveymanagement/views/survey-detail.controller.js +++ b/mdm-frontend/src/app/legacy/surveymanagement/views/survey-detail.controller.js @@ -24,7 +24,7 @@ angular.module('metadatamanagementApp') '$stateParams', 'blockUI', '$mdSidenav', - 'ExportDdiVariablesResource', + // 'ExportDdiVariablesResource', function(entity, LanguageService, CleanJSObjectService, PageMetadataService, $state, BreadcrumbService, MessageBus, SurveySearchService, SurveyAttachmentResource, Principal, @@ -32,7 +32,7 @@ angular.module('metadatamanagementApp') SurveyResponseRateImageUploadService, OutdatedVersionNotifier, DataAcquisitionProjectResource, $mdDialog, ProjectUpdateAccessService, CountryCodesResource, $stateParams, - blockUI, $mdSidenav, ExportDdiVariablesResource) { + blockUI, $mdSidenav) { blockUI.start(); SearchResultNavigatorService .setSearchIndex($stateParams['search-result-index']); @@ -54,8 +54,8 @@ angular.module('metadatamanagementApp') ctrl.responseRateImage = null; ctrl.enableJsonView = Principal .hasAnyAuthority(['ROLE_PUBLISHER', 'ROLE_ADMIN']); - ctrl.enableVariableExport = Principal - .hasAnyAuthority(['ROLE_PUBLISHER', 'ROLE_ADMIN']); + // ctrl.enableVariableExport = Principal + // .hasAnyAuthority(['ROLE_PUBLISHER', 'ROLE_ADMIN']); ctrl.surveyId = null; entity.promise.then(function(survey) { @@ -182,21 +182,5 @@ angular.module('metadatamanagementApp') ctrl.toggleSidenav = function() { $mdSidenav('SideNavBar').toggle(); }; - - /** - * Exports Variables metadata as DDI Codebook XML. - */ - ctrl.exportVariables = function() { - ExportDdiVariablesResource.exportVariablesAsXml(ctrl.surveyId).then(function(res) { - var blob = new Blob([res],{ - type: "application/xml;charset=utf-8;" - }); - var downloadLink = document.createElement('a'); - downloadLink.setAttribute('download', 'variable_ddi_mdm_export.xml'); - downloadLink.setAttribute('href', window.URL.createObjectURL(blob)); - downloadLink.click(); - $scope.isDownloadingData = false; - }) - } }]); diff --git a/mdm-frontend/src/app/legacy/variablemanagement/resources/variable.resource.js b/mdm-frontend/src/app/legacy/variablemanagement/resources/variable.resource.js index 4cf7ef21b40..cb060358881 100644 --- a/mdm-frontend/src/app/legacy/variablemanagement/resources/variable.resource.js +++ b/mdm-frontend/src/app/legacy/variablemanagement/resources/variable.resource.js @@ -17,18 +17,3 @@ angular.module('metadatamanagementApp') }); }]); - /** - * Method to request the endpoint that downloads all variables of the study as a DDI-formatted XML file. - * We could not put this into the "VariableResource"-schema (above), because the URLs would not match. - */ - angular.module('metadatamanagementApp').factory('ExportDdiVariablesResource', ['$http', - function($http) { - return { - exportVariablesAsXml: function(surveyId) { - return $http.get('api/variables/exportDDI/xml/' + surveyId).then(function(response) { - return response.data; - }); - } - } - }]); - From f3eff2e2a68ab4d2891a34327897e21026cac292 Mon Sep 17 00:00:00 2001 From: theresa Date: Wed, 18 Sep 2024 13:29:00 +0200 Subject: [PATCH 16/33] #3304 added real values to DDI export --- .../domain/ddiCodebook/Catgry.java | 2 + .../domain/ddiCodebook/CodeBook.java | 30 ++- .../domain/ddiCodebook/DataDscr.java | 2 + .../domain/ddiCodebook/Var.java | 4 + .../service/DataPackageDdiService.java | 244 +++++++++++------- 5 files changed, 191 insertions(+), 91 deletions(-) diff --git a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/Catgry.java b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/Catgry.java index e6acdbcb9b3..612a9083e37 100644 --- a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/Catgry.java +++ b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/Catgry.java @@ -2,6 +2,7 @@ import java.util.List; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper; import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; import lombok.AllArgsConstructor; @@ -12,6 +13,7 @@ public class Catgry { String catValu; @JacksonXmlProperty(localName = "labl") + @JacksonXmlElementWrapper(useWrapping = false) List labl; } diff --git a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/CodeBook.java b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/CodeBook.java index dff4a1cafd5..bf759e2a84d 100644 --- a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/CodeBook.java +++ b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/CodeBook.java @@ -2,21 +2,43 @@ import java.util.List; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper; import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement; -import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.Setter; /** - * Record defining the mapping of the DDI Codebook standard. + * Defines the mapping of the DDI Codebook standard. */ -@AllArgsConstructor -@JacksonXmlRootElement(namespace = "ddi:codebook:2_5", localName = "codeBook") +@Getter +@Setter +@JacksonXmlRootElement(localName = "codeBook", namespace = "ddi:codebook:2_5") public class CodeBook { + public CodeBook(StdyDscr stdyDscr, List fileDscr, DataDscr dataDscr) { + this.stdyDscr = stdyDscr; + this.fileDscr = fileDscr; + this.dataDscr = dataDscr; + } + +// @JacksonXmlProperty(localName = "xmlns", isAttribute = true) +// private String nameSpace = "ddi:codebook:2_5"; + +// @JacksonXmlProperty(localName = "xmlns:xsi", isAttribute = true) +// private String xsi = "http://www.w3.org/2001/XMLSchema-instance"; +// +// @JacksonXmlProperty(localName = "xmlns:xs", isAttribute = true) +// private String xs = "http://www.w3.org/2001/XMLSchema"; +// +// @JacksonXmlProperty(localName = "xsi:schemaLocation", isAttribute = true) +// private String schemaLocation = "http://www.ddialliance.org/Specification/DDI-Codebook/2.5/XMLSchema/codebook.xsd"; + @JacksonXmlProperty(localName = "stdyDscr") private StdyDscr stdyDscr; @JacksonXmlProperty(localName = "fileDscr") + @JacksonXmlElementWrapper(useWrapping = false) private List fileDscr; @JacksonXmlProperty(localName = "dataDscr") diff --git a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/DataDscr.java b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/DataDscr.java index 9a4e0a57e21..8612681a6ea 100644 --- a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/DataDscr.java +++ b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/DataDscr.java @@ -2,6 +2,7 @@ import java.util.List; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper; import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; import lombok.AllArgsConstructor; @@ -9,6 +10,7 @@ public class DataDscr { @JacksonXmlProperty(localName = "var") + @JacksonXmlElementWrapper(useWrapping = false) List var; } diff --git a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/Var.java b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/Var.java index 27bc7970bbe..764ab58f729 100644 --- a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/Var.java +++ b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/Var.java @@ -2,6 +2,7 @@ import java.util.List; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper; import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; import lombok.AllArgsConstructor; @@ -15,12 +16,15 @@ public class Var { String files; @JacksonXmlProperty(localName = "labl") + @JacksonXmlElementWrapper(useWrapping = false) List labl; @JacksonXmlProperty(localName = "qstn") + @JacksonXmlElementWrapper(useWrapping = false) List qstn; @JacksonXmlProperty(localName = "catgry") + @JacksonXmlElementWrapper(useWrapping = false) List catgry; } diff --git a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/service/DataPackageDdiService.java b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/service/DataPackageDdiService.java index bfff6659a15..9effc907e4c 100644 --- a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/service/DataPackageDdiService.java +++ b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/service/DataPackageDdiService.java @@ -1,13 +1,18 @@ package eu.dzhw.fdz.metadatamanagement.datapackagemanagement.service; import java.io.IOException; +import java.time.format.DateTimeFormatter; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.dataformat.xml.XmlMapper; import com.google.gson.Gson; +import eu.dzhw.fdz.metadatamanagement.common.domain.I18nString; +import eu.dzhw.fdz.metadatamanagement.common.domain.Person; import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.DataPackage; +import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.Tags; import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddiCodebook.Catgry; import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddiCodebook.Citation; import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddiCodebook.CodeBook; @@ -19,9 +24,29 @@ import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddiCodebook.TextElement; import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddiCodebook.TitlStmt; import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddiCodebook.Var; +import eu.dzhw.fdz.metadatamanagement.questionmanagement.domain.Question; +import eu.dzhw.fdz.metadatamanagement.searchmanagement.documents.DataPackageSearchDocument; +import eu.dzhw.fdz.metadatamanagement.searchmanagement.documents.DataSetSubDocument; +import eu.dzhw.fdz.metadatamanagement.searchmanagement.documents.QuestionSearchDocument; +import eu.dzhw.fdz.metadatamanagement.searchmanagement.documents.RelatedQuestionSubDocument; +import eu.dzhw.fdz.metadatamanagement.searchmanagement.documents.VariableSearchDocument; +import eu.dzhw.fdz.metadatamanagement.searchmanagement.documents.VariableSubDocument; +import eu.dzhw.fdz.metadatamanagement.variablemanagement.domain.Missing; +import eu.dzhw.fdz.metadatamanagement.variablemanagement.domain.RelatedQuestion; +import eu.dzhw.fdz.metadatamanagement.variablemanagement.domain.ScaleLevels; +import eu.dzhw.fdz.metadatamanagement.variablemanagement.domain.ValidResponse; +import eu.dzhw.fdz.metadatamanagement.variablemanagement.domain.projections.RelatedQuestionSubDocumentProjection; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.SearchHits; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.springframework.core.io.ByteArrayResource; import org.springframework.data.rest.core.annotation.RepositoryEventHandler; @@ -70,119 +95,164 @@ public ResponseEntity exportDdiVariablesAsXML(String dataPackageId) { * @return the metadata */ private CodeBook getDdiVariablesMetadata(String dataPackageId) throws JsonProcessingException { - StdyDscr stdyDscr = this.getDdiStdyDscr(); + // get Data + SearchRequest dataPackageRequest = new SearchRequest(); + SearchSourceBuilder builderSurveys = new SearchSourceBuilder(); + builderSurveys.query(QueryBuilders.termsQuery("id", dataPackageId)); + dataPackageRequest.source(builderSurveys); + dataPackageRequest.indices("data_packages"); - List fileDscrList = new ArrayList<>(); - // for each dataset of the data package - for (int i = 0; i < 3; i++) { - //todo make a list of all added datasets and do add there variables later on - fileDscrList.add(this.getDdiFileDsrc()); - } + try { + SearchResponse surveyResponse = client.search(dataPackageRequest, RequestOptions.DEFAULT); + List hits = Arrays.asList(surveyResponse.getHits().getHits()); + if (hits.size() != 1) { + throw new ElasticsearchException( + String.format("Expected one data package for id '%s', but found %d", dataPackageId, hits.size())); + } + DataPackageSearchDocument dataPackageDoc = gson.fromJson( + hits.get(0).getSourceAsString(), DataPackageSearchDocument.class); + if (dataPackageDoc.getRelease() == null || + (dataPackageDoc.getRelease() != null && dataPackageDoc.getRelease().getIsPreRelease())) { + throw new ElasticsearchException( + String.format("DDI codebook export failed. Can only export released data packages. " + + "Data package with id '%s' is not released.", dataPackageDoc.getId()) + ); + } + + StdyDscr stdyDscr = this.getDdiStdyDscr(dataPackageDoc); + List fileDscrList = new ArrayList<>(); + for (DataSetSubDocument dataset : dataPackageDoc.getDataSets()) { + fileDscrList.add(this.getDdiFileDsrc(dataset)); + } - List varList = new ArrayList<>(); - // for each variable of the data package - for (int j = 0; j < 10; j++) { - varList.add(this.getDdiVar()); + List varList = new ArrayList<>(); + for (VariableSubDocument variable : dataPackageDoc.getVariables()) { + varList.add(this.getDdiVar(variable)); + } + DataDscr dataDscr = new DataDscr(varList); + CodeBook codeBook = new CodeBook(stdyDscr, fileDscrList, dataDscr); + + return codeBook; + + } catch (IOException e) { + log.error("An error occurred while querying the ES. ", e); } - DataDscr dataDscr = new DataDscr(varList); - CodeBook codeBook = new CodeBook(stdyDscr, fileDscrList, dataDscr); - - return codeBook; - - -// SearchRequest surveyRequest = new SearchRequest(); -// SearchSourceBuilder builderSurveys = new SearchSourceBuilder(); -// builderSurveys.query(QueryBuilders.termsQuery("id", surveyId)); -// surveyRequest.source(builderSurveys); -// surveyRequest.indices("surveys"); -// -// SearchRequest variablesRequest = new SearchRequest(); -// SearchSourceBuilder builderVariables = new SearchSourceBuilder(); -// builderVariables.query(QueryBuilders.termsQuery("surveyIds", surveyId)) -// .size(10000); -// variablesRequest.source(builderVariables); -// variablesRequest.indices("variables"); -// try { -// SearchResponse surveyResponse = client.search(surveyRequest, RequestOptions.DEFAULT); -// List surveyHits = Arrays.asList(surveyResponse.getHits().getHits()); -// if (surveyHits.size() == 0) { -// throw new ElasticsearchException(String.format("Could not find survey with id '%s'", surveyId)); -// } -// SurveySearchDocument surveyDoc = gson.fromJson( -// surveyHits.get(0).getSourceAsString(), SurveySearchDocument.class); -// // toDo: Decide on language usage -// CodeBook.StdyDscr.Citation.TitlStmt titlStmt = new CodeBook.StdyDscr.Citation.TitlStmt(surveyDoc.getTitle().getEn()); -// CodeBook.StdyDscr.Citation citation = new CodeBook.StdyDscr.Citation(titlStmt); -// CodeBook.StdyDscr study = new CodeBook.StdyDscr(citation); -// SearchResponse variableResponse = client.search(variablesRequest, RequestOptions.DEFAULT); -// List hits = Arrays.asList(variableResponse.getHits().getHits()); -// if (hits.size() > 0) { -// log.info(String.format("Found %d variables for survey '%s'", hits.size(),surveyId)); -// List variableList = new ArrayList<>(); -// for (var variable : hits) { -// VariableSearchDocument variableDoc = gson.fromJson( -// variable.getSourceAsString(), VariableSearchDocument.class); -// CodeBook.DataDscr.Var varMetadata = new CodeBook.DataDscr.Var(variableDoc.getName()); -// variableList.add(varMetadata); -// } -// CodeBook.DataDscr dataDscr = new CodeBook.DataDscr(variableList); -// return new CodeBook(study, dataDscr); -// } else { -// log.info(String.format("No variables found for studyId '%s'", surveyId)); -// } -// } catch (IOException e) { -// throw new ElasticsearchIoException(e); -// } - //return null; + return null; } /** * * @return */ - private Var getDdiVar() { - String name = "Variable ID"; - String files = "Dataset ID"; + private Var getDdiVar(VariableSubDocument variableDoc) { + String name = variableDoc.getId(); + String files = variableDoc.getDataSetId(); List varLablList = new ArrayList<>(); - varLablList.add(new TextElement(LanguageEnum.de, "Variable Label in German")); - varLablList.add(new TextElement(LanguageEnum.en, "Variable Label in English")); + varLablList.add(new TextElement(LanguageEnum.de, variableDoc.getLabel().getDe())); + varLablList.add(new TextElement(LanguageEnum.en, variableDoc.getLabel().getEn())); List qstnList = new ArrayList<>(); - // possibly empty - qstnList.add(new TextElement(LanguageEnum.de, "Related Question label in German")); - qstnList.add(new TextElement(LanguageEnum.en, "Related Question label in English")); + if (variableDoc.getRelatedQuestions() != null && variableDoc.getRelatedQuestions().size() > 0) { + for (RelatedQuestionSubDocumentProjection relQuest : variableDoc.getRelatedQuestions()) { + SearchRequest questionRequest = new SearchRequest(); + SearchSourceBuilder builder = new SearchSourceBuilder(); + builder.query(QueryBuilders.termsQuery("id", relQuest.getQuestionId())); + questionRequest.source(builder); + questionRequest.indices("questions"); + try { + SearchResponse questionResponse = client.search(questionRequest, RequestOptions.DEFAULT); + List hits = Arrays.asList(questionResponse.getHits().getHits()); + if (hits.size() == 0) { + throw new ElasticsearchException( + String.format("Could not find question for id '%s'", relQuest.getQuestionId())); + } + for (SearchHit hit : hits) { + QuestionSearchDocument relatedQuestion = gson.fromJson( + hit.getSourceAsString(), QuestionSearchDocument.class); + if (relatedQuestion.getQuestionText() != null && relatedQuestion.getQuestionText().getDe() != null) { + qstnList.add(new TextElement(LanguageEnum.de, relatedQuestion.getQuestionText().getDe())); + } + if (relatedQuestion.getQuestionText() != null && relatedQuestion.getQuestionText().getEn() != null) { + qstnList.add(new TextElement(LanguageEnum.en, relatedQuestion.getQuestionText().getEn())); + } + } + } catch (IOException e) { + log.error("An exception occurred requestion questions index. ", e); + } + } + } + + SearchRequest variableRequest = new SearchRequest(); + SearchSourceBuilder builderVar = new SearchSourceBuilder(); + builderVar.query(QueryBuilders.termsQuery("id", variableDoc.getId())); + variableRequest.source(builderVar); + variableRequest.indices("variables"); List catgryList = new ArrayList<>(); - for (int k = 0; k < 5; k++) { - // only for nominal/ordinal - String catValu = "Value"; - List catLablList = new ArrayList<>(); - catLablList.add(new TextElement(LanguageEnum.de, "Value Label in German")); - catLablList.add(new TextElement(LanguageEnum.en, "Value Label in English")); - catgryList.add(new Catgry(catValu, catLablList)); + try { + SearchResponse variableResponse = client.search(variableRequest, RequestOptions.DEFAULT); + List hits = Arrays.asList(variableResponse.getHits().getHits()); + if (hits.size() == 0) { + throw new ElasticsearchException( + String.format("Could not find question for id '%s'", variableDoc.getId())); + } + for (SearchHit hit : hits) { + VariableSearchDocument varDoc = gson.fromJson( + hit.getSourceAsString(), VariableSearchDocument.class); + if ((varDoc.getScaleLevel().equals(ScaleLevels.NOMINAL) || varDoc.getScaleLevel().equals(ScaleLevels.ORDINAL)) + && varDoc.getDistribution() != null + && varDoc.getDistribution().getValidResponses() != null) { + for (ValidResponse validResponse : varDoc.getDistribution().getValidResponses()) { + String catValu = validResponse.getValue(); + List catLablList = new ArrayList<>(); + if (validResponse.getLabel() != null && validResponse.getLabel().getDe() != null) { + catLablList.add(new TextElement(LanguageEnum.de, validResponse.getLabel().getDe())); + } + if (validResponse.getLabel() != null && validResponse.getLabel().getEn() != null) { + catLablList.add(new TextElement(LanguageEnum.en, validResponse.getLabel().getEn())); + } + catgryList.add(new Catgry(catValu, catLablList)); + } + // missing values + if (varDoc.getDistribution() != null && varDoc.getDistribution().getMissings() != null) + for (Missing missing : varDoc.getDistribution().getMissings()) { + String catValu = missing.getCode(); + List catLablList = new ArrayList<>(); + if (missing.getLabel() != null && missing.getLabel().getDe() != null) { + catLablList.add(new TextElement(LanguageEnum.de, missing.getLabel().getDe())); + } + if (missing.getLabel() != null && missing.getLabel().getEn() != null) { + catLablList.add(new TextElement(LanguageEnum.en, missing.getLabel().getEn())); + } + catgryList.add(new Catgry(catValu, catLablList)); + } + } + } + } catch (IOException e) { + log.error("Error ...", e); } - return new Var(name, files, varLablList, qstnList, catgryList); + return new Var(name, files, varLablList, qstnList.size() > 0 ? qstnList : null, catgryList); } /** * * @return */ - private FileDscr getDdiFileDsrc() { - String id = "Dataset ID"; - String fileName = "Dataset ID"; - TextElement fileCont = new TextElement(LanguageEnum.de, "Dataset Title in German"); + private FileDscr getDdiFileDsrc(DataSetSubDocument dataset) { + String id = dataset.getId(); + String fileName = dataset.getId(); + // todo: Clarify why this is just in German and if it is the correct field + TextElement fileCont = new TextElement(LanguageEnum.de, dataset.getDescription().getDe()); FileTxt fileTxt = new FileTxt(fileName, fileCont); return new FileDscr(id, fileTxt); } /** - * - * @return + * Creates the DDI element stdyDscr with data from the data package. + * @return the stdyDscr element */ - private StdyDscr getDdiStdyDscr() { - TextElement titl = new TextElement(LanguageEnum.de, "Data Package Title in German"); - TextElement parTitl = new TextElement(LanguageEnum.en, "Data Package Title in English"); + private StdyDscr getDdiStdyDscr(DataPackageSearchDocument doc) { + TextElement titl = new TextElement(LanguageEnum.de, doc.getTitle().getDe()); + TextElement parTitl = new TextElement(LanguageEnum.en, doc.getTitle().getEn()); Citation citation = new Citation(new TitlStmt(titl, parTitl)); return new StdyDscr(citation); } - } From f0da9241253e4c4eada5b880b332f9b7e1ac3163 Mon Sep 17 00:00:00 2001 From: theresa Date: Thu, 19 Sep 2024 15:22:53 +0200 Subject: [PATCH 17/33] #3304 switched to JAXB for handling XML generation --- .../domain/ddiCodebook/Catgry.java | 11 ++++--- .../domain/ddiCodebook/Citation.java | 9 ++++-- .../domain/ddiCodebook/CodeBook.java | 31 +++++-------------- .../domain/ddiCodebook/DataDscr.java | 9 +++--- .../domain/ddiCodebook/FileDscr.java | 10 ++++-- .../domain/ddiCodebook/FileTxt.java | 9 ++++-- .../domain/ddiCodebook/StdyDscr.java | 7 +++-- .../domain/ddiCodebook/TextElement.java | 14 ++++++--- .../domain/ddiCodebook/TitlStmt.java | 13 +++++--- .../domain/ddiCodebook/Var.java | 20 ++++++------ .../domain/ddiCodebook/package-info.java | 21 +++++++++++++ .../service/DataPackageDdiService.java | 28 +++++++++-------- 12 files changed, 107 insertions(+), 75 deletions(-) create mode 100644 src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/package-info.java diff --git a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/Catgry.java b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/Catgry.java index 612a9083e37..abe09a709d6 100644 --- a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/Catgry.java +++ b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/Catgry.java @@ -2,18 +2,19 @@ import java.util.List; -import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper; -import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; +import javax.xml.bind.annotation.XmlElement; + import lombok.AllArgsConstructor; @AllArgsConstructor public class Catgry { - @JacksonXmlProperty(localName = "catValu") + public Catgry() {} + + @XmlElement(name="catValu") String catValu; - @JacksonXmlProperty(localName = "labl") - @JacksonXmlElementWrapper(useWrapping = false) + @XmlElement(name="labl") List labl; } diff --git a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/Citation.java b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/Citation.java index 716c886175e..9fdaa46a78c 100644 --- a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/Citation.java +++ b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/Citation.java @@ -1,12 +1,15 @@ package eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddiCodebook; -import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; +import javax.xml.bind.annotation.XmlElement; + import lombok.AllArgsConstructor; @AllArgsConstructor public class Citation { - @JacksonXmlProperty(localName = "titlStmt") - private TitlStmt titlStmt; + public Citation() {} + + @XmlElement(name="titlStmt") + TitlStmt titlStmt; } diff --git a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/CodeBook.java b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/CodeBook.java index bf759e2a84d..e111d858620 100644 --- a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/CodeBook.java +++ b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/CodeBook.java @@ -2,18 +2,14 @@ import java.util.List; -import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper; -import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; -import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement; -import lombok.Getter; -import lombok.Setter; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; /** * Defines the mapping of the DDI Codebook standard. */ -@Getter -@Setter -@JacksonXmlRootElement(localName = "codeBook", namespace = "ddi:codebook:2_5") + +@XmlRootElement(name="codeBook") public class CodeBook { public CodeBook(StdyDscr stdyDscr, List fileDscr, DataDscr dataDscr) { @@ -22,26 +18,15 @@ public CodeBook(StdyDscr stdyDscr, List fileDscr, DataDscr dataDscr) { this.dataDscr = dataDscr; } -// @JacksonXmlProperty(localName = "xmlns", isAttribute = true) -// private String nameSpace = "ddi:codebook:2_5"; - -// @JacksonXmlProperty(localName = "xmlns:xsi", isAttribute = true) -// private String xsi = "http://www.w3.org/2001/XMLSchema-instance"; -// -// @JacksonXmlProperty(localName = "xmlns:xs", isAttribute = true) -// private String xs = "http://www.w3.org/2001/XMLSchema"; -// -// @JacksonXmlProperty(localName = "xsi:schemaLocation", isAttribute = true) -// private String schemaLocation = "http://www.ddialliance.org/Specification/DDI-Codebook/2.5/XMLSchema/codebook.xsd"; + public CodeBook() {} - @JacksonXmlProperty(localName = "stdyDscr") + @XmlElement(name="stdyDscr") private StdyDscr stdyDscr; - @JacksonXmlProperty(localName = "fileDscr") - @JacksonXmlElementWrapper(useWrapping = false) + @XmlElement(name="fileDscr") private List fileDscr; - @JacksonXmlProperty(localName = "dataDscr") + @XmlElement(name="dataDscr") private DataDscr dataDscr; } diff --git a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/DataDscr.java b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/DataDscr.java index 8612681a6ea..9ba62f23277 100644 --- a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/DataDscr.java +++ b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/DataDscr.java @@ -2,15 +2,16 @@ import java.util.List; -import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper; -import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; +import javax.xml.bind.annotation.XmlElement; + import lombok.AllArgsConstructor; @AllArgsConstructor public class DataDscr { - @JacksonXmlProperty(localName = "var") - @JacksonXmlElementWrapper(useWrapping = false) + public DataDscr() {} + + @XmlElement(name="var") List var; } diff --git a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/FileDscr.java b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/FileDscr.java index c4ede7e849a..242126a4d8d 100644 --- a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/FileDscr.java +++ b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/FileDscr.java @@ -1,14 +1,18 @@ package eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddiCodebook; -import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; +import javax.xml.bind.annotation.XmlAttribute; +import javax.xml.bind.annotation.XmlElement; + import lombok.AllArgsConstructor; @AllArgsConstructor public class FileDscr { - @JacksonXmlProperty(localName = "ID", isAttribute = true) + public FileDscr() {} + + @XmlAttribute(name="ID") String id; - @JacksonXmlProperty(localName = "fileTxt") + @XmlElement(name="fileTxt") FileTxt fileTxt; } diff --git a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/FileTxt.java b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/FileTxt.java index 5f9e18db2ba..8950dfeb2db 100644 --- a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/FileTxt.java +++ b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/FileTxt.java @@ -1,15 +1,18 @@ package eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddiCodebook; -import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; +import javax.xml.bind.annotation.XmlElement; + import lombok.AllArgsConstructor; @AllArgsConstructor public class FileTxt { - @JacksonXmlProperty(localName = "fileName") + public FileTxt() {} + + @XmlElement(name="fileName") String fileName; - @JacksonXmlProperty(localName = "fileCont") + @XmlElement(name="fileCont") TextElement fileCont; } diff --git a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/StdyDscr.java b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/StdyDscr.java index 08bf01808e0..55fc9fe8ad8 100644 --- a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/StdyDscr.java +++ b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/StdyDscr.java @@ -1,12 +1,15 @@ package eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddiCodebook; -import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; +import javax.xml.bind.annotation.XmlElement; + import lombok.AllArgsConstructor; @AllArgsConstructor public class StdyDscr { - @JacksonXmlProperty(localName = "citation") + public StdyDscr() {} + + @XmlElement(name="citation") Citation citation; } diff --git a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/TextElement.java b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/TextElement.java index e38c29434b7..91d3020315d 100644 --- a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/TextElement.java +++ b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/TextElement.java @@ -1,7 +1,10 @@ package eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddiCodebook; -import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; -import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlText; +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlAttribute; +import javax.xml.bind.annotation.XmlValue; + import lombok.AllArgsConstructor; /** @@ -9,12 +12,15 @@ * e.g. labels (labl in DDI standard). */ @AllArgsConstructor +@XmlAccessorType(XmlAccessType.NONE) public class TextElement { - @JacksonXmlProperty(localName = "xml:lang", isAttribute = true) + public TextElement() {} + + @XmlAttribute(name="xml:lang") private LanguageEnum lang; - @JacksonXmlText + @XmlValue private String value; public String getValue() { diff --git a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/TitlStmt.java b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/TitlStmt.java index d5172cfd010..b050aded1ce 100644 --- a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/TitlStmt.java +++ b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/TitlStmt.java @@ -1,15 +1,18 @@ package eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddiCodebook; -import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; +import javax.xml.bind.annotation.XmlElement; + import lombok.AllArgsConstructor; @AllArgsConstructor public class TitlStmt { - @JacksonXmlProperty(localName = "titl") - private TextElement titl; + public TitlStmt() {} + + @XmlElement(name="titl") + TextElement titl; - @JacksonXmlProperty(localName = "parTitl") - private TextElement parTitle; + @XmlElement(name="parTitl") + TextElement parTitle; } diff --git a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/Var.java b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/Var.java index 764ab58f729..07937829c7b 100644 --- a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/Var.java +++ b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/Var.java @@ -2,29 +2,29 @@ import java.util.List; -import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper; -import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; +import javax.xml.bind.annotation.XmlAttribute; +import javax.xml.bind.annotation.XmlElement; + import lombok.AllArgsConstructor; @AllArgsConstructor public class Var { - @JacksonXmlProperty(localName = "name", isAttribute = true) + public Var() {} + + @XmlAttribute(name="name") String name; - @JacksonXmlProperty(localName = "files", isAttribute = true) + @XmlAttribute(name="files") String files; - @JacksonXmlProperty(localName = "labl") - @JacksonXmlElementWrapper(useWrapping = false) + @XmlElement(name="labl") List labl; - @JacksonXmlProperty(localName = "qstn") - @JacksonXmlElementWrapper(useWrapping = false) + @XmlElement(name="qstn") List qstn; - @JacksonXmlProperty(localName = "catgry") - @JacksonXmlElementWrapper(useWrapping = false) + @XmlElement(name="catgry") List catgry; } diff --git a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/package-info.java b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/package-info.java new file mode 100644 index 00000000000..5df93d28d08 --- /dev/null +++ b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/package-info.java @@ -0,0 +1,21 @@ + +/** + * Configuration for JAXB + */ + +@XmlSchema( + xmlns = { + @XmlNs(prefix = "", namespaceURI = "ddi:codebook:2_5"), + @XmlNs(prefix = "xs", namespaceURI = "http://www.w3.org/2001/XMLSchema") + }, + namespace = "ddi:codebook:2_5", + elementFormDefault = XmlNsForm.QUALIFIED) +package eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddiCodebook; + +import javax.xml.bind.annotation.XmlNs; +import javax.xml.bind.annotation.XmlNsForm; +import javax.xml.bind.annotation.XmlSchema; + + + + diff --git a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/service/DataPackageDdiService.java b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/service/DataPackageDdiService.java index 9effc907e4c..7cb3f4816c8 100644 --- a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/service/DataPackageDdiService.java +++ b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/service/DataPackageDdiService.java @@ -1,18 +1,18 @@ package eu.dzhw.fdz.metadatamanagement.datapackagemanagement.service; +import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import javax.xml.bind.JAXBContext; +import javax.xml.bind.JAXBException; +import javax.xml.bind.Marshaller; + import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.dataformat.xml.XmlMapper; import com.google.gson.Gson; -import eu.dzhw.fdz.metadatamanagement.common.domain.I18nString; -import eu.dzhw.fdz.metadatamanagement.common.domain.Person; import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.DataPackage; -import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.Tags; import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddiCodebook.Catgry; import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddiCodebook.Citation; import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddiCodebook.CodeBook; @@ -24,15 +24,12 @@ import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddiCodebook.TextElement; import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddiCodebook.TitlStmt; import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddiCodebook.Var; -import eu.dzhw.fdz.metadatamanagement.questionmanagement.domain.Question; import eu.dzhw.fdz.metadatamanagement.searchmanagement.documents.DataPackageSearchDocument; import eu.dzhw.fdz.metadatamanagement.searchmanagement.documents.DataSetSubDocument; import eu.dzhw.fdz.metadatamanagement.searchmanagement.documents.QuestionSearchDocument; -import eu.dzhw.fdz.metadatamanagement.searchmanagement.documents.RelatedQuestionSubDocument; import eu.dzhw.fdz.metadatamanagement.searchmanagement.documents.VariableSearchDocument; import eu.dzhw.fdz.metadatamanagement.searchmanagement.documents.VariableSubDocument; import eu.dzhw.fdz.metadatamanagement.variablemanagement.domain.Missing; -import eu.dzhw.fdz.metadatamanagement.variablemanagement.domain.RelatedQuestion; import eu.dzhw.fdz.metadatamanagement.variablemanagement.domain.ScaleLevels; import eu.dzhw.fdz.metadatamanagement.variablemanagement.domain.ValidResponse; import eu.dzhw.fdz.metadatamanagement.variablemanagement.domain.projections.RelatedQuestionSubDocumentProjection; @@ -45,7 +42,6 @@ import org.elasticsearch.client.RestHighLevelClient; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.SearchHit; -import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.springframework.core.io.ByteArrayResource; @@ -78,14 +74,20 @@ public class DataPackageDdiService { public ResponseEntity exportDdiVariablesAsXML(String dataPackageId) { try { CodeBook variableMetadata = this.getDdiVariablesMetadata(dataPackageId); - XmlMapper mapper = new XmlMapper(); - ByteArrayResource resource = new ByteArrayResource(mapper.writeValueAsBytes(variableMetadata)); + JAXBContext context = JAXBContext.newInstance(CodeBook.class); + Marshaller mar = context.createMarshaller(); + mar.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, Boolean.TRUE); + mar.setProperty(Marshaller.JAXB_SCHEMA_LOCATION, "http://www.ddialliance.org/Specification/DDI-Codebook/2.5/XMLSchema/codebook.xsd");; + ByteArrayOutputStream res = new ByteArrayOutputStream(); + mar.marshal(variableMetadata, res); + ByteArrayResource resource = new ByteArrayResource(res.toByteArray()); HttpHeaders headers = new HttpHeaders(); - headers.add("Content-Disposition", "attachment; filename=Variables_PID_MDM_Export.xml"); + headers.add("Content-Disposition", "attachment; filename=Variables_DDI_MDM_Export.xml"); return ResponseEntity.ok() .headers(headers) .body(resource); - } catch (IOException ex) { + } catch (JAXBException | JsonProcessingException ex) { + log.error("Error generating XML: " + ex); return new ResponseEntity<>(null, null, HttpStatus.NOT_FOUND); } } From 6cdd615811ae8c7968e37586c586bf036b434b77 Mon Sep 17 00:00:00 2001 From: theresa Date: Mon, 23 Sep 2024 10:26:46 +0200 Subject: [PATCH 18/33] #3304 added txt element to variable element, added doc strings --- .../domain/ddiCodebook/Var.java | 3 +++ .../service/DataPackageDdiService.java | 25 ++++++++++++------- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/Var.java b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/Var.java index 07937829c7b..4e50335ce01 100644 --- a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/Var.java +++ b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/Var.java @@ -24,6 +24,9 @@ public Var() {} @XmlElement(name="qstn") List qstn; + @XmlElement(name="txt") + List txt; + @XmlElement(name="catgry") List catgry; diff --git a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/service/DataPackageDdiService.java b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/service/DataPackageDdiService.java index 7cb3f4816c8..ba26d8acf61 100644 --- a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/service/DataPackageDdiService.java +++ b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/service/DataPackageDdiService.java @@ -67,7 +67,7 @@ public class DataPackageDdiService { private final Gson gson; /** - * Exports all variables belonging to a given datapackage according to the DDI Codebook standard. + * Exports all variables metadata belonging to a given data package according to the DDI Codebook standard. * @param dataPackageId the ID of the study * @return DDI metadata as XML */ @@ -143,8 +143,8 @@ private CodeBook getDdiVariablesMetadata(String dataPackageId) throws JsonProces } /** - * - * @return + * Create the DDI var Element with data from the variable, and its related questions. + * @return the var element */ private Var getDdiVar(VariableSubDocument variableDoc) { String name = variableDoc.getId(); @@ -178,7 +178,7 @@ private Var getDdiVar(VariableSubDocument variableDoc) { } } } catch (IOException e) { - log.error("An exception occurred requestion questions index. ", e); + log.error("An exception occurred querying the questions index. ", e); } } } @@ -189,16 +189,23 @@ private Var getDdiVar(VariableSubDocument variableDoc) { variableRequest.source(builderVar); variableRequest.indices("variables"); List catgryList = new ArrayList<>(); + List txtList = new ArrayList<>(); try { SearchResponse variableResponse = client.search(variableRequest, RequestOptions.DEFAULT); List hits = Arrays.asList(variableResponse.getHits().getHits()); if (hits.size() == 0) { throw new ElasticsearchException( - String.format("Could not find question for id '%s'", variableDoc.getId())); + String.format("Could not find variable for id '%s'", variableDoc.getId())); } for (SearchHit hit : hits) { VariableSearchDocument varDoc = gson.fromJson( hit.getSourceAsString(), VariableSearchDocument.class); + if (varDoc.getAnnotations() != null && varDoc.getAnnotations().getDe() != null) { + txtList.add(new TextElement(LanguageEnum.de, varDoc.getAnnotations().getDe())); + } + if (varDoc.getAnnotations() != null && varDoc.getAnnotations().getEn() != null) { + txtList.add(new TextElement(LanguageEnum.en, varDoc.getAnnotations().getEn())); + } if ((varDoc.getScaleLevel().equals(ScaleLevels.NOMINAL) || varDoc.getScaleLevel().equals(ScaleLevels.ORDINAL)) && varDoc.getDistribution() != null && varDoc.getDistribution().getValidResponses() != null) { @@ -229,14 +236,14 @@ private Var getDdiVar(VariableSubDocument variableDoc) { } } } catch (IOException e) { - log.error("Error ...", e); + log.error("An exception occurred querying the variables index. ", e); } - return new Var(name, files, varLablList, qstnList.size() > 0 ? qstnList : null, catgryList); + return new Var(name, files, varLablList, qstnList.size() > 0 ? qstnList : null, txtList, catgryList); } /** - * - * @return + * Create the DDI element fileDscr with data from the datasets of the data package. + * @return the fileDscr element */ private FileDscr getDdiFileDsrc(DataSetSubDocument dataset) { String id = dataset.getId(); From 0b98ec987a1ec6266a334a5c68304f6883f9087d Mon Sep 17 00:00:00 2001 From: theresa Date: Mon, 23 Sep 2024 10:56:07 +0200 Subject: [PATCH 19/33] #3304 code style fixes part I --- .../domain/ddiCodebook/CodeBook.java | 32 -------- .../{ddiCodebook => ddicodebook}/Catgry.java | 12 ++- .../Citation.java | 10 ++- .../domain/ddicodebook/CodeBook.java | 41 ++++++++++ .../DataDscr.java | 10 ++- .../FileDscr.java | 9 ++- .../{ddiCodebook => ddicodebook}/FileTxt.java | 12 ++- .../LanguageEnum.java | 4 +- .../StdyDscr.java | 10 ++- .../TextElement.java | 7 +- .../TitlStmt.java | 12 ++- .../{ddiCodebook => ddicodebook}/Var.java | 20 +++-- .../package-info.java | 10 +-- .../ExportDdiVariablesResourceController.java | 2 +- .../service/DataPackageDdiService.java | 78 +++++++++++-------- .../service/DataPackageManagementService.java | 21 ----- 16 files changed, 166 insertions(+), 124 deletions(-) delete mode 100644 src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/CodeBook.java rename src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/{ddiCodebook => ddicodebook}/Catgry.java (61%) rename src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/{ddiCodebook => ddicodebook}/Citation.java (61%) create mode 100644 src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddicodebook/CodeBook.java rename src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/{ddiCodebook => ddicodebook}/DataDscr.java (63%) rename src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/{ddiCodebook => ddicodebook}/FileDscr.java (67%) rename src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/{ddiCodebook => ddicodebook}/FileTxt.java (58%) rename src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/{ddiCodebook => ddicodebook}/LanguageEnum.java (88%) rename src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/{ddiCodebook => ddicodebook}/StdyDscr.java (61%) rename src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/{ddiCodebook => ddicodebook}/TextElement.java (88%) rename src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/{ddiCodebook => ddicodebook}/TitlStmt.java (59%) rename src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/{ddiCodebook => ddicodebook}/Var.java (57%) rename src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/{ddiCodebook => ddicodebook}/package-info.java (90%) diff --git a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/CodeBook.java b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/CodeBook.java deleted file mode 100644 index e111d858620..00000000000 --- a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/CodeBook.java +++ /dev/null @@ -1,32 +0,0 @@ -package eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddiCodebook; - -import java.util.List; - -import javax.xml.bind.annotation.XmlElement; -import javax.xml.bind.annotation.XmlRootElement; - -/** - * Defines the mapping of the DDI Codebook standard. - */ - -@XmlRootElement(name="codeBook") -public class CodeBook { - - public CodeBook(StdyDscr stdyDscr, List fileDscr, DataDscr dataDscr) { - this.stdyDscr = stdyDscr; - this.fileDscr = fileDscr; - this.dataDscr = dataDscr; - } - - public CodeBook() {} - - @XmlElement(name="stdyDscr") - private StdyDscr stdyDscr; - - @XmlElement(name="fileDscr") - private List fileDscr; - - @XmlElement(name="dataDscr") - private DataDscr dataDscr; - -} diff --git a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/Catgry.java b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddicodebook/Catgry.java similarity index 61% rename from src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/Catgry.java rename to src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddicodebook/Catgry.java index abe09a709d6..27fc22794ac 100644 --- a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/Catgry.java +++ b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddicodebook/Catgry.java @@ -1,4 +1,4 @@ -package eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddiCodebook; +package eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddicodebook; import java.util.List; @@ -6,15 +6,21 @@ import lombok.AllArgsConstructor; +/** + * Class representing the DDI catgry element. + */ @AllArgsConstructor public class Catgry { + /** + * Constructor needed by JAXB. + */ public Catgry() {} - @XmlElement(name="catValu") + @XmlElement(name = "catValu") String catValu; - @XmlElement(name="labl") + @XmlElement(name = "labl") List labl; } diff --git a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/Citation.java b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddicodebook/Citation.java similarity index 61% rename from src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/Citation.java rename to src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddicodebook/Citation.java index 9fdaa46a78c..df5224931f5 100644 --- a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/Citation.java +++ b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddicodebook/Citation.java @@ -1,15 +1,21 @@ -package eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddiCodebook; +package eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddicodebook; import javax.xml.bind.annotation.XmlElement; import lombok.AllArgsConstructor; +/** + * Class representing the DDI citation element. + */ @AllArgsConstructor public class Citation { + /** + * Constructor needed by JAXB. + */ public Citation() {} - @XmlElement(name="titlStmt") + @XmlElement(name = "titlStmt") TitlStmt titlStmt; } diff --git a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddicodebook/CodeBook.java b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddicodebook/CodeBook.java new file mode 100644 index 00000000000..d68f7c92fff --- /dev/null +++ b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddicodebook/CodeBook.java @@ -0,0 +1,41 @@ +package eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddicodebook; + +import java.util.List; + +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; + +/** + * Class representing the root element codebook of the mapping of the DDI Codebook standard. + */ + +@XmlRootElement(name = "codeBook") +public class CodeBook { + + /** + * Constructor for codebook. + * @param stdyDscr the data package description in form of the DDI stdyDscr element. + * @param fileDscr the dataset metadata in form of the DDI fileDscr element. + * @param dataDscr the variable metadata in form of the DDI dataDscr element. + */ + public CodeBook(StdyDscr stdyDscr, List fileDscr, DataDscr dataDscr) { + this.stdyDscr = stdyDscr; + this.fileDscr = fileDscr; + this.dataDscr = dataDscr; + } + + /** + * Constructor needed by JAXB. + */ + public CodeBook() {} + + @XmlElement(name = "stdyDscr") + private StdyDscr stdyDscr; + + @XmlElement(name = "fileDscr") + private List fileDscr; + + @XmlElement(name = "dataDscr") + private DataDscr dataDscr; + +} diff --git a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/DataDscr.java b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddicodebook/DataDscr.java similarity index 63% rename from src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/DataDscr.java rename to src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddicodebook/DataDscr.java index 9ba62f23277..2f81233c41c 100644 --- a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/DataDscr.java +++ b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddicodebook/DataDscr.java @@ -1,4 +1,4 @@ -package eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddiCodebook; +package eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddicodebook; import java.util.List; @@ -6,12 +6,18 @@ import lombok.AllArgsConstructor; +/** + * Class representing the DDI dataDscr element. + */ @AllArgsConstructor public class DataDscr { + /** + * Constructor needed by JAXB. + */ public DataDscr() {} - @XmlElement(name="var") + @XmlElement(name = "var") List var; } diff --git a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/FileDscr.java b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddicodebook/FileDscr.java similarity index 67% rename from src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/FileDscr.java rename to src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddicodebook/FileDscr.java index 242126a4d8d..fef929c6b55 100644 --- a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/FileDscr.java +++ b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddicodebook/FileDscr.java @@ -1,18 +1,21 @@ -package eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddiCodebook; +package eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddicodebook; import javax.xml.bind.annotation.XmlAttribute; import javax.xml.bind.annotation.XmlElement; import lombok.AllArgsConstructor; +/** + * Class representing the DDI Codebook fileDscr element. + */ @AllArgsConstructor public class FileDscr { public FileDscr() {} - @XmlAttribute(name="ID") + @XmlAttribute(name = "ID") String id; - @XmlElement(name="fileTxt") + @XmlElement(name = "fileTxt") FileTxt fileTxt; } diff --git a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/FileTxt.java b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddicodebook/FileTxt.java similarity index 58% rename from src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/FileTxt.java rename to src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddicodebook/FileTxt.java index 8950dfeb2db..933ddb440d8 100644 --- a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/FileTxt.java +++ b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddicodebook/FileTxt.java @@ -1,18 +1,24 @@ -package eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddiCodebook; +package eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddicodebook; import javax.xml.bind.annotation.XmlElement; import lombok.AllArgsConstructor; +/** + * Class representing the DDI fileTxt element. + */ @AllArgsConstructor public class FileTxt { + /** + * Constructor needed by JAXB. + */ public FileTxt() {} - @XmlElement(name="fileName") + @XmlElement(name = "fileName") String fileName; - @XmlElement(name="fileCont") + @XmlElement(name = "fileCont") TextElement fileCont; } diff --git a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/LanguageEnum.java b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddicodebook/LanguageEnum.java similarity index 88% rename from src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/LanguageEnum.java rename to src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddicodebook/LanguageEnum.java index c3c914ab36c..10601ebe139 100644 --- a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/LanguageEnum.java +++ b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddicodebook/LanguageEnum.java @@ -1,10 +1,10 @@ -package eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddiCodebook; +package eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddicodebook; public enum LanguageEnum { de("de"), en("en"); /** - * The language string + * The language string. */ public final String languageString; diff --git a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/StdyDscr.java b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddicodebook/StdyDscr.java similarity index 61% rename from src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/StdyDscr.java rename to src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddicodebook/StdyDscr.java index 55fc9fe8ad8..302ff532836 100644 --- a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/StdyDscr.java +++ b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddicodebook/StdyDscr.java @@ -1,15 +1,21 @@ -package eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddiCodebook; +package eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddicodebook; import javax.xml.bind.annotation.XmlElement; import lombok.AllArgsConstructor; +/** + * Class representing the DDI stdyDscr element. + */ @AllArgsConstructor public class StdyDscr { + /** + * Constructor needed by JAXB. + */ public StdyDscr() {} - @XmlElement(name="citation") + @XmlElement(name = "citation") Citation citation; } diff --git a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/TextElement.java b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddicodebook/TextElement.java similarity index 88% rename from src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/TextElement.java rename to src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddicodebook/TextElement.java index 91d3020315d..67a9e86880f 100644 --- a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/TextElement.java +++ b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddicodebook/TextElement.java @@ -1,4 +1,4 @@ -package eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddiCodebook; +package eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddicodebook; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; @@ -15,9 +15,12 @@ @XmlAccessorType(XmlAccessType.NONE) public class TextElement { + /** + * Constructor needed by JAXB. + */ public TextElement() {} - @XmlAttribute(name="xml:lang") + @XmlAttribute(name = "xml:lang") private LanguageEnum lang; @XmlValue diff --git a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/TitlStmt.java b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddicodebook/TitlStmt.java similarity index 59% rename from src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/TitlStmt.java rename to src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddicodebook/TitlStmt.java index b050aded1ce..365e7008762 100644 --- a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/TitlStmt.java +++ b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddicodebook/TitlStmt.java @@ -1,18 +1,24 @@ -package eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddiCodebook; +package eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddicodebook; import javax.xml.bind.annotation.XmlElement; import lombok.AllArgsConstructor; +/** + * Class representing the DDI titlStmt element. + */ @AllArgsConstructor public class TitlStmt { + /** + * Constructor needed by JAXB. + */ public TitlStmt() {} - @XmlElement(name="titl") + @XmlElement(name = "titl") TextElement titl; - @XmlElement(name="parTitl") + @XmlElement(name = "parTitl") TextElement parTitle; } diff --git a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/Var.java b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddicodebook/Var.java similarity index 57% rename from src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/Var.java rename to src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddicodebook/Var.java index 4e50335ce01..cabc51cb15d 100644 --- a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/Var.java +++ b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddicodebook/Var.java @@ -1,4 +1,4 @@ -package eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddiCodebook; +package eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddicodebook; import java.util.List; @@ -7,27 +7,33 @@ import lombok.AllArgsConstructor; +/** + * Class representing the DDI Codebook var element. + */ @AllArgsConstructor public class Var { + /** + * Constructor needed by JAXB. + */ public Var() {} - @XmlAttribute(name="name") + @XmlAttribute(name = "name") String name; - @XmlAttribute(name="files") + @XmlAttribute(name = "files") String files; - @XmlElement(name="labl") + @XmlElement(name = "labl") List labl; - @XmlElement(name="qstn") + @XmlElement(name = "qstn") List qstn; - @XmlElement(name="txt") + @XmlElement(name = "txt") List txt; - @XmlElement(name="catgry") + @XmlElement(name = "catgry") List catgry; } diff --git a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/package-info.java b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddicodebook/package-info.java similarity index 90% rename from src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/package-info.java rename to src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddicodebook/package-info.java index 5df93d28d08..3c18219797c 100644 --- a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddiCodebook/package-info.java +++ b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddicodebook/package-info.java @@ -1,6 +1,5 @@ - /** - * Configuration for JAXB + * Configuration for JAXB. */ @XmlSchema( @@ -10,12 +9,9 @@ }, namespace = "ddi:codebook:2_5", elementFormDefault = XmlNsForm.QUALIFIED) -package eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddiCodebook; + +package eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddicodebook; import javax.xml.bind.annotation.XmlNs; import javax.xml.bind.annotation.XmlNsForm; import javax.xml.bind.annotation.XmlSchema; - - - - diff --git a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/rest/ExportDdiVariablesResourceController.java b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/rest/ExportDdiVariablesResourceController.java index c865ffe82f8..b814bc457dd 100644 --- a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/rest/ExportDdiVariablesResourceController.java +++ b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/rest/ExportDdiVariablesResourceController.java @@ -38,7 +38,7 @@ public class ExportDdiVariablesResourceController { public ResponseEntity exportVariablesAsXml(@PathVariable String dataPackageId) { ResponseEntity response = new ResponseEntity<>(null, null, HttpStatus.NOT_FOUND); try { - response = this.dataPackageDdiService.exportDdiVariablesAsXML(dataPackageId); + response = this.dataPackageDdiService.exportDdiVariablesAsXml(dataPackageId); } catch (PersistenceException ex) { return response; } diff --git a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/service/DataPackageDdiService.java b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/service/DataPackageDdiService.java index ba26d8acf61..297f1ceda49 100644 --- a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/service/DataPackageDdiService.java +++ b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/service/DataPackageDdiService.java @@ -13,17 +13,17 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.google.gson.Gson; import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.DataPackage; -import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddiCodebook.Catgry; -import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddiCodebook.Citation; -import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddiCodebook.CodeBook; -import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddiCodebook.DataDscr; -import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddiCodebook.FileDscr; -import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddiCodebook.FileTxt; -import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddiCodebook.LanguageEnum; -import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddiCodebook.StdyDscr; -import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddiCodebook.TextElement; -import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddiCodebook.TitlStmt; -import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddiCodebook.Var; +import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddicodebook.Catgry; +import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddicodebook.Citation; +import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddicodebook.CodeBook; +import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddicodebook.DataDscr; +import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddicodebook.FileDscr; +import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddicodebook.FileTxt; +import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddicodebook.LanguageEnum; +import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddicodebook.StdyDscr; +import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddicodebook.TextElement; +import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddicodebook.TitlStmt; +import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddicodebook.Var; import eu.dzhw.fdz.metadatamanagement.searchmanagement.documents.DataPackageSearchDocument; import eu.dzhw.fdz.metadatamanagement.searchmanagement.documents.DataSetSubDocument; import eu.dzhw.fdz.metadatamanagement.searchmanagement.documents.QuestionSearchDocument; @@ -53,8 +53,9 @@ /** * Service for construction DDI codebook metadata for all veriables of a {@link DataPackage}. - * @since Sep 2024 + * * @author Theresa Möller + * @since Sep 2024 */ @Service @RepositoryEventHandler @@ -68,16 +69,18 @@ public class DataPackageDdiService { /** * Exports all variables metadata belonging to a given data package according to the DDI Codebook standard. + * * @param dataPackageId the ID of the study * @return DDI metadata as XML */ - public ResponseEntity exportDdiVariablesAsXML(String dataPackageId) { + public ResponseEntity exportDdiVariablesAsXml(String dataPackageId) { try { CodeBook variableMetadata = this.getDdiVariablesMetadata(dataPackageId); JAXBContext context = JAXBContext.newInstance(CodeBook.class); Marshaller mar = context.createMarshaller(); mar.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, Boolean.TRUE); - mar.setProperty(Marshaller.JAXB_SCHEMA_LOCATION, "http://www.ddialliance.org/Specification/DDI-Codebook/2.5/XMLSchema/codebook.xsd");; + mar.setProperty(Marshaller.JAXB_SCHEMA_LOCATION, + "http://www.ddialliance.org/Specification/DDI-Codebook/2.5/XMLSchema/codebook.xsd"); ByteArrayOutputStream res = new ByteArrayOutputStream(); mar.marshal(variableMetadata, res); ByteArrayResource resource = new ByteArrayResource(res.toByteArray()); @@ -94,6 +97,7 @@ public ResponseEntity exportDdiVariablesAsXML(String dataPackageId) { /** * Collects variables metadata according to DDI Codebook standard. + * * @return the metadata */ private CodeBook getDdiVariablesMetadata(String dataPackageId) throws JsonProcessingException { @@ -112,12 +116,12 @@ private CodeBook getDdiVariablesMetadata(String dataPackageId) throws JsonProces String.format("Expected one data package for id '%s', but found %d", dataPackageId, hits.size())); } DataPackageSearchDocument dataPackageDoc = gson.fromJson( - hits.get(0).getSourceAsString(), DataPackageSearchDocument.class); - if (dataPackageDoc.getRelease() == null || - (dataPackageDoc.getRelease() != null && dataPackageDoc.getRelease().getIsPreRelease())) { + hits.get(0).getSourceAsString(), DataPackageSearchDocument.class); + if (dataPackageDoc.getRelease() == null + || (dataPackageDoc.getRelease() != null && dataPackageDoc.getRelease().getIsPreRelease())) { throw new ElasticsearchException( - String.format("DDI codebook export failed. Can only export released data packages. " + - "Data package with id '%s' is not released.", dataPackageDoc.getId()) + String.format("DDI codebook export failed. Can only export released data packages. " + + "Data package with id '%s' is not released.", dataPackageDoc.getId()) ); } @@ -144,15 +148,16 @@ private CodeBook getDdiVariablesMetadata(String dataPackageId) throws JsonProces /** * Create the DDI var Element with data from the variable, and its related questions. + * * @return the var element */ private Var getDdiVar(VariableSubDocument variableDoc) { String name = variableDoc.getId(); String files = variableDoc.getDataSetId(); - List varLablList = new ArrayList<>(); + List varLablList = new ArrayList(); varLablList.add(new TextElement(LanguageEnum.de, variableDoc.getLabel().getDe())); varLablList.add(new TextElement(LanguageEnum.en, variableDoc.getLabel().getEn())); - List qstnList = new ArrayList<>(); + List qstnList = new ArrayList(); if (variableDoc.getRelatedQuestions() != null && variableDoc.getRelatedQuestions().size() > 0) { for (RelatedQuestionSubDocumentProjection relQuest : variableDoc.getRelatedQuestions()) { SearchRequest questionRequest = new SearchRequest(); @@ -207,8 +212,8 @@ private Var getDdiVar(VariableSubDocument variableDoc) { txtList.add(new TextElement(LanguageEnum.en, varDoc.getAnnotations().getEn())); } if ((varDoc.getScaleLevel().equals(ScaleLevels.NOMINAL) || varDoc.getScaleLevel().equals(ScaleLevels.ORDINAL)) - && varDoc.getDistribution() != null - && varDoc.getDistribution().getValidResponses() != null) { + && varDoc.getDistribution() != null + && varDoc.getDistribution().getValidResponses() != null) { for (ValidResponse validResponse : varDoc.getDistribution().getValidResponses()) { String catValu = validResponse.getValue(); List catLablList = new ArrayList<>(); @@ -222,27 +227,31 @@ private Var getDdiVar(VariableSubDocument variableDoc) { } // missing values if (varDoc.getDistribution() != null && varDoc.getDistribution().getMissings() != null) - for (Missing missing : varDoc.getDistribution().getMissings()) { - String catValu = missing.getCode(); - List catLablList = new ArrayList<>(); - if (missing.getLabel() != null && missing.getLabel().getDe() != null) { - catLablList.add(new TextElement(LanguageEnum.de, missing.getLabel().getDe())); + for (Missing missing : varDoc.getDistribution().getMissings()) { + String catValu = missing.getCode(); + List catLablList = new ArrayList<>(); + if (missing.getLabel() != null && missing.getLabel().getDe() != null) { + catLablList.add(new TextElement(LanguageEnum.de, missing.getLabel().getDe())); + } + if (missing.getLabel() != null && missing.getLabel().getEn() != null) { + catLablList.add(new TextElement(LanguageEnum.en, missing.getLabel().getEn())); + } + catgryList.add(new Catgry(catValu, catLablList)); } - if (missing.getLabel() != null && missing.getLabel().getEn() != null) { - catLablList.add(new TextElement(LanguageEnum.en, missing.getLabel().getEn())); - } - catgryList.add(new Catgry(catValu, catLablList)); - } } } } catch (IOException e) { log.error("An exception occurred querying the variables index. ", e); } - return new Var(name, files, varLablList, qstnList.size() > 0 ? qstnList : null, txtList, catgryList); + return new Var(name, files, varLablList, + qstnList.size() > 0 ? qstnList : null, + txtList.size() > 0 ? txtList : null, + catgryList); } /** * Create the DDI element fileDscr with data from the datasets of the data package. + * * @return the fileDscr element */ private FileDscr getDdiFileDsrc(DataSetSubDocument dataset) { @@ -256,6 +265,7 @@ private FileDscr getDdiFileDsrc(DataSetSubDocument dataset) { /** * Creates the DDI element stdyDscr with data from the data package. + * * @return the stdyDscr element */ private StdyDscr getDdiStdyDscr(DataPackageSearchDocument doc) { diff --git a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/service/DataPackageManagementService.java b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/service/DataPackageManagementService.java index 1e2b7f8959f..59ee928a17b 100644 --- a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/service/DataPackageManagementService.java +++ b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/service/DataPackageManagementService.java @@ -1,41 +1,22 @@ package eu.dzhw.fdz.metadatamanagement.datapackagemanagement.service; -import java.io.IOException; -import java.util.ArrayList; import java.util.List; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; -import org.springframework.core.io.ByteArrayResource; import org.springframework.data.rest.core.annotation.HandleAfterCreate; import org.springframework.data.rest.core.annotation.HandleAfterDelete; import org.springframework.data.rest.core.annotation.HandleAfterSave; import org.springframework.data.rest.core.annotation.RepositoryEventHandler; -import org.springframework.http.HttpHeaders; -import org.springframework.http.HttpStatus; -import org.springframework.http.ResponseEntity; import org.springframework.security.access.annotation.Secured; import org.springframework.stereotype.Service; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.dataformat.xml.XmlMapper; -import com.google.gson.Gson; import eu.dzhw.fdz.metadatamanagement.analysispackagemanagement.domain.AnalysisPackage; import eu.dzhw.fdz.metadatamanagement.common.service.CrudService; import eu.dzhw.fdz.metadatamanagement.conceptmanagement.domain.Concept; import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.DataPackage; -import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddiCodebook.Catgry; -import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddiCodebook.Citation; -import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddiCodebook.DataDscr; -import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddiCodebook.FileDscr; -import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddiCodebook.FileTxt; -import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddiCodebook.LanguageEnum; -import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddiCodebook.StdyDscr; -import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddiCodebook.TextElement; -import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddiCodebook.TitlStmt; -import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddiCodebook.Var; import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.repository.DataPackageRepository; import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.service.helper.DataPackageCrudHelper; import eu.dzhw.fdz.metadatamanagement.datasetmanagement.domain.DataSet; @@ -53,11 +34,9 @@ import eu.dzhw.fdz.metadatamanagement.searchmanagement.service.ElasticsearchUpdateQueueService; import eu.dzhw.fdz.metadatamanagement.surveymanagement.domain.Survey; import eu.dzhw.fdz.metadatamanagement.usermanagement.security.AuthoritiesConstants; -import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddiCodebook.CodeBook; import eu.dzhw.fdz.metadatamanagement.variablemanagement.domain.Variable; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.elasticsearch.client.RestHighLevelClient; /** * Service for managing the domain object/aggregate {@link DataPackage}. From 5fa605ee3fa988abf7183a2ad14b4e7cb79f5ad7 Mon Sep 17 00:00:00 2001 From: theresa Date: Mon, 23 Sep 2024 11:18:47 +0200 Subject: [PATCH 20/33] #3304 code style fix part II --- .../domain/ddicodebook/LanguageEnum.java | 3 +++ .../service/DataPackageDdiService.java | 19 ++++++++++--------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddicodebook/LanguageEnum.java b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddicodebook/LanguageEnum.java index 10601ebe139..e38c80a4729 100644 --- a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddicodebook/LanguageEnum.java +++ b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/domain/ddicodebook/LanguageEnum.java @@ -1,5 +1,8 @@ package eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.ddicodebook; +/** + * Enum representing available language abbreviations. + */ public enum LanguageEnum { de("de"), en("en"); diff --git a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/service/DataPackageDdiService.java b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/service/DataPackageDdiService.java index 297f1ceda49..fdf29161ca4 100644 --- a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/service/DataPackageDdiService.java +++ b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/service/DataPackageDdiService.java @@ -52,7 +52,7 @@ import org.springframework.stereotype.Service; /** - * Service for construction DDI codebook metadata for all veriables of a {@link DataPackage}. + * Service for construction DDI codebook metadata for all variables of a {@link DataPackage}. * * @author Theresa Möller * @since Sep 2024 @@ -80,7 +80,7 @@ public ResponseEntity exportDdiVariablesAsXml(String dataPackageId) { Marshaller mar = context.createMarshaller(); mar.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, Boolean.TRUE); mar.setProperty(Marshaller.JAXB_SCHEMA_LOCATION, - "http://www.ddialliance.org/Specification/DDI-Codebook/2.5/XMLSchema/codebook.xsd"); + "http://www.ddialliance.org/Specification/DDI-Codebook/2.5/XMLSchema/codebook.xsd"); ByteArrayOutputStream res = new ByteArrayOutputStream(); mar.marshal(variableMetadata, res); ByteArrayResource resource = new ByteArrayResource(res.toByteArray()); @@ -152,8 +152,6 @@ private CodeBook getDdiVariablesMetadata(String dataPackageId) throws JsonProces * @return the var element */ private Var getDdiVar(VariableSubDocument variableDoc) { - String name = variableDoc.getId(); - String files = variableDoc.getDataSetId(); List varLablList = new ArrayList(); varLablList.add(new TextElement(LanguageEnum.de, variableDoc.getLabel().getDe())); varLablList.add(new TextElement(LanguageEnum.en, variableDoc.getLabel().getEn())); @@ -174,7 +172,7 @@ private Var getDdiVar(VariableSubDocument variableDoc) { } for (SearchHit hit : hits) { QuestionSearchDocument relatedQuestion = gson.fromJson( - hit.getSourceAsString(), QuestionSearchDocument.class); + hit.getSourceAsString(), QuestionSearchDocument.class); if (relatedQuestion.getQuestionText() != null && relatedQuestion.getQuestionText().getDe() != null) { qstnList.add(new TextElement(LanguageEnum.de, relatedQuestion.getQuestionText().getDe())); } @@ -204,7 +202,7 @@ private Var getDdiVar(VariableSubDocument variableDoc) { } for (SearchHit hit : hits) { VariableSearchDocument varDoc = gson.fromJson( - hit.getSourceAsString(), VariableSearchDocument.class); + hit.getSourceAsString(), VariableSearchDocument.class); if (varDoc.getAnnotations() != null && varDoc.getAnnotations().getDe() != null) { txtList.add(new TextElement(LanguageEnum.de, varDoc.getAnnotations().getDe())); } @@ -212,8 +210,8 @@ private Var getDdiVar(VariableSubDocument variableDoc) { txtList.add(new TextElement(LanguageEnum.en, varDoc.getAnnotations().getEn())); } if ((varDoc.getScaleLevel().equals(ScaleLevels.NOMINAL) || varDoc.getScaleLevel().equals(ScaleLevels.ORDINAL)) - && varDoc.getDistribution() != null - && varDoc.getDistribution().getValidResponses() != null) { + && varDoc.getDistribution() != null + && varDoc.getDistribution().getValidResponses() != null) { for (ValidResponse validResponse : varDoc.getDistribution().getValidResponses()) { String catValu = validResponse.getValue(); List catLablList = new ArrayList<>(); @@ -226,7 +224,7 @@ private Var getDdiVar(VariableSubDocument variableDoc) { catgryList.add(new Catgry(catValu, catLablList)); } // missing values - if (varDoc.getDistribution() != null && varDoc.getDistribution().getMissings() != null) + if (varDoc.getDistribution() != null && varDoc.getDistribution().getMissings() != null) { for (Missing missing : varDoc.getDistribution().getMissings()) { String catValu = missing.getCode(); List catLablList = new ArrayList<>(); @@ -238,11 +236,14 @@ private Var getDdiVar(VariableSubDocument variableDoc) { } catgryList.add(new Catgry(catValu, catLablList)); } + } } } } catch (IOException e) { log.error("An exception occurred querying the variables index. ", e); } + String name = variableDoc.getId(); + String files = variableDoc.getDataSetId(); return new Var(name, files, varLablList, qstnList.size() > 0 ? qstnList : null, txtList.size() > 0 ? txtList : null, From 0d9544e60ba77f86b600591b87805f9c37f71561 Mon Sep 17 00:00:00 2001 From: theresa Date: Mon, 23 Sep 2024 12:55:25 +0200 Subject: [PATCH 21/33] #3304 code style fix part III --- .../rest/ExportDdiVariablesResourceController.java | 1 - .../variablemanagement/service/VariableManagementService.java | 1 - 2 files changed, 2 deletions(-) diff --git a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/rest/ExportDdiVariablesResourceController.java b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/rest/ExportDdiVariablesResourceController.java index b814bc457dd..5886f086044 100644 --- a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/rest/ExportDdiVariablesResourceController.java +++ b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/rest/ExportDdiVariablesResourceController.java @@ -3,7 +3,6 @@ import javax.persistence.PersistenceException; import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.service.DataPackageDdiService; -import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.service.DataPackageManagementService; import eu.dzhw.fdz.metadatamanagement.usermanagement.security.AuthoritiesConstants; import lombok.RequiredArgsConstructor; diff --git a/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/service/VariableManagementService.java b/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/service/VariableManagementService.java index d6017438d6f..ed106981111 100644 --- a/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/service/VariableManagementService.java +++ b/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/service/VariableManagementService.java @@ -22,7 +22,6 @@ import eu.dzhw.fdz.metadatamanagement.variablemanagement.repository.VariableRepository; import eu.dzhw.fdz.metadatamanagement.variablemanagement.service.helper.VariableCrudHelper; import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; import org.elasticsearch.client.RestHighLevelClient; import org.springframework.data.rest.core.annotation.HandleAfterCreate; From a29ff4728c578550b91e86e09648736eed38c214 Mon Sep 17 00:00:00 2001 From: theresa Date: Tue, 24 Sep 2024 09:52:51 +0200 Subject: [PATCH 22/33] #3304 removed jackson --- pom.xml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pom.xml b/pom.xml index 32a6413cbc2..7632a09471b 100644 --- a/pom.xml +++ b/pom.xml @@ -66,10 +66,6 @@ com.fasterxml.jackson.core jackson-databind - - com.fasterxml.jackson.dataformat - jackson-dataformat-xml - com.github.cloudyrock.mongock mongock-spring-v5 From fcc11141c316bcfcacc855f1733f946a55dd272b Mon Sep 17 00:00:00 2001 From: theresa Date: Tue, 24 Sep 2024 09:53:41 +0200 Subject: [PATCH 23/33] #3304 placed export button in order menu, removed it from datapackage detail page --- .../views/data-package-detail.controller.js | 24 +------------ .../views/data-package-detail.html.tmpl | 16 ++------- .../data-package-configurator.controller.js | 34 ++++++++++++++++++- .../data-package-configurator.html.tmpl | 10 ++++++ .../configuration/translations-de.js | 4 ++- .../configuration/translations-en.js | 4 ++- .../views/survey-detail.controller.js | 8 ----- 7 files changed, 53 insertions(+), 47 deletions(-) diff --git a/mdm-frontend/src/app/legacy/datapackagemanagement/views/data-package-detail.controller.js b/mdm-frontend/src/app/legacy/datapackagemanagement/views/data-package-detail.controller.js index bce54e94843..5ae229afc46 100644 --- a/mdm-frontend/src/app/legacy/datapackagemanagement/views/data-package-detail.controller.js +++ b/mdm-frontend/src/app/legacy/datapackagemanagement/views/data-package-detail.controller.js @@ -35,7 +35,6 @@ angular.module('metadatamanagementApp') 'blockUI', '$mdSidenav', 'ContainsOnlyQualitativeDataChecker', - 'ExportDdiVariablesResource', function(entity, MessageBus, PageMetadataService, @@ -49,8 +48,7 @@ angular.module('metadatamanagementApp') ProjectUpdateAccessService, $scope, $timeout, $document, DataPackageOverviewResource, OutdatedVersionNotifier, DataPackageSearchService, $log, - blockUI, $mdSidenav, ContainsOnlyQualitativeDataChecker, - ExportDdiVariablesResource) { + blockUI, $mdSidenav, ContainsOnlyQualitativeDataChecker) { blockUI.start(); SearchResultNavigatorService .setSearchIndex($stateParams['search-result-index']); @@ -91,8 +89,6 @@ angular.module('metadatamanagementApp') .hasAnyAuthority(['ROLE_PUBLISHER', 'ROLE_ADMIN']); ctrl.showRemarks = Principal .hasAnyAuthority(['ROLE_PUBLISHER']); - ctrl.enableVariableExport = Principal - .hasAnyAuthority(['ROLE_PUBLISHER', 'ROLE_ADMIN']); var bowser = $rootScope.bowser; @@ -310,23 +306,5 @@ angular.module('metadatamanagementApp') } return true; } - - /** - * Exports Variables metadata as DDI Codebook XML. - */ - ctrl.exportVariables = function() { - console.log(ctrl.dataPackage) - ExportDdiVariablesResource.exportVariablesAsXml(ctrl.dataPackage.id).then(function(res) { - var blob = new Blob([res],{ - type: "application/xml;charset=utf-8;" - }); - var downloadLink = document.createElement('a'); - const fileName = 'mdm_export_ddi_variables_' + ctrl.dataPackage.id + '.xml' - downloadLink.setAttribute('download', fileName); - downloadLink.setAttribute('href', window.URL.createObjectURL(blob)); - downloadLink.click(); - $scope.isDownloadingData = false; - }) - } }]); diff --git a/mdm-frontend/src/app/legacy/datapackagemanagement/views/data-package-detail.html.tmpl b/mdm-frontend/src/app/legacy/datapackagemanagement/views/data-package-detail.html.tmpl index d1a447454c1..7d57fada3a7 100644 --- a/mdm-frontend/src/app/legacy/datapackagemanagement/views/data-package-detail.html.tmpl +++ b/mdm-frontend/src/app/legacy/datapackagemanagement/views/data-package-detail.html.tmpl @@ -6,20 +6,10 @@
-
+

-
- - - - Variablen als XML nach DDI Standard exportieren - - DDI - -
+
{{'data-package-management.detail.label.tags' | translate}}:
diff --git a/mdm-frontend/src/app/legacy/ordermanagement/components/data-package-configurator.controller.js b/mdm-frontend/src/app/legacy/ordermanagement/components/data-package-configurator.controller.js index 9910d52a6d6..95181230c84 100644 --- a/mdm-frontend/src/app/legacy/ordermanagement/components/data-package-configurator.controller.js +++ b/mdm-frontend/src/app/legacy/ordermanagement/components/data-package-configurator.controller.js @@ -24,6 +24,8 @@ '$q', 'dataAcquisitionProjectSearchService', 'ElasticSearchClient', + 'ExportDdiVariablesResource', + 'Principal', function ($scope, $rootScope, $location, @@ -38,7 +40,8 @@ DataPackageAccessWaysResource, $mdDialog, DataPackageCitationDialogService, CurrentDataPackageService, - DataAcquisitionProjectResource, $q, dataAcquisitionProjectSearchService, ElasticSearchClient) { + DataAcquisitionProjectResource, $q, dataAcquisitionProjectSearchService, ElasticSearchClient, + ExportDdiVariablesResource, Principal) { var $ctrl = this; var initReady = false; $ctrl.dataPackageIdVersion = {}; @@ -49,6 +52,7 @@ $ctrl.isPreReleased = false; $ctrl.variableNotAccessible = false; $ctrl.disabled = false; + $ctrl.allowedToExportVariableMetadata = false; $scope.bowser = $rootScope.bowser; $ctrl.numberOfShoppingCartProducts = ShoppingCartService.count(); @@ -89,6 +93,7 @@ $ctrl.selectedVersion = $ctrl.dataPackageIdVersion.version; loadDataPackage($ctrl.dataPackageIdVersion.masterId, $ctrl.dataPackageIdVersion.version); + $ctrl.allowedToExportVariableMetadata = Principal.isAuthenticated(); initReady = true; } @@ -390,4 +395,31 @@ } return true; } + + /** + * Checks if the latest version is selected. + * @returns true if the latest version is selected else false + */ + $ctrl.isLatestVersionSelected = function() { + return this.selectedVersion + && $ctrl.releases + && $ctrl.releases.length > 0 ? $ctrl.releases[0].version === this.selectedVersion : false; + } + + /** + * Exports Variables metadata as DDI Codebook XML. + */ + $ctrl.exportVariables = function() { + ExportDdiVariablesResource.exportVariablesAsXml($ctrl.dataPackage.id).then(function(res) { + var blob = new Blob([res],{ + type: "application/xml;charset=utf-8;" + }); + var downloadLink = document.createElement('a'); + const fileName = 'mdm_export_ddi_variables_' + $ctrl.dataPackage.id.split('$')[0] + '.xml' + downloadLink.setAttribute('download', fileName); + downloadLink.setAttribute('href', window.URL.createObjectURL(blob)); + downloadLink.click(); + $scope.isDownloadingData = false; + }) + } }]); diff --git a/mdm-frontend/src/app/legacy/ordermanagement/components/data-package-configurator.html.tmpl b/mdm-frontend/src/app/legacy/ordermanagement/components/data-package-configurator.html.tmpl index 830ccead602..5418219f905 100644 --- a/mdm-frontend/src/app/legacy/ordermanagement/components/data-package-configurator.html.tmpl +++ b/mdm-frontend/src/app/legacy/ordermanagement/components/data-package-configurator.html.tmpl @@ -113,6 +113,16 @@ + + + + {{'shopping-cart.buttons.export-ddi-variables-tooltip' | translate}} + + {{'shopping-cart.buttons.export-ddi-variables' | translate}} + + diff --git a/mdm-frontend/src/app/legacy/ordermanagement/configuration/translations-de.js b/mdm-frontend/src/app/legacy/ordermanagement/configuration/translations-de.js index 3b57a1d20dc..201194b1043 100644 --- a/mdm-frontend/src/app/legacy/ordermanagement/configuration/translations-de.js +++ b/mdm-frontend/src/app/legacy/ordermanagement/configuration/translations-de.js @@ -128,7 +128,9 @@ angular.module('metadatamanagementApp').config([ 'related-publications-tooltip': 'Klicken, um weitere Informationen zu Publikationen zu erhalten.', 'data-package-version-tooltip': 'Klicken, um weitere Informationen zur Version von Datenpaketen zu erhalten.', 'analysis-package-version-tooltip': 'Klicken, um weitere Informationen zur Version von Analysepaketen zu erhalten.', - 'data-package-access-way-tooltip': 'Klicken, um weitere Informationen zu Zugangswegen zu erhalten.' + 'data-package-access-way-tooltip': 'Klicken, um weitere Informationen zu Zugangswegen zu erhalten.', + 'export-ddi-variables': 'Variablenmetadaten exportieren', + 'export-ddi-variables-tooltip': 'Klicken, um die Variablen-Metadaten als DDI-Codebook-XML herunterzuladen.' }, 'version-info': { 'title': 'Eine Version auswählen', diff --git a/mdm-frontend/src/app/legacy/ordermanagement/configuration/translations-en.js b/mdm-frontend/src/app/legacy/ordermanagement/configuration/translations-en.js index 4dbe6f75a8e..4b63b645624 100644 --- a/mdm-frontend/src/app/legacy/ordermanagement/configuration/translations-en.js +++ b/mdm-frontend/src/app/legacy/ordermanagement/configuration/translations-en.js @@ -127,7 +127,9 @@ angular.module('metadatamanagementApp').config([ 'analysis-package-tooltip': 'Click to get more information about analysis packages', 'related-publications-tooltip': 'Click to get more information about publications', 'analysis-package-version-tooltip': 'Click to get more information about versions of analysis packages', - 'data-package-access-way-tooltip': 'Click to get more information about access ways' + 'data-package-access-way-tooltip': 'Click to get more information about access ways', + 'export-ddi-variables': 'Export variable metadata', + 'export-ddi-variables-tooltip': 'Click to download the metadata of all variables as a DDI codebook XML.' }, 'version-info': { 'title': 'Select a Version', diff --git a/mdm-frontend/src/app/legacy/surveymanagement/views/survey-detail.controller.js b/mdm-frontend/src/app/legacy/surveymanagement/views/survey-detail.controller.js index d8bfe997cbc..ef66dc214c1 100644 --- a/mdm-frontend/src/app/legacy/surveymanagement/views/survey-detail.controller.js +++ b/mdm-frontend/src/app/legacy/surveymanagement/views/survey-detail.controller.js @@ -24,7 +24,6 @@ angular.module('metadatamanagementApp') '$stateParams', 'blockUI', '$mdSidenav', - // 'ExportDdiVariablesResource', function(entity, LanguageService, CleanJSObjectService, PageMetadataService, $state, BreadcrumbService, MessageBus, SurveySearchService, SurveyAttachmentResource, Principal, @@ -54,19 +53,12 @@ angular.module('metadatamanagementApp') ctrl.responseRateImage = null; ctrl.enableJsonView = Principal .hasAnyAuthority(['ROLE_PUBLISHER', 'ROLE_ADMIN']); - // ctrl.enableVariableExport = Principal - // .hasAnyAuthority(['ROLE_PUBLISHER', 'ROLE_ADMIN']); - ctrl.surveyId = null; entity.promise.then(function(survey) { var fetchFn = SurveySearchService.findShadowByIdAndVersion .bind(null, survey.masterId, null, ['nested*','variables','questions', 'instruments', 'dataSets', 'relatedPublications','concepts']); OutdatedVersionNotifier.checkVersionAndNotify(survey, fetchFn); -; - if (survey.release) { - ctrl.surveyId = survey.id + '-' + survey.release.version; - } if (Principal .hasAnyAuthority(['ROLE_PUBLISHER', 'ROLE_DATA_PROVIDER'])) { From e832333813a8af2702d7476a08de799f5e262583 Mon Sep 17 00:00:00 2001 From: theresa Date: Tue, 24 Sep 2024 09:54:28 +0200 Subject: [PATCH 24/33] #3304 optimized id representation in export --- .../service/DataPackageDdiService.java | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/service/DataPackageDdiService.java b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/service/DataPackageDdiService.java index fdf29161ca4..450eb7e67b8 100644 --- a/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/service/DataPackageDdiService.java +++ b/src/main/java/eu/dzhw/fdz/metadatamanagement/datapackagemanagement/service/DataPackageDdiService.java @@ -242,8 +242,8 @@ private Var getDdiVar(VariableSubDocument variableDoc) { } catch (IOException e) { log.error("An exception occurred querying the variables index. ", e); } - String name = variableDoc.getId(); - String files = variableDoc.getDataSetId(); + String name = variableDoc.getId().split("\\$")[0]; + String files = variableDoc.getDataSetId().split("\\$")[0]; return new Var(name, files, varLablList, qstnList.size() > 0 ? qstnList : null, txtList.size() > 0 ? txtList : null, @@ -256,11 +256,9 @@ private Var getDdiVar(VariableSubDocument variableDoc) { * @return the fileDscr element */ private FileDscr getDdiFileDsrc(DataSetSubDocument dataset) { - String id = dataset.getId(); - String fileName = dataset.getId(); - // todo: Clarify why this is just in German and if it is the correct field + String id = dataset.getId().split("\\$")[0]; TextElement fileCont = new TextElement(LanguageEnum.de, dataset.getDescription().getDe()); - FileTxt fileTxt = new FileTxt(fileName, fileCont); + FileTxt fileTxt = new FileTxt(id, fileCont); return new FileDscr(id, fileTxt); } From b492a3b18d0bd833ba7e9b0f471e2df0ac2d38a4 Mon Sep 17 00:00:00 2001 From: theresa Date: Tue, 24 Sep 2024 10:09:56 +0200 Subject: [PATCH 25/33] #3304 fixed spotbugs unused fields error --- .../variablemanagement/service/VariableManagementService.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/service/VariableManagementService.java b/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/service/VariableManagementService.java index ed106981111..0d14f331885 100644 --- a/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/service/VariableManagementService.java +++ b/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/service/VariableManagementService.java @@ -49,10 +49,6 @@ public class VariableManagementService implements CrudService { private final VariableCrudHelper crudHelper; - private final RestHighLevelClient client; - - private final Gson gson; - /** * Delete all variables when the dataAcquisitionProject was deleted. * From e23c654f8f060e0ebf0d621835778685099c6323 Mon Sep 17 00:00:00 2001 From: theresa Date: Tue, 24 Sep 2024 14:22:32 +0200 Subject: [PATCH 26/33] #3303 optimized access ways selection for PID registration --- .../service/VariableManagementService.java | 36 +++++++++---------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/service/VariableManagementService.java b/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/service/VariableManagementService.java index 6803868b7ff..23329035a6d 100644 --- a/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/service/VariableManagementService.java +++ b/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/service/VariableManagementService.java @@ -9,6 +9,8 @@ import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Optional; @@ -64,17 +66,11 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.RestHighLevelClient; -import org.elasticsearch.client.core.CountRequest; -import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.ExistsQueryBuilder; -import org.elasticsearch.index.query.MatchQueryBuilder; -import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.query.TermQueryBuilder; import org.elasticsearch.search.SearchHit; -import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.objectweb.asm.TypeReference; /** * Service for managing the domain object/aggregate {@link Variable}. @@ -320,7 +316,6 @@ private ArrayNode getPidVariablesMetadata() { List hitsDataPackages = Arrays.asList(responseDataPackage.getHits().getHits()); assert hitsDataPackages.size() == 1; - // 3. ES-Query Variables shadow=true, dataAcquisitionProjectId=project.id-project.version SearchRequest variablesRequest = new SearchRequest(); SearchSourceBuilder builderVariables = new SearchSourceBuilder(); builderVariables.query(QueryBuilders.boolQuery() @@ -366,18 +361,13 @@ private ObjectNode getPidMetadataOfVariable( ObjectNode variableObj = objectMapper.createObjectNode(); variableObj.put("studyDOI", dataPackage.getDoi()); variableObj.put("variableName", variable.getName()); - // todo: Clarify which language to use - variableObj.put("variableLabel", variable.getLabel().getEn() != null - ? variable.getLabel().getEn() : - variable.getLabel().getDe()); + variableObj.put("variableLabel", variable.getLabel().getEn()); variableObj.put("pidProposal", pidPrefix + project.getId() + "_" + variable.getName() + ":" + project.getRelease().getVersion()); - // todo: Clarify which language to use variableObj.put("landingPage", landingPageBaseUrl + variable.getMasterId().replace("$", "") + "?version=" + project.getRelease().getVersion()); variableObj.put("resourceType", "Variable"); - // todo: Clarify which language to use variableObj.put("title", variable.getName() + ": " + variable.getLabel().getEn()); variableObj.set("creators", this.compileCreators(dataPackage.getProjectContributors())); variableObj.put("publisher", "FDZ-DZHW"); @@ -455,15 +445,23 @@ private ArrayNode compileCreators(List projectContributors) { } /** - * Compiles the given list of accessway entries into a readable string. + * Selects the availablity type according to the available access ways. + * All download, remote and onsite access ways are mapped to "delivery". + * All not available access ways are mapped to "not available". + * Every other access ways us mapped as "unknown". * @param accessWays list of access ways - * @return a readable string of access ways + * @return the availability string */ private String compileAccessWays(List accessWays) { - List strList = new ArrayList<>(); - for (var access : accessWays) { - strList.add(AccessWays.displayAccessWay(access)); + if (accessWays.contains(AccessWays.DOWNLOAD_CUF) + || accessWays.contains(AccessWays.DOWNLOAD_SUF) + || accessWays.contains(AccessWays.REMOTE_DESKTOP) + || accessWays.contains(AccessWays.ONSITE_SUF)) { + return "delivery"; + } else if (accessWays.contains(AccessWays.NOT_ACCESSIBLE)) { + return "not available"; + } else { + return "unknown"; } - return String.join(", ", strList); } } From 39ba60e6989f45077700c727a62cd8ff415732fe Mon Sep 17 00:00:00 2001 From: theresa Date: Tue, 24 Sep 2024 15:07:29 +0200 Subject: [PATCH 27/33] #3303 fixed code style --- .../health/health.controller.js | 1 - .../service/VariableManagementService.java | 19 ++----------------- 2 files changed, 2 insertions(+), 18 deletions(-) diff --git a/mdm-frontend/src/app/legacy/administration/health/health.controller.js b/mdm-frontend/src/app/legacy/administration/health/health.controller.js index 298e1dbee5f..1f2408272ce 100644 --- a/mdm-frontend/src/app/legacy/administration/health/health.controller.js +++ b/mdm-frontend/src/app/legacy/administration/health/health.controller.js @@ -85,7 +85,6 @@ angular.module('metadatamanagementApp').controller('HealthController', [ downloadLink.setAttribute('download', 'variable_pid_mdm_export.json'); downloadLink.setAttribute('href', window.URL.createObjectURL(blob)); downloadLink.click(); - console.log("Download", res); $scope.isDownloadingData = false; }) } diff --git a/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/service/VariableManagementService.java b/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/service/VariableManagementService.java index 23329035a6d..3ee40f06f5b 100644 --- a/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/service/VariableManagementService.java +++ b/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/service/VariableManagementService.java @@ -1,18 +1,9 @@ package eu.dzhw.fdz.metadatamanagement.variablemanagement.service; -import java.io.File; import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.rmi.server.ExportException; import java.time.format.DateTimeFormatter; -import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; import java.util.List; -import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -29,28 +20,22 @@ import org.springframework.security.access.annotation.Secured; import org.springframework.stereotype.Service; -import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.gson.Gson; import eu.dzhw.fdz.metadatamanagement.common.domain.Person; -import eu.dzhw.fdz.metadatamanagement.common.domain.projections.IdAndVersionProjection; import eu.dzhw.fdz.metadatamanagement.common.service.CrudService; import eu.dzhw.fdz.metadatamanagement.conceptmanagement.domain.Concept; import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.DataPackage; -import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.repository.DataPackageRepository; import eu.dzhw.fdz.metadatamanagement.datasetmanagement.domain.DataSet; import eu.dzhw.fdz.metadatamanagement.instrumentmanagement.domain.Instrument; import eu.dzhw.fdz.metadatamanagement.projectmanagement.domain.DataAcquisitionProject; -import eu.dzhw.fdz.metadatamanagement.projectmanagement.domain.Release; -import eu.dzhw.fdz.metadatamanagement.projectmanagement.repository.DataAcquisitionProjectRepository; import eu.dzhw.fdz.metadatamanagement.questionmanagement.domain.Question; import eu.dzhw.fdz.metadatamanagement.questionmanagement.repository.QuestionRepository; import eu.dzhw.fdz.metadatamanagement.searchmanagement.dao.exception.ElasticsearchIoException; import eu.dzhw.fdz.metadatamanagement.searchmanagement.documents.DataAcquisitionProjectSearchDocument; import eu.dzhw.fdz.metadatamanagement.searchmanagement.documents.DataPackageSearchDocument; -import eu.dzhw.fdz.metadatamanagement.searchmanagement.documents.DataPackageSubDocument; import eu.dzhw.fdz.metadatamanagement.searchmanagement.documents.VariableSearchDocument; import eu.dzhw.fdz.metadatamanagement.searchmanagement.service.ElasticsearchType; import eu.dzhw.fdz.metadatamanagement.searchmanagement.service.ElasticsearchUpdateQueueService; @@ -97,8 +82,8 @@ public class VariableManagementService implements CrudService { private final Gson gson; - private final String landingPageBaseUrl = "https://metadata.fdz.dzhw.eu/en/variables/"; - private final String pidPrefix = "21.T11998/dzhw:"; + private static final String landingPageBaseUrl = "https://metadata.fdz.dzhw.eu/en/variables/"; + private static final String pidPrefix = "21.T11998/dzhw:"; From e37bfd0937f4144a33f8297849fdd256181dbe01 Mon Sep 17 00:00:00 2001 From: theresa Date: Wed, 25 Sep 2024 11:01:54 +0200 Subject: [PATCH 28/33] #3304 fixed pmd error --- .../variablemanagement/service/VariableManagementService.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/service/VariableManagementService.java b/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/service/VariableManagementService.java index 0d14f331885..3d6d4d0ac9d 100644 --- a/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/service/VariableManagementService.java +++ b/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/service/VariableManagementService.java @@ -5,7 +5,6 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -import com.google.gson.Gson; import eu.dzhw.fdz.metadatamanagement.common.service.CrudService; import eu.dzhw.fdz.metadatamanagement.conceptmanagement.domain.Concept; import eu.dzhw.fdz.metadatamanagement.datapackagemanagement.domain.DataPackage; @@ -22,7 +21,6 @@ import eu.dzhw.fdz.metadatamanagement.variablemanagement.repository.VariableRepository; import eu.dzhw.fdz.metadatamanagement.variablemanagement.service.helper.VariableCrudHelper; import lombok.RequiredArgsConstructor; -import org.elasticsearch.client.RestHighLevelClient; import org.springframework.data.rest.core.annotation.HandleAfterCreate; import org.springframework.data.rest.core.annotation.HandleAfterDelete; From 2ae4fc86abf5b5283814c7fcc944a97281bd4ace Mon Sep 17 00:00:00 2001 From: elias Date: Wed, 3 Apr 2024 11:44:23 +0200 Subject: [PATCH 29/33] #3302: changed content of 'access way'-infowindow --- .../app/legacy/ordermanagement/configuration/translations-de.js | 2 +- .../app/legacy/ordermanagement/configuration/translations-en.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mdm-frontend/src/app/legacy/ordermanagement/configuration/translations-de.js b/mdm-frontend/src/app/legacy/ordermanagement/configuration/translations-de.js index 3b57a1d20dc..fb168a86abf 100644 --- a/mdm-frontend/src/app/legacy/ordermanagement/configuration/translations-de.js +++ b/mdm-frontend/src/app/legacy/ordermanagement/configuration/translations-de.js @@ -140,7 +140,7 @@ angular.module('metadatamanagementApp').config([ }, 'access-way-info': { 'title': 'Einen Zugangsweg auswählen', - 'content': '

Für unsere Datenpakete gibt es bis zu vier Zugangswege. Diese unterscheiden sich hinsichtlich:

  1. des Nutzungszwecks (Campus Use File (CUF) für Lehre vs. Scientific Use File (SUF) für Forschung),
  2. des Grades der statistischen Anonymisierung (stärker, moderat, geringer) und
  3. des technischen Weges des Datenzugangs (Download, Remote-Desktop, Onsite) verbunden mit unterschiedlich starken technisch-organisatorischen Datensicherheits- und Kontrollmaßnahmen (geringer, moderat, stärker).

Die Nutzung der Daten zum Download bedeutet die geringsten Aufwände im Rahmen der Nutzung und sollte präferiert werden, sofern das Datenpaket dieses Zugangswegs für die Nutzung ausreicht. Sofern diese Daten nicht ausreichen, können ebenfalls die anderen Zugangswege gewählt werden.

Unsere Datenpaketkonfiguration ist gegliedert in:

  • CUF: Download nur für Lehrzwecke; stärkerer statistischer Anonymisierungsgrad; zum Download nach Antragsbewilligung
  • SUF: Download stärkerer statistischer Anonymisierungsgrad; zum Download nach Vertragsabschluss
  • SUF: Remote-Desktop moderater statistischer Anonymisierungsgrad; Zugang nur über virtuelle Desktops per Internet mit technischen Kontrollen nach Vertragsabschluss
  • SUF: On-Site geringer statistischer Anonymisierungsgrad; Zugang nur vor Ort im FDZ-DZHW in Hannover mit technischen Kontrollen nach Vertragsabschluss

Mehr Informationen: https://www.fdz.dzhw.eu/de/datennutzung

' + 'content': '

Für unsere Datenpakete gibt es bis zu vier Zugangswege.

  • CUF: Download nur für Lehrzwecke; stärkerer statistischer Anonymisierungsgrad; zum Download nach Antragsbewilligung
  • SUF: Download stärkerer statistischer Anonymisierungsgrad; zum Download nach Vertragsabschluss
  • SUF: Remote-Desktop moderater statistischer Anonymisierungsgrad; Zugang nur über virtuelle Desktops per Internet mit technischen Kontrollen nach Vertragsabschluss
  • SUF: On-Site geringer statistischer Anonymisierungsgrad; Zugang nur vor Ort im FDZ-DZHW in Hannover mit technischen Kontrollen nach Vertragsabschluss

Mehr Informationen: https://www.fdz.dzhw.eu/de/datennutzung

' } }, 'order-menu': { diff --git a/mdm-frontend/src/app/legacy/ordermanagement/configuration/translations-en.js b/mdm-frontend/src/app/legacy/ordermanagement/configuration/translations-en.js index 4dbe6f75a8e..c5652733fb1 100644 --- a/mdm-frontend/src/app/legacy/ordermanagement/configuration/translations-en.js +++ b/mdm-frontend/src/app/legacy/ordermanagement/configuration/translations-en.js @@ -139,7 +139,7 @@ angular.module('metadatamanagementApp').config([ }, 'access-way-info': { 'title': 'Select an Access Way', - 'content': '

There are up to four access ways for our data packages. These differ in terms of:

  1. the purpose of use (Campus Use File (CUF) for teaching vs. Scientific Use File (SUF) for research),
  2. the degree of statistical anonymisation (stronger, moderate, lower) and
  3. the technical way of data access (Download, Remote-Desktop, Onsite) combined with varying degrees of technical and organisational data security and control measures (lower, moderate, stronger).

The use of data for download means the least effort in the context of use and should be preferred. If the data package of this access way is not sufficient, the other access ways can also be chosen.

Our data package configuration is divided into:

  • CUF: Download for teaching purposes only; higher degree of statistical anonymity; for download after application approval
  • SUF: Download higher degree of statistical anonymity; for download after conclusion of contract
  • SUF: Remote-Desktop moderate statistical anonymisation level; access only via virtual desktops via Internet with technical checks after conclusion of contract
  • SUF: On-Site low degree of statistical anonymity; access only on site at the FDZ-DZHW in Hanover with technical checks after conclusion of contract

More information: https://www.fdz.dzhw.eu/en/data-usage

' + 'content': '

There are up to four access ways for our data packages.

  • CUF: Download for teaching purposes only; higher degree of statistical anonymity; for download after application approval
  • SUF: Download higher degree of statistical anonymity; for download after conclusion of contract
  • SUF: Remote-Desktop moderate statistical anonymisation level; access only via virtual desktops via Internet with technical checks after conclusion of contract
  • SUF: On-Site low degree of statistical anonymity; access only on site at the FDZ-DZHW in Hanover with technical checks after conclusion of contract

More information: https://www.fdz.dzhw.eu/en/data-usage

' } }, 'order-menu': { From 270e24e2fc40123e71fa09e51e6abb9ea29250f5 Mon Sep 17 00:00:00 2001 From: elias Date: Thu, 2 May 2024 11:22:49 +0200 Subject: [PATCH 30/33] #3302: further adjustments to the info text --- .../app/legacy/ordermanagement/configuration/translations-de.js | 2 +- .../app/legacy/ordermanagement/configuration/translations-en.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mdm-frontend/src/app/legacy/ordermanagement/configuration/translations-de.js b/mdm-frontend/src/app/legacy/ordermanagement/configuration/translations-de.js index fb168a86abf..93cf5cee134 100644 --- a/mdm-frontend/src/app/legacy/ordermanagement/configuration/translations-de.js +++ b/mdm-frontend/src/app/legacy/ordermanagement/configuration/translations-de.js @@ -140,7 +140,7 @@ angular.module('metadatamanagementApp').config([ }, 'access-way-info': { 'title': 'Einen Zugangsweg auswählen', - 'content': '

Für unsere Datenpakete gibt es bis zu vier Zugangswege.

  • CUF: Download nur für Lehrzwecke; stärkerer statistischer Anonymisierungsgrad; zum Download nach Antragsbewilligung
  • SUF: Download stärkerer statistischer Anonymisierungsgrad; zum Download nach Vertragsabschluss
  • SUF: Remote-Desktop moderater statistischer Anonymisierungsgrad; Zugang nur über virtuelle Desktops per Internet mit technischen Kontrollen nach Vertragsabschluss
  • SUF: On-Site geringer statistischer Anonymisierungsgrad; Zugang nur vor Ort im FDZ-DZHW in Hannover mit technischen Kontrollen nach Vertragsabschluss

Mehr Informationen: https://www.fdz.dzhw.eu/de/datennutzung

' + 'content': '

Für unsere Datenpakete gibt es bis zu vier Zugangswege.

  • CUF: Download nur für Lehrzwecke; sehr hohe statistischer Anonymisierungsgrad; zum Download nach Antragsbewilligung
  • SUF: Download hoher statistischer Anonymisierungsgrad; zum Download nach Vertragsabschluss
  • SUF: Remote-Desktop Bitte prüfen Sie die erlaubten Nutzungszwecke auf der Datenpaketdetailseite
  • SUF: On-Site geringer statistischer Anonymisierungsgrad; Zugang nur vor Ort im FDZ-DZHW in Hannover mit technischen Kontrollen nach Vertragsabschluss

Mehr Informationen: https://www.fdz.dzhw.eu/de/datennutzung

' } }, 'order-menu': { diff --git a/mdm-frontend/src/app/legacy/ordermanagement/configuration/translations-en.js b/mdm-frontend/src/app/legacy/ordermanagement/configuration/translations-en.js index c5652733fb1..cae8e89e74c 100644 --- a/mdm-frontend/src/app/legacy/ordermanagement/configuration/translations-en.js +++ b/mdm-frontend/src/app/legacy/ordermanagement/configuration/translations-en.js @@ -139,7 +139,7 @@ angular.module('metadatamanagementApp').config([ }, 'access-way-info': { 'title': 'Select an Access Way', - 'content': '

There are up to four access ways for our data packages.

  • CUF: Download for teaching purposes only; higher degree of statistical anonymity; for download after application approval
  • SUF: Download higher degree of statistical anonymity; for download after conclusion of contract
  • SUF: Remote-Desktop moderate statistical anonymisation level; access only via virtual desktops via Internet with technical checks after conclusion of contract
  • SUF: On-Site low degree of statistical anonymity; access only on site at the FDZ-DZHW in Hanover with technical checks after conclusion of contract

More information: https://www.fdz.dzhw.eu/en/data-usage

' + 'content': '

There are up to four access ways for our data packages.

  • CUF: Download for teaching purposes only; very high degree of statistical anonymity; for download after application approval
  • SUF: Download high degree of statistical anonymity; for download after conclusion of contract
  • SUF: Remote-Desktop Please check the approved uses on the data package detail page
  • SUF: On-Site low degree of statistical anonymity; access only on site at the FDZ-DZHW in Hanover with technical checks after conclusion of contract

More information: https://www.fdz.dzhw.eu/en/data-usage

' } }, 'order-menu': { From 14c9ece7c7ce7c87cfe058867671f5b411b8b7a2 Mon Sep 17 00:00:00 2001 From: elias Date: Mon, 8 Jul 2024 09:16:07 +0200 Subject: [PATCH 31/33] #3302: final adjustments on content of info window according to the specifications of Anne Weber (26.06.2024) --- .../app/legacy/ordermanagement/configuration/translations-de.js | 2 +- .../app/legacy/ordermanagement/configuration/translations-en.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mdm-frontend/src/app/legacy/ordermanagement/configuration/translations-de.js b/mdm-frontend/src/app/legacy/ordermanagement/configuration/translations-de.js index 93cf5cee134..7123cccbd29 100644 --- a/mdm-frontend/src/app/legacy/ordermanagement/configuration/translations-de.js +++ b/mdm-frontend/src/app/legacy/ordermanagement/configuration/translations-de.js @@ -140,7 +140,7 @@ angular.module('metadatamanagementApp').config([ }, 'access-way-info': { 'title': 'Einen Zugangsweg auswählen', - 'content': '

Für unsere Datenpakete gibt es bis zu vier Zugangswege.

  • CUF: Download nur für Lehrzwecke; sehr hohe statistischer Anonymisierungsgrad; zum Download nach Antragsbewilligung
  • SUF: Download hoher statistischer Anonymisierungsgrad; zum Download nach Vertragsabschluss
  • SUF: Remote-Desktop Bitte prüfen Sie die erlaubten Nutzungszwecke auf der Datenpaketdetailseite
  • SUF: On-Site geringer statistischer Anonymisierungsgrad; Zugang nur vor Ort im FDZ-DZHW in Hannover mit technischen Kontrollen nach Vertragsabschluss

Mehr Informationen: https://www.fdz.dzhw.eu/de/datennutzung

' + 'content': '

Für unsere Datenpakete gibt es bis zu vier Zugangswege:

  • CUF: Download erstellt für die wissenschaftliche Lehre und Übungszwecke; sehr hoher statistischer Anonymisierungsgrad; nach Antragsbewilligung verfügbar per Download
  • SUF: Download erstellt für wissenschaftliche Forschungszwecke; hoher statistischer Anonymisierungsgrad; nach Vertragsabschluss verfügbar per Download
  • SUF: Remote-Desktop erstellt für wissenschaftliche Forschungszwecke; moderater statistischer Anonymisierungsgrad; nach Vertragsabschluss verfügbar per Internet über virtuellen Desktop mit Input-/Output-Prüfungen
  • SUF: On-Site erstellt für wissenschaftliche Forschungszwecke; geringer statistischer Anonymisierungsgrad; nach Vertragsabschluss verfügbar vor Ort im FDZ-DZHW in Hannover mit Input-/Output-Prüfungen

Mehr Informationen: https://www.fdz.dzhw.eu/de/datennutzung

' } }, 'order-menu': { diff --git a/mdm-frontend/src/app/legacy/ordermanagement/configuration/translations-en.js b/mdm-frontend/src/app/legacy/ordermanagement/configuration/translations-en.js index cae8e89e74c..97a50c41f35 100644 --- a/mdm-frontend/src/app/legacy/ordermanagement/configuration/translations-en.js +++ b/mdm-frontend/src/app/legacy/ordermanagement/configuration/translations-en.js @@ -139,7 +139,7 @@ angular.module('metadatamanagementApp').config([ }, 'access-way-info': { 'title': 'Select an Access Way', - 'content': '

There are up to four access ways for our data packages.

  • CUF: Download for teaching purposes only; very high degree of statistical anonymity; for download after application approval
  • SUF: Download high degree of statistical anonymity; for download after conclusion of contract
  • SUF: Remote-Desktop Please check the approved uses on the data package detail page
  • SUF: On-Site low degree of statistical anonymity; access only on site at the FDZ-DZHW in Hanover with technical checks after conclusion of contract

More information: https://www.fdz.dzhw.eu/en/data-usage

' + 'content': '

There are up to four access ways for our data packages:

  • CUF: Download created for academic teaching and training purposes; very high statistical anonymization level; available for download after application approval
  • SUF: Download created for scientific research purposes; high statistical anonymization level; available for download after conclusion of data use agreement
  • SUF: Remote-Desktop created for scientific research purposes; moderate statistical anonymization level; available via internet using virtual desktop with input/output checks after conclusion of data use agreement
  • SUF: On-Site created for scientific research purposes; low statistical anonymization level; available on site at the FDZ-DZHW in Hanover with input/output checks after conclusion of data use agreement

More information: https://www.fdz.dzhw.eu/en/data-usage

' } }, 'order-menu': { From a745f203587d855e2884569269a027014269d368 Mon Sep 17 00:00:00 2001 From: elias Date: Mon, 8 Jul 2024 11:32:42 +0200 Subject: [PATCH 32/33] #3256: edited default description according to the specifications of Anne Weber (26.06.2024) --- .../configuration/translations-de.js | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/mdm-frontend/src/app/legacy/datasetmanagement/configuration/translations-de.js b/mdm-frontend/src/app/legacy/datasetmanagement/configuration/translations-de.js index 285d59ff90b..5f22207e6a7 100644 --- a/mdm-frontend/src/app/legacy/datasetmanagement/configuration/translations-de.js +++ b/mdm-frontend/src/app/legacy/datasetmanagement/configuration/translations-de.js @@ -219,20 +219,20 @@ angular.module('metadatamanagementApp').config([ 'move-sub-data-set-down-tooltip': 'Klicken, um den ausgewählten Subdatensatz nach oben zu verschieben', 'sub-data-set-default-description': { 'download-cuf': { - 'de': 'Kann nach einer Registrierung ohne Datennutzungsvertrag heruntergeladen werden', - 'en': 'can be downloaded after registration without a data use agreement' + 'de': 'Erstellt für die wissenschaftliche Lehre und Übungszwecke; sehr hoher statistischer Anonymisierungsgrad; nach Antragsbewilligung verfügbar per Download', + 'en': 'created for academic teaching and training purposes; very high statistical anonymization level; available for download after application approval' }, 'download-suf': { - 'de': 'Kann nach Abschluss eines Datennutzungsvertrags heruntergeladen werden', - 'en': 'can be downloaded after the conclusion of a data use agreement' + 'de': 'Erstellt für wissenschaftliche Forschungszwecke; hoher statistischer Anonymisierungsgrad; nach Vertragsabschluss verfügbar per Download', + 'en': 'created for scientific research purposes; high statistical anonymization level; available for download after conclusion of data use agreement' }, 'remote-desktop-suf': { - 'de': 'Kann über Remote Desktop genutzt werden', - 'en': 'can be used via remote desktop' + 'de': 'Erstellt für wissenschaftliche Forschungszwecke; moderater statistischer Anonymisierungsgrad; nach Vertragsabschluss verfügbar per Internet über virtuellen Desktop mit Input-/Output-Prüfungen', + 'en': 'created for scientific research purposes; moderate statistical anonymization level; available via internet using virtual desktop with input/output checks after conclusion of data use agreement' }, 'onsite-suf': { - 'de': 'Ist am Gastwissenschaftler*innenarbeitsplatz im DZHW in Hannover verfügbar', - 'en': 'available at the workspace for visiting scientists at the DZHW in Hannover' + 'de': 'Erstellt für wissenschaftliche Forschungszwecke; geringer statistischer Anonymisierungsgrad; nach Vertragsabschluss verfügbar vor Ort im FDZ-DZHW in Hannover mit Input-/Output-Prüfungen', + 'en': 'created for scientific research purposes; low statistical anonymization level; available on site at the FDZ-DZHW in Hanover with input/output checks after conclusion of data use agreement' } }, 'label': { From 74b68f1d515997aae95d094511c367a12df239cc Mon Sep 17 00:00:00 2001 From: Tilo Villwock Date: Wed, 23 Oct 2024 15:25:44 +0200 Subject: [PATCH 33/33] #3303 Implemented a fallback for missing variable label translations --- .../service/VariableManagementService.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/service/VariableManagementService.java b/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/service/VariableManagementService.java index 3ee40f06f5b..9854f60b286 100644 --- a/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/service/VariableManagementService.java +++ b/src/main/java/eu/dzhw/fdz/metadatamanagement/variablemanagement/service/VariableManagementService.java @@ -346,7 +346,11 @@ private ObjectNode getPidMetadataOfVariable( ObjectNode variableObj = objectMapper.createObjectNode(); variableObj.put("studyDOI", dataPackage.getDoi()); variableObj.put("variableName", variable.getName()); - variableObj.put("variableLabel", variable.getLabel().getEn()); + if (variable.getLabel().getEn() == null || variable.getLabel().getEn().isEmpty()) { + variableObj.put("variableLabel", variable.getLabel().getDe()); + } else { + variableObj.put("variableLabel", variable.getLabel().getEn()); + } variableObj.put("pidProposal", pidPrefix + project.getId() + "_" + variable.getName() + ":" + project.getRelease().getVersion()); variableObj.put("landingPage", landingPageBaseUrl