From 225fbb1c234bcc4e2b5ae7cb5565951794e817a6 Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Tue, 7 May 2024 10:55:57 +0200 Subject: [PATCH 1/7] setup dev branch --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 7f819b9..f33ee0c 100644 --- a/pom.xml +++ b/pom.xml @@ -1,5 +1,5 @@ - 4.0.0 From cb39a424f459d51efb6c1cbfb43e5bdf34021394 Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Fri, 24 May 2024 11:10:29 +0200 Subject: [PATCH 2/7] Add standard dropbox for handling of data scanner output (#1) * add dropbox for handling data scanner output --------- Co-authored-by: Steffengreiner Co-authored-by: Tobias Koch Co-authored-by: Sven F. Co-authored-by: Tobias Koch --- .gitignore | 49 +-- mvnw | 310 ------------------ mvnw.cmd | 182 ---------- pom.xml | 230 ++++++++----- src/main/groovy/.DS_Store | Bin 6148 -> 0 bytes src/main/groovy/life/qbic/.DS_Store | Bin 6148 -> 0 bytes .../springminimaltemplate/AppConfig.groovy | 32 -- .../CodingPrayersMessageService.groovy | 29 -- .../DeveloperNews.groovy | 20 -- .../MessageService.groovy | 18 - .../springminimaltemplate/NewsMedia.groovy | 17 - .../SpringMinimalTemplateApplication.groovy | 23 -- .../openbis/DataSetProvenance.java | 94 ++++++ .../openbis/OpenBisDropboxETL.java | 81 +++++ .../registration/openbis/OpenBisSearch.java | 24 ++ .../openbis/OpenBisSearchImpl.java | 69 ++++ .../openbis/ProvenanceParser.java | 26 ++ .../fail/DatasetTypeMappingException.java | 10 + .../fail/MeasurementHasDataException.java | 10 + .../fail/ProvenanceParseException.java | 10 + .../fail/TooManyMeasurementsException.java | 10 + .../fail/UnknownSampleTypeException.java | 10 + .../retry/NoMeasurementsFoundException.java | 13 + .../openbis/types/QDatasetType.java | 44 +++ .../openbis/types/QPropertyType.java | 42 +++ .../openbis/types/QSampleType.java | 37 +++ src/main/resources/application.properties | 1 - src/main/resources/messages.txt | 5 - ...ringMinimalTemplateApplicationTests.groovy | 26 -- .../openbis/ProvenanceParserTest.groovy | 25 ++ src/test/resources/valid-provenance.json | 14 + 31 files changed, 688 insertions(+), 773 deletions(-) delete mode 100755 mvnw delete mode 100644 mvnw.cmd delete mode 100644 src/main/groovy/.DS_Store delete mode 100644 src/main/groovy/life/qbic/.DS_Store delete mode 100644 src/main/groovy/life/qbic/springminimaltemplate/AppConfig.groovy delete mode 100644 src/main/groovy/life/qbic/springminimaltemplate/CodingPrayersMessageService.groovy delete mode 100644 src/main/groovy/life/qbic/springminimaltemplate/DeveloperNews.groovy delete mode 100644 src/main/groovy/life/qbic/springminimaltemplate/MessageService.groovy delete mode 100644 src/main/groovy/life/qbic/springminimaltemplate/NewsMedia.groovy delete mode 100644 src/main/groovy/life/qbic/springminimaltemplate/SpringMinimalTemplateApplication.groovy create mode 100644 src/main/java/life/qbic/registration/openbis/DataSetProvenance.java create mode 100644 src/main/java/life/qbic/registration/openbis/OpenBisDropboxETL.java create mode 100644 src/main/java/life/qbic/registration/openbis/OpenBisSearch.java create mode 100644 src/main/java/life/qbic/registration/openbis/OpenBisSearchImpl.java create mode 100644 src/main/java/life/qbic/registration/openbis/ProvenanceParser.java create mode 100644 src/main/java/life/qbic/registration/openbis/exceptions/fail/DatasetTypeMappingException.java create mode 100644 src/main/java/life/qbic/registration/openbis/exceptions/fail/MeasurementHasDataException.java create mode 100644 src/main/java/life/qbic/registration/openbis/exceptions/fail/ProvenanceParseException.java create mode 100644 src/main/java/life/qbic/registration/openbis/exceptions/fail/TooManyMeasurementsException.java create mode 100644 src/main/java/life/qbic/registration/openbis/exceptions/fail/UnknownSampleTypeException.java create mode 100644 src/main/java/life/qbic/registration/openbis/exceptions/retry/NoMeasurementsFoundException.java create mode 100644 src/main/java/life/qbic/registration/openbis/types/QDatasetType.java create mode 100644 src/main/java/life/qbic/registration/openbis/types/QPropertyType.java create mode 100644 src/main/java/life/qbic/registration/openbis/types/QSampleType.java delete mode 100644 src/main/resources/application.properties delete mode 100644 src/main/resources/messages.txt delete mode 100644 src/test/groovy/life/qbic/springminimaltemplate/SpringMinimalTemplateApplicationTests.groovy create mode 100644 src/test/java/life/qbic/registration/openbis/ProvenanceParserTest.groovy create mode 100644 src/test/resources/valid-provenance.json diff --git a/.gitignore b/.gitignore index e3b191b..0dc03db 100644 --- a/.gitignore +++ b/.gitignore @@ -1,29 +1,34 @@ -# Compiled class file -*.class +# Eclipse +.classpath +.project +.settings/ -# Log file -*.log +# VS Code +.vscode -# Maven targets -target/* +# Intellij +.idea/ +*.iml +*.iws -# BlueJ files -*.ctxt +# Mac +.DS_Store -# Mobile Tools for Java (J2ME) -.mtj.tmp/ +# Maven +target/ +**/.flattened-pom.xml -# Package Files # -*.jar -*.war -*.nar -*.ear -*.zip -*.tar.gz -*.rar +# Some of our applications generate logs, which should be ignored by git +logs/ -# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml -hs_err_pid* +# Other +**/.factorypath -# misc -*.DS_STORE +# compiled Vaadin widgetsets +**/VAADIN/widgetsets + +# From docs +**/__pycache__ + +# Properties, which may contain sensitive information +**/developer.properties diff --git a/mvnw b/mvnw deleted file mode 100755 index a16b543..0000000 --- a/mvnw +++ /dev/null @@ -1,310 +0,0 @@ -#!/bin/sh -# ---------------------------------------------------------------------------- -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# ---------------------------------------------------------------------------- - -# ---------------------------------------------------------------------------- -# Maven Start Up Batch script -# -# Required ENV vars: -# ------------------ -# JAVA_HOME - location of a JDK home dir -# -# Optional ENV vars -# ----------------- -# M2_HOME - location of maven2's installed home dir -# MAVEN_OPTS - parameters passed to the Java VM when running Maven -# e.g. to debug Maven itself, use -# set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 -# MAVEN_SKIP_RC - flag to disable loading of mavenrc files -# ---------------------------------------------------------------------------- - -if [ -z "$MAVEN_SKIP_RC" ] ; then - - if [ -f /etc/mavenrc ] ; then - . /etc/mavenrc - fi - - if [ -f "$HOME/.mavenrc" ] ; then - . "$HOME/.mavenrc" - fi - -fi - -# OS specific support. $var _must_ be set to either true or false. -cygwin=false; -darwin=false; -mingw=false -case "`uname`" in - CYGWIN*) cygwin=true ;; - MINGW*) mingw=true;; - Darwin*) darwin=true - # Use /usr/libexec/java_home if available, otherwise fall back to /Library/Java/Home - # See https://developer.apple.com/library/mac/qa/qa1170/_index.html - if [ -z "$JAVA_HOME" ]; then - if [ -x "/usr/libexec/java_home" ]; then - export JAVA_HOME="`/usr/libexec/java_home`" - else - export JAVA_HOME="/Library/Java/Home" - fi - fi - ;; -esac - -if [ -z "$JAVA_HOME" ] ; then - if [ -r /etc/gentoo-release ] ; then - JAVA_HOME=`java-config --jre-home` - fi -fi - -if [ -z "$M2_HOME" ] ; then - ## resolve links - $0 may be a link to maven's home - PRG="$0" - - # need this for relative symlinks - while [ -h "$PRG" ] ; do - ls=`ls -ld "$PRG"` - link=`expr "$ls" : '.*-> \(.*\)$'` - if expr "$link" : '/.*' > /dev/null; then - PRG="$link" - else - PRG="`dirname "$PRG"`/$link" - fi - done - - saveddir=`pwd` - - M2_HOME=`dirname "$PRG"`/.. - - # make it fully qualified - M2_HOME=`cd "$M2_HOME" && pwd` - - cd "$saveddir" - # echo Using m2 at $M2_HOME -fi - -# For Cygwin, ensure paths are in UNIX format before anything is touched -if $cygwin ; then - [ -n "$M2_HOME" ] && - M2_HOME=`cygpath --unix "$M2_HOME"` - [ -n "$JAVA_HOME" ] && - JAVA_HOME=`cygpath --unix "$JAVA_HOME"` - [ -n "$CLASSPATH" ] && - CLASSPATH=`cygpath --path --unix "$CLASSPATH"` -fi - -# For Mingw, ensure paths are in UNIX format before anything is touched -if $mingw ; then - [ -n "$M2_HOME" ] && - M2_HOME="`(cd "$M2_HOME"; pwd)`" - [ -n "$JAVA_HOME" ] && - JAVA_HOME="`(cd "$JAVA_HOME"; pwd)`" -fi - -if [ -z "$JAVA_HOME" ]; then - javaExecutable="`which javac`" - if [ -n "$javaExecutable" ] && ! [ "`expr \"$javaExecutable\" : '\([^ ]*\)'`" = "no" ]; then - # readlink(1) is not available as standard on Solaris 10. - readLink=`which readlink` - if [ ! `expr "$readLink" : '\([^ ]*\)'` = "no" ]; then - if $darwin ; then - javaHome="`dirname \"$javaExecutable\"`" - javaExecutable="`cd \"$javaHome\" && pwd -P`/javac" - else - javaExecutable="`readlink -f \"$javaExecutable\"`" - fi - javaHome="`dirname \"$javaExecutable\"`" - javaHome=`expr "$javaHome" : '\(.*\)/bin'` - JAVA_HOME="$javaHome" - export JAVA_HOME - fi - fi -fi - -if [ -z "$JAVACMD" ] ; then - if [ -n "$JAVA_HOME" ] ; then - if [ -x "$JAVA_HOME/jre/sh/java" ] ; then - # IBM's JDK on AIX uses strange locations for the executables - JAVACMD="$JAVA_HOME/jre/sh/java" - else - JAVACMD="$JAVA_HOME/bin/java" - fi - else - JAVACMD="`which java`" - fi -fi - -if [ ! -x "$JAVACMD" ] ; then - echo "Error: JAVA_HOME is not defined correctly." >&2 - echo " We cannot execute $JAVACMD" >&2 - exit 1 -fi - -if [ -z "$JAVA_HOME" ] ; then - echo "Warning: JAVA_HOME environment variable is not set." -fi - -CLASSWORLDS_LAUNCHER=org.codehaus.plexus.classworlds.launcher.Launcher - -# traverses directory structure from process work directory to filesystem root -# first directory with .mvn subdirectory is considered project base directory -find_maven_basedir() { - - if [ -z "$1" ] - then - echo "Path not specified to find_maven_basedir" - return 1 - fi - - basedir="$1" - wdir="$1" - while [ "$wdir" != '/' ] ; do - if [ -d "$wdir"/.mvn ] ; then - basedir=$wdir - break - fi - # workaround for JBEAP-8937 (on Solaris 10/Sparc) - if [ -d "${wdir}" ]; then - wdir=`cd "$wdir/.."; pwd` - fi - # end of workaround - done - echo "${basedir}" -} - -# concatenates all lines of a file -concat_lines() { - if [ -f "$1" ]; then - echo "$(tr -s '\n' ' ' < "$1")" - fi -} - -BASE_DIR=`find_maven_basedir "$(pwd)"` -if [ -z "$BASE_DIR" ]; then - exit 1; -fi - -########################################################################################## -# Extension to allow automatically downloading the maven-wrapper.jar from Maven-central -# This allows using the maven wrapper in projects that prohibit checking in binary data. -########################################################################################## -if [ -r "$BASE_DIR/.mvn/wrapper/maven-wrapper.jar" ]; then - if [ "$MVNW_VERBOSE" = true ]; then - echo "Found .mvn/wrapper/maven-wrapper.jar" - fi -else - if [ "$MVNW_VERBOSE" = true ]; then - echo "Couldn't find .mvn/wrapper/maven-wrapper.jar, downloading it ..." - fi - if [ -n "$MVNW_REPOURL" ]; then - jarUrl="$MVNW_REPOURL/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar" - else - jarUrl="https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar" - fi - while IFS="=" read key value; do - case "$key" in (wrapperUrl) jarUrl="$value"; break ;; - esac - done < "$BASE_DIR/.mvn/wrapper/maven-wrapper.properties" - if [ "$MVNW_VERBOSE" = true ]; then - echo "Downloading from: $jarUrl" - fi - wrapperJarPath="$BASE_DIR/.mvn/wrapper/maven-wrapper.jar" - if $cygwin; then - wrapperJarPath=`cygpath --path --windows "$wrapperJarPath"` - fi - - if command -v wget > /dev/null; then - if [ "$MVNW_VERBOSE" = true ]; then - echo "Found wget ... using wget" - fi - if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then - wget "$jarUrl" -O "$wrapperJarPath" - else - wget --http-user=$MVNW_USERNAME --http-password=$MVNW_PASSWORD "$jarUrl" -O "$wrapperJarPath" - fi - elif command -v curl > /dev/null; then - if [ "$MVNW_VERBOSE" = true ]; then - echo "Found curl ... using curl" - fi - if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then - curl -o "$wrapperJarPath" "$jarUrl" -f - else - curl --user $MVNW_USERNAME:$MVNW_PASSWORD -o "$wrapperJarPath" "$jarUrl" -f - fi - - else - if [ "$MVNW_VERBOSE" = true ]; then - echo "Falling back to using Java to download" - fi - javaClass="$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.java" - # For Cygwin, switch paths to Windows format before running javac - if $cygwin; then - javaClass=`cygpath --path --windows "$javaClass"` - fi - if [ -e "$javaClass" ]; then - if [ ! -e "$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.class" ]; then - if [ "$MVNW_VERBOSE" = true ]; then - echo " - Compiling MavenWrapperDownloader.java ..." - fi - # Compiling the Java class - ("$JAVA_HOME/bin/javac" "$javaClass") - fi - if [ -e "$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.class" ]; then - # Running the downloader - if [ "$MVNW_VERBOSE" = true ]; then - echo " - Running MavenWrapperDownloader.java ..." - fi - ("$JAVA_HOME/bin/java" -cp .mvn/wrapper MavenWrapperDownloader "$MAVEN_PROJECTBASEDIR") - fi - fi - fi -fi -########################################################################################## -# End of extension -########################################################################################## - -export MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-"$BASE_DIR"} -if [ "$MVNW_VERBOSE" = true ]; then - echo $MAVEN_PROJECTBASEDIR -fi -MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS" - -# For Cygwin, switch paths to Windows format before running java -if $cygwin; then - [ -n "$M2_HOME" ] && - M2_HOME=`cygpath --path --windows "$M2_HOME"` - [ -n "$JAVA_HOME" ] && - JAVA_HOME=`cygpath --path --windows "$JAVA_HOME"` - [ -n "$CLASSPATH" ] && - CLASSPATH=`cygpath --path --windows "$CLASSPATH"` - [ -n "$MAVEN_PROJECTBASEDIR" ] && - MAVEN_PROJECTBASEDIR=`cygpath --path --windows "$MAVEN_PROJECTBASEDIR"` -fi - -# Provide a "standardized" way to retrieve the CLI args that will -# work with both Windows and non-Windows executions. -MAVEN_CMD_LINE_ARGS="$MAVEN_CONFIG $@" -export MAVEN_CMD_LINE_ARGS - -WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain - -exec "$JAVACMD" \ - $MAVEN_OPTS \ - -classpath "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" \ - "-Dmaven.home=${M2_HOME}" "-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \ - ${WRAPPER_LAUNCHER} $MAVEN_CONFIG "$@" diff --git a/mvnw.cmd b/mvnw.cmd deleted file mode 100644 index c8d4337..0000000 --- a/mvnw.cmd +++ /dev/null @@ -1,182 +0,0 @@ -@REM ---------------------------------------------------------------------------- -@REM Licensed to the Apache Software Foundation (ASF) under one -@REM or more contributor license agreements. See the NOTICE file -@REM distributed with this work for additional information -@REM regarding copyright ownership. The ASF licenses this file -@REM to you under the Apache License, Version 2.0 (the -@REM "License"); you may not use this file except in compliance -@REM with the License. You may obtain a copy of the License at -@REM -@REM https://www.apache.org/licenses/LICENSE-2.0 -@REM -@REM Unless required by applicable law or agreed to in writing, -@REM software distributed under the License is distributed on an -@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -@REM KIND, either express or implied. See the License for the -@REM specific language governing permissions and limitations -@REM under the License. -@REM ---------------------------------------------------------------------------- - -@REM ---------------------------------------------------------------------------- -@REM Maven Start Up Batch script -@REM -@REM Required ENV vars: -@REM JAVA_HOME - location of a JDK home dir -@REM -@REM Optional ENV vars -@REM M2_HOME - location of maven2's installed home dir -@REM MAVEN_BATCH_ECHO - set to 'on' to enable the echoing of the batch commands -@REM MAVEN_BATCH_PAUSE - set to 'on' to wait for a keystroke before ending -@REM MAVEN_OPTS - parameters passed to the Java VM when running Maven -@REM e.g. to debug Maven itself, use -@REM set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 -@REM MAVEN_SKIP_RC - flag to disable loading of mavenrc files -@REM ---------------------------------------------------------------------------- - -@REM Begin all REM lines with '@' in case MAVEN_BATCH_ECHO is 'on' -@echo off -@REM set title of command window -title %0 -@REM enable echoing by setting MAVEN_BATCH_ECHO to 'on' -@if "%MAVEN_BATCH_ECHO%" == "on" echo %MAVEN_BATCH_ECHO% - -@REM set %HOME% to equivalent of $HOME -if "%HOME%" == "" (set "HOME=%HOMEDRIVE%%HOMEPATH%") - -@REM Execute a user defined script before this one -if not "%MAVEN_SKIP_RC%" == "" goto skipRcPre -@REM check for pre script, once with legacy .bat ending and once with .cmd ending -if exist "%HOME%\mavenrc_pre.bat" call "%HOME%\mavenrc_pre.bat" -if exist "%HOME%\mavenrc_pre.cmd" call "%HOME%\mavenrc_pre.cmd" -:skipRcPre - -@setlocal - -set ERROR_CODE=0 - -@REM To isolate internal variables from possible post scripts, we use another setlocal -@setlocal - -@REM ==== START VALIDATION ==== -if not "%JAVA_HOME%" == "" goto OkJHome - -echo. -echo Error: JAVA_HOME not found in your environment. >&2 -echo Please set the JAVA_HOME variable in your environment to match the >&2 -echo location of your Java installation. >&2 -echo. -goto error - -:OkJHome -if exist "%JAVA_HOME%\bin\java.exe" goto init - -echo. -echo Error: JAVA_HOME is set to an invalid directory. >&2 -echo JAVA_HOME = "%JAVA_HOME%" >&2 -echo Please set the JAVA_HOME variable in your environment to match the >&2 -echo location of your Java installation. >&2 -echo. -goto error - -@REM ==== END VALIDATION ==== - -:init - -@REM Find the project base dir, i.e. the directory that contains the folder ".mvn". -@REM Fallback to current working directory if not found. - -set MAVEN_PROJECTBASEDIR=%MAVEN_BASEDIR% -IF NOT "%MAVEN_PROJECTBASEDIR%"=="" goto endDetectBaseDir - -set EXEC_DIR=%CD% -set WDIR=%EXEC_DIR% -:findBaseDir -IF EXIST "%WDIR%"\.mvn goto baseDirFound -cd .. -IF "%WDIR%"=="%CD%" goto baseDirNotFound -set WDIR=%CD% -goto findBaseDir - -:baseDirFound -set MAVEN_PROJECTBASEDIR=%WDIR% -cd "%EXEC_DIR%" -goto endDetectBaseDir - -:baseDirNotFound -set MAVEN_PROJECTBASEDIR=%EXEC_DIR% -cd "%EXEC_DIR%" - -:endDetectBaseDir - -IF NOT EXIST "%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config" goto endReadAdditionalConfig - -@setlocal EnableExtensions EnableDelayedExpansion -for /F "usebackq delims=" %%a in ("%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config") do set JVM_CONFIG_MAVEN_PROPS=!JVM_CONFIG_MAVEN_PROPS! %%a -@endlocal & set JVM_CONFIG_MAVEN_PROPS=%JVM_CONFIG_MAVEN_PROPS% - -:endReadAdditionalConfig - -SET MAVEN_JAVA_EXE="%JAVA_HOME%\bin\java.exe" -set WRAPPER_JAR="%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.jar" -set WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain - -set DOWNLOAD_URL="https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar" - -FOR /F "tokens=1,2 delims==" %%A IN ("%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.properties") DO ( - IF "%%A"=="wrapperUrl" SET DOWNLOAD_URL=%%B -) - -@REM Extension to allow automatically downloading the maven-wrapper.jar from Maven-central -@REM This allows using the maven wrapper in projects that prohibit checking in binary data. -if exist %WRAPPER_JAR% ( - if "%MVNW_VERBOSE%" == "true" ( - echo Found %WRAPPER_JAR% - ) -) else ( - if not "%MVNW_REPOURL%" == "" ( - SET DOWNLOAD_URL="%MVNW_REPOURL%/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar" - ) - if "%MVNW_VERBOSE%" == "true" ( - echo Couldn't find %WRAPPER_JAR%, downloading it ... - echo Downloading from: %DOWNLOAD_URL% - ) - - powershell -Command "&{"^ - "$webclient = new-object System.Net.WebClient;"^ - "if (-not ([string]::IsNullOrEmpty('%MVNW_USERNAME%') -and [string]::IsNullOrEmpty('%MVNW_PASSWORD%'))) {"^ - "$webclient.Credentials = new-object System.Net.NetworkCredential('%MVNW_USERNAME%', '%MVNW_PASSWORD%');"^ - "}"^ - "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; $webclient.DownloadFile('%DOWNLOAD_URL%', '%WRAPPER_JAR%')"^ - "}" - if "%MVNW_VERBOSE%" == "true" ( - echo Finished downloading %WRAPPER_JAR% - ) -) -@REM End of extension - -@REM Provide a "standardized" way to retrieve the CLI args that will -@REM work with both Windows and non-Windows executions. -set MAVEN_CMD_LINE_ARGS=%* - -%MAVEN_JAVA_EXE% %JVM_CONFIG_MAVEN_PROPS% %MAVEN_OPTS% %MAVEN_DEBUG_OPTS% -classpath %WRAPPER_JAR% "-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%" %WRAPPER_LAUNCHER% %MAVEN_CONFIG% %* -if ERRORLEVEL 1 goto error -goto end - -:error -set ERROR_CODE=1 - -:end -@endlocal & set ERROR_CODE=%ERROR_CODE% - -if not "%MAVEN_SKIP_RC%" == "" goto skipRcPost -@REM check for post script, once with legacy .bat ending and once with .cmd ending -if exist "%HOME%\mavenrc_post.bat" call "%HOME%\mavenrc_post.bat" -if exist "%HOME%\mavenrc_post.cmd" call "%HOME%\mavenrc_post.cmd" -:skipRcPost - -@REM pause the script if MAVEN_BATCH_PAUSE is set to 'on' -if "%MAVEN_BATCH_PAUSE%" == "on" pause - -if "%MAVEN_TERMINATE_CMD%" == "on" exit %ERROR_CODE% - -exit /B %ERROR_CODE% diff --git a/pom.xml b/pom.xml index f33ee0c..7717d9b 100644 --- a/pom.xml +++ b/pom.xml @@ -1,97 +1,151 @@ - - 4.0.0 - - org.springframework.boot - spring-boot-starter-parent - 2.5.6 - - - life.qbic - spring-minimal-template - 1.0.2 - spring-minimal-template - Demo project for Spring Boot - - 17 - 3.0.8 - - - - org.springframework.boot - spring-boot-starter - - - org.codehaus.groovy - groovy - + + 4.0.0 - - org.springframework.boot - spring-boot-starter-test - test - + OpenBIS ETL routines written in Java + https://github.com/qbicsoftware/openbis20-java-dropboxes + ETL routines written in Java for OpenBIS version 20 - - org.spockframework - spock-core - 2.0-groovy-3.0 - test - + life.qbic.registration + java-openbis20-dropboxes + 1.0.0 + jar - - org.spockframework - spock-spring - 2.0-groovy-3.0 - test - - + + UTF-8 + 11 + 11 + 20.10.7.3 + r1700646105 + - + + + central + https://repo.maven.apache.org/maven2 + + false + + + + nexus-snapshots + QBiC Snapshots + + false + + + fail + true + always + + https://qbic-repo.qbic.uni-tuebingen.de/repository/maven-snapshots + + + nexus-releases + QBiC Releases + + fail + true + always + + + false + + https://qbic-repo.qbic.uni-tuebingen.de/repository/maven-releases + + + + + life.qbic + openbis-api + ${openbis.version} + ${openbis.classifier} + + + life.qbic + openbis-api-dropbox + ${openbis.version} + ${openbis.classifier} + + + life.qbic + openbis-api-commonbase + ${openbis.version} + ${openbis.classifier} + + + + life.qbic + openbis-dss + 18.06.2 + provided + + + life.qbic + openbis-api-etl + 18.06.2 + provided + + + + life.qbic + hotdeploy-plugins + 13.01.0 + + + org.spockframework + spock-core + 2.4-M1-groovy-4.0 + test + + + + - - - true - nexus-releases - QBiC Releases - https://qbic-repo.qbic.uni-tuebingen.de/repository/maven-releases - - - false - nexus-snapshots - QBiC Snapshots - https://qbic-repo.qbic.uni-tuebingen.de/repository/maven-snapshots - - - - - - - org.springframework.boot - spring-boot-maven-plugin - - - org.codehaus.gmavenplus - gmavenplus-plugin - 1.13.0 - - - - addSources - addTestSources - generateStubs - compile - generateTestStubs - compileTests - removeStubs - removeTestStubs - - - - - - + + + true + nexus-releases + QBiC Releases + https://qbic-repo.qbic.uni-tuebingen.de/repository/maven-releases + + + false + nexus-snapshots + QBiC Snapshots + https://qbic-repo.qbic.uni-tuebingen.de/repository/maven-snapshots + + + + + + maven-assembly-plugin + + + + + + + jar-with-dependencies + + + + + make-assembly + package + + single + + + + + + diff --git a/src/main/groovy/.DS_Store b/src/main/groovy/.DS_Store deleted file mode 100644 index f94f0281fb9b43b076bd9f9eb9a68ba8543a379c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeH~O=`nH427Q>F9O|ondNMHfZkvT=?Qv)ke@=pkVVOUj-DrtHgUTSCOm=kjWiav z-@;=7uNbSFMM%*>b%m~g=l&Zpnwbhut8FVfyR;3<8?Y(KXJDIf);fE17d zQeZ&}+P_@gl09MAO)nrOo3I;M?e3+=>N_CvlgXN zKnlE=0ygZvcU!(xo~?gg&+GfF`nu7{xSZkPCxD3`#T$AU_lqycnrxk{(DWk^GAKxa Hrz&s<%q|k8 diff --git a/src/main/groovy/life/qbic/.DS_Store b/src/main/groovy/life/qbic/.DS_Store deleted file mode 100644 index 73c441a7f0ca095ed89d5aad8f063030dfe7352a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHKOG*P#5UkcL0&cPqT)r!KgCWEdcmbmk2~12dx}R0fm7`hx0STjS`(O43dWNP&w|z$S~w#hkBHy>;|*-fJ8EiS9K= rx*Ox5aEW$IjCRb8x8u7g%DUz;&%44QG3d+(ov5Dy*F`1;{#$`>9%mX( diff --git a/src/main/groovy/life/qbic/springminimaltemplate/AppConfig.groovy b/src/main/groovy/life/qbic/springminimaltemplate/AppConfig.groovy deleted file mode 100644 index 70c3187..0000000 --- a/src/main/groovy/life/qbic/springminimaltemplate/AppConfig.groovy +++ /dev/null @@ -1,32 +0,0 @@ -package life.qbic.springminimaltemplate - -import org.springframework.beans.factory.annotation.Value -import org.springframework.context.annotation.Bean -import org.springframework.context.annotation.Configuration -import org.springframework.context.annotation.PropertySource - -/** - * Spring configuration class - * - *

Reads properties from a properties file and creates beans for the application.

- * - * @since 0.1.0 - */ -@Configuration -@PropertySource("application.properties") -class AppConfig { - - @Value('${messages.file}') - public String messagesFile - - @Bean - MessageService messageService() { - return new CodingPrayersMessageService(messagesFile) - } - - @Bean - NewsMedia newsMedia() { - return new DeveloperNews(messageService()) - } - -} diff --git a/src/main/groovy/life/qbic/springminimaltemplate/CodingPrayersMessageService.groovy b/src/main/groovy/life/qbic/springminimaltemplate/CodingPrayersMessageService.groovy deleted file mode 100644 index 54c29f1..0000000 --- a/src/main/groovy/life/qbic/springminimaltemplate/CodingPrayersMessageService.groovy +++ /dev/null @@ -1,29 +0,0 @@ -package life.qbic.springminimaltemplate - -/** - * Example implementation of a {@link MessageService} - * - * @since 0.1.0 - */ -class CodingPrayersMessageService implements MessageService { - - private List messages - - CodingPrayersMessageService() { - this.messages = new ArrayList<>() - } - - CodingPrayersMessageService(String filePath) { - this.messages = readMessagesFromClassPath(filePath) - } - - @Override - String collectMessage() { - return messages.get(new Random().nextInt(messages.size())) - } - - private List readMessagesFromClassPath(String path) { - URL url = getClass().getClassLoader().getResource(path) - return url.readLines().each { it.trim() }.collect() - } -} diff --git a/src/main/groovy/life/qbic/springminimaltemplate/DeveloperNews.groovy b/src/main/groovy/life/qbic/springminimaltemplate/DeveloperNews.groovy deleted file mode 100644 index 04ed5d1..0000000 --- a/src/main/groovy/life/qbic/springminimaltemplate/DeveloperNews.groovy +++ /dev/null @@ -1,20 +0,0 @@ -package life.qbic.springminimaltemplate - -/** - * An example {@link NewsMedia} implementation for developer news. - * - * @since 0.1.0 - */ -class DeveloperNews implements NewsMedia { - - private MessageService service - - DeveloperNews(MessageService service) { - this.service = service - } - - @Override - String getNews() { - return service.collectMessage() - } -} diff --git a/src/main/groovy/life/qbic/springminimaltemplate/MessageService.groovy b/src/main/groovy/life/qbic/springminimaltemplate/MessageService.groovy deleted file mode 100644 index eaadf3e..0000000 --- a/src/main/groovy/life/qbic/springminimaltemplate/MessageService.groovy +++ /dev/null @@ -1,18 +0,0 @@ -package life.qbic.springminimaltemplate - -/** - * Small toy interface that represents message services - * - *

Message services shall provide access to received messages.

- * - * @since 0.1.0 - */ -interface MessageService { - - /** - * Collects the latest message - * @return the latest message - * @since 0.1.0 - */ - String collectMessage() -} diff --git a/src/main/groovy/life/qbic/springminimaltemplate/NewsMedia.groovy b/src/main/groovy/life/qbic/springminimaltemplate/NewsMedia.groovy deleted file mode 100644 index d335196..0000000 --- a/src/main/groovy/life/qbic/springminimaltemplate/NewsMedia.groovy +++ /dev/null @@ -1,17 +0,0 @@ -package life.qbic.springminimaltemplate - -/** - * Example interface for a news media - * - * @since 0.1.0 - */ -interface NewsMedia { - - /** - * Returns latest news - * @return news stuff! - * @since 0.1.0 - */ - String getNews() - -} \ No newline at end of file diff --git a/src/main/groovy/life/qbic/springminimaltemplate/SpringMinimalTemplateApplication.groovy b/src/main/groovy/life/qbic/springminimaltemplate/SpringMinimalTemplateApplication.groovy deleted file mode 100644 index e9ba274..0000000 --- a/src/main/groovy/life/qbic/springminimaltemplate/SpringMinimalTemplateApplication.groovy +++ /dev/null @@ -1,23 +0,0 @@ -package life.qbic.springminimaltemplate - -import org.springframework.boot.SpringApplication -import org.springframework.boot.autoconfigure.SpringBootApplication -import org.springframework.context.annotation.AnnotationConfigApplicationContext - -@SpringBootApplication -class SpringMinimalTemplateApplication { - - static void main(String[] args) { - SpringApplication.run(SpringMinimalTemplateApplication, args) - - AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(AppConfig.class) - - NewsMedia media = context.getBean("newsMedia", NewsMedia.class) - println "####################### Message of the day ##################" - println media.getNews() - println "##############################################################" - - context.close() - } - -} diff --git a/src/main/java/life/qbic/registration/openbis/DataSetProvenance.java b/src/main/java/life/qbic/registration/openbis/DataSetProvenance.java new file mode 100644 index 0000000..aa82b8b --- /dev/null +++ b/src/main/java/life/qbic/registration/openbis/DataSetProvenance.java @@ -0,0 +1,94 @@ +package life.qbic.registration.openbis; + +import com.fasterxml.jackson.annotation.JsonProperty; +import java.io.Serializable; +import java.util.Arrays; +import java.util.Objects; + +/** + * Class holding provenance information provided by the user. This is a record of immutable data. + * + * @since 1.0.0 + */ +public class DataSetProvenance implements Serializable { + + private static final long serialVersionUID = -1597156104025439195L; + + @JsonProperty("origin") + private String origin; + @JsonProperty("user") + private String user; + @JsonProperty("measurementId") + private String measurementId; + @JsonProperty("datasetFiles") + private String[] datasetFiles; + @JsonProperty("taskId") + private String taskId; + @JsonProperty("history") + private String[] history; + + + private DataSetProvenance() { + + } + public DataSetProvenance(String origin, String user, String measurementId, String[] datasetFiles, + String taskId, String[] history) { + this.origin = origin; + this.user = user; + this.measurementId = measurementId; + this.datasetFiles = datasetFiles; + this.taskId = taskId; + this.history = history; + } + + public String origin() { + return origin; + } + + public String user() { + return user; + } + + public String measurementId() { + return measurementId; + } + + public String[] datasetFiles() { + return datasetFiles; + } + + public String taskId() { + return taskId; + } + + public String[] history() { + return history; + } + + @Override + public final boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof DataSetProvenance)) { + return false; + } + + DataSetProvenance that = (DataSetProvenance) o; + return Objects.equals(origin, that.origin) && Objects.equals(user, that.user) + && Objects.equals(measurementId, that.measurementId) && Arrays.equals( + datasetFiles, that.datasetFiles) && Objects.equals(taskId, that.taskId) + && Arrays.equals(history, that.history); + } + + @Override + public int hashCode() { + int result = Objects.hashCode(origin); + result = 31 * result + Objects.hashCode(user); + result = 31 * result + Objects.hashCode(measurementId); + result = 31 * result + Arrays.hashCode(datasetFiles); + result = 31 * result + Objects.hashCode(taskId); + result = 31 * result + Arrays.hashCode(history); + return result; + } +} diff --git a/src/main/java/life/qbic/registration/openbis/OpenBisDropboxETL.java b/src/main/java/life/qbic/registration/openbis/OpenBisDropboxETL.java new file mode 100644 index 0000000..be29f86 --- /dev/null +++ b/src/main/java/life/qbic/registration/openbis/OpenBisDropboxETL.java @@ -0,0 +1,81 @@ +package life.qbic.registration.openbis; + +import ch.systemsx.cisd.common.exceptions.NotImplementedException; +import ch.systemsx.cisd.etlserver.registrator.DataSetRegistrationContext; +import ch.systemsx.cisd.etlserver.registrator.api.v2.AbstractJavaDataSetRegistrationDropboxV2; +import ch.systemsx.cisd.etlserver.registrator.api.v2.IDataSet; +import ch.systemsx.cisd.etlserver.registrator.api.v2.IDataSetRegistrationTransactionV2; +import ch.systemsx.cisd.openbis.dss.generic.shared.api.internal.v2.ISampleImmutable; +import java.io.File; +import life.qbic.registration.openbis.exceptions.fail.MeasurementHasDataException; +import life.qbic.registration.openbis.exceptions.fail.UnknownSampleTypeException; +import life.qbic.registration.openbis.types.QDatasetType; +import life.qbic.registration.openbis.types.QPropertyType; +import life.qbic.registration.openbis.types.QSampleType; + +/** + * The Dropbox ETL process. + *

+ * This process is responsible for + *

    + *
  1. fetching the measurement sample from openbis + *
  2. creating a data set linked to the sample + *
  3. moving files into the dataset + *
+ * Some constraints are taken care of during this process. These constraints being: + *
    + *
  • the measurement sample already exists in openbis + *
  • only one measurement sample exists with the provided identifier + *
  • the measurement sample has no data set linked as one measurement can only have one dataset. + *
+ * + * @version 1.0.0 + */ +public class OpenBisDropboxETL extends AbstractJavaDataSetRegistrationDropboxV2 { + + private static final String PROVENANCE_FILE_NAME = "provenance.json"; + + public OpenBisDropboxETL(){ + } + + + public interface WithRetryOption { + } + + @Override + public boolean shouldRetryProcessing(DataSetRegistrationContext context, Exception problem) + throws NotImplementedException { + return problem instanceof WithRetryOption || super.shouldRetryProcessing(context, problem); + } + + @Override + public void process(IDataSetRegistrationTransactionV2 transaction) { + DataSetProvenance dataSetProvenance = ProvenanceParser.parseProvenanceJson( + new File(transaction.getIncoming(), PROVENANCE_FILE_NAME)); + + String measurementId = dataSetProvenance.measurementId(); + + OpenBisSearchImpl openBisSearch = new OpenBisSearchImpl(transaction.getSearchService()); + + ISampleImmutable measurementSample = openBisSearch.getMeasurementSample(measurementId); + + if (openBisSearch.doesMeasurementHaveData(measurementSample)) { + throw new MeasurementHasDataException("Measurement " + measurementId + " has data attached."); + } + IDataSet newDataSet = transaction.createNewDataSet(); + newDataSet.setSample(measurementSample); + newDataSet.setPropertyValue(QPropertyType.Q_SUBMITTER.getOpenBisPropertyName(), dataSetProvenance.user()); + newDataSet.setPropertyValue(QPropertyType.Q_TASK_ID.getOpenBisPropertyName(), dataSetProvenance.taskId()); + QDatasetType qDatasetType = getDatasetType(measurementSample); + newDataSet.setDataSetType(qDatasetType.name()); + + transaction.moveFile(transaction.getIncoming().getAbsolutePath(), newDataSet); + } + + private static QDatasetType getDatasetType(ISampleImmutable measurementSample) { + QSampleType qSampleType = QSampleType.lookup(measurementSample.getSampleType()) + .orElseThrow(() -> new UnknownSampleTypeException( + "Unknown sample type: " + measurementSample.getSampleType())); + return QDatasetType.fromQSampleType(qSampleType); + } +} diff --git a/src/main/java/life/qbic/registration/openbis/OpenBisSearch.java b/src/main/java/life/qbic/registration/openbis/OpenBisSearch.java new file mode 100644 index 0000000..93dd6da --- /dev/null +++ b/src/main/java/life/qbic/registration/openbis/OpenBisSearch.java @@ -0,0 +1,24 @@ +package life.qbic.registration.openbis; + +import ch.systemsx.cisd.openbis.dss.generic.shared.api.internal.v2.ISampleImmutable; + +/** + * Interface that allows for searching openbis infrastructure + */ +public interface OpenBisSearch { + + /** + * Searches for a sample with the measurement code in openbis. Assumes that there is one sample and only one; Fails otherwise. + * @param measurementId the identifier of the measurement + * @return the existing measurement + */ + ISampleImmutable getMeasurementSample(String measurementId); + + /** + * Checks for datasets attached to a measurement sample + * @param measurementSample + * @return true if a dataset is contained in the measurement; false otherwise. + */ + boolean doesMeasurementHaveData(ISampleImmutable measurementSample); + +} diff --git a/src/main/java/life/qbic/registration/openbis/OpenBisSearchImpl.java b/src/main/java/life/qbic/registration/openbis/OpenBisSearchImpl.java new file mode 100644 index 0000000..ad95d63 --- /dev/null +++ b/src/main/java/life/qbic/registration/openbis/OpenBisSearchImpl.java @@ -0,0 +1,69 @@ +package life.qbic.registration.openbis; + +import ch.systemsx.cisd.openbis.dss.generic.shared.api.internal.v2.IDataSetImmutable; +import ch.systemsx.cisd.openbis.dss.generic.shared.api.internal.v2.ISampleImmutable; +import ch.systemsx.cisd.openbis.dss.generic.shared.api.internal.v2.ISearchService; +import ch.systemsx.cisd.openbis.generic.shared.api.v1.dto.SearchCriteria; +import ch.systemsx.cisd.openbis.generic.shared.api.v1.dto.SearchCriteria.MatchClause; +import ch.systemsx.cisd.openbis.generic.shared.api.v1.dto.SearchCriteria.MatchClauseAttribute; +import ch.systemsx.cisd.openbis.generic.shared.api.v1.dto.SearchSubCriteria; +import java.util.List; +import life.qbic.registration.openbis.exceptions.fail.TooManyMeasurementsException; +import life.qbic.registration.openbis.exceptions.retry.NoMeasurementsFoundException; + +/** + * Implements the Openbis access + */ +public class OpenBisSearchImpl implements OpenBisSearch { + + private final ISearchService searchService; + + public OpenBisSearchImpl(ISearchService searchService) { + this.searchService = searchService; + } + + @Override + public ISampleImmutable getMeasurementSample(String measurementId) { + return findMeasurementSample(measurementId, searchService); + } + + private static ISampleImmutable findMeasurementSample(String measurementId, + ISearchService searchService) { + SearchCriteria measurementSearchCriteria = new SearchCriteria(); + measurementSearchCriteria.addMatchClause( + MatchClause.createAttributeMatch(MatchClauseAttribute.CODE, measurementId)); + List immutableSamples = searchService.searchForSamples( + measurementSearchCriteria); + + if (immutableSamples.isEmpty()) { + //measurement not found + throw new NoMeasurementsFoundException("Measurement '" + measurementId + "' not found"); + } + if (immutableSamples.size() > 1) { + throw new TooManyMeasurementsException( + "Multiple measurement with id '" + measurementId + "' found"); + } + return immutableSamples.get(0); + } + + @Override + public boolean doesMeasurementHaveData(ISampleImmutable sample) { + return doesMeasurementHaveData(sample, searchService); + } + + private static boolean doesMeasurementHaveData(ISampleImmutable sample, + ISearchService searchService) { + SearchCriteria parentSampleSearchCriteria = new SearchCriteria(); + parentSampleSearchCriteria.addMatchClause( + MatchClause.createAttributeMatch(MatchClauseAttribute.PERM_ID, sample.getPermId())); + + SearchCriteria dataSetSearchCriteria = new SearchCriteria(); + dataSetSearchCriteria.addSubCriteria( + SearchSubCriteria.createDataSetContainerCriteria(parentSampleSearchCriteria)); + List existingDataSets = searchService.searchForDataSets( + dataSetSearchCriteria); + return !existingDataSets.isEmpty(); + } + + +} diff --git a/src/main/java/life/qbic/registration/openbis/ProvenanceParser.java b/src/main/java/life/qbic/registration/openbis/ProvenanceParser.java new file mode 100644 index 0000000..b46e69e --- /dev/null +++ b/src/main/java/life/qbic/registration/openbis/ProvenanceParser.java @@ -0,0 +1,26 @@ +package life.qbic.registration.openbis; + +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.File; +import java.io.IOException; +import life.qbic.registration.openbis.exceptions.fail.ProvenanceParseException; + +/** + * Parses a provenance file and produces a java object from it. + */ +public class ProvenanceParser { + + + private static final Class DATA_SET_PROVENANCE_CLASS = DataSetProvenance.class; + + static DataSetProvenance parseProvenanceJson(File provenanceFile) { + try { + return new ObjectMapper().readValue(provenanceFile, + DATA_SET_PROVENANCE_CLASS); + } catch (IOException e) { + throw new ProvenanceParseException( + "Could not parse '" + provenanceFile.getAbsolutePath() + "'", e); + } + } + +} diff --git a/src/main/java/life/qbic/registration/openbis/exceptions/fail/DatasetTypeMappingException.java b/src/main/java/life/qbic/registration/openbis/exceptions/fail/DatasetTypeMappingException.java new file mode 100644 index 0000000..675debb --- /dev/null +++ b/src/main/java/life/qbic/registration/openbis/exceptions/fail/DatasetTypeMappingException.java @@ -0,0 +1,10 @@ +package life.qbic.registration.openbis.exceptions.fail; + +/** + * Thrown when the dataset type cannot be determined. + */ +public class DatasetTypeMappingException extends RuntimeException { + public DatasetTypeMappingException(String message) { + super(message); + } +} diff --git a/src/main/java/life/qbic/registration/openbis/exceptions/fail/MeasurementHasDataException.java b/src/main/java/life/qbic/registration/openbis/exceptions/fail/MeasurementHasDataException.java new file mode 100644 index 0000000..cf8c7c9 --- /dev/null +++ b/src/main/java/life/qbic/registration/openbis/exceptions/fail/MeasurementHasDataException.java @@ -0,0 +1,10 @@ +package life.qbic.registration.openbis.exceptions.fail; + +/** + * Thrown when a measurement has data already attached and this breaks an assumption. + */ +public class MeasurementHasDataException extends RuntimeException { + public MeasurementHasDataException(String message) { + super(message); + } +} diff --git a/src/main/java/life/qbic/registration/openbis/exceptions/fail/ProvenanceParseException.java b/src/main/java/life/qbic/registration/openbis/exceptions/fail/ProvenanceParseException.java new file mode 100644 index 0000000..8c9cb86 --- /dev/null +++ b/src/main/java/life/qbic/registration/openbis/exceptions/fail/ProvenanceParseException.java @@ -0,0 +1,10 @@ +package life.qbic.registration.openbis.exceptions.fail; + +/** + * Thrown when the provenance file does not match the expected format. + */ +public class ProvenanceParseException extends RuntimeException { + public ProvenanceParseException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/src/main/java/life/qbic/registration/openbis/exceptions/fail/TooManyMeasurementsException.java b/src/main/java/life/qbic/registration/openbis/exceptions/fail/TooManyMeasurementsException.java new file mode 100644 index 0000000..bd4595b --- /dev/null +++ b/src/main/java/life/qbic/registration/openbis/exceptions/fail/TooManyMeasurementsException.java @@ -0,0 +1,10 @@ +package life.qbic.registration.openbis.exceptions.fail; + +/** + * Thrown when multiple measurements exist where only one is expected. + */ +public class TooManyMeasurementsException extends RuntimeException { + public TooManyMeasurementsException(String message) { + super(message); + } +} diff --git a/src/main/java/life/qbic/registration/openbis/exceptions/fail/UnknownSampleTypeException.java b/src/main/java/life/qbic/registration/openbis/exceptions/fail/UnknownSampleTypeException.java new file mode 100644 index 0000000..d0d2414 --- /dev/null +++ b/src/main/java/life/qbic/registration/openbis/exceptions/fail/UnknownSampleTypeException.java @@ -0,0 +1,10 @@ +package life.qbic.registration.openbis.exceptions.fail; + +/** + * Thrown when the ETL is not able to parse the sample type or no sample type was provided. + */ +public class UnknownSampleTypeException extends RuntimeException { + public UnknownSampleTypeException(String message) { + super(message); + } +} diff --git a/src/main/java/life/qbic/registration/openbis/exceptions/retry/NoMeasurementsFoundException.java b/src/main/java/life/qbic/registration/openbis/exceptions/retry/NoMeasurementsFoundException.java new file mode 100644 index 0000000..ff9af6b --- /dev/null +++ b/src/main/java/life/qbic/registration/openbis/exceptions/retry/NoMeasurementsFoundException.java @@ -0,0 +1,13 @@ +package life.qbic.registration.openbis.exceptions.retry; + +import life.qbic.registration.openbis.OpenBisDropboxETL.WithRetryOption; + +/** + * Thrown whenever a measurement was not found even if it is assumed to be there. + */ +public class NoMeasurementsFoundException extends RuntimeException implements + WithRetryOption { + public NoMeasurementsFoundException(String message) { + super(message); + } +} diff --git a/src/main/java/life/qbic/registration/openbis/types/QDatasetType.java b/src/main/java/life/qbic/registration/openbis/types/QDatasetType.java new file mode 100644 index 0000000..b2b5c4e --- /dev/null +++ b/src/main/java/life/qbic/registration/openbis/types/QDatasetType.java @@ -0,0 +1,44 @@ +package life.qbic.registration.openbis.types; + +import java.util.Objects; +import life.qbic.registration.openbis.exceptions.fail.DatasetTypeMappingException; + +/** + * Controlled vocabulary for openBiS dataset types + * + * @since 1.0.0 + */ +public enum QDatasetType { + + Q_NGS_RAW_DATA("Q_NGS_RAW_DATA"), + Q_PROTEOMICS_RAW_DATA("Q_PROTEOMICS_RAW_DATA"), + ; + + private final String openBisPropertyName; + + QDatasetType(String openBisPropertyName) { + this.openBisPropertyName = openBisPropertyName; + } + + public String getOpenBisPropertyName() { + return openBisPropertyName; + } + + public static QDatasetType fromQSampleType(QSampleType qSampleType) { +// If this was Java 17+ the following code would easily solve the issue. + /* + return switch (qSampleType) { + case Q_NGS_MEASUREMENT -> Q_NGS_RAW_DATA; + case Q_PROTEOMICS_MEASUREMENT -> Q_PROTEOMICS_RAW_DATA; + }; + */ + + if (Objects.requireNonNull(qSampleType) == QSampleType.Q_NGS_MEASUREMENT) { + return Q_NGS_RAW_DATA; + } else if (qSampleType == QSampleType.Q_PROTEOMICS_MEASUREMENT) { + return Q_PROTEOMICS_RAW_DATA; + } + throw new DatasetTypeMappingException("Unknown sample type to dataset type mapping. Cannot map " + qSampleType); + } + +} diff --git a/src/main/java/life/qbic/registration/openbis/types/QPropertyType.java b/src/main/java/life/qbic/registration/openbis/types/QPropertyType.java new file mode 100644 index 0000000..aeb7f42 --- /dev/null +++ b/src/main/java/life/qbic/registration/openbis/types/QPropertyType.java @@ -0,0 +1,42 @@ +package life.qbic.registration.openbis.types; + +import java.util.Optional; + +/** + * Controlled vocabulary for openBIS property types + * + * @since 1.0.0 + */ +public enum QPropertyType { + Q_SUBMITTER("Q_SUBMITTER"), + Q_TASK_ID("Q_TASK_ID"), + ; + + private final String openBisPropertyName; + + QPropertyType(String openBisPropertyName) { + this.openBisPropertyName = openBisPropertyName; + } + + public String getOpenBisPropertyName() { + return openBisPropertyName; + } + + /** + * Looks up the enum of a given name. If none match, an empty optional is returned. + * @param name the name of the enum + * @return an optional containing the matching enum if any. + */ + public static Optional lookup(String name) { + try { + return Optional.of(valueOf(name)); + } catch (IllegalArgumentException e) { + for (var value : values()) { + if (value.openBisPropertyName.equals(name)) { + return Optional.of(value); + } + } + } + return Optional.empty(); + } +} diff --git a/src/main/java/life/qbic/registration/openbis/types/QSampleType.java b/src/main/java/life/qbic/registration/openbis/types/QSampleType.java new file mode 100644 index 0000000..99b9895 --- /dev/null +++ b/src/main/java/life/qbic/registration/openbis/types/QSampleType.java @@ -0,0 +1,37 @@ +package life.qbic.registration.openbis.types; + +import java.util.Optional; + +/** + * Controlled vocabulary for openBIS sample types + * + * @since 1.0.0 + */ +public enum QSampleType { + Q_NGS_MEASUREMENT("Q_NGS_MEASUREMENT"), + Q_PROTEOMICS_MEASUREMENT("Q_PROTEOMICS_MEASUREMENT"); + + private final String openBisTypeName; + + QSampleType(String openBisTypeName) { + this.openBisTypeName = openBisTypeName; + } + + /** + * Looks up the enum of a given name. If none match, an empty optional is returned. + * @param name the name of the enum + * @return an optional containing the matching enum if any. + */ + public static Optional lookup(String name) { + try { + return Optional.of(valueOf(name)); + } catch (IllegalArgumentException e) { + for (QSampleType value : values()) { + if (value.openBisTypeName.equals(name)) { + return Optional.of(value); + } + } + } + return Optional.empty(); + } +} diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties deleted file mode 100644 index 43f0696..0000000 --- a/src/main/resources/application.properties +++ /dev/null @@ -1 +0,0 @@ -messages.file=messages.txt diff --git a/src/main/resources/messages.txt b/src/main/resources/messages.txt deleted file mode 100644 index 76dab38..0000000 --- a/src/main/resources/messages.txt +++ /dev/null @@ -1,5 +0,0 @@ -Have you written unit tests yet? If not, do it! -Keep it simple! -Remember the single responsibility principle! -Clean architecture ftw! -Favor composition over inheritance! \ No newline at end of file diff --git a/src/test/groovy/life/qbic/springminimaltemplate/SpringMinimalTemplateApplicationTests.groovy b/src/test/groovy/life/qbic/springminimaltemplate/SpringMinimalTemplateApplicationTests.groovy deleted file mode 100644 index f472c3e..0000000 --- a/src/test/groovy/life/qbic/springminimaltemplate/SpringMinimalTemplateApplicationTests.groovy +++ /dev/null @@ -1,26 +0,0 @@ -package life.qbic.springminimaltemplate - -import org.junit.jupiter.api.Test -import org.springframework.beans.factory.annotation.Autowired -import org.springframework.boot.test.context.SpringBootTest -import spock.lang.Specification - -@SpringBootTest -class SpringMinimalTemplateApplicationTests extends Specification { - - @Test - void contextLoads() { - } - - @Autowired - private MessageService messageService - - def "autowired works"() { - when: - String messages = messageService.collectMessage() - println(messages) - then: - messages != null - } - -} diff --git a/src/test/java/life/qbic/registration/openbis/ProvenanceParserTest.groovy b/src/test/java/life/qbic/registration/openbis/ProvenanceParserTest.groovy new file mode 100644 index 0000000..38b6936 --- /dev/null +++ b/src/test/java/life/qbic/registration/openbis/ProvenanceParserTest.groovy @@ -0,0 +1,25 @@ +package life.qbic.registration.openbis + +import spock.lang.Specification + +class ProvenanceParserTest extends Specification { + + final File validFile = new File(ProvenanceParserTest.class.getClassLoader().getResource("valid-provenance.json").toURI()); + + def "parsing a valid file works"() { + when: + var resultingProvenanceObject = ProvenanceParser.parseProvenanceJson(validFile) + then: + resultingProvenanceObject.measurementId() == "NGSQTEST001AE-1234512312" + and: + resultingProvenanceObject.origin() == "/Users/myuser/registration" + and: + resultingProvenanceObject.datasetFiles() == ["file1.fastq.gz", "file2.fastq.gz"] + and: + resultingProvenanceObject.user() == "/Users/myuser" + and: + resultingProvenanceObject.taskId() == "ce36775e-0d06-471e-baa7-1e3b63de871f" + and: + resultingProvenanceObject.history() == ["/some/dir", "/some/other/dir"] + } +} diff --git a/src/test/resources/valid-provenance.json b/src/test/resources/valid-provenance.json new file mode 100644 index 0000000..99ebbfd --- /dev/null +++ b/src/test/resources/valid-provenance.json @@ -0,0 +1,14 @@ +{ + "origin": "/Users/myuser/registration", + "user": "/Users/myuser", + "measurementId": "NGSQTEST001AE-1234512312", + "datasetFiles" : [ + "file1.fastq.gz", + "file2.fastq.gz" + ], + "taskId": "ce36775e-0d06-471e-baa7-1e3b63de871f", + "history": [ + "/some/dir", + "/some/other/dir" + ] +} From 09405a5bfea837e3ae14cec9d12e3b2d93422c7a Mon Sep 17 00:00:00 2001 From: Tobias Koch Date: Fri, 24 May 2024 14:57:44 +0200 Subject: [PATCH 3/7] Fix github actions and repository setup --- .github.settings.xml | 24 +++++++++---------- .github/CODEOWNERS | 1 + .github/dependabot.yml | 11 +++++++++ .github/workflows/build_package.yml | 17 ++++++------- .github/workflows/codeql-analysis.yml | 22 ++++++++--------- .github/workflows/create-release.yml | 22 +++++++++-------- .github/workflows/label-pull-requests.yml | 12 ++++++---- .github/workflows/labeler.yml | 13 ++++++++++ .github/workflows/nexus-publish-snapshots.yml | 18 +++++++------- .github/workflows/release.yml | 20 ++++++++++++++++ .github/workflows/run_tests.yml | 15 +++++++----- LICENSE | 2 +- 12 files changed, 116 insertions(+), 61 deletions(-) create mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/labeler.yml create mode 100644 .github/workflows/release.yml diff --git a/.github.settings.xml b/.github.settings.xml index d3a0059..1059768 100644 --- a/.github.settings.xml +++ b/.github.settings.xml @@ -1,14 +1,14 @@ - - - nexus-snapshots - ${env.MAVEN_REPO_USERNAME} - ${env.MAVEN_REPO_PASSWORD} - - - nexus-releases - ${env.MAVEN_REPO_USERNAME} - ${env.MAVEN_REPO_PASSWORD} - - + + + nexus-snapshots + ${env.MAVEN_REPO_USERNAME} + ${env.MAVEN_REPO_PASSWORD} + + + nexus-releases + ${env.MAVEN_REPO_USERNAME} + ${env.MAVEN_REPO_PASSWORD} + + diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 8d7e69b..73cf965 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1 +1,2 @@ * @qbicsoftware/itss + diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..90e05c4 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,11 @@ +# To get started with Dependabot version updates, you'll need to specify which +# package ecosystems to update and where the package manifests are located. +# Please see the documentation for all configuration options: +# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates + +version: 2 +updates: + - package-ecosystem: "github-actions" # See documentation for possible values + directory: "/" # Location of package manifests + schedule: + interval: "weekly" diff --git a/.github/workflows/build_package.yml b/.github/workflows/build_package.yml index 02a177d..983ffe0 100644 --- a/.github/workflows/build_package.yml +++ b/.github/workflows/build_package.yml @@ -6,25 +6,26 @@ on: - '**' pull_request: # The branches below must be a subset of the branches above - branches: [ main, master ] + branches: [ main ] + jobs: package: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - name: Set up JDK 1.17 - uses: actions/setup-java@v1 + - name: Checkout repository + uses: actions/checkout@v4 + - name: Set up JDK 11 + uses: actions/setup-java@v4 with: - java-version: 1.17 - + distribution: 'zulu' + java-version: '11' - name: Load local Maven repository cache - uses: actions/cache@v2 + uses: actions/cache@v4 with: path: ~/.m2/repository key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} restore-keys: | ${{ runner.os }}-maven- - - name: Run mvn package run: mvn -B package --file pom.xml diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index d030d5a..8191550 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -13,16 +13,15 @@ name: "CodeQL" on: push: - branches: [ main, master, development, release/*, hotfix/* ] + branches: [ main, development, release/*, hotfix/* ] pull_request: # The branches below must be a subset of the branches above - branches: [ main, master ] + branches: [ main ] schedule: - cron: '21 1 * * 4' jobs: analyze: - name: analyze runs-on: ubuntu-latest permissions: actions: read @@ -38,15 +37,16 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v2 - - name: Set up JDK 1.17 - uses: actions/setup-java@v1 + uses: actions/checkout@v4 + - name: Set up JDK 11 + uses: actions/setup-java@v4 with: - java-version: 1.17 + distribution: 'zulu' + java-version: '11' settings-path: ${{ github.workspace }} - name: Load local Maven repository cache - uses: actions/cache@v2 + uses: actions/cache@v4 with: path: ~/.m2/repository key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} @@ -55,7 +55,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v1 + uses: github/codeql-action/init@v3 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. @@ -66,7 +66,7 @@ jobs: # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild - uses: github/codeql-action/autobuild@v1 + uses: github/codeql-action/autobuild@v3 # ℹī¸ Command-line programs to run using the OS shell. # 📚 https://git.io/JvXDl @@ -80,4 +80,4 @@ jobs: # make release - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v1 + uses: github/codeql-action/analyze@v3 diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml index 38ce57d..02b4dcd 100644 --- a/.github/workflows/create-release.yml +++ b/.github/workflows/create-release.yml @@ -11,15 +11,17 @@ jobs: release: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - name: Set up JDK 1.17 - uses: actions/setup-java@v1 + - name: Checkout repository + uses: actions/checkout@v4 + - name: Set up JDK 11 + uses: actions/setup-java@v4 with: - java-version: 1.17 + distribution: 'zulu' + java-version: '11' settings-path: ${{ github.workspace }} - name: Load local Maven repository cache - uses: actions/cache@v2 + uses: actions/cache@v4 with: path: ~/.m2/repository key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} @@ -32,17 +34,17 @@ jobs: git config --global user.name "JohnnyQ5" - name: Set version in Maven project - run: mvn versions:set -DnewVersion=${{ github.event.inputs.versionTag }} + run: mvn versions:set -DnewVersion=${{ github.event.inputs.versionTag }} - name: Build with Maven - run: mvn -B package --file pom.xml + run: mvn -B package --file pom.xml - name: Create Release Notes if: ${{ !startsWith(github.ref, 'refs/tags/') && !( contains(github.event.inputs.versionTag, 'alpha') || contains(github.event.inputs.versionTag, 'beta') || contains(github.event.inputs.versionTag, 'rc')) }} - uses: actions/github-script@v4.0.2 + uses: actions/github-script@v7 with: github-token: ${{secrets.JOHNNY_Q5_REPORTS_TOKEN}} script: | @@ -56,7 +58,7 @@ jobs: && ( contains(github.event.inputs.versionTag, 'alpha') || contains(github.event.inputs.versionTag, 'beta') || contains(github.event.inputs.versionTag, 'rc')) }} - uses: actions/github-script@v4.0.2 + uses: actions/github-script@v7 with: github-token: ${{secrets.JOHNNY_Q5_REPORTS_TOKEN}} script: | @@ -85,7 +87,7 @@ jobs: run: git push - name: Open PR with version bump - uses: actions/github-script@v4.0.2 + uses: actions/github-script@v7 with: github-token: ${{secrets.JOHNNY_Q5_REPORTS_TOKEN}} script: | diff --git a/.github/workflows/label-pull-requests.yml b/.github/workflows/label-pull-requests.yml index b03eee1..931ff6d 100644 --- a/.github/workflows/label-pull-requests.yml +++ b/.github/workflows/label-pull-requests.yml @@ -1,15 +1,17 @@ name: Label Pull Requests +permissions: + contents: read + pull-requests: write + on: pull_request: types: [ opened, edited ] jobs: - label: + Assign-Label-To-Pull-Request: runs-on: ubuntu-latest steps: - - uses: TimonVS/pr-labeler-action@v3 + - uses: actions/labeler@v5.0.0-alpha.1 with: - configuration-path: .github/pr-labels.yml # optional, .github/pr-labeler.yml is the default value - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + repo-token: "${{ secrets.GITHUB_TOKEN }}" diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml new file mode 100644 index 0000000..fdd0f42 --- /dev/null +++ b/.github/workflows/labeler.yml @@ -0,0 +1,13 @@ +feature: + - any: + - head-branch: [ '^feature' ] +fix: + - any: + - head-branch: [ '^fix', '^hotfix' ] +chore: + - any: + - head-branch: [ '^chore', '^documentation', '^docs', '^ci', '^refactor' ] +release: + - all: + - base-branch: [ 'main', 'master' ] + - head-branch: [ 'development', 'dev' ] diff --git a/.github/workflows/nexus-publish-snapshots.yml b/.github/workflows/nexus-publish-snapshots.yml index fc25f08..b975e6b 100644 --- a/.github/workflows/nexus-publish-snapshots.yml +++ b/.github/workflows/nexus-publish-snapshots.yml @@ -10,20 +10,22 @@ on: - development jobs: - publish_snapshots: + publish_snapshot: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - name: Set up JDK 1.17 - uses: actions/setup-java@v1 + - name: Checkout repository + uses: actions/checkout@v4 + - name: Set up JDK 11 + uses: actions/setup-java@v4 with: - java-version: 1.17 + distribution: 'zulu' + java-version: '11' settings-path: ${{ github.workspace }} - name: Load local Maven repository cache - uses: actions/cache@v2 + uses: actions/cache@v4 with: path: ~/.m2/repository key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} @@ -34,13 +36,13 @@ jobs: run: mvn versions:set -DremoveSnapshot # Set the SNAPSHOT for this build and deployment - name: Set version in Maven project - run: mvn versions:set -DnewVersion='${project.version}-SNAPSHOT' + run: mvn versions:set -DnewVersion='${project.version}-SNAPSHOT' - name: Build with Maven run: mvn -B package --file pom.xml - name: Publish artefact to QBiC Nexus Repository - run: mvn --settings $GITHUB_WORKSPACE/.github.settings.xml deploy + run: mvn --quiet --settings $GITHUB_WORKSPACE/.github.settings.xml deploy env: MAVEN_REPO_USERNAME: ${{ secrets.NEXUS_USERNAME }} MAVEN_REPO_PASSWORD: ${{ secrets.NEXUS_PASSWORD }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..1a928f0 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,20 @@ +changelog: + exclude: + labels: + - ignore-for-release + authors: + - JohnnyQ5 + - github-actions + categories: + - title: New Features 🚀 + labels: + - feature + - title: Bugfixes đŸĒ˛ + labels: + - fix + - title: Documentation & CI đŸĒ‚ + labels: + - chore + - title: Others 🧃 + labels: + - "*" diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index 1dde144..5fbffcd 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -6,21 +6,24 @@ on: - '**' pull_request: # The branches below must be a subset of the branches above - branches: [ main, master ] + branches: [ main ] jobs: test: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - name: Set up JDK 1.17 - uses: actions/setup-java@v1 + - name: Checkout repository + uses: actions/checkout@v4 + - name: Set up JDK 11 + uses: actions/setup-java@v4 with: - java-version: 1.17 + distribution: 'zulu' + java-version: '11' + settings-path: ${{ github.workspace }} - name: Load local Maven repository cache - uses: actions/cache@v2 + uses: actions/cache@v4 with: path: ~/.m2/repository key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} diff --git a/LICENSE b/LICENSE index 9ac8ded..d23975c 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2021 QBiC +Copyright (c) 2024, University of Tuebingen Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal From 8580f20d4084eee5d309850c86a976a49ef99421 Mon Sep 17 00:00:00 2001 From: Tobias Koch Date: Fri, 24 May 2024 15:10:29 +0200 Subject: [PATCH 4/7] fix labeler config --- .github/{workflows => }/labeler.yml | 2 +- .github/pr-labels.yml | 3 --- .github/workflows/label-pull-requests.yml | 3 ++- .github/workflows/release.yml | 20 -------------------- 4 files changed, 3 insertions(+), 25 deletions(-) rename .github/{workflows => }/labeler.yml (86%) delete mode 100644 .github/pr-labels.yml delete mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/labeler.yml b/.github/labeler.yml similarity index 86% rename from .github/workflows/labeler.yml rename to .github/labeler.yml index fdd0f42..09a2dc5 100644 --- a/.github/workflows/labeler.yml +++ b/.github/labeler.yml @@ -9,5 +9,5 @@ chore: - head-branch: [ '^chore', '^documentation', '^docs', '^ci', '^refactor' ] release: - all: - - base-branch: [ 'main', 'master' ] + - base-branch: [ 'main' ] - head-branch: [ 'development', 'dev' ] diff --git a/.github/pr-labels.yml b/.github/pr-labels.yml deleted file mode 100644 index f95cf6e..0000000 --- a/.github/pr-labels.yml +++ /dev/null @@ -1,3 +0,0 @@ -feature: ['feature/*', 'feat/*'] -fix: ['fix/*', 'hotfix'] -chore: ['chore/*', 'documentation/*', 'docs/*', 'ci/*', 'refactor/*'] diff --git a/.github/workflows/label-pull-requests.yml b/.github/workflows/label-pull-requests.yml index 931ff6d..f3f6f83 100644 --- a/.github/workflows/label-pull-requests.yml +++ b/.github/workflows/label-pull-requests.yml @@ -12,6 +12,7 @@ jobs: Assign-Label-To-Pull-Request: runs-on: ubuntu-latest steps: - - uses: actions/labeler@v5.0.0-alpha.1 + - uses: actions/labeler@v5 with: repo-token: "${{ secrets.GITHUB_TOKEN }}" + configuration-path: .github/labeler.yml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml deleted file mode 100644 index 1a928f0..0000000 --- a/.github/workflows/release.yml +++ /dev/null @@ -1,20 +0,0 @@ -changelog: - exclude: - labels: - - ignore-for-release - authors: - - JohnnyQ5 - - github-actions - categories: - - title: New Features 🚀 - labels: - - feature - - title: Bugfixes đŸĒ˛ - labels: - - fix - - title: Documentation & CI đŸĒ‚ - labels: - - chore - - title: Others 🧃 - labels: - - "*" From b2f0ca3cd25abaf733ac8ec3e07ca20e289ec816 Mon Sep 17 00:00:00 2001 From: Tobias Koch Date: Mon, 27 May 2024 08:50:36 +0200 Subject: [PATCH 5/7] Add Readme --- README.md | 227 +++++++++++++----------------------------------------- 1 file changed, 52 insertions(+), 175 deletions(-) diff --git a/README.md b/README.md index 6c46469..f96c255 100644 --- a/README.md +++ b/README.md @@ -1,187 +1,64 @@ -# Spring Boot Starter (template) - -A minimal working starter template for a Spring Boot non-web applications using the -Spring [CommandLineRunner](https://docs.spring.io/spring-boot/docs/current/api/org/springframework/boot/CommandLineRunner.html) -interface with a demonstration of Java -annotation-based [Inversion of Control](https://stackoverflow.com/questions/3058/what-is-inversion-of-control) -via [Dependency Injection](https://stackoverflow.com/questions/130794/what-is-dependency-injection). - -## Run the app - -Checkout the latest code from `main` and run the Maven goal `spring-boot:run`: - -``` -mvn spring-boot:run -``` - -## What the app does - -This small app just parses a file with a collection of good coding prayers and creates a singleton -instance of an `CodingPrayersMessageService`. This concrete implementation uses the -interface `MessageService`, that comes with only one public method: `String collectMessage()`. - -This service is used to demonstrate the IoC principle. We have defined another interface `NewsMedia` -and provide a concrete implementation `DeveloperNews`, that will call the message service to receive -recent news and forward them to the caller. - -In the main app code, we just retrieve this Singleton instance or Bean in Spring lingua from the -loaded context and call the news media `getNEws()` method. The collected message is then printed out -to the command line interface: - -``` - - . ____ _ __ _ _ - /\\ / ___'_ __ _ _(_)_ __ __ _ \ \ \ \ -( ( )\___ | '_ | '_| | '_ \/ _` | \ \ \ \ - \\/ ___)| |_)| | | | | || (_| | ) ) ) ) - ' |____| .__|_| |_|_| |_\__, | / / / / - =========|_|==============|___/=/_/_/_/ - :: Spring Boot :: (v2.5.6) - -2021-11-18 09:17:37.640 INFO 68052 --- [ main] l.q.s.SpringMinimalTemplateApplication : Starting SpringMinimalTemplateApplication using Java 17.0.1 on imperator.am10.uni-tuebingen.de with PID 68052 (/Users/sven1103/git/spring-boot-starter-template/target/classes started by sven1103 in /Users/sven1103/git/spring-boot-starter-template) -2021-11-18 09:17:37.641 INFO 68052 --- [ main] l.q.s.SpringMinimalTemplateApplication : No active profile set, falling back to default profiles: default -2021-11-18 09:17:38.164 INFO 68052 --- [ main] l.q.s.SpringMinimalTemplateApplication : Started SpringMinimalTemplateApplication in 0.808 seconds (JVM running for 1.489) -####################### Message of the day ################## -Have you written unit tests yet? If not, do it! -############################################################## - +# OpenBiS Dropboxes + +!!! abstract +After the [Data Scanner](#), data registration in openbis can be automated by dropboxes. +We use Java Dropboxes to move files into OpenBis (see +the [Documentation on Java Dropboxes](https://openbis.readthedocs.io/en/20.10.x/software-developer-documentation/server-side-extensions/dss-dropboxes.html#java-dropboxes)) + +## Context + +```mermaid +sequenceDiagram + autonumber + User ->> SFTP Folder : move files + Data Scanner ->> SFTP Folder : is new data present? + SFTP Folder ->> Data Scanner : Yes + Data Scanner ->> Dropbox Incoming : move files + Data Scanner ->> Dropbox Incoming : create marker file + Dropbox ->> Dropbox Incoming : Is marker file present? + Dropbox Incoming ->> Dropbox : Yes + Dropbox ->> OpenBiS : Register data ``` -## Realisation of IoC and DPI - -The messages collection is stored in a simple text file `messages.txt`, that is provided with the -apps `resources`. Just go ahead and change the content of the file and run the app! - -grafik - - -So how does the app know where to find this file and **load the messages content**? - -We have configured it as a **external property** in a file `application.properties` and load the -configuration on application startup! Cool eh? - -grafik -This is how the file content looks like: +### Dropbox Process -``` -messages.file=messages.txt -``` - -So how do we access the value of the `messages.file` property in our application with Spring? - -Have a look in the class `AppConfig`, there the magic happens: +The dropbox processes the data before moving it into OpenBiS. A successful processing is shown +below. -```groovy -@Configuration -@PropertySource("application.properties") -class AppConfig { +The process is triggered when a marker filed called `MARKER_is_finished_` with `` being the folder to be registered, is created. - @Value('${messages.file}') - public String messagesFile +``` mermaid +sequenceDiagram + autonumber + IncomingFolder ->> Dropbox: Folder and Markerfile exist + Dropbox ->> IncomingFolder: Read provenance.json + Dropbox ->> Dropbox: Parse measurement ID + Dropbox ->> OpenBiS DSS: Get measurement sample + OpenBiS DSS ->> Dropbox: Give measurement sample + Dropbox ->> OpenBiS DSS: Has measurement datasets? + OpenBiS DSS ->> Dropbox: No + Dropbox ->> Dropbox: Parse measurement properties + Dropbox ->> OpenBiS DSS: Create dataset + Dropbox ->> OpenBiS DSS: Send files to dataset ``` -We define the property source, which is the file `application.properties` that is provided in the -resource folder of the app and available to the classpath. We also tell with the -annotation `@Configuration` Spring, hey this is a class that holds app configuration data! +### Dropbox Configuration -With the annotation `@Value('${messages.file}')` we tell Spring, which property's value should be -injected. Here we make use of field injection, other types of injection like method and constructor -injection are also possible. +The dropbox configuration can be found +at `...` on the OpenBiS instance. Each folder within this directory creates a dropbox. -So how is the concrete implementation of the `MessageService` presented to Spring? We can use -the `@Bean` annotation here, to tell Spring: _hey, this is sth you must load on startup and provide -to the context_. +Within each folder exists a directory `lib` and a file `plugin.properties`. The `lib` folder is +where you place your dropbox `jar` file. The `plugin.properties` is where you configure your +dropbox. -```java -@Configuration -@PropertySource("application.properties") -class AppConfig { - - .... - - @Bean - MessageService messageService() { - return new CodingPrayersMessageService(messagesFile) - } +```properties title="plugin.properties" linenums="1" +incoming-dir=... +incoming-data-completeness-condition=marker-file +top-level-data-set-handler=ch.systemsx.cisd.etlserver.registrator.api.v2.JavaTopLevelDataSetHandlerV2 +program-class=life.qbic.registration.openbis.OpenBisDropboxETL +storage-processor=ch.systemsx.cisd.etlserver.DefaultStorageProcessor ``` - -That is all there is, you can now load the bean in your main application code: - -```java -@SpringBootApplication -class SpringMinimalTemplateApplication { - - static void main(String[] args) { - SpringApplication.run(SpringMinimalTemplateApplication, args) - // load the annotation context - AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(AppConfig.class) - // get the service bean - MessageService service = context.getBean("messageService", MessageService.class) - // collect the message and praise the magic - println service.collectMessage() - -``` - -### Inversion of Control - -grafik - -You might have already spotted the interface `NewsMedia` and its implementing class `DeveloperNews` -in the app's source code. Here you can see an example for the magic of inversion of control. - -The `NewsMedia` interface is just an abstraction that we will later use, because we don't care about -the actual implementation details. By this, we also do not create any dependencies to concrete -implementation details but on actual behaviour. Concrete implementations can then later be exchanged -without causing any breaking changes in the client code base. - -The interface has only one method: `String getNews()`. Now let's have a closer look into the -class `DeveloperNews` that implements this interface: - -```java -class DeveloperNews implements NewsMedia{ - - private MessageService service - - DeveloperNews(MessageService service) { - this.service = service - } - - @Override - String getNews() { - return service.collectMessage() - } -} -``` - -When you check the constructor signature, you see that this method has only one argument, which is a -reference to an object of type `MessageService`. And when the `getNews()` method is called by the -client, the class delegates this request to the message service. Since we have stored the reference -in a private field, that is super easy, we known how to call the service. - -So why is this inversion of control? - -Because the `DeveloperNews` class does not manage the instantiation of a concrete message service. -The configuration happened outside of the class, therefore the DeveloperNews class has no direct -control over the instantiation. If it had, it would look like this: - -```java -DeveloperNews(String filePathToMessages) { - this.service = new CodingPrayersMessageService(filePathToMessages) -} -``` - -That doesn't look good, does it? In order to create an instance of a message service, we would need -to know the conrete implementation and its required properties (here it is the file path to -the `messages.txt`). So the `DeveloperNews` class has the control over the message service. - -Instead, we would like to not take care about these details, so we invert the control and inject the -dependency via the constructor. - -Please find more in depth documentation on the -official [Spring website](https://spring.io/projects/spring-framework). - - - - - +!!! info +Ususally you only need to adapt the `incoming-dir` and `program-class` when setting up a new dropbox. +For more configuration options please see the [OpenBiS documentation](https://openbis.readthedocs.io/en/20.10.x/software-developer-documentation/server-side-extensions/dss-dropboxes.html#configuration). From 13a429901cdcc8769930bf1875e686202ce3c3e9 Mon Sep 17 00:00:00 2001 From: Tobias Koch Date: Mon, 27 May 2024 08:55:37 +0200 Subject: [PATCH 6/7] Add link to data-processing --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f96c255..366548e 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # OpenBiS Dropboxes !!! abstract -After the [Data Scanner](#), data registration in openbis can be automated by dropboxes. +After the [Data Scanner]((https://github.com/qbicsoftware/data-processing), data registration in openbis can be automated by dropboxes. We use Java Dropboxes to move files into OpenBis (see the [Documentation on Java Dropboxes](https://openbis.readthedocs.io/en/20.10.x/software-developer-documentation/server-side-extensions/dss-dropboxes.html#java-dropboxes)) From 0a6567e414fed57b52cee3985fb97c1893875746 Mon Sep 17 00:00:00 2001 From: Tobias Koch Date: Mon, 27 May 2024 08:55:54 +0200 Subject: [PATCH 7/7] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 366548e..bb336d9 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # OpenBiS Dropboxes !!! abstract -After the [Data Scanner]((https://github.com/qbicsoftware/data-processing), data registration in openbis can be automated by dropboxes. +After the [Data Scanner](https://github.com/qbicsoftware/data-processing), data registration in openbis can be automated by dropboxes. We use Java Dropboxes to move files into OpenBis (see the [Documentation on Java Dropboxes](https://openbis.readthedocs.io/en/20.10.x/software-developer-documentation/server-side-extensions/dss-dropboxes.html#java-dropboxes))