diff --git a/.github/workflows/meteorology-eurac-dc.yml b/.github/workflows/meteorology-eurac-dc.yml new file mode 100644 index 0000000..dd13f71 --- /dev/null +++ b/.github/workflows/meteorology-eurac-dc.yml @@ -0,0 +1,64 @@ +name: CI/CD meteorology-eurac-dc + +on: + push: + paths: + - "collectors/meteorology-eurac/**" + - ".github/workflows/meteorology-eurac-dc.yml" + +env: + AWS_REGION : eu-west-1 + AWS_EKS_CLUSTER_NAME : aws-main-eu-01 + AWS_ACCESS_KEY_ID : ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_ACCESS_SECRET_KEY : ${{ secrets.AWS_ACCESS_SECRET_KEY }} + JAVA_VERSION: 17 + WORKING_DIRECTORY: collectors/meteorology-eurac + DOCKER_IMAGE: ghcr.io/noi-techpark/odh-v2-playground/meteorology-eurac-dc + DOCKER_TAG: 0.0.0 + + +jobs: + deploy-test: + runs-on: ubuntu-latest + if: github.ref == 'refs/heads/meteorology-eurac' + concurrency: deploy-test-meteorology-eurac-dc + steps: + - name: Checkout source code + uses: actions/checkout@v4 + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ env.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ env.AWS_ACCESS_SECRET_KEY }} + aws-region: ${{ env.AWS_REGION }} + + - name: Authenticate to AWS EKS + run: aws eks --region ${{ env.AWS_REGION }} update-kubeconfig --name ${{ env.AWS_EKS_CLUSTER_NAME }} + + - name: Build libs + uses: noi-techpark/github-actions/maven-build@v2 + with: + working-directory: collectors/lib/ingress-mq + java-version: ${{ env.JAVA_VERSION }} + build-command: 'mvn clean install' + + - name: Build project + uses: noi-techpark/github-actions/maven-build@v2 + with: + working-directory: ${{ env.WORKING_DIRECTORY }} + java-version: ${{ env.JAVA_VERSION }} + build-command: 'mvn clean install' + + - name: Build and push images + uses: noi-techpark/github-actions/docker-build-and-push@v2 + with: + working-directory: ${{ env.WORKING_DIRECTORY }}/infrastructure + docker-username: ${{ github.actor }} + docker-password: ${{ secrets.GITHUB_TOKEN }} + + - name: Helm deployment + run: | + RELEASETIME=`date +%s` + cd collectors/meteorology-eurac + helm upgrade --install dc-meteorology-eurac ../../helm/generic-collector --values infrastructure/helm/values.yaml --set-string podAnnotations.releaseTime=$RELEASETIME \ No newline at end of file diff --git a/.github/workflows/meteorology-eurac-tr.yml b/.github/workflows/meteorology-eurac-tr.yml new file mode 100644 index 0000000..d9d08ac --- /dev/null +++ b/.github/workflows/meteorology-eurac-tr.yml @@ -0,0 +1,64 @@ +name: CI/CD meteorology-eurac-tr + +on: + push: + paths: + - "transformers/meteorology-eurac/**" + - ".github/workflows/meteorology-eurac-tr.yml" + +env: + AWS_REGION : eu-west-1 + AWS_EKS_CLUSTER_NAME : aws-main-eu-01 + AWS_ACCESS_KEY_ID : ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_ACCESS_SECRET_KEY : ${{ secrets.AWS_ACCESS_SECRET_KEY }} + JAVA_VERSION: 17 + WORKING_DIRECTORY: transformers/meteorology-eurac + DOCKER_IMAGE: ghcr.io/noi-techpark/odh-v2-playground/meteorology-eurac-tr + DOCKER_TAG: 0.2.0 + + +jobs: + deploy-test: + runs-on: ubuntu-latest + if: github.ref == 'refs/heads/meteorology-eurac' + concurrency: deploy-test-meteorology-eurac-tr + steps: + - name: Checkout source code + uses: actions/checkout@v4 + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ env.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ env.AWS_ACCESS_SECRET_KEY }} + aws-region: ${{ env.AWS_REGION }} + + - name: Authenticate to AWS EKS + run: aws eks --region ${{ env.AWS_REGION }} update-kubeconfig --name ${{ env.AWS_EKS_CLUSTER_NAME }} + + - name: Build libs + uses: noi-techpark/github-actions/maven-build@v2 + with: + working-directory: transformers/lib/rabbit-mongo-listener + java-version: ${{ env.JAVA_VERSION }} + build-command: 'mvn clean install' + + - name: Build project + uses: noi-techpark/github-actions/maven-build@v2 + with: + working-directory: ${{ env.WORKING_DIRECTORY }} + java-version: ${{ env.JAVA_VERSION }} + build-command: 'mvn clean install' + + - name: Build and push images + uses: noi-techpark/github-actions/docker-build-and-push@v2 + with: + working-directory: ${{ env.WORKING_DIRECTORY }}/infrastructure + docker-username: ${{ github.actor }} + docker-password: ${{ secrets.GITHUB_TOKEN }} + + - name: Helm deployment + run: | + RELEASETIME=`date +%s` + cd transformers/meteorology-eurac + helm upgrade --install tr-meteorology-eurac ../../helm/generic-collector --values infrastructure/helm/values.yaml --set-string podAnnotations.releaseTime=$RELEASETIME \ No newline at end of file diff --git a/collectors/lib/ingress-mq/src/main/java/com/opendatahub/collector/lib/ingress/mq/MsgRoute.java b/collectors/lib/ingress-mq/src/main/java/com/opendatahub/collector/lib/ingress/mq/MsgRoute.java index 7f2c7d7..6a90c57 100644 --- a/collectors/lib/ingress-mq/src/main/java/com/opendatahub/collector/lib/ingress/mq/MsgRoute.java +++ b/collectors/lib/ingress-mq/src/main/java/com/opendatahub/collector/lib/ingress/mq/MsgRoute.java @@ -12,8 +12,8 @@ public class MsgRoute extends RouteBuilder { public MsgRoute(String from) { this.from = from; } - - public String getRouteUri(){ + + public String getRouteUri() { return "direct:" + from; } @@ -22,7 +22,7 @@ public MsgRoute() { @Value("${ingress.provider:#{null}}") String provider; - + @Autowired RabbitMQConnection rabbitMQConfig; diff --git a/collectors/meteorology-eurac/.env.example b/collectors/meteorology-eurac/.env.example new file mode 100644 index 0000000..735903b --- /dev/null +++ b/collectors/meteorology-eurac/.env.example @@ -0,0 +1,8 @@ +INGRESS_PROVIDER=meteorology-eurac/test +INGRESS_RABBITMQ_CLUSTER=rabbithost.domain.tld +INGRESS_RABBITMQ_USER=dc-meteorology-eurac +INGRESS_RABBITMQ_PASS=***** +INGRESS_RABBITMQ_CLIENTNAME=meteorology-eurac-datacollector-test +CRON_STATIONS=0 */2 * * * ? +CRON_MONTHLY=0 */2 * * * ? +CRON_DAILY=0 */2 * * * ? \ No newline at end of file diff --git a/collectors/meteorology-eurac/.gitignore b/collectors/meteorology-eurac/.gitignore new file mode 100644 index 0000000..549e00a --- /dev/null +++ b/collectors/meteorology-eurac/.gitignore @@ -0,0 +1,33 @@ +HELP.md +target/ +!.mvn/wrapper/maven-wrapper.jar +!**/src/main/**/target/ +!**/src/test/**/target/ + +### STS ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans +.sts4-cache + +### IntelliJ IDEA ### +.idea +*.iws +*.iml +*.ipr + +### NetBeans ### +/nbproject/private/ +/nbbuild/ +/dist/ +/nbdist/ +/.nb-gradle/ +build/ +!**/src/main/**/build/ +!**/src/test/**/build/ + +### VS Code ### +.vscode/ diff --git a/collectors/meteorology-eurac/deploy.sh b/collectors/meteorology-eurac/deploy.sh new file mode 100755 index 0000000..7139406 --- /dev/null +++ b/collectors/meteorology-eurac/deploy.sh @@ -0,0 +1,8 @@ +#!/bin/bash +#docker login ghcr.io (create token with read/write package permissions in github developer settings) +RELEASETIME=`date +%s` +(cd ../lib/ingress-mq; mvn clean install) \ +&& mvn clean install \ +&& docker build -t ghcr.io/noi-techpark/odh-v2-playground/meteorology-eurac:0.0.0 . -f infrastructure/docker/Dockerfile \ +&& docker image push ghcr.io/noi-techpark/odh-v2-playground/meteorology-eurac:0.0.0 \ +&& helm upgrade --install dc-meteorology-eurac ../../helm/generic-collector --values infrastructure/helm/values.yaml --set-string podAnnotations.releaseTime=$RELEASETIME \ No newline at end of file diff --git a/collectors/meteorology-eurac/docker-compose.yml b/collectors/meteorology-eurac/docker-compose.yml new file mode 100644 index 0000000..b1d9110 --- /dev/null +++ b/collectors/meteorology-eurac/docker-compose.yml @@ -0,0 +1,18 @@ +version: "3" + +services: + app: + image: maven:3-openjdk-17-slim + env_file: + - .env + environment: + MAVEN_CONFIG: /var/maven/.m2 + MAVEN_OPTS: -Dmaven.repo.local=/var/maven/.m2/repository -Dspring-boot.run.jvmArguments="-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:9000" + command: > + mvn clean spring-boot:run -debug + volumes: + - ~/.m2/:/var/maven/.m2 + - ./:/code + working_dir: /code + tty: true + network_mode: host \ No newline at end of file diff --git a/collectors/meteorology-eurac/infrastructure/docker-compose.build.yml b/collectors/meteorology-eurac/infrastructure/docker-compose.build.yml new file mode 100644 index 0000000..93d36f0 --- /dev/null +++ b/collectors/meteorology-eurac/infrastructure/docker-compose.build.yml @@ -0,0 +1,8 @@ +version: "3.4" + +services: + app: + image: ${DOCKER_IMAGE}:${DOCKER_TAG} + build: + context: ../ + dockerfile: infrastructure/docker/Dockerfile diff --git a/collectors/meteorology-eurac/infrastructure/docker/Dockerfile b/collectors/meteorology-eurac/infrastructure/docker/Dockerfile new file mode 100644 index 0000000..135c5c4 --- /dev/null +++ b/collectors/meteorology-eurac/infrastructure/docker/Dockerfile @@ -0,0 +1,5 @@ +FROM eclipse-temurin:17-jre-alpine +WORKDIR /app +COPY target/app.jar app.jar +ENTRYPOINT [ "java", "-jar", "app.jar"] + diff --git a/collectors/meteorology-eurac/infrastructure/helm/values.yaml b/collectors/meteorology-eurac/infrastructure/helm/values.yaml new file mode 100644 index 0000000..f41744c --- /dev/null +++ b/collectors/meteorology-eurac/infrastructure/helm/values.yaml @@ -0,0 +1,27 @@ +# SPDX-FileCopyrightText: NOI Techpark +# +# SPDX-License-Identifier: CC0-1.0 + +nameOverride: dc-meteorology-eurac +fullnameOverride: dc-meteorology-eurac + +image: + repository: ghcr.io/noi-techpark/odh-v2-playground/meteorology-eurac-dc + pullPolicy: Always + tag: "0.0.0" # Set this when upgrading chart with --set-value + +imagePullSecrets: + - name: ghcr-odh-v2-playground-readonly + +env: + INGRESS_RABBITMQ_CLUSTER: rabbitmq-0.rabbitmq-headless.default.svc.cluster.local:5672 + INGRESS_RABBITMQ_CLIENTNAME: meteorology-eurac-datacollector + CRON_STATIONS: "0 0 */1 * * ?" + CRON_MONTHLY: "0 */5 * * * ?" + CRON_DAILY: "0 0 */24 * * ?" + INGRESS_PROVIDER: meteorology/eurac + + +envSecret: + INGRESS_RABBITMQ_USER: guest + INGRESS_RABBITMQ_PASS: guest \ No newline at end of file diff --git a/collectors/meteorology-eurac/pom.xml b/collectors/meteorology-eurac/pom.xml new file mode 100644 index 0000000..f751320 --- /dev/null +++ b/collectors/meteorology-eurac/pom.xml @@ -0,0 +1,68 @@ + + + 4.0.0 + + org.springframework.boot + spring-boot-starter-parent + 3.1.3 + + + com.opendatahub.collector + dc-meteorology-eurac + 0.0.1-SNAPSHOT + dc-meteorology-eurac + Eurac meteorology data collector + + 17 + 4.1.0 + + + + com.opendatahub.collector.lib + ingress-mq + 1.0 + + + + org.springframework.boot + spring-boot-starter-test + test + + + + org.apache.camel.springboot + camel-spring-boot-starter + ${camel.version} + + + + org.apache.camel + camel-cron + ${camel.version} + + + + org.apache.camel + camel-http + ${camel.version} + + + + + + app + + + org.graalvm.buildtools + native-maven-plugin + + + org.springframework.boot + spring-boot-maven-plugin + + + + + \ No newline at end of file diff --git a/collectors/meteorology-eurac/src/main/java/com/opendatahub/collector/meteorology/eurac/MeteorologyEuracApplication.java b/collectors/meteorology-eurac/src/main/java/com/opendatahub/collector/meteorology/eurac/MeteorologyEuracApplication.java new file mode 100644 index 0000000..6772a35 --- /dev/null +++ b/collectors/meteorology-eurac/src/main/java/com/opendatahub/collector/meteorology/eurac/MeteorologyEuracApplication.java @@ -0,0 +1,11 @@ +package com.opendatahub.collector.meteorology.eurac; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; + +@SpringBootApplication +public class MeteorologyEuracApplication { + public static void main(String[] args) { + SpringApplication.run(MeteorologyEuracApplication.class, args); + } +} diff --git a/collectors/meteorology-eurac/src/main/java/com/opendatahub/collector/meteorology/eurac/Route.java b/collectors/meteorology-eurac/src/main/java/com/opendatahub/collector/meteorology/eurac/Route.java new file mode 100644 index 0000000..090e99a --- /dev/null +++ b/collectors/meteorology-eurac/src/main/java/com/opendatahub/collector/meteorology/eurac/Route.java @@ -0,0 +1,47 @@ +package com.opendatahub.collector.meteorology.eurac; + +import org.apache.camel.builder.RouteBuilder; +import org.springframework.stereotype.Component; + +@Component +public class Route extends RouteBuilder { + + private String stationsUrl = "https://edp-portal.eurac.edu/envdb/metadata"; + private String dailyUrl = "https://edp-portal.eurac.edu/envdb/climate_daily?id=eq.%STATION_ID%&select=date,tmin,tmax,tmean,prec"; + private String monthlyUrl = "https://edp-portal.eurac.edu/envdb/climatologies?order=id"; + + private String odhStations = "https://mobility.api.dev.testingmachine.eu/v2/flat,node/MeteoStation?where=and(sorigin.eq.EURAC,sactive.eq.true)&select=smetadata.id"; + + @Override + public void configure() { + // stations + from("cron:stations?schedule={{env:CRON_STATIONS}}") + .routeId("meteorology.eurac.stations") + .to(stationsUrl) + .removeHeaders("*") + .process(e -> { + log.info("Stations..."); + e.getMessage().setHeader("route_key", "stations"); + }) + .to("direct:mq"); + + // monthly + from("cron:monthly?schedule={{env:CRON_MONTHLY}}") + .routeId("meteorology.eurac.monthly") + .to(monthlyUrl) + .removeHeaders("*") + .process(e -> { + log.info("Monthly..."); + e.getMessage().setHeader("route_key", "monthly"); + }) + .to("direct:mq"); + + // // daily + // from("cron:tab?schedule={{env:CRON_DAILY}}") + // .routeId("meteorology.eurac.daily") + // .to(odhStations) + // .to(dailyUrl) + // .removeHeaders("*") + // .to("direct:mq"); + } +} diff --git a/collectors/meteorology-eurac/src/main/resources/application.properties b/collectors/meteorology-eurac/src/main/resources/application.properties new file mode 100644 index 0000000..f559b08 --- /dev/null +++ b/collectors/meteorology-eurac/src/main/resources/application.properties @@ -0,0 +1,7 @@ +management.health.probes.enabled=true + +ingress.provider=${INGRESS_PROVIDER} +ingress.rabbitmq.cluster=${INGRESS_RABBITMQ_CLUSTER} +ingress.rabbitmq.user=${INGRESS_RABBITMQ_USER} +ingress.rabbitmq.pass=${INGRESS_RABBITMQ_PASS} +ingress.rabbitmq.clientname=${INGRESS_RABBITMQ_CLIENTNAME} \ No newline at end of file diff --git a/transformers/meteorology-eurac/.env.example b/transformers/meteorology-eurac/.env.example new file mode 100644 index 0000000..23b8408 --- /dev/null +++ b/transformers/meteorology-eurac/.env.example @@ -0,0 +1,40 @@ +# Alperia Config +provenance_name=dc-meteorology-eurac-local +app_period=600 + +# Scheduler: 10 seconds with a single job for development +SCHEDULER_CRON=*/10 * * * * * +SCHEDULER_POOL_SIZE=1 + +# ODH Core Writer Connection +BASE_URI=https://mobility.share.opendatahub.testingmachine.eu/json +authorizationUri=https://auth.opendatahub.testingmachine.eu/auth +tokenUri=https://auth.opendatahub.testingmachine.eu/auth/realms/noi/protocol/openid-connect/token +clientId=odh-mobility-datacollector +clientName=odh-mobility-datacollector +scope=openid +clientSecret= + +# Get it from your pom.xml -> project/version and project/artifactId +provenance_name=dc-meteorology-eurac-local +provenance_version=0.0.0 + +# Build, only needed to test infrastructure/docker-compose.build.yml +COMPOSE_PROJECT_NAME=meteorology-eurac +DOCKER_IMAGE=meteorology-eurac-image +DOCKER_TAG=test-1 +ARTIFACT_NAME=dc-meteorology-eurac + +# Logging level and style (text or json) +LOG_LEVEL=debug +LOG_STYLE=text + +MQ_LISTEN_HOST=localhost +MQ_LISTEN_PORT=5672 +MQ_LISTEN_USERNAME=guest +MQ_LISTEN_PASSWORD=guest +MQ_LISTEN_QUEUE=ready-q +MQ_LISTEN_KEY=meteorology.eurac +MQ_LISTEN_ACKTIMEOUT=300000 + +MONGO_CONNECTIONSTRING=mongodb://localhost:27017 \ No newline at end of file diff --git a/transformers/meteorology-eurac/deploy.sh b/transformers/meteorology-eurac/deploy.sh new file mode 100755 index 0000000..90c8924 --- /dev/null +++ b/transformers/meteorology-eurac/deploy.sh @@ -0,0 +1,7 @@ +#!/bin/bash +#docker login ghcr.io +(cd ../lib/rabbit-mongo-listener; mvn clean install) \ +&& mvn clean compile package \ +&& docker build -t ghcr.io/noi-techpark/odh-v2-playground/meteorology-eurac:0.1.0 . -f infrastructure/docker/Dockerfile \ +&& docker image push ghcr.io/noi-techpark/odh-v2-playground/meteorology-eurac:0.1.0 \ +&& helm upgrade --install tr-meteorology-eurac ../../helm/generic-collector --values infrastructure/helm/values.yaml --set-string podAnnotations.releaseTime=$RELEASETIME \ No newline at end of file diff --git a/transformers/meteorology-eurac/docker-compose.yml b/transformers/meteorology-eurac/docker-compose.yml new file mode 100644 index 0000000..e8d4207 --- /dev/null +++ b/transformers/meteorology-eurac/docker-compose.yml @@ -0,0 +1,27 @@ +# SPDX-FileCopyrightText: NOI Techpark +# +# SPDX-License-Identifier: CC0-1.0 + +version: "3.4" + +services: + app: + image: maven:3-openjdk-17-slim + env_file: + - .env + environment: + MAVEN_CONFIG: /var/maven/.m2 + MAVEN_OPTS: -Dspring-boot.run.jvmArguments="-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:9000" + command: > + mvn + -Duser.home=/var/maven + spring-boot:run + ports: + - 9000:9000 + - 9001:9001 + volumes: + - ~/.m2/:/var/maven/.m2 + - ./:/code + working_dir: /code + tty: true + network_mode: host diff --git a/transformers/meteorology-eurac/infrastructure/docker-compose.build.yml b/transformers/meteorology-eurac/infrastructure/docker-compose.build.yml new file mode 100644 index 0000000..93d36f0 --- /dev/null +++ b/transformers/meteorology-eurac/infrastructure/docker-compose.build.yml @@ -0,0 +1,8 @@ +version: "3.4" + +services: + app: + image: ${DOCKER_IMAGE}:${DOCKER_TAG} + build: + context: ../ + dockerfile: infrastructure/docker/Dockerfile diff --git a/transformers/meteorology-eurac/infrastructure/docker/Dockerfile b/transformers/meteorology-eurac/infrastructure/docker/Dockerfile new file mode 100644 index 0000000..135c5c4 --- /dev/null +++ b/transformers/meteorology-eurac/infrastructure/docker/Dockerfile @@ -0,0 +1,5 @@ +FROM eclipse-temurin:17-jre-alpine +WORKDIR /app +COPY target/app.jar app.jar +ENTRYPOINT [ "java", "-jar", "app.jar"] + diff --git a/transformers/meteorology-eurac/infrastructure/helm/values.yaml b/transformers/meteorology-eurac/infrastructure/helm/values.yaml new file mode 100644 index 0000000..946f33d --- /dev/null +++ b/transformers/meteorology-eurac/infrastructure/helm/values.yaml @@ -0,0 +1,44 @@ +# SPDX-FileCopyrightText: NOI Techpark harging-alperia +# +# SPDX-License-Identifier: CC0-1.0 + +nameOverride: tr-meteorology-eurac +fullnameOverride: tr-meteorology-eurac +image: + repository: ghcr.io/noi-techpark/odh-v2-playground/meteorology-eurac-tr + pullPolicy: Always + tag: "0.2.0" + +imagePullSecrets: + - name: ghcr-odh-v2-playground-readonly + +env: + app_dataOrigin: EURAC + app_period: 600 + + # ODH Core Writer Connection + BASE_URI: http://bdp-core.default.svc.cluster.local:8080/json + authorizationUri: https://auth.opendatahub.testingmachine.eu/auth + tokenUri: https://auth.opendatahub.testingmachine.eu/auth/realms/noi/protocol/openid-connect/token + clientId: odh-mobility-datacollector-development + clientName: odh-mobility-datacollector-development + scope: openid + clientSecret: 7bd46f8f-c296-416d-a13d-dc81e68d0830 + + # Get it from your pom.xml -> project/version and project/artifactId + provenance_name: dc-meteorology-eurac-local + provenance_version: 0.0.0 + + # Logging level and style (text or json) + LOG_LEVEL: debug + LOG_STYLE: text + + MQ_LISTEN_HOST: rabbitmq-headless.default.svc.cluster.local + MQ_LISTEN_PORT: 5672 + MQ_LISTEN_KEY: meteorology.eurac + MQ_LISTEN_USERNAME: guest + MQ_LISTEN_PASSWORD: guest + MQ_LISTEN_QUEUE: meteorology.eurac + MQ_LISTEN_ACKTIMEOUT: 300000 + + MONGO_CONNECTIONSTRING: mongodb://mongodb-headless.default.svc.cluster.local:27017 \ No newline at end of file diff --git a/transformers/meteorology-eurac/pom.xml b/transformers/meteorology-eurac/pom.xml new file mode 100644 index 0000000..ae93df1 --- /dev/null +++ b/transformers/meteorology-eurac/pom.xml @@ -0,0 +1,89 @@ + + + + + + 4.0.0 + com.opendatahub + dc-meteorology-eurac + 3.0.0 + Meteorology Eurac transformer + + + 17 + UTF-8 + 17 + 17 + app + + + + org.springframework.boot + spring-boot-starter-parent + 2.7.14 + + + + + + maven-repo.opendatahub.com + https://maven-repo.opendatahub.com/release + + + + + + com.opendatahub.transformer.lib + listener + 1.0 + + + + org.springframework.boot + spring-boot-starter + + + + org.springframework.boot + spring-boot-starter-test + test + + + + org.springframework.boot + spring-boot-starter-amqp + + + + org.springframework.boot + spring-boot-starter-data-mongodb + + + + org.apache.httpcomponents + httpclient + 4.5.14 + + + + it.bz.idm.bdp + dc-interface + 7.4.0 + + + + + + + org.springframework.boot + spring-boot-maven-plugin + + + + ${finalName} + + diff --git a/transformers/meteorology-eurac/src/main/java/com/opendatahub/tr/meteorology/eurac/Main.java b/transformers/meteorology-eurac/src/main/java/com/opendatahub/tr/meteorology/eurac/Main.java new file mode 100644 index 0000000..019af6d --- /dev/null +++ b/transformers/meteorology-eurac/src/main/java/com/opendatahub/tr/meteorology/eurac/Main.java @@ -0,0 +1,20 @@ +// SPDX-FileCopyrightText: NOI Techpark +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package com.opendatahub.tr.meteorology.eurac; + + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.scheduling.annotation.EnableScheduling; + +@EnableScheduling +@ComponentScan({"com.opendatahub.tr.meteorology.eurac", "it.bz.idm.bdp"}) +@SpringBootApplication +public class Main { + public static void main(String[] args) { + SpringApplication.run(Main.class, args); + } +} diff --git a/transformers/meteorology-eurac/src/main/java/com/opendatahub/tr/meteorology/eurac/OdhClient.java b/transformers/meteorology-eurac/src/main/java/com/opendatahub/tr/meteorology/eurac/OdhClient.java new file mode 100644 index 0000000..2a14501 --- /dev/null +++ b/transformers/meteorology-eurac/src/main/java/com/opendatahub/tr/meteorology/eurac/OdhClient.java @@ -0,0 +1,51 @@ +// SPDX-FileCopyrightText: NOI Techpark +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package com.opendatahub.tr.meteorology.eurac; + +import it.bz.idm.bdp.dto.DataMapDto; +import it.bz.idm.bdp.dto.ProvenanceDto; +import it.bz.idm.bdp.dto.RecordDtoImpl; +import it.bz.idm.bdp.json.NonBlockingJSONPusher; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Service; + +@Lazy +@Service +public class OdhClient extends NonBlockingJSONPusher { + + @Value("${odh_client.stationtype}") + private String stationtype; + + @Value("${odh_client.provenance.name}") + private String provenanceName; + + @Value("${odh_client.provenance.version}") + private String provenanceVersion; + + @Value("${odh_client.provenance.origin}") + private String provenanceOrigin; + + @Override + public DataMapDto mapData(T data) { + /* You can ignore this legacy method call */ + return null; + } + + @Override + public String initIntegreenTypology() { + return stationtype; + } + + @Override + public ProvenanceDto defineProvenance() { + return new ProvenanceDto(null, provenanceName, provenanceVersion, provenanceOrigin); + } + + public ProvenanceDto getProvenance() { + return provenance; + } + +} diff --git a/transformers/meteorology-eurac/src/main/java/com/opendatahub/tr/meteorology/eurac/Transformer.java b/transformers/meteorology-eurac/src/main/java/com/opendatahub/tr/meteorology/eurac/Transformer.java new file mode 100644 index 0000000..da9a394 --- /dev/null +++ b/transformers/meteorology-eurac/src/main/java/com/opendatahub/tr/meteorology/eurac/Transformer.java @@ -0,0 +1,115 @@ +// SPDX-FileCopyrightText: NOI Techpark +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package com.opendatahub.tr.meteorology.eurac; + +import java.util.ArrayList; +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Service; +import org.springframework.web.reactive.function.client.WebClientRequestException; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.opendatahub.tr.meteorology.eurac.dto.ClimatologyDto; +import com.opendatahub.tr.meteorology.eurac.dto.MetadataDto; +import com.opendatahub.transformer.lib.listener.MongoService; +import com.opendatahub.transformer.lib.listener.MsgDto; +import com.opendatahub.transformer.lib.listener.TransformerListener; + +import it.bz.idm.bdp.dto.DataMapDto; +import it.bz.idm.bdp.dto.DataTypeDto; +import it.bz.idm.bdp.dto.RecordDtoImpl; +import it.bz.idm.bdp.dto.StationDto; +import it.bz.idm.bdp.dto.StationList; + +@Service +public class Transformer { + private static final Logger log = LoggerFactory.getLogger(Transformer.class); + + private static final String STATION_ID_PREFIX = "EURAC_"; + + private static final String DATATYPE_ID_TMIN = "air-temperature-min"; + private static final String DATATYPE_ID_TMAX = "air-temperature-max"; + private static final String DATATYPE_ID_TMEAN = "air-temperature"; + private static final String DATATYPE_ID_PREC = "precipitation"; + + @Autowired + private ObjectMapper mapper; + + @Lazy + @Autowired + private OdhClient odhClient; + + @Autowired + private MongoService mongo; + + @TransformerListener + public void listen(String msgPayload) throws Exception { + log.info("Message Payload: {}", msgPayload); + MsgDto msg = mapper.readValue(msgPayload, MsgDto.class); + log.debug("received new event: {}", msg); + String raw = mongo.getRawPayload(msg.db, msg.collection, msg.id); + log.debug("Raw data from db: {}", raw); + syncAll(raw); + } + + public void syncAll(String raw) throws Exception { + log.info("Sync: Fetching from source"); + + MetadataDto[] euracStations = mapper.readValue(raw, MetadataDto[].class); + + List odhDataTypeList = new ArrayList<>(); + + odhDataTypeList.add(new DataTypeDto(DATATYPE_ID_TMAX, "°C", "Maximum temperature", "max")); + odhDataTypeList.add(new DataTypeDto(DATATYPE_ID_TMEAN, "°C", "Mean temperature", "mean")); + odhDataTypeList.add(new DataTypeDto(DATATYPE_ID_PREC, "mm", "Precipitation", "total")); + + StationList odhStationList = new StationList(); + for (MetadataDto s : euracStations) { + if (s.getIdSource() == null) { // id_source is null, we have to create a new station + StationDto station = new StationDto(getStationIdNOI(s), s.getName(), s.getLat(), s.getLon()); + + station.setOrigin(odhClient.getProvenance().getLineage()); + station.setElevation(s.getEle()); + + // As an exception, we add ID to the map because we need it for other methods, + // It is not in the Map by default + s.setOtherField("id", s.getId()); + station.setMetaData(s.getOtherFields()); + + odhStationList.add(station); + } + } + + try { + odhClient.syncStations(odhStationList); + odhClient.syncDataTypes(odhDataTypeList); + log.info("Cron job for stations successful"); + } catch (WebClientRequestException e) { + log.error("Cron job for stations failed: Request exception: {}", e.getMessage()); + } + log.info("Sync Data DONE!"); + } + + private String getStationIdNOI(MetadataDto euracStation) { + return getStationIdNOI(euracStation.getIdSource(), euracStation.getId()); + } + + private String getStationIdNOI(ClimatologyDto climatology) { + return getStationIdNOI(climatology.getIdSource(), climatology.getId()); + } + + private String getStationIdNOI(String stationIdSource, int stationId) { + if (stationIdSource != null) { + return stationIdSource; + } else { + return STATION_ID_PREFIX + stationId; + } + } +} diff --git a/transformers/meteorology-eurac/src/main/java/com/opendatahub/tr/meteorology/eurac/dto/ClimateDailyDto.java b/transformers/meteorology-eurac/src/main/java/com/opendatahub/tr/meteorology/eurac/dto/ClimateDailyDto.java new file mode 100644 index 0000000..990cd27 --- /dev/null +++ b/transformers/meteorology-eurac/src/main/java/com/opendatahub/tr/meteorology/eurac/dto/ClimateDailyDto.java @@ -0,0 +1,61 @@ +// SPDX-FileCopyrightText: NOI Techpark +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package com.opendatahub.tr.meteorology.eurac.dto; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + +@JsonIgnoreProperties(ignoreUnknown = true) +public class ClimateDailyDto { + + private String date; + + private Double tmin; + + private Double tmax; + + private Double tmean; + + private Double prec; + + public Double getTmin() { + return tmin; + } + + public void setTmin(Double tmin) { + this.tmin = tmin; + } + + public Double getTmax() { + return tmax; + } + + public void setTmax(Double tmax) { + this.tmax = tmax; + } + + public Double getTmean() { + return tmean; + } + + public void setTmean(Double tmean) { + this.tmean = tmean; + } + + public Double getPrec() { + return prec; + } + + public void setPrec(Double prec) { + this.prec = prec; + } + + public String getDate() { + return date; + } + + public void setDate(String date) { + this.date = date; + } +} diff --git a/transformers/meteorology-eurac/src/main/java/com/opendatahub/tr/meteorology/eurac/dto/ClimatologyDto.java b/transformers/meteorology-eurac/src/main/java/com/opendatahub/tr/meteorology/eurac/dto/ClimatologyDto.java new file mode 100644 index 0000000..3291eef --- /dev/null +++ b/transformers/meteorology-eurac/src/main/java/com/opendatahub/tr/meteorology/eurac/dto/ClimatologyDto.java @@ -0,0 +1,93 @@ +// SPDX-FileCopyrightText: NOI Techpark +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package com.opendatahub.tr.meteorology.eurac.dto; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +@JsonIgnoreProperties(ignoreUnknown = true) +public class ClimatologyDto { + + private int month; + + private Double tmin; + + private Double tmax; + + private Double tmean; + + private Double prec; + + private int id; + + private String station; + + @JsonProperty("id_source") + private String idSource; + + public int getId() { + return id; + } + + public void setId(int id) { + this.id = id; + } + + public int getMonth() { + return month; + } + + public void setMonth(int month) { + this.month = month; + } + + public Double getTmin() { + return tmin; + } + + public void setTmin(Double tmin) { + this.tmin = tmin; + } + + public Double getTmax() { + return tmax; + } + + public void setTmax(Double tmax) { + this.tmax = tmax; + } + + public Double getTmean() { + return tmean; + } + + public void setTmean(Double tmean) { + this.tmean = tmean; + } + + public Double getPrec() { + return prec; + } + + public void setPrec(Double prec) { + this.prec = prec; + } + + public String getStation() { + return station; + } + + public void setStation(String station) { + this.station = station; + } + + public String getIdSource() { + return idSource; + } + + public void setIdSource(String idSource) { + this.idSource = idSource; + } +} diff --git a/transformers/meteorology-eurac/src/main/java/com/opendatahub/tr/meteorology/eurac/dto/MetadataDto.java b/transformers/meteorology-eurac/src/main/java/com/opendatahub/tr/meteorology/eurac/dto/MetadataDto.java new file mode 100644 index 0000000..9ad6190 --- /dev/null +++ b/transformers/meteorology-eurac/src/main/java/com/opendatahub/tr/meteorology/eurac/dto/MetadataDto.java @@ -0,0 +1,89 @@ +// SPDX-FileCopyrightText: NOI Techpark +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package com.opendatahub.tr.meteorology.eurac.dto; + +import java.util.HashMap; +import java.util.Map; + +import com.fasterxml.jackson.annotation.JsonAnyGetter; +import com.fasterxml.jackson.annotation.JsonAnySetter; +import com.fasterxml.jackson.annotation.JsonProperty; + +public class MetadataDto { + + private Double lat; + + private Double lon; + + private String name; + + @JsonProperty("id_source") + private String idSource; + + private Double ele; + + private int id; + + private Map otherFields = new HashMap<>(); + + public Double getLat() { + return lat; + } + + public void setLat(Double lat) { + this.lat = lat; + } + + public Double getLon() { + return lon; + } + + public void setLon(Double lon) { + this.lon = lon; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public int getId() { + return id; + } + + public void setId(int id) { + this.id = id; + } + + public String getIdSource() { + return idSource; + } + + public void setIdSource(String idSource) { + this.idSource = idSource; + } + + public Double getEle() { + return ele; + } + + public void setEle(Double ele) { + this.ele = ele; + } + + // Capture all other fields that Jackson do not match + @JsonAnyGetter + public Map getOtherFields() { + return otherFields; + } + + @JsonAnySetter + public void setOtherField(String name, Object value) { + otherFields.put(name, value); + } +} \ No newline at end of file diff --git a/transformers/meteorology-eurac/src/main/resources/application.properties b/transformers/meteorology-eurac/src/main/resources/application.properties new file mode 100644 index 0000000..4ed0488 --- /dev/null +++ b/transformers/meteorology-eurac/src/main/resources/application.properties @@ -0,0 +1,39 @@ +# dc-interface configuration (Writer API endpoint) +bdp_host= +bdp_port=0 +bdp_endpoint= + +endpoint_host= +endpoint_port= +endpoint_ssl=yes +endpoint_path= +app_callerId= +app_dataOrigin= +app_period= + +##### Open Data Hub Configuration +# Data provenance (Where does our data come from?) +odh_client.stationtype=${ODH_CLIENT_STATIONTYPE:MeteoStation} +odh_client.provenance.name=${ODH_CLIENT_PROVENANCE_NAME:dc-meteorology-eurac} +odh_client.provenance.version=${ODH_CLIENT_PROVENANCE_VERSION:0.0.0-local-dev} +odh_client.provenance.origin=${ODH_CLIENT_PROVENANCE_ORIGIN:EURAC} +odh_client.period.climatology=${ODH_CLIENT_PERIOD_CLIMATOLOGY:31536000} +odh_client.period.climateDaily=${ODH_CLIENT_PERIOD_CLIMATEDAILY:86400} + +spring.main.web-application-type=NONE + +# Listen queue rabbitmq coordinates +mq.listen.host=${MQ_LISTEN_HOST} +mq.listen.port=${MQ_LISTEN_PORT} +mq.listen.username=${MQ_LISTEN_USERNAME} +mq.listen.password=${MQ_LISTEN_PASSWORD} +# Exchange to which the listen queue is bound. should always be "routed" +mq.listen.exchange=${MQ_LISTEN_EXCHANGE:routed} +# Routing key for listen queue. Should be db.collection as provided by the data collector +mq.listen.key=${MQ_LISTEN_KEY} +# Name of queue to listen on. Should be db.collection as provided by the data collector +mq.listen.queue=${MQ_LISTEN_QUEUE} +# Timeout for listen queue in ms. That is how much time do we have to consume the message and ACK/NACK it +mq.listen.acktimeout=${MQ_LISTEN_ACKTIMEOUT:300000} + +mongo.connectionString=${MONGO_CONNECTIONSTRING}