Merge branch 'meteorology-eurac'

.github/workflows/meteorology-eurac-dc.yml

@@ -0,0 +1,64 @@
+name: CI/CD meteorology-eurac-dc
+
+on: 
+  push:
+    paths:
+      - "collectors/meteorology-eurac/**"
+      - ".github/workflows/meteorology-eurac-dc.yml"
+
+env:
+  AWS_REGION : eu-west-1
+  AWS_EKS_CLUSTER_NAME : aws-main-eu-01
+  AWS_ACCESS_KEY_ID : ${{ secrets.AWS_ACCESS_KEY_ID }}
+  AWS_ACCESS_SECRET_KEY : ${{ secrets.AWS_ACCESS_SECRET_KEY }}
+  JAVA_VERSION: 17
+  WORKING_DIRECTORY: collectors/meteorology-eurac
+  DOCKER_IMAGE: ghcr.io/noi-techpark/odh-v2-playground/meteorology-eurac-dc
+  DOCKER_TAG: 0.0.0
+
+
+jobs:
+  deploy-test:
+    runs-on: ubuntu-latest
+    if: github.ref == 'refs/heads/meteorology-eurac'
+    concurrency: deploy-test-meteorology-eurac-dc
+    steps:
+      - name: Checkout source code
+        uses: actions/checkout@v4
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-access-key-id: ${{ env.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ env.AWS_ACCESS_SECRET_KEY }}
+          aws-region: ${{ env.AWS_REGION }}
+
+      - name: Authenticate to AWS EKS
+        run: aws eks --region ${{ env.AWS_REGION }} update-kubeconfig --name ${{ env.AWS_EKS_CLUSTER_NAME }}
+
+      - name: Build libs
+        uses: noi-techpark/github-actions/maven-build@v2
+        with:
+          working-directory: collectors/lib/ingress-mq
+          java-version: ${{ env.JAVA_VERSION }}
+          build-command: 'mvn clean install'
+
+      - name: Build project
+        uses: noi-techpark/github-actions/maven-build@v2
+        with:
+          working-directory: ${{ env.WORKING_DIRECTORY }}
+          java-version: ${{ env.JAVA_VERSION }}
+          build-command: 'mvn clean install'
+
+      - name: Build and push images
+        uses: noi-techpark/github-actions/docker-build-and-push@v2
+        with:
+          working-directory: ${{ env.WORKING_DIRECTORY }}/infrastructure
+          docker-username: ${{ github.actor }}
+          docker-password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Helm deployment
+        run: |
+            RELEASETIME=`date +%s`
+            cd collectors/meteorology-eurac
+            helm upgrade --install dc-meteorology-eurac ../../helm/generic-collector --values infrastructure/helm/values.yaml --set-string podAnnotations.releaseTime=$RELEASETIME

.github/workflows/meteorology-eurac-tr.yml

@@ -0,0 +1,64 @@
+name: CI/CD meteorology-eurac-tr
+
+on: 
+  push:
+    paths:
+      - "transformers/meteorology-eurac/**"
+      - ".github/workflows/meteorology-eurac-tr.yml"
+
+env:
+  AWS_REGION : eu-west-1
+  AWS_EKS_CLUSTER_NAME : aws-main-eu-01
+  AWS_ACCESS_KEY_ID : ${{ secrets.AWS_ACCESS_KEY_ID }}
+  AWS_ACCESS_SECRET_KEY : ${{ secrets.AWS_ACCESS_SECRET_KEY }}
+  JAVA_VERSION: 17
+  WORKING_DIRECTORY: transformers/meteorology-eurac
+  DOCKER_IMAGE: ghcr.io/noi-techpark/odh-v2-playground/meteorology-eurac-tr
+  DOCKER_TAG: 0.2.0
+
+
+jobs:
+  deploy-test:
+    runs-on: ubuntu-latest
+    if: github.ref == 'refs/heads/meteorology-eurac'
+    concurrency: deploy-test-meteorology-eurac-tr
+    steps:
+      - name: Checkout source code
+        uses: actions/checkout@v4
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-access-key-id: ${{ env.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ env.AWS_ACCESS_SECRET_KEY }}
+          aws-region: ${{ env.AWS_REGION }}
+
+      - name: Authenticate to AWS EKS
+        run: aws eks --region ${{ env.AWS_REGION }} update-kubeconfig --name ${{ env.AWS_EKS_CLUSTER_NAME }}
+
+      - name: Build libs
+        uses: noi-techpark/github-actions/maven-build@v2
+        with:
+          working-directory: transformers/lib/rabbit-mongo-listener
+          java-version: ${{ env.JAVA_VERSION }}
+          build-command: 'mvn clean install'
+
+      - name: Build project
+        uses: noi-techpark/github-actions/maven-build@v2
+        with:
+          working-directory: ${{ env.WORKING_DIRECTORY }}
+          java-version: ${{ env.JAVA_VERSION }}
+          build-command: 'mvn clean install'
+
+      - name: Build and push images
+        uses: noi-techpark/github-actions/docker-build-and-push@v2
+        with:
+          working-directory: ${{ env.WORKING_DIRECTORY }}/infrastructure
+          docker-username: ${{ github.actor }}
+          docker-password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Helm deployment
+        run: |
+            RELEASETIME=`date +%s`
+            cd transformers/meteorology-eurac
+            helm upgrade --install tr-meteorology-eurac ../../helm/generic-collector --values infrastructure/helm/values.yaml --set-string podAnnotations.releaseTime=$RELEASETIME

collectors/lib/ingress-mq/src/main/java/com/opendatahub/collector/lib/ingress/mq/MsgRoute.java

@@ -12,8 +12,8 @@ public class MsgRoute extends RouteBuilder {
     public MsgRoute(String from) {
         this.from = from;
     }
-    
-    public String getRouteUri(){
+
+    public String getRouteUri() {
         return "direct:" + from;
     }
 
@@ -22,7 +22,7 @@ public MsgRoute() {
 
     @Value("${ingress.provider:#{null}}")
     String provider;
-    
+
     @Autowired
     RabbitMQConnection rabbitMQConfig;
 

collectors/meteorology-eurac/.env.example

@@ -0,0 +1,8 @@
+INGRESS_PROVIDER=meteorology-eurac/test
+INGRESS_RABBITMQ_CLUSTER=rabbithost.domain.tld
+INGRESS_RABBITMQ_USER=dc-meteorology-eurac
+INGRESS_RABBITMQ_PASS=*****
+INGRESS_RABBITMQ_CLIENTNAME=meteorology-eurac-datacollector-test
+CRON_STATIONS=0 */2 * * * ?
+CRON_MONTHLY=0 */2 * * * ?
+CRON_DAILY=0 */2 * * * ?

collectors/meteorology-eurac/.gitignore

@@ -0,0 +1,33 @@
+HELP.md
+target/
+!.mvn/wrapper/maven-wrapper.jar
+!**/src/main/**/target/
+!**/src/test/**/target/
+
+### STS ###
+.apt_generated
+.classpath
+.factorypath
+.project
+.settings
+.springBeans
+.sts4-cache
+
+### IntelliJ IDEA ###
+.idea
+*.iws
+*.iml
+*.ipr
+
+### NetBeans ###
+/nbproject/private/
+/nbbuild/
+/dist/
+/nbdist/
+/.nb-gradle/
+build/
+!**/src/main/**/build/
+!**/src/test/**/build/
+
+### VS Code ###
+.vscode/

collectors/meteorology-eurac/deploy.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+#docker login ghcr.io (create token with read/write package permissions in github developer settings)
+RELEASETIME=`date +%s`
+(cd ../lib/ingress-mq; mvn clean install) \
+&& mvn clean install \
+&& docker build -t ghcr.io/noi-techpark/odh-v2-playground/meteorology-eurac:0.0.0 . -f infrastructure/docker/Dockerfile \
+&& docker image push ghcr.io/noi-techpark/odh-v2-playground/meteorology-eurac:0.0.0 \
+&& helm upgrade --install dc-meteorology-eurac ../../helm/generic-collector --values infrastructure/helm/values.yaml --set-string podAnnotations.releaseTime=$RELEASETIME

collectors/meteorology-eurac/docker-compose.yml

@@ -0,0 +1,18 @@
+version: "3"
+
+services:
+  app:
+    image: maven:3-openjdk-17-slim
+    env_file:
+      - .env
+    environment:
+      MAVEN_CONFIG: /var/maven/.m2
+      MAVEN_OPTS: -Dmaven.repo.local=/var/maven/.m2/repository -Dspring-boot.run.jvmArguments="-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:9000"
+    command: >
+      mvn clean spring-boot:run -debug
+    volumes:
+      - ~/.m2/:/var/maven/.m2
+      - ./:/code
+    working_dir: /code
+    tty: true
+    network_mode: host

collectors/meteorology-eurac/infrastructure/docker-compose.build.yml

@@ -0,0 +1,8 @@
+version: "3.4"
+
+services:
+  app:
+    image: ${DOCKER_IMAGE}:${DOCKER_TAG}
+    build:
+      context: ../
+      dockerfile: infrastructure/docker/Dockerfile

collectors/meteorology-eurac/infrastructure/docker/Dockerfile

@@ -0,0 +1,5 @@
+FROM eclipse-temurin:17-jre-alpine
+WORKDIR /app
+COPY target/app.jar app.jar
+ENTRYPOINT [ "java", "-jar", "app.jar"]
+

collectors/meteorology-eurac/infrastructure/helm/values.yaml

@@ -0,0 +1,27 @@
+# SPDX-FileCopyrightText: NOI Techpark <[email protected]>
+#
+# SPDX-License-Identifier: CC0-1.0
+
+nameOverride: dc-meteorology-eurac
+fullnameOverride: dc-meteorology-eurac
+
+image:
+  repository: ghcr.io/noi-techpark/odh-v2-playground/meteorology-eurac-dc
+  pullPolicy: Always
+  tag: "0.0.0" # Set this when upgrading chart with --set-value
+
+imagePullSecrets:
+  - name: ghcr-odh-v2-playground-readonly
+
+env:
+  INGRESS_RABBITMQ_CLUSTER: rabbitmq-0.rabbitmq-headless.default.svc.cluster.local:5672
+  INGRESS_RABBITMQ_CLIENTNAME: meteorology-eurac-datacollector
+  CRON_STATIONS: "0 0 */1 * * ?"
+  CRON_MONTHLY: "0 */5 * * * ?"
+  CRON_DAILY: "0 0 */24 * * ?"
+  INGRESS_PROVIDER: meteorology/eurac
+
+
+envSecret:
+  INGRESS_RABBITMQ_USER: guest
+  INGRESS_RABBITMQ_PASS: guest

collectors/meteorology-eurac/pom.xml

@@ -0,0 +1,68 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>org.springframework.boot</groupId>
+        <artifactId>spring-boot-starter-parent</artifactId>
+        <version>3.1.3</version>
+        <relativePath /> <!-- lookup parent from repository -->
+    </parent>
+    <groupId>com.opendatahub.collector</groupId>
+    <artifactId>dc-meteorology-eurac</artifactId>
+    <version>0.0.1-SNAPSHOT</version>
+    <name>dc-meteorology-eurac</name>
+    <description>Eurac meteorology data collector</description>
+    <properties>
+        <java.version>17</java.version>
+        <camel.version>4.1.0</camel.version>
+    </properties>
+    <dependencies>
+        <dependency>
+            <groupId>com.opendatahub.collector.lib</groupId>
+            <artifactId>ingress-mq</artifactId>
+            <version>1.0</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-test</artifactId>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.camel.springboot</groupId>
+            <artifactId>camel-spring-boot-starter</artifactId>
+            <version>${camel.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.camel</groupId>
+            <artifactId>camel-cron</artifactId>
+            <version>${camel.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.camel</groupId>
+            <artifactId>camel-http</artifactId>
+            <version>${camel.version}</version>
+        </dependency>
+
+    </dependencies>
+
+    <build>
+        <finalName>app</finalName>
+        <plugins>
+            <plugin>
+                <groupId>org.graalvm.buildtools</groupId>
+                <artifactId>native-maven-plugin</artifactId>
+            </plugin>
+            <plugin>
+                <groupId>org.springframework.boot</groupId>
+                <artifactId>spring-boot-maven-plugin</artifactId>
+            </plugin>
+        </plugins>
+    </build>
+
+</project>

collectors/meteorology-eurac/src/main/java/com/opendatahub/collector/meteorology/eurac/MeteorologyEuracApplication.java

@@ -0,0 +1,11 @@
+package com.opendatahub.collector.meteorology.eurac;
+
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+
+@SpringBootApplication
+public class MeteorologyEuracApplication {
+	public static void main(String[] args) {
+		SpringApplication.run(MeteorologyEuracApplication.class, args);
+	}
+}

collectors/meteorology-eurac/src/main/java/com/opendatahub/collector/meteorology/eurac/Route.java

@@ -0,0 +1,47 @@
+package com.opendatahub.collector.meteorology.eurac;
+
+import org.apache.camel.builder.RouteBuilder;
+import org.springframework.stereotype.Component;
+
+@Component
+public class Route extends RouteBuilder {
+
+    private String stationsUrl = "https://edp-portal.eurac.edu/envdb/metadata";
+    private String dailyUrl = "https://edp-portal.eurac.edu/envdb/climate_daily?id=eq.%STATION_ID%&select=date,tmin,tmax,tmean,prec";
+    private String monthlyUrl = "https://edp-portal.eurac.edu/envdb/climatologies?order=id";
+
+    private String odhStations = "https://mobility.api.dev.testingmachine.eu/v2/flat,node/MeteoStation?where=and(sorigin.eq.EURAC,sactive.eq.true)&select=smetadata.id";
+
+    @Override
+    public void configure() {
+        // stations
+        from("cron:stations?schedule={{env:CRON_STATIONS}}")
+                .routeId("meteorology.eurac.stations")
+                .to(stationsUrl)
+                .removeHeaders("*")
+                .process(e -> {
+                    log.info("Stations...");
+                    e.getMessage().setHeader("route_key", "stations");
+                })
+                .to("direct:mq");
+
+        // monthly
+        from("cron:monthly?schedule={{env:CRON_MONTHLY}}")
+                .routeId("meteorology.eurac.monthly")
+                .to(monthlyUrl)
+                .removeHeaders("*")
+                .process(e -> {
+                    log.info("Monthly...");
+                    e.getMessage().setHeader("route_key", "monthly");
+                })
+                .to("direct:mq");
+
+        // // daily
+        // from("cron:tab?schedule={{env:CRON_DAILY}}")
+        // .routeId("meteorology.eurac.daily")
+        // .to(odhStations)
+        // .to(dailyUrl)
+        // .removeHeaders("*")
+        // .to("direct:mq");
+    }
+}