dc-lorawan, new collector + workflow for testing

.github/workflows/dc-lorawan.yml

@@ -0,0 +1,63 @@
+name: CI/CD dc-lorawan
+
+on: 
+  push:
+    paths:
+      - "collectors/lorawan/**"
+      - "!collectors/lorawan/infrastructure/helm/*"
+      - "collectors/lorawan/infrastructure/helm/values.yaml"
+      - ".github/workflows/dc-lorawan.yml"
+
+env:
+  WORKING_DIRECTORY: collectors/lorawan
+  DOCKER_IMAGE: ghcr.io/noi-techpark/opendatahub-collectors/dc-lorawan
+  DOCKER_TAG: ${{ github.sha }}
+  KUBERNETES_NAMESPACE: collector
+  VALUES_YAML: infrastructure/helm/values.yaml
+  K8S_NAME: dc-lorawan
+
+jobs:
+  build:
+    runs-on: ubuntu-22.04
+    concurrency: dc-rest-poller-lorawan-build
+    steps:
+    - name: Checkout source code
+      uses: actions/checkout@v4
+
+    - name: Build and push images
+      uses: noi-techpark/github-actions/docker-build-and-push@v2
+      with:
+        working-directory: ${{ env.WORKING_DIRECTORY }}/infrastructure
+        docker-username: ${{ github.actor }}
+        docker-password: ${{ secrets.GITHUB_TOKEN }}
+
+  deploy-lorawan-test:
+    if: github.ref == 'refs/heads/main'
+    needs: build
+    runs-on: ubuntu-22.04
+    concurrency: dc-rest-poller-lorawan-test
+    environment: test
+    steps:
+      - name: Checkout source code
+        uses: actions/checkout@v4
+
+      - name: Customize values.yaml
+        working-directory: ${{ env.WORKING_DIRECTORY }}
+        run: |
+          yq -i '.image.repository = "${{ env.DOCKER_IMAGE }}"' ${{ env.VALUES_YAML }}
+          yq -i '.image.tag = "${{ env.DOCKER_TAG }}"' ${{ env.VALUES_YAML }}
+          yq -i '.image.pullPolicy = "IfNotPresent"' ${{ env.VALUES_YAML }}
+          yq -i '.envSecret.LORAWAN_PASSWORD = ${{ secrets.LORAWAN_PASSWORD }}"' ${{ env.VALUES_YAML }}
+      
+
+      - name: Deploy on cluster  
+        uses: noi-techpark/github-actions/helm-deploy@v2
+        with:
+          k8s-name: dc-lorawan
+          k8s-namespace: collector
+          chart-path: helm/generic-collector
+          values-file: ${{ env.WORKING_DIRECTORY}}/${{ env.VALUES_YAML }}
+          aws-access-key-id: ${{ secrets[vars.AWS_KEY_ID] }}
+          aws-secret-access-key: ${{ secrets[vars.AWS_KEY_SECRET] }}
+          aws-eks-cluster-name: aws-main-eu-01
+          aws-region: eu-west-1

collectors/lorawan/.env.example

@@ -0,0 +1,27 @@
+# SPDX-FileCopyrightText: 2024 NOI Techpark <[email protected]>
+#
+# SPDX-License-Identifier: CC0-1.0
+
+MQ_URI=amqp://guest:guest@rabbitmq
+MQ_CLIENT=dc-mqtt-client-dev # identify your datacollector to rabbitmq
+MQ_EXCHANGE=ingress # defaults to ingress
+LOGLEVEL=DEBUG
+PROVIDER=test/mqtt
+
+# Cron polling schedule, starting with seconds
+CRON="* * * * * *"
+# Endpoint URL
+HTTP_URL='https://echo.free.beeceptor.com/test-path?queryparam=queryval'
+
+PAGING_PARAM_TYPE=query
+PAGING_SIZE=200
+PAGING_LIMIT_NAME=limit
+PAGING_OFFSET_NAME=offset
+
+# set custom http headers.
+# must have the prefix 'HTTP_HEADER_' the part after it just has to be unique and is not used for anything
+HTTP_HEADER_TEST1='Authorization: Token abcd1231'
+HTTP_HEADER_ACCEPT='Accept: application/xml'
+
+# interpret response as binary and store as base64. defaults to false
+RAW_BINARY=false

collectors/lorawan/.gitignore

@@ -0,0 +1,5 @@
+.env
+/target
+.vscode
+__debug
+tmp/

collectors/lorawan/README.md

@@ -0,0 +1,10 @@
+<!--
+SPDX-FileCopyrightText: 2024 NOI Techpark <[email protected]>
+
+SPDX-License-Identifier: CC0-1.0
+-->
+
+# REST polling data collector
+Polls an HTTP endpoint on a cron shedule and posts the response body to rabbitmq
+
+For documentation on configuration, refer to the `.env.example` file

collectors/lorawan/docker-compose.yml

@@ -0,0 +1,27 @@
+# SPDX-FileCopyrightText: 2024 NOI Techpark <[email protected]>
+#
+# SPDX-License-Identifier: CC0-1.0
+
+services:
+  app:
+    build:
+      dockerfile: infrastructure/docker/Dockerfile
+      context: . 
+      target: dev
+    env_file:
+      - .env
+    volumes:
+      - ./src:/code
+      - pkg:/go/pkg/mod
+    working_dir: /code
+    network_mode: host
+
+  rabbitmq:
+    extends:
+      file: ../lib/docker-compose/docker-compose.rabbitmq.yml
+      service: rabbitmq
+    attach: false
+    ports:
+      - "15677:15672"
+volumes:
+  pkg:

collectors/lorawan/infrastructure/docker-compose.build.yml

@@ -0,0 +1,11 @@
+# SPDX-FileCopyrightText: 2024 NOI Techpark <[email protected]>
+#
+# SPDX-License-Identifier: CC0-1.0
+
+services:
+  app:
+    image: ${DOCKER_IMAGE}:${DOCKER_TAG}
+    build:
+      context: ../
+      dockerfile: infrastructure/docker/Dockerfile
+      target: build

collectors/lorawan/infrastructure/docker/Dockerfile

@@ -0,0 +1,29 @@
+# SPDX-FileCopyrightText: 2024 NOI Techpark <[email protected]>
+#
+# SPDX-License-Identifier: CC0-1.0
+
+FROM golang:1.23-bookworm AS base
+
+EXPOSE 8080
+
+FROM base AS build-env
+WORKDIR /app
+COPY src/ .
+RUN go mod download
+RUN CGO_ENABLED=0 GOOS=linux go build -o main
+
+# BUILD published image
+FROM alpine:3 AS build
+WORKDIR /app
+COPY --from=build-env /app/main .
+ENTRYPOINT [ "./main"]
+
+# LOCAL DEVELOPMENT
+FROM base AS dev
+WORKDIR /code
+CMD ["go", "run", "."]
+
+# TESTS
+FROM base AS test
+WORKDIR /code
+CMD ["go", "test", "."]

collectors/lorawan/infrastructure/helm/values.yaml

@@ -0,0 +1,20 @@
+image:
+  repository: ghcr.io/noi-techpark/opendatahub-collectors/rest-poller
+  pullPolicy: IfNotPresent
+  tag: "0.2.0" # Set this when upgrading chart with --set-value
+
+env:
+  MQ_CLIENTNAME: dc-lorawan
+  PROVIDER: lorawan/data
+
+  CRON: "0 */2 * * * *"
+  HTTP_URL: "http://saocompute.eurac.edu/sensordb/query?db=db_opendatahub&u=opendatahub&p=%s&q=select%%20*%%20from%%20device_frmpayload_data_message%%20WHERE%%20%%22device_name%%22%%3D%%27%s%%27%%20limit%%201"
+  HTTP_HEADER_ACCEPT: "Accept: application/json"
+
+envSecret:
+  HTTP_HEADER_AUTHORIZATION: ""
+
+envSecretRef:
+  - name: MQ_URI
+    secret: rabbitmq-svcbind
+    key: uri

collectors/lorawan/src/go.mod

@@ -0,0 +1,11 @@
+module opendatahub.com/rest-poller
+
+go 1.22.7
+
+require (
+	github.com/kelseyhightower/envconfig v1.4.0
+	github.com/rabbitmq/amqp091-go v1.10.0
+	github.com/robfig/cron/v3 v3.0.1
+)
+
+require github.com/noi-techpark/go-opendatahub-ingest v0.0.0-20241121092514-13f81a2c1241 // indirect

collectors/lorawan/src/go.sum

@@ -0,0 +1,12 @@
+github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8=
+github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg=
+github.com/noi-techpark/go-opendatahub-ingest v0.0.0-20241119065535-d68a9695ad44 h1:rr/a0Lha81kj2sase8SXQrFrWHeFduUehgMAjiZwRC0=
+github.com/noi-techpark/go-opendatahub-ingest v0.0.0-20241119065535-d68a9695ad44/go.mod h1:OR71Qot9WXWSNwLjNNuhhdIts1KbDfDzbwv2F/Q6KOk=
+github.com/noi-techpark/go-opendatahub-ingest v0.0.0-20241121092514-13f81a2c1241 h1:FI5JAF/RwPZEjcpzC8JQhbSI757SEmL2+ncl0WRCI2A=
+github.com/noi-techpark/go-opendatahub-ingest v0.0.0-20241121092514-13f81a2c1241/go.mod h1:OR71Qot9WXWSNwLjNNuhhdIts1KbDfDzbwv2F/Q6KOk=
+github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw=
+github.com/rabbitmq/amqp091-go v1.10.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o=
+github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
+github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
+go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
+go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=