Merge pull request #48 from SocialGouv/feat/dockerfiles

feat: docker environnement for ELK and NextJS + CI workflow

.env.example

@@ -5,4 +5,15 @@ ELASTIC_PASSWORD=elastic_password
 KIBANA_PASSWORD=kibana_password
 
 # Set the cluster name
-CLUSTER_NAME=docker-cluster
+CLUSTER_NAME=docker-cluster
+
+# ElasticApi key name
+NEXT_PUBLIC_ELASTIC_API_KEY_NAME=cm2d_api_key
+
+# AWS SES credentials
+AWS_ACCESS_KEY_ID=your_access_key_id_here
+AWS_SECRET_ACCESS_KEY=your_secret_access_key_here
+AWS_REGION=your_aws_region_here
+
+# The email address that sends the email
+EMAIL_SOURCE=your_email_source_here

.github/workflows/build.yml

@@ -0,0 +1,103 @@
+name: CI
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+
+jobs:
+  elk-check-files-changed:
+    runs-on: ubuntu-latest
+
+    outputs:
+      files-changed-elasticsearch: ${{ steps.set-output.outputs.files-changed-elasticsearch }}
+      files-changed-kibana: ${{ steps.set-output.outputs.files-changed-kibana }}
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Get changed files in docker for elasticsearch
+        id: check-files-elasticsearch
+        uses: tj-actions/changed-files@v37
+        with:
+          files: docker/elasticsearch
+
+      - name: Get changed files in docker for kibana
+        id: check-files-kibana
+        uses: tj-actions/changed-files@v37
+        with:
+          files: docker/kibana
+
+      - name: Set output if files changed in docker for elasticsearch and kibana
+        id: set-output
+        run: |
+          echo "files-changed-elasticsearch=${{ steps.check-files-elasticsearch.outputs.any_changed }}" >> $GITHUB_OUTPUT
+          echo "files-changed-kibana=${{ steps.check-files-kibana.outputs.any_changed }}" >> $GITHUB_OUTPUT
+
+  elasticsearch-build:
+    needs: elk-check-files-changed
+    if: ${{ needs.elk-check-files-changed.outputs.files-changed-elasticsearch == 'true' }}
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push
+        uses: docker/build-push-action@v3
+        with:
+          context: "{{defaultContext}}:docker/elasticsearch"
+          push: true
+          tags: cm2d/cm2d-elasticsearch:${{ github.sha }},cm2d/cm2d-elasticsearch:latest
+
+  kibana-build:
+    needs: elk-check-files-changed
+    if: ${{ needs.elk-check-files-changed.outputs.files-changed-kibana == 'true' }}
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push
+        uses: docker/build-push-action@v3
+        with:
+          context: "{{defaultContext}}:docker/kibana"
+          push: true
+          tags: cm2d/cm2d-kibana:${{ github.sha }},cm2d/cm2d-kibana:latest
+
+  webapp-build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push
+        uses: docker/build-push-action@v3
+        with:
+          context: "{{defaultContext}}:webapp-next"
+          push: true
+          tags: ghcr.io/socialgouv/cm2d-webapp:${{ github.sha }},ghcr.io/socialgouv/cm2d-webapp:latest

.gitignore

@@ -4,4 +4,5 @@ faker/sample_data.csv
 faker/.idea
 certificates/*
 sample_data.csv
-.env
+.env
+node_modules/

README.md

@@ -16,18 +16,17 @@ Pour démarrer la suite ELK localement :
 docker compose up -d
 ```
 
-Au premier run ELK, lancez cette suite de commande
+Pour initialiser les variables d'environnement NextJS
 
 ```
-mkdir certificates
-docker cp elasticsearch:/usr/share/elasticsearch/config/certs/ca/ca.crt ./certificates/ca.crt
+cd webapp-next
+cp .env.example .env
 ```
 
-Pour initialiser les variables d'environnement NextJS
+Au premier run ELK, lancez cette commande pour initialiser le mot de passe du user "kibana_system" (remplacer {ELASTIC_PASSWORD} et {KIBANA_PASSWORD} par les mots de passe de votre environnement) :
 
 ```
-cd webapp-next
-cp .env.example .env
+docker exec elasticsearch curl -s -X POST --cacert config/certs/ca/ca.crt -u "elastic:{ELASTIC_PASSWORD}" -H "Content-Type: application/json" https://elasticsearch:9200/_security/user/kibana_system/_password -d "{\"password\":\"{KIBANA_PASSWORD}\"}"
 ```
 
 Pour démarrer le frontend NextJS localement :
@@ -82,75 +81,14 @@ Il est maintenant possible de se connecter en utilisant l'adresse email et le mo
 
 ## Initialisation de l'environnement ELK
 
-### Indexation des certificats
-
-Création de l'index principal destiné à rassembler les informations relatives aux certificats.
-
-Rendez-vous dans "Management" > "Dev Tools" et lancez la requête suivante :
-
-```
-PUT /cm2d_certificate
-{
-  "mappings": {
-    "_meta": {
-      "created_by": "curl-user"
-    },
-    "properties": {
-      "@timestamp": {
-        "type": "date"
-      },
-      "age": {
-        "type": "long"
-      },
-      "categories_level_1": {
-        "type": "keyword"
-      },
-      "categories_level_2": {
-        "type": "keyword"
-      },
-      "coordinates": {
-        "type": "keyword"
-      },
-      "date": {
-        "type": "date",
-        "format": "iso8601"
-      },
-      "death_location": {
-        "type": "keyword"
-      },
-      "department": {
-        "type": "long"
-      },
-      "home_location": {
-        "type": "keyword"
-      },
-      "kind": {
-        "type": "keyword"
-      },
-      "sex": {
-        "type": "keyword"
-      }
-    }
-  }
-}'
-```
-
-### Index pour les attributs supplémentaires des utilisateurs
-
-Pour stocker des informations supplémentaires concernant les utilisateurs CM2D, nous devons créer un index dédié.
-
-Rendez-vous dans "Management" > "Dev Tools" et lancez la requête suivante :
-
-```
-PUT /cm2d_users
-{
-  "mappings": {
-    "properties": {
-      "username": { "type": "text" },
-      "versionCGU": { "type": "text" }
-    }
-  }
-}
+### Indexation des certificats et des utilisateurs
+
+```
+docker run --net=host --rm -ti -e NODE_TLS_REJECT_UNAUTHORIZED=0 -v ./default-indexes:/tmp --entrypoint multielasticdump elasticdump/elasticsearch-dump \
+  --direction=load \
+  --input=./tmp \
+  --output="https://elastic:${ELASTIC_PASSWORD}@localhost:9200" \
+  --tlsAuth
 ```
 
 ### Mise en place des transformations
@@ -241,3 +179,40 @@ Destination Index : `cm2d_departments`
 Continous mode
 Date field for continous mode : `@timestamp`
 Delay : `60s`
+
+
+
+### Docker production
+
+## Créer les images docker
+```
+docker build -t cm2d-elasticsearch docker/elasticsearch
+docker build -t cm2d-kibana docker/kibana
+docker build --build-arg NEXT_PUBLIC_ELASTIC_API_KEY_NAME=${NEXT_PUBLIC_ELASTIC_API_KEY_NAME} -t cm2d-webapp webapp-next
+```
+
+## Créer les réseaux docker
+```
+docker network create elastic
+docker network create webapp
+```
+
+## Elasticsearch
+```
+docker run -d -p 9200:9200 -p 9300:9300 --net elastic -v es_data:/usr/share/elasticsearch/data -v certs:/usr/share/elasticsearch/config/certs -e ELASTIC_PASSWORD=${ELASTIC_PASSWORD} --name elasticsearch cm2d-elasticsearch
+```
+
+## Attacher le réseau webapp à Elasticsearch
+```
+docker network connect webapp elasticsearch
+```
+
+## Kibana
+```
+docker run -d -p 5601:5601 --net elastic -v kibana_data:/usr/share/kibana/data -v certs:/usr/share/kibana/config/certs -e ELASTICSEARCH_PASSWORD=${KIBANA_PASSWORD} --name kibana cm2d-kibana
+```
+
+## Webapp
+```
+docker run -d -p 3000:3000 --net webapp -v certs:/app/certs --env-file ${path_fichier_environnement} --name webapp cm2d-webapp
+```

default-indexes/cm2d_certificate.mapping.json

@@ -0,0 +1 @@
+{"cm2d_certificate":{"mappings":{"_meta":{"created_by":"file-data-visualizer"},"properties":{"@timestamp":{"type":"date"},"age":{"type":"long"},"categories_level_1":{"type":"keyword"},"categories_level_2":{"type":"keyword"},"coordinates":{"type":"keyword"},"date":{"type":"date","format":"iso8601"},"death_location":{"type":"keyword"},"department":{"type":"long"},"home_location":{"type":"keyword"},"kind":{"type":"keyword"},"sex":{"type":"keyword"}}}}}

default-indexes/cm2d_certificate.settings.json

@@ -0,0 +1 @@
+"{\"cm2d_certificate\":{\"settings\":{\"index\":{\"routing\":{\"allocation\":{\"include\":{\"_tier_preference\":\"data_content\"}}},\"number_of_shards\":\"1\",\"number_of_replicas\":\"1\"}}}}"

default-indexes/cm2d_certificate.template.json

@@ -0,0 +1 @@
+"{}"

default-indexes/cm2d_users.mapping.json

@@ -0,0 +1 @@
+{"cm2d_users":{"mappings":{"properties":{"username":{"type":"text"},"versionCGU":{"type":"text"}}}}}

default-indexes/cm2d_users.settings.json

@@ -0,0 +1 @@
+"{\"cm2d_users\":{\"settings\":{\"index\":{\"routing\":{\"allocation\":{\"include\":{\"_tier_preference\":\"data_content\"}}},\"number_of_shards\":\"1\",\"number_of_replicas\":\"1\"}}}}"

default-indexes/cm2d_users.template.json

@@ -0,0 +1 @@
+"{}"