feat: imporve docker compose usage

Signed-off-by: Manuel Ruck <[email protected]>
demokratie-live · Apr 7, 2024 · df12f75 · df12f75
1 parent ef2b6a6
commit df12f75
Show file tree

Hide file tree

Showing 32 changed files with 278 additions and 454 deletions.
diff --git a/.github/workflows/build-and-push.yaml b/.github/workflows/build-and-push.yaml
@@ -29,7 +29,7 @@ jobs:
         uses: actions/checkout@v4
         with:
           fetch-depth: 0
-      - uses: pnpm/action-setup@v2
+      - uses: pnpm/action-setup@v3
         with:
           version: 8
           run_install: false

diff --git a/.github/workflows/get-changed-services.yaml b/.github/workflows/get-changed-services.yaml
@@ -17,7 +17,7 @@ jobs:
         with:
           fetch-depth: 0
 
-      - uses: pnpm/action-setup@v2
+      - uses: pnpm/action-setup@v3
         with:
           version: 8
           run_install: false

diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
@@ -60,7 +60,7 @@ jobs:
           git pull origin ${{ github.event.repository.default_branch }}
         if: ${{ contains(github.ref, github.event.repository.default_branch) }}
 
-      - uses: pnpm/action-setup@v2
+      - uses: pnpm/action-setup@v3
         with:
           version: 8
           run_install: false

diff --git a/bundestag.io/admin/README.md b/bundestag.io/admin/README.md
@@ -5,9 +5,7 @@ This is a [Next.js](https://nextjs.org/) project bootstrapped with [`create-next
 First, run the development server:
 
 ```bash
-npm run dev
-# or
-yarn dev
+pnpm dev
 ```
 
 Open [http://localhost:3000](http://localhost:3000) with your browser to see the result.

diff --git a/bundestag.io/api/package.json b/bundestag.io/api/package.json
@@ -13,7 +13,7 @@
     "prebuild": "pnpm run generate",
     "start": "node build/index.js",
     "dev": "ts-node-dev --watch=resolvers,schemas --respawn src/index.ts",
-    "lint": "yarn lint:ts && yarn lint:exports",
+    "lint": "pnpm lint:ts && pnpm lint:exports",
     "lint:es": "eslint src --ext .js,.jsx,.ts,.tsx",
     "lint:ts": "tsc --noEmit",
     "lint:exports": "ts-unused-exports ./tsconfig.json --excludePathsFromReport=generated --excludePathsFromReport=resolvers --excludePathsFromReport=/schemas --excludePathsFromReport=graphqlContext.ts",

diff --git a/bundestag.io/api/src/graphql/resolvers/Procedure.ts b/bundestag.io/api/src/graphql/resolvers/Procedure.ts
@@ -303,12 +303,15 @@ const ProcedureResolvers: Resolvers = {
           return prev;
         }, []);
       };
-      console.log('procedure', procedure);
 
       if (procedure) {
         const axiosInstance = axios.create();
         const date = procedure.voteDate;
 
+        if (!date) {
+          throw new Error('No vote date found');
+        }
+
         const conferenceWeekDetail = await ConferenceWeekDetailModel.findOne({
           'sessions.date': {
             $gte: new Date(date.getFullYear(), date.getMonth(), date.getDate()),

diff --git a/bundestag.io/api/src/index.ts b/bundestag.io/api/src/index.ts
@@ -30,7 +30,7 @@ const server = createServer({ httpServer });
 const main = async () => {
   // Connect to DB - this keeps the process running
   // IMPORTANT - This is done before any Model is registered
-  await mongoConnect();
+  await mongoConnect(CONFIG.DB_URL);
 
   await server.start();
 

diff --git a/common/bundestagio/src/index.ts b/common/bundestagio/src/index.ts
@@ -1,2 +1,2 @@
-export * from "./models";
-export * from "./utils";
+export * from './models';
+export * from './utils';
diff --git a/common/bundestagio/src/models/mongoose.ts b/common/bundestagio/src/models/mongoose.ts
@@ -2,12 +2,11 @@ import Mongoose from 'mongoose';
 const mongoose = Mongoose;
 export { mongoose };
 
-export const mongoConnect = async () =>
+export const mongoConnect = async (dbUrl: string) =>
   new Promise(async (resolve) => {
     // Mongo Debug
     mongoose.set('debug', false);
-
-    mongoose.connect(process.env.DB_URL!);
+    mongoose.connect(dbUrl);
 
     mongoose.connection.once('connected', () => {
       console.info('MongoDB is running');

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -1,4 +1,5 @@
 version: '3.9'
+
 services:
   mongo:
     image: mongo:4
@@ -10,212 +11,48 @@ services:
       - democracy
   bundestag-io:
     build:
-      context: bundestag.io/api
-      target: base_stage
-      dockerfile: ../../infra/Dockerfile
+      dockerfile: ./infra/Dockerfile.service
+      target: dev
+      args:
+        - NODE_VERSION=18.18.2
+        - SERVICE=bundestag.io
+        - SERVICE_PATH=bundestag.io/api
     ports:
       - 4000:4000
-    command: pnpm run dev
     depends_on:
       - mongo
     environment:
       - DB_URL=mongodb://mongo:27017/bundestagio
       - BIO_EDIT_TOKEN=CHANGE_ME
       - PORT=4000
     volumes:
-      - ./bundestag.io/api/src:/app/src
+      - ./bundestag.io/api/src:/app/bundestag.io/api/src
     networks:
       - democracy
   bio-admin:
     build:
-      context: bundestag.io/admin
-      target: runner
+      dockerfile: ./infra/Dockerfile.service
+      target: dev
+      args:
+        - NODE_VERSION=18.18.2
+        - SERVICE=bundestag.io-admin
+        - SERVICE_PATH=bundestag.io/admin
+        - BUNDESTAGIO_SERVER_URL=http://bundestag-io:4000
     ports:
       - 4001:3000
     depends_on:
       - bundestag-io
     environment:
-      - BUNDESTAGIO_SERVER_URL=http://bundestag-io:3100
+      - BUNDESTAGIO_SERVER_URL=http://bundestag-io:4000
       - BIO_EDIT_TOKEN=CHANGE_ME
       - NEXTAUTH_URL=http://localhost:4001
       - CREDENTIALS=username:password
-    profiles:
-      - admin
-    networks:
-      - democracy
-  crawler:
-    build:
-      context: services/cron-jobs/crawler
-      target: base_stage
-      dockerfile: ../../../infra/Dockerfile
-    command: pnpm dev
-    volumes:
-      - ./services/cron-jobs/crawler/src:/app/src
-    env_file:
-      - services/cron-jobs/crawler/.env.local
-    environment:
-      - DB_URL=mongodb://mongo:27017/bundestagio
-    depends_on:
-      - mongo
-    profiles:
-      - cronjobs
-    networks:
-      - democracy
-  sync-procedures:
-    build:
-      context: services/cron-jobs/sync-procedures
-      target: base_stage
-      dockerfile: ../../../infra/Dockerfile
-    command: pnpm dev
-    volumes:
-      - ./services/cron-jobs/sync-procedures/src:/app/src
-    env_file:
-      - services/cron-jobs/sync-procedures/.env.local
-    environment:
-      - DB_URL=mongodb://mongo/democracy
-      - BUNDESTAGIO_SERVER_URL=http://bundestag-io:4000
-    depends_on:
-      - bundestag-io
-      - mongo
-    profiles:
-      - cronjobs
-    networks:
-      - democracy
-  queue-pushs-vote-top-100:
-    build:
-      context: services/cron-jobs/queue-pushs-vote-top-100
-      target: base_stage
-      dockerfile: ../../../infra/Dockerfile
-    command: pnpm dev
-    volumes:
-      - ./services/cron-jobs/queue-pushs-vote-top-100/src:/app/src
-    env_file:
-      - services/cron-jobs/queue-pushs-vote-top-100/.env.local
-    environment:
-      - DB_URL=mongodb://mongo/democracy
-    depends_on:
-      - mongo
-    profiles:
-      - cronjobs
-    networks:
-      - democracy
-  push-send-queued:
-    build:
-      context: services/cron-jobs/push-send-queued
-      target: base_stage
-      dockerfile: ../../../infra/Dockerfile
-    command: pnpm dev
-    volumes:
-      - ./services/cron-jobs/push-send-queued/src:/app/src
-    env_file:
-      - services/cron-jobs/push-send-queued/.env
-    environment:
-      - DB_URL=mongodb://mongo/democracy
-    depends_on:
-      - mongo
-    profiles:
-      - cronjobs
-    networks:
-      - democracy
-  cleanup-push-queue:
-    build:
-      context: services/cron-jobs/cleanup-push-queue
-      target: base_stage
-      dockerfile: ../../../infra/Dockerfile
-    command: pnpm dev
-    volumes:
-      - ./services/cron-jobs/cleanup-push-queue/src:/app/src
-    env_file:
-      - services/cron-jobs/cleanup-push-queue/.env.local
-    environment:
-      - DB_URL=mongodb://mongo/democracy
-    depends_on:
-      - mongo
-    profiles:
-      - cronjobs
-    networks:
-      - democracy
-  import-named-polls:
-    build:
-      context: services/cron-jobs/import-named-polls
-      target: base_stage
-      dockerfile: ../../../infra/Dockerfile
-    command: pnpm dev
-    volumes:
-      - ./services/cron-jobs/import-named-polls/src:/app/src
-    env_file:
-      - services/cron-jobs/import-named-polls/.env.local
-    environment:
-      - DB_URL=mongodb://mongo/bundestagio
-    depends_on:
-      - mongo
-    profiles:
-      - cronjobs
-  import-named-poll-deputies:
-    build:
-      context: services/cron-jobs/import-named-poll-deputies
-      target: base_stage
-      dockerfile: ../../../infra/Dockerfile
-    command: pnpm dev
-    volumes:
-      - ./services/cron-jobs/import-named-poll-deputies/src:/app/src
-    env_file:
-      - services/cron-jobs/import-named-poll-deputies/.env.local
-    environment:
-      - DB_URL=mongodb://mongo/bundestagio
-    depends_on:
-      - mongo
-    profiles:
-      - cronjobs
-  loki:
-    image: grafana/loki:2.4.2
-    volumes:
-      - ./docker_volumes/loki:/etc/loki
-    ports:
-      - 3301:3100
-    restart: unless-stopped
-    command: --config.file=/etc/loki/loki-config.yml
-    profiles:
-      - debug
-    networks:
-      - democracy
-  promtail:
-    image: grafana/promtail:2.4.0
     volumes:
-      - /var/log:/var/log
-      - ./docker_volumes/promtail:/etc/promtail
-    # ports:
-    #   - "1514:1514" # this is only needed if you are going to send syslogs
-    restart: unless-stopped
-    command: -config.file=/etc/promtail/promtail-config.yml
+      - ./bundestag.io/admin/pages:/app/bundestag.io/admin/pages
     profiles:
-      - debug
-    networks:
-      - democracy
-  grafana:
-    image: grafana/grafana:latest
-    user: '501'
-    volumes:
-      - ./docker_volumes/grafana:/var/lib/grafana
-    ports:
-      - '3300:3000'
-    restart: unless-stopped
-    profiles:
-      - debug
-    networks:
-      - democracy
-  prometheus:
-    image: prom/prometheus:v2.34.0
-    ports:
-      - 9090:9090
-    volumes:
-      - ./docker_volumes/prometheus:/etc/prometheus
-      - /prometheus
-    command: --web.enable-lifecycle  --config.file=/etc/prometheus/prometheus.yml
-    profiles:
-      - debug
+      - admin
     networks:
       - democracy
 networks:
   democracy:
-    external: true
+    external: false
diff --git a/infra/Dockerfile.service b/infra/Dockerfile.service
@@ -37,6 +37,11 @@ RUN pnpm install
 
 # Build the project
 COPY --from=builder /app/out/full/ .
+
+FROM installer AS dev
+CMD cd $SERVICE_PATH && pnpm dev
+
+FROM installer AS installer_prod
 RUN pnpm turbo run build --filter=$SERVICE
 RUN pnpm install --prod --frozen-lockfile --ignore-scripts --prefer-offline
 
@@ -47,12 +52,12 @@ WORKDIR /app
 RUN addgroup --system --gid 1001 nodejs
 RUN adduser --system --uid 1001 runuser
 
-COPY --from=installer /app .
+COPY --from=installer_prod /app .
 
 # Create data directory and set permissions
-RUN mkdir -p $SERVICE_PATH/data
-RUN chown runuser:nodejs $SERVICE_PATH/data
+# RUN mkdir -p $SERVICE_PATH/data
+# RUN chown runuser:nodejs $SERVICE_PATH/data
 
-USER runuser
+# USER runuser
 
 CMD cd $SERVICE_PATH && pnpm start
diff --git a/package.json b/package.json
@@ -6,31 +6,13 @@
   "author": "Manuel Ruck <[email protected]>",
   "license": "Apache-2.0",
   "private": true,
-  "packageManager": "[email protected].4",
+  "packageManager": "[email protected].5",
   "scripts": {
     "build": "turbo run build",
     "lint": "turbo run lint",
-    "dev": "pnpm build:k8s:local && skaffold dev --port-forward --status-check=false",
-    "build:k8s:local": "kustomize build ./infra/kustomize/overlays/local > infra/local.yaml",
-    "build:k8s:internal": "kustomize build ./infra/kustomize/overlays/internal > infra/internal.yaml",
-    "build:k8s:alpha": "kustomize build ./infra/kustomize/overlays/alpha > infra/alpha.yaml",
-    "build:k8s:prod": "kustomize build ./infra/kustomize/overlays/prod > infra/prod.yaml",
-    "s:i:procedures": "cd services/cron-jobs/crawler && pnpm && pnpm dev",
-    "s:i:conference-weeks": "cd services/cron-jobs/import-conference-week-details && pnpm && pnpm dev",
-    "s:i:deputy-profiles": "cd services/cron-jobs/import-deputy-profiles && pnpm && pnpm dev",
-    "s:i:named-poll-deputies": "cd services/cron-jobs/import-named-poll-deputies && pnpm && pnpm dev",
-    "s:i:named-polls": "cd services/cron-jobs/import-named-polls && pnpm && pnpm dev",
-    "s:i:plenary-mitutes": "cd services/cron-jobs/import-plenary-minutes && pnpm && pnpm dev",
-    "s:i": "pnpm s:i:procedures && pnpm s:i:conference-weeks && pnpm s:i:deputy-profiles && pnpm s:i:named-poll-deputies && pnpm s:i:named-polls && pnpm s:i:plenary-mitutes",
-    "s:parse-plenary-minutes": "cd services/cron-jobs/parse-plenary-minutes && pnpm && pnpm dev",
-    "s:q:conference-weeks": "cd services/cron-jobs/queue-pushs-conference-week && pnpm && pnpm dev",
-    "s:q:vote-conference-weeks": "cd services/cron-jobs/queue-pushs-vote-conference-week && pnpm && pnpm dev",
-    "s:q:top-100": "cd services/cron-jobs/queue-pushs-vote-top-100 && pnpm && pnpm dev",
-    "s:push": "cd services/cron-jobs/push-send-queued && pnpm && pnpm dev",
-    "s:s:deputy-profiles": "cd services/cron-jobs/sync-deputy-profiles && pnpm && pnpm dev",
-    "s:s:named-polls": "cd services/cron-jobs/sync-named-polls && pnpm && pnpm dev",
-    "s:s:procedures": "cd services/cron-jobs/sync-procedures && pnpm && pnpm dev",
-    "s:s": "pnpm s:s:deputy-profiles && pnpm s:s:named-polls && pnpm s:s:procedures"
+    "dev": "turbo run dev",
+    "infra:setup": "sh infra/minikube/setup.sh || bash infra/minikube/setup.sh || cmd.exe /C infra\\minikube\\setup.sh",
+    "infra:start": "minikube start -p mk-democracy"
   },
   "devDependencies": {
     "@typescript-eslint/eslint-plugin": "^6.13.2",