Modron v0.2

nianticlabs · Oct 24, 2023 · b3ae32a · b3ae32a
1 parent 94699d7
commit b3ae32a
Show file tree

Hide file tree

Showing 234 changed files with 21,054 additions and 43,214 deletions.
diff --git a/CHANGELOG b/CHANGELOG
@@ -1,8 +1,22 @@
 # Changelog
 
+## 2023-10-24
+
+* Moved to go 1.21
+* Add jump host to access to SQL database
+* Beef up the SQL server not to slow down scans
+* New detections
+* Nagatha backend now reached with JSON and not binary protocol.
+
+## 2023-05-05
+
+* Moved to go 1.20
+* Moved to Postgresql as a backend
+* Fixed the scheduled scan runs that didn't work.
+
 ## 2022-11-18
 
 * Moved to go 1.19
 * Added automated runs for scans
 * Fixed issue where last reported observation would still appear even if newer scans reported no observations
-* Fixed group member ship resolution when checking for accesses to GCP projects
+* Fixed group member ship resolution when checking for accesses to GCP projects
diff --git a/README.md b/README.md
@@ -111,7 +111,13 @@ The Modron service is meant to work at the organization level on GCP. In order t
     "compute.backendServices.list",
     "compute.instances.list",
     "compute.regions.list",
+    "compute.sslCertificates.list",
+    "compute.sslPolicies.list",
     "compute.subnetworks.list",
+    "compute.targetHttpsProxies.list",
+    "compute.targetHttpsProxies.list",
+    "compute.targetSslProxies.list",
+    "compute.urlMaps.list",
     "compute.zones.list",
     "container.clusters.list",
     "iam.serviceAccounts.list",
@@ -139,7 +145,13 @@ resource "google_organization_iam_custom_role" "modron_lister" {
     "compute.backendServices.list",
     "compute.instances.list",
     "compute.regions.list",
+    "compute.sslCertificates.list",
+    "compute.sslPolicies.list",
     "compute.subnetworks.list",
+    "compute.targetHttpsProxies.list",
+    "compute.targetHttpsProxies.list",
+    "compute.targetSslProxies.list",
+    "compute.urlMaps.list",
     "compute.zones.list",
     "container.clusters.list",
     "iam.serviceAccounts.list",
@@ -176,7 +188,10 @@ go test ./... --short
 
 ### Integration test
 
+To run the integration test, you'll need a self signed certificate for the notification service.
+
 ```
+openssl req -x509 -newkey rsa:4096 -keyout key.pem -nodes -out cert.pem -sha256 -days 365 -subj '/CN=modron_test' -addext "subjectAltName = DNS:modron_test"
 docker-compose up --build --exit-code-from "modron_test" --abort-on-container-exit
 ```
 
@@ -188,16 +203,23 @@ docker-compose -f docker-compose.ui.yaml up --build --exit-code-from "modron_tes
 
 ### Running locally
 
-Use the docker command to run modron locally:
+Use this docker command to spin up a local deployment via docker-compose (will rebuild on every run):
+```
+docker-compose -f docker-compose.ui.yaml up --build
+```
+In case you want to clean up all the created images, services and volumes (e.g. if you suspect a caching issue or if a service does not properly shut down):
+```
+docker-compose rm -fsv # remove all images, services and volumes if needed
+```
+
+
+Alternative: Use the docker command to run modron locally (against a dev project):
 
 ```
-cd src
-chmod 644 ~/.config/gcloud/application_default_credentials.json 
-docker build -t modron-local:latest .
-export PROJECT=modron-dev
-export ORG_SUFFIX="@example.com"
-export ORG_ID="0123456789"
-docker run -e COLLECT_AND_SCAN_INTERVAL="12h" -e DATASET_ID="modron_bq" -e RESOURCE_TABLE_ID="resources" -e OBSERVATION_TABLE_ID="observations" -e OPERATION_TABLE_ID="operations" -e GCP_PROJECT_ID="$PROJECT" -e PORT="8080" -e GOOGLE_APPLICATION_CREDENTIALS="/tmp/application_default_credentials.json" -e ORG_SUFFIX="$ORG_SUFFIX" -e ORG_ID="$ORG_ID" --mount "source=$HOME/.config/gcloud/application_default_credentials.json,target=/tmp/application_default_credentials.json,type=bind" -p 8080:8080 modron-local:latest
+chmod 644 ~/.config/gcloud/application_default_credentials.json
+docker build -f Dockerfile.db -t modron-db:latest .
+docker run -e POSTGRES_PASSWORD="docker-test-password" -e POSTGRES_USER="modron" -e POSTGRES_DB="modron" -e PG_DATA="tmp_data/" -t modron-db:latest -p 5432
+GOOGLE_APPLICATION_CREDENTIALS=~/.config/gcloud/application_default_credentials.json PORT="8080" GCP_PROJECT_ID=modron-dev OPERATION_TABLE_ID="operations" OBSERVATION_TABLE_ID="observations" RESOURCE_TABLE_ID="resources" RUN_AUTOMATED_SCANS="false" ORG_SUFFIX="@example.com" STORAGE="SQL" DB_MAX_CONNECTIONS="1" SQL_BACKEND_DRIVER="postgres" SQL_CONNECT_STRING="host=localhost port=5432 user=modron password=docker-test-password database=modron sslmode=disable" go run . --logtostderr
 ```
 
 ## Future developments
@@ -209,4 +231,4 @@ docker run -e COLLECT_AND_SCAN_INTERVAL="12h" -e DATASET_ID="modron_bq" -e RESOU
 
 ## Security
 
-Report any security issue to [[email protected]](mailto:[email protected]). 
+Report any security issue to [[email protected]](mailto:[email protected]). 
diff --git a/docker-compose.ui.yaml b/docker-compose.ui.yaml
@@ -18,15 +18,24 @@ services:
     container_name: modron_fake
     build: src/
     environment:
-      PORT: 8080
-      STORAGE: "MEM"
+      RUN_AUTOMATED_SCANS: "false"
       COLLECTOR: "FAKE"
+      DB_MAX_CONNECTIONS: "1"
+      GRPC_TRACE: "all"
+      GRPC_VERBOSITY: "DEBUG"
+      OBSERVATION_TABLE_ID: "observations"
+      OPERATION_TABLE_ID: "operations"
       ORG_ID: "0123456789"
       ORG_SUFFIX: "@example.com"
-      GRPC_VERBOSITY: "DEBUG"
-      GRPC_TRACE: "all"
+      PORT: 8080
+      RESOURCE_TABLE_ID: "resources"
+      SQL_BACKEND_DRIVER: "postgres"
+      SQL_CONNECT_STRING: "host=postgres_db port=5432 user=modron password=docker-test-password database=modron sslmode=disable"
+      STORAGE: "SQL"
     networks:
       - modron
+    depends_on:
+      - postgres_db
 
   modron_ui:
     container_name: modron_ui
@@ -47,11 +56,29 @@ services:
       - modron_proxy
     environment:
       CYPRESS_BASE_URL: "http://modron_proxy:10000"
+      ELECTRON_ENABLE_LOGGING: 1
     command: npx cypress run
     volumes:
       - ./src/ui/client/cypress:/app/cypress:ro
       - ./docker/cypress/tsconfig.json:/app/tsconfig.json
       - ./docker/cypress/cypress.config.ts:/app/cypress.config.ts
+      - ./:/app/results
+    networks:
+      - modron
+
+  postgres_db:
+    container_name: postgres_db
+    build:
+      context: src/
+      dockerfile: Dockerfile.db
+    restart: always
+    environment:
+      POSTGRES_USER: "modron"
+      POSTGRES_PASSWORD: "docker-test-password"
+      POSTGRES_DB: "modron"
+      PGDATA: "/tmp/"
+    tmpfs:
+      - /tmp
     networks:
       - modron
 

diff --git a/docker-compose.yaml b/docker-compose.yaml
@@ -1,35 +1,48 @@
 version: '3'
 
 services:
-  modron:
-    container_name: modron
-    build: src/
+  postgres_db:
+    container_name: postgres_db
+    build:
+      context: src/
+      dockerfile: Dockerfile.db
+    restart: always
     environment:
-      PORT: 8080
-      ENVIRONMENT: "E2E_GRPC_TESTING"
-      STORAGE: "MEM"
-      COLLECTOR: "FAKE"
-      ORG_ID: "0123456789"
-      ORG_SUFFIX: "@example.com"
-    ports:
-      - "8080:8080"
+      POSTGRES_USER: "modron"
+      POSTGRES_PASSWORD: "docker-test-password"
+      POSTGRES_DB: "modron"
+      PGDATA: "/tmp/"
+    tmpfs:
+      - /tmp
     networks:
       - modron
 
   modron_fake:
     container_name: modron_fake
     build: src/
     environment:
-      PORT: 8081
-      ENVIRONMENT: "E2E_GRPC_TESTING"
-      STORAGE: "MEM"
       COLLECTOR: "FAKE"
+      DB_BATCH_SIZE: "1"
+      DB_MAX_CONNECTIONS: "1"
+      ENVIRONMENT: "E2E_GRPC_TESTING"
+      GLOG_v: "10"
+      NOTIFICATION_SERVICE: "modron_test:8082"
+      OBSERVATION_TABLE_ID: "observations"
+      OPERATION_TABLE_ID: "operations"
       ORG_ID: "0123456789"
       ORG_SUFFIX: "@example.com"
+      PORT: 8081
+      RESOURCE_TABLE_ID: "resources"
+      RUN_AUTOMATED_SCANS: "false"
+      SQL_BACKEND_DRIVER: "postgres"
+      SQL_CONNECT_STRING: "host=postgres_db port=5432 user=modron password=docker-test-password database=modron sslmode=disable"
+      STORAGE: "SQL"
     ports:
       - "8081:8081"
     networks:
       - modron
+    depends_on:
+      - postgres_db
 
   modron_test:
     container_name: e2e_test
@@ -39,19 +52,12 @@ services:
     environment:
       BACKEND_ADDRESS: "modron:8080"
       FAKE_BACKEND_ADDRESS: "modron_fake:8081"
-    volumes:
-      - stats:/app/stats
-      - "./secrets:/app/secrets"
+      FAKE_NOTIFICATION_SERVICE_PORT: "8082"
     networks:
       - modron
     depends_on:
-      - modron
       - modron_fake
 
-volumes:
-  stats:
-    external: false
-
 networks:
   modron:
     driver: bridge
diff --git a/docker/cypress/cypress.config.ts b/docker/cypress/cypress.config.ts
@@ -1,20 +1,25 @@
-import { defineConfig } from 'cypress'
+import { defineConfig } from "cypress"
 
 export default defineConfig({
 
   e2e: {
-    baseUrl: 'http://localhost:8080',
+    baseUrl: "http://localhost:8080",
     supportFile: false
   },
   video: false,
   screenshotOnRunFailure: false,
-
   component: {
     devServer: {
-      framework: 'angular',
-      bundler: 'webpack',
+      framework: "angular",
+      bundler: "webpack",
     },
-    specPattern: '**/*.cy.ts'
-  }
+    specPattern: "**/*.cy.ts"
+  },
+
+  reporter: "junit",
+  reporterOptions: {
+    mochaFile: "/app/results/modron-e2e-ui-junit.xml",
+    toConsole: false,
+  },
 
 })
diff --git a/docker/cypress/tsconfig.json b/docker/cypress/tsconfig.json
@@ -1,15 +1,38 @@
+/* To learn more about this file see: https://angular.io/config/tsconfig. */
 {
+  "compileOnSave": false,
   "compilerOptions": {
     "baseUrl": "./",
-    "target": "es5",
-    "module": "commonjs",
-    "skipLibCheck": true,
+    "outDir": "./dist/out-tsc",
+    "forceConsistentCasingInFileNames": true,
     "strict": true,
-    "types": [
-      "cypress"
-    ]
+    "noImplicitOverride": true,
+    "noPropertyAccessFromIndexSignature": true,
+    "noImplicitReturns": true,
+    "noFallthroughCasesInSwitch": true,
+    "sourceMap": true,
+    "declaration": false,
+    "downlevelIteration": true,
+    "experimentalDecorators": true,
+    "moduleResolution": "node",
+    "importHelpers": true,
+    "target": "ES2022",
+    "module": "ES2022",
+    "lib": [
+      "es2020",
+      "dom"
+    ],
   },
-  "include": [
-    "cypress/**/*.ts"
+  "angularCompilerOptions": {
+    "enableI18nLegacyMessageIdFormat": false,
+    "strictInjectionParameters": true,
+    "strictInputAccessModifiers": true,
+    "strictTemplates": true
+  },
+  "exclude": [
+    "cypress.config.ts"
+  ],
+  "files": [
+    "cypress.config.ts"
   ]
 }
diff --git a/docker/envoyproxy/envoy.yaml b/docker/envoyproxy/envoy.yaml
@@ -0,0 +1,60 @@
+static_resources:
+  listeners:
+    - name: listener_0
+      address:
+        socket_address: { address: 0.0.0.0, port_value: 10000 }
+
+      filter_chains:
+        - filters:
+            - name: envoy.http_connection_manager
+              typed_config:
+                "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
+                codec_type: AUTO
+                stat_prefix: ingress_http
+                route_config:
+                  name: local_route
+                  virtual_hosts:
+                    - name: modron
+                      domains: ["*"]
+                      routes:
+                        - match: { prefix: "/api/" }
+                          route: { prefix_rewrite: "/", cluster: modron_fake }
+                        - match: { prefix: "/" }
+                          route: { cluster: modron_ui }
+                http_filters:
+                  - name: envoy.filters.http.router
+                    typed_config:
+                      "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router
+                access_log:
+                  - name: envoy.access_loggers.file
+                    typed_config:
+                      "@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog
+                      path: "/dev/stdout"
+
+  clusters:
+    - name: modron_ui
+      connect_timeout: 0.25s
+      type: STRICT_DNS
+      lb_policy: ROUND_ROBIN
+      load_assignment:
+        cluster_name: modron_ui
+        endpoints:
+          - lb_endpoints:
+              - endpoint:
+                  address:
+                    socket_address:
+                      address: modron_ui
+                      port_value: 8080
+    - name: modron_fake
+      connect_timeout: 0.25s
+      type: STRICT_DNS
+      lb_policy: ROUND_ROBIN
+      load_assignment:
+        cluster_name: modron_fake
+        endpoints:
+          - lb_endpoints:
+              - endpoint:
+                  address:
+                    socket_address:
+                      address: modron_fake
+                      port_value: 8080
diff --git a/go.mod b/go.mod
@@ -1,4 +1,4 @@
 // This file is required for gosec to work.
 module github.com/nianticlabs/modron
 
-go 1.19
+go 1.21