feat: Makefile + push pre-built images to ghcr

.env

@@ -0,0 +1,2 @@
+IMAGE_NAME=ghcr.io/hyperdxio/hyperdx
+IMAGE_VERSION=0.0.1

.github/workflows/main.yml

@@ -28,7 +28,7 @@ jobs:
           ~/.vector/bin/vector --version
           ln -s ~/.vector/bin/vector /usr/local/bin/vector
       - name: Run lint + type check
-        run: yarn ci:lint
+        run: make ci-lint
   integration:
     timeout-minutes: 8
     runs-on: ubuntu-20.04
@@ -46,4 +46,4 @@ jobs:
           docker buildx create --use --driver=docker-container
           docker buildx bake -f ./docker-compose.ci.yml --set *.cache-to="type=gha" --set *.cache-from="type=gha" --load
       - name: Run integration tests
-        run: yarn ci:int
+        run: make ci-int

.gitignore

@@ -11,11 +11,12 @@
 **/lerna-debug.log*
 
 # dotenv environment variable files
-**/.env
 **/.env.development.local
 **/.env.test.local
 **/.env.production.local
 **/.env.local
+**/.dockerhub.env
+**/.ghcr.env
 
 # Next.js build output
 packages/app/.next

Makefile

@@ -0,0 +1,40 @@
+LATEST_VERSION := $$(sed -n 's/.*"version": "\([^"]*\)".*/\1/p' package.json)
+BUILD_PLATFORMS = linux/arm64/v8,linux/arm/v7,linux/amd64
+
+include .env
+
+.PHONY: all
+all: install-tools
+
+.PHONY: install-tools
+install-tools:
+	yarn install
+	@echo "All tools installed"
+
+.PHONY: dev-lint
+dev-lint:
+	./docker/ingestor/run_linting.sh && yarn workspaces run lint
+
+.PHONY: ci-lint
+ci-lint:
+	./docker/ingestor/run_linting.sh && yarn workspaces run ci:lint
+
+.PHONY: dev-int
+dev-int:
+	docker compose -p int -f ./docker-compose.ci.yml run --rm api dev:int
+
+.PHONY: ci-int
+ci-int:
+	docker compose -p int -f ./docker-compose.ci.yml run --rm api ci:int
+
+
+.PHONY: build-and-push-ghcr
+build-and-push-ghcr:
+	docker buildx build --platform ${BUILD_PLATFORMS} . -f ./packages/miner/Dockerfile -t ${IMAGE_NAME}:${LATEST_VERSION}-miner  --target dev --push &
+	docker buildx build --platform ${BUILD_PLATFORMS} . -f ./docker/hostmetrics/Dockerfile -t ${IMAGE_NAME}:${LATEST_VERSION}-hostmetrics  --target dev --push &
+	docker buildx build --platform ${BUILD_PLATFORMS} . -f ./docker/ingestor/Dockerfile -t ${IMAGE_NAME}:${LATEST_VERSION}-ingestor  --target dev --push &
+	docker buildx build --platform ${BUILD_PLATFORMS} . -f ./docker/otel-collector/Dockerfile -t ${IMAGE_NAME}:${LATEST_VERSION}-otel-collector  --target dev --push &
+	docker buildx build --platform ${BUILD_PLATFORMS} . -f ./packages/api/Dockerfile -t ${IMAGE_NAME}:${LATEST_VERSION}-api  --target dev --push &
+	docker buildx build --platform ${BUILD_PLATFORMS} . -f ./packages/app/Dockerfile -t ${IMAGE_NAME}:${LATEST_VERSION}-app  --target prod --push
+
+

docker-compose.dev.yml

@@ -0,0 +1,257 @@
+version: '3'
+services:
+  miner:
+    image: ${IMAGE_NAME}:${IMAGE_VERSION}-miner
+    container_name: hdx-oss-dev-miner
+    build:
+      context: .
+      dockerfile: ./packages/miner/Dockerfile
+      target: dev
+      cache_from:
+        - ${IMAGE_NAME}:${IMAGE_VERSION}-miner
+    environment:
+      HYPERDX_API_KEY: ${HYPERDX_API_KEY}
+      HYPERDX_ENABLE_ADVANCED_NETWORK_CAPTURE: 1
+      OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318
+      OTEL_LOG_LEVEL: ERROR
+      OTEL_SERVICE_NAME: hdx-oss-dev-miner
+    volumes:
+      - ./packages/miner/src:/app/src
+    ports:
+      - 5123:5123
+    networks:
+      - internal
+  hostmetrics:
+    image: ${IMAGE_NAME}:${IMAGE_VERSION}-hostmetrics
+    container_name: hdx-oss-dev-hostmetrics
+    build:
+      context: ./docker/hostmetrics
+      target: dev
+      cache_from:
+        - ${IMAGE_NAME}:${IMAGE_VERSION}-hostmetrics
+    volumes:
+      - ./docker/hostmetrics/config.dev.yaml:/etc/otelcol-contrib/config.yaml
+    environment:
+      HYPERDX_API_KEY: ${HYPERDX_API_KEY}
+      OTEL_SERVICE_NAME: hostmetrics
+    restart: always
+    networks:
+      - internal
+  ingestor:
+    image: ${IMAGE_NAME}:${IMAGE_VERSION}-ingestor
+    container_name: hdx-oss-dev-ingestor
+    build:
+      context: ./docker/ingestor
+      target: dev
+      cache_from:
+        - ${IMAGE_NAME}:${IMAGE_VERSION}-ingestor
+    volumes:
+      - ./docker/ingestor:/app
+      - .volumes/ingestor_data:/var/lib/vector
+    ports:
+      - 8002:8002 # http-generic
+      - 8686:8686 # healthcheck
+    environment:
+      RUST_BACKTRACE: full
+      VECTOR_LOG: debug
+      VECTOR_OPENSSL_LEGACY_PROVIDER: false
+    restart: always
+    networks:
+      - internal
+  redis:
+    image: redis:7.0.11-alpine
+    container_name: hdx-oss-dev-redis
+    volumes:
+      - .volumes/redis:/data
+    ports:
+      - 6379:6379
+    networks:
+      - internal
+  db:
+    image: mongo:5.0.14-focal
+    container_name: hdx-oss-dev-db
+    volumes:
+      - .volumes/db:/data/db
+    ports:
+      - 27017:27017
+    networks:
+      - internal
+  otel-collector:
+    image: ${IMAGE_NAME}:${IMAGE_VERSION}-otel-collector
+    container_name: hdx-oss-dev-otel-collector
+    build:
+      context: ./docker/otel-collector
+      target: dev
+      cache_from:
+        - ${IMAGE_NAME}:${IMAGE_VERSION}-otel-collector
+    volumes:
+      - ./docker/otel-collector/config.yaml:/etc/otelcol-contrib/config.yaml
+    ports:
+      - '13133:13133' # health_check extension
+      - '1888:1888' # pprof extension
+      - '24225:24225' # fluentd receiver
+      - '4317:4317' # OTLP gRPC receiver
+      - '4318:4318' # OTLP http receiver
+      - '55679:55679' # zpages extension
+      - '8888:8888' # metrics extension
+      - '9411:9411' # zipkin
+    restart: always
+    networks:
+      - internal
+  aggregator:
+    image: ${IMAGE_NAME}:${IMAGE_VERSION}-api
+    container_name: hdx-oss-dev-aggregator
+    build:
+      context: .
+      dockerfile: ./packages/api/Dockerfile
+      target: dev
+      cache_from:
+        - ${IMAGE_NAME}:${IMAGE_VERSION}-api
+    ports:
+      - 8001:8001
+    environment:
+      APP_TYPE: 'aggregator'
+      CLICKHOUSE_HOST: http://ch-server:8123
+      CLICKHOUSE_PASSWORD: aggregator
+      CLICKHOUSE_USER: aggregator
+      FRONTEND_URL: 'http://localhost:8080' # need to be localhost (CORS)
+      MONGO_URI: 'mongodb://db:27017/hyperdx'
+      NODE_ENV: development
+      PORT: 8001
+      REDIS_URL: redis://redis:6379
+      SERVER_URL: 'http://localhost:8000'
+    volumes:
+      - ./packages/api/src:/app/src
+    networks:
+      - internal
+    depends_on:
+      - db
+      - redis
+      - ch-server
+  task-check-alerts:
+    image: ${IMAGE_NAME}:${IMAGE_VERSION}-api
+    container_name: hdx-oss-dev-task-check-alerts
+    build:
+      context: .
+      dockerfile: ./packages/api/Dockerfile
+      target: dev
+      cache_from:
+        - ${IMAGE_NAME}:${IMAGE_VERSION}-api
+    entrypoint: 'yarn'
+    command: 'dev:task check-alerts'
+    environment:
+      APP_TYPE: 'scheduled-task'
+      CLICKHOUSE_HOST: http://ch-server:8123
+      CLICKHOUSE_LOG_LEVEL: trace
+      CLICKHOUSE_PASSWORD: worker
+      CLICKHOUSE_USER: worker
+      FRONTEND_URL: 'http://localhost:8080' # need to be localhost (CORS)
+      HDX_NODE_ADVANCED_NETWORK_CAPTURE: 1
+      HDX_NODE_BETA_MODE: 0
+      HDX_NODE_CONSOLE_CAPTURE: 1
+      HYPERDX_API_KEY: ${HYPERDX_API_KEY}
+      HYPERDX_INGESTOR_ENDPOINT: 'http://ingestor:8002'
+      MINER_API_URL: 'http://miner:5123'
+      MONGO_URI: 'mongodb://db:27017/hyperdx'
+      NODE_ENV: development
+      OTEL_EXPORTER_OTLP_ENDPOINT: 'http://otel-collector:4318'
+      OTEL_SERVICE_NAME: 'hdx-oss-dev-task-check-alerts'
+      REDIS_URL: redis://redis:6379
+    volumes:
+      - ./packages/api/src:/app/src
+    restart: always
+    networks:
+      - internal
+    depends_on:
+      - ch-server
+      - db
+      - redis
+  api:
+    image: ${IMAGE_NAME}:${IMAGE_VERSION}-api
+    container_name: hdx-oss-dev-api
+    build:
+      context: .
+      dockerfile: ./packages/api/Dockerfile
+      target: dev
+      cache_from:
+        - ${IMAGE_NAME}:${IMAGE_VERSION}-api
+    ports:
+      - 8000:8000
+    environment:
+      APP_TYPE: 'api'
+      CLICKHOUSE_HOST: http://ch-server:8123
+      CLICKHOUSE_LOG_LEVEL: trace
+      CLICKHOUSE_PASSWORD: api
+      CLICKHOUSE_USER: api
+      EXPRESS_SESSION_SECRET: 'hyperdx is cool 👋'
+      FRONTEND_URL: 'http://localhost:8080' # need to be localhost (CORS)
+      HDX_NODE_ADVANCED_NETWORK_CAPTURE: 1
+      HDX_NODE_BETA_MODE: 1
+      HDX_NODE_CONSOLE_CAPTURE: 1
+      HYPERDX_API_KEY: ${HYPERDX_API_KEY}
+      HYPERDX_INGESTOR_ENDPOINT: 'http://ingestor:8002'
+      MINER_API_URL: 'http://miner:5123'
+      MONGO_URI: 'mongodb://db:27017/hyperdx'
+      NODE_ENV: development
+      OTEL_EXPORTER_OTLP_ENDPOINT: 'http://otel-collector:4318'
+      OTEL_SERVICE_NAME: 'hdx-oss-dev-api'
+      PORT: 8000
+      REDIS_URL: redis://redis:6379
+      SERVER_URL: 'http://localhost:8000'
+    volumes:
+      - ./packages/api/src:/app/src
+    networks:
+      - internal
+    depends_on:
+      - ch-server
+      - db
+      - redis
+  app:
+    image: ${IMAGE_NAME}:${IMAGE_VERSION}-app
+    container_name: hdx-oss-dev-app
+    build:
+      context: .
+      dockerfile: ./packages/app/Dockerfile
+      target: dev
+      cache_from:
+        - ${IMAGE_NAME}:${IMAGE_VERSION}-app
+    ports:
+      - 8080:8080
+    environment:
+      NEXT_PUBLIC_API_SERVER_URL: 'http://localhost:8000' # need to be localhost (CORS)
+      NEXT_PUBLIC_HDX_API_KEY: ${HYPERDX_API_KEY}
+      NEXT_PUBLIC_HDX_COLLECTOR_URL: 'http://localhost:4318'
+      NEXT_PUBLIC_HDX_SERVICE_NAME: 'hdx-oss-dev-app'
+      NODE_ENV: development
+      PORT: 8080
+    volumes:
+      - ./packages/app/pages:/app/pages
+      - ./packages/app/public:/app/public
+      - ./packages/app/src:/app/src
+      - ./packages/app/styles:/app/styles
+      - ./packages/app/mdx.d.ts:/app/mdx.d.ts
+      - ./packages/app/next-env.d.ts:/app/next-env.d.ts
+      - ./packages/app/next.config.js:/app/next.config.js
+    networks:
+      - internal
+    depends_on:
+      - api
+  ch-server:
+    image: clickhouse/clickhouse-server:23.7.1-alpine
+    container_name: hdx-oss-dev-ch-server
+    ports:
+      - 8123:8123 # http api
+      - 9000:9000 # native
+    environment:
+      # default settings
+      CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1
+    volumes:
+      - ./docker/clickhouse/local/config.xml:/etc/clickhouse-server/config.xml
+      - ./docker/clickhouse/local/users.xml:/etc/clickhouse-server/users.xml
+      - .volumes/ch_data:/var/lib/clickhouse
+      - .volumes/ch_logs:/var/log/clickhouse-server
+    restart: on-failure
+    networks:
+      - internal
+networks:
+  internal: