Merge pull request #5 from zeeshanakram3/docker-image-and-github-ci-s…

…etup

Docker image and GitHub CI setup

.docker.env

@@ -0,0 +1,8 @@
+# DEFAULT DOCKER ENVIRONMENT
+
+# DB_HOST is placed here instead of .env file, because the migration script
+# loads .env and it would cause migration script to use `squid_db` instead
+# of `localhost` as DB_HOST, causing the migration script to fail.
+
+# Docker db config
+DB_HOST=squid_db

.env

@@ -1,12 +1,8 @@
 # Db config
-DB_HOST=squid_db
 DB_NAME=squid
 DB_PASS=squid
 DB_PORT=23332
 
-# Processor service host
-PROCESSOR_HOST=squid_processor
-
 # Processor service prometheus port
 PROCESSOR_PROMETHEUS_PORT=3337
 

.github/workflows/checks.yml

@@ -0,0 +1,38 @@
+name: Checks
+on: [push, pull_request]
+
+jobs:
+  local:
+    name: Local build, linting and formatting
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        node-version: [16.x]
+      fail-fast: true
+    steps:
+      - uses: actions/checkout@v2
+      - name: Use Node.js ${{matrix.node-version}}
+        uses: actions/setup-node@v1
+        with:
+          node-version: ${{matrix.node-version}}
+      - name: Install npm packages
+        run: npm ci
+      - name: Run checks
+        run: npm run checks
+  docker:
+    name: Docker build check
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        node-version: [16.x]
+      fail-fast: true
+    steps:
+      - uses: actions/checkout@v2
+      - name: Use Node.js ${{matrix.node-version}}
+        uses: actions/setup-node@v1
+        with:
+          node-version: ${{matrix.node-version}}
+      - name: Build docker image
+        run: make build-docker

.github/workflows/docker.yml

@@ -0,0 +1,60 @@
+name: Publish Storage-Squid Docker image
+
+on:
+  workflow_dispatch:
+
+jobs:
+  build_and_publish:
+    name: Build and Publish
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - uses: actions/setup-node@v3
+        with:
+          node-version: '18.x'
+
+      - name: Extract Package Version
+        id: extract_version
+        shell: bash
+        run: |
+          echo "squid_version=$(cat package.json | jq -r '.version')" >> $GITHUB_OUTPUT
+
+      - name: Make some space
+        shell: bash
+        run: |
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /opt/ghc
+          sudo rm -rf "/usr/local/share/boost"
+          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      - name: Login to DockerHub
+        if: github.event_name == 'workflow_dispatch'
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_PASSWORD }}
+
+      # docker/build-push-action doc:
+      # Be careful because any file mutation in the steps that precede the
+      # build step will be ignored, including processing of the .dockerignore file
+      # since the context is based on the Git reference. However, you can use
+      # the Path context using the context input alongside the actions/checkout action
+      # to remove this restriction.
+      - name: Build storage-squid
+        uses: docker/build-push-action@v3
+        with:
+          # Do not use local dir context to ensure we can build from a commit directly
+          # context: .
+          file: Dockerfile
+          push: false
+          load: true
+          tags: joystream/storage-squid:latest
+
+      - name: Push storage-squid
+        run: |
+          docker image tag joystream/storage-squid:latest joystream/storage-squid:${{ steps.extract_version.outputs.squid_version }}
+          docker push joystream/storage-squid:${{ steps.extract_version.outputs.squid_version }}

.gitignore

@@ -10,5 +10,4 @@ src/model/generated
 src/types/
 /schema.graphql
 /db/persisted
-/scripts/orion-v1-migration/data
 /db/export

.prettierignore

@@ -7,5 +7,4 @@ src/types
 src/model/generated
 db/migrations/*.js
 schema.graphql
-/scripts/orion-v1-migration/data
 /db/export

CHANGELOG.md

@@ -0,0 +1,3 @@
+# 1.0.0
+
+- Initial release of `storage-squid` package.

Dockerfile

@@ -0,0 +1,47 @@
+FROM node:18-alpine AS node
+
+FROM node AS node-with-gyp
+RUN apk add g++ make python3
+
+FROM node-with-gyp AS builder
+WORKDIR /squid
+ADD package.json .
+ADD package-lock.json .
+ADD assets assets
+RUN npm ci
+ADD tsconfig.json .
+ADD src src
+ADD schema schema
+ADD scripts scripts
+RUN npx squid-typeorm-codegen
+RUN npm run build
+
+FROM node-with-gyp AS deps
+WORKDIR /squid
+ADD package.json .
+ADD package-lock.json .
+ADD assets assets
+RUN npm ci --omit=dev
+
+FROM node AS squid
+WORKDIR /squid
+COPY --from=deps /squid/package.json .
+COPY --from=deps /squid/package-lock.json .
+COPY --from=deps /squid/node_modules node_modules
+COPY --from=builder /squid/lib lib
+RUN echo -e "loglevel=silent\nupdate-notifier=false" > /squid/.npmrc
+ADD db db
+ADD assets assets
+ADD schema schema
+ADD scripts scripts
+ENV PROCESSOR_PROMETHEUS_PORT 3000
+EXPOSE 3000
+EXPOSE 4000
+
+
+FROM squid AS processor
+CMD ["npm", "run", "processor-start"]
+
+
+FROM squid AS query-node
+CMD ["npm", "run", "graphql-server-start"]

Makefile

@@ -9,7 +9,7 @@ build:
 	@npm run build
 
 build-docker:
-	@docker build . -t joystream/orion
+	@docker build . -t joystream/storage-squid
 
 serve:
 	@npx squid-graphql-server --subscriptions
@@ -20,6 +20,13 @@ migrate:
 dbgen:
 	@npx squid-typeorm-migration generate
 
+generate-migrations: 
+	@rm db/migrations/*-Data.js || true
+	@docker-compose down -v
+	@docker network create joystream_default || true
+	@docker-compose up -d squid_db
+	@npx squid-typeorm-migration generate
+
 codegen:
 	@npm run generate:schema || true
 	@npx squid-typeorm-codegen

docker-compose.yml

@@ -3,7 +3,7 @@ version: '3'
 services:
   squid_db:
     container_name: squid_db
-    hostname: squid_db
+    hostname: squid-db
     image: postgres:14
     restart: unless-stopped
     environment:
@@ -28,11 +28,12 @@ services:
 
   squid_processor:
     container_name: squid_processor
-    hostname: squid_processor
+    hostname: squid-processor
     image: node:18
     restart: unless-stopped
     env_file:
       - .env
+      - .docker.env
     ports:
       - '127.0.0.1:${PROCESSOR_PROMETHEUS_PORT}:${PROCESSOR_PROMETHEUS_PORT}'
       - '[::1]:${PROCESSOR_PROMETHEUS_PORT}:${PROCESSOR_PROMETHEUS_PORT}'
@@ -49,11 +50,12 @@ services:
 
   squid_graphql-server:
     container_name: squid_graphql-server
-    hostname: squid_graphql-server
+    hostname: squid-graphql-server
     image: node:18
     restart: unless-stopped
     env_file:
       - .env
+      - .docker.env
     environment:
       - SQD_TRACE=authentication
     depends_on:

package.json

@@ -9,7 +9,7 @@
     "build": "npm run generate:schema; rm -rf lib && tsc",
     "lint": "eslint --ext .ts ./src",
     "format": "prettier --write .",
-    "checks": "prettier --check . && npm run lint && make codegen && tsc --noEmit --pretty",
+    "checks": "prettier --check . && npm run lint && make prepare && tsc --noEmit --pretty",
     "db:migrate": "npx squid-typeorm-migration apply",
     "processor-start": "node -r dotenv-expand/config lib/processor.js",
     "graphql-server-start": "NODE_ENV=production patch-package --patch-dir assets/patches && squid-graphql-server --subscriptions",