From 2e4752b5f9a9222387792def154e6915ed7c6805 Mon Sep 17 00:00:00 2001 From: Zeeshan Akram <97m.zeeshan@gmail.com> Date: Wed, 13 Dec 2023 23:25:26 +0500 Subject: [PATCH 1/4] add makefile command to generate db migrations --- .docker.env | 8 ++++++++ .env | 4 ---- Makefile | 7 +++++++ docker-compose.yml | 8 +++++--- 4 files changed, 20 insertions(+), 7 deletions(-) create mode 100644 .docker.env diff --git a/.docker.env b/.docker.env new file mode 100644 index 0000000..b7c20a3 --- /dev/null +++ b/.docker.env @@ -0,0 +1,8 @@ +# DEFAULT DOCKER ENVIRONMENT + +# DB_HOST is placed here instead of .env file, because the migration script +# loads .env and it would cause migration script to use `squid_db` instead +# of `localhost` as DB_HOST, causing the migration script to fail. + +# Docker db config +DB_HOST=squid_db diff --git a/.env b/.env index 65f54e8..1fbb87d 100644 --- a/.env +++ b/.env @@ -1,12 +1,8 @@ # Db config -DB_HOST=squid_db DB_NAME=squid DB_PASS=squid DB_PORT=23332 -# Processor service host -PROCESSOR_HOST=squid_processor - # Processor service prometheus port PROCESSOR_PROMETHEUS_PORT=3337 diff --git a/Makefile b/Makefile index d7aa816..74f4963 100644 --- a/Makefile +++ b/Makefile @@ -20,6 +20,13 @@ migrate: dbgen: @npx squid-typeorm-migration generate +generate-migrations: + @rm db/migrations/*-Data.js || true + @docker-compose down -v + @docker network create joystream_default || true + @docker-compose up -d squid_db + @npx squid-typeorm-migration generate + codegen: @npm run generate:schema || true @npx squid-typeorm-codegen diff --git a/docker-compose.yml b/docker-compose.yml index d6397f2..5f87789 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,7 +3,7 @@ version: '3' services: squid_db: container_name: squid_db - hostname: squid_db + hostname: squid-db image: postgres:14 restart: unless-stopped environment: @@ -28,11 +28,12 @@ services: squid_processor: container_name: squid_processor - hostname: squid_processor + hostname: squid-processor image: node:18 restart: unless-stopped env_file: - .env + - .docker.env ports: - '127.0.0.1:${PROCESSOR_PROMETHEUS_PORT}:${PROCESSOR_PROMETHEUS_PORT}' - '[::1]:${PROCESSOR_PROMETHEUS_PORT}:${PROCESSOR_PROMETHEUS_PORT}' @@ -49,11 +50,12 @@ services: squid_graphql-server: container_name: squid_graphql-server - hostname: squid_graphql-server + hostname: squid-graphql-server image: node:18 restart: unless-stopped env_file: - .env + - .docker.env environment: - SQD_TRACE=authentication depends_on: From 2f27f244ffe886e4cd9046e20f4d8df8134fbe10 Mon Sep 17 00:00:00 2001 From: Zeeshan Akram <97m.zeeshan@gmail.com> Date: Wed, 13 Dec 2023 23:28:25 +0500 Subject: [PATCH 2/4] remove unused configuration files --- .gitignore | 1 - .prettierignore | 1 - scripts/docker-run.sh | 5 ----- squid.yaml | 18 ------------------ 4 files changed, 25 deletions(-) delete mode 100644 scripts/docker-run.sh delete mode 100644 squid.yaml diff --git a/.gitignore b/.gitignore index c484964..1c5a466 100644 --- a/.gitignore +++ b/.gitignore @@ -10,5 +10,4 @@ src/model/generated src/types/ /schema.graphql /db/persisted -/scripts/orion-v1-migration/data /db/export diff --git a/.prettierignore b/.prettierignore index c196d42..8a8350b 100644 --- a/.prettierignore +++ b/.prettierignore @@ -7,5 +7,4 @@ src/types src/model/generated db/migrations/*.js schema.graphql -/scripts/orion-v1-migration/data /db/export \ No newline at end of file diff --git a/scripts/docker-run.sh b/scripts/docker-run.sh deleted file mode 100644 index 1947733..0000000 --- a/scripts/docker-run.sh +++ /dev/null @@ -1,5 +0,0 @@ -set -e -docker build . --target processor -t squid-processor -# make sure the port matches .env. -# For Linux, add --add-host=host.docker.internal:host-gateway -docker run --rm -e DB_HOST=host.docker.internal --env-file=.env squid-processor \ No newline at end of file diff --git a/squid.yaml b/squid.yaml deleted file mode 100644 index 759cd73..0000000 --- a/squid.yaml +++ /dev/null @@ -1,18 +0,0 @@ -manifestVersion: subsquid.io/v0.1 -name: jostream-content-qn -version: 1 -description: Substrate squid template -build: -deploy: - secrets: - - RPC_ENDPOINT - addons: - postgres: - processor: - cmd: - - sqd - - process:prod - api: - cmd: - - sqd - - serve:prod From 65e11a968a9be009319d3dbdc1e77c4f0f76b9bd Mon Sep 17 00:00:00 2001 From: Zeeshan Akram <97m.zeeshan@gmail.com> Date: Wed, 13 Dec 2023 23:28:52 +0500 Subject: [PATCH 3/4] added docker file --- Dockerfile | 47 +++++++++++++++++++++++++++++++++++++++++++++++ Makefile | 2 +- 2 files changed, 48 insertions(+), 1 deletion(-) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..be823b0 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,47 @@ +FROM node:18-alpine AS node + +FROM node AS node-with-gyp +RUN apk add g++ make python3 + +FROM node-with-gyp AS builder +WORKDIR /squid +ADD package.json . +ADD package-lock.json . +ADD assets assets +RUN npm ci +ADD tsconfig.json . +ADD src src +ADD schema schema +ADD scripts scripts +RUN npx squid-typeorm-codegen +RUN npm run build + +FROM node-with-gyp AS deps +WORKDIR /squid +ADD package.json . +ADD package-lock.json . +ADD assets assets +RUN npm ci --omit=dev + +FROM node AS squid +WORKDIR /squid +COPY --from=deps /squid/package.json . +COPY --from=deps /squid/package-lock.json . +COPY --from=deps /squid/node_modules node_modules +COPY --from=builder /squid/lib lib +RUN echo -e "loglevel=silent\nupdate-notifier=false" > /squid/.npmrc +ADD db db +ADD assets assets +ADD schema schema +ADD scripts scripts +ENV PROCESSOR_PROMETHEUS_PORT 3000 +EXPOSE 3000 +EXPOSE 4000 + + +FROM squid AS processor +CMD ["npm", "run", "processor-start"] + + +FROM squid AS query-node +CMD ["npm", "run", "graphql-server-start"] diff --git a/Makefile b/Makefile index 74f4963..e3cb260 100644 --- a/Makefile +++ b/Makefile @@ -9,7 +9,7 @@ build: @npm run build build-docker: - @docker build . -t joystream/orion + @docker build . -t joystream/storage-squid serve: @npx squid-graphql-server --subscriptions From 274f7e109c7bc33bd58e5831d3a312184a5ec58b Mon Sep 17 00:00:00 2001 From: Zeeshan Akram <97m.zeeshan@gmail.com> Date: Wed, 13 Dec 2023 23:55:15 +0500 Subject: [PATCH 4/4] added github workflows for build checks as well as docker publishing --- .github/workflows/checks.yml | 38 +++++++++++++++++++++++ .github/workflows/docker.yml | 60 ++++++++++++++++++++++++++++++++++++ CHANGELOG.md | 3 ++ package.json | 2 +- 4 files changed, 102 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/checks.yml create mode 100644 .github/workflows/docker.yml create mode 100644 CHANGELOG.md diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml new file mode 100644 index 0000000..0a1e192 --- /dev/null +++ b/.github/workflows/checks.yml @@ -0,0 +1,38 @@ +name: Checks +on: [push, pull_request] + +jobs: + local: + name: Local build, linting and formatting + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest] + node-version: [16.x] + fail-fast: true + steps: + - uses: actions/checkout@v2 + - name: Use Node.js ${{matrix.node-version}} + uses: actions/setup-node@v1 + with: + node-version: ${{matrix.node-version}} + - name: Install npm packages + run: npm ci + - name: Run checks + run: npm run checks + docker: + name: Docker build check + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest] + node-version: [16.x] + fail-fast: true + steps: + - uses: actions/checkout@v2 + - name: Use Node.js ${{matrix.node-version}} + uses: actions/setup-node@v1 + with: + node-version: ${{matrix.node-version}} + - name: Build docker image + run: make build-docker diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 0000000..5b2c1d3 --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,60 @@ +name: Publish Storage-Squid Docker image + +on: + workflow_dispatch: + +jobs: + build_and_publish: + name: Build and Publish + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + + - uses: actions/setup-node@v3 + with: + node-version: '18.x' + + - name: Extract Package Version + id: extract_version + shell: bash + run: | + echo "squid_version=$(cat package.json | jq -r '.version')" >> $GITHUB_OUTPUT + + - name: Make some space + shell: bash + run: | + sudo rm -rf /usr/share/dotnet + sudo rm -rf /opt/ghc + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Login to DockerHub + if: github.event_name == 'workflow_dispatch' + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + + # docker/build-push-action doc: + # Be careful because any file mutation in the steps that precede the + # build step will be ignored, including processing of the .dockerignore file + # since the context is based on the Git reference. However, you can use + # the Path context using the context input alongside the actions/checkout action + # to remove this restriction. + - name: Build storage-squid + uses: docker/build-push-action@v3 + with: + # Do not use local dir context to ensure we can build from a commit directly + # context: . + file: Dockerfile + push: false + load: true + tags: joystream/storage-squid:latest + + - name: Push storage-squid + run: | + docker image tag joystream/storage-squid:latest joystream/storage-squid:${{ steps.extract_version.outputs.squid_version }} + docker push joystream/storage-squid:${{ steps.extract_version.outputs.squid_version }} diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..a569057 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,3 @@ +# 1.0.0 + +- Initial release of `storage-squid` package. \ No newline at end of file diff --git a/package.json b/package.json index fcc1f77..cbdfbc5 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ "build": "npm run generate:schema; rm -rf lib && tsc", "lint": "eslint --ext .ts ./src", "format": "prettier --write .", - "checks": "prettier --check . && npm run lint && make codegen && tsc --noEmit --pretty", + "checks": "prettier --check . && npm run lint && make prepare && tsc --noEmit --pretty", "db:migrate": "npx squid-typeorm-migration apply", "processor-start": "node -r dotenv-expand/config lib/processor.js", "graphql-server-start": "NODE_ENV=production patch-package --patch-dir assets/patches && squid-graphql-server --subscriptions",