feat: introduce OpenSearch (#3612)

Co-authored-by: Bruno Zorić <[email protected]>
webiny · Dec 4, 2023 · 5c7a76b · 5c7a76b
1 parent b56dde1
commit 5c7a76b
Show file tree

Hide file tree

Showing 37 changed files with 1,429 additions and 849 deletions.
diff --git a/.github/workflows/pullRequests.yml b/.github/workflows/pullRequests.yml
@@ -281,6 +281,64 @@ jobs:
     permissions:
       id-token: write
     if: needs.init.outputs.is-fork-pr != 'true'
+  jestTestsDdbOs:
+    needs: init
+    name: ${{ matrix.package.cmd }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os:
+          - ubuntu-latest
+        node:
+          - 18
+        package: >-
+          ${{ fromJson('[{"cmd":"packages/api-aco
+          --storage=ddb-os,ddb","storage":"ddb-os","id":"api-aco_ddb-os_ddb"},{"cmd":"packages/api-audit-logs
+          --storage=ddb-os,ddb","storage":"ddb-os","id":"api-audit-logs_ddb-os_ddb"},{"cmd":"packages/api-elasticsearch","storage":["ddb-es","ddb-os"],"id":"api-elasticsearch"},{"cmd":"packages/api-file-manager
+          --storage=ddb-os,ddb","storage":"ddb-os","id":"api-file-manager_ddb-os_ddb"},{"cmd":"packages/api-form-builder
+          --storage=ddb-os,ddb","storage":"ddb-os","id":"api-form-builder_ddb-os_ddb"},{"cmd":"packages/api-form-builder-so-ddb-es
+          --storage=ddb-os,ddb","storage":"ddb-os","id":"api-form-builder-so-ddb-es_ddb-os_ddb"},{"cmd":"packages/api-headless-cms
+          --storage=ddb-os,ddb","storage":"ddb-os","id":"api-headless-cms_ddb-os_ddb"},{"cmd":"packages/api-headless-cms-ddb-es
+          --storage=ddb-os,ddb","storage":"ddb-os","id":"api-headless-cms-ddb-es_ddb-os_ddb"},{"cmd":"packages/api-mailer
+          --storage=ddb-os,ddb","storage":"ddb-os","id":"api-mailer_ddb-os_ddb"},{"cmd":"packages/api-page-builder
+          --storage=ddb-os,ddb","storage":"ddb-os","id":"api-page-builder_ddb-os_ddb"},{"cmd":"packages/api-page-builder-aco
+          --storage=ddb-os,ddb","storage":"ddb-os","id":"api-page-builder-aco_ddb-os_ddb"},{"cmd":"packages/api-page-builder-so-ddb-es
+          --storage=ddb-os,ddb","storage":"ddb-os","id":"api-page-builder-so-ddb-es_ddb-os_ddb"},{"cmd":"packages/migrations","storage":["ddb-es","ddb-os"],"id":"migrations"}]')
+          }}
+    runs-on: ${{ matrix.os }}
+    env:
+      NODE_OPTIONS: '--max_old_space_size=4096'
+      YARN_ENABLE_IMMUTABLE_INSTALLS: false
+      AWS_ELASTIC_SEARCH_DOMAIN_NAME: ${{ secrets.AWS_OPEN_SEARCH_DOMAIN_NAME }}
+      ELASTIC_SEARCH_ENDPOINT: ${{ secrets.OPEN_SEARCH_ENDPOINT }}
+      ELASTIC_SEARCH_INDEX_PREFIX: ${{ matrix.package.id }}
+    steps:
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 18
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: arn:aws:iam::726952677045:role/GitHubActionsWebinyJs
+          aws-region: eu-central-1
+      - uses: actions/checkout@v3
+      - uses: actions/cache@v3
+        with:
+          path: .yarn/cache
+          key: yarn-${{ runner.os }}-${{ hashFiles('**/yarn.lock') }}
+      - uses: actions/cache@v3
+        with:
+          path: .webiny/cached-packages
+          key: packages-cache-${{ needs.init.outputs.ts }}
+      - name: Install dependencies
+        run: yarn --immutable
+      - name: Build packages
+        run: yarn build:quick
+      - name: Run tests
+        run: yarn test ${{ matrix.package.cmd }}
+    permissions:
+      id-token: write
+    if: needs.init.outputs.is-fork-pr != 'true'
   verdaccioPublish:
     name: Publish to Verdaccio
     needs: init

diff --git a/.github/workflows/pullRequestsCommandCypress.yml b/.github/workflows/pullRequestsCommandCypress.yml
@@ -534,3 +534,251 @@ jobs:
           yarn cypress run --browser chrome --spec "${{ matrix.cypress-folder
           }}"
     runs-on: ubuntu-latest
+  e2e-wby-cms-ddb-os-init:
+    needs: checkComment
+    name: E2E (DDB-OS) - Init
+    outputs:
+      day: ${{ steps.get-day.outputs.day }}
+      ts: ${{ steps.get-timestamp.outputs.ts }}
+      cypress-folders: ${{ steps.list-cypress-folders.outputs.cypress-folders }}
+    steps:
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 18
+      - uses: actions/checkout@v3
+      - name: Install Hub Utility
+        run: sudo apt-get install -y hub
+      - name: Checkout Pull Request
+        working-directory: ''
+        run: hub pr checkout ${{ github.event.issue.number }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
+      - name: Get day of the month
+        id: get-day
+        run: >-
+          echo "day=$(node --eval "console.log(new Date().getDate())")" >>
+          $GITHUB_OUTPUT
+      - name: Get timestamp
+        id: get-timestamp
+        run: >-
+          echo "ts=$(node --eval "console.log(new Date().getTime())")" >>
+          $GITHUB_OUTPUT
+      - name: List Cypress tests folders
+        id: list-cypress-folders
+        run: >-
+          echo "cypress-folders=$(node scripts/listCypressTestsFolders.js)" >>
+          $GITHUB_OUTPUT
+    runs-on: ubuntu-latest
+    env:
+      NODE_OPTIONS: '--max_old_space_size=4096'
+      YARN_ENABLE_IMMUTABLE_INSTALLS: false
+  e2e-wby-cms-ddb-os-project-setup:
+    needs: e2e-wby-cms-ddb-os-init
+    name: E2E (DDB-OS) - Project setup
+    outputs:
+      cypress-config: ${{ steps.save-cypress-config.outputs.cypress-config }}
+    environment: next
+    env:
+      NODE_OPTIONS: '--max_old_space_size=4096'
+      YARN_ENABLE_IMMUTABLE_INSTALLS: 'false'
+      CYPRESS_MAILOSAUR_API_KEY: ${{ secrets.CYPRESS_MAILOSAUR_API_KEY }}
+      PULUMI_CONFIG_PASSPHRASE: ${{ secrets.PULUMI_CONFIG_PASSPHRASE }}
+      PULUMI_SECRETS_PROVIDER: ${{ secrets.PULUMI_SECRETS_PROVIDER }}
+      WEBINY_PULUMI_BACKEND: >-
+        ${{ secrets.WEBINY_PULUMI_BACKEND }}${{
+        needs.e2e-wby-cms-ddb-os-init.outputs.ts }}_ddb
+      AWS_ELASTIC_SEARCH_DOMAIN_NAME: ${{ secrets.AWS_OPEN_SEARCH_DOMAIN_NAME }}
+      ELASTIC_SEARCH_ENDPOINT: ${{ secrets.OPEN_SEARCH_ENDPOINT }}
+      ELASTIC_SEARCH_INDEX_PREFIX: ${{ needs.e2e-wby-cms-ddb-os-init.outputs.ts }}_
+    steps:
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 18
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: arn:aws:iam::726952677045:role/GitHubActionsWebinyJs
+          aws-region: eu-central-1
+      - uses: actions/checkout@v3
+        with:
+          path: dev
+      - name: Install Hub Utility
+        run: sudo apt-get install -y hub
+      - name: Checkout Pull Request
+        working-directory: dev
+        run: hub pr checkout ${{ github.event.issue.number }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
+      - uses: actions/cache@v3
+        id: yarn-cache
+        with:
+          path: dev/.yarn/cache
+          key: yarn-${{ runner.os }}-${{ hashFiles('dev/**/yarn.lock') }}
+      - uses: actions/cache@v3
+        id: cached-packages
+        with:
+          path: dev/.webiny/cached-packages
+          key: >-
+            ${{ runner.os }}-${{ needs.e2e-wby-cms-ddb-os-init.outputs.day
+            }}-${{ secrets.RANDOM_CACHE_KEY_SUFFIX }}
+      - name: Install dependencies
+        working-directory: dev
+        run: yarn --immutable
+      - name: Build packages
+        working-directory: dev
+        run: yarn build:quick
+      - uses: actions/cache@v3
+        id: packages-cache
+        with:
+          path: dev/.webiny/cached-packages
+          key: packages-cache-${{ needs.e2e-wby-cms-ddb-os-init.outputs.ts }}
+      - name: Start Verdaccio local server
+        working-directory: dev
+        run: >-
+          yarn add pm2 verdaccio && npx pm2 start verdaccio -- -c
+          .verdaccio.yaml
+      - name: Configure NPM to use local registry
+        run: npm config set registry http://localhost:4873
+      - name: Set git email
+        run: git config --global user.email "[email protected]"
+      - name: Set git username
+        run: git config --global user.name "webiny-bot"
+      - name: Create ".npmrc" file in the project root, with a dummy auth token
+        working-directory: dev
+        run: echo '//localhost:4873/:_authToken="dummy-auth-token"' > .npmrc
+      - name: Version and publish to Verdaccio
+        working-directory: dev
+        run: yarn release --type=verdaccio
+      - name: Create verdaccio-files artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: verdaccio-files-ddb-os
+          retention-days: 1
+          path: |
+            dev/.verdaccio/
+            dev/.verdaccio.yaml
+      - name: Create directory
+        run: mkdir xyz
+      - name: Disable Webiny telemetry
+        run: >
+          mkdir ~/.webiny && echo '{ "id": "ci", "telemetry": false }' >
+          ~/.webiny/config
+      - name: Create a new Webiny project
+        working-directory: xyz
+        run: >
+          npx create-webiny-project@local-npm test-project --tag local-npm
+          --no-interactive --assign-to-yarnrc
+          '{"npmRegistryServer":"http://localhost:4873","unsafeHttpWhitelist":["localhost"]}'
+          --template-options '{"region":"${{ env.AWS_REGION
+          }}","storageOperations":"ddb-os"}'
+      - name: Print CLI version
+        working-directory: xyz/test-project
+        run: yarn webiny --version
+      - name: Create project-files artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: project-files-ddb-os
+          retention-days: 1
+          path: |
+            xyz/test-project/
+            !xyz/test-project/node_modules/**/*
+            !xyz/test-project/**/node_modules/**/*
+            !xyz/test-project/.yarn/cache/**/*
+      - name: Deploy Core
+        working-directory: xyz/test-project
+        run: yarn webiny deploy apps/core --env dev
+      - name: Deploy API
+        working-directory: xyz/test-project
+        run: yarn webiny deploy apps/api --env dev
+      - name: Deploy Admin Area
+        working-directory: xyz/test-project
+        run: yarn webiny deploy apps/admin --env dev
+      - name: Deploy Website
+        working-directory: xyz/test-project
+        run: yarn webiny deploy apps/website --env dev
+      - name: Create Cypress config
+        working-directory: dev
+        run: yarn setup-cypress --projectFolder ../xyz/test-project
+      - name: Save Cypress config
+        id: save-cypress-config
+        working-directory: dev
+        run: >-
+          echo "cypress-config=$(cat cypress-tests/cypress.config.ts | tr -d
+          '\t\n\r')" >> $GITHUB_OUTPUT
+      - name: Cypress - run installation wizard test
+        working-directory: dev/cypress-tests
+        run: >-
+          yarn cypress run --browser chrome --spec
+          "cypress/e2e/adminInstallation/**/*.cy.js"
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+  e2e-wby-cms-ddb-os-cypress-tests:
+    name: >-
+      ${{ matrix.cypress-folder }} (ddb-os, ${{ matrix.os }}, Node v${{
+      matrix.node }})
+    needs:
+      - e2e-wby-cms-ddb-os-init
+      - e2e-wby-cms-ddb-os-project-setup
+    strategy:
+      fail-fast: false
+      matrix:
+        os:
+          - ubuntu-latest
+        node:
+          - 18
+        cypress-folder: ${{ fromJson(needs.e2e-wby-cms-ddb-os-init.outputs.cypress-folders) }}
+    environment: next
+    env:
+      NODE_OPTIONS: '--max_old_space_size=4096'
+      YARN_ENABLE_IMMUTABLE_INSTALLS: 'false'
+      CYPRESS_MAILOSAUR_API_KEY: ${{ secrets.CYPRESS_MAILOSAUR_API_KEY }}
+      PULUMI_CONFIG_PASSPHRASE: ${{ secrets.PULUMI_CONFIG_PASSPHRASE }}
+      PULUMI_SECRETS_PROVIDER: ${{ secrets.PULUMI_SECRETS_PROVIDER }}
+      WEBINY_PULUMI_BACKEND: >-
+        ${{ secrets.WEBINY_PULUMI_BACKEND }}${{
+        needs.e2e-wby-cms-ddb-os-init.outputs.ts }}_ddb
+      AWS_ELASTIC_SEARCH_DOMAIN_NAME: ${{ secrets.AWS_OPEN_SEARCH_DOMAIN_NAME }}
+      ELASTIC_SEARCH_ENDPOINT: ${{ secrets.OPEN_SEARCH_ENDPOINT }}
+      ELASTIC_SEARCH_INDEX_PREFIX: ${{ needs.e2e-wby-cms-ddb-os-init.outputs.ts }}_
+    steps:
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 18
+      - uses: actions/checkout@v3
+        with:
+          path: dev
+      - name: Install Hub Utility
+        run: sudo apt-get install -y hub
+      - name: Checkout Pull Request
+        working-directory: dev
+        run: hub pr checkout ${{ github.event.issue.number }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
+      - uses: actions/cache@v3
+        with:
+          path: dev/.webiny/cached-packages
+          key: packages-cache-${{ needs.e2e-wby-cms-ddb-os-init.outputs.ts }}
+      - uses: actions/cache@v3
+        with:
+          path: dev/.yarn/cache
+          key: yarn-${{ runner.os }}-${{ hashFiles('dev/**/yarn.lock') }}
+      - name: Install dependencies
+        working-directory: dev
+        run: yarn --immutable
+      - name: Build packages
+        working-directory: dev
+        run: yarn build:quick
+      - name: Set up Cypress config
+        working-directory: dev
+        run: >-
+          echo '${{
+          needs.e2e-wby-cms-ddb-os-project-setup.outputs.cypress-config }}' >
+          cypress-tests/cypress.config.ts
+      - name: Cypress - run "${{ matrix.cypress-folder }}" tests
+        working-directory: dev/cypress-tests
+        timeout-minutes: 40
+        run: >-
+          yarn cypress run --browser chrome --spec "${{ matrix.cypress-folder
+          }}"
+    runs-on: ubuntu-latest