Merge pull request #645 from permitio/roe/per-10476-write-opal-applic…

…ation-tests Introduce docker & bash based application tests
permitio · Aug 29, 2024 · 539f2d9 · 539f2d9
2 parents 5a091e6 + 2dce27b
commit 539f2d9
Show file tree

Hide file tree

Showing 10 changed files with 299 additions and 22 deletions.
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -97,21 +97,30 @@ jobs:
           tags: |
             permitio/opal-server:test
 
-      # TEST PHASE
-      - name: Create modified docker compose file
-        run: sed 's/:latest/:test/g' docker/docker-compose-with-callbacks.yml > docker/docker-compose-test.yml
-
-      - name: Bring up stack
-        run: docker-compose -f docker/docker-compose-test.yml up -d
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: "3.10"
 
-      - name: Check if OPA is healthy
-        run: ./scripts/wait-for.sh -t 2 http://localhost:8181/v1/data/users -- sleep 10 && curl -s "http://localhost:8181/v1/data/users" | jq '.result.bob.location.country == "US"'
+      - name: Install opal packages
+        run: |
+          python -m pip install -e ./packages/opal-common
+          python -m pip install -e ./packages/opal-client
+          python -m pip install -e ./packages/opal-server
+
+      - name: App Tests
+        working-directory: ./app-tests
+        env:
+          OPAL_IMAGE_TAG: test
+          OPAL_TESTS_POLICY_REPO_DEPLOY_KEY: ${{ secrets.OPAL_TESTS_POLICY_REPO_DEPLOY_KEY }}
+        run: |
+          # Prepare git for using tests policy repo
+          export OPAL_POLICY_REPO_SSH_KEY_PATH=$(realpath ./opal-tests-policy-repo-key)
+          echo "$OPAL_TESTS_POLICY_REPO_DEPLOY_KEY" > $OPAL_POLICY_REPO_SSH_KEY_PATH
+          chmod 400 $OPAL_POLICY_REPO_SSH_KEY_PATH
 
-      - name: Output container logs
-        run: docker-compose -f docker/docker-compose-test.yml logs
+          git config --global core.sshCommand "ssh -i $OPAL_POLICY_REPO_SSH_KEY_PATH -o IdentitiesOnly=yes"
+          git config --global user.name "$GITHUB_ACTOR"
+          git config --global user.email "<>"
 
-      - name: check if opal-client was brought up successfully
-        run: |
-          docker-compose -f docker/docker-compose-test.yml logs opal_client | grep "Connected to PubSub server"
-          docker-compose -f docker/docker-compose-test.yml logs opal_client | grep "Got policy bundle"
-          docker-compose -f docker/docker-compose-test.yml logs opal_client | grep 'PUT /v1/data/static -> 204'
+          ./run.sh
diff --git a/app-tests/README.md b/app-tests/README.md
@@ -0,0 +1,51 @@
+# OPAL Application Tests
+
+To fully test OPAL's core features as part of our CI flow,
+We're using a bash script and a docker-compose configuration that enables most of OPAL's important features.
+
+## How To Run Locally
+
+### Controlling the image tag
+
+By default, tests would run with the `latest` image tag (for both server & client).
+
+To configure another specific version:
+
+```bash
+export OPAL_IMAGE_TAG=0.7.1
+```
+
+Or if you want to test locally built images
+```bash
+make docker-build-next
+export OPAL_IMAGE_TAG=next
+```
+
+### Using a policy repo
+
+To test opal's git tracking capabilities, `run.sh` uses a dedicated GitHub repo ([opal-tests-policy-repo](https://github.com/permitio/opal-tests-policy-repo)) in which it creates branches and pushes new commits.
+
+If you're not accessible to that repo (not in `Permit.io`), Please fork our public [opal-example-policy-repo](https://github.com/permitio/opal-example-policy-repo), and override the repo URL to be used:
+```bash
+export [email protected]:your-org/your-repo.git
+```
+
+As `run.sh` requires push permissions, and as `opal-server` itself might need to authenticate GitHub (if your repo is private). If your GitHub ssh private key is not stored at `~/.ssh/id_rsa`, provide it using:
+```bash
+# Use an absolute path
+export OPAL_POLICY_REPO_SSH_KEY_PATH=$(realpath ./your_github_ssh_private_key)
+```
+
+
+### Putting it all together
+
+```bash
+make docker-build-next # To locally build opal images
+export OPAL_IMAGE_TAG=next # Otherwise would default to "latest"
+
+export [email protected]:your-org/your-repo.git # To use your own repo for testing (if you're not an Permit.io employee yet...)
+export OPAL_POLICY_REPO_SSH_KEY_PATH=$(realpath ./your_github_ssh_private_key) # If your GitHub ssh key isn't in "~.ssh/id_rsa"
+
+cd app-tests
+./run.sh
+```
diff --git a/app-tests/docker-compose-app-tests.yml b/app-tests/docker-compose-app-tests.yml
@@ -0,0 +1,58 @@
+services:
+  broadcast_channel:
+    image: postgres:alpine
+    environment:
+      - POSTGRES_DB=postgres
+      - POSTGRES_USER=postgres
+      - POSTGRES_PASSWORD=postgres
+
+  opal_server:
+    image: permitio/opal-server:${OPAL_IMAGE_TAG:-latest}
+    deploy:
+      mode: replicated
+      replicas: 2
+      endpoint_mode: vip
+    environment:
+      - OPAL_BROADCAST_URI=postgres://postgres:postgres@broadcast_channel:5432/postgres
+      - UVICORN_NUM_WORKERS=4
+      - OPAL_POLICY_REPO_URL=${OPAL_POLICY_REPO_URL:[email protected]:permitio/opal-tests-policy-repo.git}
+      - OPAL_POLICY_REPO_MAIN_BRANCH=${POLICY_REPO_BRANCH}
+      - OPAL_POLICY_REPO_SSH_KEY=${OPAL_POLICY_REPO_SSH_KEY}
+      - OPAL_DATA_CONFIG_SOURCES={"config":{"entries":[{"url":"http://opal_server:7002/policy-data","config":{"headers":{"Authorization":"Bearer ${OPAL_CLIENT_TOKEN}"}},"topics":["policy_data"],"dst_path":"/static"}]}}
+      - OPAL_LOG_FORMAT_INCLUDE_PID=true
+      - OPAL_POLICY_REPO_WEBHOOK_SECRET=xxxxx
+      - OPAL_POLICY_REPO_WEBHOOK_PARAMS={"secret_header_name":"x-webhook-token","secret_type":"token","secret_parsing_regex":"(.*)","event_request_key":"gitEvent","push_event_value":"git.push"}
+      - OPAL_AUTH_PUBLIC_KEY=${OPAL_AUTH_PUBLIC_KEY}
+      - OPAL_AUTH_PRIVATE_KEY=${OPAL_AUTH_PRIVATE_KEY}
+      - OPAL_AUTH_MASTER_TOKEN=${OPAL_AUTH_MASTER_TOKEN}
+      - OPAL_AUTH_JWT_AUDIENCE=https://api.opal.ac/v1/
+      - OPAL_AUTH_JWT_ISSUER=https://opal.ac/
+      - OPAL_STATISTICS_ENABLED=true
+    ports:
+      - "7002-7003:7002"
+    depends_on:
+      - broadcast_channel
+
+  opal_client:
+    image: permitio/opal-client:${OPAL_IMAGE_TAG:-latest}
+    deploy:
+      mode: replicated
+      replicas: 2
+      endpoint_mode: vip
+    environment:
+      - OPAL_SERVER_URL=http://opal_server:7002
+      - OPAL_LOG_FORMAT_INCLUDE_PID=true
+      - OPAL_INLINE_OPA_LOG_FORMAT=http
+      - OPAL_SHOULD_REPORT_ON_DATA_UPDATES=True
+      - OPAL_DEFAULT_UPDATE_CALLBACKS={"callbacks":[["http://opal_server:7002/data/callback_report",{"method":"post","process_data":false,"headers":{"Authorization":"Bearer ${OPAL_CLIENT_TOKEN}","content-type":"application/json"}}]]}
+      - OPAL_OPA_HEALTH_CHECK_POLICY_ENABLED=True
+      - OPAL_CLIENT_TOKEN=${OPAL_CLIENT_TOKEN}
+      - OPAL_AUTH_JWT_AUDIENCE=https://api.opal.ac/v1/
+      - OPAL_AUTH_JWT_ISSUER=https://opal.ac/
+      - OPAL_STATISTICS_ENABLED=true
+    ports:
+      - "7766-7767:7000"
+      - "8181-8182:8181"
+    depends_on:
+      - opal_server
+    command: sh -c "exec ./wait-for.sh opal_server:7002 --timeout=20 -- ./start.sh"
diff --git a/app-tests/run.sh b/app-tests/run.sh
@@ -0,0 +1,159 @@
+#!/bin/bash
+set -e
+
+export OPAL_AUTH_PUBLIC_KEY
+export OPAL_AUTH_PRIVATE_KEY
+export OPAL_AUTH_MASTER_TOKEN
+export OPAL_CLIENT_TOKEN
+export OPAL_DATA_SOURCE_TOKEN
+
+function generate_opal_keys {
+  echo "- Generating OPAL keys"
+
+  ssh-keygen -q -t rsa -b 4096 -m pem -f opal_crypto_key -N ""
+  OPAL_AUTH_PUBLIC_KEY="$(cat opal_crypto_key.pub)"
+  OPAL_AUTH_PRIVATE_KEY="$(tr '\n' '_' < opal_crypto_key)"
+  rm opal_crypto_key.pub opal_crypto_key
+
+  OPAL_AUTH_MASTER_TOKEN="$(openssl rand -hex 16)"
+  OPAL_AUTH_JWT_AUDIENCE=https://api.opal.ac/v1/ OPAL_AUTH_JWT_ISSUER=https://opal.ac/ OPAL_REPO_WATCHER_ENABLED=0 \
+    opal-server run &
+  sleep 2;
+
+  OPAL_CLIENT_TOKEN="$(opal-client obtain-token "$OPAL_AUTH_MASTER_TOKEN" --type client)"
+  OPAL_DATA_SOURCE_TOKEN="$(opal-client obtain-token "$OPAL_AUTH_MASTER_TOKEN" --type datasource)"
+  # shellcheck disable=SC2009
+  ps -ef | grep opal-server | grep -v grep | awk '{print $2}' | xargs kill
+  sleep 5;
+
+  echo "- Create .env file"
+  rm -f .env
+  (
+    echo "OPAL_AUTH_PUBLIC_KEY=\"$OPAL_AUTH_PUBLIC_KEY\"";
+    echo "OPAL_AUTH_PRIVATE_KEY=\"$OPAL_AUTH_PRIVATE_KEY\"";
+    echo "OPAL_AUTH_MASTER_TOKEN=\"$OPAL_AUTH_MASTER_TOKEN\"";
+    echo "OPAL_CLIENT_TOKEN=\"$OPAL_CLIENT_TOKEN\"";
+    echo "OPAL_AUTH_PRIVATE_KEY_PASSPHRASE=\"$OPAL_AUTH_PRIVATE_KEY_PASSPHRASE\""
+  ) > .env
+}
+
+function prepare_policy_repo {
+  echo "- Clone tests policy repo to create test's branch"
+  export OPAL_POLICY_REPO_URL
+  export POLICY_REPO_BRANCH
+  OPAL_POLICY_REPO_URL=${OPAL_POLICY_REPO_URL:-git@github.com:permitio/opal-tests-policy-repo.git}
+  POLICY_REPO_BRANCH=test-$RANDOM$RANDOM
+  rm -rf ./opal-tests-policy-repo
+  git clone "$OPAL_POLICY_REPO_URL"
+  cd opal-tests-policy-repo
+  git checkout -b $POLICY_REPO_BRANCH
+  git push --set-upstream origin $POLICY_REPO_BRANCH
+  cd -
+
+  # That's for the docker-compose to use, set ssh key from "~/.ssh/id_rsa", unless another path/key data was configured
+  export OPAL_POLICY_REPO_SSH_KEY
+  OPAL_POLICY_REPO_SSH_KEY_PATH=${OPAL_POLICY_REPO_SSH_KEY_PATH:-~/.ssh/id_rsa}
+  OPAL_POLICY_REPO_SSH_KEY=${OPAL_POLICY_REPO_SSH_KEY:-$(cat "$OPAL_POLICY_REPO_SSH_KEY_PATH")}
+}
+
+function compose {
+  docker compose -f ./docker-compose-app-tests.yml --env-file .env "$@"
+}
+
+function check_clients_logged {
+  echo "- Looking for msg '$1' in client's logs"
+  compose logs --index 1 opal_client | grep -q "$1"
+  compose logs --index 2 opal_client | grep -q "$1"
+}
+
+function check_no_error {
+  # Without index would output all replicas
+  if compose logs opal_client | grep -q 'ERROR'; then
+    echo "- Found error in logs"
+    exit 1
+  fi
+}
+
+function clean_up {
+    ARG=$?
+    if [[ "$ARG" -ne 0 ]]; then
+      echo "*** Test Failed ***"
+      echo ""
+      compose logs
+    else
+      echo "*** Test Passed ***"
+      echo ""
+    fi
+    compose down
+    cd opal-tests-policy-repo; git push -d origin $POLICY_REPO_BRANCH; cd - # Remove remote tests branch
+    rm -rf ./opal-tests-policy-repo
+    exit $ARG
+}
+
+function test_push_policy {
+  echo "- Testing pushing policy $1"
+  regofile="$1.rego"
+  cd opal-tests-policy-repo
+  echo "package $1" > "$regofile"
+  git add "$regofile"
+  git commit -m "Add $regofile"
+  git push
+  cd -
+
+  curl -s --request POST 'http://localhost:7002/webhook' --header 'Content-Type: application/json' --header 'x-webhook-token: xxxxx' --data-raw '{"gitEvent":"git.push","repository":{"git_url":"'"$OPAL_POLICY_REPO_URL"'"}}'
+  sleep 5
+  check_clients_logged "PUT /v1/policies/$regofile -> 200"
+}
+
+function test_data_publish {
+  echo "- Testing data publish for user $1"
+  user=$1
+  OPAL_CLIENT_TOKEN=$OPAL_DATA_SOURCE_TOKEN opal-client publish-data-update --src-url https://api.country.is/23.54.6.78 -t policy_data --dst-path "/users/$user/location"
+  sleep 5
+  check_clients_logged "PUT /v1/data/users/$user/location -> 204"
+}
+
+function test_statistics {
+    echo "- Testing statistics feature"
+    # Make sure 2 servers & 2 clients (repeat few times cause different workers might response)
+    for _ in {1..10}; do
+      curl -s 'http://localhost:7002/stats' --header "Authorization: Bearer $OPAL_DATA_SOURCE_TOKEN" | grep '"client_count":2,"server_count":2'
+    done
+}
+
+function main {
+  # Setup
+  generate_opal_keys
+  prepare_policy_repo
+
+  trap clean_up EXIT
+
+  # Bring up OPAL containers
+  compose down --remove-orphans
+  compose up -d
+  sleep 10
+
+  # Check containers started correctly
+  check_clients_logged "Connected to PubSub server"
+  check_clients_logged "Got policy bundle"
+  check_clients_logged 'PUT /v1/data/static -> 204'
+  check_no_error
+
+  # Test functionality
+  test_data_publish "bob"
+  test_push_policy "something"
+  test_statistics
+
+  echo "- Testing broadcast channel disconnection"
+  compose restart broadcast_channel
+  sleep 10
+
+  test_data_publish "alice"
+  test_push_policy "another"
+  test_data_publish "sunil"
+  test_data_publish "eve"
+  test_push_policy "best_one_yet"
+  # TODO: Test statistics feature again after broadcaster restart (should first fix statistics bug)
+}
+
+main
diff --git a/docker/docker-compose-api-policy-source-example.yml b/docker/docker-compose-api-policy-source-example.yml
@@ -36,7 +36,7 @@ services:
       # configures from where the opal client should initially fetch data (when it first goes up, after disconnection, etc).
       # the data sources represents from where the opal clients should get a "complete picture" of the data they need.
       # after the initial sources are fetched, the client will subscribe only to update notifications sent by the server.
-      - OPAL_DATA_CONFIG_SOURCES={"config":{"entries":[{"url":"http://opal-server:7002/policy-data","topics":["policy_data"],"dst_path":"/static"}]}}
+      - OPAL_DATA_CONFIG_SOURCES={"config":{"entries":[{"url":"http://opal_server:7002/policy-data","topics":["policy_data"],"dst_path":"/static"}]}}
       - OPAL_LOG_FORMAT_INCLUDE_PID=true
     ports:
       # exposes opal server on the host machine, you can access the server at: http://localhost:7002

diff --git a/docker/docker-compose-with-kafka-example.yml b/docker/docker-compose-with-kafka-example.yml
@@ -69,7 +69,7 @@ services:
       # configures from where the opal client should initially fetch data (when it first goes up, after disconnection, etc).
       # the data sources represents from where the opal clients should get a "complete picture" of the data they need.
       # after the initial sources are fetched, the client will subscribe only to update notifications sent by the server.
-      - OPAL_DATA_CONFIG_SOURCES={"config":{"entries":[{"url":"http://opal-server:7002/policy-data","topics":["policy_data"],"dst_path":"/static"}]}}
+      - OPAL_DATA_CONFIG_SOURCES={"config":{"entries":[{"url":"http://opal_server:7002/policy-data","topics":["policy_data"],"dst_path":"/static"}]}}
       - OPAL_LOG_FORMAT_INCLUDE_PID=true
     ports:
       # exposes opal server on the host machine, you can access the server at: http://localhost:7002

diff --git a/docker/docker-compose-with-rate-limiting.yml b/docker/docker-compose-with-rate-limiting.yml
@@ -30,7 +30,7 @@ services:
       # configures from where the opal client should initially fetch data (when it first goes up, after disconnection, etc).
       # the data sources represents from where the opal clients should get a "complete picture" of the data they need.
       # after the initial sources are fetched, the client will subscribe only to update notifications sent by the server.
-      - OPAL_DATA_CONFIG_SOURCES={"config":{"entries":[{"url":"http://opal-server:7002/policy-data","topics":["policy_data"],"dst_path":"/static"}]}}
+      - OPAL_DATA_CONFIG_SOURCES={"config":{"entries":[{"url":"http://opal_server:7002/policy-data","topics":["policy_data"],"dst_path":"/static"}]}}
       - OPAL_LOG_FORMAT_INCLUDE_PID=true
       # Turns on rate limiting in the server
       # supported formats documented here: https://limits.readthedocs.io/en/stable/quickstart.html#rate-limit-string-notation

diff --git a/docker/docker-compose-with-security.yml b/docker/docker-compose-with-security.yml
@@ -45,7 +45,7 @@ services:
       # after the initial sources are fetched, the client will subscribe only to update notifications sent by the server.
       # please notice - since we fetch data entries from the OPAL server itself, we need to authenticate to that endpoint
       # with the client's token (JWT).
-      - OPAL_DATA_CONFIG_SOURCES={"config":{"entries":[{"url":"http://opal-server:7002/policy-data","config":{"headers":{"Authorization":"Bearer ${OPAL_CLIENT_TOKEN}"}},"topics":["policy_data"],"dst_path":"/static"}]}}
+      - OPAL_DATA_CONFIG_SOURCES={"config":{"entries":[{"url":"http://opal_server:7002/policy-data","config":{"headers":{"Authorization":"Bearer ${OPAL_CLIENT_TOKEN}"}},"topics":["policy_data"],"dst_path":"/static"}]}}
       - OPAL_LOG_FORMAT_INCLUDE_PID=true
       # --------------------------------------------------------------------------------
       # the jwt audience and jwt issuer are not typically necessary in real setups

diff --git a/docker/docker-compose-with-statistics.yml b/docker/docker-compose-with-statistics.yml
@@ -31,7 +31,7 @@ services:
       # configures from where the opal client should initially fetch data (when it first goes up, after disconnection, etc).
       # the data sources represents from where the opal clients should get a "complete picture" of the data they need.
       # after the initial sources are fetched, the client will subscribe only to update notifications sent by the server.
-      - OPAL_DATA_CONFIG_SOURCES={"config":{"entries":[{"url":"http://opal-server:7002/policy-data","topics":["policy_data"],"dst_path":"/static"}]}}
+      - OPAL_DATA_CONFIG_SOURCES={"config":{"entries":[{"url":"http://opal_server:7002/policy-data","topics":["policy_data"],"dst_path":"/static"}]}}
       - OPAL_LOG_FORMAT_INCLUDE_PID=true
       # turning on statistics collection on the server side
       - OPAL_STATISTICS_ENABLED=true

diff --git a/documentation/docs/opal-plus/features.mdx b/documentation/docs/opal-plus/features.mdx
@@ -39,13 +39,13 @@ Configure the [OPAL_STATISTICS_ENABLED=true](/getting-started/configuration) env
 
 You can then monitor the state of your OPAL+ cluster by calling the `/stats` API route on the server.
 ```bash
-curl http://opal-server:8181/stats -H "Authorization: Bearer <token>"
+curl http://opal_server:8181/stats -H "Authorization: Bearer <token>"
 # { "uptime": "2024-07-14T14:55:02.710Z", "version": "0.7.8", "client_count": 1, "server_count": 1 }
 ```
 
 You can also get detailed information about the OPAL+ clients and servers by calling the `/statistics` API route on the server.
 ```bash
-curl http://opal-server:8181/statistics -H "Authorization: Bear <token>"
+curl http://opal_server:8181/statistics -H "Authorization: Bear <token>"
 ```
 ```json
 {