Enable OAuth2 authentication.

.github/workflows/sync_opal_plus.yml

@@ -60,6 +60,15 @@ jobs:
       - name: Create Pull Request for opal-plus
         working-directory: opal-plus
         run: |
-          gh pr create --repo permitio/opal-plus --assignee "$GITHUB_ACTOR" --reviewer "$GITHUB_ACTOR" --base master --head public-${{ github.ref_name }} --title "Sync changes from public OPAL repository" --body "This PR synchronizes changes from the public OPAL repository to the private OPAL Plus repository."
+          set -e
+          PR_NUMBER=$(gh pr list --repo permitio/opal-plus --base master --head public-master --json number --jq '.[0].number')
+          if [ -n "$PR_NUMBER" ]; then
+            echo "PR already exists: #$PR_NUMBER"
+            gh pr edit "$PR_NUMBER" --repo permitio/opal-plus --add-reviewer "$GITHUB_ACTOR" || true
+          else
+            gh pr create --repo permitio/opal-plus --assignee "$GITHUB_ACTOR" --reviewer "$GITHUB_ACTOR" --base master --head public-master --title "Sync changes from public OPAL repository" --body "This PR synchronizes changes from the public OPAL repository to the private OPAL Plus repository." || true
+            echo "New PR created."
+          fi
+        shell: bash
         env:
           GITHUB_TOKEN: ${{ steps.get_workflow_token.outputs.token }}

.github/workflows/tests.yml

@@ -25,6 +25,7 @@ jobs:
           --health-timeout 5s
           --health-retries 5
     runs-on: ubuntu-latest
+    timeout-minutes: 60
     strategy:
       matrix:
         python-version: ["3.9", "3.10", "3.11", "3.12"]
@@ -53,6 +54,7 @@ jobs:
 
   test-docker:
     runs-on: ubuntu-latest
+    timeout-minutes: 60
     steps:
       # BUILD PHASE
       - name: Checkout
@@ -97,21 +99,30 @@ jobs:
           tags: |
             permitio/opal-server:test
 
-      # TEST PHASE
-      - name: Create modified docker compose file
-        run: sed 's/:latest/:test/g' docker/docker-compose-with-callbacks.yml > docker/docker-compose-test.yml
-
-      - name: Bring up stack
-        run: docker-compose -f docker/docker-compose-test.yml up -d
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: "3.10"
 
-      - name: Check if OPA is healthy
-        run: ./scripts/wait-for.sh -t 2 http://localhost:8181/v1/data/users -- sleep 10 && curl -s "http://localhost:8181/v1/data/users" | jq '.result.bob.location.country == "US"'
+      - name: Install opal packages
+        run: |
+          python -m pip install -e ./packages/opal-common
+          python -m pip install -e ./packages/opal-client
+          python -m pip install -e ./packages/opal-server
+
+      - name: App Tests
+        working-directory: ./app-tests
+        env:
+          OPAL_IMAGE_TAG: test
+          OPAL_TESTS_POLICY_REPO_DEPLOY_KEY: ${{ secrets.OPAL_TESTS_POLICY_REPO_DEPLOY_KEY }}
+        run: |
+          # Prepare git for using tests policy repo
+          export OPAL_POLICY_REPO_SSH_KEY_PATH=$(realpath ./opal-tests-policy-repo-key)
+          echo "$OPAL_TESTS_POLICY_REPO_DEPLOY_KEY" > $OPAL_POLICY_REPO_SSH_KEY_PATH
+          chmod 400 $OPAL_POLICY_REPO_SSH_KEY_PATH
 
-      - name: Output container logs
-        run: docker-compose -f docker/docker-compose-test.yml logs
+          git config --global core.sshCommand "ssh -i $OPAL_POLICY_REPO_SSH_KEY_PATH -o IdentitiesOnly=yes"
+          git config --global user.name "$GITHUB_ACTOR"
+          git config --global user.email "<>"
 
-      - name: check if opal-client was brought up successfully
-        run: |
-          docker-compose -f docker/docker-compose-test.yml logs opal_client | grep "Connected to PubSub server"
-          docker-compose -f docker/docker-compose-test.yml logs opal_client | grep "Got policy bundle"
-          docker-compose -f docker/docker-compose-test.yml logs opal_client | grep 'PUT /v1/data/static -> 204'
+          ./run.sh

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.4.0
+    rev: v5.0.0
     hooks:
       - id: check-yaml
       - id: end-of-file-fixer

README.md

@@ -36,25 +36,46 @@ Open Policy Administration Layer
 
 OPAL is an administration layer for Policy Engines such as <a target="_blank" href="https://www.openpolicyagent.org/">Open Policy Agent (OPA)</a>, and <a target="_blank" href="https://github.com/permitio/cedar-agent">AWS' Cedar Agent</a> detecting changes to both policy and policy data in realtime and pushing live updates to your agents. OPAL brings open-policy up to the speed needed by live applications.
 
-As your application state changes (whether it's via your APIs, DBs, git, S3 or 3rd-party SaaS services), OPAL will make sure your services are always in sync with the authorization data and policy they need (and only those they need).
+As your app's data state changes (whether it's via your APIs, DBs, git, S3 or 3rd-party SaaS services), OPAL will make sure your services are always in sync with the authorization data and policy they need (and only those they need).
 
-Check out our main site at <a target="_blank" href="https://opal.ac">OPAL.ac</a>, <a target="_blank" href="https://youtu.be/tG8jrdcc7Zo">this video</a> briefly explaining OPAL and how it works with OPA, and a deeper dive into it at [this OWASP DevSlop talk](https://www.youtube.com/watch?v=1_Iz0tRQCH4).
+Check out OPAL's main site at <a target="_blank" href="https://opal.ac">OPAL.ac</a>
 
-## Why use OPAL?
+## OPAL Use Cases
 
 OPAL is the easiest way to keep your solution's authorization layer up-to-date in realtime. It aggregates policy and data from across the field and integrates them seamlessly into the authorization layer, and is microservices and cloud-native.
 
-## OPA + OPAL == 💜
+Here are some of the main use cases for using OPAL:
+* **End-to-End [Fine-Grained Authorization](https://www.permit.io/blog/what-is-fine-grained-authorization-fga) service** that can be used with any policy language or data store
+* [Google-Zanzibar](https://www.permit.io/blog/what-is-google-zanzibar) support for Policy as Code engines such as OPA and AWS Cedar
+* Streamline permissions in microservice architectures using [centralized policy configuration with decentralized data](https://www.permit.io/blog/best-practices-for-implementing-hybrid-cloud-security) sources and policy engines
+* Manage and automate the deployment of multiple Open Policy Agent engines in a Cloud-Native environment
+
+<img src="https://github.com/permitio/opal/assets/4082578/99d3dd95-a7ff-45c2-805e-3d533f8b1e8c" alt="simplified" border="0">
+
+OPAL  uses a client-server stateless architecture. OPAL-Servers publish policy and data updates over a lightweight (websocket) PubSub Channel, which OPAL-clients subscribe to via topics. Upon updates, each client fetches data directly (from the source) to load it into its managed Policy Engine instance.
+
+
+### OPA + OPAL == 💜
 
 While OPA (Open Policy Agent) decouples policy from code in a highly-performant and elegant way, the challenge of keeping policy agents up-to-date remains.
 This is especially true in applications, where each user interaction or API call may affect access-control decisions.
-OPAL runs in the background, supercharging policy-agents, keeping them in sync with events in realtime.
+OPAL runs in the background, supercharging policy agents and keeping them in sync with events in real time.
 
-## AWS Cedar + OPAL == 💪
+### AWS Cedar + OPAL == 💪
 
 Cedar is a very powerful policy language, which powers AWS' AVP (Amazon Verified Permissions) - but what if you want to enjoy the power of Cedar on another cloud, locally, or on premise?
 This is where [Cedar-Agent](https://github.com/permitio/cedar-agent) and OPAL come in.
 
+This [video](https://youtu.be/tG8jrdcc7Zo) briefly explains OPAL and how it works with OPA, and a deeper dive into it at [this OWASP DevSlop talk](https://www.youtube.com/watch?v=1_Iz0tRQCH4).
+
+## Who's Using OPAL?
+OPAL is being used as the core engine of Permit.io Authorization Service and serves in production:
+* \> 10,000 policy engines deployment
+* \> 100,000 policy changes and data synchronizations every day
+* \> 10,000,000 authorization checks every day
+
+Besides Permit, OPAL is being used in Production in **Tesla**, **Walmart**, **The NBA**, **Intel**, **Cisco**, **Live-Oak Bank**, and thousands of other development teams and companies of all sizes.
+
 ## Documentation
 
 - 📃 &nbsp; [Full documentation is available here](https://docs.opal.ac)
@@ -104,22 +125,18 @@ curl -L https://raw.githubusercontent.com/permitio/opal/master/docker/docker-com
 
 - 🎨 &nbsp; [Key concepts and design](https://docs.opal.ac/overview/design)
 - 🏗️ &nbsp; [Architecture](https://docs.opal.ac/overview/architecture)
-<be>
-<br>
-OPAL  uses a client-server stateless architecture. OPAL-Servers publish policy and data updates over a lightweight (websocket) PubSub Channel, which OPAL-clients subscribe to via topics. Upon updates each client fetches data directly (from source) to load it in to its managed OPA instance.
-  <br>
-<img src="https://github.com/permitio/opal/assets/4082578/99d3dd95-a7ff-45c2-805e-3d533f8b1e8c" alt="simplified" border="0">
 <br>
-📖 &nbsp; For further reading check out our [Blog](https://bit.ly/opal_blog).
+
+📖 For further reading, check out our [Blog](https://io.permit.io/opal-readme-blog)
 
 ## Community
 
-Come talk to us about OPAL, or authorization in general - we would love to hear from you ❤️
+ We would love to chat with you about OPAL. [Join our Slack community](https://io.permit.io/opal-readme-slack) to chat about authorization, open-source, realtime communication, tech, or anything else!
 
-You can raise questions and ask for features to be added to the road-map in our [**Github discussions**](https://github.com/permitio/opal/discussions), report issues in [**Github issues**](https://github.com/permitio/opal/issues), follow us on Twitter to get the latest OPAL updates, and join our Slack community to chat about authorization, open-source, realtime communication, tech, or anything else!
+You can raise questions and ask for features to be added to the road-map in our [**Github discussions**](https://github.com/permitio/opal/discussions), report issues in [**Github issues**](https://github.com/permitio/opal/issues)
 </br>
 </br>
-If you are using our project, please consider giving us a ⭐️
+If you like our project, please consider giving us a ⭐️
 </br>
 
 [![Button][join-slack-link]][badge-slack-link] </br> [![Button][follow-twitter-link]][badge-twitter-link]

app-tests/README.md

@@ -0,0 +1,51 @@
+# OPAL Application Tests
+
+To fully test OPAL's core features as part of our CI flow,
+We're using a bash script and a docker-compose configuration that enables most of OPAL's important features.
+
+## How To Run Locally
+
+### Controlling the image tag
+
+By default, tests would run with the `latest` image tag (for both server & client).
+
+To configure another specific version:
+
+```bash
+export OPAL_IMAGE_TAG=0.7.1
+```
+
+Or if you want to test locally built images
+```bash
+make docker-build-next
+export OPAL_IMAGE_TAG=next
+```
+
+### Using a policy repo
+
+To test opal's git tracking capabilities, `run.sh` uses a dedicated GitHub repo ([opal-tests-policy-repo](https://github.com/permitio/opal-tests-policy-repo)) in which it creates branches and pushes new commits.
+
+If you're not accessible to that repo (not in `Permit.io`), Please fork our public [opal-example-policy-repo](https://github.com/permitio/opal-example-policy-repo), and override the repo URL to be used:
+```bash
+export [email protected]:your-org/your-repo.git
+```
+
+As `run.sh` requires push permissions, and as `opal-server` itself might need to authenticate GitHub (if your repo is private). If your GitHub ssh private key is not stored at `~/.ssh/id_rsa`, provide it using:
+```bash
+# Use an absolute path
+export OPAL_POLICY_REPO_SSH_KEY_PATH=$(realpath ./your_github_ssh_private_key)
+```
+
+
+### Putting it all together
+
+```bash
+make docker-build-next # To locally build opal images
+export OPAL_IMAGE_TAG=next # Otherwise would default to "latest"
+
+export [email protected]:your-org/your-repo.git # To use your own repo for testing (if you're not an Permit.io employee yet...)
+export OPAL_POLICY_REPO_SSH_KEY_PATH=$(realpath ./your_github_ssh_private_key) # If your GitHub ssh key isn't in "~.ssh/id_rsa"
+
+cd app-tests
+./run.sh
+```

app-tests/docker-compose-app-tests.yml

@@ -0,0 +1,58 @@
+services:
+  broadcast_channel:
+    image: postgres:alpine
+    environment:
+      - POSTGRES_DB=postgres
+      - POSTGRES_USER=postgres
+      - POSTGRES_PASSWORD=postgres
+
+  opal_server:
+    image: permitio/opal-server:${OPAL_IMAGE_TAG:-latest}
+    deploy:
+      mode: replicated
+      replicas: 2
+      endpoint_mode: vip
+    environment:
+      - OPAL_BROADCAST_URI=postgres://postgres:postgres@broadcast_channel:5432/postgres
+      - UVICORN_NUM_WORKERS=4
+      - OPAL_POLICY_REPO_URL=${OPAL_POLICY_REPO_URL:[email protected]:permitio/opal-tests-policy-repo.git}
+      - OPAL_POLICY_REPO_MAIN_BRANCH=${POLICY_REPO_BRANCH}
+      - OPAL_POLICY_REPO_SSH_KEY=${OPAL_POLICY_REPO_SSH_KEY}
+      - OPAL_DATA_CONFIG_SOURCES={"config":{"entries":[{"url":"http://opal_server:7002/policy-data","config":{"headers":{"Authorization":"Bearer ${OPAL_CLIENT_TOKEN}"}},"topics":["policy_data"],"dst_path":"/static"}]}}
+      - OPAL_LOG_FORMAT_INCLUDE_PID=true
+      - OPAL_POLICY_REPO_WEBHOOK_SECRET=xxxxx
+      - OPAL_POLICY_REPO_WEBHOOK_PARAMS={"secret_header_name":"x-webhook-token","secret_type":"token","secret_parsing_regex":"(.*)","event_request_key":"gitEvent","push_event_value":"git.push"}
+      - OPAL_AUTH_PUBLIC_KEY=${OPAL_AUTH_PUBLIC_KEY}
+      - OPAL_AUTH_PRIVATE_KEY=${OPAL_AUTH_PRIVATE_KEY}
+      - OPAL_AUTH_MASTER_TOKEN=${OPAL_AUTH_MASTER_TOKEN}
+      - OPAL_AUTH_JWT_AUDIENCE=https://api.opal.ac/v1/
+      - OPAL_AUTH_JWT_ISSUER=https://opal.ac/
+      - OPAL_STATISTICS_ENABLED=true
+    ports:
+      - "7002-7003:7002"
+    depends_on:
+      - broadcast_channel
+
+  opal_client:
+    image: permitio/opal-client:${OPAL_IMAGE_TAG:-latest}
+    deploy:
+      mode: replicated
+      replicas: 2
+      endpoint_mode: vip
+    environment:
+      - OPAL_SERVER_URL=http://opal_server:7002
+      - OPAL_LOG_FORMAT_INCLUDE_PID=true
+      - OPAL_INLINE_OPA_LOG_FORMAT=http
+      - OPAL_SHOULD_REPORT_ON_DATA_UPDATES=True
+      - OPAL_DEFAULT_UPDATE_CALLBACKS={"callbacks":[["http://opal_server:7002/data/callback_report",{"method":"post","process_data":false,"headers":{"Authorization":"Bearer ${OPAL_CLIENT_TOKEN}","content-type":"application/json"}}]]}
+      - OPAL_OPA_HEALTH_CHECK_POLICY_ENABLED=True
+      - OPAL_CLIENT_TOKEN=${OPAL_CLIENT_TOKEN}
+      - OPAL_AUTH_JWT_AUDIENCE=https://api.opal.ac/v1/
+      - OPAL_AUTH_JWT_ISSUER=https://opal.ac/
+      - OPAL_STATISTICS_ENABLED=true
+    ports:
+      - "7766-7767:7000"
+      - "8181-8182:8181"
+    depends_on:
+      - opal_server
+    command: sh -c "exec ./wait-for.sh opal_server:7002 --timeout=20 -- ./start.sh"