diff --git a/.github/workflows/sync_opal_plus.yml b/.github/workflows/sync_opal_plus.yml index ab92edaa6..18bbad2ce 100644 --- a/.github/workflows/sync_opal_plus.yml +++ b/.github/workflows/sync_opal_plus.yml @@ -60,6 +60,15 @@ jobs: - name: Create Pull Request for opal-plus working-directory: opal-plus run: | - gh pr create --repo permitio/opal-plus --assignee "$GITHUB_ACTOR" --reviewer "$GITHUB_ACTOR" --base master --head public-${{ github.ref_name }} --title "Sync changes from public OPAL repository" --body "This PR synchronizes changes from the public OPAL repository to the private OPAL Plus repository." + set -e + PR_NUMBER=$(gh pr list --repo permitio/opal-plus --base master --head public-master --json number --jq '.[0].number') + if [ -n "$PR_NUMBER" ]; then + echo "PR already exists: #$PR_NUMBER" + gh pr edit "$PR_NUMBER" --repo permitio/opal-plus --add-reviewer "$GITHUB_ACTOR" || true + else + gh pr create --repo permitio/opal-plus --assignee "$GITHUB_ACTOR" --reviewer "$GITHUB_ACTOR" --base master --head public-master --title "Sync changes from public OPAL repository" --body "This PR synchronizes changes from the public OPAL repository to the private OPAL Plus repository." || true + echo "New PR created." + fi + shell: bash env: GITHUB_TOKEN: ${{ steps.get_workflow_token.outputs.token }} diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 8638cba36..5d324cd39 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -25,6 +25,7 @@ jobs: --health-timeout 5s --health-retries 5 runs-on: ubuntu-latest + timeout-minutes: 60 strategy: matrix: python-version: ["3.9", "3.10", "3.11", "3.12"] @@ -53,6 +54,7 @@ jobs: test-docker: runs-on: ubuntu-latest + timeout-minutes: 60 steps: # BUILD PHASE - name: Checkout @@ -97,21 +99,30 @@ jobs: tags: | permitio/opal-server:test - # TEST PHASE - - name: Create modified docker compose file - run: sed 's/:latest/:test/g' docker/docker-compose-with-callbacks.yml > docker/docker-compose-test.yml - - - name: Bring up stack - run: docker-compose -f docker/docker-compose-test.yml up -d + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: "3.10" - - name: Check if OPA is healthy - run: ./scripts/wait-for.sh -t 2 http://localhost:8181/v1/data/users -- sleep 10 && curl -s "http://localhost:8181/v1/data/users" | jq '.result.bob.location.country == "US"' + - name: Install opal packages + run: | + python -m pip install -e ./packages/opal-common + python -m pip install -e ./packages/opal-client + python -m pip install -e ./packages/opal-server + + - name: App Tests + working-directory: ./app-tests + env: + OPAL_IMAGE_TAG: test + OPAL_TESTS_POLICY_REPO_DEPLOY_KEY: ${{ secrets.OPAL_TESTS_POLICY_REPO_DEPLOY_KEY }} + run: | + # Prepare git for using tests policy repo + export OPAL_POLICY_REPO_SSH_KEY_PATH=$(realpath ./opal-tests-policy-repo-key) + echo "$OPAL_TESTS_POLICY_REPO_DEPLOY_KEY" > $OPAL_POLICY_REPO_SSH_KEY_PATH + chmod 400 $OPAL_POLICY_REPO_SSH_KEY_PATH - - name: Output container logs - run: docker-compose -f docker/docker-compose-test.yml logs + git config --global core.sshCommand "ssh -i $OPAL_POLICY_REPO_SSH_KEY_PATH -o IdentitiesOnly=yes" + git config --global user.name "$GITHUB_ACTOR" + git config --global user.email "<>" - - name: check if opal-client was brought up successfully - run: | - docker-compose -f docker/docker-compose-test.yml logs opal_client | grep "Connected to PubSub server" - docker-compose -f docker/docker-compose-test.yml logs opal_client | grep "Got policy bundle" - docker-compose -f docker/docker-compose-test.yml logs opal_client | grep 'PUT /v1/data/static -> 204' + ./run.sh diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 481d33c18..e57f8ef62 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v5.0.0 hooks: - id: check-yaml - id: end-of-file-fixer diff --git a/README.md b/README.md index cd35f2a50..71fd09b50 100644 --- a/README.md +++ b/README.md @@ -36,25 +36,46 @@ Open Policy Administration Layer OPAL is an administration layer for Policy Engines such as Open Policy Agent (OPA), and AWS' Cedar Agent detecting changes to both policy and policy data in realtime and pushing live updates to your agents. OPAL brings open-policy up to the speed needed by live applications. -As your application state changes (whether it's via your APIs, DBs, git, S3 or 3rd-party SaaS services), OPAL will make sure your services are always in sync with the authorization data and policy they need (and only those they need). +As your app's data state changes (whether it's via your APIs, DBs, git, S3 or 3rd-party SaaS services), OPAL will make sure your services are always in sync with the authorization data and policy they need (and only those they need). -Check out our main site at OPAL.ac, this video briefly explaining OPAL and how it works with OPA, and a deeper dive into it at [this OWASP DevSlop talk](https://www.youtube.com/watch?v=1_Iz0tRQCH4). +Check out OPAL's main site at OPAL.ac -## Why use OPAL? +## OPAL Use Cases OPAL is the easiest way to keep your solution's authorization layer up-to-date in realtime. It aggregates policy and data from across the field and integrates them seamlessly into the authorization layer, and is microservices and cloud-native. -## OPA + OPAL == 💜 +Here are some of the main use cases for using OPAL: +* **End-to-End [Fine-Grained Authorization](https://www.permit.io/blog/what-is-fine-grained-authorization-fga) service** that can be used with any policy language or data store +* [Google-Zanzibar](https://www.permit.io/blog/what-is-google-zanzibar) support for Policy as Code engines such as OPA and AWS Cedar +* Streamline permissions in microservice architectures using [centralized policy configuration with decentralized data](https://www.permit.io/blog/best-practices-for-implementing-hybrid-cloud-security) sources and policy engines +* Manage and automate the deployment of multiple Open Policy Agent engines in a Cloud-Native environment + +simplified + +OPAL uses a client-server stateless architecture. OPAL-Servers publish policy and data updates over a lightweight (websocket) PubSub Channel, which OPAL-clients subscribe to via topics. Upon updates, each client fetches data directly (from the source) to load it into its managed Policy Engine instance. + + +### OPA + OPAL == 💜 While OPA (Open Policy Agent) decouples policy from code in a highly-performant and elegant way, the challenge of keeping policy agents up-to-date remains. This is especially true in applications, where each user interaction or API call may affect access-control decisions. -OPAL runs in the background, supercharging policy-agents, keeping them in sync with events in realtime. +OPAL runs in the background, supercharging policy agents and keeping them in sync with events in real time. -## AWS Cedar + OPAL == 💪 +### AWS Cedar + OPAL == 💪 Cedar is a very powerful policy language, which powers AWS' AVP (Amazon Verified Permissions) - but what if you want to enjoy the power of Cedar on another cloud, locally, or on premise? This is where [Cedar-Agent](https://github.com/permitio/cedar-agent) and OPAL come in. +This [video](https://youtu.be/tG8jrdcc7Zo) briefly explains OPAL and how it works with OPA, and a deeper dive into it at [this OWASP DevSlop talk](https://www.youtube.com/watch?v=1_Iz0tRQCH4). + +## Who's Using OPAL? +OPAL is being used as the core engine of Permit.io Authorization Service and serves in production: +* \> 10,000 policy engines deployment +* \> 100,000 policy changes and data synchronizations every day +* \> 10,000,000 authorization checks every day + +Besides Permit, OPAL is being used in Production in **Tesla**, **Walmart**, **The NBA**, **Intel**, **Cisco**, **Live-Oak Bank**, and thousands of other development teams and companies of all sizes. + ## Documentation - 📃   [Full documentation is available here](https://docs.opal.ac) @@ -104,22 +125,18 @@ curl -L https://raw.githubusercontent.com/permitio/opal/master/docker/docker-com - 🎨   [Key concepts and design](https://docs.opal.ac/overview/design) - 🏗️   [Architecture](https://docs.opal.ac/overview/architecture) - -
-OPAL uses a client-server stateless architecture. OPAL-Servers publish policy and data updates over a lightweight (websocket) PubSub Channel, which OPAL-clients subscribe to via topics. Upon updates each client fetches data directly (from source) to load it in to its managed OPA instance. -
-simplified
-📖   For further reading check out our [Blog](https://bit.ly/opal_blog). + +📖 For further reading, check out our [Blog](https://io.permit.io/opal-readme-blog) ## Community -Come talk to us about OPAL, or authorization in general - we would love to hear from you ❤️ + We would love to chat with you about OPAL. [Join our Slack community](https://io.permit.io/opal-readme-slack) to chat about authorization, open-source, realtime communication, tech, or anything else! -You can raise questions and ask for features to be added to the road-map in our [**Github discussions**](https://github.com/permitio/opal/discussions), report issues in [**Github issues**](https://github.com/permitio/opal/issues), follow us on Twitter to get the latest OPAL updates, and join our Slack community to chat about authorization, open-source, realtime communication, tech, or anything else! +You can raise questions and ask for features to be added to the road-map in our [**Github discussions**](https://github.com/permitio/opal/discussions), report issues in [**Github issues**](https://github.com/permitio/opal/issues)

-If you are using our project, please consider giving us a ⭐️ +If you like our project, please consider giving us a ⭐️
[![Button][join-slack-link]][badge-slack-link]
[![Button][follow-twitter-link]][badge-twitter-link] diff --git a/app-tests/README.md b/app-tests/README.md new file mode 100644 index 000000000..3d54d2d98 --- /dev/null +++ b/app-tests/README.md @@ -0,0 +1,51 @@ +# OPAL Application Tests + +To fully test OPAL's core features as part of our CI flow, +We're using a bash script and a docker-compose configuration that enables most of OPAL's important features. + +## How To Run Locally + +### Controlling the image tag + +By default, tests would run with the `latest` image tag (for both server & client). + +To configure another specific version: + +```bash +export OPAL_IMAGE_TAG=0.7.1 +``` + +Or if you want to test locally built images +```bash +make docker-build-next +export OPAL_IMAGE_TAG=next +``` + +### Using a policy repo + +To test opal's git tracking capabilities, `run.sh` uses a dedicated GitHub repo ([opal-tests-policy-repo](https://github.com/permitio/opal-tests-policy-repo)) in which it creates branches and pushes new commits. + +If you're not accessible to that repo (not in `Permit.io`), Please fork our public [opal-example-policy-repo](https://github.com/permitio/opal-example-policy-repo), and override the repo URL to be used: +```bash +export OPAL_POLICY_REPO_URL=git@github.com:your-org/your-repo.git +``` + +As `run.sh` requires push permissions, and as `opal-server` itself might need to authenticate GitHub (if your repo is private). If your GitHub ssh private key is not stored at `~/.ssh/id_rsa`, provide it using: +```bash +# Use an absolute path +export OPAL_POLICY_REPO_SSH_KEY_PATH=$(realpath ./your_github_ssh_private_key) +``` + + +### Putting it all together + +```bash +make docker-build-next # To locally build opal images +export OPAL_IMAGE_TAG=next # Otherwise would default to "latest" + +export OPAL_POLICY_REPO_URL=git@github.com:your-org/your-repo.git # To use your own repo for testing (if you're not an Permit.io employee yet...) +export OPAL_POLICY_REPO_SSH_KEY_PATH=$(realpath ./your_github_ssh_private_key) # If your GitHub ssh key isn't in "~.ssh/id_rsa" + +cd app-tests +./run.sh +``` diff --git a/app-tests/docker-compose-app-tests.yml b/app-tests/docker-compose-app-tests.yml new file mode 100644 index 000000000..b12e5309a --- /dev/null +++ b/app-tests/docker-compose-app-tests.yml @@ -0,0 +1,58 @@ +services: + broadcast_channel: + image: postgres:alpine + environment: + - POSTGRES_DB=postgres + - POSTGRES_USER=postgres + - POSTGRES_PASSWORD=postgres + + opal_server: + image: permitio/opal-server:${OPAL_IMAGE_TAG:-latest} + deploy: + mode: replicated + replicas: 2 + endpoint_mode: vip + environment: + - OPAL_BROADCAST_URI=postgres://postgres:postgres@broadcast_channel:5432/postgres + - UVICORN_NUM_WORKERS=4 + - OPAL_POLICY_REPO_URL=${OPAL_POLICY_REPO_URL:-git@github.com:permitio/opal-tests-policy-repo.git} + - OPAL_POLICY_REPO_MAIN_BRANCH=${POLICY_REPO_BRANCH} + - OPAL_POLICY_REPO_SSH_KEY=${OPAL_POLICY_REPO_SSH_KEY} + - OPAL_DATA_CONFIG_SOURCES={"config":{"entries":[{"url":"http://opal_server:7002/policy-data","config":{"headers":{"Authorization":"Bearer ${OPAL_CLIENT_TOKEN}"}},"topics":["policy_data"],"dst_path":"/static"}]}} + - OPAL_LOG_FORMAT_INCLUDE_PID=true + - OPAL_POLICY_REPO_WEBHOOK_SECRET=xxxxx + - OPAL_POLICY_REPO_WEBHOOK_PARAMS={"secret_header_name":"x-webhook-token","secret_type":"token","secret_parsing_regex":"(.*)","event_request_key":"gitEvent","push_event_value":"git.push"} + - OPAL_AUTH_PUBLIC_KEY=${OPAL_AUTH_PUBLIC_KEY} + - OPAL_AUTH_PRIVATE_KEY=${OPAL_AUTH_PRIVATE_KEY} + - OPAL_AUTH_MASTER_TOKEN=${OPAL_AUTH_MASTER_TOKEN} + - OPAL_AUTH_JWT_AUDIENCE=https://api.opal.ac/v1/ + - OPAL_AUTH_JWT_ISSUER=https://opal.ac/ + - OPAL_STATISTICS_ENABLED=true + ports: + - "7002-7003:7002" + depends_on: + - broadcast_channel + + opal_client: + image: permitio/opal-client:${OPAL_IMAGE_TAG:-latest} + deploy: + mode: replicated + replicas: 2 + endpoint_mode: vip + environment: + - OPAL_SERVER_URL=http://opal_server:7002 + - OPAL_LOG_FORMAT_INCLUDE_PID=true + - OPAL_INLINE_OPA_LOG_FORMAT=http + - OPAL_SHOULD_REPORT_ON_DATA_UPDATES=True + - OPAL_DEFAULT_UPDATE_CALLBACKS={"callbacks":[["http://opal_server:7002/data/callback_report",{"method":"post","process_data":false,"headers":{"Authorization":"Bearer ${OPAL_CLIENT_TOKEN}","content-type":"application/json"}}]]} + - OPAL_OPA_HEALTH_CHECK_POLICY_ENABLED=True + - OPAL_CLIENT_TOKEN=${OPAL_CLIENT_TOKEN} + - OPAL_AUTH_JWT_AUDIENCE=https://api.opal.ac/v1/ + - OPAL_AUTH_JWT_ISSUER=https://opal.ac/ + - OPAL_STATISTICS_ENABLED=true + ports: + - "7766-7767:7000" + - "8181-8182:8181" + depends_on: + - opal_server + command: sh -c "exec ./wait-for.sh opal_server:7002 --timeout=20 -- ./start.sh" diff --git a/app-tests/run.sh b/app-tests/run.sh new file mode 100755 index 000000000..79293815a --- /dev/null +++ b/app-tests/run.sh @@ -0,0 +1,159 @@ +#!/bin/bash +set -e + +export OPAL_AUTH_PUBLIC_KEY +export OPAL_AUTH_PRIVATE_KEY +export OPAL_AUTH_MASTER_TOKEN +export OPAL_CLIENT_TOKEN +export OPAL_DATA_SOURCE_TOKEN + +function generate_opal_keys { + echo "- Generating OPAL keys" + + ssh-keygen -q -t rsa -b 4096 -m pem -f opal_crypto_key -N "" + OPAL_AUTH_PUBLIC_KEY="$(cat opal_crypto_key.pub)" + OPAL_AUTH_PRIVATE_KEY="$(tr '\n' '_' < opal_crypto_key)" + rm opal_crypto_key.pub opal_crypto_key + + OPAL_AUTH_MASTER_TOKEN="$(openssl rand -hex 16)" + OPAL_AUTH_JWT_AUDIENCE=https://api.opal.ac/v1/ OPAL_AUTH_JWT_ISSUER=https://opal.ac/ OPAL_REPO_WATCHER_ENABLED=0 \ + opal-server run & + sleep 2; + + OPAL_CLIENT_TOKEN="$(opal-client obtain-token "$OPAL_AUTH_MASTER_TOKEN" --type client)" + OPAL_DATA_SOURCE_TOKEN="$(opal-client obtain-token "$OPAL_AUTH_MASTER_TOKEN" --type datasource)" + # shellcheck disable=SC2009 + ps -ef | grep opal-server | grep -v grep | awk '{print $2}' | xargs kill + sleep 5; + + echo "- Create .env file" + rm -f .env + ( + echo "OPAL_AUTH_PUBLIC_KEY=\"$OPAL_AUTH_PUBLIC_KEY\""; + echo "OPAL_AUTH_PRIVATE_KEY=\"$OPAL_AUTH_PRIVATE_KEY\""; + echo "OPAL_AUTH_MASTER_TOKEN=\"$OPAL_AUTH_MASTER_TOKEN\""; + echo "OPAL_CLIENT_TOKEN=\"$OPAL_CLIENT_TOKEN\""; + echo "OPAL_AUTH_PRIVATE_KEY_PASSPHRASE=\"$OPAL_AUTH_PRIVATE_KEY_PASSPHRASE\"" + ) > .env +} + +function prepare_policy_repo { + echo "- Clone tests policy repo to create test's branch" + export OPAL_POLICY_REPO_URL + export POLICY_REPO_BRANCH + OPAL_POLICY_REPO_URL=${OPAL_POLICY_REPO_URL:-git@github.com:permitio/opal-tests-policy-repo.git} + POLICY_REPO_BRANCH=test-$RANDOM$RANDOM + rm -rf ./opal-tests-policy-repo + git clone "$OPAL_POLICY_REPO_URL" + cd opal-tests-policy-repo + git checkout -b $POLICY_REPO_BRANCH + git push --set-upstream origin $POLICY_REPO_BRANCH + cd - + + # That's for the docker-compose to use, set ssh key from "~/.ssh/id_rsa", unless another path/key data was configured + export OPAL_POLICY_REPO_SSH_KEY + OPAL_POLICY_REPO_SSH_KEY_PATH=${OPAL_POLICY_REPO_SSH_KEY_PATH:-~/.ssh/id_rsa} + OPAL_POLICY_REPO_SSH_KEY=${OPAL_POLICY_REPO_SSH_KEY:-$(cat "$OPAL_POLICY_REPO_SSH_KEY_PATH")} +} + +function compose { + docker compose -f ./docker-compose-app-tests.yml --env-file .env "$@" +} + +function check_clients_logged { + echo "- Looking for msg '$1' in client's logs" + compose logs --index 1 opal_client | grep -q "$1" + compose logs --index 2 opal_client | grep -q "$1" +} + +function check_no_error { + # Without index would output all replicas + if compose logs opal_client | grep -q 'ERROR'; then + echo "- Found error in logs" + exit 1 + fi +} + +function clean_up { + ARG=$? + if [[ "$ARG" -ne 0 ]]; then + echo "*** Test Failed ***" + echo "" + compose logs + else + echo "*** Test Passed ***" + echo "" + fi + compose down + cd opal-tests-policy-repo; git push -d origin $POLICY_REPO_BRANCH; cd - # Remove remote tests branch + rm -rf ./opal-tests-policy-repo + exit $ARG +} + +function test_push_policy { + echo "- Testing pushing policy $1" + regofile="$1.rego" + cd opal-tests-policy-repo + echo "package $1" > "$regofile" + git add "$regofile" + git commit -m "Add $regofile" + git push + cd - + + curl -s --request POST 'http://localhost:7002/webhook' --header 'Content-Type: application/json' --header 'x-webhook-token: xxxxx' --data-raw '{"gitEvent":"git.push","repository":{"git_url":"'"$OPAL_POLICY_REPO_URL"'"}}' + sleep 5 + check_clients_logged "PUT /v1/policies/$regofile -> 200" +} + +function test_data_publish { + echo "- Testing data publish for user $1" + user=$1 + OPAL_CLIENT_TOKEN=$OPAL_DATA_SOURCE_TOKEN opal-client publish-data-update --src-url https://api.country.is/23.54.6.78 -t policy_data --dst-path "/users/$user/location" + sleep 5 + check_clients_logged "PUT /v1/data/users/$user/location -> 204" +} + +function test_statistics { + echo "- Testing statistics feature" + # Make sure 2 servers & 2 clients (repeat few times cause different workers might response) + for _ in {1..10}; do + curl -s 'http://localhost:7002/stats' --header "Authorization: Bearer $OPAL_DATA_SOURCE_TOKEN" | grep '"client_count":2,"server_count":2' + done +} + +function main { + # Setup + generate_opal_keys + prepare_policy_repo + + trap clean_up EXIT + + # Bring up OPAL containers + compose down --remove-orphans + compose up -d + sleep 10 + + # Check containers started correctly + check_clients_logged "Connected to PubSub server" + check_clients_logged "Got policy bundle" + check_clients_logged 'PUT /v1/data/static -> 204' + check_no_error + + # Test functionality + test_data_publish "bob" + test_push_policy "something" + test_statistics + + echo "- Testing broadcast channel disconnection" + compose restart broadcast_channel + sleep 10 + + test_data_publish "alice" + test_push_policy "another" + test_data_publish "sunil" + test_data_publish "eve" + test_push_policy "best_one_yet" + # TODO: Test statistics feature again after broadcaster restart (should first fix statistics bug) +} + +main diff --git a/docker/Dockerfile b/docker/Dockerfile index 35f1144a6..da5c7383c 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -25,6 +25,8 @@ RUN CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse cargo build --release FROM python:3.10-slim-bookworm AS common # copy libraries from build stage (This won't copy redundant libraries we used in build-stage) +# also remove the default python site-packages that has older versions of packages that won't be overridden +RUN rm -r /usr/local/lib/python3.10/site-packages COPY --from=build-stage /usr/local /usr/local # Add non-root user (with home dir at /opal) diff --git a/docker/docker-compose-api-policy-source-example.yml b/docker/docker-compose-api-policy-source-example.yml index 219381598..eb202a74e 100644 --- a/docker/docker-compose-api-policy-source-example.yml +++ b/docker/docker-compose-api-policy-source-example.yml @@ -36,7 +36,7 @@ services: # configures from where the opal client should initially fetch data (when it first goes up, after disconnection, etc). # the data sources represents from where the opal clients should get a "complete picture" of the data they need. # after the initial sources are fetched, the client will subscribe only to update notifications sent by the server. - - OPAL_DATA_CONFIG_SOURCES={"config":{"entries":[{"url":"http://opal-server:7002/policy-data","topics":["policy_data"],"dst_path":"/static"}]}} + - OPAL_DATA_CONFIG_SOURCES={"config":{"entries":[{"url":"http://opal_server:7002/policy-data","topics":["policy_data"],"dst_path":"/static"}]}} - OPAL_LOG_FORMAT_INCLUDE_PID=true ports: # exposes opal server on the host machine, you can access the server at: http://localhost:7002 diff --git a/docker/docker-compose-with-kafka-example.yml b/docker/docker-compose-with-kafka-example.yml index 1289e0592..70a479d65 100644 --- a/docker/docker-compose-with-kafka-example.yml +++ b/docker/docker-compose-with-kafka-example.yml @@ -69,7 +69,7 @@ services: # configures from where the opal client should initially fetch data (when it first goes up, after disconnection, etc). # the data sources represents from where the opal clients should get a "complete picture" of the data they need. # after the initial sources are fetched, the client will subscribe only to update notifications sent by the server. - - OPAL_DATA_CONFIG_SOURCES={"config":{"entries":[{"url":"http://opal-server:7002/policy-data","topics":["policy_data"],"dst_path":"/static"}]}} + - OPAL_DATA_CONFIG_SOURCES={"config":{"entries":[{"url":"http://opal_server:7002/policy-data","topics":["policy_data"],"dst_path":"/static"}]}} - OPAL_LOG_FORMAT_INCLUDE_PID=true ports: # exposes opal server on the host machine, you can access the server at: http://localhost:7002 diff --git a/docker/docker-compose-with-oauth-jwt-token.yml b/docker/docker-compose-with-oauth-jwt-token.yml new file mode 100644 index 000000000..b62197241 --- /dev/null +++ b/docker/docker-compose-with-oauth-jwt-token.yml @@ -0,0 +1,93 @@ +services: + # When scaling the opal-server to multiple nodes and/or multiple workers, we use + # a *broadcast* channel to sync between all the instances of opal-server. + # Under the hood, this channel is implemented by encode/broadcaster (see link below). + # At the moment, the broadcast channel can be either: postgresdb, redis or kafka. + # The format of the broadcaster URI string (the one we pass to opal server as `OPAL_BROADCAST_URI`) is specified here: + # https://github.com/encode/broadcaster#available-backends + broadcast_channel: + image: postgres:alpine + environment: + - POSTGRES_DB=postgres + - POSTGRES_USER=postgres + - POSTGRES_PASSWORD=postgres + opal_server: + # by default we run opal-server from latest official image + image: permitio/opal-server:latest + environment: + # the broadcast backbone uri used by opal server workers (see comments above for: broadcast_channel) + - OPAL_BROADCAST_URI=postgres://postgres:postgres@broadcast_channel:5432/postgres + # number of uvicorn workers to run inside the opal-server container + - UVICORN_NUM_WORKERS=4 + # the git repo hosting our policy + # - if this repo is not public, you can pass an ssh key via `OPAL_POLICY_REPO_SSH_KEY`) + # - the repo we pass in this example is *public* and acts as an example repo with dummy rego policy + # - for more info, see: https://docs.opal.ac/tutorials/track_a_git_repo + - OPAL_POLICY_REPO_URL=https://github.com/permitio/opal-example-policy-repo + # in this example we will use a polling interval of 30 seconds to check for new policy updates (git commits affecting the rego policy). + # however, it is better to utilize a git *webhook* to trigger the server to check for changes only when the repo has new commits. + # for more info see: https://docs.opal.ac/tutorials/track_a_git_repo + - OPAL_POLICY_REPO_POLLING_INTERVAL=30 + # configures from where the opal client should initially fetch data (when it first goes up, after disconnection, etc). + # the data sources represents from where the opal clients should get a "complete picture" of the data they need. + # after the initial sources are fetched, the client will subscribe only to update notifications sent by the server. + - OPAL_DATA_CONFIG_SOURCES={"config":{"entries":[{"url":"http://opal_server:7002/policy-data","topics":["policy_data"],"dst_path":"/static"}]}} + - OPAL_LOG_FORMAT_INCLUDE_PID=true + # to protect resources with OAuth2 Opaque token provided by dedicated server + - OPAL_AUTH_TYPE=oauth2 + # URL to generate new OAuth 2.0 Client Credentials Grant token + - OPAL_OAUTH2_TOKEN_URL=https://example/oauth2/token + # JWT validation + - OPAL_OAUTH2_OPENID_CONFIGURATION_URL=https://example/.well-known/openid-configuration + - OPAL_OAUTH2_EXACT_MATCH_CLAIMS=aud=some_audience,iss=some_issuer + - OPAL_OAUTH2_REQUIRED_CLAIMS=sub,iat,exp + - OPAL_OAUTH2_JWT_ALGORITHM=RS256 + - OPAL_OAUTH2_JWT_AUDIENCE=some_audience + - OPAL_OAUTH2_JWT_ISSUER=https://example/issuer + ports: + # exposes opal server on the host machine, you can access the server at: http://localhost:7002 + - "7002:7002" + depends_on: + - broadcast_channel + opal_client: + # by default we run opal-client from latest official image + image: permitio/opal-client:latest + environment: + - OPAL_SERVER_URL=http://opal_server:7002 + - OPAL_LOG_FORMAT_INCLUDE_PID=true + - OPAL_INLINE_OPA_LOG_FORMAT=http + # to protect resources with OAuth2 Opaque token provided by dedicated server + - OPAL_AUTH_TYPE=oauth2 + # client credentials + - OPAL_OAUTH2_CLIENT_ID=some_client_id + - OPAL_OAUTH2_CLIENT_SECRET=some_client_secret + # URL to generate new OAuth 2.0 Client Credentials Grant token + - OPAL_OAUTH2_TOKEN_URL=https://example/oauth2/token + # JWT validation + - OPAL_OAUTH2_OPENID_CONFIGURATION_URL=https://example/.well-known/openid-configuration + - OPAL_OAUTH2_EXACT_MATCH_CLAIMS=aud=some_audience,iss=some_issuer + - OPAL_OAUTH2_REQUIRED_CLAIMS=sub,iat,exp + - OPAL_OAUTH2_JWT_ALGORITHM=RS256 + - OPAL_OAUTH2_JWT_AUDIENCE=some_audience + - OPAL_OAUTH2_JWT_ISSUER=https://example/issuer + # Enable Authorization / Authentication in OPA + - 'OPAL_INLINE_OPA_CONFIG={"authentication":"token", "authorization":"basic", "files": ["authz.rego"]}' + volumes: + # The goal is to create an initial authorization rego that allows OPAL to write the first policy from the POLICY_REPO_URL. + # This is achieved through policy overwrite based on the "id" attribute. + # When the authz.rego file is placed in the root directory of OPA, it is given the id 'authz.rego'. + # Similarly, if there is another authz.rego file in the root of POLICY_REPO_URL, it will also be given the id 'authz.rego'. + # Therefore, if the authz.rego file from the POLICY_REPO_URL exists, it will overwrite the initial authz.rego file. + - ./docker_files/policy_test/authz.rego:/opal/authz.rego + ports: + # exposes opal client on the host machine, you can access the client at: http://localhost:7766 + - "7766:7000" + # exposes the OPA agent (being run by OPAL) on the host machine + # you can access the OPA api that you know and love at: http://localhost:8181 + # OPA api docs are at: https://www.openpolicyagent.org/docs/latest/rest-api/ + - "8181:8181" + depends_on: + - opal_server + # this command is not necessary when deploying OPAL for real, it is simply a trick for dev environments + # to make sure that opal-server is already up before starting the client. + command: sh -c "exec ./wait-for.sh opal_server:7002 --timeout=20 -- ./start.sh" diff --git a/docker/docker-compose-with-oauth-opaque-token.yml b/docker/docker-compose-with-oauth-opaque-token.yml new file mode 100644 index 000000000..7641cd0e8 --- /dev/null +++ b/docker/docker-compose-with-oauth-opaque-token.yml @@ -0,0 +1,83 @@ +services: + # When scaling the opal-server to multiple nodes and/or multiple workers, we use + # a *broadcast* channel to sync between all the instances of opal-server. + # Under the hood, this channel is implemented by encode/broadcaster (see link below). + # At the moment, the broadcast channel can be either: postgresdb, redis or kafka. + # The format of the broadcaster URI string (the one we pass to opal server as `OPAL_BROADCAST_URI`) is specified here: + # https://github.com/encode/broadcaster#available-backends + broadcast_channel: + image: postgres:alpine + environment: + - POSTGRES_DB=postgres + - POSTGRES_USER=postgres + - POSTGRES_PASSWORD=postgres + opal_server: + # by default we run opal-server from latest official image + image: permitio/opal-server:latest + environment: + # the broadcast backbone uri used by opal server workers (see comments above for: broadcast_channel) + - OPAL_BROADCAST_URI=postgres://postgres:postgres@broadcast_channel:5432/postgres + # number of uvicorn workers to run inside the opal-server container + - UVICORN_NUM_WORKERS=4 + # the git repo hosting our policy + # - if this repo is not public, you can pass an ssh key via `OPAL_POLICY_REPO_SSH_KEY`) + # - the repo we pass in this example is *public* and acts as an example repo with dummy rego policy + # - for more info, see: https://docs.opal.ac/tutorials/track_a_git_repo + - OPAL_POLICY_REPO_URL=https://github.com/permitio/opal-example-policy-repo + # in this example we will use a polling interval of 30 seconds to check for new policy updates (git commits affecting the rego policy). + # however, it is better to utilize a git *webhook* to trigger the server to check for changes only when the repo has new commits. + # for more info see: https://docs.opal.ac/tutorials/track_a_git_repo + - OPAL_POLICY_REPO_POLLING_INTERVAL=30 + # configures from where the opal client should initially fetch data (when it first goes up, after disconnection, etc). + # the data sources represents from where the opal clients should get a "complete picture" of the data they need. + # after the initial sources are fetched, the client will subscribe only to update notifications sent by the server. + - OPAL_DATA_CONFIG_SOURCES={"config":{"entries":[{"url":"http://opal_server:7002/policy-data","topics":["policy_data"],"dst_path":"/static"}]}} + - OPAL_LOG_FORMAT_INCLUDE_PID=true + # to protect resources with OAuth2 Opaque token provided by dedicated server + - OPAL_AUTH_TYPE=oauth2 + # URL to generate new OAuth 2.0 Client Credentials Grant token + - OPAL_OAUTH2_TOKEN_URL=https://example/oauth2/token + # introspect URL for Opaque token validation + - OPAL_OAUTH2_INTROSPECT_URL=https://example/oauth2/introspect + ports: + # exposes opal server on the host machine, you can access the server at: http://localhost:7002 + - "7002:7002" + depends_on: + - broadcast_channel + opal_client: + # by default we run opal-client from latest official image + image: permitio/opal-client:latest + environment: + - OPAL_SERVER_URL=http://opal_server:7002 + - OPAL_LOG_FORMAT_INCLUDE_PID=true + - OPAL_INLINE_OPA_LOG_FORMAT=http + # to protect resources with OAuth2 Opaque token provided by dedicated server + - OPAL_AUTH_TYPE=oauth2 + # client credentials + - OPAL_OAUTH2_CLIENT_ID=some_client_id + - OPAL_OAUTH2_CLIENT_SECRET=some_client_secret + # URL to generate new OAuth 2.0 Client Credentials Grant token + - OPAL_OAUTH2_TOKEN_URL=https://example/oauth2/token + # introspect URL for Opaque token validation + - OPAL_OAUTH2_INTROSPECT_URL=https://example/oauth2/introspect + # Enable Authorization / Authentication in OPA + - 'OPAL_INLINE_OPA_CONFIG={"authentication":"token", "authorization":"basic", "files": ["authz.rego"]}' + volumes: + # The goal is to create an initial authorization rego that allows OPAL to write the first policy from the POLICY_REPO_URL. + # This is achieved through policy overwrite based on the "id" attribute. + # When the authz.rego file is placed in the root directory of OPA, it is given the id 'authz.rego'. + # Similarly, if there is another authz.rego file in the root of POLICY_REPO_URL, it will also be given the id 'authz.rego'. + # Therefore, if the authz.rego file from the POLICY_REPO_URL exists, it will overwrite the initial authz.rego file. + - ./docker_files/policy_test/authz.rego:/opal/authz.rego + ports: + # exposes opal client on the host machine, you can access the client at: http://localhost:7766 + - "7766:7000" + # exposes the OPA agent (being run by OPAL) on the host machine + # you can access the OPA api that you know and love at: http://localhost:8181 + # OPA api docs are at: https://www.openpolicyagent.org/docs/latest/rest-api/ + - "8181:8181" + depends_on: + - opal_server + # this command is not necessary when deploying OPAL for real, it is simply a trick for dev environments + # to make sure that opal-server is already up before starting the client. + command: sh -c "exec ./wait-for.sh opal_server:7002 --timeout=20 -- ./start.sh" diff --git a/docker/docker-compose-with-rate-limiting.yml b/docker/docker-compose-with-rate-limiting.yml index 6f10caf5e..8ac8c8c6e 100644 --- a/docker/docker-compose-with-rate-limiting.yml +++ b/docker/docker-compose-with-rate-limiting.yml @@ -30,7 +30,7 @@ services: # configures from where the opal client should initially fetch data (when it first goes up, after disconnection, etc). # the data sources represents from where the opal clients should get a "complete picture" of the data they need. # after the initial sources are fetched, the client will subscribe only to update notifications sent by the server. - - OPAL_DATA_CONFIG_SOURCES={"config":{"entries":[{"url":"http://opal-server:7002/policy-data","topics":["policy_data"],"dst_path":"/static"}]}} + - OPAL_DATA_CONFIG_SOURCES={"config":{"entries":[{"url":"http://opal_server:7002/policy-data","topics":["policy_data"],"dst_path":"/static"}]}} - OPAL_LOG_FORMAT_INCLUDE_PID=true # Turns on rate limiting in the server # supported formats documented here: https://limits.readthedocs.io/en/stable/quickstart.html#rate-limit-string-notation diff --git a/docker/docker-compose-with-security.yml b/docker/docker-compose-with-security.yml index 2c27a711f..840b8dae5 100644 --- a/docker/docker-compose-with-security.yml +++ b/docker/docker-compose-with-security.yml @@ -45,7 +45,7 @@ services: # after the initial sources are fetched, the client will subscribe only to update notifications sent by the server. # please notice - since we fetch data entries from the OPAL server itself, we need to authenticate to that endpoint # with the client's token (JWT). - - OPAL_DATA_CONFIG_SOURCES={"config":{"entries":[{"url":"http://opal-server:7002/policy-data","config":{"headers":{"Authorization":"Bearer ${OPAL_CLIENT_TOKEN}"}},"topics":["policy_data"],"dst_path":"/static"}]}} + - OPAL_DATA_CONFIG_SOURCES={"config":{"entries":[{"url":"http://opal_server:7002/policy-data","config":{"headers":{"Authorization":"Bearer ${OPAL_CLIENT_TOKEN}"}},"topics":["policy_data"],"dst_path":"/static"}]}} - OPAL_LOG_FORMAT_INCLUDE_PID=true # -------------------------------------------------------------------------------- # the jwt audience and jwt issuer are not typically necessary in real setups diff --git a/docker/docker-compose-with-statistics.yml b/docker/docker-compose-with-statistics.yml index eb26daef4..1a81d93c7 100644 --- a/docker/docker-compose-with-statistics.yml +++ b/docker/docker-compose-with-statistics.yml @@ -31,7 +31,7 @@ services: # configures from where the opal client should initially fetch data (when it first goes up, after disconnection, etc). # the data sources represents from where the opal clients should get a "complete picture" of the data they need. # after the initial sources are fetched, the client will subscribe only to update notifications sent by the server. - - OPAL_DATA_CONFIG_SOURCES={"config":{"entries":[{"url":"http://opal-server:7002/policy-data","topics":["policy_data"],"dst_path":"/static"}]}} + - OPAL_DATA_CONFIG_SOURCES={"config":{"entries":[{"url":"http://opal_server:7002/policy-data","topics":["policy_data"],"dst_path":"/static"}]}} - OPAL_LOG_FORMAT_INCLUDE_PID=true # turning on statistics collection on the server side - OPAL_STATISTICS_ENABLED=true diff --git a/documentation/docs/getting-started/configuration.mdx b/documentation/docs/getting-started/configuration.mdx index 5cd2ed4c9..d3baa47b0 100644 --- a/documentation/docs/getting-started/configuration.mdx +++ b/documentation/docs/getting-started/configuration.mdx @@ -24,7 +24,8 @@ Please use this table as a reference. | LOG_FILE_RETENTION | | | | LOG_FILE_COMPRESSION | | | | LOG_FILE_SERIALIZE | Serialize log messages in file into json format (useful for log aggregation platforms) | | -| LOG_FILE_LEVEL | | | +| LOG_FILE_LEVEL | +| LOG_DIAGNOSE | Include diagnosis in log messages | | | STATISTICS_ENABLED | Collect statistics about OPAL clients. | | | STATISTICS_ADD_CLIENT_CHANNEL | The topic to update about the new OPAL clients connection. | | | STATISTICS_REMOVE_CLIENT_CHANNEL | The topic to update about the OPAL clients disconnection. | | diff --git a/documentation/docs/getting-started/running-opal/run-opal-client/obtain-jwt-token.mdx b/documentation/docs/getting-started/running-opal/run-opal-client/obtain-jwt-token.mdx index 0540eadc4..bde4707b0 100644 --- a/documentation/docs/getting-started/running-opal/run-opal-client/obtain-jwt-token.mdx +++ b/documentation/docs/getting-started/running-opal/run-opal-client/obtain-jwt-token.mdx @@ -30,7 +30,7 @@ curl --request POST 'https://opal.yourdomain.com/token' \ --header 'Authorization: Bearer MY_MASTER_TOKEN' \ --header 'Content-Type: application/json' \ --data-raw '{ - "type": "client", + "type": "client" }' ``` diff --git a/documentation/docs/getting-started/running-opal/run-opal-server/broadcast-interface.mdx b/documentation/docs/getting-started/running-opal/run-opal-server/broadcast-interface.mdx index 5558a65bf..403b074a7 100644 --- a/documentation/docs/getting-started/running-opal/run-opal-server/broadcast-interface.mdx +++ b/documentation/docs/getting-started/running-opal/run-opal-server/broadcast-interface.mdx @@ -40,14 +40,6 @@ Declaring the broadcast uri is optional, depending on whether you deployed a bro . -
  • - Note that password authentication is not supported in the - broadcaster URI when using redis, see{" "} - - source - - . -
  • Example value:{" "} OPAL_BROADCAST_URI=postgres://localhost/mydb diff --git a/documentation/docs/getting-started/running-opal/run-opal-server/security-parameters.mdx b/documentation/docs/getting-started/running-opal/run-opal-server/security-parameters.mdx index 1862e7afd..8a0fba6ed 100644 --- a/documentation/docs/getting-started/running-opal/run-opal-server/security-parameters.mdx +++ b/documentation/docs/getting-started/running-opal/run-opal-server/security-parameters.mdx @@ -105,3 +105,5 @@ You must then configure the master token like so | Env Var Name | Function | | :--------------------- | :------------------------------------------------------------------- | | OPAL_AUTH_MASTER_TOKEN | the master token generated by the cli (or any other secret you pick) | + +Ensure LOG_DIAGNOSE is set to False to disable diagnostic logging that may expose sensitive information. diff --git a/documentation/docs/opal-plus/features.mdx b/documentation/docs/opal-plus/features.mdx index bc3a0249f..0d8b75e58 100644 --- a/documentation/docs/opal-plus/features.mdx +++ b/documentation/docs/opal-plus/features.mdx @@ -39,13 +39,13 @@ Configure the [OPAL_STATISTICS_ENABLED=true](/getting-started/configuration) env You can then monitor the state of your OPAL+ cluster by calling the `/stats` API route on the server. ```bash -curl http://opal-server:8181/stats -H "Authorization: Bearer " +curl http://opal_server:8181/stats -H "Authorization: Bearer " # { "uptime": "2024-07-14T14:55:02.710Z", "version": "0.7.8", "client_count": 1, "server_count": 1 } ``` You can also get detailed information about the OPAL+ clients and servers by calling the `/statistics` API route on the server. ```bash -curl http://opal-server:8181/statistics -H "Authorization: Bear " +curl http://opal_server:8181/statistics -H "Authorization: Bear " ``` ```json { diff --git a/documentation/docs/tutorials/run_opal_with_pulsar.mdx b/documentation/docs/tutorials/run_opal_with_pulsar.mdx new file mode 100644 index 000000000..4433d7a65 --- /dev/null +++ b/documentation/docs/tutorials/run_opal_with_pulsar.mdx @@ -0,0 +1,115 @@ +--- +sidebar_position: 12 +title: Run OPAL with Apache Pulsar +--- + +# Running OPAL-server with Apache Pulsar + +## Introduction + +OPAL-server supports multiple backbone pub/sub solutions for connecting distributed server instances. This guide explains how to set up and use Apache Pulsar as the backbone pub/sub (broadcast channel) for OPAL-server. + +## Apache Pulsar as the Backbone Pub/Sub + +### What is a backbone pub/sub? + +OPAL-server can scale out both in number of worker processes per server and across multiple servers. While OPAL provides a lightweight websocket pub/sub for OPAL-clients, multiple servers are linked together by a more robust messaging solution like Apache Pulsar, Kafka, Redis, or Postgres Listen/Notify. + +### Broadcaster Module + +Support for multiple backbone solutions is provided by the Permit's port of the [Python Broadcaster package](https://pypi.org/project/permit-broadcaster/). To use it with Apache Pulsar, install the `permit-broadcaster[pulsar]` module: + +```bash +pip install permit-broadcaster[pulsar] +``` + +## Setting Up OPAL-server with Apache Pulsar + +### Configuration + +To use Apache Pulsar as the backbone, set the `OPAL_BROADCAST_URI` environment variable: + +```bash +OPAL_BROADCAST_URI=pulsar://pulsar-host-name:6650 +``` + +The "pulsar://" prefix tells OPAL-server to use Apache Pulsar. + +### Pulsar Topic + +OPAL-server uses a single Pulsar topic named 'broadcast' for all communication. This topic is automatically created when the producer and consumer are initialized. + +## Docker Compose Example + +Here's an example `docker-compose.yml` configuration that includes Apache Pulsar: + +```yaml +version: '3' +services: + pulsar: + image: apachepulsar/pulsar:3.3.1 + command: bin/pulsar standalone + ports: + - 6650:6650 + - 8080:8080 + volumes: + - pulsardata:/pulsar/data + - pulsarconf:/pulsar/conf + + opal-server: + image: permitio/opal-server:latest + environment: + - OPAL_BROADCAST_URI=pulsar://pulsar:6650 + depends_on: + - pulsar + +volumes: + pulsardata: + pulsarconf: +``` + +Run this configuration with: + +```bash +docker-compose up --force-recreate +``` + +Allow a few seconds for Apache Pulsar and OPAL to start up before testing connectivity. + +## Triggering Events + +You can trigger events using the OPAL CLI: + +```bash +opal-client publish-data-update --src-url https://api.country.is/23.54.6.78 -t policy_data --dst-path /users/bob/location +``` + +You should see the effect in: +- OPAL-server logs: "Broadcasting incoming event" +- OPAL-client: Receiving and acting on the event +- Pulsar: Event data in the 'broadcast' topic + +## Supported Backends + +| Backend | Environment Variable | Docker Compose Service | +|----------|---------------------------------------------------------|------------------------| +| Kafka | `BROADCAST_URL=kafka://localhost:9092` | `docker-compose up kafka` | +| Redis | `BROADCAST_URL=redis://localhost:6379` | `docker-compose up redis` | +| Postgres | `BROADCAST_URL=postgres://localhost:5432/broadcaster` | `docker-compose up postgres` | +| Pulsar | `BROADCAST_URL=pulsar://localhost:6650` | `docker-compose up pulsar` | + +## Advanced: Publishing Events Directly to Pulsar + +You can trigger events by publishing messages directly to the 'broadcast' topic in Pulsar. Ensure the message format follows the OPAL-server schema for backbone events. + +## Conclusion + +This guide covered setting up and using Apache Pulsar as the backbone pub/sub for OPAL-server. By following these instructions, you can effectively scale your OPAL deployment across multiple servers. + +## Further Resources + +- [OPAL Documentation](https://www.opal.ac/docs/) +- [Apache Pulsar Documentation](https://pulsar.apache.org/docs/en/standalone/) +- [Python Broadcaster Package](https://pypi.org/project/broadcaster/) + +For more information or support, please refer to the OPAL community forums or contact the maintainers. diff --git a/documentation/package-lock.json b/documentation/package-lock.json index 262b5fe96..6de0a1b39 100644 --- a/documentation/package-lock.json +++ b/documentation/package-lock.json @@ -11,14 +11,14 @@ "@docusaurus/core": "3.0.0", "@docusaurus/preset-classic": "3.0.0", "@mdx-js/react": "^3.0.0", - "axios": "^1.6.4", + "axios": "^1.7.5", "clsx": "^1.2.1", "docusaurus-plugin-sass": "^0.2.5", - "micromatch": "^4.0.6", + "micromatch": "^4.0.8", "node": "^18.0.0", "prism-react-renderer": "^2.1.0", - "react": "^18.0.0", - "react-dom": "^18.0.0", + "react": "^18.3.0", + "react-dom": "^18.3.0", "sass": "^1.71.1" }, "devDependencies": { @@ -3352,24 +3352,6 @@ "@types/ms": "*" } }, - "node_modules/@types/eslint": { - "version": "8.56.9", - "resolved": "https://registry.npmjs.org/@types/eslint/-/eslint-8.56.9.tgz", - "integrity": "sha512-W4W3KcqzjJ0sHg2vAq9vfml6OhsJ53TcUjUqfzzZf/EChUtwspszj/S0pzMxnfRcO55/iGq47dscXw71Fxc4Zg==", - "dependencies": { - "@types/estree": "*", - "@types/json-schema": "*" - } - }, - "node_modules/@types/eslint-scope": { - "version": "3.7.7", - "resolved": "https://registry.npmjs.org/@types/eslint-scope/-/eslint-scope-3.7.7.tgz", - "integrity": "sha512-MzMFlSLBqNF2gcHWO0G1vP/YQyfvrxZ0bF+u7mzUdZ1/xK4A4sru+nraZz5i3iEIk1l1uyicaDVTB4QbbEkAYg==", - "dependencies": { - "@types/eslint": "*", - "@types/estree": "*" - } - }, "node_modules/@types/estree": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.5.tgz", @@ -3852,10 +3834,10 @@ "node": ">=0.4.0" } }, - "node_modules/acorn-import-assertions": { - "version": "1.9.0", - "resolved": "https://registry.npmjs.org/acorn-import-assertions/-/acorn-import-assertions-1.9.0.tgz", - "integrity": "sha512-cmMwop9x+8KFhxvKrKfPYmN6/pKTYYHBqLa0DfvVZcKMJWNyWLnaqND7dx/qn66R7ewM1UX5XMaDVP5wlVTaVA==", + "node_modules/acorn-import-attributes": { + "version": "1.9.5", + "resolved": "https://registry.npmjs.org/acorn-import-attributes/-/acorn-import-attributes-1.9.5.tgz", + "integrity": "sha512-n02Vykv5uA3eHGM/Z2dQrcD56kL8TyDb2p1+0P83PClMnC/nc+anbQRhIOWnSq4Ke/KvDPrY3C9hDtC/A3eHnQ==", "peerDependencies": { "acorn": "^8" } @@ -4128,9 +4110,9 @@ } }, "node_modules/axios": { - "version": "1.7.3", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.7.3.tgz", - "integrity": "sha512-Ar7ND9pU99eJ9GpoGQKhKf58GpUOgnzuaB7ueNQ5BMi0p+LZ5oaEnfF999fAArcTIBwXTCHAmGcHOZJaWPq9Nw==", + "version": "1.7.5", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.7.5.tgz", + "integrity": "sha512-fZu86yCo+svH3uqJ/yTdQ0QHpQu5oL+/QE+QPSv6BZSkDAoky9vytxp7u5qk83OJFS3kEBcesWni9WTZAv3tSw==", "dependencies": { "follow-redirects": "^1.15.6", "form-data": "^4.0.0", @@ -5819,9 +5801,9 @@ } }, "node_modules/enhanced-resolve": { - "version": "5.16.0", - "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.16.0.tgz", - "integrity": "sha512-O+QWCviPNSSLAD9Ucn8Awv+poAkqn3T1XY5/N7kR7rQO9yfSGWkYZDwpJ+iKF7B8rxaQKWngSqACpgzeapSyoA==", + "version": "5.17.1", + "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.17.1.tgz", + "integrity": "sha512-LMHl3dXhTcfv8gM4kEzIUeTQ+7fpdA0l2tUf34BddXPkz2A5xJ5L/Pchd5BL6rdccM9QGvu0sWZzK1Z1t4wwyg==", "dependencies": { "graceful-fs": "^4.2.4", "tapable": "^2.2.0" @@ -10382,9 +10364,9 @@ ] }, "node_modules/micromatch": { - "version": "4.0.7", - "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.7.tgz", - "integrity": "sha512-LPP/3KorzCwBxfeUuZmaR6bG2kdeHSbe0P2tY3FLRU4vYrjYz5hI4QZwV0njUx3jeuKe67YukQ1LSPZBKDqO/Q==", + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz", + "integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==", "dependencies": { "braces": "^3.0.3", "picomatch": "^2.3.1" @@ -11983,9 +11965,9 @@ } }, "node_modules/react": { - "version": "18.2.0", - "resolved": "https://registry.npmjs.org/react/-/react-18.2.0.tgz", - "integrity": "sha512-/3IjMdb2L9QbBdWiW5e3P2/npwMBaU9mHCSCUzNln0ZCYbcfTsGbTJrU/kGemdH2IWmB2ioZ+zkxtmq6g09fGQ==", + "version": "18.3.0", + "resolved": "https://registry.npmjs.org/react/-/react-18.3.0.tgz", + "integrity": "sha512-RPutkJftSAldDibyrjuku7q11d3oy6wKOyPe5K1HA/HwwrXcEqBdHsLypkC2FFYjP7bPUa6gbzSBhw4sY2JcDg==", "dependencies": { "loose-envify": "^1.1.0" }, @@ -12123,15 +12105,15 @@ } }, "node_modules/react-dom": { - "version": "18.2.0", - "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-18.2.0.tgz", - "integrity": "sha512-6IMTriUmvsjHUjNtEDudZfuDQUoWXVxKHhlEGSk81n4YFS+r/Kl99wXiwlVXtPBtJenozv2P+hxDsw9eA7Xo6g==", + "version": "18.3.0", + "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-18.3.0.tgz", + "integrity": "sha512-zaKdLBftQJnvb7FtDIpZtsAIb2MZU087RM8bRDZU8LVCCFYjPTsDZJNFUWPcVz3HFSN1n/caxi0ca4B/aaVQGQ==", "dependencies": { "loose-envify": "^1.1.0", - "scheduler": "^0.23.0" + "scheduler": "^0.23.1" }, "peerDependencies": { - "react": "^18.2.0" + "react": "^18.3.0" } }, "node_modules/react-error-overlay": { @@ -12913,9 +12895,9 @@ "integrity": "sha512-0s+oAmw9zLl1V1cS9BtZN7JAd0cW5e0QH4W3LWEK6a4LaLEA2OTpGYWDY+6XasBLtz6wkm3u1xRw95mRuJ59WA==" }, "node_modules/scheduler": { - "version": "0.23.0", - "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.23.0.tgz", - "integrity": "sha512-CtuThmgHNg7zIZWAXi3AsyIzA3n4xx7aNyjwC2VJldO2LMVDhFK+63xGqq6CsJH4rTAt6/M+N4GhZiDYPx9eUw==", + "version": "0.23.2", + "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.23.2.tgz", + "integrity": "sha512-UOShsPwz7NrMUqhR6t0hWjFduvOzbtv7toDH1/hIrfRNIDBnnBWd0CwJTGvTpngVlmwGCdP9/Zl/tVrDqcuYzQ==", "dependencies": { "loose-envify": "^1.1.0" } @@ -14592,20 +14574,19 @@ "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" }, "node_modules/webpack": { - "version": "5.91.0", - "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.91.0.tgz", - "integrity": "sha512-rzVwlLeBWHJbmgTC/8TvAcu5vpJNII+MelQpylD4jNERPwpBJOE2lEcko1zJX3QJeLjTTAnQxn/OJ8bjDzVQaw==", + "version": "5.94.0", + "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.94.0.tgz", + "integrity": "sha512-KcsGn50VT+06JH/iunZJedYGUJS5FGjow8wb9c0v5n1Om8O1g4L6LjtfxwlXIATopoQu+vOXXa7gYisWxCoPyg==", "dependencies": { - "@types/eslint-scope": "^3.7.3", "@types/estree": "^1.0.5", "@webassemblyjs/ast": "^1.12.1", "@webassemblyjs/wasm-edit": "^1.12.1", "@webassemblyjs/wasm-parser": "^1.12.1", "acorn": "^8.7.1", - "acorn-import-assertions": "^1.9.0", + "acorn-import-attributes": "^1.9.5", "browserslist": "^4.21.10", "chrome-trace-event": "^1.0.2", - "enhanced-resolve": "^5.16.0", + "enhanced-resolve": "^5.17.1", "es-module-lexer": "^1.2.1", "eslint-scope": "5.1.1", "events": "^3.2.0", diff --git a/documentation/package.json b/documentation/package.json index 89ac97334..4584124ea 100644 --- a/documentation/package.json +++ b/documentation/package.json @@ -21,11 +21,11 @@ "docusaurus-plugin-sass": "^0.2.5", "node": "^18.0.0", "prism-react-renderer": "^2.1.0", - "react": "^18.0.0", - "react-dom": "^18.0.0", + "react": "^18.3.0", + "react-dom": "^18.3.0", "sass": "^1.71.1", - "axios": "^1.6.4", - "micromatch": "^4.0.6" + "axios": "^1.7.5", + "micromatch": "^4.0.8" }, "devDependencies": { "@docusaurus/module-type-aliases": "3.0.0" diff --git a/packages/opal-client/opal_client/callbacks/api.py b/packages/opal-client/opal_client/callbacks/api.py index 49cb0853a..b1e22d7f1 100644 --- a/packages/opal-client/opal_client/callbacks/api.py +++ b/packages/opal-client/opal_client/callbacks/api.py @@ -3,8 +3,8 @@ from fastapi import APIRouter, Depends, HTTPException, Response, status from opal_client.callbacks.register import CallbacksRegister from opal_client.config import opal_client_config +from opal_common.authentication.authenticator import Authenticator from opal_common.authentication.authz import require_peer_type -from opal_common.authentication.deps import JWTAuthenticator from opal_common.authentication.types import JWTClaims from opal_common.authentication.verifier import Unauthorized from opal_common.logger import logger @@ -13,7 +13,7 @@ from starlette.status import HTTP_500_INTERNAL_SERVER_ERROR -def init_callbacks_api(authenticator: JWTAuthenticator, register: CallbacksRegister): +def init_callbacks_api(authenticator: Authenticator, register: CallbacksRegister): async def require_listener_token(claims: JWTClaims = Depends(authenticator)): try: require_peer_type( diff --git a/packages/opal-client/opal_client/client.py b/packages/opal-client/opal_client/client.py index be8e5ca49..cdc562509 100644 --- a/packages/opal-client/opal_client/client.py +++ b/packages/opal-client/opal_client/client.py @@ -2,9 +2,7 @@ import functools import os import signal -import tempfile import uuid -from logging import disable from typing import Awaitable, Callable, List, Literal, Optional, Union import aiofiles @@ -17,8 +15,8 @@ from opal_client.callbacks.register import CallbacksRegister from opal_client.config import PolicyStoreTypes, opal_client_config from opal_client.data.api import init_data_router -from opal_client.data.fetcher import DataFetcher from opal_client.data.updater import DataUpdater +from opal_client.data.updater_factory import DataUpdaterFactory from opal_client.engine.options import CedarServerOptions, OpaServerOptions from opal_client.engine.runner import CedarRunner, OpaRunner from opal_client.limiter import StartupLoadLimiter @@ -29,8 +27,8 @@ from opal_client.policy_store.policy_store_client_factory import ( PolicyStoreClientFactory, ) -from opal_common.authentication.deps import JWTAuthenticator -from opal_common.authentication.verifier import JWTVerifier +from opal_common.authentication.authenticator import Authenticator +from opal_common.authentication.authenticator_factory import AuthenticatorFactory from opal_common.config import opal_common_config from opal_common.logger import configure_logs, logger from opal_common.middleware import configure_middleware @@ -49,7 +47,7 @@ def __init__( inline_opa_options: OpaServerOptions = None, inline_cedar_enabled: bool = None, inline_cedar_options: CedarServerOptions = None, - verifier: Optional[JWTVerifier] = None, + authenticator: Optional[Authenticator] = None, store_backup_path: Optional[str] = None, store_backup_interval: Optional[int] = None, offline_mode_enabled: bool = False, @@ -64,6 +62,10 @@ def __init__( data_updater (DataUpdater, optional): Defaults to None. policy_updater (PolicyUpdater, optional): Defaults to None. """ + if authenticator is not None: + self.authenticator = authenticator + else: + self.authenticator = AuthenticatorFactory.create() self._shard_id = shard_id # defaults policy_store_type: PolicyStoreTypes = ( @@ -119,6 +121,7 @@ def __init__( policy_store=self.policy_store, callbacks_register=self._callbacks_register, opal_client_id=opal_client_identifier, + authenticator=self.authenticator, ) else: self.policy_updater = None @@ -134,12 +137,13 @@ def __init__( else opal_client_config.DATA_TOPICS ) - self.data_updater = DataUpdater( + self.data_updater = DataUpdaterFactory.create( policy_store=self.policy_store, data_topics=data_topics, callbacks_register=self._callbacks_register, opal_client_id=opal_client_identifier, shard_id=self._shard_id, + authenticator=self.authenticator, ) else: self.data_updater = None @@ -162,19 +166,6 @@ def __init__( "OPAL client is configured to trust self-signed certificates" ) - if verifier is not None: - self.verifier = verifier - else: - self.verifier = JWTVerifier( - public_key=opal_common_config.AUTH_PUBLIC_KEY, - algorithm=opal_common_config.AUTH_JWT_ALGORITHM, - audience=opal_common_config.AUTH_JWT_AUDIENCE, - issuer=opal_common_config.AUTH_JWT_ISSUER, - ) - if not self.verifier.enabled: - logger.info( - "API authentication disabled (public encryption key was not provided)" - ) self.store_backup_path = ( store_backup_path or opal_client_config.STORE_BACKUP_PATH ) @@ -250,13 +241,11 @@ def _init_fast_api_app(self): def _configure_api_routes(self, app: FastAPI): """mounts the api routes on the app object.""" - authenticator = JWTAuthenticator(self.verifier) - # Init api routers with required dependencies policy_router = init_policy_router(policy_updater=self.policy_updater) data_router = init_data_router(data_updater=self.data_updater) - policy_store_router = init_policy_store_router(authenticator) - callbacks_router = init_callbacks_api(authenticator, self._callbacks_register) + policy_store_router = init_policy_store_router(self.authenticator) + callbacks_router = init_callbacks_api(self.authenticator, self._callbacks_register) # mount the api routes on the app object app.include_router(policy_router, tags=["Policy Updater"]) diff --git a/packages/opal-client/opal_client/config.py b/packages/opal-client/opal_client/config.py index 58d7ae2c8..f4ec738b9 100644 --- a/packages/opal-client/opal_client/config.py +++ b/packages/opal-client/opal_client/config.py @@ -192,7 +192,7 @@ def load_policy_store(): # opal server auth token CLIENT_TOKEN = confi.str( "CLIENT_TOKEN", - "THIS_IS_A_DEV_SECRET", + None, description="opal server auth token", flags=["-t"], ) diff --git a/packages/opal-client/opal_client/data/oauth2_updater.py b/packages/opal-client/opal_client/data/oauth2_updater.py new file mode 100644 index 000000000..12adfd47a --- /dev/null +++ b/packages/opal-client/opal_client/data/oauth2_updater.py @@ -0,0 +1,35 @@ +import aiohttp +from aiohttp.client import ClientSession +from opal_client.logger import logger +from urllib.parse import urlencode, urlparse, parse_qs + +from .updater import DefaultDataUpdater + + +class OAuth2DataUpdater(DefaultDataUpdater): + async def _load_policy_data_config(self, url: str, headers) -> aiohttp.ClientResponse: + await self._authenticator.authenticate(headers) + + async with ClientSession(headers=headers) as session: + response = await session.get(url, **self._ssl_context_kwargs, allow_redirects=False) + + if response.status == 307: + return await self._load_redirected_policy_data_config(response.headers['location'], headers) + else: + return response + + async def _load_redirected_policy_data_config(self, url: str, headers): + redirect_url = self.__redirect_url(url) + + logger.info("Redirecting to data-sources configuration '{source}'", source=redirect_url) + + async with ClientSession(headers=headers) as session: + return await session.get(redirect_url, **self._ssl_context_kwargs, allow_redirects=False) + + def __redirect_url(self, url: str) -> str: + u = urlparse(url) + query = parse_qs(u.query, keep_blank_values=True) + query.pop('token', None) + u = u._replace(query=urlencode(query, True)) + + return u.geturl() \ No newline at end of file diff --git a/packages/opal-client/opal_client/data/updater.py b/packages/opal-client/opal_client/data/updater.py index e288b5963..8d7aa5aaf 100644 --- a/packages/opal-client/opal_client/data/updater.py +++ b/packages/opal-client/opal_client/data/updater.py @@ -24,6 +24,8 @@ DEFAULT_POLICY_STORE_GETTER, ) from opal_common.async_utils import TakeANumberQueue, TasksPool, repeated_call +from opal_common.authentication.authenticator import Authenticator +from opal_common.authentication.authenticator_factory import AuthenticatorFactory from opal_common.config import opal_common_config from opal_common.fetcher.events import FetcherConfig from opal_common.http_utils import is_http_error_response @@ -41,6 +43,54 @@ class DataUpdater: + async def trigger_data_update(self, update: DataUpdate): + raise NotImplementedError() + + async def get_policy_data_config(self, url: str = None) -> DataSourceConfig: + raise NotImplementedError() + + async def get_base_policy_data( + self, config_url: str = None, data_fetch_reason="Initial load" + ): + raise NotImplementedError() + + async def on_connect(self, client: PubSubClient, channel: RpcChannel): + raise NotImplementedError() + + async def on_disconnect(self, channel: RpcChannel): + raise NotImplementedError() + + async def start(self): + raise NotImplementedError() + + async def stop(self): + raise NotImplementedError() + + async def wait_until_done(self): + raise NotImplementedError() + + @staticmethod + def calc_hash(data): + """Calculate an hash (sah256) on the given data, if data isn't a + string, it will be converted to JSON. + + String are encoded as 'utf-8' prior to hash calculation. + Returns: + the hash of the given data (as a a hexdigit string) or '' on failure to process. + """ + try: + if not isinstance(data, str): + data = json.dumps(data, default=pydantic_encoder) + return hashlib.sha256(data.encode("utf-8")).hexdigest() + except: + logger.exception("Failed to calculate hash for data {data}", data=data) + return "" + + @property + def callbacks_reporter(self) -> CallbacksReporter: + raise NotImplementedError() + +class DefaultDataUpdater(DataUpdater): def __init__( self, token: str = None, @@ -54,6 +104,7 @@ def __init__( callbacks_register: Optional[CallbacksRegister] = None, opal_client_id: str = None, shard_id: Optional[str] = None, + authenticator: Optional[Authenticator] = None, ): """Keeps policy-stores (e.g. OPA) up to date with relevant data Obtains data configuration on startup from OPAL-server Uses Pub/Sub to @@ -132,6 +183,10 @@ def __init__( self._updates_storing_queue = TakeANumberQueue(logger) self._tasks = TasksPool() self._polling_update_tasks = [] + if authenticator is not None: + self._authenticator = authenticator + else: + self._authenticator = AuthenticatorFactory.create() async def __aenter__(self): await self.start() @@ -177,20 +232,30 @@ async def get_policy_data_config(self, url: str = None) -> DataSourceConfig: if url is None: url = self._data_sources_config_url logger.info("Getting data-sources configuration from '{source}'", source=url) + + headers = {} + if self._extra_headers is not None: + headers = self._extra_headers.copy() + headers['Accept'] = "application/json" + try: - async with ClientSession(headers=self._extra_headers) as session: - response = await session.get(url, **self._ssl_context_kwargs) - if response.status == 200: - return DataSourceConfig.parse_obj(await response.json()) - else: - error_details = await response.json() - raise ClientError( - f"Fetch data sources failed with status code {response.status}, error: {error_details}" - ) + response = await self._load_policy_data_config(url, headers) + + if response.status == 200: + return DataSourceConfig.parse_obj(await response.json()) + else: + error_details = await response.text() + raise ClientError( + f"Fetch data sources failed with status code {response.status}, error: {error_details}" + ) except: logger.exception(f"Failed to load data sources config") raise + async def _load_policy_data_config(self, url: str, headers) -> aiohttp.ClientResponse: + async with ClientSession(headers=headers) as session: + return await session.get(url, **self._ssl_context_kwargs) + async def get_base_policy_data( self, config_url: str = None, data_fetch_reason="Initial load" ): @@ -274,12 +339,18 @@ async def _subscriber(self): """Coroutine meant to be spunoff with create_task to listen in the background for data events and pass them to the data_fetcher.""" logger.info("Subscribing to topics: {topics}", topics=self._data_topics) + + headers = {} + if self._extra_headers is not None: + headers = self._extra_headers.copy() + await self._authenticator.authenticate(headers) + self._client = PubSubClient( self._data_topics, self._update_policy_data_callback, methods_class=TenantAwareRpcEventClientMethods, on_connect=[self.on_connect], - extra_headers=self._extra_headers, + extra_headers=headers, keep_alive=opal_client_config.KEEP_ALIVE_INTERVAL, server_uri=self._server_url, **self._ssl_context_kwargs, @@ -338,23 +409,6 @@ async def wait_until_done(self): if self._subscriber_task is not None: await self._subscriber_task - @staticmethod - def calc_hash(data): - """Calculate an hash (sah256) on the given data, if data isn't a - string, it will be converted to JSON. - - String are encoded as 'utf-8' prior to hash calculation. - Returns: - the hash of the given data (as a a hexdigit string) or '' on failure to process. - """ - try: - if not isinstance(data, str): - data = json.dumps(data, default=pydantic_encoder) - return hashlib.sha256(data.encode("utf-8")).hexdigest() - except: - logger.exception("Failed to calculate hash for data {data}", data=data) - return "" - async def _update_policy_data( self, update: DataUpdate, @@ -467,7 +521,7 @@ async def _store_fetched_update(self, update_item): policy_data = result # Create a report on the data-fetching report = DataEntryReport( - entry=entry, hash=self.calc_hash(policy_data), fetched=True + entry=entry, hash=DataUpdater.calc_hash(policy_data), fetched=True ) try: diff --git a/packages/opal-client/opal_client/data/updater_factory.py b/packages/opal-client/opal_client/data/updater_factory.py new file mode 100644 index 000000000..46f66349e --- /dev/null +++ b/packages/opal-client/opal_client/data/updater_factory.py @@ -0,0 +1,62 @@ +from typing import List, Optional + +from opal_client.callbacks.register import CallbacksRegister +from opal_client.data.fetcher import DataFetcher +from opal_client.policy_store.base_policy_store_client import BasePolicyStoreClient +from opal_common.authentication.authenticator import Authenticator +from opal_common.config import opal_common_config +from opal_common.logger import logger + +from .oauth2_updater import OAuth2DataUpdater +from .updater import DataUpdater, DefaultDataUpdater + + +class DataUpdaterFactory: + @staticmethod + def create( + token: str = None, + pubsub_url: str = None, + data_sources_config_url: str = None, + fetch_on_connect: bool = True, + data_topics: List[str] = None, + policy_store: BasePolicyStoreClient = None, + should_send_reports=None, + data_fetcher: Optional[DataFetcher] = None, + callbacks_register: Optional[CallbacksRegister] = None, + opal_client_id: str = None, + shard_id: Optional[str] = None, + authenticator: Optional[Authenticator] = None, + ) -> DataUpdater: + if opal_common_config.AUTH_TYPE == "oauth2": + logger.info( + "OPAL is running in secure mode - will authenticate Datasource requests with OAuth2 tokens." + ) + return OAuth2DataUpdater( + token, + pubsub_url, + data_sources_config_url, + fetch_on_connect, + data_topics, + policy_store, + should_send_reports, + data_fetcher, + callbacks_register, + opal_client_id, + shard_id, + authenticator, + ) + else: + return DefaultDataUpdater( + token, + pubsub_url, + data_sources_config_url, + fetch_on_connect, + data_topics, + policy_store, + should_send_reports, + data_fetcher, + callbacks_register, + opal_client_id, + shard_id, + authenticator, + ) diff --git a/packages/opal-client/opal_client/engine/runner.py b/packages/opal-client/opal_client/engine/runner.py index 9cca62c28..762472232 100644 --- a/packages/opal-client/opal_client/engine/runner.py +++ b/packages/opal-client/opal_client/engine/runner.py @@ -136,8 +136,8 @@ async def _pipe_logs_stream(stream: asyncio.StreamReader): line = b"" - await asyncio.wait( - [ + await asyncio.gather( + *[ _pipe_logs_stream(self._process.stdout), _pipe_logs_stream(self._process.stderr), ] diff --git a/packages/opal-client/opal_client/policy/fetcher.py b/packages/opal-client/opal_client/policy/fetcher.py index a435370b1..5be4a6e88 100644 --- a/packages/opal-client/opal_client/policy/fetcher.py +++ b/packages/opal-client/opal_client/policy/fetcher.py @@ -4,6 +4,8 @@ from fastapi import HTTPException, status from opal_client.config import opal_client_config from opal_client.logger import logger +from opal_common.authentication.authenticator import Authenticator +from opal_common.authentication.authenticator_factory import AuthenticatorFactory from opal_common.schemas.policy import PolicyBundle from opal_common.security.sslcontext import get_custom_ssl_context from opal_common.utils import ( @@ -28,15 +30,27 @@ def force_valid_bundle(bundle) -> PolicyBundle: class PolicyFetcher: """fetches policy from backend.""" - def __init__(self, backend_url=None, token=None): + def __init__( + self, + backend_url=None, + token=None, + authenticator: Optional[Authenticator] = None, + ): """ Args: backend_url (str): Defaults to opal_client_config.SERVER_URL. token ([type], optional): [description]. Defaults to opal_client_config.CLIENT_TOKEN. """ + if authenticator is not None: + self._authenticator = authenticator + else: + self._authenticator = AuthenticatorFactory.create() self._token = token or opal_client_config.CLIENT_TOKEN self._backend_url = backend_url or opal_client_config.SERVER_URL - self._auth_headers = tuple_to_dict(get_authorization_header(self._token)) + if self._token is not None: + self._auth_headers = tuple_to_dict(get_authorization_header(self._token)) + else: + self._auth_headers = dict() self._retry_config = ( opal_client_config.POLICY_UPDATER_CONN_RETRY.toTenacityConfig() @@ -82,10 +96,15 @@ async def _fetch_policy_bundle( May throw, in which case we retry again. """ + headers = {} + if self._auth_headers is not None: + headers = self._auth_headers.copy() + await self._authenticator.authenticate(headers) + params = {"path": directories} if base_hash is not None: params["base_hash"] = base_hash - async with aiohttp.ClientSession() as session: + async with aiohttp.ClientSession(headers=headers) as session: logger.info( "Fetching policy bundle from {url}", url=self._policy_endpoint_url, diff --git a/packages/opal-client/opal_client/policy/updater.py b/packages/opal-client/opal_client/policy/updater.py index 57d93099f..9de7528d9 100644 --- a/packages/opal-client/opal_client/policy/updater.py +++ b/packages/opal-client/opal_client/policy/updater.py @@ -16,6 +16,8 @@ DEFAULT_POLICY_STORE_GETTER, ) from opal_common.async_utils import TakeANumberQueue, TasksPool +from opal_common.authentication.authenticator import Authenticator +from opal_common.authentication.authenticator_factory import AuthenticatorFactory from opal_common.config import opal_common_config from opal_common.schemas.data import DataUpdateReport from opal_common.schemas.policy import PolicyBundle, PolicyUpdateMessage @@ -43,6 +45,7 @@ def __init__( data_fetcher: Optional[DataFetcher] = None, callbacks_register: Optional[CallbacksRegister] = None, opal_client_id: str = None, + authenticator: Optional[Authenticator] = None, ): """inits the policy updater. @@ -64,6 +67,10 @@ def __init__( self._opal_client_id = opal_client_id self._scope_id = opal_client_config.SCOPE_ID + if authenticator is not None: + self._authenticator = authenticator + else: + self._authenticator = AuthenticatorFactory.create() # The policy store we'll save policy modules into (i.e: OPA) self._policy_store = policy_store or DEFAULT_POLICY_STORE_GETTER() # pub/sub server url and authentication data @@ -87,7 +94,7 @@ def __init__( self._policy_update_task = None self._stopping = False # policy fetcher - fetches policy bundles - self._policy_fetcher = PolicyFetcher() + self._policy_fetcher = PolicyFetcher(authenticator=self._authenticator) # callbacks on policy changes self._data_fetcher = data_fetcher or DataFetcher() self._callbacks_register = callbacks_register or CallbacksRegister() @@ -240,12 +247,18 @@ async def _subscriber(self): update_policy() callback (which will fetch the relevant policy bundle from the server and update the policy store).""" logger.info("Subscribing to topics: {topics}", topics=self._topics) + + headers = {} + if self._extra_headers is not None: + headers = self._extra_headers.copy() + await self._authenticator.authenticate(headers) + self._client = PubSubClient( topics=self._topics, callback=self._update_policy_callback, on_connect=[self._on_connect], on_disconnect=[self._on_disconnect], - extra_headers=self._extra_headers, + extra_headers=headers, keep_alive=opal_client_config.KEEP_ALIVE_INTERVAL, server_uri=self._server_url, **self._ssl_context_kwargs, diff --git a/packages/opal-client/opal_client/policy_store/api.py b/packages/opal-client/opal_client/policy_store/api.py index b27d83d70..97113f109 100644 --- a/packages/opal-client/opal_client/policy_store/api.py +++ b/packages/opal-client/opal_client/policy_store/api.py @@ -1,15 +1,15 @@ from fastapi import APIRouter, Depends from opal_client.config import opal_client_config from opal_client.policy_store.schemas import PolicyStoreAuth, PolicyStoreDetails +from opal_common.authentication.authenticator import Authenticator from opal_common.authentication.authz import require_peer_type -from opal_common.authentication.deps import JWTAuthenticator from opal_common.authentication.types import JWTClaims from opal_common.authentication.verifier import Unauthorized from opal_common.logger import logger from opal_common.schemas.security import PeerType -def init_policy_store_router(authenticator: JWTAuthenticator): +def init_policy_store_router(authenticator: Authenticator): router = APIRouter() @router.get( diff --git a/packages/opal-client/opal_client/tests/data_updater_test.py b/packages/opal-client/opal_client/tests/data_updater_test.py index 2b34804d0..ff863c7c7 100644 --- a/packages/opal-client/opal_client/tests/data_updater_test.py +++ b/packages/opal-client/opal_client/tests/data_updater_test.py @@ -21,7 +21,7 @@ from opal_client.config import opal_client_config from opal_client.data.rpc import TenantAwareRpcEventClientMethods -from opal_client.data.updater import DataSourceEntry, DataUpdate, DataUpdater +from opal_client.data.updater import DataSourceEntry, DataUpdate, DataUpdater, DefaultDataUpdater from opal_client.policy_store.policy_store_client_factory import ( PolicyStoreClientFactory, ) @@ -167,7 +167,7 @@ async def test_data_updater(server): server trigger a Data-update and check our policy store gets the update.""" # config to use mock OPA policy_store = PolicyStoreClientFactory.create(store_type=PolicyStoreTypes.MOCK) - updater = DataUpdater( + updater = DefaultDataUpdater( pubsub_url=UPDATES_URL, policy_store=policy_store, fetch_on_connect=False, @@ -233,7 +233,7 @@ async def test_data_updater_with_report_callback(server): server trigger a Data-update and check our policy store gets the update.""" # config to use mock OPA policy_store = PolicyStoreClientFactory.create(store_type=PolicyStoreTypes.MOCK) - updater = DataUpdater( + updater = DefaultDataUpdater( pubsub_url=UPDATES_URL, policy_store=policy_store, fetch_on_connect=False, @@ -248,6 +248,7 @@ async def test_data_updater_with_report_callback(server): res = await session.get(CHECK_DATA_UPDATE_CALLBACK_URL) current_callback_count = await res.json() + proc2 = None try: proc = multiprocessing.Process(target=trigger_update, daemon=True) proc.start() @@ -283,7 +284,8 @@ async def test_data_updater_with_report_callback(server): finally: await updater.stop() proc.terminate() - proc2.terminate() + if proc2: + proc2.terminate() @pytest.mark.asyncio @@ -291,7 +293,7 @@ async def test_client_get_initial_data(server): """Connect to OPAL-server and make sure data is fetched on-connect.""" # config to use mock OPA policy_store = PolicyStoreClientFactory.create(store_type=PolicyStoreTypes.MOCK) - updater = DataUpdater( + updater = DefaultDataUpdater( pubsub_url=UPDATES_URL, data_sources_config_url=DATA_CONFIG_URL, policy_store=policy_store, diff --git a/packages/opal-client/opal_client/tests/server_to_client_intergation_test.py b/packages/opal-client/opal_client/tests/server_to_client_intergation_test.py index a3372c56f..5ce829fd7 100644 --- a/packages/opal-client/opal_client/tests/server_to_client_intergation_test.py +++ b/packages/opal-client/opal_client/tests/server_to_client_intergation_test.py @@ -18,7 +18,7 @@ from opal_client import OpalClient from opal_client.data.rpc import TenantAwareRpcEventClientMethods -from opal_client.data.updater import DataSourceEntry, DataUpdate, DataUpdater +from opal_client.data.updater import DataSourceEntry, DataUpdate, DefaultDataUpdater from opal_client.policy_store.mock_policy_store_client import MockPolicyStoreClient from opal_client.policy_store.policy_store_client_factory import ( PolicyStoreClientFactory, @@ -76,7 +76,7 @@ async def startup_event(): def setup_client(event): # config to use mock OPA policy_store = PolicyStoreClientFactory.create(store_type=PolicyStoreTypes.MOCK) - data_updater = DataUpdater( + data_updater = DefaultDataUpdater( pubsub_url=UPDATES_URL, data_sources_config_url=DATA_CONFIG_URL, policy_store=policy_store, diff --git a/packages/opal-client/requires.txt b/packages/opal-client/requires.txt index e29158b99..0fb2499eb 100644 --- a/packages/opal-client/requires.txt +++ b/packages/opal-client/requires.txt @@ -2,6 +2,5 @@ aiofiles>=0.8.0,<1 aiohttp>=3.9.2,<4 psutil>=5.9.1,<6 tenacity>=8.0.1,<9 -websockets>=10.3,<11 dpath>=2.1.5,<3 jsonpatch>=1.33,<2 diff --git a/packages/opal-common/opal_common/authentication/authenticator.py b/packages/opal-common/opal_common/authentication/authenticator.py new file mode 100644 index 000000000..87e210ad4 --- /dev/null +++ b/packages/opal-common/opal_common/authentication/authenticator.py @@ -0,0 +1,15 @@ +from typing import Optional + +from opal_common.authentication.signer import JWTSigner + + +class Authenticator: + @property + def enabled(self) -> bool: + raise NotImplementedError() + + def signer(self) -> Optional[JWTSigner]: + raise NotImplementedError() + + async def authenticate(self, headers): + raise NotImplementedError() diff --git a/packages/opal-common/opal_common/authentication/authenticator_factory.py b/packages/opal-common/opal_common/authentication/authenticator_factory.py new file mode 100644 index 000000000..c0bcc40ad --- /dev/null +++ b/packages/opal-common/opal_common/authentication/authenticator_factory.py @@ -0,0 +1,34 @@ +from opal_common.authentication.authenticator import Authenticator +from opal_common.authentication.deps import JWTAuthenticator +from opal_common.authentication.verifier import JWTVerifier, Unauthorized +from opal_common.config import opal_common_config +from opal_common.logger import logger + +from .oauth2 import CachedOAuth2Authenticator, OAuth2ClientCredentialsAuthenticator + + +class AuthenticatorFactory: + @staticmethod + def create() -> Authenticator: + if opal_common_config.AUTH_TYPE == "oauth2": + logger.info( + "OPAL is running in secure mode - will authenticate API requests with OAuth2 tokens." + ) + return CachedOAuth2Authenticator(OAuth2ClientCredentialsAuthenticator()) + else: + return JWTAuthenticator(AuthenticatorFactory.__verifier()) + + @staticmethod + def __verifier() -> JWTVerifier: + verifier = JWTVerifier( + public_key=opal_common_config.AUTH_PUBLIC_KEY, + algorithm=opal_common_config.AUTH_JWT_ALGORITHM, + audience=opal_common_config.AUTH_JWT_AUDIENCE, + issuer=opal_common_config.AUTH_JWT_ISSUER, + ) + if not verifier.enabled: + logger.info( + "API authentication disabled (public encryption key was not provided)" + ) + + return verifier diff --git a/packages/opal-common/opal_common/authentication/authz.py b/packages/opal-common/opal_common/authentication/authz.py index 742304bf5..822497e64 100644 --- a/packages/opal-common/opal_common/authentication/authz.py +++ b/packages/opal-common/opal_common/authentication/authz.py @@ -1,4 +1,4 @@ -from opal_common.authentication.deps import JWTAuthenticator +from opal_common.authentication.authenticator import Authenticator from opal_common.authentication.types import JWTClaims from opal_common.authentication.verifier import Unauthorized from opal_common.schemas.data import DataUpdate @@ -6,7 +6,7 @@ def require_peer_type( - authenticator: JWTAuthenticator, claims: JWTClaims, required_type: PeerType + authenticator: Authenticator, claims: JWTClaims, required_type: PeerType ): if not authenticator.enabled: return @@ -28,7 +28,7 @@ def require_peer_type( def restrict_optional_topics_to_publish( - authenticator: JWTAuthenticator, claims: JWTClaims, update: DataUpdate + authenticator: Authenticator, claims: JWTClaims, update: DataUpdate ): if not authenticator.enabled: return diff --git a/packages/opal-common/opal_common/authentication/deps.py b/packages/opal-common/opal_common/authentication/deps.py index 390e8ce2d..1cff6cd01 100644 --- a/packages/opal-common/opal_common/authentication/deps.py +++ b/packages/opal-common/opal_common/authentication/deps.py @@ -4,6 +4,8 @@ from fastapi import Header from fastapi.exceptions import HTTPException from fastapi.security.utils import get_authorization_scheme_param +from opal_common.authentication.authenticator import Authenticator +from opal_common.authentication.signer import JWTSigner from opal_common.authentication.types import JWTClaims from opal_common.authentication.verifier import JWTVerifier, Unauthorized from opal_common.logger import logger @@ -67,7 +69,7 @@ def verify_logged_in(verifier: JWTVerifier, token: Optional[str]) -> JWTClaims: raise -class _JWTAuthenticator: +class _JWTAuthenticator(Authenticator): def __init__(self, verifier: JWTVerifier): self._verifier = verifier @@ -75,10 +77,16 @@ def __init__(self, verifier: JWTVerifier): def verifier(self) -> JWTVerifier: return self._verifier + def signer(self) -> Optional[JWTSigner]: + return self._verifier + @property def enabled(self) -> JWTVerifier: return self._verifier.enabled + async def authenticate(self, headers): + pass + class JWTAuthenticator(_JWTAuthenticator): """bearer token authentication for http(s) api endpoints. diff --git a/packages/opal-common/opal_common/authentication/jwk.py b/packages/opal-common/opal_common/authentication/jwk.py new file mode 100644 index 000000000..182b5cdb9 --- /dev/null +++ b/packages/opal-common/opal_common/authentication/jwk.py @@ -0,0 +1,45 @@ +import jwt +import httpx + +from cachetools import TTLCache +from opal_common.authentication.verifier import Unauthorized + +class JWKManager: + def __init__(self, openid_configuration_url, jwt_algorithm, cache_maxsize, cache_ttl): + self._openid_configuration_url = openid_configuration_url + self._jwt_algorithm = jwt_algorithm + self._cache = TTLCache(maxsize=cache_maxsize, ttl=cache_ttl) + + def public_key(self, token): + header = jwt.get_unverified_header(token) + kid = header['kid'] + + public_key = self._cache.get(kid) + if public_key is None: + public_key = self._fetch_public_key(token) + self._cache[kid] = public_key + + return public_key + + def _fetch_public_key(self, token: str): + try: + return self._jwks_client().get_signing_key_from_jwt(token).key + except Exception: + raise Unauthorized(description="unknown JWT error") + + def _jwks_client(self): + oidc_config = self._openid_configuration() + signing_algorithms = oidc_config["id_token_signing_alg_values_supported"] + if self._jwt_algorithm.name not in signing_algorithms: + raise Unauthorized(description="unknown JWT algorithm") + if "jwks_uri" not in oidc_config: + raise Unauthorized(description="missing 'jwks_uri' property") + return jwt.PyJWKClient(oidc_config["jwks_uri"]) + + def _openid_configuration(self): + response = httpx.get(self._openid_configuration_url) + + if response.status_code != httpx.codes.OK: + raise Unauthorized(description=f"invalid status code {response.status_code}") + + return response.json() diff --git a/packages/opal-common/opal_common/authentication/oauth2.py b/packages/opal-common/opal_common/authentication/oauth2.py new file mode 100644 index 000000000..33bc4647a --- /dev/null +++ b/packages/opal-common/opal_common/authentication/oauth2.py @@ -0,0 +1,164 @@ +import asyncio +import httpx +import time + +from cachetools import cached, TTLCache +from fastapi import Header +from httpx import AsyncClient, BasicAuth +from opal_common.authentication.authenticator import Authenticator +from opal_common.authentication.deps import get_token_from_header +from opal_common.authentication.jwk import JWKManager +from opal_common.authentication.signer import JWTSigner +from opal_common.authentication.verifier import JWTVerifier, Unauthorized +from opal_common.config import opal_common_config +from typing import Optional + +class _OAuth2Authenticator(Authenticator): + async def authenticate(self, headers): + if "Authorization" not in headers: + token = await self.token() + headers['Authorization'] = f"Bearer {token}" + + +class OAuth2ClientCredentialsAuthenticator(_OAuth2Authenticator): + def __init__(self) -> None: + self._client_id = opal_common_config.OAUTH2_CLIENT_ID + self._client_secret = opal_common_config.OAUTH2_CLIENT_SECRET + self._token_url = opal_common_config.OAUTH2_TOKEN_URL + self._introspect_url = opal_common_config.OAUTH2_INTROSPECT_URL + self._jwt_algorithm = opal_common_config.OAUTH2_JWT_ALGORITHM + self._jwt_audience = opal_common_config.OAUTH2_JWT_AUDIENCE + self._jwt_issuer = opal_common_config.OAUTH2_JWT_ISSUER + self._jwk_manager = JWKManager( + opal_common_config.OAUTH2_OPENID_CONFIGURATION_URL, + opal_common_config.OAUTH2_JWT_ALGORITHM, + opal_common_config.OAUTH2_JWK_CACHE_MAXSIZE, + opal_common_config.OAUTH2_JWK_CACHE_TTL, + ) + + cfg = opal_common_config.OAUTH2_EXACT_MATCH_CLAIMS + if cfg is None: + self._exact_match_claims = {} + else: + self._exact_match_claims = dict(map(lambda x: x.split("="), cfg.split(","))) + + cfg = opal_common_config.OAUTH2_REQUIRED_CLAIMS + if cfg is None: + self._required_claims = [] + else: + self._required_claims = cfg.split(",") + + @property + def enabled(self): + return True + + def signer(self) -> Optional[JWTSigner]: + return None + + async def token(self): + auth = BasicAuth(self._client_id, self._client_secret) + data = {"grant_type": "client_credentials"} + + async with AsyncClient() as client: + response = await client.post(self._token_url, auth=auth, data=data) + return (response.json())['access_token'] + + def __call__(self, authorization: Optional[str] = Header(None)) -> {}: + token = get_token_from_header(authorization) + return self.verify(token) + + def verify(self, token: str) -> {}: + if self._introspect_url is not None: + claims = self._verify_opaque(token) + else: + claims = self._verify_jwt(token) + + self._verify_exact_match_claims(claims) + self._verify_required_claims(claims) + + return claims + + def _verify_opaque(self, token: str) -> {}: + response = httpx.post(self._introspect_url, data={'token': token}) + + if response.status_code != httpx.codes.OK: + raise Unauthorized(description=f"invalid status code {response.status_code}") + + claims = response.json() + active = claims.get("active", False) + if not active: + raise Unauthorized(description="inactive token") + + return claims or {} + + def _verify_jwt(self, token: str) -> {}: + public_key = self._jwk_manager.public_key(token) + + verifier = JWTVerifier( + public_key=public_key, + algorithm=self._jwt_algorithm, + audience=self._jwt_audience, + issuer=self._jwt_issuer, + ) + claims = verifier.verify(token) + + return claims or {} + + def _verify_exact_match_claims(self, claims): + for key, value in self._exact_match_claims.items(): + if key not in claims: + raise Unauthorized(description=f"missing required '{key}' claim") + elif claims[key] != value: + raise Unauthorized(description=f"invalid '{key}' claim value") + + def _verify_required_claims(self, claims): + for claim in self._required_claims: + if claim not in claims: + raise Unauthorized(description=f"missing required '{claim}' claim") + + +class CachedOAuth2Authenticator(_OAuth2Authenticator): + lock = asyncio.Lock() + + def __init__(self, delegate: OAuth2ClientCredentialsAuthenticator) -> None: + self._token = None + self._exp = None + self._exp_margin = opal_common_config.OAUTH2_EXP_MARGIN + self._delegate = delegate + + @property + def enabled(self): + return self._delegate.enabled + + def signer(self) -> Optional[JWTSigner]: + return self._delegate.signer() + + def _expired(self): + if self._token is None: + return True + + now = int(time.time()) + return now > self._exp - self._exp_margin + + async def token(self): + if not self._expired(): + return self._token + + async with CachedOAuth2Authenticator.lock: + if not self._expired(): + return self._token + + token = await self._delegate.token() + claims = self._delegate.verify(token) + + self._token = token + self._exp = claims['exp'] + + return self._token + + @cached(cache=TTLCache( + maxsize=opal_common_config.OAUTH2_TOKEN_VERIFY_CACHE_MAXSIZE, + ttl=opal_common_config.OAUTH2_TOKEN_VERIFY_CACHE_TTL + )) + def __call__(self, authorization: Optional[str] = Header(None)) -> {}: + return self._delegate(authorization) diff --git a/packages/opal-common/opal_common/config.py b/packages/opal-common/opal_common/config.py index ab18dd0cb..7414cd6cc 100644 --- a/packages/opal-common/opal_common/config.py +++ b/packages/opal-common/opal_common/config.py @@ -159,6 +159,68 @@ class OpalCommonConfig(Confi): [".rego"], description="List of extensions to serve as policy modules", ) + AUTH_TYPE = confi.str( + "AUTH_TYPE", + None, + description="Authentication type. Available options are oauth2 for validating access token via either OAUTH2_INTROSPECT_URL or OPAL_OAUTH2_OPENID_CONFIGURATION_URL or anything else if you prefer OPAL to do the job.", + ) + OAUTH2_CLIENT_ID = confi.str( + "OAUTH2_CLIENT_ID", None, description="OAuth2 Client ID." + ) + OAUTH2_CLIENT_SECRET = confi.str( + "OAUTH2_CLIENT_SECRET", None, description="OAuth2 Client Secret." + ) + OAUTH2_TOKEN_URL = confi.str( + "OAUTH2_TOKEN_URL", None, description="OAuth2 Token URL." + ) + OAUTH2_INTROSPECT_URL = confi.str( + "OAUTH2_INTROSPECT_URL", None, description="OAuth2 introspect URL." + ) + OAUTH2_OPENID_CONFIGURATION_URL = confi.str( + "OAUTH2_OPENID_CONFIGURATION_URL", + None, + description="OAuth2 OpenID configuration URL.", + ) + OAUTH2_TOKEN_VERIFY_CACHE_MAXSIZE = confi.int( + "OAUTH2_TOKEN_VERIFY_CACHE_MAXSIZE", + 100, + description="OAuth2 token validation cache maxsize.", + ) + OAUTH2_TOKEN_VERIFY_CACHE_TTL = confi.int( + "OAUTH2_TOKEN_VERIFY_CACHE_TTL", + 5 * 60, + description="OAuth2 token validation cache TTL.", + ) + + OAUTH2_EXP_MARGIN = confi.int( + "OAUTH2_EXP_MARGIN", 5 * 60, description="OAuth2 expiration margin." + ) + OAUTH2_EXACT_MATCH_CLAIMS = confi.str( + "OAUTH2_EXACT_MATCH_CLAIMS", None, description="OAuth2 exact match claims." + ) + OAUTH2_REQUIRED_CLAIMS = confi.str( + "OAUTH2_REQUIRED_CLAIMS", + None, + description="Comma separated list of required claims.", + ) + OAUTH2_JWT_ALGORITHM = confi.enum( + "OAUTH2_JWT_ALGORITHM", + JWTAlgorithm, + getattr(JWTAlgorithm, "RS256"), + description="jwt algorithm, possible values: see: https://pyjwt.readthedocs.io/en/stable/algorithms.html", + ) + OAUTH2_JWT_AUDIENCE = confi.str( + "OAUTH2_JWT_AUDIENCE", None, description="OAuth2 required audience" + ) + OAUTH2_JWT_ISSUER = confi.str( + "OAUTH2_JWT_ISSUER", None, description="OAuth2 required issuer" + ) + OAUTH2_JWK_CACHE_MAXSIZE = confi.int( + "OAUTH2_JWK_CACHE_MAXSIZE", 100, description="OAuth2 JWKS cache maxsize." + ) + OAUTH2_JWK_CACHE_TTL = confi.int( + "OAUTH2_JWK_CACHE_TTL", 7 * 24 * 60 * 60, description="OAuth2 JWKS cache TTL." + ) ENABLE_METRICS = confi.bool("ENABLE_METRICS", False) diff --git a/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py b/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py index fc74223ed..0ca9354c0 100644 --- a/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py +++ b/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py @@ -1,10 +1,12 @@ """Simple HTTP get data fetcher using requests supports.""" from enum import Enum -from typing import Any, Union, cast +from typing import Any, Optional, Union, cast import httpx from aiohttp import ClientResponse, ClientSession +from opal_common.authentication.authenticator import Authenticator +from opal_common.authentication.authenticator_factory import AuthenticatorFactory from opal_common.config import opal_common_config from opal_common.fetcher.events import FetcherConfig, FetchEvent from opal_common.fetcher.fetch_provider import BaseFetchProvider @@ -52,6 +54,8 @@ class HttpFetchEvent(FetchEvent): class HttpFetchProvider(BaseFetchProvider): + _authenticator: Optional[Authenticator] = None + def __init__(self, event: HttpFetchEvent) -> None: self._event: HttpFetchEvent if event.config is None: @@ -64,6 +68,9 @@ def __init__(self, event: HttpFetchEvent) -> None: if self._custom_ssl_context is not None else {} ) + if HttpFetchProvider._authenticator is None: + HttpFetchProvider._authenticator = AuthenticatorFactory.create() + self._authenticator = HttpFetchProvider._authenticator def parse_event(self, event: FetchEvent) -> HttpFetchEvent: return HttpFetchEvent(**event.dict(exclude={"config"}), config=event.config) @@ -71,7 +78,10 @@ def parse_event(self, event: FetchEvent) -> HttpFetchEvent: async def __aenter__(self): headers = {} if self._event.config.headers is not None: - headers = self._event.config.headers + headers = self._event.config.headers.copy() + + await self._authenticator.authenticate(headers) + if opal_common_config.HTTP_FETCHER_PROVIDER_CLIENT == "httpx": self._session = httpx.AsyncClient(headers=headers) else: diff --git a/packages/opal-common/requires.txt b/packages/opal-common/requires.txt index 90d21f1df..57198ba7b 100644 --- a/packages/opal-common/requires.txt +++ b/packages/opal-common/requires.txt @@ -9,5 +9,6 @@ tenacity>=8.0.1,<9 datadog>=0.44.0, <1 ddtrace>=2.8.1,<3 certifi>=2023.7.22 # not directly required, pinned by Snyk to avoid a vulnerability -requests>=2.31.0 # not directly required, pinned by Snyk to avoid a vulnerability -httpx==0.27.0 +requests>=2.32.0 # not directly required, pinned by Snyk to avoid a vulnerability +httpx>=0.27.0 +urllib3>=2.2.2 # not directly required, pinned by Snyk to avoid a vulnerability diff --git a/packages/opal-server/opal_server/authentication/__init__.py b/packages/opal-server/opal_server/authentication/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/packages/opal-server/opal_server/authentication/authenticator.py b/packages/opal-server/opal_server/authentication/authenticator.py new file mode 100644 index 000000000..def646382 --- /dev/null +++ b/packages/opal-server/opal_server/authentication/authenticator.py @@ -0,0 +1,18 @@ +from typing import Optional + +from fastapi import Header +from fastapi.exceptions import HTTPException +from opal_common.authentication.authenticator import Authenticator +from opal_common.authentication.types import JWTClaims +from opal_common.authentication.verifier import JWTVerifier, Unauthorized + + +class WebsocketServerAuthenticator(Authenticator): + def __init__(self, delegate: Authenticator) -> None: + self._delegate = delegate + + def __call__(self, authorization: Optional[str] = Header(None)) -> JWTClaims: + try: + return self._delegate(authorization) + except (Unauthorized, HTTPException): + return None diff --git a/packages/opal-server/opal_server/authentication/authenticator_factory.py b/packages/opal-server/opal_server/authentication/authenticator_factory.py new file mode 100644 index 000000000..43f4f092f --- /dev/null +++ b/packages/opal-server/opal_server/authentication/authenticator_factory.py @@ -0,0 +1,49 @@ +from opal_common.authentication.authenticator import Authenticator +from opal_common.authentication.deps import JWTAuthenticator +from opal_common.authentication.oauth2 import ( + CachedOAuth2Authenticator, + OAuth2ClientCredentialsAuthenticator, +) +from opal_common.authentication.signer import JWTSigner +from opal_common.config import opal_common_config +from opal_common.logger import logger +from opal_server.config import opal_server_config + +from .authenticator import WebsocketServerAuthenticator + + +class ServerAuthenticatorFactory: + @staticmethod + def create() -> Authenticator: + if opal_common_config.AUTH_TYPE == "oauth2": + logger.info( + "OPAL is running in secure mode - will verify API requests with OAuth2 tokens." + ) + return CachedOAuth2Authenticator(OAuth2ClientCredentialsAuthenticator()) + else: + return JWTAuthenticator(ServerAuthenticatorFactory.__signer()) + + @staticmethod + def __signer() -> JWTSigner: + signer = JWTSigner( + private_key=opal_server_config.AUTH_PRIVATE_KEY, + public_key=opal_common_config.AUTH_PUBLIC_KEY, + algorithm=opal_common_config.AUTH_JWT_ALGORITHM, + audience=opal_common_config.AUTH_JWT_AUDIENCE, + issuer=opal_common_config.AUTH_JWT_ISSUER, + ) + if signer.enabled: + logger.info( + "OPAL is running in secure mode - will verify API requests with JWT tokens." + ) + else: + logger.info( + "OPAL was not provided with JWT encryption keys, cannot verify api requests!" + ) + return signer + + +class WebsocketServerAuthenticatorFactory: + @staticmethod + def create() -> Authenticator: + return WebsocketServerAuthenticator(ServerAuthenticatorFactory.create()) diff --git a/packages/opal-server/opal_server/config.py b/packages/opal-server/opal_server/config.py index b272915ad..e02cadffa 100644 --- a/packages/opal-server/opal_server/config.py +++ b/packages/opal-server/opal_server/config.py @@ -28,7 +28,7 @@ class ServerRole(str, Enum): class OpalServerConfig(Confi): # ws server OPAL_WS_LOCAL_URL = confi.str("WS_LOCAL_URL", "ws://localhost:7002/ws") - OPAL_WS_TOKEN = confi.str("WS_TOKEN", "THIS_IS_A_DEV_SECRET") + OPAL_WS_TOKEN = confi.str("WS_TOKEN", None) CLIENT_LOAD_LIMIT_NOTATION = confi.str( "CLIENT_LOAD_LIMIT_NOTATION", None, diff --git a/packages/opal-server/opal_server/data/api.py b/packages/opal-server/opal_server/data/api.py index da5d043a9..45d953b41 100644 --- a/packages/opal-server/opal_server/data/api.py +++ b/packages/opal-server/opal_server/data/api.py @@ -6,7 +6,8 @@ require_peer_type, restrict_optional_topics_to_publish, ) -from opal_common.authentication.deps import JWTAuthenticator, get_token_from_header +from opal_common.authentication.authenticator import Authenticator +from opal_common.authentication.deps import get_token_from_header from opal_common.authentication.types import JWTClaims from opal_common.authentication.verifier import Unauthorized from opal_common.logger import logger @@ -25,7 +26,7 @@ def init_data_updates_router( data_update_publisher: DataUpdatePublisher, data_sources_config: ServerDataSourceConfig, - authenticator: JWTAuthenticator, + authenticator: Authenticator, ): router = APIRouter() diff --git a/packages/opal-server/opal_server/policy/webhook/api.py b/packages/opal-server/opal_server/policy/webhook/api.py index c19595ad2..ef54c81b4 100644 --- a/packages/opal-server/opal_server/policy/webhook/api.py +++ b/packages/opal-server/opal_server/policy/webhook/api.py @@ -3,7 +3,7 @@ from fastapi import APIRouter, Depends, Request, status from fastapi_websocket_pubsub.pub_sub_server import PubSubEndpoint -from opal_common.authentication.deps import JWTAuthenticator +from opal_common.authentication.authenticator import Authenticator from opal_common.logger import logger from opal_common.schemas.webhook import GitWebhookRequestParams from opal_server.config import PolicySourceTypes, opal_server_config @@ -15,7 +15,7 @@ def init_git_webhook_router( - pubsub_endpoint: PubSubEndpoint, authenticator: JWTAuthenticator + pubsub_endpoint: PubSubEndpoint, authenticator: Authenticator ): async def dummy_affected_repo_urls(request: Request) -> List[str]: return [] diff --git a/packages/opal-server/opal_server/pubsub.py b/packages/opal-server/opal_server/pubsub.py index 26d47c422..3b5c18f70 100644 --- a/packages/opal-server/opal_server/pubsub.py +++ b/packages/opal-server/opal_server/pubsub.py @@ -21,13 +21,12 @@ WebSocketRpcEventNotifier, ) from fastapi_websocket_rpc import RpcChannel -from opal_common.authentication.deps import WebsocketJWTAuthenticator -from opal_common.authentication.signer import JWTSigner from opal_common.authentication.types import JWTClaims from opal_common.authentication.verifier import Unauthorized from opal_common.confi.confi import load_conf_if_none from opal_common.config import opal_common_config from opal_common.logger import logger +from opal_server.authentication.authenticator import WebsocketServerAuthenticator from opal_server.config import opal_server_config from pydantic import BaseModel from starlette.datastructures import QueryParams @@ -121,7 +120,11 @@ class PubSub: """Wrapper for the Pub/Sub channel used for both policy and data updates.""" - def __init__(self, signer: JWTSigner, broadcaster_uri: str = None): + def __init__( + self, + broadcaster_uri: str = None, + authenticator: Optional[WebsocketServerAuthenticator] = None, + ): """ Args: broadcaster_uri (str, optional): Which server/medium should the PubSub use for broadcasting. Defaults to BROADCAST_URI. @@ -159,7 +162,6 @@ def __init__(self, signer: JWTSigner, broadcaster_uri: str = None): not opal_server_config.BROADCAST_CONN_LOSS_BUGFIX_EXPERIMENT_ENABLED ), ) - authenticator = WebsocketJWTAuthenticator(signer) @self.api_router.get( "/pubsub_client_info", response_model=Dict[str, ClientInfo] diff --git a/packages/opal-server/opal_server/scopes/api.py b/packages/opal-server/opal_server/scopes/api.py index 95181866a..e2975f4e6 100644 --- a/packages/opal-server/opal_server/scopes/api.py +++ b/packages/opal-server/opal_server/scopes/api.py @@ -16,12 +16,14 @@ from fastapi_websocket_pubsub import PubSubEndpoint from git import InvalidGitRepositoryError from opal_common.async_utils import run_sync +from opal_common.authentication.authenticator import Authenticator from opal_common.authentication.authz import ( require_peer_type, restrict_optional_topics_to_publish, ) +from opal_common.authentication.authenticator import Authenticator from opal_common.authentication.casting import cast_private_key -from opal_common.authentication.deps import JWTAuthenticator, get_token_from_header +from opal_common.authentication.deps import get_token_from_header from opal_common.authentication.types import EncryptionKeyFormat, JWTClaims from opal_common.authentication.verifier import Unauthorized from opal_common.logger import logger @@ -78,7 +80,7 @@ def verify_private_key_or_throw(scope_in: Scope): def init_scope_router( scopes: ScopeRepository, - authenticator: JWTAuthenticator, + authenticator: Authenticator, pubsub_endpoint: PubSubEndpoint, ): router = APIRouter() diff --git a/packages/opal-server/opal_server/security/jwks.py b/packages/opal-server/opal_server/security/jwks.py index c55dfe5f3..3da016ecb 100644 --- a/packages/opal-server/opal_server/security/jwks.py +++ b/packages/opal-server/opal_server/security/jwks.py @@ -1,5 +1,6 @@ import json from pathlib import Path +from typing import Optional from fastapi import FastAPI from fastapi.staticfiles import StaticFiles @@ -11,7 +12,7 @@ class JwksStaticEndpoint: def __init__( self, - signer: JWTSigner, + signer: Optional[JWTSigner], jwks_url: str, jwks_static_dir: str, ): @@ -25,7 +26,7 @@ def configure_app(self, app: FastAPI): # get the jwks contents from the signer jwks_contents = {} - if self._signer.enabled: + if self._signer is not None and self._signer.enabled: jwk = json.loads(self._signer.get_jwk()) jwks_contents = {"keys": [jwk]} diff --git a/packages/opal-server/opal_server/server.py b/packages/opal-server/opal_server/server.py index 34d9905c3..37f41f5c7 100644 --- a/packages/opal-server/opal_server/server.py +++ b/packages/opal-server/opal_server/server.py @@ -8,8 +8,8 @@ from fastapi import Depends, FastAPI from fastapi_websocket_pubsub.event_broadcaster import EventBroadcasterContextManager -from opal_common.authentication.deps import JWTAuthenticator, StaticBearerAuthenticator -from opal_common.authentication.signer import JWTSigner +from opal_common.authentication.authenticator import Authenticator +from opal_common.authentication.deps import StaticBearerAuthenticator from opal_common.confi.confi import load_conf_if_none from opal_common.config import opal_common_config from opal_common.logger import configure_logs, logger @@ -22,6 +22,11 @@ ServerSideTopicPublisher, TopicPublisher, ) +from opal_server.authentication.authenticator import WebsocketServerAuthenticator +from opal_server.authentication.authenticator_factory import ( + ServerAuthenticatorFactory, + WebsocketServerAuthenticatorFactory, +) from opal_server.config import opal_server_config from opal_server.data.api import init_data_updates_router from opal_server.data.data_update_publisher import DataUpdatePublisher @@ -49,7 +54,8 @@ def __init__( init_publisher: bool = None, data_sources_config: Optional[ServerDataSourceConfig] = None, broadcaster_uri: str = None, - signer: Optional[JWTSigner] = None, + authenticator: Optional[Authenticator] = None, + websocketAuthenticator: Optional[WebsocketServerAuthenticator] = None, enable_jwks_endpoint=True, jwks_url: str = None, jwks_static_dir: str = None, @@ -117,33 +123,26 @@ def __init__( self.broadcaster_uri = broadcaster_uri self.master_token = master_token - if signer is not None: - self.signer = signer - else: - self.signer = JWTSigner( - private_key=opal_server_config.AUTH_PRIVATE_KEY, - public_key=opal_common_config.AUTH_PUBLIC_KEY, - algorithm=opal_common_config.AUTH_JWT_ALGORITHM, - audience=opal_common_config.AUTH_JWT_AUDIENCE, - issuer=opal_common_config.AUTH_JWT_ISSUER, - ) - if self.signer.enabled: - logger.info( - "OPAL is running in secure mode - will verify API requests with JWT tokens." - ) + if authenticator is not None: + self.authenticator = authenticator else: - logger.info( - "OPAL was not provided with JWT encryption keys, cannot verify api requests!" - ) + self.authenticator = ServerAuthenticatorFactory.create() if enable_jwks_endpoint: self.jwks_endpoint = JwksStaticEndpoint( - signer=self.signer, jwks_url=jwks_url, jwks_static_dir=jwks_static_dir + signer=self.authenticator.signer(), + jwks_url=jwks_url, + jwks_static_dir=jwks_static_dir, ) else: self.jwks_endpoint = None - self.pubsub = PubSub(signer=self.signer, broadcaster_uri=broadcaster_uri) + _websocketAuthenticator = websocketAuthenticator + if _websocketAuthenticator is None: + _websocketAuthenticator = WebsocketServerAuthenticatorFactory.create() + self.pubsub = PubSub( + broadcaster_uri=broadcaster_uri, authenticator=_websocketAuthenticator + ) self.publisher: Optional[TopicPublisher] = None self.broadcast_keepalive: Optional[PeriodicPublisher] = None @@ -219,19 +218,19 @@ def _configure_monitoring(self): def _configure_api_routes(self, app: FastAPI): """mounts the api routes on the app object.""" - authenticator = JWTAuthenticator(self.signer) - data_update_publisher: Optional[DataUpdatePublisher] = None if self.publisher is not None: data_update_publisher = DataUpdatePublisher(self.publisher) # Init api routers with required dependencies data_updates_router = init_data_updates_router( - data_update_publisher, self.data_sources_config, authenticator + data_update_publisher, self.data_sources_config, self.authenticator + ) + webhook_router = init_git_webhook_router( + self.pubsub.endpoint, self.authenticator ) - webhook_router = init_git_webhook_router(self.pubsub.endpoint, authenticator) security_router = init_security_router( - self.signer, StaticBearerAuthenticator(self.master_token) + self.authenticator.signer(), StaticBearerAuthenticator(self.master_token) ) statistics_router = init_statistics_router(self.opal_statistics) loadlimit_router = init_loadlimit_router(self.loadlimit_notation) @@ -240,7 +239,7 @@ def _configure_api_routes(self, app: FastAPI): app.include_router( bundles_router, tags=["Bundle Server"], - dependencies=[Depends(authenticator)], + dependencies=[Depends(self.authenticator)], ) app.include_router(data_updates_router, tags=["Data Updates"]) app.include_router(webhook_router, tags=["Github Webhook"]) @@ -249,22 +248,24 @@ def _configure_api_routes(self, app: FastAPI): app.include_router( self.pubsub.api_router, tags=["Pub/Sub"], - dependencies=[Depends(authenticator)], + dependencies=[Depends(self.authenticator)], ) app.include_router( statistics_router, tags=["Server Statistics"], - dependencies=[Depends(authenticator)], + dependencies=[Depends(self.authenticator)], ) app.include_router( loadlimit_router, tags=["Client Load Limiting"], - dependencies=[Depends(authenticator)], + dependencies=[Depends(self.authenticator)], ) if opal_server_config.SCOPES: app.include_router( - init_scope_router(self._scopes, authenticator, self.pubsub.endpoint), + init_scope_router( + self._scopes, self.authenticator, self.pubsub.endpoint + ), tags=["Scopes"], prefix="/scopes", ) diff --git a/packages/opal-server/requires.txt b/packages/opal-server/requires.txt index 3a78295dc..ff3e5cb13 100644 --- a/packages/opal-server/requires.txt +++ b/packages/opal-server/requires.txt @@ -2,7 +2,6 @@ click>=8.1.3,<9 permit-broadcaster[postgres,redis,kafka]==0.2.5 gitpython>=3.1.32,<4 pyjwt[crypto]>=2.1.0,<3 -websockets>=10.3,<11 slowapi>=0.1.5,<1 # slowapi is stuck on and old `redis`, so fix that and switch from aioredis to redis pygit2>=1.14.1,<1.15 diff --git a/packages/requires.txt b/packages/requires.txt index 43f5a740d..98580b47d 100644 --- a/packages/requires.txt +++ b/packages/requires.txt @@ -2,10 +2,13 @@ idna>=3.3,<4 typer>=0.4.1,<1 fastapi>=0.109.1,<1 fastapi_websocket_pubsub==0.3.7 -fastapi_websocket_rpc>=0.1.21,<1 +fastapi_websocket_rpc==0.1.27 +websockets>=10.3,<14 gunicorn>=22.0.0,<23 pydantic[email]>=1.9.1,<2 typing-extensions;python_version<'3.8' uvicorn[standard]>=0.17.6,<1 fastapi-utils>=0.2.1,<1 setuptools>=70.0.0 # not directly required, pinned by Snyk to avoid a vulnerability +anyio>=4.4.0 # not directly required, pinned by Snyk to avoid a vulnerability +cachetools>=5.3.3 diff --git a/requirements.txt b/requirements.txt index 69533e9fd..656fe7c60 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,5 +7,5 @@ pytest-asyncio pytest-rerunfailures wheel>=0.38.0 twine -setuptools>=65.5.1 # not directly required, pinned by Snyk to avoid a vulnerability +setuptools>=70.0.0 # not directly required, pinned by Snyk to avoid a vulnerability zipp>=3.19.1 # not directly required, pinned by Snyk to avoid a vulnerability