diff --git a/docs/changelog/1_0_a2.rst b/docs/changelog/1_0_a2.rst index 52d317d315f..2b15f77c5ce 100644 --- a/docs/changelog/1_0_a2.rst +++ b/docs/changelog/1_0_a2.rst @@ -7,7 +7,7 @@ =========== This changelog summarizes new features and breaking changes in -`EdgeDB 1.0 alpha 2 `_. +`EdgeDB 1.0 alpha 2 `_. New JavaScript Driver @@ -47,7 +47,7 @@ and it is ready for use: main(); -The documentation can be found `here `_. +The documentation can be found :ref:`here `. Standard Library diff --git a/docs/changelog/1_0_a3.rst b/docs/changelog/1_0_a3.rst index ba09e503917..b47dfbab9de 100644 --- a/docs/changelog/1_0_a3.rst +++ b/docs/changelog/1_0_a3.rst @@ -8,7 +8,7 @@ This changelog summarizes new features and breaking changes in `EdgeDB 1.0 alpha 3 "Proxima Centauri" -`_. +`_. CLI diff --git a/docs/changelog/1_0_a4.rst b/docs/changelog/1_0_a4.rst index aab1fd55d20..22f7d5637e4 100644 --- a/docs/changelog/1_0_a4.rst +++ b/docs/changelog/1_0_a4.rst @@ -8,7 +8,7 @@ This changelog summarizes new features and breaking changes in `EdgeDB 1.0 alpha 4 "Barnard's Star" -`_. +`_. EdgeQL diff --git a/docs/changelog/1_0_a5.rst b/docs/changelog/1_0_a5.rst index ea31a0fe742..7de1a2b48a6 100644 --- a/docs/changelog/1_0_a5.rst +++ b/docs/changelog/1_0_a5.rst @@ -7,7 +7,7 @@ =========== This changelog summarizes new features and breaking changes in -`EdgeDB 1.0 alpha 5 "Luhman" `_. +`EdgeDB 1.0 alpha 5 "Luhman" `_. EdgeQL @@ -108,8 +108,7 @@ CLI Bindings ======== -* Add transaction `API - `_ to JS binding +* Add transaction :ref:`API ` to JS binding (`#61 `_). Here's an example of using transactions: diff --git a/docs/changelog/2_x.rst b/docs/changelog/2_x.rst index bf90b535af3..4d3fc4525a4 100644 --- a/docs/changelog/2_x.rst +++ b/docs/changelog/2_x.rst @@ -91,7 +91,7 @@ upgrade. - ``edgedb@0.21.0`` * - :ref:`Python ` - ``edgedb@0.24.0`` - * - `Golang `_ + * - :ref:`Golang ` - ``edgedb@0.12.0`` * - `Rust `_ - ``edgedb-tokio@0.3.0`` diff --git a/docs/clients/dart/index.rst b/docs/clients/dart/index.rst index 7c154cd450b..073632aca15 100644 --- a/docs/clients/dart/index.rst +++ b/docs/clients/dart/index.rst @@ -1,3 +1,5 @@ +.. _edgedb-dart-intro: + ==== Dart ==== diff --git a/docs/clients/dotnet/index.rst b/docs/clients/dotnet/index.rst index df9789d7e4d..d5818190afe 100644 --- a/docs/clients/dotnet/index.rst +++ b/docs/clients/dotnet/index.rst @@ -1,3 +1,5 @@ +.. _edgedb-dotnet-intro: + ==== .NET ==== diff --git a/docs/clients/go/index.rst b/docs/clients/go/index.rst index 1edad36671f..fbc70705e0f 100644 --- a/docs/clients/go/index.rst +++ b/docs/clients/go/index.rst @@ -1,3 +1,5 @@ +.. _edgedb-go-intro: + == Go == diff --git a/docs/clients/rust/getting_started.rst b/docs/clients/rust/getting_started.rst index b29d18b3f3e..b73a1f78476 100644 --- a/docs/clients/rust/getting_started.rst +++ b/docs/clients/rust/getting_started.rst @@ -120,14 +120,14 @@ the `blog post introducing the EdgeDB projects CLI`_. and ``edgedb migrate``. - a ``/migrations`` folder with ``.edgeql`` files named starting at - ``00001``. These hold the `ddl`_ commands that were used to migrate your - schema. A new file shows up in this directory every time your schema - is migrated. + ``00001``. These hold the :ref:`ddl ` commands that were used + to migrate your schema. A new file shows up in this directory every time + your schema is migrated. If you are running EdgeDB 3.0 and above, you also have the option of using -the `edgedb watch`_ command. Doing so starts a long-running process that -keeps an eye on changes in ``/dbschema``, automatically applying these -changes in real time. +the :ref:`edgedb watch ` command. Doing so starts a +long-running process that keeps an eye on changes in ``/dbschema``, +automatically applying these changes in real time. Now that you have the right dependencies and an EdgeDB instance, you can create a client. @@ -135,9 +135,7 @@ you can create a client. .. _`blog post introducing the EdgeDB projects CLI`: https://www.edgedb.com/blog/introducing-edgedb-projects .. _`bridging methods`: https://tokio.rs/tokio/topics/bridging -.. _`ddl`: https://www.edgedb.com/docs/reference/ddl/index .. _`edgedb-derive`: https://docs.rs/edgedb-derive/latest/edgedb_derive/ .. _`edgedb-protocol`: https://docs.rs/edgedb-protocol/latest/edgedb_protocol .. _`edgedb-tokio`: https://docs.rs/edgedb-tokio/latest/edgedb_tokio -.. _`edgedb watch`: https://www.edgedb.com/docs/cli/edgedb_watch .. _`examples repo`: https://github.com/Dhghomon/edgedb_rust_client_examples \ No newline at end of file diff --git a/docs/clients/rust/transactions.rst b/docs/clients/rust/transactions.rst index 75d3e660201..6924b761f60 100644 --- a/docs/clients/rust/transactions.rst +++ b/docs/clients/rust/transactions.rst @@ -4,7 +4,7 @@ Transactions ------------ The client also has a ``.transaction()`` method that -allows for atomic `transactions`_. +allows for atomic :ref:`transactions `. Wikipedia has a good example of a scenario requiring a transaction which we can then implement: @@ -63,11 +63,9 @@ another's would look like this: .. note:: What often may seem to require an atomic transaction can instead be - achieved with links and `backlinks`_ which are both idiomatic and easy to - use in EdgeDB. For example, if one object holds a ``required link`` to two + achieved with links and :ref:`backlinks ` which + are both idiomatic and easy to use in EdgeDB. + For example, if one object holds a ``required link`` to two other objects and each of these two objects has a single backlink to the first one, simply updating the first object will effectively change the state of the other two instantaneously. - -.. _`backlinks`: https://www.edgedb.com/docs/edgeql/paths#backlinks -.. _`transactions`: https://www.edgedb.com/docs/edgeql/transactions \ No newline at end of file diff --git a/docs/conf.py b/docs/conf.py index 36f929e97c0..0d7e1847a33 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -108,6 +108,8 @@ # If true, keep warnings as "system message" paragraphs in the built documents. # keep_warnings = False +suppress_warnings = ['image.not_readable'] + # If true, `todo` and `todoList` produce output, else they produce nothing. todo_include_todos = False diff --git a/docs/datamodel/objects.rst b/docs/datamodel/objects.rst index 9ae25a4d4a6..0b1a9d84a9b 100644 --- a/docs/datamodel/objects.rst +++ b/docs/datamodel/objects.rst @@ -157,7 +157,7 @@ of the same type and cardinality. .. note:: Refer to the dedicated pages on :ref:`Indexes `, - :ref:`Constraints `, :ref:`Object-Level Security + :ref:`Constraints `, :ref:`Access Policies `, and :ref:`Annotations ` for documentation on these concepts. diff --git a/docs/edgeql/index.rst b/docs/edgeql/index.rst index 3fff601efe1..99ee4d9a460 100644 --- a/docs/edgeql/index.rst +++ b/docs/edgeql/index.rst @@ -65,15 +65,15 @@ like code and less like word soup. to write deep, performant queries that traverse links, no ``JOINs`` required. **Composable**. `Unlike SQL -`_, EdgeQL's syntax is -readily composable; queries can be cleanly nested without worrying about -Cartesian explosion. +`_, +EdgeQL's syntax is readily composable; queries can be cleanly nested without +worrying about Cartesian explosion. .. note:: For a detailed writeup on the design of SQL, see `We Can Do Better Than SQL - `_ on the EdgeDB - blog. + `_ + on the EdgeDB blog. Follow along ------------ diff --git a/docs/edgeql/parameters.rst b/docs/edgeql/parameters.rst index 1a579529a94..f799056944f 100644 --- a/docs/edgeql/parameters.rst +++ b/docs/edgeql/parameters.rst @@ -91,6 +91,8 @@ Refer to the Datatypes page of your preferred :ref:`client library language-native types. +.. _ref_eql_params_types: + Parameter types and JSON ------------------------ diff --git a/docs/edgeql/sets.rst b/docs/edgeql/sets.rst index 2b023892a68..768c9efa8ba 100644 --- a/docs/edgeql/sets.rst +++ b/docs/edgeql/sets.rst @@ -131,8 +131,8 @@ of data; in EdgeDB the absence of data is just an empty set. permeates all of SQL and is often handled inconsistently in different circumstances. A number of specific inconsistencies are documented in detail in the `We Can Do Better Than SQL - `_ post on the - EdgeDB blog. For broader context, see Tony Hoare's talk + `_ + post on the EdgeDB blog. For broader context, see Tony Hoare's talk `"The Billion Dollar Mistake" `_. diff --git a/docs/edgeql/transactions.rst b/docs/edgeql/transactions.rst index f201b778a91..df010ea18ce 100644 --- a/docs/edgeql/transactions.rst +++ b/docs/edgeql/transactions.rst @@ -96,8 +96,7 @@ Using the querybuilder: await query2.run(tx); }); -Full documentation at `Client Libraries > TypeScript/JS -`_; +Full documentation at :ref:`Client Libraries > TypeScript/JS `; Python ^^^^^^ @@ -113,8 +112,7 @@ Python filter .name = 'Customer2' set { bank_balance := .bank_balance +10 };""") -Full documentation at `Client Libraries > Python -`_; +Full documentation at :ref:`Client Libraries > Python `; Golang ^^^^^^ @@ -140,7 +138,7 @@ Golang log.Fatal(err) } -Full documentation at `Client Libraries > Go `_. +Full documentation at :ref:`Client Libraries > Go `. Rust ^^^^ @@ -164,4 +162,4 @@ Rust .await .expect("Transaction should have worked"); -Full documentation at `Client Libraries > Rust `_. +Full documentation at :ref:`Client Libraries > Rust `. diff --git a/docs/guides/cheatsheet/index.rst b/docs/guides/cheatsheet/index.rst index 6ad9a79bc74..eb221bbaf5c 100644 --- a/docs/guides/cheatsheet/index.rst +++ b/docs/guides/cheatsheet/index.rst @@ -12,7 +12,8 @@ After familiarizing yourself with them, feel free to dive into more EdgeDB via our longer `interactive tutorial `_ and **much** longer `Easy EdgeDB textbook `_. -EdgeQL: +EdgeQL +====== * :ref:`select ` -- Retrieve or compute a set of values. * :ref:`insert ` -- Create new database objects. @@ -21,8 +22,11 @@ EdgeQL: * :ref:`GraphQL ` -- GraphQL queries supported natively out of the box. -Schema: +Schema +====== +* :ref:`Booleans ` -- Boolean expressions can be + tricky sometimes, so here are a handful of tips and gotchas. * :ref:`Object Types ` -- Make your own object and abstract types on top of existing system types. * :ref:`User Defined Functions ` -- Write and @@ -31,8 +35,11 @@ Schema: new types and modify existing ones on the fly. * :ref:`Schema Annotations ` -- Add human readable descriptions to items in your schema. +* :ref:`Link Properties ` -- Links can contain properties + used to store metadata about the link. -CLI/Admin: +CLI/Admin +========= * :ref:`CLI Usage ` -- Getting your database started. * :ref:`Interactive Shell ` -- Shortcuts for diff --git a/docs/guides/cloud/index.rst b/docs/guides/cloud/index.rst index fa49949831d..83feefff091 100644 --- a/docs/guides/cloud/index.rst +++ b/docs/guides/cloud/index.rst @@ -39,7 +39,7 @@ better! `_ for information on what may be causing it. * Report any bugs you find by `submitting a support ticket - `_. Note: when using EdgeDB Cloud + `_. Note: when using EdgeDB Cloud through the CLI, setting the ``RUST_LOG`` environment variable to ``info``, ``debug``, or ``trace`` may provide additional debugging information which will be useful to include with your ticket. diff --git a/docs/guides/contributing/documentation.rst b/docs/guides/contributing/documentation.rst index 4bb831bd864..e71acbf21be 100644 --- a/docs/guides/contributing/documentation.rst +++ b/docs/guides/contributing/documentation.rst @@ -85,13 +85,12 @@ find it alongside the client itself. These clients will also have documentation stubs inside the edgedb repository directing you to the documentation's location. -The `EdgeDB tutorial `_ is part of `our web +The `EdgeDB tutorial `_ is part of `our web site repository `_. You'll find it in `the tutorial directory `_. -Finally, our book for beginners titled `Easy EdgeDB -`_ lives in `its own repo -`_. +Finally, our book for beginners titled `Easy EdgeDB `_ lives in +`its own repo `_. How to Build It diff --git a/docs/guides/datamigrations/postgres.rst b/docs/guides/datamigrations/postgres.rst index 596cf396874..3d36feb026c 100644 --- a/docs/guides/datamigrations/postgres.rst +++ b/docs/guides/datamigrations/postgres.rst @@ -146,7 +146,7 @@ for now. Incidentally, even if the ``id`` was specified as a ``uuid`` value the recommended process is to record it as ``app_id`` as opposed to try and replicate it as the main object ``id``. It is, however, also possible to bring it over as the main ``id`` by adjusting certain :ref:`client connection -` settings. The column ``client_settings`` +settings `. The column ``client_settings`` would become a :eql:type:`json` property. The columns ``badge_name`` and ``status_id`` reference ``badges`` and ``statuses`` respectively and will become *links* in EdgeDB instead of *properties*, even though a property would diff --git a/docs/guides/deployment/fly_io.rst b/docs/guides/deployment/fly_io.rst index b98d0688c16..6d75eec713a 100644 --- a/docs/guides/deployment/fly_io.rst +++ b/docs/guides/deployment/fly_io.rst @@ -164,6 +164,7 @@ to Postgres: ... ALTER ROLE +.. _ref_guide_deployment_fly_io_start_edgedb: Start EdgeDB ============ @@ -293,10 +294,10 @@ something like this: restart_limit = 0 timeout = "2s" -In the same directory, `redeploy the EdgeDB app <#start-edgedb>`_. -This makes the EdgeDB port available to the outside world. You can now -access the instance from any host via the following public DSN: -``edgedb://edgedb:$PASSWORD@$EDB_APP.fly.dev``. +In the same directory, :ref:`redeploy the EdgeDB app +`. This makes the EdgeDB port +available to the outside world. You can now access the instance from any host +via the following public DSN: ``edgedb://edgedb:$PASSWORD@$EDB_APP.fly.dev``. To secure communication between the server and the client, you will also need to set the ``EDGEDB_TLS_CA`` environment secret in your application. diff --git a/docs/guides/deployment/heroku.rst b/docs/guides/deployment/heroku.rst index 9e549e371c3..7358c4e6312 100644 --- a/docs/guides/deployment/heroku.rst +++ b/docs/guides/deployment/heroku.rst @@ -42,14 +42,12 @@ First copy the code, initialize a new git repo, and create a new heroku app. $ heroku apps:create --buildpack heroku/nodejs $ edgedb project init --non-interactive -If you are using the `JS query builder for EdgeDB `_ then +If you are using the :ref:`JS query builder for EdgeDB ` then you will need to check the ``dbschema/edgeql-js`` directory in to your git repo after running ``yarn edgeql-js``. The ``edgeql-js`` command cannot be run during the build step on Heroku because it needs access to a running EdgeDB instance which is not available at build time on Heroku. -.. _js-query-builder: https://www.edgedb.com/docs/clients/js/index - .. code-block:: bash $ yarn install && npx @edgedb/generate edgeql-js diff --git a/docs/guides/migrations/guide.rst b/docs/guides/migrations/guide.rst index 7187be4d981..72dd9363e53 100644 --- a/docs/guides/migrations/guide.rst +++ b/docs/guides/migrations/guide.rst @@ -612,7 +612,7 @@ require appending ``--allow-empty`` to the command. Just do the following: schema file manually, copy the suggested name into the migration hash and type ``edgedb migrate`` again. -The `EdgeDB tutorial `_ is a good example of a database +The `EdgeDB tutorial `_ is a good example of a database set up with both a schema migration and a data migration. Setting up a database with `schema changes in one file and default data in a second file `_ is a nice way to separate the two operations @@ -1204,7 +1204,7 @@ want to accept all of the suggestions provided by the server. This process is in fact still used to migrate even today; the CLI just facilitates it by making it easy to respond to the generated suggestions. -`Early EdgeDB migrations took place inside a transaction `_ +:ref:`Early EdgeDB migrations took place inside a transaction ` handled by the user that essentially went like this: .. code-block:: @@ -1353,11 +1353,5 @@ migration and give it a proper ``.edgeql`` file in the same way we did above in the "So you really wanted to use DDL but now regret it?" section. -.. lint-off - .. _rfc: https://github.com/edgedb/rfcs/blob/master/text/1000-migrations.rst -.. _transaction: https://www.edgedb.com/docs/reference/ddl/migrations -.. _tutorial: https://www.edgedb.com/tutorial -.. _tutorial_files: https://github.com/edgedb/website/tree/main/content/tutorial/dbschema/migrations - -.. lint-on \ No newline at end of file +.. _tutorial_files: https://github.com/edgedb/website/tree/main/content/tutorial/dbschema/migrations \ No newline at end of file diff --git a/docs/guides/tutorials/chatgpt_bot.rst b/docs/guides/tutorials/chatgpt_bot.rst index 4424c7bddea..f550c8a4e4a 100644 --- a/docs/guides/tutorials/chatgpt_bot.rst +++ b/docs/guides/tutorials/chatgpt_bot.rst @@ -457,8 +457,8 @@ database because of the ``pgvector`` extension. In order to use it in our schema, we have to activate the ``ext::pgvector`` module with ``using extension pgvector`` at the beginning of the schema file. This module gives us access to the ``ext::pgvector::vector`` data type as well as few similarity functions and -indexes we can use later to retrieve embeddings. Read our `pgvector -documentation `_ for more details +indexes we can use later to retrieve embeddings. Read our :ref:`pgvector +documentation ` for more details on the extension. Just below that, we can start building our module by creating a new scalar @@ -603,8 +603,8 @@ libraries that will help us. The ``@edgedb/generate`` package provides a set of code generation tools that are useful when developing an EdgeDB-backed applications with -TypeScript/JavaScript. We're going to write queries using our `query builder -`_, but before we can, we +TypeScript/JavaScript. We're going to write queries using our +:ref:`query builder `, but before we can, we need to run the query builder generator. .. code-block:: bash diff --git a/docs/guides/tutorials/cloudflare_workers.rst b/docs/guides/tutorials/cloudflare_workers.rst index 993c099783a..0c920b4484b 100644 --- a/docs/guides/tutorials/cloudflare_workers.rst +++ b/docs/guides/tutorials/cloudflare_workers.rst @@ -27,11 +27,10 @@ Prerequisites Ensure you have the following installed: - `Node.js`_ -- `EdgeDB CLI`_ +- :ref:`EdgeDB CLI ` .. _Sign up for a Cloudflare account: https://dash.cloudflare.com/sign-up .. _Node.js: https://nodejs.org/en/ -.. _EdgeDB CLI: https://www.edgedb.com/docs/intro/cli Setup and configuration ----------------------- diff --git a/docs/guides/tutorials/graphql_apis_with_strawberry.rst b/docs/guides/tutorials/graphql_apis_with_strawberry.rst index 2a855b82e5e..572bfceb75b 100644 --- a/docs/guides/tutorials/graphql_apis_with_strawberry.rst +++ b/docs/guides/tutorials/graphql_apis_with_strawberry.rst @@ -9,7 +9,7 @@ extension. It enables you to expose GraphQL-driven CRUD APIs for all object types, their properties, links, and aliases. This opens up the scope for creating backend-less applications where the users will directly communicate with the database. You can learn more about that in the -`GraphQL `_ section of the docs. +:ref:`GraphQL ` section of the docs. However, as of now, EdgeDB is not ready to be used as a standalone backend. You shouldn't expose your EdgeDB instance directly to the application’s frontend; @@ -23,7 +23,7 @@ like schema, query, mutation, resolver, validator, etc, and have used GraphQL with some other technology before. We'll build the same movie organization system that we used in the Flask -`tutorial `_ +:ref:`tutorial ` and expose the objects and relationships as a GraphQL API. Using the GraphQL interface, you'll be able to fetch, create, update, and delete movie and actor objects in the database. `Strawberry `_ is a Python @@ -378,7 +378,7 @@ This exposes the webserver in port 5000. Now, in your browser, go to find that the HTTP basic auth requires us to provide the username and password. .. image:: - https://www.edgedb.com/docs/tutorials/strawberry/http_basic.png + /docs/tutorials/strawberry/http_basic.png :alt: HTTP basic auth prompt :width: 100% @@ -389,7 +389,7 @@ like this: .. image:: - https://www.edgedb.com/docs/tutorials/strawberry/graphiql.png + /docs/tutorials/strawberry/graphiql.png :alt: GraphiQL interface :width: 100% @@ -410,7 +410,7 @@ following query does that: The following response will appear on the right panel of the GraphiQL explorer: .. image:: - https://www.edgedb.com/docs/tutorials/strawberry/query_actors.png + /docs/tutorials/strawberry/query_actors.png :alt: Query actors :width: 100% @@ -494,7 +494,7 @@ and show all three attributes— ``name``, ``age``, and ``height`` of the create actor in the response payload. Here's the response: .. image:: - https://www.edgedb.com/docs/tutorials/strawberry/create_actor.png + /docs/tutorials/strawberry/create_actor.png :alt: Create an actor :width: 100% @@ -503,7 +503,7 @@ to fetch the actors. Running the ``ActorQuery`` will give you the following response: .. image:: - https://www.edgedb.com/docs/tutorials/strawberry/query_actors_2.png + /docs/tutorials/strawberry/query_actors_2.png :alt: Query actors :width: 100% @@ -704,7 +704,7 @@ actor ``Robert Downey Jr.`` with the movie: It'll return: .. image:: - https://www.edgedb.com/docs/tutorials/strawberry/create_movie.png + /docs/tutorials/strawberry/create_movie.png :alt: Create a movie :width: 100% @@ -727,7 +727,7 @@ Now you can fetch the movies with a simple query like this one: You'll then see an output similar to this: .. image:: - https://www.edgedb.com/docs/tutorials/strawberry/query_movies.png + /docs/tutorials/strawberry/query_movies.png :alt: Query movies :width: 100% diff --git a/docs/guides/tutorials/nextjs.rst b/docs/guides/tutorials/nextjs.rst index f474a3bbf91..3d007e930fe 100644 --- a/docs/guides/tutorials/nextjs.rst +++ b/docs/guides/tutorials/nextjs.rst @@ -117,7 +117,7 @@ something like this. .. image:: - https://www.edgedb.com/docs/tutorials/nextjs/basic_home.png + /docs/tutorials/nextjs/basic_home.png :alt: Basic blog homepage with static content :width: 100% @@ -441,7 +441,7 @@ We're also using a utility called ``$infer`` to extract the inferred type of this query. In VSCode you can hover over ``Posts`` to see what this type is. .. image:: - https://www.edgedb.com/docs/tutorials/nextjs/inference.png + /docs/tutorials/nextjs/inference.png :alt: Inferred type of posts query :width: 100% @@ -549,7 +549,7 @@ Now, click on one of the blog post links on the homepage. This should bring you to ``/post/``, which should display something like this: .. image:: - https://www.edgedb.com/docs/tutorials/nextjs/post.png + /docs/tutorials/nextjs/post.png :alt: Basic blog homepage with static content :width: 100% @@ -560,14 +560,17 @@ Deploying to Vercel First deploy an EdgeDB instance on your preferred cloud provider: -- `AWS `_ -- `Google Cloud `_ -- `Azure `_ -- `DigitalOcean `_ -- `Fly.io `_ -- `Docker `_ - (cloud-agnostic) +- :ref:`AWS ` +- :ref:`Azure ` +- :ref:`DigitalOcean ` +- :ref:`Fly.io ` +- :ref:`Google Cloud ` +- :ref:`Heroku ` +or use a cloud-agnostic deployment method: + +- :ref:`Docker ` +- :ref:`Bare metal ` **#2. Find your instance's DSN** @@ -640,7 +643,7 @@ When prompted: tutorial. .. image:: - https://www.edgedb.com/docs/tutorials/nextjs/env.png + /docs/tutorials/nextjs/env.png :alt: Setting environment variables in Vercel :width: 100% diff --git a/docs/guides/tutorials/phoenix_github_oauth.rst b/docs/guides/tutorials/phoenix_github_oauth.rst index 3150713195c..5c09f020771 100644 --- a/docs/guides/tutorials/phoenix_github_oauth.rst +++ b/docs/guides/tutorials/phoenix_github_oauth.rst @@ -6,10 +6,11 @@ Phoenix :edb-alt-title: Building a GitHub OAuth application -There is a community-supported `Elixir driver -`_ for EdgeDB. In this tutorial, we'll look at + +In this tutorial, we'll look at how you can create an application with authorization through GitHub using -`Phoenix `_ and EdgeDB. +`Phoenix `_ and :ref:`the official EdgeDB Elixir +driver `. This tutorial is a simplified version of the `LiveBeats `_ application from diff --git a/docs/guides/tutorials/rest_apis_with_fastapi.rst b/docs/guides/tutorials/rest_apis_with_fastapi.rst index 4c6452a5c03..fb651016e0b 100644 --- a/docs/guides/tutorials/rest_apis_with_fastapi.rst +++ b/docs/guides/tutorials/rest_apis_with_fastapi.rst @@ -1133,7 +1133,7 @@ your browser and head over to API navigator like this: .. image:: - https://www.edgedb.com/docs/tutorials/fastapi/openapi.png + /docs/tutorials/fastapi/openapi.png :alt: FastAPI docs navigator :width: 100% @@ -1143,7 +1143,7 @@ and then click on the **Try it out** button. You can do it in the UI as follows: .. image:: - https://www.edgedb.com/docs/tutorials/fastapi/put.png + /docs/tutorials/fastapi/put.png :alt: FastAPI docs PUT events API :width: 100% @@ -1151,7 +1151,7 @@ Clicking the **execute** button will make the request and return the following payload: .. image:: - https://www.edgedb.com/docs/tutorials/fastapi/put_result.png + /docs/tutorials/fastapi/put_result.png :alt: FastAPI docs PUT events API result :width: 100% @@ -1170,6 +1170,6 @@ Discord `_. It's a great community of helpful folks, all passionate about being part of the next generation of databases. If you like what you see and want to dive deeper into EdgeDB and what it can -do, check out our `Easy EdgeDB book `_. In +do, check out our `Easy EdgeDB book `_. In it, you'll get to learn more about EdgeDB as we build an imaginary role-playing game based on Bram Stoker's Dracula. diff --git a/docs/intro/clients.rst b/docs/intro/clients.rst index 4a536ac0872..c5c6c16e2a6 100644 --- a/docs/intro/clients.rst +++ b/docs/intro/clients.rst @@ -34,14 +34,14 @@ Available libraries To execute queries from your application code, use one of EdgeDB's *client libraries* for the following languages. - -- `JavaScript/TypeScript `_ -- `Go `_ -- `Python `_ -- `Rust `_ -- `.NET `_ -- `Java `_ -- `Elixir `_ +- :ref:`JavaScript/TypeScript ` +- :ref:`Go ` +- :ref:`Python ` +- :ref:`Rust ` +- :ref:`C# and F# ` +- :ref:`Java ` +- :ref:`Dart ` +- :ref:`Elixir ` Usage ===== @@ -394,6 +394,8 @@ projects ` guide to get started. Using environment variables --------------------------- +.. _ref_intro_clients_connection_cloud: + For EdgeDB Cloud ^^^^^^^^^^^^^^^^ @@ -466,7 +468,7 @@ DSNs can also contain the following query parameters. necessary when attempting to connect via TLS to a remote instance with a self-signed certificate. -These parameters can be added to any DSN using Web-standard query string +These parameters can be added to any DSN using web-standard query string notation. .. code-block:: @@ -513,8 +515,13 @@ Other mechanisms "tls_cert_data": "-----BEGIN CERTIFICATE-----\nabcdef..." } -``EDGEDB_INSTANCE`` (local only) - The name of a local instance. Only useful in development. +``EDGEDB_INSTANCE`` (local/EdgeDB Cloud only) + The name of an instance. Useful only for local or EdgeDB Cloud instances. + + .. note:: + + For more on EdgeDB Cloud instances, see the :ref:`EdgeDB Cloud instance + connection section ` above. Reference --------- diff --git a/docs/intro/edgeql.rst b/docs/intro/edgeql.rst index 84e2a3b3b67..93384b5daa6 100644 --- a/docs/intro/edgeql.rst +++ b/docs/intro/edgeql.rst @@ -542,8 +542,8 @@ known as a *backlink* and it has special syntax. })); // {name: string; acted_in: {title: string}[];}[] -See :ref:`Docs > EdgeQL > Select > Computed ` and -:ref:`Docs > EdgeQL > Select > Backlinks `. +See :ref:`Docs > EdgeQL > Select > Computed fields ` and +:ref:`Docs > EdgeQL > Select > Backlinks `. Update objects ^^^^^^^^^^^^^^ diff --git a/docs/intro/index.rst b/docs/intro/index.rst index 94d5fd0d9d5..40b760c1f5f 100644 --- a/docs/intro/index.rst +++ b/docs/intro/index.rst @@ -19,8 +19,8 @@ Get Started clients EdgeDB is a next-generation `graph-relational database -`_ designed as a spiritual -successor to the relational database. +`_ designed +as a spiritual successor to the relational database. It inherits the strengths of SQL databases: type safety, performance, reliability, and transactionality. But instead of modeling data in a diff --git a/docs/intro/migrations.rst b/docs/intro/migrations.rst index 5d3cf2618da..db844230a1c 100644 --- a/docs/intro/migrations.rst +++ b/docs/intro/migrations.rst @@ -300,5 +300,5 @@ Further reading Further information can be found in the :ref:`CLI reference ` or the `Beta 1 blog post -`_, which -describes the design of the migration system. \ No newline at end of file +`_, +which describes the design of the migration system. \ No newline at end of file diff --git a/docs/intro/quickstart.rst b/docs/intro/quickstart.rst index 3d315083831..a8319c9af0e 100644 --- a/docs/intro/quickstart.rst +++ b/docs/intro/quickstart.rst @@ -42,9 +42,10 @@ completes, you may need to **restart your terminal** before you can use the .. note:: Check out our additional installation methods `for various Linux distros\ - `_, `via Homebrew on macOS\ - `_, and `for the Windows Command Prompt\ - `_. + `_, `via Homebrew on + macOS\ `_, and `for the + Windows Command Prompt\ + `_. Now let's set up your EdgeDB project. @@ -435,12 +436,17 @@ will look something like this: EdgeDB UI is a useful development tool, but in practice your application will likely be using one of EdgeDB's *client libraries* to execute queries. EdgeDB -provides official libraries for -`JavaScript/TypeScript `__, -`Go `__, -`Python `__, -`Rust `__, and -`C# and F# `_. +provides official libraries for many langauges: + +- :ref:`JavaScript/TypeScript ` +- :ref:`Go ` +- :ref:`Python ` +- :ref:`Rust ` +- :ref:`C# and F# ` +- :ref:`Java ` +- :ref:`Dart ` +- :ref:`Elixir ` + Check out the :ref:`Clients ` guide to get started with the language of your choice. @@ -459,9 +465,10 @@ and used a client library. the other pages in the Getting Started section, which will cover important topics like migrations, the schema language, and EdgeQL in greater detail. -- For guided tours of major concepts, check out the - showcase pages for `Data Modeling `_, - `EdgeQL `_, and `Migrations `_. +- For guided tours of major concepts, check out the showcase pages for `Data + Modeling `_, `EdgeQL + `_, and `Migrations + `_. - For a deep dive into the EdgeQL query language, check out the `Interactive Tutorial `_. @@ -471,10 +478,14 @@ and used a client library. total beginner through EdgeDB, from the basics all the way through advanced concepts. -- To start building an application using the language of your choice, check - out our client libraries for - `JavaScript/TypeScript `__, - `Python `__, and - `Go `__. - -- Or just jump into the :ref:`docs `! +- To start building an application using the language of your choice, check out + our client libraries: + + - :ref:`JavaScript/TypeScript ` + - :ref:`Go ` + - :ref:`Python ` + - :ref:`Rust ` + - :ref:`C# and F# ` + - :ref:`Java ` + - :ref:`Dart ` + - :ref:`Elixir ` diff --git a/docs/reference/connection.rst b/docs/reference/connection.rst index 0e282b8aca9..048bc877076 100644 --- a/docs/reference/connection.rst +++ b/docs/reference/connection.rst @@ -148,7 +148,7 @@ Let's dig into each of these a bit more. instance's credentials, and connect automatically. For more information on how this works, check out the `release post - `_ for ``edgedb project``. + `_ for ``edgedb project``. .. _ref_reference_connection_priority: diff --git a/edb/edgeql/compiler/context.py b/edb/edgeql/compiler/context.py index 019fc88b18b..4386fdf9030 100644 --- a/edb/edgeql/compiler/context.py +++ b/edb/edgeql/compiler/context.py @@ -767,10 +767,16 @@ def maybe_create_anchor( else: return ir - # Return an additional key for any compilation caches that may - # vary based on "security contexts" such as whether we are in an - # access policy. def get_security_context(self) -> object: + '''Compute an additional compilation cache key. + + Return an additional key for any compilation caches that may + vary based on "security contexts" such as whether we are in an + access policy. + ''' + # N.B: Whether we are compiling a trigger is not included here + # since we clear cached rewrites when compiling them in the + # *pgsql* compiler. return bool(self.suppress_rewrites) diff --git a/edb/edgeql/parser/grammar/commondl.py b/edb/edgeql/parser/grammar/commondl.py index a75265873fe..825439bc59a 100644 --- a/edb/edgeql/parser/grammar/commondl.py +++ b/edb/edgeql/parser/grammar/commondl.py @@ -183,15 +183,6 @@ def reduce_OnExpr(self, expr): pass -class OptOnTypeExpr(Nonterm): - def reduce_empty(self): - self.val = None - - @parsing.inline(1) - def reduce_ON_FullTypeExpr(self, _, expr): - pass - - class OptExceptExpr(Nonterm): def reduce_empty(self): self.val = None @@ -780,38 +771,3 @@ def _process_sql_body(self, block, *, optional_using: bool=False): props['commands'] = commands return props - - -class IndexType(Nonterm, ProcessIndexMixin): - - def reduce_NodeName_LPAREN_OptIndexArgList_RPAREN( - self, name, _l, args, _r - ): - kwargs = self._process_arguments(args.val) - self.val = qlast.IndexType(name=name, kwargs=kwargs) - - def reduce_NodeName(self, name): - self.val = qlast.IndexType(name=name, kwargs={}) - - -class IndexTypeList(parsing.ListNonterm, element=IndexType, - separator=tokens.T_COMMA): - pass - - -class OptIndexTypeList(Nonterm): - - @parsing.inline(1) - def reduce_USING_IndexTypeList(self, _, list): - pass - - def reduce_empty(self): - self.val = [] - - -class UsingSQLIndex(Nonterm): - - def reduce_USING_Identifier_BaseStringConstant(self, _, ident, const): - lang = _parse_language(ident) - code = const.val.value - self.val = qlast.IndexCode(language=lang, code=code) diff --git a/edb/edgeql/parser/grammar/ddl.py b/edb/edgeql/parser/grammar/ddl.py index c613e5fa8df..583ae6b81a6 100644 --- a/edb/edgeql/parser/grammar/ddl.py +++ b/edb/edgeql/parser/grammar/ddl.py @@ -271,11 +271,6 @@ def reduce_DropIndexStmt(self, *_): pass -class InnerDDLStmtBlock(parsing.ListNonterm, element=InnerDDLStmt, - separator=Semicolons): - pass - - class PointerName(Nonterm): @parsing.inline(0) def reduce_PtrNodeName(self, *kids): @@ -812,12 +807,6 @@ def reduce_AlterBranchStmt(self, *kids): # -commands_block( - 'CreateBranch', - SetFieldStmt, -) - - class CreateBranchStmt(Nonterm): def reduce_CREATE_EMPTY_BRANCH_DatabaseName(self, *kids): self.val = qlast.CreateDatabase( diff --git a/edb/edgeql/parser/grammar/expressions.py b/edb/edgeql/parser/grammar/expressions.py index 812cb953348..9be0c03f03d 100644 --- a/edb/edgeql/parser/grammar/expressions.py +++ b/edb/edgeql/parser/grammar/expressions.py @@ -413,29 +413,6 @@ def reduce_LBRACE_ShapeElementList_RBRACE(self, *kids): pass -class OptShape(Nonterm): - @parsing.inline(0) - def reduce_Shape(self, *kids): - pass - - def reduce_empty(self, *kids): - self.val = [] - - -class TypedShape(Nonterm): - def reduce_NodeName_OptShape(self, *kids): - self.val = qlast.Shape( - expr=qlast.Path( - steps=[qlast.ObjectRef( - name=kids[0].val.name, - module=kids[0].val.module, - context=kids[0].context) - ] - ), - elements=kids[1].val - ) - - class FreeShape(Nonterm): def reduce_LBRACE_FreeComputableShapePointerList_RBRACE(self, *kids): self.val = qlast.Shape(elements=kids[1].val) @@ -2115,10 +2092,6 @@ def reduce_BaseName(self, base_name): name=base_name.val[-1]) -class NodeNameList(ListNonterm, element=NodeName, separator=tokens.T_COMMA): - pass - - class PtrNodeName(Nonterm): # NOTE: Generic short of fully-qualified name. # @@ -2192,6 +2165,10 @@ def reduce_AnyIdentifier(self, *kids): class Keyword(parsing.Nonterm): + """Base class for the different classes of keywords. + + Not a real nonterm on its own. + """ def __init_subclass__( cls, *, type, is_internal=False, **kwargs): super().__init_subclass__(is_internal=is_internal, **kwargs) diff --git a/edb/edgeql/parser/grammar/sdl.py b/edb/edgeql/parser/grammar/sdl.py index afa84a42639..ec288b76f2e 100644 --- a/edb/edgeql/parser/grammar/sdl.py +++ b/edb/edgeql/parser/grammar/sdl.py @@ -1277,15 +1277,6 @@ def reduce_CreateLink(self, *kids): ) -class OptPtrKind(Nonterm): - @parsing.inline(0) - def reduce_LINK(self, *kids): - pass - - def reduce_empty(self): - self.val = None - - sdl_commands_block( 'CreateConcreteLink', Using, diff --git a/edb/edgeql/parser/grammar/statements.py b/edb/edgeql/parser/grammar/statements.py index b2d3ca549e1..96f9461a1ab 100644 --- a/edb/edgeql/parser/grammar/statements.py +++ b/edb/edgeql/parser/grammar/statements.py @@ -280,15 +280,6 @@ def reduce_DESCRIBE_CURRENT_MIGRATION(self, *kids): ) -class OptAnalyze(Nonterm): - - def reduce_ANALYZE(self, *kids): - self.val = True - - def reduce_empty(self, *kids): - self.val = False - - class AnalyzeStmt(Nonterm): def reduce_ANALYZE_NamedTuple_ExprStmt(self, *kids): diff --git a/edb/edgeql/parser/grammar/tokens.py b/edb/edgeql/parser/grammar/tokens.py index 0c4b56d6245..4b1606eb251 100644 --- a/edb/edgeql/parser/grammar/tokens.py +++ b/edb/edgeql/parser/grammar/tokens.py @@ -208,10 +208,6 @@ class T_NAMEDONLY(Token, lextoken='named only'): pass -class T_SETANNOTATION(Token, lextoken='set annotation'): - pass - - class T_SETTYPE(Token, lextoken='set type'): pass @@ -248,10 +244,6 @@ class T_SCONST(Token): pass -class T_RSCONST(Token): - pass - - class T_DISTINCTFROM(Token, lextoken="?!="): pass @@ -276,10 +268,6 @@ class T_IDENT(Token): pass -class T_SUBSTITUTION(Token): - pass - - class T_EOF(Token): pass diff --git a/edb/ir/ast.py b/edb/ir/ast.py index ef347914c1a..c26fe7848f0 100644 --- a/edb/ir/ast.py +++ b/edb/ir/ast.py @@ -154,7 +154,7 @@ class TypeRef(ImmutableBase): union: typing.Optional[typing.FrozenSet[TypeRef]] = None # Whether the union is specified by an exhaustive list of # types, and type inheritance should not be considered. - union_is_concrete: bool = False + union_is_exhaustive: bool = False # If this is an intersection type, this would be a set of # intersection elements. intersection: typing.Optional[typing.FrozenSet[TypeRef]] = None @@ -239,7 +239,7 @@ class BasePointerRef(ImmutableBase): children: typing.FrozenSet[BasePointerRef] = frozenset() union_components: typing.Optional[typing.Set[BasePointerRef]] = None intersection_components: typing.Optional[typing.Set[BasePointerRef]] = None - union_is_concrete: bool = False + union_is_exhaustive: bool = False has_properties: bool = False is_derived: bool = False is_computable: bool = False diff --git a/edb/ir/typeutils.py b/edb/ir/typeutils.py index 0936000b848..39f69e38863 100644 --- a/edb/ir/typeutils.py +++ b/edb/ir/typeutils.py @@ -313,10 +313,10 @@ def _typeref( schema) for c in union_of.objects(schema) } - non_overlapping, union_is_concrete = ( + non_overlapping, union_is_exhaustive = ( s_utils.get_non_overlapping_union(schema, union_types) ) - if union_is_concrete: + if union_is_exhaustive: non_overlapping = frozenset({ t for t in non_overlapping if t.is_material_object_type(schema) @@ -326,7 +326,7 @@ def _typeref( _typeref(c) for c in non_overlapping ) else: - union_is_concrete = False + union_is_exhaustive = False union = None intersection_of = t.get_intersection_of(schema) @@ -399,7 +399,7 @@ def _typeref( children=children, ancestors=ancestors, union=union, - union_is_concrete=union_is_concrete, + union_is_exhaustive=union_is_exhaustive, intersection=intersection, element_name=_name, is_scalar=t.is_scalar(), @@ -657,7 +657,7 @@ def ptrref_from_ptrcls( union_components: Optional[Set[irast.BasePointerRef]] = None union_of = ptrcls.get_union_of(schema) - union_is_concrete = False + union_is_exhaustive = False if union_of: union_ptrs = set() @@ -666,9 +666,11 @@ def ptrref_from_ptrcls( schema, material_comp = component.material_type(schema) union_ptrs.add(material_comp) - non_overlapping, union_is_concrete = s_utils.get_non_overlapping_union( - schema, - union_ptrs, + non_overlapping, union_is_exhaustive = ( + s_utils.get_non_overlapping_union( + schema, + union_ptrs, + ) ) union_components = { @@ -740,25 +742,27 @@ def ptrref_from_ptrcls( else: children = frozenset() - kwargs.update(dict( - out_source=out_source, - out_target=out_target, - name=ptrcls.get_name(schema), - shortname=ptrcls.get_shortname(schema), - std_parent_name=std_parent_name, - source_ptr=source_ptr, - base_ptr=base_ptr, - material_ptr=material_ptr, - children=children, - is_derived=ptrcls.get_is_derived(schema), - is_computable=ptrcls.get_computable(schema), - union_components=union_components, - intersection_components=intersection_components, - union_is_concrete=union_is_concrete, - has_properties=ptrcls.has_user_defined_properties(schema), - in_cardinality=in_cardinality, - out_cardinality=out_cardinality, - )) + kwargs.update( + dict( + out_source=out_source, + out_target=out_target, + name=ptrcls.get_name(schema), + shortname=ptrcls.get_shortname(schema), + std_parent_name=std_parent_name, + source_ptr=source_ptr, + base_ptr=base_ptr, + material_ptr=material_ptr, + children=children, + is_derived=ptrcls.get_is_derived(schema), + is_computable=ptrcls.get_computable(schema), + union_components=union_components, + intersection_components=intersection_components, + union_is_exhaustive=union_is_exhaustive, + has_properties=ptrcls.has_user_defined_properties(schema), + in_cardinality=in_cardinality, + out_cardinality=out_cardinality, + ) + ) ptrref = ircls(**kwargs) diff --git a/edb/pgsql/compiler/relctx.py b/edb/pgsql/compiler/relctx.py index 088778c37d4..a6a1376514e 100644 --- a/edb/pgsql/compiler/relctx.py +++ b/edb/pgsql/compiler/relctx.py @@ -1677,14 +1677,14 @@ def range_for_typeref( for child in typeref.union: mat_child = child.material_type or child if mat_child.id in seen: - assert typeref.union_is_concrete + assert typeref.union_is_exhaustive continue seen.add(mat_child.id) c_rvar = range_for_typeref( child, path_id=path_id, - include_descendants=not typeref.union_is_concrete, + include_descendants=not typeref.union_is_exhaustive, for_mutation=for_mutation, dml_source=dml_source, lateral=lateral, @@ -1957,7 +1957,7 @@ def range_for_ptrref( overlays = get_ptr_rel_overlays( ptrref, dml_source=dml_source, ctx=ctx) - include_descendants = not ptrref.union_is_concrete + include_descendants = not ptrref.union_is_exhaustive assert isinstance(ptrref.out_source.name_hint, sn.QualName) # expand_inhviews helps support EXPLAIN. see diff --git a/edb/server/args.py b/edb/server/args.py index 199a6277d45..8771d9d8611 100644 --- a/edb/server/args.py +++ b/edb/server/args.py @@ -258,6 +258,7 @@ class ServerConfig(NamedTuple): tls_cert_file: pathlib.Path tls_key_file: pathlib.Path tls_cert_mode: ServerTlsCertMode + tls_client_ca_file: Optional[pathlib.Path] jws_key_file: pathlib.Path jose_key_mode: JOSEKeyMode @@ -501,8 +502,21 @@ def _validate_default_auth_method( if method in {ServerAuthMethod.Auto, ServerAuthMethod.Scram}: pass else: - for m in methods: - methods[m] = method + for t in methods: + # HTTP_METRICS and HTTP_HEALTH support only mTLS, but for + # backward compatibility, default them to `auto` if unsupported + # method is passed explicitly. + if t in ( + ServerConnTransport.HTTP_METRICS, + ServerConnTransport.HTTP_HEALTH, + ): + if method not in ( + ServerAuthMethod.Trust, + ServerAuthMethod.mTLS, + ): + continue + + methods[t] = method elif "," not in value and ":" not in value: raise click.BadParameter( f"invalid authentication method: {value}, " @@ -815,6 +829,18 @@ def resolve_envvar_value(self, ctx: click.Context): '"require_file" when the --security option is set to "strict", ' 'and "generate_self_signed" when the --security option is set to ' '"insecure_dev_mode"'), + click.option( + '--tls-client-ca-file', + type=PathPath(), + envvar='EDGEDB_SERVER_TLS_CLIENT_CA_FILE', + help='Specifies a path to a file containing a TLS CA certificate to ' + 'verify client certificates on demand. When set, the default ' + 'authentication method of HTTP_METRICS(/metrics) and HTTP_HEALTH' + '(/server/*) will also become "mTLS", unless explicitly set in ' + '--default-auth-method. Note, the protection of such HTTP ' + 'endpoints is only complete if --http-endpoint-security is also ' + 'set to `tls`, or they are still accessible in plaintext HTTP.' + ), click.option( '--generate-self-signed-cert', type=bool, default=False, is_flag=True, help='DEPRECATED.\n\n' @@ -1174,16 +1200,47 @@ def parse_args(**kwargs: Any): if kwargs['http_endpoint_security'] == 'default': kwargs['http_endpoint_security'] = 'optional' if not kwargs['default_auth_method']: - kwargs['default_auth_method'] = { + kwargs['default_auth_method'] = ServerAuthMethods({ t: ServerAuthMethod.Trust for t in ServerConnTransport.__members__.values() - } + }) if kwargs['tls_cert_mode'] == 'default': kwargs['tls_cert_mode'] = 'generate_self_signed' elif not kwargs['default_auth_method']: kwargs['default_auth_method'] = DEFAULT_AUTH_METHODS + methods = dict(kwargs['default_auth_method'].items()) + for transport in ServerConnTransport.__members__.values(): + method = methods[transport] + if method is ServerAuthMethod.Auto: + if transport in ( + ServerConnTransport.HTTP_METRICS, + ServerConnTransport.HTTP_HEALTH, + ): + if kwargs['tls_client_ca_file'] is None: + method = ServerAuthMethod.Trust + else: + method = ServerAuthMethod.mTLS + else: + method = DEFAULT_AUTH_METHODS.get(transport) + methods[transport] = method + elif transport in ( + ServerConnTransport.HTTP_METRICS, + ServerConnTransport.HTTP_HEALTH, + ): + if method is ServerAuthMethod.mTLS: + if kwargs['tls_client_ca_file'] is None: + abort('--tls-client-ca-file is required ' + 'for mTLS authentication') + elif method is not ServerAuthMethod.Trust: + abort( + f'--default-auth-method of {transport} can only be one ' + f'of: {ServerAuthMethod.Trust}, {ServerAuthMethod.mTLS} ' + f'or {ServerAuthMethod.Auto}' + ) + kwargs['default_auth_method'] = ServerAuthMethods(methods) + if kwargs['binary_endpoint_security'] == 'default': kwargs['binary_endpoint_security'] = 'tls' diff --git a/edb/server/compiler/rpc.pyx b/edb/server/compiler/rpc.pyx index 4a9a28821ad..79b5ba954f2 100644 --- a/edb/server/compiler/rpc.pyx +++ b/edb/server/compiler/rpc.pyx @@ -80,6 +80,26 @@ cdef class CompilationRequest: ): self._serializer = compilation_config_serializer + def __copy__(self): + cdef CompilationRequest rv = CompilationRequest(self._serializer) + rv.source = self.source + rv.protocol_version = self.protocol_version + rv.output_format = self.output_format + rv.json_parameters = self.json_parameters + rv.expect_one = self.expect_one + rv.implicit_limit = self.implicit_limit + rv.inline_typeids = self.inline_typeids + rv.inline_typenames = self.inline_typenames + rv.inline_objectids = self.inline_objectids + rv.modaliases = self.modaliases + rv.session_config = self.session_config + rv.database_config = self.database_config + rv.system_config = self.system_config + rv.schema_version = self.schema_version + rv.serialized_cache = self.serialized_cache + rv.cache_key = self.cache_key + return rv + def update( self, source: edgeql.Source, diff --git a/edb/server/compiler_pool/pool.py b/edb/server/compiler_pool/pool.py index a3040ee4284..e5407ceb7cb 100644 --- a/edb/server/compiler_pool/pool.py +++ b/edb/server/compiler_pool/pool.py @@ -885,6 +885,7 @@ def __init__(self, *, pool_size, **kwargs): self._expected_num_workers = 0 self._scale_down_handle = None self._max_num_workers = pool_size + self._cleanups = {} async def _start(self): async with asyncio.TaskGroup() as g: @@ -897,6 +898,8 @@ async def _stop(self): for transport in transports.values(): await transport._wait() transport.close() + for cleanup in list(self._cleanups.values()): + await cleanup async def _acquire_worker( self, *, condition=None, weighter=None, **compiler_args @@ -940,12 +943,21 @@ def _release_worker(self, worker, *, put_in_front: bool = True): self._scale_down, ) + async def _wait_on_dying(self, pid, trans): + await trans._wait() + self._cleanups.pop(pid) + def worker_disconnected(self, pid): num_workers_before = len(self._workers) super().worker_disconnected(pid) trans = self._worker_transports.pop(pid, None) if trans: trans.close() + # amsg.Server notifies us when the *pipe* to the worker closes, + # so we need to fire off a task to make sure that we wait for + # the worker to exit, in order to avoid a warning. + self._cleanups[pid] = ( + self._loop.create_task(self._wait_on_dying(pid, trans))) if not self._running: return if len(self._workers) < self._pool_size: diff --git a/edb/server/dbview/dbview.pxd b/edb/server/dbview/dbview.pxd index b83357c3a4a..badcaed1e97 100644 --- a/edb/server/dbview/dbview.pxd +++ b/edb/server/dbview/dbview.pxd @@ -41,6 +41,7 @@ cdef class CompiledQuery: cdef public object extra_counts cdef public object extra_blobs cdef public object request + cdef public object recompiled_cache cdef class DatabaseIndex: diff --git a/edb/server/dbview/dbview.pyx b/edb/server/dbview/dbview.pyx index 4cfc366b040..0551238d8cd 100644 --- a/edb/server/dbview/dbview.pyx +++ b/edb/server/dbview/dbview.pyx @@ -22,6 +22,7 @@ from typing import ( import asyncio import base64 +import copy import json import os.path import pickle @@ -85,12 +86,14 @@ cdef class CompiledQuery: extra_counts=(), extra_blobs=(), request=None, + recompiled_cache=None, ): self.query_unit_group = query_unit_group self.first_extra = first_extra self.extra_counts = extra_counts self.extra_blobs = extra_blobs self.request = request + self.recompiled_cache = recompiled_cache cdef class Database: @@ -197,7 +200,7 @@ cdef class Database: query_req, unit_group = self._eql_to_compiled.cleanup_one() if len(unit_group) == 1: keys.append(query_req.get_cache_key()) - if keys: + if keys and debug.flags.persistent_cache: self.tenant.create_task( self.tenant.evict_query_cache(self.name, keys), interruptable=True, @@ -909,39 +912,24 @@ cdef class DatabaseConnectionView: self._reset_tx_state() return side_effects - async def clear_cache_keys(self, conn) -> list[rpc.CompilationRequest]: - rows = await conn.sql_fetch(b'SELECT "edgedb"."_clear_query_cache"()') - rv = [] - for row in rows: - query_req = rpc.CompilationRequest( - self.server.compilation_config_serializer - ).deserialize(row[0], "") - rv.append(query_req) - self._db._eql_to_compiled.pop(query_req, None) - execute.signal_query_cache_changes(self) - return rv - - async def recompile_all( - self, conn, requests: typing.Iterable[rpc.CompilationRequest] - ): - # Assume the size of compiler pool is 100, we'll issue 50 concurrent - # compilation requests at the same time, cache up to 150 results and - # persist in one backend round-trip, in parallel. + async def recompile_cached_queries(self, user_schema, schema_version): compiler_pool = self.server.get_compiler_pool() compile_concurrency = max(1, compiler_pool.get_size_hint() // 2) concurrency_control = asyncio.Semaphore(compile_concurrency) - persist_batch_size = compile_concurrency * 3 - compiled_queue = asyncio.Queue(persist_batch_size) + rv = [] async def recompile_request(query_req: rpc.CompilationRequest): async with concurrency_control: try: - schema_version = self.schema_version database_config = self.get_database_config() system_config = self.get_compilation_system_config() - result = await compiler_pool.compile( + query_req = copy.copy(query_req) + query_req.set_schema_version(schema_version) + query_req.set_database_config(database_config) + query_req.set_system_config(system_config) + unit_group, _, _ = await compiler_pool.compile( self.dbname, - self.get_user_schema_pickle(), + user_schema, self.get_global_schema_pickle(), self.reflection_cache, database_config, @@ -951,49 +939,16 @@ cdef class DatabaseConnectionView: client_id=self.tenant.client_id, ) except Exception: - # discard cache entry that cannot be recompiled - self._db._eql_to_compiled.pop(query_req, None) + # ignore cache entry that cannot be recompiled + pass else: - # schema_version, database_config and system_config are not - # serialized but only affect the cache key. We only update - # these values *after* the compilation so that we can evict - # the in-memory cache by the right key when recompilation - # fails in the `except` branch above. - query_req.set_schema_version(schema_version) - query_req.set_database_config(database_config) - query_req.set_system_config(system_config) - - await compiled_queue.put((query_req, result[0])) - - async def persist_cache_task(): - if not debug.flags.func_cache: - # TODO(fantix): sync _query_cache in one implicit tx - await conn.sql_fetch(b'SELECT "edgedb"."_clear_query_cache"()') - - buf = [] - running = True - while running: - while len(buf) < persist_batch_size: - item = await compiled_queue.get() - if item is None: - running = False - break - buf.append(item) - if buf: - await execute.persist_cache( - conn, self, [item[:2] for item in buf] - ) - for query_req, query_unit_group in buf: - self._db._cache_compiled_query( - query_req, query_unit_group) - buf.clear() + rv.append((query_req, unit_group)) async with asyncio.TaskGroup() as g: - g.create_task(persist_cache_task()) - async with asyncio.TaskGroup() as compile_group: - for req in requests: - compile_group.create_task(recompile_request(req)) - await compiled_queue.put(None) + for req, grp in self._db._eql_to_compiled.items(): + if len(grp) == 1: + g.create_task(recompile_request(req)) + return rv async def apply_config_ops(self, conn, ops): settings = self.get_config_spec() @@ -1124,6 +1079,41 @@ cdef class DatabaseConnectionView: if not lock._waiters: del lock_table[query_req] + recompiled_cache = None + if ( + not self.in_tx() + or len(query_unit_group) > 0 + and query_unit_group[0].tx_commit + ): + # Recompile all cached queries if: + # * Issued a DDL or committing a tx with DDL (recompilation + # before in-tx DDL needs to fix _in_tx_with_ddl caching 1st) + # * Config.auto_rebuild_query_cache is turned on + # + # Ideally we should compute the proper user_schema, database_config + # and system_config for recompilation from server/compiler.py with + # proper handling of config values. For now we just use the values + # in the current dbview and not support certain marginal cases. + user_schema = None + user_schema_version = None + for unit in query_unit_group: + if unit.tx_rollback: + break + if unit.user_schema: + user_schema = unit.user_schema + user_schema_version = unit.user_schema_version + if user_schema and not self.server.config_lookup( + "auto_rebuild_query_cache", + self.get_session_config(), + self.get_database_config(), + self.get_system_config(), + ): + user_schema = None + if user_schema: + recompiled_cache = await self.recompile_cached_queries( + user_schema, user_schema_version + ) + if use_metrics: metrics.edgeql_query_compilations.inc( 1.0, self.tenant.get_instance_name(), 'compiler' @@ -1136,6 +1126,7 @@ cdef class DatabaseConnectionView: extra_counts=source.extra_counts(), extra_blobs=source.extra_blobs(), request=query_req, + recompiled_cache=recompiled_cache, ) cdef inline _check_in_tx_error(self, query_unit_group): diff --git a/edb/server/main.py b/edb/server/main.py index b8270b2380b..369b02bfcab 100644 --- a/edb/server/main.py +++ b/edb/server/main.py @@ -272,7 +272,11 @@ async def _run_server( return ss.init_tls( - args.tls_cert_file, args.tls_key_file, tls_cert_newly_generated) + args.tls_cert_file, + args.tls_key_file, + tls_cert_newly_generated, + args.tls_client_ca_file, + ) ss.init_jwcrypto(args.jws_key_file, jws_keys_newly_generated) @@ -290,7 +294,11 @@ def load_configuration(_signum): try: if args.readiness_state_file: tenant.reload_readiness_state() - ss.reload_tls(args.tls_cert_file, args.tls_key_file) + ss.reload_tls( + args.tls_cert_file, + args.tls_key_file, + args.tls_client_ca_file, + ) ss.load_jwcrypto(args.jws_key_file) except Exception: logger.critical( diff --git a/edb/server/multitenant.py b/edb/server/multitenant.py index 6341c616ca6..bb9a5a38d10 100644 --- a/edb/server/multitenant.py +++ b/edb/server/multitenant.py @@ -417,7 +417,10 @@ async def run_server( tls_cert_newly_generated, jws_keys_newly_generated ) = await ss.maybe_generate_pki(args, ss) ss.init_tls( - args.tls_cert_file, args.tls_key_file, tls_cert_newly_generated + args.tls_cert_file, + args.tls_key_file, + tls_cert_newly_generated, + args.tls_client_ca_file, ) ss.init_jwcrypto(args.jws_key_file, jws_keys_newly_generated) @@ -433,7 +436,11 @@ def load_configuration(_signum): logger.info("reloading configuration") try: - ss.reload_tls(args.tls_cert_file, args.tls_key_file) + ss.reload_tls( + args.tls_cert_file, + args.tls_key_file, + args.tls_client_ca_file, + ) ss.load_jwcrypto(args.jws_key_file) ss.reload_tenants() except Exception: diff --git a/edb/server/protocol/auth_ext/_static/webauthn-authenticate.js b/edb/server/protocol/auth_ext/_static/webauthn-authenticate.js index 787afcd385f..c4bdb61ff93 100644 --- a/edb/server/protocol/auth_ext/_static/webauthn-authenticate.js +++ b/edb/server/protocol/auth_ext/_static/webauthn-authenticate.js @@ -30,20 +30,32 @@ async function onAuthenticateSubmit(form) { const redirectOnFailure = formData.get("redirect_on_failure"); const redirectTo = formData.get("redirect_to"); - if (redirectTo === null) { - throw new Error("Missing redirect_to parameter"); + const missingFields = Object.entries({ + email, + challenge, + redirectTo, + }).filter(([k, v]) => !v); + if (missingFields.length > 0) { + throw new Error( + "Missing required parameters: " + missingFields.map(([k]) => k).join(", ") + ); } try { - const maybeCode = await authenticate({ + const response = await authenticate({ email, provider, challenge, }); const redirectUrl = new URL(redirectTo); - if (maybeCode !== null) { - redirectUrl.searchParams.append("code", maybeCode); + if ("code" in response) { + redirectUrl.searchParams.append("code", response.code); + } else if ("verification_email_sent_at" in response) { + redirectUrl.searchParams.append( + "verification_email_sent_at", + response.verification_email_sent_at + ); } window.location.href = redirectUrl.href; @@ -72,8 +84,7 @@ const WEBAUTHN_AUTHENTICATE_URL = new URL( * @param {string} props.email - Email address to register * @param {string} props.provider - WebAuthn provider * @param {string} props.challenge - PKCE challenge - * @returns {Promise} - The PKCE code or null if the application - * requires email verification + * @returns {Promise} - The server response */ export async function authenticate({ email, provider, challenge }) { // Check if WebAuthn is supported @@ -98,13 +109,11 @@ export async function authenticate({ email, provider, challenge }) { }); // Register the credentials on the server - const registerResult = await authenticateAssertion({ + return await authenticateAssertion({ email, assertion, challenge, }); - - return registerResult.code ?? null; } /** diff --git a/edb/server/protocol/auth_ext/_static/webauthn-register.js b/edb/server/protocol/auth_ext/_static/webauthn-register.js index 72c4fbcda6e..f3d8b67a5ee 100644 --- a/edb/server/protocol/auth_ext/_static/webauthn-register.js +++ b/edb/server/protocol/auth_ext/_static/webauthn-register.js @@ -45,7 +45,7 @@ export async function onRegisterSubmit(form) { ); } - const maybeCode = await register({ + const response = await register({ email, provider, challenge, @@ -54,8 +54,13 @@ export async function onRegisterSubmit(form) { const redirectUrl = new URL(redirectTo); redirectUrl.searchParams.append("isSignUp", "true"); - if (maybeCode !== null) { - redirectUrl.searchParams.append("code", maybeCode); + if ("code" in response) { + redirectUrl.searchParams.append("code", response.code); + } else if ("verification_email_sent_at" in response) { + redirectUrl.searchParams.append( + "verification_email_sent_at", + response.verification_email_sent_at + ); } window.location.href = redirectUrl.href; @@ -83,8 +88,7 @@ const WEBAUTHN_REGISTER_URL = new URL("../webauthn/register", window.location); * @param {string} props.provider - WebAuthn provider * @param {string} props.challenge - PKCE challenge * @param {string} props.verifyUrl - URL to verify email after registration - * @returns {Promise} - The PKCE code or null if the application - * requires email verification + * @returns {Promise} - The server response */ export async function register({ email, provider, challenge, verifyUrl }) { // Check if WebAuthn is supported @@ -109,15 +113,13 @@ export async function register({ email, provider, challenge, verifyUrl }) { }); // Register the credentials on the server - const registerResult = await registerCredentials({ + return await registerCredentials({ email, credentials, provider, challenge, verifyUrl, }); - - return registerResult.code ?? null; } /** diff --git a/edb/server/protocol/auth_ext/http.py b/edb/server/protocol/auth_ext/http.py index 3604c8f7cb1..e051bcd0cf6 100644 --- a/edb/server/protocol/auth_ext/http.py +++ b/edb/server/protocol/auth_ext/http.py @@ -1479,20 +1479,17 @@ async def handle_ui_verify(self, request: Any, response: Any): case _: maybe_pkce_code = None - match maybe_redirect_to: - case str(rt): - redirect_to = ( - _with_appended_qs( - rt, - { - "code": [maybe_pkce_code], - }, - ) - if maybe_pkce_code - else rt - ) - case _: - redirect_to = cast(str, ui_config.redirect_to) + redirect_to = maybe_redirect_to or redirect_to + redirect_to = ( + _with_appended_qs( + redirect_to, + { + "code": [maybe_pkce_code], + }, + ) + if maybe_pkce_code + else redirect_to + ) except errors.VerificationTokenExpired: app_details_config = self._get_app_details_config() diff --git a/edb/server/protocol/auth_helpers.pxd b/edb/server/protocol/auth_helpers.pxd index f5ad83c9203..e43d6c4f504 100644 --- a/edb/server/protocol/auth_helpers.pxd +++ b/edb/server/protocol/auth_helpers.pxd @@ -25,3 +25,5 @@ cdef scram_get_verifier(tenant, str user) cdef parse_basic_auth(str auth_payload) cdef extract_http_user(scheme, auth_payload, params) cdef auth_basic(tenant, str username, str password) +cdef auth_mtls(transport) +cdef auth_mtls_with_user(transport, str username) diff --git a/edb/server/protocol/auth_helpers.pyx b/edb/server/protocol/auth_helpers.pyx index c7ecbb32b01..ec8f1aeb52f 100644 --- a/edb/server/protocol/auth_helpers.pyx +++ b/edb/server/protocol/auth_helpers.pyx @@ -28,9 +28,6 @@ import logging from jwcrypto import jwt from edb import errors -from edb.common import debug - -from edb.server.protocol cimport args_ser cdef object logger = logging.getLogger('edb.server') @@ -235,3 +232,36 @@ cdef auth_basic(tenant, username: str, password: str): verifier, mock_auth = scram_get_verifier(tenant, username) if not scram_verify_password(password, verifier) or mock_auth: raise errors.AuthenticationError('authentication failed') + + +cdef auth_mtls(transport): + sslobj = transport.get_extra_info('ssl_object') + if sslobj is None: + raise errors.AuthenticationError( + "mTLS authentication is not supported over plaintext transport") + cert_data = sslobj.getpeercert() + if not cert_data: # None or empty dict + # If --tls-client-ca-file is specified, the SSLContext used here would + # have done load_verify_locations() in `server/server.py`, and we will + # have a valid client certificate (non-empty dict) now if one was + # provided by the client and passed validation; empty dict otherwise. + # `None` just means the peer didn't send a client certificate. + raise errors.AuthenticationError( + "valid client certificate required") + return cert_data + + +cdef auth_mtls_with_user(transport, str username): + cert_data = auth_mtls(transport) + try: + for rdn in cert_data["subject"]: + if rdn[0][0] == 'commonName': + if rdn[0][1] == username: + return + except Exception as ex: + raise errors.AuthenticationError( + "bad client certificate") from ex + + raise errors.AuthenticationError( + f"Common Name of client certificate doesn't match {username!r}", + ) diff --git a/edb/server/protocol/binary.pxd b/edb/server/protocol/binary.pxd index fb3322d3751..d6d480834d6 100644 --- a/edb/server/protocol/binary.pxd +++ b/edb/server/protocol/binary.pxd @@ -109,3 +109,4 @@ cdef class VirtualTransport: cdef: WriteBuffer buf bint closed + object transport diff --git a/edb/server/protocol/binary.pyx b/edb/server/protocol/binary.pyx index 293dc75d4ce..e578f37974c 100644 --- a/edb/server/protocol/binary.pyx +++ b/edb/server/protocol/binary.pyx @@ -838,7 +838,8 @@ cdef class EdgeConnection(frontend.FrontendConnection): if len(units) == 1 and units[0].cache_sql: conn = await self.get_pgcon() try: - await execute.persist_cache(conn, _dbview, [(query_req, units)]) + g = execute.build_cache_persistence_units([(query_req, units)]) + await g.execute(conn, _dbview) finally: self.maybe_release_pgcon(conn) @@ -1722,9 +1723,10 @@ cdef class EdgeConnection(frontend.FrontendConnection): @cython.final cdef class VirtualTransport: - def __init__(self): + def __init__(self, transport): self.buf = WriteBuffer.new() self.closed = False + self.transport = transport def write(self, data): self.buf.write_bytes(bytes(data)) @@ -1741,6 +1743,9 @@ cdef class VirtualTransport: def abort(self): self.closed = True + def get_extra_info(self, name, default=None): + return self.transport.get_extra_info(name, default) + async def eval_buffer( server, @@ -1751,12 +1756,13 @@ async def eval_buffer( protocol_version: edbdef.ProtocolVersion, auth_data: bytes, transport: srvargs.ServerConnTransport, + tcp_transport: asyncio.Transport, ): cdef: VirtualTransport vtr EdgeConnection proto - vtr = VirtualTransport() + vtr = VirtualTransport(tcp_transport) proto = new_edge_connection( server, diff --git a/edb/server/protocol/execute.pyx b/edb/server/protocol/execute.pyx index f35584da29c..02f491bdde1 100644 --- a/edb/server/protocol/execute.pyx +++ b/edb/server/protocol/execute.pyx @@ -92,7 +92,7 @@ cdef class ExecutionGroup: if state is not None: await be_conn.wait_for_state_resp(state, state_sync=0) for i, unit in enumerate(self.group): - if unit.output_format == FMT_NONE: + if unit.output_format == FMT_NONE and unit.ddl_stmt_id is None: for sql in unit.sql: await be_conn.wait_for_command( unit, parse_array[i], dbver, ignore_data=True @@ -108,7 +108,7 @@ cdef class ExecutionGroup: return rv -cdef ExecutionGroup build_cache_persistence_units( +cpdef ExecutionGroup build_cache_persistence_units( pairs: list[tuple[rpc.CompilationRequest, compiler.QueryUnitGroup]], ExecutionGroup group = None, ): @@ -118,8 +118,7 @@ cdef ExecutionGroup build_cache_persistence_units( INSERT INTO "edgedb"."_query_cache" ("key", "schema_version", "input", "output", "evict") VALUES ($1, $2, $3, $4, $5) - ON CONFLICT (key) DO UPDATE SET - "schema_version"=$2, "input"=$3, "output"=$4, "evict"=$5 + ON CONFLICT (key) DO NOTHING ''' sql_hash = hashlib.sha1(insert_sql).hexdigest().encode('latin1') for request, units in pairs: @@ -152,35 +151,6 @@ cdef ExecutionGroup build_cache_persistence_units( return group -async def persist_cache( - be_conn: pgcon.PGConnection, - dbv: dbview.DatabaseConnectionView, - pairs: list[tuple[rpc.CompilationRequest, compiler.QueryUnitGroup]], -): - cdef group = build_cache_persistence_units(pairs) - - try: - await group.execute(be_conn, dbv) - except Exception as ex: - if ( - isinstance(ex, pgerror.BackendError) - and ex.code_is(pgerror.ERROR_SERIALIZATION_FAILURE) - # If we are in a transaction, we have to let the error - # propagate, since we can't do anything else after the error. - # Hopefully the client will retry, hit the cache, and - # everything will be fine. - and not dbv.in_tx() - ): - # XXX: Is it OK to just ignore it? Can we rely on the conflict - # having set it to the same thing? - pass - else: - dbv.on_error() - raise - else: - signal_query_cache_changes(dbv) - - # TODO: can we merge execute and execute_script? async def execute( be_conn: pgcon.PGConnection, @@ -195,6 +165,7 @@ async def execute( bytes state = None, orig_state = None WriteBuffer bound_args_buf ExecutionGroup group + bint persist_cache, persist_recompiled_query_cache query_unit = compiled.query_unit_group[0] @@ -205,6 +176,16 @@ async def execute( server = dbv.server tenant = dbv.tenant + # If we have both the compilation request and a pair of SQLs for the cache + # (persist, evict), we should follow the persistent cache route. + persist_cache = bool(compiled.request and query_unit.cache_sql) + + # Recompilation is a standalone feature than persistent cache. + # This flag indicates both features are in use, and we actually have + # recompiled the query cache to persist. + persist_recompiled_query_cache = bool( + debug.flags.persistent_cache and compiled.recompiled_cache) + data = None try: @@ -231,7 +212,24 @@ async def execute( if query_unit.sql: if query_unit.user_schema: - ddl_ret = await be_conn.run_ddl(query_unit, state) + if persist_recompiled_query_cache: + # If we have recompiled the query cache, writeback to + # the cache table here in an implicit transaction (if + # not in one already), so that whenever the transaction + # commits, we flip to using the new cache at once. + group = build_cache_persistence_units( + compiled.recompiled_cache) + group.append(query_unit) + if query_unit.ddl_stmt_id is None: + await group.execute(be_conn, dbv) + ddl_ret = None + else: + ddl_ret = be_conn.load_ddl_return( + query_unit, + await group.execute(be_conn, dbv, state=state), + ) + else: + ddl_ret = await be_conn.run_ddl(query_unit, state) if ddl_ret and ddl_ret['new_types']: new_types = ddl_ret['new_types'] else: @@ -245,7 +243,10 @@ async def execute( read_data = ( query_unit.needs_readback or query_unit.is_explain) - if compiled.request and query_unit.cache_sql: + if persist_cache: + # Persistent cache needs to happen before the actual + # query because the query may depend on the function + # created persisting the cache entry. group = build_cache_persistence_units( [(compiled.request, compiled.query_unit_group)] ) @@ -358,28 +359,11 @@ async def execute( # 1. An orphan ROLLBACK command without a paring start tx # 2. There was no SQL, so the state can't have been synced. be_conn.last_state = state - if ( - debug.flags.persistent_cache - and not dbv.in_tx() - and not query_unit.tx_rollback - and query_unit.user_schema - and server.config_lookup( - "auto_rebuild_query_cache", - dbv.get_session_config(), - dbv.get_database_config(), - dbv.get_system_config(), - ) - ): - # TODO(fantix): recompile first and update cache in tx - if debug.flags.func_cache: - recompile_requests = await dbv.clear_cache_keys(be_conn) - else: - recompile_requests = [ - req - for req, grp in dbv._db._eql_to_compiled.items() - if len(grp) == 1 - ] - await dbv.recompile_all(be_conn, recompile_requests) + if compiled.recompiled_cache: + for req, qu_group in compiled.recompiled_cache: + dbv.cache_compiled_query(req, qu_group) + if persist_cache or persist_recompiled_query_cache: + signal_query_cache_changes(dbv) finally: if query_unit.drop_db: tenant.allow_database_connections(query_unit.drop_db) @@ -564,28 +548,6 @@ async def execute_script( conn.last_state = state if unit_group.state_serializer is not None: dbv.set_state_serializer(unit_group.state_serializer) - if ( - debug.flags.persistent_cache - and not in_tx - and any(query_unit.user_schema for query_unit in unit_group) - and dbv.server.config_lookup( - "auto_rebuild_query_cache", - dbv.get_session_config(), - dbv.get_database_config(), - dbv.get_system_config(), - ) - ): - # TODO(fantix): recompile first and update cache in tx - if debug.flags.func_cache: - recompile_requests = await dbv.clear_cache_keys(conn) - else: - recompile_requests = [ - req - for req, grp in dbv._db._eql_to_compiled.items() - if len(grp) == 1 - ] - - await dbv.recompile_all(conn, recompile_requests) finally: if sent and not sync: diff --git a/edb/server/protocol/frontend.pyx b/edb/server/protocol/frontend.pyx index c3b99900e26..9713084722f 100644 --- a/edb/server/protocol/frontend.pyx +++ b/edb/server/protocol/frontend.pyx @@ -591,6 +591,8 @@ cdef class FrontendConnection(AbstractFrontendConnection): 'Simple password authentication required but it is only ' 'supported for HTTP endpoints' ) + elif authmethod_name == 'mTLS': + auth_helpers.auth_mtls_with_user(self._transport, user) else: raise errors.InternalServerError( f'unimplemented auth method: {authmethod_name}') diff --git a/edb/server/protocol/protocol.pxd b/edb/server/protocol/protocol.pxd index 8fe8e444e26..56a7ee62776 100644 --- a/edb/server/protocol/protocol.pxd +++ b/edb/server/protocol/protocol.pxd @@ -74,6 +74,8 @@ cdef class HttpProtocol: str message = ?) cdef _bad_request(self, HttpRequest request, HttpResponse response, str message) + cdef _unauthorized(self, HttpRequest request, HttpResponse response, + str message) cdef _return_binary_error(self, binary.EdgeConnection proto) cdef _write(self, bytes req_version, bytes resp_status, bytes content_type, dict custom_headers, bytes body, diff --git a/edb/server/protocol/protocol.pyx b/edb/server/protocol/protocol.pyx index d0f70de8721..0584088b073 100644 --- a/edb/server/protocol/protocol.pyx +++ b/edb/server/protocol/protocol.pyx @@ -572,6 +572,7 @@ cdef class HttpProtocol: protocol_version=proto_ver, auth_data=self.current_request.authorization, transport=srvargs.ServerConnTransport.HTTP, + tcp_transport=self.transport, ) response.status = http.HTTPStatus.OK response.content_type = PROTO_MIME @@ -660,6 +661,13 @@ cdef class HttpProtocol: self.tenant, ) elif route == 'server': + if not await self._authenticate_for_default_conn_transport( + request, + response, + srvargs.ServerConnTransport.HTTP_HEALTH, + ): + return + # System API request await system_api.handle_request( request, @@ -669,6 +677,13 @@ cdef class HttpProtocol: self.tenant, ) elif path_parts == ['metrics'] and request.method == b'GET': + if not await self._authenticate_for_default_conn_transport( + request, + response, + srvargs.ServerConnTransport.HTTP_METRICS, + ): + return + # Quoting the Open Metrics spec: # Implementers MUST expose metrics in the OpenMetrics # text format in response to a simple HTTP GET request @@ -779,6 +794,16 @@ cdef class HttpProtocol: return False + cdef _unauthorized( + self, + HttpRequest request, + HttpResponse response, + str message, + ): + response.body = message.encode("utf-8") + response.status = http.HTTPStatus.UNAUTHORIZED + response.close_connection = True + async def _check_http_auth( self, HttpRequest request, @@ -822,6 +847,13 @@ cdef class HttpProtocol: 'authentication failed: ' 'SCRAM authentication required but not supported for HTTP' ) + elif authmethod_name == 'mTLS': + if ( + self.http_endpoint_security + is srvargs.ServerEndpointSecurityMode.Tls + or self.is_tls + ): + auth_helpers.auth_mtls_with_user(self.transport, username) else: raise errors.AuthenticationError( 'authentication failed: wrong method used') @@ -830,9 +862,7 @@ cdef class HttpProtocol: if debug.flags.server: markup.dump(ex) - response.body = str(ex).encode() - response.status = http.HTTPStatus.UNAUTHORIZED - response.close_connection = True + self._unauthorized(request, response, str(ex)) # If no scheme was specified, add a WWW-Authenticate header if scheme == '': @@ -844,6 +874,39 @@ cdef class HttpProtocol: return True + async def _authenticate_for_default_conn_transport( + self, + HttpRequest request, + HttpResponse response, + transport: srvargs.ServerConnTransport, + ): + try: + auth_method = self.server.get_default_auth_method(transport) + + # If the auth method and the provided auth information match, + # try to resolve the authentication. + if auth_method is srvargs.ServerAuthMethod.Trust: + pass + elif auth_method is srvargs.ServerAuthMethod.mTLS: + if ( + self.http_endpoint_security + is srvargs.ServerEndpointSecurityMode.Tls + or self.is_tls + ): + auth_helpers.auth_mtls(self.transport) + else: + raise errors.AuthenticationError( + 'authentication failed: wrong method used') + + except Exception as ex: + if debug.flags.server: + markup.dump(ex) + + self._unauthorized(request, response, str(ex)) + + return False + + return True def get_request_url(request, is_tls): request_url = request.url diff --git a/edb/server/server.py b/edb/server/server.py index 452c8df5b82..94c340edfbb 100644 --- a/edb/server/server.py +++ b/edb/server/server.py @@ -783,7 +783,7 @@ def _sni_callback(self, sslobj, server_name, sslctx): # Used in multi-tenant server only. This method must not fail. pass - def reload_tls(self, tls_cert_file, tls_key_file): + def reload_tls(self, tls_cert_file, tls_key_file, client_ca_file): logger.info("loading TLS certificates") tls_password_needed = False if self._tls_certs_reload_retry_handle is not None: @@ -843,6 +843,16 @@ def _tls_private_key_password(): raise StartupError(f"Cannot load TLS certificates - {e}") from e + if client_ca_file is not None: + try: + sslctx.load_verify_locations(client_ca_file) + sslctx_pgext.load_verify_locations(client_ca_file) + except ssl.SSLError as e: + raise StartupError( + f"Cannot load client CA certificates - {e}") from e + sslctx.verify_mode = ssl.CERT_OPTIONAL + sslctx_pgext.verify_mode = ssl.CERT_OPTIONAL + sslctx.set_alpn_protocols(['edgedb-binary', 'http/1.1']) sslctx.sni_callback = self._sni_callback sslctx_pgext.sni_callback = self._sni_callback @@ -854,16 +864,17 @@ def init_tls( tls_cert_file, tls_key_file, tls_cert_newly_generated, + client_ca_file, ): assert self._sslctx is self._sslctx_pgext is None - self.reload_tls(tls_cert_file, tls_key_file) + self.reload_tls(tls_cert_file, tls_key_file, client_ca_file) self._tls_cert_file = str(tls_cert_file) self._tls_cert_newly_generated = tls_cert_newly_generated def reload_tls(_file_modified, _event, retry=0): try: - self.reload_tls(tls_cert_file, tls_key_file) + self.reload_tls(tls_cert_file, tls_key_file, client_ca_file) except (StartupError, FileNotFoundError) as e: if retry > defines._TLS_CERT_RELOAD_MAX_RETRIES: logger.critical(str(e)) @@ -891,6 +902,8 @@ def reload_tls(_file_modified, _event, retry=0): self.monitor_fs(tls_cert_file, reload_tls) if tls_cert_file != tls_key_file: self.monitor_fs(tls_key_file, reload_tls) + if client_ca_file is not None: + self.monitor_fs(client_ca_file, reload_tls) def load_jwcrypto(self, jws_key_file: pathlib.Path) -> None: try: diff --git a/edb/testbase/server.py b/edb/testbase/server.py index 8c32ada5589..d9ef5ddaa73 100644 --- a/edb/testbase/server.py +++ b/edb/testbase/server.py @@ -53,6 +53,7 @@ import shlex import socket import ssl +import struct import subprocess import sys import tempfile @@ -75,6 +76,7 @@ from edb.common import retryloop from edb.common import secretkey +from edb import protocol from edb.protocol import protocol as test_protocol from edb.testbase import serutils @@ -382,13 +384,22 @@ def get_api_prefix(cls): @classmethod @contextlib.contextmanager - def http_con(cls, server, keep_alive=True, server_hostname=None): + def http_con( + cls, + server, + keep_alive=True, + server_hostname=None, + client_cert_file=None, + client_key_file=None, + ): conn_args = server.get_connect_args() tls_context = ssl.create_default_context( ssl.Purpose.SERVER_AUTH, cafile=conn_args["tls_ca_file"], ) tls_context.check_hostname = False + if any((client_cert_file, client_key_file)): + tls_context.load_cert_chain(client_cert_file, client_key_file) if keep_alive: ConCls = StubbornHttpConnection else: @@ -495,6 +506,53 @@ def http_con_json_request( return result, headers, status + @classmethod + def http_con_binary_request( + cls, + http_con: http.client.HTTPConnection, + query: str, + proto_ver=edgedb_defines.CURRENT_PROTOCOL, + bearer_token: Optional[str] = None, + user: str = "edgedb", + database: str = "main", + ): + proto_ver_str = f"v_{proto_ver[0]}_{proto_ver[1]}" + mime_type = f"application/x.edgedb.{proto_ver_str}.binary" + headers = {"Content-Type": mime_type, "X-EdgeDB-User": user} + if bearer_token: + headers["Authorization"] = f"Bearer {bearer_token}" + content, headers, status = cls.http_con_request( + http_con, + method="POST", + path=f"db/{database}", + prefix="", + body=protocol.Execute( + annotations=[], + allowed_capabilities=protocol.Capability.ALL, + compilation_flags=protocol.CompilationFlag(0), + implicit_limit=0, + command_text=query, + output_format=protocol.OutputFormat.JSON, + expected_cardinality=protocol.Cardinality.AT_MOST_ONE, + input_typedesc_id=b"\0" * 16, + output_typedesc_id=b"\0" * 16, + state_typedesc_id=b"\0" * 16, + arguments=b"", + state_data=b"", + ).dump() + protocol.Sync().dump(), + headers=headers, + ) + content = memoryview(content) + uint32_unpack = struct.Struct("!L").unpack + msgs = [] + while content: + mtype = content[0] + (msize,) = uint32_unpack(content[1:5]) + msg = protocol.ServerMessage.parse(mtype, content[5: msize + 1]) + msgs.append(msg) + content = content[msize + 1 :] + return msgs, headers, status + _default_cluster = None @@ -792,13 +850,22 @@ async def assertRaisesRegexTx(self, exception, regex, msg=None, **kwargs): @classmethod @contextlib.contextmanager - def http_con(cls, server=None, keep_alive=True, server_hostname=None): + def http_con( + cls, + server=None, + keep_alive=True, + server_hostname=None, + client_cert_file=None, + client_key_file=None, + ): if server is None: server = cls with super().http_con( server, keep_alive=keep_alive, server_hostname=server_hostname, + client_cert_file=client_cert_file, + client_key_file=client_key_file, ) as http_con: yield http_con @@ -2026,8 +2093,10 @@ def __init__( runstate_dir: Optional[str] = None, reset_auth: Optional[bool] = None, tenant_id: Optional[str] = None, - security: Optional[edgedb_args.ServerSecurityMode] = None, - default_auth_method: Optional[edgedb_args.ServerAuthMethod] = None, + security: edgedb_args.ServerSecurityMode, + default_auth_method: Optional[ + edgedb_args.ServerAuthMethod | edgedb_args.ServerAuthMethods + ] = None, binary_endpoint_security: Optional[ edgedb_args.ServerEndpointSecurityMode] = None, http_endpoint_security: Optional[ @@ -2039,6 +2108,7 @@ def __init__( tls_key_file: Optional[os.PathLike] = None, tls_cert_mode: edgedb_args.ServerTlsCertMode = ( edgedb_args.ServerTlsCertMode.SelfSigned), + tls_client_ca_file: Optional[os.PathLike] = None, jws_key_file: Optional[os.PathLike] = None, jwt_sub_allowlist_file: Optional[os.PathLike] = None, jwt_revocation_list_file: Optional[os.PathLike] = None, @@ -2072,6 +2142,7 @@ def __init__( self.tls_cert_file = tls_cert_file self.tls_key_file = tls_key_file self.tls_cert_mode = tls_cert_mode + self.tls_client_ca_file = tls_client_ca_file self.jws_key_file = jws_key_file self.jwt_sub_allowlist_file = jwt_sub_allowlist_file self.jwt_revocation_list_file = jwt_revocation_list_file @@ -2230,11 +2301,14 @@ async def __aenter__(self): if self.tls_key_file: cmd += ['--tls-key-file', self.tls_key_file] + if self.tls_client_ca_file: + cmd += ['--tls-client-ca-file', str(self.tls_client_ca_file)] + if self.readiness_state_file: cmd += ['--readiness-state-file', self.readiness_state_file] if self.jws_key_file: - cmd += ['--jws-key-file', self.jws_key_file] + cmd += ['--jws-key-file', str(self.jws_key_file)] if self.jwt_sub_allowlist_file: cmd += ['--jwt-sub-allowlist-file', self.jwt_sub_allowlist_file] @@ -2354,8 +2428,11 @@ def start_edgedb_server( data_dir: Optional[str] = None, reset_auth: Optional[bool] = None, tenant_id: Optional[str] = None, - security: Optional[edgedb_args.ServerSecurityMode] = None, - default_auth_method: Optional[edgedb_args.ServerAuthMethod] = None, + security: edgedb_args.ServerSecurityMode = ( + edgedb_args.ServerSecurityMode.Strict), + default_auth_method: Optional[ + edgedb_args.ServerAuthMethod | edgedb_args.ServerAuthMethods + ] = None, binary_endpoint_security: Optional[ edgedb_args.ServerEndpointSecurityMode] = None, http_endpoint_security: Optional[ @@ -2367,6 +2444,7 @@ def start_edgedb_server( tls_key_file: Optional[os.PathLike] = None, tls_cert_mode: edgedb_args.ServerTlsCertMode = ( edgedb_args.ServerTlsCertMode.SelfSigned), + tls_client_ca_file: Optional[os.PathLike] = None, jws_key_file: Optional[os.PathLike] = None, jwt_sub_allowlist_file: Optional[os.PathLike] = None, jwt_revocation_list_file: Optional[os.PathLike] = None, @@ -2433,6 +2511,7 @@ def start_edgedb_server( tls_cert_file=tls_cert_file, tls_key_file=tls_key_file, tls_cert_mode=tls_cert_mode, + tls_client_ca_file=tls_client_ca_file, jws_key_file=jws_key_file, jwt_sub_allowlist_file=jwt_sub_allowlist_file, jwt_revocation_list_file=jwt_revocation_list_file, diff --git a/tests/test_docs.py b/tests/test_docs.py index 23def70d0d3..77f98457a27 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -465,7 +465,6 @@ def test_doc_full_build(self): sys.executable, '-m', 'sphinx', '-n', - '-W', # fail on warnings '-b', 'xml', '-q', '-D', 'master_doc=index', @@ -483,3 +482,19 @@ def test_doc_full_build(self): f'STDOUT:\n{proc.stdout}\n\n' f'STDERR:\n{proc.stderr}\n' ) + + errors = [] + ignored_errors = re.compile( + r'^.* WARNING: undefined label: edgedb-' + r'(python|js|go|dart|dotnet|elixir|java)-.*$' + ) + for line in proc.stderr.splitlines(): + if not ignored_errors.match(line): + errors.append(line) + + if len(errors) > 0: + errors = '\n'.join(errors) + raise AssertionError( + f'Unable to build docs with Sphinx.\n\n' + f'{errors}\n\n' + ) diff --git a/tests/test_edgeql_triggers.py b/tests/test_edgeql_triggers.py index 77f8d3fa51b..5105c52b2a8 100644 --- a/tests/test_edgeql_triggers.py +++ b/tests/test_edgeql_triggers.py @@ -1317,3 +1317,34 @@ async def test_edgeql_triggers_when_bad(self): ); }; ''') + + async def test_edgeql_triggers_cached_global_01(self): + # Install FOR ALL triggers for everything + await self.con.execute(''' + create alias CA := count(InsertTest); + create global CG := count(InsertTest); + create type X { + create access policy asdf allow all using (global CG > 0) + }; + alter type InsertTest { + create trigger log after insert for each + do ( + insert Note { + name := assert_single(CA), + note := (global CG), + } + ); + }; + ''') + + await self.con.execute(''' + insert InsertTest { name := ((global CG)) }; + ''') + await self.assert_query_result( + ''' + select Note { name, note } + ''', + [ + {'name': '1', 'note': '1'}, + ], + ) diff --git a/tests/test_http_auth.py b/tests/test_http_auth.py index a787c076c27..5a19aa051e7 100644 --- a/tests/test_http_auth.py +++ b/tests/test_http_auth.py @@ -18,7 +18,6 @@ import base64 -import struct import urllib import edgedb @@ -117,8 +116,8 @@ def _scram_auth_expect_failure(self, user, password): content, headers, status, - _sid, - _expected_server_sig, + sid, + expected_server_sig, ) = self._scram_auth(user, password) self.assertEqual(status, 401) self.assertEqual(content, b"Authentication failed") @@ -144,46 +143,21 @@ def test_http_auth_scram_valid(self): server_final = base64.b64decode(values["data"]) server_sig = scram.parse_server_final_message(server_final) self.assertEqual(server_sig, expected_server_sig) + proto_ver = edbdef.CURRENT_PROTOCOL proto_ver_str = f"v_{proto_ver[0]}_{proto_ver[1]}" mime_type = f"application/x.edgedb.{proto_ver_str}.binary" with self.http_con() as con: - con.request( - "POST", - f"/db/{args['database']}", - body=protocol.Execute( - annotations=[], - allowed_capabilities=protocol.Capability.ALL, - compilation_flags=protocol.CompilationFlag(0), - implicit_limit=0, - command_text="SELECT 42", - output_format=protocol.OutputFormat.JSON, - expected_cardinality=protocol.Cardinality.AT_MOST_ONE, - input_typedesc_id=b"\0" * 16, - output_typedesc_id=b"\0" * 16, - state_typedesc_id=b"\0" * 16, - arguments=b"", - state_data=b"", - ).dump() - + protocol.Sync().dump(), - headers={ - "Content-Type": mime_type, - "Authorization": f"Bearer {token.decode('ascii')}", - "X-EdgeDB-User": args["user"], - }, + msgs, headers, status = self.http_con_binary_request( + con, + "SELECT 42", + bearer_token=token.decode("ascii"), + user=args["user"], + database=args["database"], ) - content, headers, status = self.http_con_read_response(con) self.assertEqual(status, 200) self.assertEqual(headers, headers | {"content-type": mime_type}) - uint32_unpack = struct.Struct("!L").unpack - msgs = [] - while content: - mtype = content[0] - (msize,) = uint32_unpack(content[1:5]) - msg = protocol.ServerMessage.parse(mtype, content[5 : msize + 1]) - msgs.append(msg) - content = content[msize + 1:] self.assertIsInstance(msgs[0], protocol.CommandDataDescription) self.assertIsInstance(msgs[1], protocol.Data) self.assertEqual(bytes(msgs[1].data[0].data), b"42") diff --git a/tests/test_server_auth.py b/tests/test_server_auth.py index dccb9ccd90c..c9f01e84d1a 100644 --- a/tests/test_server_auth.py +++ b/tests/test_server_auth.py @@ -20,15 +20,19 @@ import os import pathlib import signal +import ssl import tempfile import unittest import urllib.error import urllib.request +import asyncpg import jwcrypto.jwk import edgedb +from edb import errors +from edb import protocol from edb.common import secretkey from edb.server import args from edb.server import cluster as edbcluster @@ -303,22 +307,39 @@ async def _basic_http_request( resp_status = resp.status return resp_body, resp_status - async def _jwt_http_request( - self, server, sk, username='edgedb', db='edgedb', proto='edgeql' + async def _http_request( + self, + server, + sk=None, + username='edgedb', + db='edgedb', + proto='edgeql', + client_cert_file=None, + client_key_file=None, ): - with self.http_con(server, keep_alive=False) as con: + with self.http_con( + server, + keep_alive=False, + client_cert_file=client_cert_file, + client_key_file=client_key_file, + ) as con: + headers = {'X-EdgeDB-User': username} + if sk is not None: + headers['Authorization'] = f'bearer {sk}' return self.http_con_request( con, path=f'/db/{db}/{proto}', # ... the graphql ones will produce an error, but that's # still a 200 params=dict(query='select 1'), - headers={ - 'Authorization': f'bearer {sk}', - 'X-EdgeDB-User': username, - }, + headers=headers, ) + async def _jwt_http_request( + self, server, sk, username='edgedb', db='edgedb', proto='edgeql' + ): + return await self._http_request(server, sk, username, db, proto) + def _jwt_gql_request(self, server, sk): return self._jwt_http_request(server, sk, proto='graphql') @@ -560,3 +581,139 @@ async def test_server_auth_in_transaction(self): await self.con.query(''' DROP ROLE foo; ''') + + @unittest.skipIf( + "EDGEDB_SERVER_MULTITENANT_CONFIG_FILE" in os.environ, + "cannot use CONFIGURE INSTANCE in multi-tenant mode", + ) + async def test_server_auth_mtls(self): + if not self.has_create_role: + self.skipTest('create role is not supported by the backend') + + certs = pathlib.Path(__file__).parent / 'certs' + client_ca_cert_file = certs / 'client_ca.cert.pem' + client_ssl_cert_file = certs / 'client.cert.pem' + client_ssl_key_file = certs / 'client.key.pem' + async with tb.start_edgedb_server( + tls_client_ca_file=client_ca_cert_file, + security=args.ServerSecurityMode.Strict, + ) as sd: + # Setup mTLS and extensions + conn = await sd.connect() + try: + await conn.query("CREATE SUPERUSER ROLE ssl_user;") + await conn.query("CREATE EXTENSION edgeql_http;") + await self._test_mtls( + sd, client_ssl_cert_file, client_ssl_key_file, False) + await conn.query(""" + CONFIGURE INSTANCE INSERT Auth { + comment := 'test', + priority := 0, + method := (INSERT mTLS { + transports := { + cfg::ConnectionTransport.TCP, + cfg::ConnectionTransport.TCP_PG, + cfg::ConnectionTransport.HTTP, + cfg::ConnectionTransport.SIMPLE_HTTP, + }, + }), + } + """) + await self._test_mtls( + sd, client_ssl_cert_file, client_ssl_key_file, True) + finally: + await conn.aclose() + + async def _test_mtls( + self, sd, client_ssl_cert_file, client_ssl_key_file, granted + ): + # Verifies mTLS authentication on edgeql_http + if granted: + body, _, code = await self._http_request(sd, username="ssl_user") + self.assertEqual(code, 401, f"Wrong result: {body}") + body, _, code = await self._http_request( + sd, + username="ssl_user", + client_cert_file=client_ssl_cert_file, + client_key_file=client_ssl_key_file, + ) + if granted: + self.assertEqual(code, 200, f"Wrong result: {body}") + else: + self.assertEqual(code, 401, f"Wrong result: {body}") + + # Verifies mTLS authentication on the binary protocol + if granted: + with self.assertRaisesRegex( + edgedb.AuthenticationError, + 'client certificate required', + ): + await sd.connect() + # FIXME: add mTLS support in edgedb-python + + # Verifies mTLS authentication on binary protocol over HTTP + if granted: + with self.http_con( + sd, + keep_alive=False, + ) as con: + msgs, _, status = self.http_con_binary_request( + con, "select 42", user="ssl_user") + self.assertEqual(status, 200) + self.assertIsInstance(msgs[0], protocol.ErrorResponse) + self.assertEqual( + msgs[0].error_code, errors.AuthenticationError.get_code()) + with self.http_con( + sd, + keep_alive=False, + client_cert_file=client_ssl_cert_file, + client_key_file=client_ssl_key_file, + ) as con: + msgs, _, status = self.http_con_binary_request( + con, "select 42", user="ssl_user") + if granted: + self.assertEqual(status, 200) + self.assertIsInstance(msgs[0], protocol.CommandDataDescription) + self.assertIsInstance(msgs[1], protocol.Data) + self.assertEqual(bytes(msgs[1].data[0].data), b"42") + self.assertIsInstance(msgs[2], protocol.CommandComplete) + self.assertEqual(msgs[2].status, "SELECT") + self.assertIsInstance(msgs[3], protocol.ReadyForCommand) + else: + self.assertEqual(status, 200) + self.assertIsInstance(msgs[0], protocol.ErrorResponse) + self.assertEqual( + msgs[0].error_code, errors.AuthenticationError.get_code()) + + # Verifies mTLS authentication on emulated Postgres protocol + conargs = sd.get_connect_args() + tls_context = ssl.create_default_context( + ssl.Purpose.SERVER_AUTH, + cafile=conargs["tls_ca_file"], + ) + tls_context.check_hostname = False + conargs = dict( + host=conargs['host'], + port=conargs['port'], + user="ssl_user", + database=conargs.get('database', 'main'), + ssl=tls_context, + ) + if granted: + with self.assertRaisesRegex( + asyncpg.InvalidAuthorizationSpecificationError, + 'client certificate required', + ): + await asyncpg.connect(**conargs) + tls_context.load_cert_chain( + client_ssl_cert_file, client_ssl_key_file) + if granted: + conn = await asyncpg.connect(**conargs) + self.assertEqual(await conn.fetchval("select 42"), 42) + await conn.close() + else: + with self.assertRaisesRegex( + asyncpg.InvalidAuthorizationSpecificationError, + 'authentication failed', + ): + await asyncpg.connect(**conargs) diff --git a/tests/test_server_config.py b/tests/test_server_config.py index f030ff37e68..9bf3bd81b7a 100644 --- a/tests/test_server_config.py +++ b/tests/test_server_config.py @@ -1631,6 +1631,7 @@ async def test_server_config_default_branch_01(self): DBNAME = 'asdf' async with tb.start_edgedb_server( http_endpoint_security=args.ServerEndpointSecurityMode.Optional, + security=args.ServerSecurityMode.InsecureDevMode, default_branch=DBNAME, ) as sd: def check(mode, name, current, ok=True): diff --git a/tests/test_server_ops.py b/tests/test_server_ops.py index 606eda90053..d467743639c 100644 --- a/tests/test_server_ops.py +++ b/tests/test_server_ops.py @@ -37,6 +37,7 @@ import uuid import edgedb +import httpx from edgedb import errors from edb import protocol @@ -783,6 +784,29 @@ async def test_server_ops_cleartext_http_allowed(self): finally: await con.aclose() + async def test_server_ops_mtls_http_transports(self): + certs = pathlib.Path(__file__).parent / 'certs' + client_ca_cert_file = certs / 'client_ca.cert.pem' + client_ssl_cert_file = certs / 'client.cert.pem' + client_ssl_key_file = certs / 'client.key.pem' + async with tb.start_edgedb_server( + tls_client_ca_file=client_ca_cert_file, + security=args.ServerSecurityMode.Strict, + ) as sd: + def test(url): + resp = httpx.get(url, verify=sd.tls_cert_file) + self.assertFalse(resp.is_success) + + resp = httpx.get( + url, + verify=sd.tls_cert_file, + cert=(str(client_ssl_cert_file), str(client_ssl_key_file)), + ) + self.assertTrue(resp.is_success) + + test(f'https://{sd.host}:{sd.port}/metrics') + test(f'https://{sd.host}:{sd.port}/server/status/alive') + @unittest.skipIf( "EDGEDB_SERVER_MULTITENANT_CONFIG_FILE" in os.environ, "--readiness-state-file is not allowed in multi-tenant mode",