Configure HttpHook's auth_type from Connection

airflow/providers_manager.py

@@ -1011,7 +1011,7 @@ def _import_hook(
         :param package_name: provider package - only needed in case connection_type is missing
         : return
         """
-        from wtforms import BooleanField, IntegerField, PasswordField, StringField
+        from wtforms import BooleanField, IntegerField, PasswordField, SelectField, StringField, TextAreaField
 
         if connection_type is None and hook_class_name is None:
             raise ValueError("Either connection_type or hook_class_name must be set")
@@ -1031,7 +1031,14 @@ def _import_hook(
                 raise ValueError(
                     f"Provider package name is not set when hook_class_name ({hook_class_name}) is used"
                 )
-        allowed_field_classes = [IntegerField, PasswordField, StringField, BooleanField]
+        allowed_field_classes = [
+            IntegerField,
+            PasswordField,
+            StringField,
+            BooleanField,
+            TextAreaField,
+            SelectField,
+        ]
         hook_class: type[BaseHook] | None = _correctness_check(package_name, hook_class_name, provider_info)
         if hook_class is None:
             return None

airflow/www/templates/airflow/conn_create.html

@@ -25,7 +25,6 @@
 <meta name="rest_api_enabled" content="{{ rest_api_enabled }}">
 <meta name="test_url"
   content="{{ url_for('/api/v1.airflow_api_connexion_endpoints_connection_endpoint_test_connection') }}">
-
 {# required for codemirror #}
 <link rel="stylesheet" type="text/css" href="{{ url_for_asset('codemirror.css') }}">
 <link rel="stylesheet" type="text/css" href="{{ url_for_asset('lint.css') }}">

docs/apache-airflow-providers-http/configurations-ref.rst

@@ -0,0 +1,18 @@
+ .. Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+ ..   http://www.apache.org/licenses/LICENSE-2.0
+
+ .. Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+.. include:: ../exts/includes/providers-configurations-ref.rst

docs/apache-airflow-providers-http/connections/http.rst

@@ -24,35 +24,114 @@ HTTP Connection
 
 The HTTP connection enables connections to HTTP services.
 
-Authenticating with HTTP
-------------------------
-
-Login and Password authentication can be used along with any authentication method using headers.
-Headers can be given in json format in the Extras field.
-
 Default Connection IDs
 ----------------------
 
 The HTTP operators and hooks use ``http_default`` by default.
 
+Authentication
+--------------
+
+ .. _auth_basic:
+
+Authenticating via Basic auth
+.............................
+The simplest way to authenticate is to specify a *Login* and *Password* in the
+Connection.
+
+.. image:: /img/connection_username_password.png
+
+By default, when a *Login* or *Password* is provided, the HTTP operators and
+Hooks will perform a basic authentication via the
+``requests.auth.HTTPBasicAuth`` class.
+
+Authenticating via Headers
+..........................
+If :ref:`Basic authentication<auth_basic>` is not enough, you can also add
+*Headers* to the requests performed by the HTTP operators and Hooks.
+
+Headers can be passed in json format in the *Headers* field:
+
+.. image:: /img/connection_headers.png
+
+.. note:: Login and Password authentication can be used along custom Headers.
+
+Authenticating via Auth class
+.............................
+For more complex use-cases, you can inject a Auth class into the HTTP operators
+and Hooks via the *Auth type* setting. This is particularly useful when you
+need token refresh or advanced authentication methods like kerberos, oauth, ...
+
+.. image:: /img/connection_auth_type.png
+
+By default, only `requests Auth classes <https://github.com/psf/requests/blob/main/src/requests/auth.py>`_
+are available. But you can install any classes based on ``requests.auth.AuthBase``
+into your Airflow instance (via pip install), and then specify those classes in
+``extra_auth_types`` :doc:`configuration setting<../configurations-ref>` to
+make them available in the Connection UI.
+
+If the Auth class requires more than a *Username* and a *Password*, you can
+pass extra keywords arguments with the *Auth kwargs* setting.
+
+Example with the ``HTTPKerberosAuth`` from `requests-kerberos <https://pypi.org/project/requests-kerberos>`_ :
+
+.. image:: /img/connection_auth_kwargs.png
+
+.. tip::
+
+    You probably don't need to write an entire custom HttpOperator or HttpHook
+    to customize the connection. Simply extend the ``requests.auth.AuthBase``
+    class and configure a Connection with it.
+
 Configuring the Connection
 --------------------------
 
+Via the Admin panel
+...................
+
+Configuring the Connection via the Airflow Admin panel offers more
+possibilities than via :ref:`environment variables<env-variable>`.
+
 Login (optional)
-    Specify the login for the http service you would like to connect too.
+    The login (username) of the http service you would like to connect too.
+    If provided, by default, the HttpHook perform a Basic authentication.
 
 Password (optional)
-    Specify the password for the http service you would like to connect too.
+    The password of the http service you would like to connect too.
+    If provided, by default, the HttpHook perform a Basic authentication.
 
 Host (optional)
     Specify the entire url or the base of the url for the service.
 
 Port (optional)
-    Specify a port number if applicable.
+    A port number if applicable.
 
 Schema (optional)
-    Specify the service type etc: http/https.
+    The service type. E.g: http/https.
 
+Auth type (optional)
+    Python class used by the HttpHook (and the underlying requests library) to
+    authenticate. If provided, the *Login* and *Password* are passed as the two
+    first arguments to this class. If *Login* and/or *Password* are provided
+    without any Auth type, the HttpHook will by default perform a basic
+    authentication via the ``requests.auth.HTTPBasicAuth`` class.
+
+    Extra classes can be added via the ``extra_auth_types``
+    :doc:`configuration setting<../configurations-ref>`.
+
+Auth kwargs (optional)
+    Extra key-value parameters passed to the Auth type class.
+
+Headers (optional)
+    Extra key-value parameters added to the Headers in JSON format.
+
+Extras (optional - deprecated)
+    *Deprecated*: Specify headers in json format.
+
+ .. _env-variable:
+
+Via environment variable
+........................
 Extras (optional)
     Specify headers and default requests parameters in json format.
     Following default requests parameters are taken into account:
@@ -68,10 +147,9 @@ Extras (optional)
 When specifying the connection in environment variable you should specify
 it using URI syntax.
 
-Note that all components of the URI should be URL-encoded.
-
-For example:
+.. note:: All components of the URI should be **URL-encoded**.
 
 .. code-block:: bash
+   :caption: Example:
 
-   export AIRFLOW_CONN_HTTP_DEFAULT='http://username:password@servvice.com:80/https?headers=header'
+   export AIRFLOW_CONN_HTTP_DEFAULT='http://username:password@service.com:80/https?headers=header'

docs/apache-airflow-providers-http/index.rst

@@ -42,6 +42,7 @@
     :maxdepth: 1
     :caption: References
 
+    Configuration <configurations-ref>
     Python API <_api/airflow/providers/http/index>
 
 .. toctree::

docs/spelling_wordlist.txt

@@ -25,6 +25,7 @@ afterall
 AgentKey
 aio
 aiobotocore
+aiohttp
 AioSession
 aiplatform
 Airbnb

providers/src/airflow/providers/apache/livy/hooks/livy.py

@@ -81,18 +81,24 @@ class LivyHook(HttpHook, LoggingMixin):
     conn_type = "livy"
     hook_name = "Apache Livy"
 
+    @classmethod
+    def get_connection_form_widgets(cls) -> dict[str, Any]:
+        return super().get_connection_form_widgets()
+
+    @classmethod
+    def get_ui_field_behaviour(cls) -> dict[str, Any]:
+        return super().get_ui_field_behaviour()
+
     def __init__(
         self,
         livy_conn_id: str = default_conn_name,
         extra_options: dict[str, Any] | None = None,
         extra_headers: dict[str, Any] | None = None,
         auth_type: Any | None = None,
     ) -> None:
-        super().__init__(http_conn_id=livy_conn_id)
+        super().__init__(http_conn_id=livy_conn_id, auth_type=auth_type)
         self.extra_headers = extra_headers or {}
         self.extra_options = extra_options or {}
-        if auth_type:
-            self.auth_type = auth_type
 
     def get_conn(self, headers: dict[str, Any] | None = None) -> Any:
         """