Improve system api to perform HTTP requests (#2631)

## Motivation

Linera applications can perform HTTP requests to external services, but
the available `http_post` API was very limited, and prevented
applications from configuring the request. This is often needed in order
to add custom headers, for example for authentication.

## Proposal

Replace the `http_post` API with a broader `http_request` API, which
allows sending more configurable `http::Request`s, and returns a more
detailed `http::Response`.

## Test Plan

Tested with the demo for
[Atoma](https://github.com/linera-io/atoma-demo).

## Release Plan

- Backporting is not possible but we may want to deploy a new `devnet`
and release a new
      SDK soon.
- Contains breaking changes to the WIT interface and to the `linera-sdk`
API.

## Links

- [reviewer
checklist](https://github.com/linera-io/linera-protocol/blob/main/CONTRIBUTING.md#reviewer-checklist)

---------

Signed-off-by: Janito Vaqueiro Ferreira Filho <[email protected]>
Co-authored-by: Andreas Fackler <[email protected]>

linera-base/Cargo.toml

@@ -12,8 +12,9 @@ repository.workspace = true
 version.workspace = true
 
 [features]
-test = ["test-strategy", "proptest"]
 metrics = ["prometheus"]
+reqwest = ["dep:reqwest"]
+test = ["test-strategy", "proptest"]
 web = [
     "getrandom/js",
     "rand/getrandom",
@@ -47,6 +48,7 @@ linera-witty = { workspace = true, features = ["macros"] }
 prometheus = { workspace = true, optional = true }
 proptest = { workspace = true, optional = true, features = ["alloc"] }
 rand.workspace = true
+reqwest = { workspace = true, optional = true }
 secp256k1.workspace = true
 serde.workspace = true
 serde-name.workspace = true

linera-base/build.rs

@@ -6,6 +6,7 @@ fn main() {
         web: { all(target_arch = "wasm32", feature = "web") },
         chain: { all(target_arch = "wasm32", not(web)) },
         with_metrics: { all(not(target_arch = "wasm32"), feature = "metrics") },
+        with_reqwest: { feature = "reqwest" },
         with_testing: { any(test, feature = "test") },
 
         // the old version of `getrandom` we pin here is available on all targets, but

linera-base/src/data_types.rs

@@ -30,7 +30,7 @@ use thiserror::Error;
 use crate::prometheus_util::{bucket_latencies, register_histogram_vec, MeasureLatency};
 use crate::{
     crypto::{BcsHashable, CryptoHash},
-    doc_scalar, hex_debug,
+    doc_scalar, hex_debug, http,
     identifiers::{
         ApplicationId, BlobId, BlobType, BytecodeId, ChainId, Destination, EventId,
         GenericApplicationId, MessageId, StreamId, UserApplicationId,
@@ -770,12 +770,8 @@ pub enum OracleResponse {
         #[serde(with = "serde_bytes")]
         Vec<u8>,
     ),
-    /// The response from an HTTP POST request.
-    Post(
-        #[debug(with = "hex_debug")]
-        #[serde(with = "serde_bytes")]
-        Vec<u8>,
-    ),
+    /// The response from an HTTP request.
+    Http(http::Response),
     /// A successful read or write of a blob.
     Blob(BlobId),
     /// An assertion oracle that passed.

linera-base/src/http.rs

@@ -0,0 +1,212 @@
+// Copyright (c) Zefchain Labs, Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+//! Types used when performing HTTP requests.
+
+use custom_debug_derive::Debug;
+use linera_witty::{WitLoad, WitStore, WitType};
+use serde::{Deserialize, Serialize};
+
+use crate::hex_debug;
+
+/// An HTTP request.
+#[derive(Clone, Debug, Eq, PartialEq, WitLoad, WitStore, WitType)]
+#[witty(name = "http-request")]
+pub struct Request {
+    /// The [`Method`] used for the HTTP request.
+    pub method: Method,
+
+    /// The URL this request is intended to.
+    pub url: String,
+
+    /// The headers that should be included in the request.
+    pub headers: Vec<Header>,
+
+    /// The body of the request.
+    #[debug(with = "hex_debug")]
+    pub body: Vec<u8>,
+}
+
+impl Request {
+    /// Creates an HTTP GET [`Request`] for a `url`.
+    pub fn get(url: impl Into<String>) -> Self {
+        Request {
+            method: Method::Get,
+            url: url.into(),
+            headers: vec![],
+            body: vec![],
+        }
+    }
+
+    /// Creates an HTTP POST [`Request`] for a `url` with a `payload` that's arbitrary bytes.
+    pub fn post(url: impl Into<String>, payload: impl Into<Vec<u8>>) -> Self {
+        Request {
+            method: Method::Post,
+            url: url.into(),
+            headers: vec![],
+            body: payload.into(),
+        }
+    }
+
+    /// Creates an HTTP POST [`Request`] for a `url` with a body that's the `payload` serialized to
+    /// JSON.
+    pub fn post_json(
+        url: impl Into<String>,
+        payload: &impl Serialize,
+    ) -> Result<Self, serde_json::Error> {
+        Ok(Request {
+            method: Method::Post,
+            url: url.into(),
+            headers: vec![Header::new("Content-Type", b"application/json")],
+            body: serde_json::to_vec(payload)?,
+        })
+    }
+
+    /// Adds a header to this [`Request`].
+    pub fn with_header(mut self, name: impl Into<String>, value: impl Into<Vec<u8>>) -> Self {
+        self.headers.push(Header::new(name, value));
+        self
+    }
+}
+
+/// The method used in an HTTP request.
+#[derive(Clone, Copy, Debug, Eq, PartialEq, WitLoad, WitStore, WitType)]
+#[witty(name = "http-method")]
+pub enum Method {
+    /// A GET request.
+    Get,
+
+    /// A POST request.
+    Post,
+
+    /// A PUT request.
+    Put,
+
+    /// A DELETE request.
+    Delete,
+
+    /// A HEAD request.
+    Head,
+
+    /// A OPTIONS request.
+    Options,
+
+    /// A CONNECT request.
+    Connect,
+
+    /// A PATCH request.
+    Patch,
+
+    /// A TRACE request.
+    Trace,
+}
+
+#[cfg(with_reqwest)]
+impl From<Method> for reqwest::Method {
+    fn from(method: Method) -> Self {
+        match method {
+            Method::Get => reqwest::Method::GET,
+            Method::Post => reqwest::Method::POST,
+            Method::Put => reqwest::Method::PUT,
+            Method::Delete => reqwest::Method::DELETE,
+            Method::Head => reqwest::Method::HEAD,
+            Method::Options => reqwest::Method::OPTIONS,
+            Method::Connect => reqwest::Method::CONNECT,
+            Method::Patch => reqwest::Method::PATCH,
+            Method::Trace => reqwest::Method::TRACE,
+        }
+    }
+}
+
+/// A response for an HTTP request.
+#[derive(Clone, Debug, Deserialize, Eq, Hash, PartialEq, Serialize, WitLoad, WitStore, WitType)]
+#[witty(name = "http-response")]
+pub struct Response {
+    /// The status code of the HTTP response.
+    pub status: u16,
+
+    /// The headers included in the response.
+    pub headers: Vec<Header>,
+
+    /// The body of the response.
+    #[debug(with = "hex_debug")]
+    #[serde(with = "serde_bytes")]
+    pub body: Vec<u8>,
+}
+
+impl Response {
+    /// Creates an HTTP [`Response`] with a user defined `status_code`.
+    pub fn new(status_code: u16) -> Self {
+        Response {
+            status: status_code,
+            headers: vec![],
+            body: vec![],
+        }
+    }
+
+    /// Creates an HTTP [`Response`] with an OK status code and the provided `body`.
+    pub fn ok(body: impl Into<Vec<u8>>) -> Self {
+        Response {
+            status: 200,
+            headers: vec![],
+            body: body.into(),
+        }
+    }
+
+    /// Creates an HTTP [`Response`] with an Unauthorized status code.
+    pub fn unauthorized() -> Self {
+        Response {
+            status: 401,
+            headers: vec![],
+            body: vec![],
+        }
+    }
+
+    /// Adds a header to this [`Response`].
+    pub fn with_header(mut self, name: impl Into<String>, value: impl Into<Vec<u8>>) -> Self {
+        self.headers.push(Header::new(name, value));
+        self
+    }
+}
+
+#[cfg(with_reqwest)]
+impl Response {
+    /// Creates a [`Response`] from a [`reqwest::Response`], waiting for it to be fully
+    /// received.
+    pub async fn from_reqwest(response: reqwest::Response) -> reqwest::Result<Self> {
+        let headers = response
+            .headers()
+            .into_iter()
+            .map(|(name, value)| Header::new(name.to_string(), value.as_bytes()))
+            .collect();
+
+        Ok(Response {
+            status: response.status().as_u16(),
+            headers,
+            body: response.bytes().await?.to_vec(),
+        })
+    }
+}
+
+/// A header for an HTTP request or response.
+#[derive(Clone, Debug, Deserialize, Eq, Hash, PartialEq, Serialize, WitLoad, WitStore, WitType)]
+#[witty(name = "http-header")]
+pub struct Header {
+    /// The header name.
+    pub name: String,
+
+    /// The value of the header.
+    #[debug(with = "hex_debug")]
+    #[serde(with = "serde_bytes")]
+    pub value: Vec<u8>,
+}
+
+impl Header {
+    /// Creates a new [`Header`] with the provided `name` and `value`.
+    pub fn new(name: impl Into<String>, value: impl Into<Vec<u8>>) -> Self {
+        Header {
+            name: name.into(),
+            value: value.into(),
+        }
+    }
+}

linera-base/src/lib.rs

@@ -22,6 +22,7 @@ pub mod data_types;
 pub mod dyn_convert;
 mod graphql;
 pub mod hashed;
+pub mod http;
 pub mod identifiers;
 mod limited_writer;
 pub mod ownership;

linera-execution/Cargo.toml

@@ -56,7 +56,7 @@ dyn-clone.workspace = true
 futures.workspace = true
 hex = { workspace = true, optional = true }
 js-sys = { workspace = true, optional = true }
-linera-base.workspace = true
+linera-base = { workspace = true, features = ["reqwest"] }
 linera-views.workspace = true
 linera-views-derive.workspace = true
 linera-witty = { workspace = true, features = ["log", "macros"] }

linera-execution/src/execution_state_actor.rs

@@ -12,15 +12,15 @@ use futures::channel::mpsc;
 use linera_base::prometheus_util::{bucket_latencies, register_histogram_vec, MeasureLatency as _};
 use linera_base::{
     data_types::{Amount, ApplicationPermissions, BlobContent, Timestamp},
-    hex_debug, hex_vec_debug,
+    hex_debug, hex_vec_debug, http,
     identifiers::{Account, AccountOwner, BlobId, MessageId, Owner},
     ownership::ChainOwnership,
 };
 use linera_views::{batch::Batch, context::Context, views::View};
 use oneshot::Sender;
 #[cfg(with_metrics)]
 use prometheus::HistogramVec;
-use reqwest::{header::CONTENT_TYPE, Client};
+use reqwest::{header::HeaderMap, Client};
 
 use crate::{
     system::{CreateApplicationResult, OpenChainConfig, Recipient},
@@ -329,21 +329,20 @@ where
                 callback.respond(bytes);
             }
 
-            HttpPost {
-                url,
-                content_type,
-                payload,
-                callback,
-            } => {
-                let res = Client::new()
-                    .post(url)
-                    .body(payload)
-                    .header(CONTENT_TYPE, content_type)
+            PerformHttpRequest { request, callback } => {
+                let headers = request
+                    .headers
+                    .into_iter()
+                    .map(|http::Header { name, value }| Ok((name.parse()?, value.try_into()?)))
+                    .collect::<Result<HeaderMap, ExecutionError>>()?;
+
+                let response = Client::new()
+                    .request(request.method.into(), request.url)
+                    .body(request.body)
+                    .headers(headers)
                     .send()
                     .await?;
-                let body = res.bytes().await?;
-                let bytes = body.as_ref().to_vec();
-                callback.respond(bytes);
+                callback.respond(http::Response::from_reqwest(response).await?);
             }
 
             ReadBlobContent { blob_id, callback } => {
@@ -524,13 +523,10 @@ pub enum ExecutionRequest {
         callback: Sender<Vec<u8>>,
     },
 
-    HttpPost {
-        url: String,
-        content_type: String,
-        #[debug(with = hex_debug)]
-        payload: Vec<u8>,
+    PerformHttpRequest {
+        request: http::Request,
         #[debug(skip)]
-        callback: Sender<Vec<u8>>,
+        callback: Sender<http::Response>,
     },
 
     ReadBlobContent {