Storing call_data in quotes and order_quotes tables (#3124)

# Description
Added two new non-null columns: `metadata` (json) and `verified`
(boolean) into tables `quotes` and `order_quotes`.
Existing rows in database will be updated with `{}` and `false` values.
`metadata` column is dedicated to store any additional data for the
quote in json format. Currently it is used to store interactions
received for `/quote` response from the solvers, but it is prepared to
store other data in future through versioned `QuoteMetadata` struct as
required.
Database module only checks json validity, upper layers (orderbook,
autopilot) uses `QuoteMetadata` struct for storing specific data in the
database.
Metadata can be used for verification/auditing of quotes/orders.
Added database migration scripts (update and revert) and updated
database readme file.

## How to test
Existing tests, also added dedicated e2e tests and unit tests for this
new functionality .

crates/autopilot/src/database/auction.rs

@@ -23,7 +23,7 @@ impl QuoteStoring for Postgres {
             .start_timer();
 
         let mut ex = self.pool.acquire().await?;
-        let row = create_quote_row(data);
+        let row = create_quote_row(data)?;
         let id = database::quotes::save(&mut ex, &row).await?;
         Ok(id)
     }

crates/autopilot/src/database/onchain_order_events/mod.rs

@@ -487,6 +487,8 @@ async fn parse_general_onchain_order_placement_data<'a>(
                     sell_amount: u256_to_big_decimal(&quote.sell_amount),
                     buy_amount: u256_to_big_decimal(&quote.buy_amount),
                     solver: ByteArray(quote.data.solver.0),
+                    verified: quote.data.verified,
+                    metadata: quote.data.metadata.try_into()?,
                 }),
                 Err(err) => {
                     let err_label = err.to_metrics_label();
@@ -1187,6 +1189,8 @@ mod test {
             sell_amount: u256_to_big_decimal(&quote.sell_amount),
             buy_amount: u256_to_big_decimal(&quote.buy_amount),
             solver: ByteArray(quote.data.solver.0),
+            verified: quote.data.verified,
+            metadata: quote.data.metadata.try_into().unwrap(),
         };
         assert_eq!(result.1, vec![Some(expected_quote)]);
         assert_eq!(

crates/database/Cargo.toml

@@ -13,6 +13,7 @@ futures = { workspace = true }
 hex = { workspace = true }
 sqlx = { workspace = true }
 strum = { workspace = true }
+serde_json = { workspace = true }
 
 [dev-dependencies]
 maplit = { workspace = true }

crates/database/src/orders.rs

@@ -329,6 +329,8 @@ pub struct Quote {
     pub sell_amount: BigDecimal,
     pub buy_amount: BigDecimal,
     pub solver: Address,
+    pub verified: bool,
+    pub metadata: serde_json::Value,
 }
 
 pub async fn insert_quotes(ex: &mut PgConnection, quotes: &[Quote]) -> Result<(), sqlx::Error> {
@@ -346,9 +348,11 @@ INSERT INTO order_quotes (
     sell_token_price,
     sell_amount,
     buy_amount,
-    solver
+    solver,
+    verified,
+    metadata
 )
-VALUES ($1, $2, $3, $4, $5, $6, $7)"#;
+VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)"#;
 
 pub async fn insert_quote_and_update_on_conflict(
     ex: &mut PgConnection,
@@ -362,7 +366,7 @@ pub async fn insert_quote_and_update_on_conflict(
         " ON CONFLICT (order_uid) DO UPDATE
 SET gas_amount = $2, gas_price = $3,
 sell_token_price = $4, sell_amount = $5,
-buy_amount = $6
+buy_amount = $6, verified = $8, metadata = $9
     "
     );
     sqlx::query(QUERY)
@@ -373,6 +377,8 @@ buy_amount = $6
         .bind(&quote.sell_amount)
         .bind(&quote.buy_amount)
         .bind(quote.solver)
+        .bind(quote.verified)
+        .bind(&quote.metadata)
         .execute(ex)
         .await?;
     Ok(())
@@ -387,6 +393,8 @@ pub async fn insert_quote(ex: &mut PgConnection, quote: &Quote) -> Result<(), sq
         .bind(&quote.sell_amount)
         .bind(&quote.buy_amount)
         .bind(quote.solver)
+        .bind(quote.verified)
+        .bind(&quote.metadata)
         .execute(ex)
         .await?;
     Ok(())
@@ -498,6 +506,8 @@ pub struct FullOrderWithQuote {
     pub quote_gas_amount: Option<f64>,
     pub quote_gas_price: Option<f64>,
     pub quote_sell_token_price: Option<f64>,
+    pub quote_verified: Option<bool>,
+    pub quote_metadata: Option<serde_json::Value>,
     pub solver: Option<Address>,
 }
 
@@ -591,6 +601,8 @@ pub async fn single_full_order_with_quote(
         ", o_quotes.gas_amount as quote_gas_amount",
         ", o_quotes.gas_price as quote_gas_price",
         ", o_quotes.sell_token_price as quote_sell_token_price",
+        ", o_quotes.verified as quote_verified",
+        ", o_quotes.metadata as quote_metadata",
         ", o_quotes.solver as solver",
         " FROM ", FROM,
         " LEFT JOIN order_quotes o_quotes ON o.uid = o_quotes.order_uid",
@@ -1201,6 +1213,8 @@ mod tests {
             sell_amount: 4.into(),
             buy_amount: 5.into(),
             solver: ByteArray([1; 20]),
+            verified: false,
+            metadata: Default::default(),
         };
         insert_quote(&mut db, &quote).await.unwrap();
         insert_quote_and_update_on_conflict(&mut db, &quote)
@@ -1253,6 +1267,20 @@ mod tests {
         let mut db = db.begin().await.unwrap();
         crate::clear_DANGER_(&mut db).await.unwrap();
 
+        let metadata: serde_json::Value = serde_json::from_str(
+            r#"{ "version":"1.0", "interactions": [ {
+            "target": "0x0102030405060708091011121314151617181920",
+            "value": "1",
+            "callData": "0x0A0B0C102030"
+            },{
+            "target": "0xFF02030405060708091011121314151617181920",
+            "value": "2",
+            "callData": "0xFF0B0C102030"
+            }]
+        }"#,
+        )
+        .unwrap();
+
         let quote = Quote {
             order_uid: Default::default(),
             gas_amount: 1.,
@@ -1261,6 +1289,8 @@ mod tests {
             sell_amount: 4.into(),
             buy_amount: 5.into(),
             solver: ByteArray([1; 20]),
+            verified: true,
+            metadata,
         };
         insert_quote(&mut db, &quote).await.unwrap();
         let quote_ = read_quote(&mut db, &quote.order_uid)
@@ -1287,6 +1317,8 @@ mod tests {
             sell_amount: 4.into(),
             buy_amount: 5.into(),
             solver: ByteArray([1; 20]),
+            verified: false,
+            metadata: Default::default(),
         };
         insert_quote(&mut db, &quote).await.unwrap();
         let order_with_quote = single_full_order_with_quote(&mut db, &quote.order_uid)
@@ -2142,4 +2174,36 @@ mod tests {
             ]
         );
     }
+
+    #[tokio::test]
+    #[ignore]
+    async fn postgres_get_quote_with_no_metadata_and_validity() {
+        // This test checks backward compatibility
+        let mut db = PgConnection::connect("postgresql://").await.unwrap();
+        let mut db = db.begin().await.unwrap();
+        crate::clear_DANGER_(&mut db).await.unwrap();
+
+        let quote = Quote {
+            order_uid: Default::default(),
+            gas_amount: 1.,
+            gas_price: 2.,
+            sell_token_price: 3.,
+            sell_amount: 4.into(),
+            buy_amount: 5.into(),
+            solver: ByteArray([1; 20]),
+            verified: false,
+            metadata: Default::default(),
+        };
+
+        // insert quote with verified and metadata fields stored as NULL
+        insert_quote_and_update_on_conflict(&mut db, &quote)
+            .await
+            .unwrap();
+
+        let quote_ = read_quote(&mut db, &quote.order_uid)
+            .await
+            .unwrap()
+            .unwrap();
+        assert_eq!(quote, quote_);
+    }
 }

crates/database/src/quotes.rs

@@ -34,6 +34,8 @@ pub struct Quote {
     pub expiration_timestamp: DateTime<Utc>,
     pub quote_kind: QuoteKind,
     pub solver: Address,
+    pub verified: bool,
+    pub metadata: serde_json::Value,
 }
 
 /// Stores the quote and returns the id. The id of the quote parameter is not
@@ -51,9 +53,11 @@ INSERT INTO quotes (
     order_kind,
     expiration_timestamp,
     quote_kind,
-    solver
+    solver,
+    verified,
+    metadata
 )
-VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)
+VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)
 RETURNING id
     "#;
     let (id,) = sqlx::query_as(QUERY)
@@ -68,6 +72,8 @@ RETURNING id
         .bind(quote.expiration_timestamp)
         .bind(&quote.quote_kind)
         .bind(quote.solver)
+        .bind(quote.verified)
+        .bind(&quote.metadata)
         .fetch_one(ex)
         .await?;
     Ok(id)
@@ -181,6 +187,8 @@ mod tests {
             expiration_timestamp: now,
             quote_kind: QuoteKind::Standard,
             solver: ByteArray([1; 20]),
+            verified: false,
+            metadata: Default::default(),
         };
         let id = save(&mut db, &quote).await.unwrap();
         quote.id = id;
@@ -214,6 +222,8 @@ mod tests {
             expiration_timestamp: now,
             quote_kind: QuoteKind::Standard,
             solver: ByteArray([1; 20]),
+            verified: false,
+            metadata: Default::default(),
         };
 
         let token_b = ByteArray([2; 20]);
@@ -230,6 +240,8 @@ mod tests {
             expiration_timestamp: now,
             quote_kind: QuoteKind::Standard,
             solver: ByteArray([2; 20]),
+            verified: false,
+            metadata: Default::default(),
         };
 
         // Save two measurements for token_a
@@ -401,6 +413,8 @@ mod tests {
                 expiration_timestamp: now,
                 quote_kind: QuoteKind::Eip1271OnchainOrder,
                 solver: ByteArray([1; 20]),
+                verified: false,
+                metadata: Default::default(),
             };
             let id = save(&mut db, &quote).await.unwrap();
             quote.id = id;
@@ -422,4 +436,48 @@ mod tests {
         search_a.quote_kind = QuoteKind::Standard;
         assert_eq!(find(&mut db, &search_a).await.unwrap(), None,);
     }
+
+    #[tokio::test]
+    #[ignore]
+    async fn postgres_insert_quote_metadata() {
+        let mut db = PgConnection::connect("postgresql://").await.unwrap();
+        let mut db = db.begin().await.unwrap();
+        crate::clear_DANGER_(&mut db).await.unwrap();
+
+        let metadata: serde_json::Value = serde_json::from_str(
+            r#"{ "version":"1.0", "interactions": [ {
+                "target": "0x0102030405060708091011121314151617181920",
+                "value": "1",
+                "callData": "0x0A0B0C102030"
+            },{
+            "target": "0xFF02030405060708091011121314151617181920",
+            "value": "2",
+            "callData": "0xFF0B0C102030"
+            }]
+        }"#,
+        )
+        .unwrap();
+
+        let quote = Quote {
+            id: Default::default(),
+            sell_token: ByteArray([1; 20]),
+            buy_token: ByteArray([2; 20]),
+            sell_amount: 3.into(),
+            buy_amount: 4.into(),
+            gas_amount: 5.,
+            gas_price: 6.,
+            sell_token_price: 7.,
+            order_kind: OrderKind::Sell,
+            expiration_timestamp: low_precision_now(),
+            quote_kind: QuoteKind::Standard,
+            solver: ByteArray([1; 20]),
+            verified: false,
+            metadata: metadata.clone(),
+        };
+        // store quote in database
+        let id = save(&mut db, &quote).await.unwrap();
+
+        let stored_quote = get(&mut db, id).await.unwrap().unwrap();
+        assert_eq!(stored_quote.metadata, metadata);
+    }
 }

crates/e2e/tests/e2e/database.rs

@@ -19,6 +19,17 @@ pub async fn events_of_order(db: &Db, uid: &OrderUid) -> Vec<order_events::Order
         .unwrap()
 }
 
+/// Returns quote.
+pub async fn quote_metadata(db: &Db, quote_id: i64) -> Option<(serde_json::Value,)> {
+    const QUERY: &str = "SELECT metadata FROM quotes WHERE id = $1";
+    let mut db = db.acquire().await.unwrap();
+    sqlx::query_as(QUERY)
+        .bind(quote_id)
+        .fetch_optional(db.deref_mut())
+        .await
+        .unwrap()
+}
+
 #[allow(dead_code)]
 #[derive(Clone, Debug, sqlx::FromRow)]
 pub struct AuctionTransaction {

crates/e2e/tests/e2e/main.rs

@@ -21,6 +21,7 @@ mod order_cancellation;
 mod partial_fill;
 mod partially_fillable_balance;
 mod partially_fillable_pool;
+mod place_order_with_quote;
 mod protocol_fee;
 mod quote_verification;
 mod quoting;