[backup-service][commtest] implement integration tests for uploading …

…User Keys and User Data separately

Summary:
Part of [ENG-9673](https://linear.app/comm/issue/ENG-9673/implement-backupsuser-data).

Tracked in [ENG-9650](https://linear.app/comm/issue/ENG-9650/implement-integration-tests-to-new-backup-service-client-api).

I think I covered most of the cases, this is described in the code comments.

The endpoint to upload both User Keys and User Data at once is tested in `backup_integration_test`.

If anyone has any ideas about what test case can be added, I would be happy to do it.

Depends on D13878

Test Plan: Run tests

Reviewers: varun, bartek, tomek, angelika

Reviewed By: bartek

Subscribers: ashoat, tomek

Differential Revision: https://phab.comm.dev/D13879

services/commtest/src/backup/backup_utils.rs

@@ -5,10 +5,13 @@ use backup_client::{BackupData, Error as BackupClientError};
 use bytesize::ByteSize;
 use comm_lib::auth::UserIdentity;
 use comm_lib::backup::UploadLogRequest;
+use rand::Rng;
 use reqwest::StatusCode;
 use uuid::Uuid;
 
-pub fn generate_backup_data(predefined_byte_value: u8) -> BackupData {
+pub fn generate_backup_data(predefined_byte_value: Option<u8>) -> BackupData {
+  let predefined_byte_value =
+    predefined_byte_value.unwrap_or(rand::thread_rng().gen::<u8>());
   BackupData {
     backup_id: Uuid::new_v4().to_string(),
     user_keys: Some(generate_stable_nbytes(
@@ -60,7 +63,7 @@ pub fn generate_backup_data_with_logs(
   predefined_byte_values
     .into_iter()
     .map(|byte_value| {
-      let backup_data = generate_backup_data(byte_value);
+      let backup_data = generate_backup_data(Some(byte_value));
       let log_data = generate_log_data(&backup_data.backup_id, byte_value);
       (backup_data, log_data)
     })

services/commtest/tests/backup_performance_test.rs

@@ -26,7 +26,7 @@ async fn backup_performance_test() -> Result<(), Error> {
   );
 
   let backup_data: Vec<_> = (0..number_of_threads)
-    .map(|i| generate_backup_data(i as u8))
+    .map(|i| generate_backup_data(Some(i as u8)))
     .collect();
 
   let device_info_1 = register_user_device(None, Some(DeviceType::Ios)).await;

services/commtest/tests/backup_upload_test.rs

@@ -0,0 +1,278 @@
+use backup_client::{
+  BackupClient, BackupData, BackupDescriptor, RequestedData,
+};
+use comm_lib::backup::LatestBackupInfoResponse;
+use commtest::backup::backup_utils::{
+  assert_reqwest_error, create_user_identity, generate_backup_data,
+};
+use commtest::identity::device::register_user_device;
+use commtest::{service_addr, tools::Error};
+use grpc_clients::identity::DeviceType;
+use reqwest::StatusCode;
+
+#[tokio::test]
+async fn backup_upload_user_keys() -> Result<(), Error> {
+  let backup_client = BackupClient::new(service_addr::BACKUP_SERVICE_HTTP)?;
+
+  let device_info = register_user_device(None, Some(DeviceType::Ios)).await;
+  let user_identity = create_user_identity(device_info.clone());
+
+  let backup_data = BackupData {
+    user_data: None,
+    ..generate_backup_data(None)
+  };
+
+  // Upload backup (User Keys)
+  backup_client
+    .upload_backup(&user_identity, backup_data.clone())
+    .await?;
+
+  // Test User Keys download
+  let backup_descriptor = BackupDescriptor::BackupID {
+    backup_id: backup_data.backup_id.clone(),
+    user_identity: user_identity.clone(),
+  };
+  let user_keys = backup_client
+    .download_backup_data(&backup_descriptor, RequestedData::UserKeys)
+    .await?;
+
+  assert_eq!(Some(user_keys), backup_data.user_keys);
+
+  // Test latest backup lookup without User Data
+  let latest_backup_descriptor = BackupDescriptor::Latest {
+    user_identifier: device_info.username,
+  };
+  let backup_info_response = backup_client
+    .download_backup_data(&latest_backup_descriptor, RequestedData::BackupInfo)
+    .await?;
+  let response: LatestBackupInfoResponse =
+    serde_json::from_slice(&backup_info_response)?;
+
+  assert_eq!(response.backup_id, backup_data.backup_id);
+  assert_eq!(response.user_id, device_info.user_id);
+
+  let user_keys = backup_client
+    .download_backup_data(&latest_backup_descriptor, RequestedData::UserKeys)
+    .await?;
+  assert_eq!(Some(user_keys), backup_data.user_keys);
+
+  // Test backup cleanup for User Keys only
+  let new_backup_data = BackupData {
+    user_data: None,
+    ..generate_backup_data(None)
+  };
+
+  backup_client
+    .upload_backup(&user_identity, new_backup_data.clone())
+    .await?;
+
+  // Test Data download for old `backup_id` -> should be not found
+  let user_data_response = backup_client
+    .download_backup_data(&backup_descriptor, RequestedData::UserData)
+    .await;
+  let user_keys_response = backup_client
+    .download_backup_data(&backup_descriptor, RequestedData::UserKeys)
+    .await;
+
+  assert_reqwest_error(user_data_response, StatusCode::NOT_FOUND);
+  assert_reqwest_error(user_keys_response, StatusCode::NOT_FOUND);
+
+  Ok(())
+}
+
+#[tokio::test]
+async fn backup_upload_the_same_user_keys() -> Result<(), Error> {
+  let backup_client = BackupClient::new(service_addr::BACKUP_SERVICE_HTTP)?;
+
+  let device_info = register_user_device(None, Some(DeviceType::Ios)).await;
+  let user_identity = create_user_identity(device_info);
+
+  let backup_data = BackupData {
+    user_data: None,
+    ..generate_backup_data(None)
+  };
+
+  // Upload backup twice (User Keys only)
+  backup_client
+    .upload_backup(&user_identity, backup_data.clone())
+    .await?;
+
+  backup_client
+    .upload_backup(&user_identity, backup_data.clone())
+    .await?;
+
+  // Test User Keys download
+  let backup_descriptor = BackupDescriptor::BackupID {
+    backup_id: backup_data.backup_id.clone(),
+    user_identity: user_identity.clone(),
+  };
+  let user_keys = backup_client
+    .download_backup_data(&backup_descriptor, RequestedData::UserKeys)
+    .await?;
+
+  assert_eq!(Some(user_keys), backup_data.user_keys);
+
+  Ok(())
+}
+
+#[tokio::test]
+async fn backup_upload_user_data_without_user_keys() -> Result<(), Error> {
+  let backup_client = BackupClient::new(service_addr::BACKUP_SERVICE_HTTP)?;
+
+  let device_info = register_user_device(None, Some(DeviceType::Ios)).await;
+  let user_identity = create_user_identity(device_info);
+
+  let backup_data = BackupData {
+    user_keys: None,
+    ..generate_backup_data(None)
+  };
+
+  // Upload backup (User Data) -> should fail,
+  // there is no corresponding User Keys
+  let response = backup_client
+    .upload_backup(&user_identity, backup_data.clone())
+    .await;
+
+  assert_reqwest_error(response, StatusCode::NOT_FOUND);
+
+  Ok(())
+}
+
+#[tokio::test]
+async fn backup_upload_user_keys_and_user_data() -> Result<(), Error> {
+  let backup_client = BackupClient::new(service_addr::BACKUP_SERVICE_HTTP)?;
+
+  let device_info = register_user_device(None, Some(DeviceType::Ios)).await;
+  let user_identity = create_user_identity(device_info.clone());
+
+  let backup_data = generate_backup_data(None);
+  let user_keys = BackupData {
+    user_data: None,
+    ..backup_data.clone()
+  };
+  let user_data = BackupData {
+    user_keys: None,
+    ..backup_data.clone()
+  };
+
+  // Upload backups (User Keys and User Data)
+  backup_client
+    .upload_backup(&user_identity, user_keys.clone())
+    .await?;
+
+  backup_client
+    .upload_backup(&user_identity, user_data.clone())
+    .await?;
+
+  // Test User Keys download
+  let backup_descriptor = BackupDescriptor::BackupID {
+    backup_id: backup_data.backup_id.clone(),
+    user_identity: user_identity.clone(),
+  };
+  let user_keys = backup_client
+    .download_backup_data(&backup_descriptor, RequestedData::UserKeys)
+    .await?;
+
+  assert_eq!(Some(user_keys), backup_data.user_keys);
+
+  // Test User Data download
+  let user_data = backup_client
+    .download_backup_data(&backup_descriptor, RequestedData::UserData)
+    .await?;
+  assert_eq!(Some(user_data), backup_data.user_data);
+
+  // Upload new User Data
+  let new_backup_data = generate_backup_data(None);
+  let new_user_data = BackupData {
+    // Important we using the same `backup_id`
+    backup_id: backup_data.backup_id.clone(),
+    user_keys: None,
+    user_data: new_backup_data.user_data.clone(),
+    attachments: new_backup_data.attachments,
+    siwe_backup_msg: None,
+  };
+
+  backup_client
+    .upload_backup(&user_identity, new_user_data.clone())
+    .await?;
+
+  // Test User Keys download again -> should remain unchanged
+  let backup_descriptor = BackupDescriptor::BackupID {
+    backup_id: new_user_data.backup_id.clone(),
+    user_identity: user_identity.clone(),
+  };
+  let user_keys = backup_client
+    .download_backup_data(&backup_descriptor, RequestedData::UserKeys)
+    .await?;
+
+  assert_eq!(Some(user_keys), backup_data.user_keys);
+
+  // Test User Data download, should be updated
+  let user_data = backup_client
+    .download_backup_data(&backup_descriptor, RequestedData::UserData)
+    .await?;
+
+  assert_eq!(Some(user_data), new_backup_data.user_data);
+
+  // Upload new User Keys -> should override User Keys and keep User Data unchanged
+  let new_user_keys = BackupData {
+    user_data: None,
+    ..generate_backup_data(None)
+  };
+
+  backup_client
+    .upload_backup(&user_identity, new_user_keys.clone())
+    .await?;
+
+  // Test latest backup -> should return newest `backup_id`
+  let latest_backup_descriptor = BackupDescriptor::Latest {
+    user_identifier: device_info.username,
+  };
+  let backup_info_response = backup_client
+    .download_backup_data(&latest_backup_descriptor, RequestedData::BackupInfo)
+    .await?;
+  let response: LatestBackupInfoResponse =
+    serde_json::from_slice(&backup_info_response)?;
+
+  assert_eq!(response.backup_id, new_user_keys.backup_id);
+  assert_eq!(response.user_id, device_info.user_id);
+
+  // Test User Keys download -> should be updated
+  let backup_descriptor = BackupDescriptor::BackupID {
+    backup_id: new_user_keys.backup_id.clone(),
+    user_identity: user_identity.clone(),
+  };
+  let user_keys = backup_client
+    .download_backup_data(&backup_descriptor, RequestedData::UserKeys)
+    .await?;
+
+  assert_eq!(Some(user_keys), new_user_keys.user_keys);
+
+  // Test User Data download -> should be the old one
+  let backup_descriptor = BackupDescriptor::BackupID {
+    backup_id: new_user_keys.backup_id.clone(),
+    user_identity: user_identity.clone(),
+  };
+  let user_data = backup_client
+    .download_backup_data(&backup_descriptor, RequestedData::UserData)
+    .await?;
+
+  assert_eq!(Some(user_data), new_backup_data.user_data);
+
+  // Test Data download for old `backup_id` -> should be not found
+  let removed_backup_descriptor = BackupDescriptor::BackupID {
+    backup_id: backup_data.backup_id.clone(),
+    user_identity: user_identity.clone(),
+  };
+  let user_data_response = backup_client
+    .download_backup_data(&removed_backup_descriptor, RequestedData::UserData)
+    .await;
+  let user_keys_response = backup_client
+    .download_backup_data(&removed_backup_descriptor, RequestedData::UserKeys)
+    .await;
+
+  assert_reqwest_error(user_data_response, StatusCode::NOT_FOUND);
+  assert_reqwest_error(user_keys_response, StatusCode::NOT_FOUND);
+
+  Ok(())
+}