From 6dbee0230acc4f81df580f28c0ac1b7a3cd3640b Mon Sep 17 00:00:00 2001 From: Facundo Lopez Janza <56484504+Linker44@users.noreply.github.com> Date: Fri, 13 Dec 2024 12:16:13 -0300 Subject: [PATCH] Improve testing for stripe and hubspot (#5584) --- data/saas/config/hubspot_config.yml | 9 +- data/saas/dataset/hubspot_dataset.yml | 5 + .../hubspot_request_overrides.py | 48 - tests/fixtures/saas/hubspot_fixtures.py | 102 +- tests/fixtures/saas/stripe_fixtures.py | 584 ++++--- .../saas/connector_runner.py | 5 + .../saas/test_hubspot_task.py | 309 ++-- .../saas/test_stripe_task.py | 1393 +++-------------- .../test_saas_privacy_requests.py | 1 + 9 files changed, 764 insertions(+), 1692 deletions(-) delete mode 100644 src/fides/api/service/saas_request/override_implementations/hubspot_request_overrides.py diff --git a/data/saas/config/hubspot_config.yml b/data/saas/config/hubspot_config.yml index cbb57414b7..e38c168c35 100644 --- a/data/saas/config/hubspot_config.yml +++ b/data/saas/config/hubspot_config.yml @@ -4,7 +4,7 @@ saas_config: type: hubspot description: A sample schema representing the HubSpot connector for Fides user_guide: https://docs.ethyca.com/user-guides/integrations/saas-integrations/hubspot - version: 0.0.6 + version: 0.0.7 connector_params: - name: domain @@ -56,7 +56,12 @@ saas_config: source: body path: paging.next.link update: - request_override: hubspot_contacts_update + path: /crm/v3/objects/contacts/ + method: PATCH + body: | + { + + } param_values: - name: contactId references: diff --git a/data/saas/dataset/hubspot_dataset.yml b/data/saas/dataset/hubspot_dataset.yml index 94f0bf43f9..e82c3452af 100644 --- a/data/saas/dataset/hubspot_dataset.yml +++ b/data/saas/dataset/hubspot_dataset.yml @@ -26,6 +26,11 @@ dataset: data_categories: [user.contact.email] fidesops_meta: data_type: string + masking_strategy_override: + strategy: random_string_rewrite + configuration: + format_preservation: + suffix: "@company.com" - name: firstname data_categories: [user.name] fidesops_meta: diff --git a/src/fides/api/service/saas_request/override_implementations/hubspot_request_overrides.py b/src/fides/api/service/saas_request/override_implementations/hubspot_request_overrides.py deleted file mode 100644 index ba08164b3f..0000000000 --- a/src/fides/api/service/saas_request/override_implementations/hubspot_request_overrides.py +++ /dev/null @@ -1,48 +0,0 @@ -from json import dumps -from typing import Any, Dict, List - -from fides.api.models.policy import Policy -from fides.api.models.privacy_request import PrivacyRequest -from fides.api.schemas.saas.shared_schemas import HTTPMethod, SaaSRequestParams -from fides.api.service.connectors.saas.authenticated_client import AuthenticatedClient -from fides.api.service.saas_request.saas_request_override_factory import ( - SaaSRequestType, - register, -) -from fides.api.util.saas_util import PRIVACY_REQUEST_ID - - -@register("hubspot_contacts_update", [SaaSRequestType.UPDATE]) -def hubspot_contacts_update( - client: AuthenticatedClient, - param_values_per_row: List[Dict[str, Any]], - policy: Policy, - privacy_request: PrivacyRequest, - secrets: Dict[str, Any], -) -> int: - rows_updated = 0 - # each update_params dict correspond to a record that needs to be updated - for row_param_values in param_values_per_row: - # check if the privacy_request targeted emails for erasure, - # if so rewrite with a format that can be accepted by hubspot - # regardless of the masking strategy in use - masked_object_fields = row_param_values["masked_object_fields"] - - if "email" in masked_object_fields["properties"]: - privacy_request_id = row_param_values[PRIVACY_REQUEST_ID] - masked_object_fields["properties"][ - "email" - ] = f"{privacy_request_id}@company.com" - - update_body = dumps(masked_object_fields) - contact_id = row_param_values["contactId"] - client.send( - SaaSRequestParams( - method=HTTPMethod.PATCH, - headers={"Content-Type": "application/json"}, - path=f"/crm/v3/objects/contacts/{contact_id}", - body=update_body, - ) - ) - rows_updated += 1 - return rows_updated diff --git a/tests/fixtures/saas/hubspot_fixtures.py b/tests/fixtures/saas/hubspot_fixtures.py index a90afffe5d..6df14261d8 100644 --- a/tests/fixtures/saas/hubspot_fixtures.py +++ b/tests/fixtures/saas/hubspot_fixtures.py @@ -1,3 +1,4 @@ +from time import sleep from typing import Any, Dict, Generator import pydash @@ -5,7 +6,6 @@ import requests from sqlalchemy.orm import Session -from fides.api.cryptography import cryptographic_util from fides.api.models.connectionconfig import ( AccessLevel, ConnectionConfig, @@ -17,6 +17,10 @@ load_config_with_replacement, load_dataset_with_replacement, ) +from tests.ops.integration_tests.saas.connector_runner import ( + ConnectorRunner, + generate_random_email, +) from tests.ops.test_helpers.saas_test_utils import poll_for_existence from tests.ops.test_helpers.vault_client import get_secrets @@ -33,15 +37,8 @@ def hubspot_secrets(saas_config): @pytest.fixture(scope="function") -def hubspot_identity_email(saas_config): - return ( - pydash.get(saas_config, "hubspot.identity_email") or secrets["identity_email"] - ) - - -@pytest.fixture(scope="session") -def hubspot_erasure_identity_email(): - return f"{cryptographic_util.generate_secure_random_string(13)}@email.com" +def hubspot_identity_email(): + return generate_random_email() @pytest.fixture @@ -115,8 +112,7 @@ class HubspotTestClient: headers: object = {} base_url: str = "" - def __init__(self, connection_config_hubspot: ConnectionConfig): - hubspot_secrets = connection_config_hubspot.secrets + def __init__(self, hubspot_secrets: Dict[str, str]): self.headers = { "Authorization": f"Bearer {hubspot_secrets['private_app_token']}", } @@ -192,6 +188,13 @@ def delete_contact(self, contact_id: str) -> requests.Response: ) return contact_response + def delete_user(self, user_id: str) -> requests.Response: + user_response: requests.Response = requests.delete( + url=f"{self.base_url}/settings/v3/users/{user_id}", + headers=self.headers, + ) + return user_response + def get_email_subscriptions(self, email: str) -> requests.Response: email_subscriptions: requests.Response = requests.get( url=f"{self.base_url}/communication-preferences/v3/status/email/{email}", @@ -202,9 +205,9 @@ def get_email_subscriptions(self, email: str) -> requests.Response: @pytest.fixture(scope="function") def hubspot_test_client( - connection_config_hubspot: HubspotTestClient, + hubspot_secrets, ) -> Generator: - test_client = HubspotTestClient(connection_config_hubspot=connection_config_hubspot) + test_client = HubspotTestClient(hubspot_secrets) yield test_client @@ -236,26 +239,14 @@ def user_exists(user_id: str, hubspot_test_client: HubspotTestClient) -> Any: return user_body -@pytest.fixture(scope="function") -def hubspot_erasure_data( - hubspot_test_client: HubspotTestClient, - hubspot_erasure_identity_email: str, -) -> Generator: - """ - Gets the current value of the resource and restores it after the test is complete. - Used for erasure tests. - """ +def create_hubspot_data(test_client, email): # create contact - contacts_response = hubspot_test_client.create_contact( - email=hubspot_erasure_identity_email - ) + contacts_response = test_client.create_contact(email=email) contacts_body = contacts_response.json() contact_id = contacts_body["id"] # create user - users_response = hubspot_test_client.create_user( - email=hubspot_erasure_identity_email - ) + users_response = test_client.create_user(email=email) users_body = users_response.json() user_id = users_body["id"] @@ -266,7 +257,7 @@ def hubspot_erasure_data( ) poll_for_existence( _contact_exists, - (contact_id, hubspot_erasure_identity_email, hubspot_test_client), + (contact_id, email, test_client), error_message=error_message, interval=60, ) @@ -274,14 +265,32 @@ def hubspot_erasure_data( error_message = f"User with user id {user_id} could not be added to Hubspot" poll_for_existence( user_exists, - (user_id, hubspot_test_client), + (user_id, test_client), error_message=error_message, ) + sleep(3) + return contact_id, user_id + + +@pytest.fixture(scope="function") +def hubspot_data( + hubspot_test_client: HubspotTestClient, + hubspot_identity_email: str, +) -> Generator: + + contact_id, user_id = create_hubspot_data( + hubspot_test_client, email=hubspot_identity_email + ) + random_email = generate_random_email() + random_contact_id, random_user_id = create_hubspot_data( + hubspot_test_client, email=random_email + ) yield contact_id, user_id # delete contact hubspot_test_client.delete_contact(contact_id=contact_id) + hubspot_test_client.delete_user(user_id=user_id) # verify contact is deleted error_message = ( @@ -289,7 +298,36 @@ def hubspot_erasure_data( ) poll_for_existence( _contact_exists, - (contact_id, hubspot_erasure_identity_email, hubspot_test_client), + (contact_id, hubspot_identity_email, hubspot_test_client), error_message=error_message, existence_desired=False, ) + + # delete random contact + hubspot_test_client.delete_contact(contact_id=random_contact_id) + hubspot_test_client.delete_user(user_id=random_user_id) + + # verify random contact is deleted + error_message = ( + f"Contact with contact id {random_contact_id} could not be deleted from Hubspot" + ) + poll_for_existence( + _contact_exists, + (random_contact_id, random_email, hubspot_test_client), + error_message=error_message, + existence_desired=False, + ) + + +@pytest.fixture +def hubspot_runner( + db, + cache, + hubspot_secrets, +) -> ConnectorRunner: + return ConnectorRunner( + db, + cache, + "hubspot", + hubspot_secrets, + ) diff --git a/tests/fixtures/saas/stripe_fixtures.py b/tests/fixtures/saas/stripe_fixtures.py index 7ff6bb27dc..a0232c7aeb 100644 --- a/tests/fixtures/saas/stripe_fixtures.py +++ b/tests/fixtures/saas/stripe_fixtures.py @@ -1,4 +1,5 @@ from datetime import datetime +from time import sleep from typing import Any, Dict, Generator import pydash @@ -20,6 +21,11 @@ load_config_with_replacement, load_dataset_with_replacement, ) +from tests.ops.integration_tests.saas.connector_runner import ( + ConnectorRunner, + generate_random_email, + generate_random_phone_number, +) from tests.ops.test_helpers.vault_client import get_secrets secrets = get_secrets("stripe") @@ -35,22 +41,19 @@ def stripe_secrets(saas_config): } -@pytest.fixture(scope="session") -def stripe_identity_email(saas_config): - return pydash.get(saas_config, "stripe.identity_email") or secrets["identity_email"] +@pytest.fixture +def stripe_identity_email(): + return generate_random_email() -@pytest.fixture(scope="session") -def stripe_identity_phone_number(saas_config): - return ( - pydash.get(saas_config, "stripe.identity_phone_number") - or secrets["identity_phone_number"] - ) +@pytest.fixture +def stripe_identity_phone_number(): + return generate_random_phone_number() -@pytest.fixture(scope="session") +@pytest.fixture def stripe_erasure_identity_email(): - return f"{cryptographic_util.generate_secure_random_string(13)}@email.com" + return generate_random_email() @pytest.fixture @@ -115,18 +118,326 @@ def stripe_dataset_config( ctl_dataset.delete(db=db) +class StripeTestClient: + + def __init__(self, stripe_secrets: Dict[str, Any]): + self.base_url = f"https://{stripe_secrets['domain']}" + self.headers = { + "Content-Type": "application/x-www-form-urlencoded", + "Authorization": f"Bearer {stripe_secrets['api_key']}", + } + + def create_customer(self, customer_data: Dict[str, Any]) -> Dict[str, Any]: + + response = requests.post( + url=f"{self.base_url}/v1/customers", + headers=self.headers, + data=multidimensional_urlencode(customer_data), + ) + assert response.ok + return response.json() + + def create_dispute(self, customer_id, customer_data): + # create dispute by adding a fraudulent card and charging it + response = requests.post( + url=f"{self.base_url}/v1/customers/{customer_id}", + headers=self.headers, + data=multidimensional_urlencode({"source": "tok_createDispute"}), + ) + assert response.ok + card = response.json()["sources"]["data"][0] + card_id = card["id"] + + # update card name to have something to mask + response = requests.post( + url=f"{self.base_url}/v1/customers/{customer_id}/sources/{card_id}", + headers=self.headers, + data=multidimensional_urlencode({"name": customer_data["name"]}), + ) + assert response.ok + + # charge + response = requests.post( + url=f"{self.base_url}/v1/charges", + headers=self.headers, + data=multidimensional_urlencode( + { + "customer": customer_id, + "source": card_id, + "amount": 1000, + "currency": "usd", + } + ), + ) + assert response.ok + + # charge + response = requests.post( + url=f"{self.base_url}/v1/charges", + headers=self.headers, + data=multidimensional_urlencode( + { + "customer": customer_id, + "source": card_id, + "amount": 1000, + "currency": "usd", + } + ), + ) + assert response.ok + + return card_id + + def create_bank_account(self, customer_id, customer_data): + response = requests.post( + url=f"{self.base_url}/v1/customers/{customer_id}/sources", + headers=self.headers, + data=multidimensional_urlencode({"source": "btok_us_verified"}), + ) + assert response.ok + bank_account = response.json() + bank_account_id = bank_account["id"] + # update bank account holder name to have something to mask + response = requests.post( + url=f"{self.base_url}/v1/customers/{customer_id}/sources/{bank_account_id}", + headers=self.headers, + data=multidimensional_urlencode( + {"account_holder_name": customer_data["name"]} + ), + ) + assert response.ok + + def create_invoice(self, customer_id): + # invoice item + response = requests.post( + url=f"{self.base_url}/v1/invoiceitems", + headers=self.headers, + params={"customer": customer_id}, + data=multidimensional_urlencode({"amount": 200, "currency": "usd"}), + ) + assert response.ok + + # pulls in the previously created invoice item automatically to create the invoice + response = requests.post( + url=f"{self.base_url}/v1/invoices", + headers=self.headers, + params={"customer": customer_id}, + ) + assert response.ok + invoice = response.json() + invoice_id = invoice["id"] + + # finalize invoice + response = requests.post( + url=f"{self.base_url}/v1/invoices/{invoice_id}/finalize", + headers=self.headers, + ) + assert response.ok + + return invoice + + def create_credit_note(self, invoice): + response = requests.post( + url=f"{self.base_url}/v1/credit_notes", + headers=self.headers, + params={"invoice": invoice["id"]}, + data=multidimensional_urlencode( + { + "lines[0]": { + "type": "invoice_line_item", + "invoice_line_item": invoice["lines"]["data"][0]["id"], + "quantity": 1, + } + } + ), + ) + assert response.ok + + def create_balance_transaction(self, customer_id): + response = requests.post( + url=f"{self.base_url}/v1/customers/{customer_id}/balance_transactions", + headers=self.headers, + data=multidimensional_urlencode({"amount": -500, "currency": "usd"}), + ) + assert response.ok + + def create_payment_intent(self, customer_id): + response = requests.post( + url=f"{self.base_url}/v1/payment_intents", + headers=self.headers, + data=multidimensional_urlencode( + { + "customer": customer_id, + "amount": 2000, + "currency": "usd", + "payment_method_types[]": "card", + "confirm": True, + } + ), + ) + assert response.ok + + response = requests.post( + url=f"{self.base_url}/v1/setup_intents", + params={"customer": customer_id, "payment_method_types[]": "card"}, + headers=self.headers, + ) + assert response.ok + + def create_payment_method(self, customer_id, customer_name): + response = requests.post( + url=f"{self.base_url}/v1/payment_methods", + headers=self.headers, + data=multidimensional_urlencode( + { + "type": "card", + "card": { + "number": 4242424242424242, + "exp_month": 4, + "exp_year": datetime.today().year + 1, + "cvc": 314, + }, + "billing_details": {"name": customer_name}, + } + ), + ) + assert response.ok + payment_method = response.json() + payment_method_id = payment_method["id"] + + response = requests.post( + url=f"{self.base_url}/v1/payment_methods/{payment_method_id}/attach", + params={"customer": customer_id}, + headers=self.headers, + ) + assert response.ok + + def create_subscription(self, customer_id): + response = requests.get( + url=f"{self.base_url}/v1/prices", + params={"type": "recurring"}, + headers=self.headers, + ) + assert response.ok + price = response.json()["data"][0] + price_id = price["id"] + + response = requests.post( + url=f"{self.base_url}/v1/subscriptions", + headers=self.headers, + data=multidimensional_urlencode( + {"customer": customer_id, "items[0]": {"price": price_id}} + ), + ) + assert response.ok + subscription = response.json() + return subscription["id"] + + def create_tax(self, customer_id): + # tax id + response = requests.post( + url=f"{self.base_url}/v1/customers/{customer_id}/tax_ids", + headers=self.headers, + data=multidimensional_urlencode({"type": "us_ein", "value": "000000000"}), + ) + assert response.ok + tax = response.json() + return tax["id"] + + def delete_customer(self, customer_id): + requests.delete( + url=f"{self.base_url}/v1/customers/{customer_id}", headers=self.headers + ) + + def delete_card(self, customer_id, card_id): + requests.get( + url=f"{self.base_url}/v1/customers/{customer_id}/sources/{card_id}", + headers=self.headers, + ) + + def delete_tax_id(self, customer_id, tax_id): + requests.get( + url=f"{self.base_url}/v1/customers/{customer_id}/tax_ids/{tax_id}", + headers=self.headers, + ) + + def delete_subscription(self, subscription_id): + requests.get( + url=f"{self.base_url}/v1/subscriptions/{subscription_id}", + headers=self.headers, + ) + + def get_customer(self, email): + response = requests.get( + url=f"{self.base_url}/v1/customers", + headers=self.headers, + params={"email": email}, + ) + customer = response.json()["data"][0] + return customer + + def get_card(self, customer_id): + response = requests.get( + url=f"{self.base_url}/v1/customers/{customer_id}/sources", + headers=self.headers, + params={"object": "card"}, + ) + cards = response.json()["data"] + return cards + + def get_payment_method(self, customer_id): + response = requests.get( + url=f"{self.base_url}/v1/customers/{customer_id}/payment_methods", + headers=self.headers, + params={"type": "card"}, + ) + payment_methods = response.json()["data"] + return payment_methods + + def get_bank_account(self, customer_id): + response = requests.get( + url=f"{self.base_url}/v1/customers/{customer_id}/sources", + headers=self.headers, + params={"object": "bank_account"}, + ) + bank_account = response.json()["data"][0] + return bank_account + + def get_tax_ids(self, customer_id): + response = requests.get( + url=f"{self.base_url}/v1/customers/{customer_id}/tax_ids", + headers=self.headers, + ) + tax_ids = response.json()["data"] + return tax_ids + + def get_invoice_items(self, customer_id): + response = requests.get( + url=f"{self.base_url}/v1/invoiceitems", + headers=self.headers, + params={"customer": {customer_id}}, + ) + invoice_item = response.json()["data"] + return invoice_item + + def get_subscription(self, customer_id): + response = requests.get( + url=f"{self.base_url}/v1/customers/{customer_id}/subscriptions", + headers=self.headers, + ) + subscriptions = response.json()["data"] + return subscriptions + + @pytest.fixture(scope="function") -def stripe_create_erasure_data( - stripe_connection_config: ConnectionConfig, stripe_erasure_identity_email +def stripe_test_client( + stripe_secrets, ) -> Generator: - stripe_secrets = stripe_connection_config.secrets + test_client = StripeTestClient(stripe_secrets) + yield test_client - base_url = f"https://{stripe_secrets['domain']}" - headers = { - "Content-Type": "application/x-www-form-urlencoded", - "Authorization": f"Bearer {stripe_secrets['api_key']}", - } +def stripe_generate_data(client, email, phone_number): # customer customer_data = { @@ -140,7 +451,8 @@ def stripe_create_erasure_data( }, "balance": 0, "description": "RTF Test Customer", - "email": stripe_erasure_identity_email, + "email": email, + "phone": phone_number, "name": "Ethyca RTF", "preferred_locales": ["en-US"], "shipping": { @@ -156,197 +468,69 @@ def stripe_create_erasure_data( }, } - response = requests.post( - url=f"{base_url}/v1/customers", - headers=headers, - data=multidimensional_urlencode(customer_data), - ) - assert response.ok - customer = response.json() + customer = client.create_customer(customer_data) + customer_id = customer["id"] - # create dispute by adding a fraudulent card and charging it - response = requests.post( - url=f"{base_url}/v1/customers/{customer['id']}", - headers=headers, - data=multidimensional_urlencode({"source": "tok_createDispute"}), - ) - assert response.ok - card = response.json()["sources"]["data"][0] - card_id = card["id"] - - # update card name to have something to mask - response = requests.post( - url=f"{base_url}/v1/customers/{customer_id}/sources/{card_id}", - headers=headers, - data=multidimensional_urlencode({"name": customer_data["name"]}), - ) - assert response.ok - - # charge - response = requests.post( - url=f"{base_url}/v1/charges", - headers=headers, - data=multidimensional_urlencode( - { - "customer": customer_id, - "source": card_id, - "amount": 1000, - "currency": "usd", - } - ), - ) - assert response.ok + card_id = client.create_dispute(customer_id, customer_data) - # bank account - response = requests.post( - url=f"{base_url}/v1/customers/{customer_id}/sources", - headers=headers, - data=multidimensional_urlencode({"source": "btok_us_verified"}), - ) - assert response.ok - bank_account = response.json() - bank_account_id = bank_account["id"] - # update bank account holder name to have something to mask - response = requests.post( - url=f"{base_url}/v1/customers/{customer_id}/sources/{bank_account_id}", - headers=headers, - data=multidimensional_urlencode({"account_holder_name": customer_data["name"]}), - ) - assert response.ok - - # invoice item - response = requests.post( - url=f"{base_url}/v1/invoiceitems", - headers=headers, - params={"customer": customer_id}, - data=multidimensional_urlencode({"amount": 200, "currency": "usd"}), - ) - assert response.ok + client.create_bank_account(customer_id, customer_data) - # pulls in the previously created invoice item automatically to create the invoice - response = requests.post( - url=f"{base_url}/v1/invoices", - headers=headers, - params={"customer": customer_id}, - ) - assert response.ok - invoice = response.json() - invoice_id = invoice["id"] + invoice = client.create_invoice(customer_id) - # finalize invoice - response = requests.post( - url=f"{base_url}/v1/invoices/{invoice_id}/finalize", headers=headers - ) - assert response.ok - - # credit note - response = requests.post( - url=f"{base_url}/v1/credit_notes", - headers=headers, - params={"invoice": invoice_id}, - data=multidimensional_urlencode( - { - "lines[0]": { - "type": "invoice_line_item", - "invoice_line_item": invoice["lines"]["data"][0]["id"], - "quantity": 1, - } - } - ), - ) - assert response.ok + client.create_credit_note(invoice) - # customer balance transaction - response = requests.post( - url=f"{base_url}/v1/customers/{customer_id}/balance_transactions", - headers=headers, - data=multidimensional_urlencode({"amount": -500, "currency": "usd"}), - ) - assert response.ok - - # payment intent - response = requests.post( - url=f"{base_url}/v1/payment_intents", - headers=headers, - data=multidimensional_urlencode( - { - "customer": customer_id, - "amount": 2000, - "currency": "usd", - "payment_method_types[]": "card", - "confirm": True, - } - ), - ) - assert response.ok - - # create and attach payment method to customer - response = requests.post( - url=f"{base_url}/v1/payment_methods", - headers=headers, - data=multidimensional_urlencode( - { - "type": "card", - "card": { - "number": 4242424242424242, - "exp_month": 4, - "exp_year": datetime.today().year + 1, - "cvc": 314, - }, - "billing_details": {"name": customer_data["name"]}, - } - ), - ) - assert response.ok - payment_method = response.json() - payment_method_id = payment_method["id"] - - response = requests.post( - url=f"{base_url}/v1/payment_methods/{payment_method_id}/attach", - params={"customer": customer_id}, - headers=headers, - ) - assert response.ok + client.create_balance_transaction(customer_id) - # setup intent - response = requests.post( - url=f"{base_url}/v1/setup_intents", - params={"customer": customer_id, "payment_method_types[]": "card"}, - headers=headers, - ) - assert response.ok + client.create_payment_intent(customer_id) + + client.create_payment_method(customer_id, customer_data["name"]) + + subscription_id = client.create_subscription(customer_id) - # get an existing price and use it to create a subscription - response = requests.get( - url=f"{base_url}/v1/prices", - params={"type": "recurring"}, - headers=headers, + tax_id = client.create_tax(customer_id) + + sleep(3) + + return { + "customer_id": customer_id, + "card_id": card_id, + "subscription_id": subscription_id, + "tax_id": tax_id, + } + + +@pytest.fixture(scope="function") +def stripe_create_data( + stripe_test_client: StripeTestClient, + stripe_identity_email: str, + stripe_identity_phone_number: str, +) -> Generator: + customer = stripe_generate_data( + stripe_test_client, stripe_identity_email, stripe_identity_phone_number ) - assert response.ok - price = response.json()["data"][0] - price_id = price["id"] - - response = requests.post( - url=f"{base_url}/v1/subscriptions", - headers=headers, - data=multidimensional_urlencode( - {"customer": customer_id, "items[0]": {"price": price_id}} - ), + random_customer = stripe_generate_data( + stripe_test_client, generate_random_email(), generate_random_phone_number() ) - assert response.ok - # tax id - response = requests.post( - url=f"{base_url}/v1/customers/{customer_id}/tax_ids", - headers=headers, - data=multidimensional_urlencode({"type": "us_ein", "value": "000000000"}), - ) - assert response.ok + yield - yield customer + for data in [customer, random_customer]: + stripe_test_client.delete_customer(data["customer_id"]) + stripe_test_client.delete_card(data["customer_id"], data["card_id"]) + stripe_test_client.delete_subscription(data["subscription_id"]) + stripe_test_client.delete_tax_id(data["customer_id"], data["tax_id"]) - response = requests.delete( - url=f"{base_url}/v1/customers/{customer_id}", headers=headers + +@pytest.fixture +def stripe_runner( + db, + cache, + stripe_secrets, +) -> ConnectorRunner: + return ConnectorRunner( + db, + cache, + "stripe", + stripe_secrets, ) - assert response.ok diff --git a/tests/ops/integration_tests/saas/connector_runner.py b/tests/ops/integration_tests/saas/connector_runner.py index 0f9e6e80e9..81c09005df 100644 --- a/tests/ops/integration_tests/saas/connector_runner.py +++ b/tests/ops/integration_tests/saas/connector_runner.py @@ -8,6 +8,7 @@ from fides.api.graph.config import CollectionAddress, GraphDataset from fides.api.graph.graph import DatasetGraph from fides.api.graph.traversal import Traversal, TraversalNode +from fides.api.models.application_config import ApplicationConfig from fides.api.models.connectionconfig import ( AccessLevel, ConnectionConfig, @@ -168,6 +169,7 @@ async def strict_erasure_request( # store the existing masking_strict value so we can reset it at the end of the test masking_strict = CONFIG.execution.masking_strict CONFIG.execution.masking_strict = True + ApplicationConfig.update_config_set(self.db, CONFIG) access_results, erasure_results = await self._base_erasure_request( access_policy, erasure_policy, identities, privacy_request_id @@ -175,6 +177,7 @@ async def strict_erasure_request( # reset masking_strict value CONFIG.execution.masking_strict = masking_strict + ApplicationConfig.update_config_set(self.db, CONFIG) return access_results, erasure_results async def non_strict_erasure_request( @@ -194,6 +197,7 @@ async def non_strict_erasure_request( # store the existing masking_strict value so we can reset it at the end of the test masking_strict = CONFIG.execution.masking_strict CONFIG.execution.masking_strict = False + ApplicationConfig.update_config_set(self.db, CONFIG) access_results, erasure_results = await self._base_erasure_request( access_policy, @@ -205,6 +209,7 @@ async def non_strict_erasure_request( # reset masking_strict value CONFIG.execution.masking_strict = masking_strict + ApplicationConfig.update_config_set(self.db, CONFIG) return access_results, erasure_results async def old_consent_request( diff --git a/tests/ops/integration_tests/saas/test_hubspot_task.py b/tests/ops/integration_tests/saas/test_hubspot_task.py index c5061ac280..90d2de37bd 100644 --- a/tests/ops/integration_tests/saas/test_hubspot_task.py +++ b/tests/ops/integration_tests/saas/test_hubspot_task.py @@ -1,219 +1,106 @@ import pytest -from fides.api.graph.graph import DatasetGraph -from fides.api.schemas.redis_cache import Identity -from fides.api.service.connectors import get_connector -from fides.api.task.filter_results import filter_data_categories -from fides.api.task.graph_task import get_cached_data_for_erasures -from fides.config import CONFIG -from tests.conftest import access_runner_tester, erasure_runner_tester +from fides.api.models.policy import Policy from tests.fixtures.saas.hubspot_fixtures import HubspotTestClient, user_exists -from tests.ops.graph.graph_test_util import assert_rows_match +from tests.ops.integration_tests.saas.connector_runner import ConnectorRunner from tests.ops.test_helpers.saas_test_utils import poll_for_existence @pytest.mark.integration_saas -def test_hubspot_connection_test(connection_config_hubspot) -> None: - get_connector(connection_config_hubspot).test_connection() - - -@pytest.mark.integration_saas -@pytest.mark.asyncio -@pytest.mark.parametrize( - "dsr_version", - ["use_dsr_3_0", "use_dsr_2_0"], -) -async def test_hubspot_access_request_task( - db, - dsr_version, - request, - policy, - connection_config_hubspot, - dataset_config_hubspot, - hubspot_identity_email, - privacy_request, -) -> None: - """Full access request based on the Hubspot SaaS config""" - request.getfixturevalue(dsr_version) # REQUIRED to test both DSR 3.0 and 2.0 - - identity_attribute = "email" - identity_value = hubspot_identity_email - identity_kwargs = {identity_attribute: identity_value} - identity = Identity(**identity_kwargs) - privacy_request.cache_identity(identity) - - dataset_name = connection_config_hubspot.get_saas_config().fides_key - merged_graph = dataset_config_hubspot.get_graph() - graph = DatasetGraph(merged_graph) - - v = access_runner_tester( - privacy_request, - policy, - graph, - [connection_config_hubspot], - {"email": hubspot_identity_email}, - db, - ) - - assert_rows_match( - v[f"{dataset_name}:contacts"], - min_size=1, - keys=["archived", "createdAt", "id", "properties"], - ) - assert_rows_match( - v[f"{dataset_name}:subscription_preferences"], - min_size=1, - keys=["recipient", "subscriptionStatuses"], - ) - assert_rows_match( - v[f"{dataset_name}:users"], - min_size=1, - keys=["id", "email"], - ) - assert_rows_match( - v[f"{dataset_name}:owners"], - min_size=1, - keys=["id", "email", "lastName", "updatedAt", "firstName", "userId"], - ) - - target_categories = {"user"} - filtered_results = filter_data_categories( - v, - target_categories, - graph, - ) - - assert set(filtered_results.keys()) == { - f"{dataset_name}:contacts", - f"{dataset_name}:subscription_preferences", - f"{dataset_name}:users", - f"{dataset_name}:owners", - } - assert set(filtered_results[f"{dataset_name}:contacts"][0].keys()) == { - "id", - "properties", - } - - results_set = set( - filtered_results[f"{dataset_name}:contacts"][0]["properties"].keys() - ) - assert "lastname" in results_set - assert "firstname" in results_set - assert "email" in results_set - - assert set( - filtered_results[f"{dataset_name}:subscription_preferences"][0].keys() - ) == {"recipient"} - assert ( - filtered_results[f"{dataset_name}:subscription_preferences"][0]["recipient"] - == hubspot_identity_email - ) - assert set(filtered_results[f"{dataset_name}:users"][0].keys()) == {"email", "id"} - assert ( - filtered_results[f"{dataset_name}:users"][0]["email"] == hubspot_identity_email - ) - assert set(filtered_results[f"{dataset_name}:owners"][0].keys()) == { - "email", - "id", - "userId", - "firstName", - "lastName", - } - assert ( - filtered_results[f"{dataset_name}:owners"][0]["email"] == hubspot_identity_email - ) - - -@pytest.mark.integration_saas -@pytest.mark.asyncio -@pytest.mark.usefixtures( - "use_dsr_3_0" -) # Only testing on DSR 3.0 not 2.0 - because of fixtures taking too long to settle down -async def test_hubspot_erasure_request_task( - db, - privacy_request, - erasure_policy_string_rewrite_name_and_email, - connection_config_hubspot, - dataset_config_hubspot, - hubspot_erasure_identity_email, - hubspot_erasure_data, - hubspot_test_client: HubspotTestClient, -) -> None: - """Full erasure request based on the Hubspot SaaS config""" - - privacy_request.policy_id = erasure_policy_string_rewrite_name_and_email.id - privacy_request.save(db) - contact_id, user_id = hubspot_erasure_data - - identity_attribute = "email" - identity_kwargs = {identity_attribute: (hubspot_erasure_identity_email)} - identity = Identity(**identity_kwargs) - privacy_request.cache_identity(identity) - - dataset_name = connection_config_hubspot.get_saas_config().fides_key - merged_graph = dataset_config_hubspot.get_graph() - graph = DatasetGraph(merged_graph) - - v = access_runner_tester( - privacy_request, - erasure_policy_string_rewrite_name_and_email, - graph, - [connection_config_hubspot], - identity_kwargs, - db, - ) - - assert_rows_match( - v[f"{dataset_name}:contacts"], - min_size=1, - keys=["archived", "createdAt", "id", "properties"], - ) - assert_rows_match( - v[f"{dataset_name}:subscription_preferences"], - min_size=1, - keys=["recipient", "subscriptionStatuses"], - ) - - temp_masking = CONFIG.execution.masking_strict - CONFIG.execution.masking_strict = False # Allow delete - x = erasure_runner_tester( - privacy_request, +class TestHubspotConnector: + + def test_hubspot_connection_test(self, hubspot_runner: ConnectorRunner) -> None: + hubspot_runner.test_connection() + + @pytest.mark.parametrize( + "dsr_version", + ["use_dsr_3_0", "use_dsr_2_0"], + ) + async def test_hubspot_access_request_task( + self, + hubspot_runner: ConnectorRunner, + dsr_version, + request, + policy: Policy, + hubspot_identity_email, + hubspot_data, + ) -> None: + """Full access request based on the Hubspot SaaS config""" + request.getfixturevalue(dsr_version) # REQUIRED to test both DSR 3.0 and 2.0 + + dataset_name = hubspot_runner.dataset_config.fides_key + + access_results = await hubspot_runner.access_request( + access_policy=policy, identities={"email": hubspot_identity_email} + ) + + assert len(access_results[f"{dataset_name}:users"]) == 1 + + assert ( + access_results[f"{dataset_name}:subscription_preferences"][0]["recipient"] + == hubspot_identity_email + ) + assert ( + access_results[f"{dataset_name}:users"][0]["email"] + == hubspot_identity_email + ) + assert ( + access_results[f"{dataset_name}:owners"][0]["email"] + == hubspot_identity_email + ) + + @pytest.mark.usefixtures( + "use_dsr_3_0" + ) # Only testing on DSR 3.0 not 2.0 - because of fixtures taking too long to settle down + async def test_hubspot_erasure_request_task( + self, + hubspot_runner: ConnectorRunner, + policy: Policy, erasure_policy_string_rewrite_name_and_email, - graph, - [connection_config_hubspot], - identity_kwargs, - get_cached_data_for_erasures(privacy_request.id), - db, - ) - CONFIG.execution.masking_strict = temp_masking - - # Masking request only issued to "contacts", "subscription_preferences", and "users" endpoints - assert x == { - "hubspot_instance:contacts": 1, - "hubspot_instance:owners": 0, - "hubspot_instance:subscription_preferences": 2, - "hubspot_instance:users": 1, - } - - # Verify the user has been assigned to None - contact_response = hubspot_test_client.get_contact(contact_id=contact_id) - contact_body = contact_response.json() - assert contact_body["properties"]["firstname"] == "MASKED" - assert contact_body["properties"]["email"] == f"{privacy_request.id}@company.com" - - # verify user is unsubscribed - email_subscription_response = hubspot_test_client.get_email_subscriptions( - email=hubspot_erasure_identity_email - ) - subscription_body = email_subscription_response.json() - for subscription_status in subscription_body["subscriptionStatuses"]: - assert subscription_status["status"] == "NOT_SUBSCRIBED" - - # verify user is deleted - error_message = f"User with user id {user_id} could not be deleted from Hubspot" - poll_for_existence( - user_exists, - (user_id, hubspot_test_client), - error_message=error_message, - existence_desired=False, - ) + hubspot_identity_email, + hubspot_data, + hubspot_test_client: HubspotTestClient, + ) -> None: + """Full erasure request based on the Hubspot SaaS config""" + + contact_id, user_id = hubspot_data + + ( + _, + erasure_results, + ) = await hubspot_runner.non_strict_erasure_request( + access_policy=policy, + erasure_policy=erasure_policy_string_rewrite_name_and_email, + identities={"email": hubspot_identity_email}, + ) + + # Masking request only issued to "contacts", "subscription_preferences", and "users" endpoints + assert erasure_results == { + "hubspot_instance:contacts": 1, + "hubspot_instance:owners": 0, + "hubspot_instance:subscription_preferences": 2, + "hubspot_instance:users": 1, + } + + # Verify the user has been assigned to None + contact_response = hubspot_test_client.get_contact(contact_id=contact_id) + contact_body = contact_response.json() + assert contact_body["properties"]["firstname"] == "MASKED" + assert contact_body["properties"]["email"].endswith("@company.com") + + # verify user is unsubscribed + email_subscription_response = hubspot_test_client.get_email_subscriptions( + email=hubspot_identity_email + ) + subscription_body = email_subscription_response.json() + for subscription_status in subscription_body["subscriptionStatuses"]: + assert subscription_status["status"] == "NOT_SUBSCRIBED" + + # verify user is deleted + error_message = f"User with user id {user_id} could not be deleted from Hubspot" + poll_for_existence( + user_exists, + (user_id, hubspot_test_client), + error_message=error_message, + existence_desired=False, + ) diff --git a/tests/ops/integration_tests/saas/test_stripe_task.py b/tests/ops/integration_tests/saas/test_stripe_task.py index 1c51330a88..ceb7f449b8 100644 --- a/tests/ops/integration_tests/saas/test_stripe_task.py +++ b/tests/ops/integration_tests/saas/test_stripe_task.py @@ -1,1216 +1,211 @@ from typing import List import pytest -import requests -from fides.api.graph.graph import DatasetGraph -from fides.api.schemas.redis_cache import Identity -from fides.api.service.connectors import get_connector -from fides.api.task.filter_results import filter_data_categories -from fides.api.task.graph_task import get_cached_data_for_erasures -from fides.config import CONFIG -from tests.conftest import access_runner_tester, erasure_runner_tester -from tests.ops.graph.graph_test_util import assert_rows_match -from tests.ops.test_helpers.cache_secrets_helper import clear_cache_identities +from fides.api.models.policy import Policy +from tests.ops.integration_tests.saas.connector_runner import ConnectorRunner @pytest.mark.integration_saas -def test_stripe_connection_test(stripe_connection_config) -> None: - get_connector(stripe_connection_config).test_connection() - - -@pytest.mark.integration_saas -@pytest.mark.asyncio -@pytest.mark.parametrize( - "dsr_version", - ["use_dsr_3_0", "use_dsr_2_0"], -) -async def test_stripe_access_request_task_with_email( - db, - policy, - dsr_version, - request, - privacy_request, - stripe_connection_config, - stripe_dataset_config, - stripe_identity_email, -) -> None: - """Full access request based on the Stripe SaaS config""" - request.getfixturevalue(dsr_version) # REQUIRED to test both DSR 3.0 and 2.0 - - identity = Identity(**{"email": stripe_identity_email}) - privacy_request.cache_identity(identity) - - dataset_name = stripe_connection_config.get_saas_config().fides_key - merged_graph = stripe_dataset_config.get_graph() - graph = DatasetGraph(merged_graph) - - v = access_runner_tester( - privacy_request, - policy, - graph, - [stripe_connection_config], - {"email": stripe_identity_email}, - db, - ) - - # verify all collections are returned with the expected number of rows and fields - - assert_rows_match( - v[f"{dataset_name}:bank_account"], - min_size=1, - keys=[ - "account_holder_name", - "account_holder_type", - "account_type", - "bank_name", - "country", - "currency", - "customer", - "fingerprint", - "id", - "last4", - "object", - "routing_number", - "status", - ], - ) - - assert_rows_match( - v[f"{dataset_name}:card"], - min_size=1, - keys=[ - "address_city", - "address_country", - "address_line1", - "address_line1_check", - "address_line2", - "address_state", - "address_zip", - "address_zip_check", - "brand", - "country", - "customer", - "cvc_check", - "dynamic_last4", - "exp_month", - "exp_year", - "fingerprint", - "funding", - "id", - "last4", - "name", - "object", - "tokenization_method", - ], - ) - - assert_rows_match( - v[f"{dataset_name}:charge"], - min_size=2, - keys=[ - "amount", - "amount_captured", - "amount_refunded", - "application", - "application_fee", - "application_fee_amount", - "balance_transaction", - "billing_details", - "calculated_statement_descriptor", - "captured", - "created", - "currency", - "customer", - "description", - "disputed", - "failure_balance_transaction", - "failure_code", - "failure_message", - "fraud_details", - "id", - "invoice", - "livemode", - "object", - "on_behalf_of", - "order", - "paid", - "payment_intent", - "payment_method", - "payment_method_details", - "receipt_email", - "receipt_number", - "receipt_url", - "refunded", - "review", - "shipping", - "source_transfer", - "statement_descriptor", - "statement_descriptor_suffix", - "status", - "transfer_data", - "transfer_group", - ], - ) - - assert_rows_match( - v[f"{dataset_name}:credit_note"], - min_size=1, - keys=[ - "amount", - "created", - "currency", - "customer", - "customer_balance_transaction", - "discount_amount", - "discount_amounts", - "id", - "invoice", - "livemode", - "memo", - "number", - "object", - "out_of_band_amount", - "pdf", - "reason", - "refund", - "status", - "subtotal", - "tax_amounts", - "total", - "type", - "voided_at", - ], - ) - - assert_rows_match( - v[f"{dataset_name}:customer"], - min_size=1, - keys=[ - "address", - "balance", - "created", - "currency", - "default_source", - "delinquent", - "description", - "discount", - "email", - "id", - "invoice_prefix", - "invoice_settings", - "livemode", - "name", - "next_invoice_sequence", - "object", - "phone", - "preferred_locales", - "shipping", - "tax_exempt", - "test_clock", - ], - ) - - assert_rows_match( - v[f"{dataset_name}:customer_balance_transaction"], - min_size=2, - keys=[ - "amount", - "created", - "credit_note", - "currency", - "customer", - "description", - "ending_balance", - "id", - "invoice", - "livemode", - "object", - "type", - ], - ) - - assert_rows_match( - v[f"{dataset_name}:dispute"], - min_size=3, - keys=[ - "amount", - "balance_transactions", - "charge", - "created", - "currency", - "evidence", - "evidence_details", - "id", - "is_charge_refundable", - "livemode", - "object", - "payment_intent", - "reason", - "status", - ], - ) - - assert_rows_match( - v[f"{dataset_name}:invoice"], - min_size=1, - keys=[ - "account_country", - "account_name", - "amount_due", - "amount_paid", - "amount_remaining", - "application_fee_amount", - "attempt_count", - "attempted", - "auto_advance", - "automatic_tax", - "billing_reason", - "charge", - "collection_method", - "created", - "currency", - "custom_fields", - "customer", - "customer_address", - "customer_email", - "customer_name", - "customer_phone", - "customer_shipping", - "customer_tax_exempt", - "customer_tax_ids", - "default_payment_method", - "default_source", - "default_tax_rates", - "description", - "discount", - "discounts", - "due_date", - "ending_balance", - "footer", - "hosted_invoice_url", - "id", - "invoice_pdf", - "last_finalization_error", - "livemode", - "next_payment_attempt", - "number", - "object", - "on_behalf_of", - "paid", - "paid_out_of_band", - "payment_intent", - "payment_settings", - "period_end", - "period_start", - "post_payment_credit_notes_amount", - "pre_payment_credit_notes_amount", - "quote", - "receipt_number", - "starting_balance", - "statement_descriptor", - "status", - "status_transitions", - "subscription", - "subtotal", - "tax", - "test_clock", - "total", - "total_tax_amounts", - "transfer_data", - "webhooks_delivered_at", - ], - ) - - assert_rows_match( - v[f"{dataset_name}:invoice_item"], - min_size=1, - keys=[ - "amount", - "currency", - "customer", - "date", - "description", - "discountable", - "discounts", - "id", - "invoice", - "livemode", - "object", - "period", - "price", - "proration", - "quantity", - "subscription", - "tax_rates", - "test_clock", - "unit_amount", - "unit_amount_decimal", - ], - ) - - assert_rows_match( - v[f"{dataset_name}:payment_intent"], - min_size=1, - keys=[ - "amount", - "amount_capturable", - "amount_received", - "application", - "application_fee_amount", - "automatic_payment_methods", - "canceled_at", - "cancellation_reason", - "capture_method", - "client_secret", - "confirmation_method", - "created", - "currency", - "customer", - "description", - "id", - "invoice", - "last_payment_error", - "livemode", - "next_action", - "object", - "on_behalf_of", - "payment_method", - "payment_method_options", - "payment_method_types", - "processing", - "receipt_email", - "review", - "setup_future_usage", - "shipping", - "statement_descriptor", - "statement_descriptor_suffix", - "status", - "transfer_data", - "transfer_group", - ], - ) - - assert_rows_match( - v[f"{dataset_name}:payment_method"], - min_size=2, - keys=[ - "billing_details", - "created", - "customer", - "id", - "livemode", - "object", - "type", - ], - ) - - assert_rows_match( - v[f"{dataset_name}:tax_id"], - min_size=1, - keys=[ - "country", - "created", - "customer", - "id", - "livemode", - "object", - "type", - "value", - "verification", - ], - ) - - # verify we only returned data for our identity email - assert v[f"{dataset_name}:customer"][0]["email"] == stripe_identity_email - customer_id: str = v[f"{dataset_name}:customer"][0]["id"] - - for bank_account in v[f"{dataset_name}:bank_account"]: - assert bank_account["customer"] == customer_id - - for card in v[f"{dataset_name}:card"]: - assert card["customer"] == customer_id - - charge_ids: List[str] = [] - for charge in v[f"{dataset_name}:charge"]: - assert charge["customer"] == customer_id - charge_ids.append(charge["id"]) - - payment_intent_ids: List[str] = [] - for payment_intent in v[f"{dataset_name}:payment_intent"]: - assert payment_intent["customer"] == customer_id - payment_intent_ids.append(payment_intent["id"]) - - for credit_note in v[f"{dataset_name}:credit_note"]: - assert credit_note["customer"] == customer_id +class TestStripeConnector: + + def test_stripe_connection_test(self, stripe_runner: ConnectorRunner) -> None: + stripe_runner.test_connection() + + @pytest.mark.parametrize( + "dsr_version", + ["use_dsr_3_0", "use_dsr_2_0"], + ) + async def test_stripe_access_request_task_with_email( + self, + stripe_runner: ConnectorRunner, + policy: Policy, + dsr_version, + request, + stripe_identity_email, + stripe_create_data, + ) -> None: + """Full access request based on the Stripe SaaS config""" + request.getfixturevalue(dsr_version) # REQUIRED to test both DSR 3.0 and 2.0 + + dataset_name = stripe_runner.dataset_config.fides_key + + access_results = await stripe_runner.access_request( + access_policy=policy, identities={"email": stripe_identity_email} + ) - for bank_account in v[f"{dataset_name}:bank_account"]: - assert bank_account["customer"] == customer_id + # verify we only returned data for our identity email - for customer_balance_transaction in v[ - f"{dataset_name}:customer_balance_transaction" - ]: - assert customer_balance_transaction["customer"] == customer_id + assert len(access_results[f"{dataset_name}:customer"]) == 1 - # disputes are retrieved by charge.id or payment_intent.id - for dispute in v[f"{dataset_name}:dispute"]: assert ( - dispute["charge"] in charge_ids - or dispute["payment_intent"] in payment_intent_ids + access_results[f"{dataset_name}:customer"][0]["email"] + == stripe_identity_email + ) + customer_id: str = access_results[f"{dataset_name}:customer"][0]["id"] + + for bank_account in access_results[f"{dataset_name}:bank_account"]: + assert bank_account["customer"] == customer_id + + for card in access_results[f"{dataset_name}:card"]: + assert card["customer"] == customer_id + + charge_ids: List[str] = [] + for charge in access_results[f"{dataset_name}:charge"]: + assert charge["customer"] == customer_id + charge_ids.append(charge["id"]) + + payment_intent_ids: List[str] = [] + for payment_intent in access_results[f"{dataset_name}:payment_intent"]: + assert payment_intent["customer"] == customer_id + payment_intent_ids.append(payment_intent["id"]) + + for credit_note in access_results[f"{dataset_name}:credit_note"]: + assert credit_note["customer"] == customer_id + + for bank_account in access_results[f"{dataset_name}:bank_account"]: + assert bank_account["customer"] == customer_id + + for customer_balance_transaction in access_results[ + f"{dataset_name}:customer_balance_transaction" + ]: + assert customer_balance_transaction["customer"] == customer_id + + # disputes are retrieved by charge.id or payment_intent.id + for dispute in access_results[f"{dataset_name}:dispute"]: + assert ( + dispute["charge"] in charge_ids + or dispute["payment_intent"] in payment_intent_ids + ) + + for invoice in access_results[f"{dataset_name}:invoice"]: + assert invoice["customer"] == customer_id + + for invoice_item in access_results[f"{dataset_name}:invoice_item"]: + assert invoice_item["customer"] == customer_id + + for payment_method in access_results[f"{dataset_name}:payment_method"]: + assert payment_method["customer"] == customer_id + + for subscription in access_results[f"{dataset_name}:subscription"]: + assert subscription["customer"] == customer_id + + for tax_id in access_results[f"{dataset_name}:tax_id"]: + assert tax_id["customer"] == customer_id + + @pytest.mark.parametrize( + "dsr_version", + ["use_dsr_3_0", "use_dsr_2_0"], + ) + async def test_stripe_access_request_task_with_phone_number( + self, + stripe_runner: ConnectorRunner, + policy: Policy, + dsr_version, + request, + stripe_identity_email, + stripe_identity_phone_number, + stripe_create_data, + ) -> None: + """Full access request based on the Stripe SaaS config""" + request.getfixturevalue(dsr_version) # REQUIRED to test both DSR 3.0 and 2.0 + # The Privacy request fixture we're using already has an email/phone cached + # so I'm clearing that first + dataset_name = stripe_runner.dataset_config.fides_key + + access_results = await stripe_runner.access_request( + access_policy=policy, + identities={"phone_number": stripe_identity_phone_number}, ) - for invoice in v[f"{dataset_name}:invoice"]: - assert invoice["customer"] == customer_id - - for invoice_item in v[f"{dataset_name}:invoice_item"]: - assert invoice_item["customer"] == customer_id - - for payment_method in v[f"{dataset_name}:payment_method"]: - assert payment_method["customer"] == customer_id - - for subscription in v[f"{dataset_name}:subscription"]: - assert subscription["customer"] == customer_id - - for tax_id in v[f"{dataset_name}:tax_id"]: - assert tax_id["customer"] == customer_id - - # verify we keep the expected fields after filtering by the user data category - target_categories = {"user"} - filtered_results = filter_data_categories(v, target_categories, graph) - - assert set(filtered_results.keys()) == { - f"{dataset_name}:bank_account", - f"{dataset_name}:card", - f"{dataset_name}:charge", - f"{dataset_name}:credit_note", - f"{dataset_name}:customer", - f"{dataset_name}:customer_balance_transaction", - f"{dataset_name}:dispute", - f"{dataset_name}:invoice", - f"{dataset_name}:invoice_item", - f"{dataset_name}:payment_intent", - f"{dataset_name}:payment_method", - f"{dataset_name}:tax_id", - } - - assert set(filtered_results[f"{dataset_name}:bank_account"][0].keys()) == { - "account_holder_name", - "bank_name", - "country", - "last4", - "routing_number", - } - - assert set(filtered_results[f"{dataset_name}:card"][0].keys()) == { - "address_city", - "address_country", - "address_line1", - "address_line2", - "address_state", - "address_zip", - "country", - "dynamic_last4", - "last4", - "name", - } - - assert set(filtered_results[f"{dataset_name}:charge"][0].keys()) == { - "amount", - "amount_captured", - "amount_refunded", - "billing_details", - "payment_method_details", - "receipt_email", - "source", - } - - assert set(filtered_results[f"{dataset_name}:credit_note"][0].keys()) == { - "amount", - "discount_amount", - "subtotal", - "total", - } - - assert set(filtered_results[f"{dataset_name}:customer"][0].keys()) == { - "address", - "balance", - "email", - "name", - "phone", - "preferred_locales", - "shipping", - } - - assert set( - filtered_results[f"{dataset_name}:customer_balance_transaction"][0].keys() - ) == {"ending_balance"} - - assert set(filtered_results[f"{dataset_name}:dispute"][0].keys()) == { - "amount", - "evidence", - } - - assert set(filtered_results[f"{dataset_name}:invoice"][0].keys()) == { - "account_country", - "account_name", - "amount_due", - "amount_paid", - "amount_remaining", - "customer_address", - "customer_email", - "customer_name", - "customer_phone", - "customer_shipping", - "discount", - "starting_balance", - "subtotal", - "total", - } - - assert set(filtered_results[f"{dataset_name}:invoice_item"][0].keys()) == { - "amount", - "unit_amount", - "unit_amount_decimal", - } - - assert set(filtered_results[f"{dataset_name}:payment_intent"][0].keys()) == { - "amount", - "amount_capturable", - "amount_received", - "receipt_email", - "shipping", - } - - assert set(filtered_results[f"{dataset_name}:payment_method"][0].keys()) == { - "billing_details", - "card", - } - - assert set(filtered_results[f"{dataset_name}:tax_id"][0].keys()) == { - "country", - "verification", - } - - -@pytest.mark.integration_saas -@pytest.mark.asyncio -@pytest.mark.parametrize( - "dsr_version", - ["use_dsr_3_0", "use_dsr_2_0"], -) -async def test_stripe_access_request_task_with_phone_number( - db, - policy, - dsr_version, - request, - privacy_request, - stripe_connection_config, - stripe_dataset_config, - stripe_identity_email, - stripe_identity_phone_number, -) -> None: - """Full access request based on the Stripe SaaS config""" - request.getfixturevalue(dsr_version) # REQUIRED to test both DSR 3.0 and 2.0 - # The Privacy request fixture we're using already has an email/phone cached - # so I'm clearing that first - clear_cache_identities(privacy_request.id) - - identity = Identity(**{"phone_number": stripe_identity_phone_number}) - privacy_request.cache_identity(identity) - - dataset_name = stripe_connection_config.get_saas_config().fides_key - merged_graph = stripe_dataset_config.get_graph() - graph = DatasetGraph(merged_graph) - - v = access_runner_tester( - privacy_request, - policy, - graph, - [stripe_connection_config], - {"phone_number": stripe_identity_phone_number}, - db, - ) - - assert_rows_match( - v[f"{dataset_name}:customer"], - min_size=1, - keys=[ - "address", - "balance", - "created", - "currency", - "default_source", - "delinquent", - "description", - "discount", - "email", - "id", - "invoice_prefix", - "invoice_settings", - "livemode", - "name", - "next_invoice_sequence", - "object", - "phone", - "preferred_locales", - "shipping", - "tax_exempt", - "test_clock", - ], - ) - - # verify we only returned data for our identity phone number and that - # it is the same customer that we retrieved using the identity email - assert v[f"{dataset_name}:customer"][0]["phone"] == stripe_identity_phone_number - assert v[f"{dataset_name}:customer"][0]["email"] == stripe_identity_email - - -@pytest.mark.integration_saas -@pytest.mark.asyncio -@pytest.mark.parametrize( - "dsr_version", - ["use_dsr_3_0", "use_dsr_2_0"], -) -async def test_stripe_erasure_request_task( - db, - privacy_request, - dsr_version, - request, - erasure_policy_string_rewrite, - stripe_connection_config, - stripe_dataset_config, - stripe_erasure_identity_email, - stripe_create_erasure_data, -) -> None: - """Full erasure request based on the Stripe SaaS config""" - request.getfixturevalue(dsr_version) # REQUIRED to test both DSR 3.0 and 2.0 - - privacy_request.policy_id = erasure_policy_string_rewrite.id - privacy_request.save(db) - - identity = Identity(**{"email": stripe_erasure_identity_email}) - privacy_request.cache_identity(identity) - - dataset_name = stripe_connection_config.get_saas_config().fides_key - merged_graph = stripe_dataset_config.get_graph() - graph = DatasetGraph(merged_graph) - - v = access_runner_tester( - privacy_request, - erasure_policy_string_rewrite, - graph, - [stripe_connection_config], - {"email": stripe_erasure_identity_email}, - db, - ) - - # verify staged data is available for erasure - assert_rows_match( - v[f"{dataset_name}:bank_account"], - min_size=1, - keys=[ - "account_holder_name", - "account_holder_type", - "account_type", - "bank_name", - "country", - "currency", - "customer", - "fingerprint", - "id", - "last4", - "object", - "routing_number", - "status", - ], - ) - - assert_rows_match( - v[f"{dataset_name}:card"], - min_size=1, - keys=[ - "address_city", - "address_country", - "address_line1", - "address_line1_check", - "address_line2", - "address_state", - "address_zip", - "address_zip_check", - "brand", - "country", - "customer", - "cvc_check", - "dynamic_last4", - "exp_month", - "exp_year", - "fingerprint", - "funding", - "id", - "last4", - "name", - "object", - "tokenization_method", - ], - ) - - assert_rows_match( - v[f"{dataset_name}:charge"], - min_size=2, - keys=[ - "amount", - "amount_captured", - "amount_refunded", - "application", - "application_fee", - "application_fee_amount", - "balance_transaction", - "billing_details", - "calculated_statement_descriptor", - "captured", - "created", - "currency", - "customer", - "description", - "disputed", - "failure_balance_transaction", - "failure_code", - "failure_message", - "fraud_details", - "id", - "invoice", - "livemode", - "object", - "on_behalf_of", - "order", - "paid", - "payment_intent", - "payment_method", - "payment_method_details", - "receipt_email", - "receipt_number", - "receipt_url", - "refunded", - "review", - "shipping", - "source_transfer", - "statement_descriptor", - "statement_descriptor_suffix", - "status", - "transfer_data", - "transfer_group", - ], - ) - - assert_rows_match( - v[f"{dataset_name}:credit_note"], - min_size=1, - keys=[ - "amount", - "created", - "currency", - "customer", - "customer_balance_transaction", - "discount_amount", - "discount_amounts", - "id", - "invoice", - "livemode", - "memo", - "number", - "object", - "out_of_band_amount", - "pdf", - "reason", - "refund", - "status", - "subtotal", - "tax_amounts", - "total", - "type", - "voided_at", - ], - ) - - assert_rows_match( - v[f"{dataset_name}:customer"], - min_size=1, - keys=[ - "address", - "balance", - "created", - "currency", - "default_source", - "delinquent", - "description", - "discount", - "email", - "id", - "invoice_prefix", - "invoice_settings", - "livemode", - "name", - "next_invoice_sequence", - "object", - "phone", - "preferred_locales", - "shipping", - "tax_exempt", - "test_clock", - ], - ) - - assert_rows_match( - v[f"{dataset_name}:customer_balance_transaction"], - min_size=2, - keys=[ - "amount", - "created", - "credit_note", - "currency", - "customer", - "description", - "ending_balance", - "id", - "invoice", - "livemode", - "object", - "type", - ], - ) - - assert_rows_match( - v[f"{dataset_name}:dispute"], - min_size=3, - keys=[ - "amount", - "balance_transactions", - "charge", - "created", - "currency", - "evidence", - "evidence_details", - "id", - "is_charge_refundable", - "livemode", - "object", - "payment_intent", - "reason", - "status", - ], - ) - - assert_rows_match( - v[f"{dataset_name}:invoice"], - min_size=1, - keys=[ - "account_country", - "account_name", - "amount_due", - "amount_paid", - "amount_remaining", - "application_fee_amount", - "attempt_count", - "attempted", - "auto_advance", - "automatic_tax", - "billing_reason", - "charge", - "collection_method", - "created", - "currency", - "custom_fields", - "customer", - "customer_address", - "customer_email", - "customer_name", - "customer_phone", - "customer_shipping", - "customer_tax_exempt", - "customer_tax_ids", - "default_payment_method", - "default_source", - "default_tax_rates", - "description", - "discount", - "discounts", - "due_date", - "ending_balance", - "footer", - "hosted_invoice_url", - "id", - "invoice_pdf", - "last_finalization_error", - "livemode", - "next_payment_attempt", - "number", - "object", - "on_behalf_of", - "paid", - "paid_out_of_band", - "payment_intent", - "payment_settings", - "period_end", - "period_start", - "post_payment_credit_notes_amount", - "pre_payment_credit_notes_amount", - "quote", - "receipt_number", - "starting_balance", - "statement_descriptor", - "status", - "status_transitions", - "subscription", - "subtotal", - "tax", - "test_clock", - "total", - "total_tax_amounts", - "transfer_data", - "webhooks_delivered_at", - ], - ) - - assert_rows_match( - v[f"{dataset_name}:invoice_item"], - min_size=1, - keys=[ - "amount", - "currency", - "customer", - "date", - "description", - "discountable", - "discounts", - "id", - "invoice", - "livemode", - "object", - "period", - "price", - "proration", - "quantity", - "subscription", - "tax_rates", - "test_clock", - "unit_amount", - "unit_amount_decimal", - ], - ) - - assert_rows_match( - v[f"{dataset_name}:payment_intent"], - min_size=1, - keys=[ - "amount", - "amount_capturable", - "amount_received", - "application", - "application_fee_amount", - "automatic_payment_methods", - "canceled_at", - "cancellation_reason", - "capture_method", - "client_secret", - "confirmation_method", - "created", - "currency", - "customer", - "description", - "id", - "invoice", - "last_payment_error", - "livemode", - "next_action", - "object", - "on_behalf_of", - "payment_method", - "payment_method_options", - "payment_method_types", - "processing", - "receipt_email", - "review", - "setup_future_usage", - "shipping", - "statement_descriptor", - "statement_descriptor_suffix", - "status", - "transfer_data", - "transfer_group", - ], - ) - - assert_rows_match( - v[f"{dataset_name}:payment_method"], - min_size=2, - keys=[ - "billing_details", - "created", - "customer", - "id", - "livemode", - "object", - "type", - ], - ) - - assert_rows_match( - v[f"{dataset_name}:subscription"], - min_size=1, - keys=[ - "application_fee_percent", - "automatic_tax", - "billing_cycle_anchor", - "billing_thresholds", - "cancel_at", - "cancel_at_period_end", - "canceled_at", - "collection_method", - "created", - "current_period_end", - "current_period_start", - "customer", - "days_until_due", - "default_payment_method", - "default_source", - "default_tax_rates", - "discount", - "ended_at", - "id", - "latest_invoice", - "livemode", - "next_pending_invoice_item_invoice", - "object", - "pause_collection", - "payment_settings", - "pending_invoice_item_interval", - "pending_setup_intent", - "pending_update", - "schedule", - "start_date", - "status", - "test_clock", - "transfer_data", - "trial_end", - "trial_start", - ], - ) - - assert_rows_match( - v[f"{dataset_name}:tax_id"], - min_size=1, - keys=[ - "country", - "created", - "customer", - "id", - "livemode", - "object", - "type", - "value", - "verification", - ], - ) + # verify we only returned data for our identity phone number and that + # it is the same customer that we retrieved using the identity email + assert len(access_results[f"{dataset_name}:customer"]) == 1 - # Run erasure with masking_strict = False so both update and delete actions can be used - masking_strict = CONFIG.execution.masking_strict - CONFIG.execution.masking_strict = False + assert ( + access_results[f"{dataset_name}:customer"][0]["phone"] + == stripe_identity_phone_number + ) + assert ( + access_results[f"{dataset_name}:customer"][0]["email"] + == stripe_identity_email + ) - x = erasure_runner_tester( - privacy_request, + @pytest.mark.integration_saas + @pytest.mark.asyncio + @pytest.mark.parametrize( + "dsr_version", + ["use_dsr_3_0", "use_dsr_2_0"], + ) + async def test_non_strict_erasure_request_with_email( + self, + stripe_runner: ConnectorRunner, + policy: Policy, + dsr_version, + request, erasure_policy_string_rewrite, - graph, - [stripe_connection_config], - {"email": stripe_erasure_identity_email}, - get_cached_data_for_erasures(privacy_request.id), - db, - ) - - # verify masking request was issued for endpoints with both update/delete actions - assert x == { - f"{dataset_name}:customer": 1, - f"{dataset_name}:tax_id": 1, - f"{dataset_name}:invoice_item": 1, - f"{dataset_name}:charge": 0, - f"{dataset_name}:invoice": 2, - f"{dataset_name}:card": 1, - f"{dataset_name}:customer_balance_transaction": 0, - f"{dataset_name}:payment_intent": 0, - f"{dataset_name}:payment_method": 3, - f"{dataset_name}:credit_note": 0, - f"{dataset_name}:bank_account": 1, - f"{dataset_name}:subscription": 1, - f"{dataset_name}:dispute": 0, - } - - stripe_secrets = stripe_connection_config.secrets - base_url = f"https://{stripe_secrets['domain']}" - headers = { - "Content-Type": "application/x-www-form-urlencoded", - "Authorization": f"Bearer {stripe_secrets['api_key']}", - } - - # customer - response = requests.get( - url=f"{base_url}/v1/customers", - headers=headers, - params={"email": stripe_erasure_identity_email}, - ) - customer = response.json()["data"][0] - customer_id = customer["id"] - assert customer["shipping"]["name"] == "MASKED" - - # card - response = requests.get( - url=f"{base_url}/v1/customers/{customer_id}/sources", - headers=headers, - params={"object": "card"}, - ) - cards = response.json()["data"] - assert cards == [] - - # payment method - response = requests.get( - url=f"{base_url}/v1/customers/{customer_id}/payment_methods", - headers=headers, - params={"type": "card"}, - ) - payment_methods = response.json()["data"] - for payment_method in payment_methods: - assert payment_method["billing_details"]["name"] == "MASKED" - - # bank account - response = requests.get( - url=f"{base_url}/v1/customers/{customer_id}/sources", - headers=headers, - params={"object": "bank_account"}, - ) - bank_account = response.json()["data"][0] - assert bank_account["account_holder_name"] == "MASKED" - - # tax_id - response = requests.get( - url=f"{base_url}/v1/customers/{customer_id}/tax_ids", headers=headers - ) - tax_ids = response.json()["data"] - assert tax_ids == [] - - # invoice_item - response = requests.get( - url=f"{base_url}/v1/invoiceitems", - headers=headers, - params={"customer": {customer_id}}, - ) - invoice_item = response.json()["data"] - # Can't delete an invoice item that is attached to an invoice that is no longer editable - assert len(invoice_item) == 1 - - # subscription - response = requests.get( - url=f"{base_url}/v1/customers/{customer_id}/subscriptions", headers=headers - ) - subscriptions = response.json()["data"] - assert subscriptions == [] + stripe_identity_email, + stripe_test_client, + stripe_create_data, + ) -> None: + """Full erasure request based on the Stripe SaaS config""" + request.getfixturevalue(dsr_version) # REQUIRED to test both DSR 3.0 and 2.0 + + dataset_name = stripe_runner.dataset_config.fides_key + + ( + _, + erasure_results, + ) = await stripe_runner.non_strict_erasure_request( + access_policy=policy, + erasure_policy=erasure_policy_string_rewrite, + identities={"email": stripe_identity_email}, + ) - # reset - CONFIG.execution.masking_strict = masking_strict + # verify masking request was issued for endpoints with both update/delete actions + assert erasure_results == { + f"{dataset_name}:customer": 1, + f"{dataset_name}:tax_id": 1, + f"{dataset_name}:invoice_item": 1, + f"{dataset_name}:charge": 0, + f"{dataset_name}:invoice": 2, + f"{dataset_name}:card": 1, + f"{dataset_name}:customer_balance_transaction": 0, + f"{dataset_name}:payment_intent": 0, + f"{dataset_name}:payment_method": 3, + f"{dataset_name}:credit_note": 0, + f"{dataset_name}:bank_account": 1, + f"{dataset_name}:subscription": 1, + f"{dataset_name}:dispute": 0, + } + + # customer + customer = stripe_test_client.get_customer(stripe_identity_email) + customer_id = customer["id"] + assert customer["shipping"]["name"] == "MASKED" + + # card + cards = stripe_test_client.get_card(customer_id) + assert cards == [] + + # payment method + payment_methods = stripe_test_client.get_payment_method(customer_id) + for payment_method in payment_methods: + assert payment_method["billing_details"]["name"] == "MASKED" + + # bank account + bank_account = stripe_test_client.get_bank_account(customer_id) + assert bank_account["account_holder_name"] == "MASKED" + + # tax_id + tax_ids = stripe_test_client.get_tax_ids(customer_id) + assert tax_ids == [] + + # invoice_item + invoice_item = stripe_test_client.get_invoice_items(customer_id) + # Can't delete an invoice item that is attached to an invoice that is no longer editable + assert len(invoice_item) == 1 + + # subscription + subscriptions = stripe_test_client.get_subscription(customer_id) + assert subscriptions == [] diff --git a/tests/ops/service/privacy_request/test_saas_privacy_requests.py b/tests/ops/service/privacy_request/test_saas_privacy_requests.py index 8816037e2e..e2f5e4bea2 100644 --- a/tests/ops/service/privacy_request/test_saas_privacy_requests.py +++ b/tests/ops/service/privacy_request/test_saas_privacy_requests.py @@ -153,6 +153,7 @@ def test_create_and_process_access_request_saas_hubspot( dsr_version, request, hubspot_identity_email, + hubspot_data, run_privacy_request_task, ): request.getfixturevalue(dsr_version) # REQUIRED to test both DSR 3.0 and 2.0