Merge branch 'main' into main

tariqajyusuf · Jul 18, 2024 · 4f06a2e · 4f06a2e
2 parents 7ea79b0 + a24f3f6
commit 4f06a2e
Show file tree

Hide file tree

Showing 97 changed files with 2,823 additions and 1,658 deletions.
diff --git a/.fides/db_dataset.yml b/.fides/db_dataset.yml
@@ -2131,6 +2131,8 @@ dataset:
       data_categories: [system]
     - name: databases
       data_categories: [system]
+    - name: excluded_databases
+      data_categories: [system]
     - name: monitor_execution_trigger
       data_categories: [system]
     - name: datasource_params

diff --git a/.github/workflows/backend_checks.yml b/.github/workflows/backend_checks.yml
@@ -361,6 +361,8 @@ jobs:
           SNOWFLAKE_TEST_ACCOUNT_IDENTIFIER: ${{ secrets.SNOWFLAKE_TEST_ACCOUNT_IDENTIFIER }}
           SNOWFLAKE_TEST_DATABASE_NAME: ${{ secrets.SNOWFLAKE_TEST_DATABASE_NAME }}
           SNOWFLAKE_TEST_PASSWORD: ${{ secrets.SNOWFLAKE_TEST_PASSWORD }}
+          SNOWFLAKE_TEST_PRIVATE_KEY: ${{ secrets.SNOWFLAKE_TEST_PRIVATE_KEY }}
+          SNOWFLAKE_TEST_PRIVATE_KEY_PASSPHRASE: ${{ secrets.SNOWFLAKE_TEST_PRIVATE_KEY_PASSPHRASE }}
           SNOWFLAKE_TEST_SCHEMA_NAME: ${{ secrets.SNOWFLAKE_TEST_SCHEMA_NAME }}
           SNOWFLAKE_TEST_USER_LOGIN_NAME: ${{ secrets.SNOWFLAKE_TEST_USER_LOGIN_NAME }}
           SNOWFLAKE_TEST_WAREHOUSE_NAME: ${{ secrets.SNOWFLAKE_TEST_WAREHOUSE_NAME }}

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -15,25 +15,31 @@ The types of changes are:
 - `Fixed` for any bug fixes.
 - `Security` in case of vulnerabilities.
 
-## [Unreleased](https://github.com/ethyca/fides/compare/2.40.0...main)
+## [2.41.0](https://github.com/ethyca/fides/compare/2.40.0...2.41.0)
 
 - Add AWS Tags in the meta field for Fides system when using `fides generate` [#4998](https://github.com/ethyca/fides/pull/4998).
 
 ### Added
 - Added erasure support for Alchemer integration [#4925](https://github.com/ethyca/fides/pull/4925)
-
-### Developer Experience
-- Upgrade to React 18 and Chakra 2, including other dependencies [#5036](https://github.com/ethyca/fides/pull/5036)
-- Added support for "output templates" in read SaaS requests [#5054](https://github.com/ethyca/fides/pull/5054)
-
+- Added new columns and action buttons to discovery monitors table [#5068](https://github.com/ethyca/fides/pull/5068)
+- Added field to exclude databases on MonitorConfig [#5080](https://github.com/ethyca/fides/pull/5080)
+- Added key pair authentication for the Snowflake integration [#5079](https://github.com/ethyca/fides/pull/5079)
 
 ### Changed
 - Updated the sample dataset for the Amplitude integration [#5063](https://github.com/ethyca/fides/pull/5063)
+- Updated System's page to display a table that uses a paginated endpoint [#5084](https://github.com/ethyca/fides/pull/5084)
 - Messaging page now shows a notice if you have properties without any templates [#5077](https://github.com/ethyca/fides/pull/5077)
 - Endpoints for listing systems (GET /system) and datasets (GET /dataset) now support optional pagination [#5071](https://github.com/ethyca/fides/pull/5071)
+- Messaging page will now show a notice about using global mode [#5090](https://github.com/ethyca/fides/pull/5090)
+- Changed behavior of project selection modal in discovery monitor form [#5092](https://github.com/ethyca/fides/pull/5092)
+- Data category selector for Discovery results won't show disabled categories [#5102](https://github.com/ethyca/fides/pull/5102)
+
 
 ### Developer Experience
 - Upgrade to React 18 and Chakra 2, including other dependencies [#5036](https://github.com/ethyca/fides/pull/5036)
+- Added support for "output templates" in read SaaS requests [#5054](https://github.com/ethyca/fides/pull/5054)
+- URL for deployment instructions when the webserver is running [#5088](https://github.com/ethyca/fides/pull/5088)
+- Optimize TCF bundle with just-in-time GVL translations [#5074](https://github.com/ethyca/fides/pull/5074)
 
 ### Fixed
 - Fixed bug with unescaped table names in mysql queries [#5072](https://github.com/ethyca/fides/pull/5072/)

diff --git a/clients/admin-ui/cypress/e2e/auth.cy.ts b/clients/admin-ui/cypress/e2e/auth.cy.ts
@@ -60,12 +60,12 @@ describe("User Authentication", () => {
       cy.getByTestId("User Management");
 
       cy.visit(SYSTEM_ROUTE);
-      cy.getByTestId("Systems");
+      cy.getByTestId("system-management");
     });
 
     it("lets them log out", () => {
       cy.visit(SYSTEM_ROUTE);
-      cy.getByTestId("Systems");
+      cy.getByTestId("system-management");
 
       cy.intercept("POST", "/api/v1/logout", {
         statusCode: 204,

diff --git a/clients/admin-ui/cypress/e2e/discovery-detection.cy.ts b/clients/admin-ui/cypress/e2e/discovery-detection.cy.ts
@@ -348,8 +348,8 @@ describe("discovery and detection", () => {
 
       it("allows classifications to be changed using the dropdown", () => {
         cy.intercept("GET", "/api/v1/data_category", [
-          { fides_key: "system" },
-          { fides_key: "user.contact" },
+          { fides_key: "system", active: true },
+          { fides_key: "user.contact", active: true },
         ]);
         cy.intercept("PATCH", "/api/v1/plus/discovery-monitor/*/results").as(
           "patchClassification"

diff --git a/clients/admin-ui/cypress/e2e/integration-management.cy.ts b/clients/admin-ui/cypress/e2e/integration-management.cy.ts
@@ -220,8 +220,14 @@ describe("Integration management for data detection & discovery", () => {
           fixture: "detection-discovery/monitors/monitor_list.json",
         }).as("getMonitors");
         cy.intercept("/api/v1/plus/discovery-monitor/databases", {
-          fixture: "detection-discovery/monitors/database_list.json",
-        }).as("getDatabases");
+          fixture: "detection-discovery/monitors/database_list_page_1.json",
+        }).as("getDatabasesPage1");
+        cy.intercept("POST", "/api/v1/plus/discovery-monitor/*/execute").as(
+          "executeMonitor"
+        );
+        cy.intercept("DELETE", "/api/v1/plus/discovery-monitor/*").as(
+          "deleteMonitor"
+        );
         cy.getByTestId("tab-Data discovery").click();
         cy.wait("@getMonitors");
       });
@@ -238,7 +244,7 @@ describe("Integration management for data detection & discovery", () => {
         cy.selectOption("input-execution_frequency", "Daily");
         cy.getByTestId("input-execution_start_date").type("2034-06-03T10:00");
         cy.getByTestId("next-btn").click();
-        cy.wait("@getDatabases");
+        cy.wait("@getDatabasesPage1");
         cy.getByTestId("prj-bigquery-000001-checkbox").click();
         cy.getByTestId("save-btn").click();
         cy.wait("@putMonitor").then((interception) => {
@@ -247,6 +253,50 @@ describe("Integration management for data detection & discovery", () => {
         cy.wait("@getMonitors");
       });
 
+      it("can exclude databases", () => {
+        cy.intercept("PUT", "/api/v1/plus/discovery-monitor*").as("putMonitor");
+        cy.getByTestId("add-monitor-btn").click();
+        cy.getByTestId("add-modal-content").should("be.visible");
+        cy.getByTestId("input-name").type("A new monitor");
+        cy.selectOption("input-execution_frequency", "Daily");
+        cy.getByTestId("input-execution_start_date").type("2034-06-03T10:00");
+        cy.getByTestId("next-btn").click();
+        cy.wait("@getDatabasesPage1");
+        cy.getByTestId("select-all-checkbox").click();
+        cy.getByTestId("prj-bigquery-000001-checkbox").should(
+          "have.attr",
+          "data-checked"
+        );
+        cy.getByTestId("prj-bigquery-000001-checkbox").click();
+        cy.getByTestId("save-btn").click();
+        cy.wait("@putMonitor").then((interception) => {
+          expect(interception.request.body.excluded_databases).to.contain(
+            "prj-bigquery-000001"
+          );
+        });
+      });
+
+      it("can load more databases", () => {
+        cy.intercept("PUT", "/api/v1/plus/discovery-monitor*").as("putMonitor");
+        cy.getByTestId("add-monitor-btn").click();
+        cy.getByTestId("add-modal-content").should("be.visible");
+        cy.getByTestId("input-name").type("A new monitor");
+        cy.selectOption("input-execution_frequency", "Daily");
+        cy.getByTestId("input-execution_start_date").type("2034-06-03T10:00");
+        cy.getByTestId("next-btn").click();
+        cy.wait("@getDatabasesPage1");
+        cy.getByTestId("select-all-checkbox").click();
+        cy.intercept("POST", "/api/v1/plus/discovery-monitor/databases", {
+          fixture: "detection-discovery/monitors/database_list_page_2.json",
+        }).as("getDatabasesPage2");
+        cy.getByTestId("load-more-btn").click();
+        cy.wait("@getDatabasesPage2");
+        cy.getByTestId("prj-bigquery-000026-checkbox").should(
+          "have.attr",
+          "data-checked"
+        );
+      });
+
       it("can edit an existing monitor by clicking the edit button", () => {
         cy.intercept("PUT", "/api/v1/plus/discovery-monitor*").as("putMonitor");
         cy.getByTestId("row-test monitor 1").within(() => {
@@ -268,7 +318,7 @@ describe("Integration management for data detection & discovery", () => {
         cy.getByTestId("prj-bigquery-000003-checkbox").click();
         cy.getByTestId("save-btn").click();
         cy.wait("@putMonitor").then((interception) => {
-          expect(interception.request.body.databases).to.length(0);
+          expect(interception.request.body.databases).to.length(3);
         });
       });
 
@@ -280,6 +330,31 @@ describe("Integration management for data detection & discovery", () => {
           "Weekly"
         );
       });
+
+      it("can execute a monitor", () => {
+        cy.getByTestId("row-test monitor 1").within(() => {
+          cy.getByTestId("action-Scan").click();
+        });
+        cy.wait("@executeMonitor");
+      });
+
+      it("can delete a monitor", () => {
+        cy.getByTestId("row-test monitor 1").within(() => {
+          cy.getByTestId("delete-monitor-btn").click();
+        });
+        cy.getByTestId("confirmation-modal").within(() => {
+          cy.getByTestId("continue-btn").click();
+        });
+        cy.wait("@deleteMonitor");
+      });
+
+      it("can enable/disable a monitor", () => {
+        cy.intercept("PUT", "/api/v1/plus/discovery-monitor*").as("putMonitor");
+        cy.getByTestId("row-test monitor 1").within(() => {
+          cy.getByTestId("toggle-switch").click();
+        });
+        cy.wait("@putMonitor");
+      });
     });
 
     describe("data discovery tab with no projects/databases", () => {

diff --git a/clients/admin-ui/cypress/e2e/nav-bar.cy.ts b/clients/admin-ui/cypress/e2e/nav-bar.cy.ts
@@ -13,7 +13,7 @@ describe("Nav Bar", () => {
       cy.getByTestId("Home-nav-link");
     });
     cy.getByTestId("Data inventory-nav-group").within(() => {
-      cy.getByTestId("Systems & vendors-nav-link");
+      cy.getByTestId("System inventory-nav-link");
       cy.getByTestId("Add systems-nav-link");
       cy.getByTestId("Manage datasets-nav-link");
     });
@@ -50,18 +50,18 @@ describe("Nav Bar", () => {
     cy.getByTestId("Home-nav-link")
       .should("have.css", "background-color")
       .should("eql", ACTIVE_COLOR);
-    cy.getByTestId("Systems & vendors-nav-link")
+    cy.getByTestId("System inventory-nav-link")
       .should("have.css", "background-color")
       .should("not.eql", ACTIVE_COLOR);
 
     // Navigate by clicking a nav link.
-    cy.getByTestId("Systems & vendors-nav-link").click();
+    cy.getByTestId("System inventory-nav-link").click();
 
     // The nav should update which page is active.
     cy.getByTestId("Home-nav-link")
       .should("have.css", "background-color")
       .should("not.eql", ACTIVE_COLOR);
-    cy.getByTestId("Systems & vendors-nav-link")
+    cy.getByTestId("System inventory-nav-link")
       .should("have.css", "background-color")
       .should("eql", ACTIVE_COLOR);
   });
@@ -75,7 +75,7 @@ describe("Nav Bar", () => {
     cy.getByTestId("Request manager-nav-link").should("not.be.visible");
 
     // Move to another page
-    cy.getByTestId("Systems & vendors-nav-link").click();
+    cy.getByTestId("System inventory-nav-link").click();
     cy.getByTestId("Request manager-nav-link").should("not.be.visible");
   });
 });
diff --git a/clients/admin-ui/cypress/e2e/system-integrations-plus.cy.ts b/clients/admin-ui/cypress/e2e/system-integrations-plus.cy.ts
@@ -5,8 +5,8 @@ import { SYSTEM_ROUTE } from "~/features/common/nav/v2/routes";
 describe("System integrations", () => {
   beforeEach(() => {
     cy.login();
-    cy.intercept("GET", "/api/v1/system", {
-      fixture: "systems/systems.json",
+    cy.intercept("GET", "/api/v1/system*", {
+      fixture: "systems/systems_paginated.json",
     }).as("getSystems");
     cy.intercept("GET", "/api/v1/connection_type*", {
       fixture: "connectors/connection_types.json",
@@ -24,7 +24,6 @@ describe("System integrations", () => {
 
   it("should render the integration configuration panel when navigating to integrations tab", () => {
     cy.getByTestId("system-fidesctl_system").within(() => {
-      cy.getByTestId("more-btn").click();
       cy.getByTestId("edit-btn").click();
     });
     cy.wait("@getDict");
@@ -35,7 +34,6 @@ describe("System integrations", () => {
   describe("Integration search", () => {
     beforeEach(() => {
       cy.getByTestId("system-fidesctl_system").within(() => {
-        cy.getByTestId("more-btn").click();
         cy.getByTestId("edit-btn").click();
       });
       cy.getByTestId("tab-Integrations").click();
@@ -60,7 +58,6 @@ describe("System integrations", () => {
   describe("Integration form contents", () => {
     beforeEach(() => {
       cy.getByTestId("system-fidesctl_system").within(() => {
-        cy.getByTestId("more-btn").click();
         cy.getByTestId("edit-btn").click();
       });
       cy.getByTestId("tab-Integrations").click();

diff --git a/clients/admin-ui/cypress/e2e/system-integrations.cy.ts b/clients/admin-ui/cypress/e2e/system-integrations.cy.ts
@@ -5,9 +5,10 @@ import { SYSTEM_ROUTE } from "~/features/common/nav/v2/routes";
 describe("System integrations", () => {
   beforeEach(() => {
     cy.login();
-    cy.intercept("GET", "/api/v1/system", {
-      fixture: "systems/systems.json",
+    cy.intercept("GET", "/api/v1/system*", {
+      fixture: "systems/systems_paginated.json",
     }).as("getSystems");
+
     cy.intercept("GET", "/api/v1/connection_type*", {
       fixture: "connectors/connection_types.json",
     }).as("getConnectionTypes");
@@ -21,7 +22,6 @@ describe("System integrations", () => {
 
   it("should render the integration configuration panel when navigating to integrations tab", () => {
     cy.getByTestId("system-fidesctl_system").within(() => {
-      cy.getByTestId("more-btn").click();
       cy.getByTestId("edit-btn").click();
     });
     cy.getByTestId("tab-Integrations").click();
@@ -31,7 +31,6 @@ describe("System integrations", () => {
   describe("Integration search", () => {
     beforeEach(() => {
       cy.getByTestId("system-fidesctl_system").within(() => {
-        cy.getByTestId("more-btn").click();
         cy.getByTestId("edit-btn").click();
       });
       cy.getByTestId("tab-Integrations").click();
@@ -56,7 +55,6 @@ describe("System integrations", () => {
   describe("Integration form contents", () => {
     beforeEach(() => {
       cy.getByTestId("system-fidesctl_system").within(() => {
-        cy.getByTestId("more-btn").click();
         cy.getByTestId("edit-btn").click();
       });
       cy.getByTestId("tab-Integrations").click();