Merge branch 'main' of https://github.com/holonym-foundation/passport

app/__tests__/context/ceramicContext.test.tsx

@@ -284,7 +284,7 @@ describe("CeramicContextProvider syncs stamp state with ceramic", () => {
       await waitFor(() => {
         expect(addStampsMock).toHaveBeenCalled();
         expect(addStampMock).toHaveBeenCalledWith(stamps);
-        expect(console.log).toHaveBeenCalledWith("error setting ceramic stamps", new Error("Error"));
+        expect(console.log).toHaveBeenCalledWith("error adding ceramic stamps", new Error("Error"));
       });
     } finally {
       console.log = oldConsoleLog;

app/context/ceramicContext.tsx

@@ -1,15 +1,18 @@
 import { createContext, useContext, useEffect, useState, useRef, useMemo } from "react";
 import {
+  ComposeDBMetadataRequest,
+  ComposeDBSaveStatus,
   Passport,
   PassportLoadResponse,
   PassportLoadStatus,
   PLATFORM_ID,
   PROVIDER_ID,
+  SecondaryStorageBulkPatchResponse,
   Stamp,
   StampPatch,
 } from "@gitcoin/passport-types";
 import { ProviderSpec } from "../config/providers";
-import { CeramicStorage, DataStorageBase, ComposeDatabase, PassportDatabase } from "@gitcoin/passport-database-client";
+import { DataStorageBase, ComposeDatabase, PassportDatabase } from "@gitcoin/passport-database-client";
 import { datadogLogs } from "@datadog/browser-logs";
 import { datadogRum } from "@datadog/browser-rum";
 import { useWalletStore } from "./walletStore";
@@ -292,7 +295,7 @@ export const CeramicContext = createContext(startingState);
 
 export const cleanPassport = (
   passport: Passport,
-  database: CeramicStorage | DataStorageBase,
+  database: DataStorageBase,
   allProvidersState: AllProvidersState
 ): {
   passport: Passport;
@@ -410,7 +413,7 @@ export const CeramicContextProvider = ({ children }: { children: any }) => {
   }, [ceramicClient]);
 
   const passportLoadSuccess = (
-    database: ComposeDatabase | PassportDatabase,
+    database: PassportDatabase,
     passport?: Passport,
     skipLoadingState?: boolean
   ): Passport => {
@@ -441,7 +444,7 @@ export const CeramicContextProvider = ({ children }: { children: any }) => {
 
   const handlePassportUpdate = async (
     passportResponse: PassportLoadResponse,
-    database: ComposeDatabase | PassportDatabase,
+    database: PassportDatabase,
     skipLoadingState?: boolean,
     isInitialLoad?: boolean
   ) => {
@@ -475,7 +478,7 @@ export const CeramicContextProvider = ({ children }: { children: any }) => {
   };
 
   const fetchPassport = async (
-    database: ComposeDatabase | PassportDatabase,
+    database: PassportDatabase,
     skipLoadingState?: boolean,
     isInitialLoad?: boolean
   ): Promise<Passport | undefined> => {
@@ -507,9 +510,19 @@ export const CeramicContextProvider = ({ children }: { children: any }) => {
         handlePassportUpdate(addResponse, database);
 
         if (ceramicClient && addResponse.passport) {
-          ceramicClient
-            .addStamps(addResponse.passport.stamps)
-            .catch((e) => console.log("error setting ceramic stamps", e));
+          (async () => {
+            try {
+              const composeDBAddResponse = await ceramicClient.addStamps(stamps);
+              const composeDBMetadata = processComposeDBMetadata(addResponse.passport, {
+                adds: composeDBAddResponse,
+                deletes: [],
+              });
+              await database.patchStampComposeDBMetadata(composeDBMetadata);
+            } catch (e) {
+              console.log("error adding ceramic stamps", e);
+              datadogLogs.logger.error("Error adding ceramic stamps", { stamps, error: e });
+            }
+          })();
         }
         if (dbAccessToken) {
           refreshScore(address, dbAccessToken);
@@ -521,6 +534,51 @@ export const CeramicContextProvider = ({ children }: { children: any }) => {
     }
   };
 
+  const processComposeDBMetadata = (
+    updatedPassport: Passport | undefined,
+    composeDBPatchResponse: SecondaryStorageBulkPatchResponse
+  ): ComposeDBMetadataRequest[] => {
+    composeDBPatchResponse.deletes.forEach(({ secondaryStorageId, secondaryStorageError }) => {
+      if (secondaryStorageError) {
+        console.log(`Failed to delete stamp ${secondaryStorageId} from secondary storage: ${secondaryStorageError}`);
+        datadogLogs.logger.error(
+          `Failed to delete stamp ${secondaryStorageId} from secondary storage: ${secondaryStorageError}`
+        );
+      }
+    });
+
+    return composeDBPatchResponse.adds
+      .map((addResponse) => {
+        const { provider, secondaryStorageId, secondaryStorageError } = addResponse;
+
+        if (secondaryStorageError) {
+          console.log(
+            `Failed to add stamp ${secondaryStorageId} to secondary storage, error: ${secondaryStorageError}`
+          );
+          datadogLogs.logger.error("Error adding stamp to secondary storage", {
+            stamp: addResponse,
+            error: secondaryStorageError,
+          });
+        }
+
+        const primaryStorageId = updatedPassport?.stamps.find((stamp) => stamp.provider === provider)?.id;
+
+        if (!primaryStorageId) {
+          console.log(`Stamp ID not found for provider ${provider} when adding to secondary storage`);
+          datadogLogs.logger.error(`Stamp ID not found for provider ${provider} when adding to secondary storage`);
+        } else {
+          const compose_db_save_status: ComposeDBSaveStatus =
+            !secondaryStorageError && secondaryStorageId ? "saved" : "failed";
+          return {
+            id: primaryStorageId,
+            compose_db_stream_id: secondaryStorageId,
+            compose_db_save_status,
+          };
+        }
+      })
+      .filter((v?: ComposeDBMetadataRequest): v is ComposeDBMetadataRequest => Boolean(v));
+  };
+
   const handlePatchStamps = async (stampPatches: StampPatch[]): Promise<void> => {
     try {
       if (database) {
@@ -530,9 +588,12 @@ export const CeramicContextProvider = ({ children }: { children: any }) => {
         if (ceramicClient && patchResponse.passport) {
           (async () => {
             try {
-              await ceramicClient.patchStamps(stampPatches);
+              const composeDBPatchResponse = await ceramicClient.patchStamps(stampPatches);
+              const composeDBMetadata = processComposeDBMetadata(patchResponse.passport, composeDBPatchResponse);
+              await database.patchStampComposeDBMetadata(composeDBMetadata);
             } catch (e) {
               console.log("error patching ceramic stamps", e);
+              datadogLogs.logger.error("Error patching ceramic stamps", { stampPatches, error: e });
             }
           })();
         }
@@ -553,7 +614,15 @@ export const CeramicContextProvider = ({ children }: { children: any }) => {
         const deleteResponse = await database.deleteStamps(providerIds);
         handlePassportUpdate(deleteResponse, database);
         if (ceramicClient && deleteResponse.status === "Success" && deleteResponse.passport?.stamps) {
-          ceramicClient.deleteStamps(providerIds).catch((e) => console.log("error setting ceramic stamps", e));
+          (async () => {
+            try {
+              const responses = await ceramicClient.deleteStamps(providerIds);
+              processComposeDBMetadata(deleteResponse.passport, { adds: [], deletes: responses });
+            } catch (e) {
+              console.log("error deleting ceramic stamps", e);
+              datadogLogs.logger.error("Error deleting ceramic stamps", { providerIds, error: e });
+            }
+          })();
         }
         if (dbAccessToken) {
           refreshScore(address, dbAccessToken);

database-client/__tests__/composeDatabase.test.ts

@@ -153,6 +153,21 @@ const mockComposeVc = {
     },
   },
 };
+
+const mockWrapperId = "456";
+const executeQueryMockReturn = {
+  data: {
+    createGitcoinPassportStamp: {
+      document,
+    },
+    createGitcoinPassportStampWrapper: {
+      document: {
+        id: mockWrapperId,
+      },
+    },
+  },
+};
+
 describe("Compose Database", () => {
   beforeEach(() => {
     database = new ComposeDatabase({ id: "id" } as unknown as DID);
@@ -164,47 +179,39 @@ describe("Compose Database", () => {
   });
   describe("adding stamps", () => {
     it("should add a single stamp successfully", async () => {
-      jest
-        .spyOn(ComposeClient.prototype, "executeQuery")
-        .mockResolvedValue({ data: { createGitcoinPassportStamp: { document } } });
+      jest.spyOn(ComposeClient.prototype, "executeQuery").mockResolvedValue(executeQueryMockReturn);
 
       const result = await database.addStamp(mockStamps[0] as unknown as Stamp);
-      expect(result).toEqual({ status: "Success" });
+      console.log("RESULT", result);
+      expect(result.secondaryStorageError).toBeUndefined();
+      expect(result.secondaryStorageId).toEqual(mockWrapperId);
     });
     it("should indicate that an error was thrown from the add vc request", async () => {
       jest.spyOn(ComposeClient.prototype, "executeQuery").mockResolvedValueOnce(mockComposeError);
-      await expect(async () => {
-        return await database.addStamp(mockStamps[0] as unknown as Stamp);
-      }).rejects.toThrow(
-        new Error(
-          `[ComposeDB] error thrown from mutation CreateGitcoinPassportVc, error: ` +
-            JSON.stringify(mockComposeError.errors)
-        )
+      const addStampResponse = await database.addStamp(mockStamps[0] as unknown as Stamp);
+      expect(addStampResponse.secondaryStorageError).toEqual(
+        `[ComposeDB] error from mutation CreateGitcoinPassportVc, error: ` + JSON.stringify(mockComposeError.errors)
       );
     });
     it("should indicate that an error was thrown from the add wrapper request", async () => {
-      jest
-        .spyOn(ComposeClient.prototype, "executeQuery")
-        .mockResolvedValueOnce({ data: { createGitcoinPassportStamp: { document } } });
+      jest.spyOn(ComposeClient.prototype, "executeQuery").mockResolvedValueOnce(executeQueryMockReturn);
       jest.spyOn(ComposeClient.prototype, "executeQuery").mockResolvedValueOnce(mockComposeError);
-      await expect(async () => {
-        return await database.addStamp(mockStamps[0] as unknown as Stamp);
-      }).rejects.toThrow(
-        new Error(
-          `[ComposeDB] error thrown from mutation CreateGitcoinStampWrapper, vcID: ${
-            document.id
-          } error: ${JSON.stringify(mockComposeError.errors)}`
-        )
+      const addStampResponse = await database.addStamp(mockStamps[0] as unknown as Stamp);
+      expect(addStampResponse.secondaryStorageError).toEqual(
+        `[ComposeDB] error thrown from mutation CreateGitcoinStampWrapper, vcID: ${document.id} error: ${JSON.stringify(
+          mockComposeError.errors
+        )}`
       );
     });
 
     it("should allow bulk addition of stamps", async () => {
-      jest
-        .spyOn(ComposeClient.prototype, "executeQuery")
-        .mockResolvedValue({ data: { createGitcoinPassportStamp: { document } } });
+      jest.spyOn(ComposeClient.prototype, "executeQuery").mockResolvedValue(executeQueryMockReturn);
 
       const result = await database.addStamps(mockStamps as unknown as Stamp[]);
-      expect(result).toEqual({ status: "Success" });
+      result.forEach((stamp) => {
+        expect(stamp.secondaryStorageError).toBeUndefined();
+        expect(stamp.secondaryStorageId).toEqual(mockWrapperId);
+      });
     });
 
     it("should indicate where errors were thrown when creating bulk addition of stamps", async () => {
@@ -216,12 +223,12 @@ describe("Compose Database", () => {
           if (i > 3) {
             return await Promise.resolve(mockComposeError);
           } else {
-            return await Promise.resolve({ data: { createGitcoinPassportStamp: { document } } });
+            return await Promise.resolve(executeQueryMockReturn);
           }
         });
       const result = await database.addStamps(mockStamps as unknown as Stamp[]);
-      expect(result.status).toEqual("ExceptionRaised");
-      expect(result.errorDetails?.messages?.length).toEqual(3);
+      const errorResults = result.filter(({ secondaryStorageError }) => secondaryStorageError);
+      expect(errorResults.length).toEqual(3);
     });
   });
   describe("getting stamps", () => {
@@ -249,7 +256,10 @@ describe("Compose Database", () => {
         .mockResolvedValueOnce({ data: { deleteGitcoinPassportStamp: { document: { id: "123" } } } });
       const results = await database.deleteStamps(mockStamps.map((stamp) => stamp.provider as PROVIDER_ID));
       expect(ComposeClient.prototype.executeQuery).toHaveBeenCalled();
-      expect(results).toEqual({ status: "Success" });
+      results.forEach((stamp) => {
+        expect(stamp.secondaryStorageError).toBeUndefined();
+        expect(stamp.secondaryStorageId).toEqual(mockWrapperId);
+      });
     });
     it("indicate the an error occurred while querying", async () => {
       jest
@@ -270,19 +280,22 @@ describe("Compose Database", () => {
         });
       // jest.spyOn(ComposeClient.prototype, "executeQuery").mockResolvedValueOnce(mockComposeError);
       const result = await database.deleteStamps(mockStamps.map((stamp) => stamp.provider as PROVIDER_ID));
-      expect(result.status).toEqual("ExceptionRaised");
+      const errorResults = result.filter(({ secondaryStorageError }) => secondaryStorageError);
+      expect(errorResults.length).toEqual(1);
       expect(ComposeClient.prototype.executeQuery).toHaveBeenCalledTimes(2);
     });
   });
   describe("patching stamps", () => {
     it("should patch stamps successfully when passport has no existing stamps", async () => {
-      jest
-        .spyOn(ComposeClient.prototype, "executeQuery")
-        .mockResolvedValue({ data: { createGitcoinPassportStamp: { document: { id: "123" } } } });
+      jest.spyOn(ComposeClient.prototype, "executeQuery").mockResolvedValue(executeQueryMockReturn);
 
       const result = await database.patchStamps(mockStamps as unknown as StampPatch[]);
       expect(ComposeClient.prototype.executeQuery).toHaveBeenCalledTimes(7);
-      expect(result.status).toEqual("Success");
+      const errorResults = [
+        ...result.adds.filter(({ secondaryStorageError }) => secondaryStorageError),
+        ...result.deletes.filter(({ secondaryStorageError }) => secondaryStorageError),
+      ];
+      expect(errorResults.length).toEqual(0);
     });
 
     it("should delete existing stamps and create new ones", async () => {
@@ -299,12 +312,16 @@ describe("Compose Database", () => {
               },
             });
           } else {
-            return await Promise.resolve({ data: { createGitcoinPassportStamp: { document: { id: "123" } } } });
+            return await Promise.resolve(executeQueryMockReturn);
           }
         });
       const result = await database.patchStamps(mockStamps as unknown as StampPatch[]);
       expect(ComposeClient.prototype.executeQuery).toHaveBeenCalledTimes(8);
-      expect(result.status).toEqual("Success");
+      const errorResults = [
+        ...result.adds.filter(({ secondaryStorageError }) => secondaryStorageError),
+        ...result.deletes.filter(({ secondaryStorageError }) => secondaryStorageError),
+      ];
+      expect(errorResults.length).toEqual(0);
     });
   });
 });

database-client/integration-tests/ceramicDatabaseStampValidity.ts

@@ -65,7 +65,7 @@ describe("assuming a valid stamp is stored in ceramic", () => {
 
     // Step 2: Write the stamp to compose
     const addRequest = await composeDatabase.addStamps(stampsToAdd);
-    expect(addRequest.status).toEqual("Success");
+    expect(addRequest[0].secondaryStorageError).toBeUndefined();
 
     // Step 3: Read the user passport
     const result = await composeDatabase.getPassport();

database-client/integration-tests/ceramicDatabaseTest.ts

@@ -42,12 +42,12 @@ describe("adding and deleting stamps", () => {
     expect(result.passport.stamps.length).toEqual(1);
     expect(result.passport.stamps[0].provider).toEqual(stampsToAdd[0].provider);
   });
+
   it("should indicate an error when adding a stamp", async () => {
     const result = await composeDatabase.addStamps([badStamp]);
-    expect(result.status).toEqual("ExceptionRaised");
-    expect(result.errorDetails).toBeDefined();
-    expect(result.errorDetails?.messages.length).toEqual(1);
+    expect(result.filter(({ secondaryStorageError }) => secondaryStorageError).length).toEqual(1);
   });
+
   it("should delete stamps from compose-db", async () => {
     let passportResult = await composeDatabase.getPassport();
     expect(passportResult.status).toEqual("Success");
@@ -64,9 +64,7 @@ describe("adding and deleting stamps", () => {
     expect(passportResult.passport.stamps.length).toEqual(0);
 
     const result = await composeDatabase.addStamps([badStamp, stampsToAdd[0]]);
-    expect(result.status).toEqual("ExceptionRaised");
-    expect(result.errorDetails).toBeDefined();
-    expect(result.errorDetails?.messages.length).toEqual(1);
+    expect(result.filter(({ secondaryStorageError }) => secondaryStorageError).length).toEqual(1);
 
     let newPassport = await composeDatabase.getPassport();
     expect(newPassport.status).toEqual("Success");
@@ -126,8 +124,6 @@ describe("updating an existing passport", () => {
   });
   it("should indicate that an error was thrown while patching stamps", async () => {
     const result = await composeDatabase.patchStamps([badStamp]);
-    expect(result.status).toEqual("ExceptionRaised");
-    expect(result.errorDetails).toBeDefined();
-    expect(result.errorDetails?.messages.length).toEqual(1);
+    expect(result.adds.filter(({ secondaryStorageError }) => secondaryStorageError).length).toEqual(1);
   });
 });