Fix subgraph template IPFS URL (#997)

* fix ci attempt * reorder * move type tests to lint * fix thegraph ipfs fetch, speed up tests, add --skip-install command * chore: changeset * rm log
ponder-sh · Jul 24, 2024 · 0afd6fa · 0afd6fa
1 parent 44f7bbc
commit 0afd6fa
Show file tree

Hide file tree

Showing 7 changed files with 142 additions and 122 deletions.
diff --git a/.changeset/strong-guests-tan.md b/.changeset/strong-guests-tan.md
@@ -0,0 +1,5 @@
+---
+"create-ponder": patch
+---
+
+Fixed an issue where the `create-ponder` subgraph template would not fetch IPFS files correctly. Added a `--skip-install` option to `create-ponder`.
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -27,6 +27,9 @@ jobs:
       - name: Typecheck
         run: pnpm typecheck
 
+      - name: Test (types)
+        run: pnpm --filter core test:typecheck
+
       - name: Lint
         run: pnpm lint
 
@@ -67,7 +70,7 @@ jobs:
         run: cd packages/core/ && pnpm wagmi generate
 
       - name: Test
-        run: pnpm --filter core test:typecheck
+        run: pnpm --filter core test
         env:
           DATABASE_URL: ${{ matrix.database == 'Postgres' && steps.postgres.outputs.connection-uri || '' }}
 

diff --git a/docs/pages/docs/api-reference/create-ponder.mdx b/docs/pages/docs/api-reference/create-ponder.mdx
@@ -37,13 +37,14 @@ Usage:
 Options:
   -t, --template [id]             Use a template
   --etherscan [url]               Use the Etherscan template with the specified contract URL
+  --etherscan-api-key [key]       Etherscan API key for Etherscan template
   --subgraph [id]                 Use the subgraph template with the specified subgraph ID
   --subgraph-provider [provider]  Specify the subgraph provider
   --npm                           Use npm as your package manager
   --pnpm                          Use pnpm as your package manager
   --yarn                          Use yarn as your package manager
   --skip-git                      Skip initializing a git repository
-  --etherscan-api-key [key]       Etherscan API key for Etherscan template
+  --skip-install                  Skip installing packages
   -h, --help                      Display this message
   -v, --version                   Display version number
 ```

diff --git a/packages/core/package.json b/packages/core/package.json
@@ -29,7 +29,7 @@
   "scripts": {
     "build": "tsup",
     "test": "vitest",
-    "test:typecheck": "vitest --typecheck",
+    "test:typecheck": "vitest --typecheck.only",
     "typecheck": "tsc --noEmit"
   },
   "peerDependencies": {

diff --git a/packages/create-ponder/src/_test/cli.test.ts b/packages/create-ponder/src/_test/cli.test.ts
@@ -18,74 +18,93 @@ test("create empty", async () => {
 
   await run({
     args: [rootDir],
-    options: { template: "empty", skipGit: true },
+    options: { template: "empty", skipGit: true, skipInstall: true },
   });
 
-  const templateFiles = (
-    readdirSync(path.join(__dirname, "..", "..", "templates", "empty"), {
-      recursive: true,
-    }) as string[]
-  )
-    .map((filePath) =>
-      filePath === "_dot_env.local"
-        ? ".env.local"
-        : filePath === "_dot_eslintrc.json"
-          ? ".eslintrc.json"
-          : filePath === "_dot_gitignore"
-            ? ".gitignore"
-            : filePath,
-    )
-    .sort();
-
-  const generatedFiles = (readdirSync(rootDir, { recursive: true }) as string[])
-    .filter((f) => !f.startsWith("node_modules") && !f.startsWith("pnpm-lock"))
-    .sort();
-  expect(generatedFiles).toStrictEqual(templateFiles);
+  const files = readdirSync(rootDir, { recursive: true, encoding: "utf8" });
+
+  expect(files).toEqual(
+    expect.arrayContaining([
+      ".env.local",
+      ".eslintrc.json",
+      ".gitignore",
+      "package.json",
+      "ponder-env.d.ts",
+      "ponder.config.ts",
+      "ponder.schema.ts",
+      "tsconfig.json",
+      "abis/ExampleContractAbi.ts",
+      "src/index.ts",
+      "src/api/index.ts",
+    ]),
+  );
 });
 
-test("create subgraph id", async () => {
+test("create subgraph thegraph", async () => {
   const rootDir = path.join(tempDir, "subgraph-id");
 
   await run({
     args: [rootDir],
     options: {
       template: "subgraph",
-      skipGit: true,
       subgraph: "QmeCy8bjyudQYwddetkgKDKQTsFxWomvbgsDifnbWFEMK9",
+      subgraphProvider: "thegraph",
+      skipGit: true,
+      skipInstall: true,
+    },
+  });
+
+  const files = readdirSync(rootDir, { recursive: true, encoding: "utf8" });
+
+  expect(files).toEqual(
+    expect.arrayContaining([
+      ".env.local",
+      ".eslintrc.json",
+      ".gitignore",
+      "package.json",
+      "ponder-env.d.ts",
+      "ponder.config.ts",
+      "ponder.schema.ts",
+      "tsconfig.json",
+      "abis/ERC20Abi.ts",
+      "abis/ERC721Abi.ts",
+      "abis/EntryPointAbi.ts",
+      "src/EntryPoint.ts",
+      "src/EntryPointV0.6.0.ts",
+    ]),
+  );
+});
+
+test("create subgraph satsuma", async () => {
+  const rootDir = path.join(tempDir, "subgraph-id");
+
+  await run({
+    args: [rootDir],
+    options: {
+      template: "subgraph",
+      subgraph: "QmbjiXHX5E7VypxH2gRcdySEXsvSUo7Aocuypr7m9u6Na9",
+      subgraphProvider: "satsuma",
+      skipGit: true,
+      skipInstall: true,
     },
   });
 
-  const templateFiles = (
-    readdirSync(path.join(__dirname, "..", "..", "templates", "subgraph"), {
-      recursive: true,
-    }) as string[]
-  )
-    .concat([
+  const files = readdirSync(rootDir, { recursive: true, encoding: "utf8" });
+
+  expect(files).toEqual(
+    expect.arrayContaining([
+      ".env.local",
+      ".eslintrc.json",
+      ".gitignore",
+      "package.json",
+      "ponder-env.d.ts",
       "ponder.config.ts",
-      path.join("abis", "ERC20Abi.ts"),
-      path.join("abis", "ERC721Abi.ts"),
-      path.join("abis", "EntryPointAbi.ts"),
-      path.join("src", "EntryPoint.ts"),
-      path.join("src", "EntryPointV0.6.0.ts"),
-      "abis",
-      "src",
-    ])
-    // _gitignore is renamed to .gitignore
-    .map((filePath) =>
-      filePath === "_dot_env.local"
-        ? ".env.local"
-        : filePath === "_dot_eslintrc.json"
-          ? ".eslintrc.json"
-          : filePath === "_dot_gitignore"
-            ? ".gitignore"
-            : filePath,
-    )
-    .sort();
-
-  const generatedFiles = (readdirSync(rootDir, { recursive: true }) as string[])
-    .filter((f) => !f.startsWith("node_modules") && !f.startsWith("pnpm-lock"))
-    .sort();
-  expect(generatedFiles).toStrictEqual(templateFiles);
+      "ponder.schema.ts",
+      "tsconfig.json",
+      "abis/AchievementNFTAbi.ts",
+      "src/AchievementNFT.ts",
+    ]),
+  );
 });
 
 test("create etherscan", async () => {
@@ -95,39 +114,28 @@ test("create etherscan", async () => {
     args: [rootDir],
     options: {
       template: "etherscan",
-      skipGit: true,
       etherscanApiKey: process.env.ETHERSCAN_API_KEY!,
       etherscan:
         "https://etherscan.io/address/0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2",
+      skipGit: true,
+      skipInstall: true,
     },
   });
 
-  const templateFiles = (
-    readdirSync(path.join(__dirname, "..", "..", "templates", "etherscan"), {
-      recursive: true,
-    }) as string[]
-  )
-    .concat([
+  const files = readdirSync(rootDir, { recursive: true, encoding: "utf8" });
+
+  expect(files).toEqual(
+    expect.arrayContaining([
+      ".env.local",
+      ".eslintrc.json",
+      ".gitignore",
+      "package.json",
+      "ponder-env.d.ts",
       "ponder.config.ts",
-      path.join("abis", "WETH9Abi.ts"),
-      path.join("src", "WETH9.ts"),
-      "abis",
-      "src",
-    ])
-    // _gitignore is renamed to .gitignore
-    .map((filePath) =>
-      filePath === "_dot_env.local"
-        ? ".env.local"
-        : filePath === "_dot_eslintrc.json"
-          ? ".eslintrc.json"
-          : filePath === "_dot_gitignore"
-            ? ".gitignore"
-            : filePath,
-    )
-    .sort();
-
-  const generatedFiles = (readdirSync(rootDir, { recursive: true }) as string[])
-    .filter((f) => !f.startsWith("node_modules") && !f.startsWith("pnpm-lock"))
-    .sort();
-  expect(generatedFiles).toStrictEqual(templateFiles);
+      "ponder.schema.ts",
+      "tsconfig.json",
+      expect.stringMatching(/src\/(?:WETH9|UnverifiedContract)\.ts/),
+      expect.stringMatching(/abis\/(?:WETH9|UnverifiedContract)Abi\.ts/),
+    ]),
+  );
 });
diff --git a/packages/create-ponder/src/index.ts b/packages/create-ponder/src/index.ts
@@ -443,24 +443,26 @@ export async function run({
     "install",
     packageManager === "npm" ? "--quiet" : "--silent",
   ];
-  await oraPromise(
-    execa(packageManager, installArgs, {
-      cwd: projectPath,
-      env: {
-        ...process.env,
-        ADBLOCK: "1",
-        DISABLE_OPENCOLLECTIVE: "1",
-        // we set NODE_ENV to development as pnpm skips dev
-        // dependencies when production
-        NODE_ENV: "development",
+  if (!options.skipInstall) {
+    await oraPromise(
+      execa(packageManager, installArgs, {
+        cwd: projectPath,
+        env: {
+          ...process.env,
+          ADBLOCK: "1",
+          DISABLE_OPENCOLLECTIVE: "1",
+          // we set NODE_ENV to development as pnpm skips dev
+          // dependencies when production
+          NODE_ENV: "development",
+        },
+      }),
+      {
+        text: `Installing packages with ${pico.bold(packageManager)}. This may take a few seconds.`,
+        failText: "Failed to install packages.",
+        successText: `Installed packages with ${pico.bold(packageManager)}.`,
       },
-    }),
-    {
-      text: `Installing packages with ${pico.bold(packageManager)}. This may take a few seconds.`,
-      failText: "Failed to install packages.",
-      successText: `Installed packages with ${pico.bold(packageManager)}.`,
-    },
-  );
+    );
+  }
 
   // Create git repository
   if (!options.skipGit) {
@@ -526,6 +528,10 @@ export async function run({
       `Use a template. Options: ${templates.map(({ id }) => id).join(", ")}`,
     )
     .option("--etherscan [url]", "Use the Etherscan template")
+    .option(
+      "--etherscan-api-key [key]",
+      "Etherscan API key for Etherscan template",
+    )
     .option("--subgraph [id]", "Use the subgraph template")
     .option(
       "--subgraph-provider [provider]",
@@ -535,10 +541,7 @@ export async function run({
     .option("--pnpm", "Use pnpm as your package manager")
     .option("--yarn", "Use yarn as your package manager")
     .option("--skip-git", "Skip initializing a git repository")
-    .option(
-      "--etherscan-api-key [key]",
-      "Etherscan API key for Etherscan template",
-    )
+    .option("--skip-install", "Skip installing packages")
     .help();
 
   // Check Nodejs version

diff --git a/packages/create-ponder/src/subgraph.ts b/packages/create-ponder/src/subgraph.ts
@@ -14,46 +14,46 @@ export const subgraphProviders = [
   {
     id: "thegraph",
     name: "The Graph",
-    getUrl: (cid: string) =>
-      `https://ipfs.network.thegraph.com/api/v0/cat?arg=${cid}`,
+    // Used to be https://ipfs.network.thegraph.com/api/v0/cat?arg=${cid}
+    // Also used to accept GET requests for some reason
+    fetchIpfs: async (cid: string) => {
+      const response = await fetch(
+        `https://api.thegraph.com/ipfs/api/v0/cat?arg=${cid}`,
+        { method: "POST" },
+      );
+      return await response.text();
+    },
   },
   {
     id: "satsuma",
     name: "Alchemy Subgraph (Satsuma)",
-    getUrl: (cid: string) => `https://ipfs.satsuma.xyz/ipfs/${cid}`,
+    fetchIpfs: async (cid: string) => {
+      const response = await fetch(`https://ipfs.satsuma.xyz/ipfs/${cid}`);
+      return await response.text();
+    },
   },
 ] as const;
 
 type SubgraphProvider = (typeof subgraphProviders)[number];
 
 export type SubgraphProviderId = SubgraphProvider["id"];
 
-const fetchIpfsFile = async (
-  cid: string,
-  subgraphProvider: SubgraphProvider,
-) => {
-  const url = subgraphProvider.getUrl(cid);
-  const response = await fetch(url);
-  const contentRaw = await response.text();
-  return contentRaw;
-};
-
 export const fromSubgraphId = async ({
   rootDir,
   subgraphId,
-  subgraphProvider = "thegraph",
+  subgraphProvider,
 }: {
   rootDir: string;
   subgraphId: string;
-  subgraphProvider?: SubgraphProviderId;
+  subgraphProvider: SubgraphProviderId;
 }) => {
   // Find provider
   const provider = subgraphProviders.find((p) => p.id === subgraphProvider);
   if (!provider)
     throw new Error(`Unknown subgraph provider: ${subgraphProvider}`);
 
   // Fetch the manifest file.
-  const manifestRaw = await fetchIpfsFile(subgraphId, provider);
+  const manifestRaw = await provider.fetchIpfs(subgraphId);
 
   const manifest = parse(manifestRaw);
 
@@ -84,7 +84,7 @@ export const fromSubgraphId = async ({
 
   await Promise.all(
     abiFiles.map(async (abi) => {
-      const abiContent = await fetchIpfsFile(abi.file["/"].slice(6), provider);
+      const abiContent = await provider.fetchIpfs(abi.file["/"].slice(6));
       const abiPath = path.join(rootDir, `./abis/${abi.name}Abi.ts`);
       writeFileSync(
         abiPath,