Fetch proxy updates

packages/fetch/src/fetch.ts

@@ -5,7 +5,7 @@ import { HttpsProxyAgent } from "https-proxy-agent";
 import { BodyInit, RequestInit, Response } from "node-fetch";
 import { getAgentOptions } from "./getAgentOptions.js";
 import fetch from "./node-fetch-patch.js";
-import { getProxyFromEnv, shouldBypassProxy } from "./util.js";
+import { getProxy, shouldBypassProxy } from "./util.js";
 
 const { http, https } = (followRedirects as any).default;
 
@@ -92,15 +92,10 @@ export async function fetchwithRequestOptions(
   const agentOptions = getAgentOptions(requestOptions);
 
   // Get proxy from options or environment variables
-  let proxy = requestOptions?.proxy;
-  if (!proxy) {
-    proxy = getProxyFromEnv(url.protocol);
-  }
+  const proxy = getProxy(url.protocol, requestOptions);
 
   // Check if should bypass proxy based on requestOptions or NO_PROXY env var
-  const shouldBypass =
-    requestOptions?.noProxy?.includes(url.hostname) ||
-    shouldBypassProxy(url.hostname);
+  const shouldBypass = shouldBypassProxy(url.hostname, requestOptions);
 
   // Create agent
   const protocol = url.protocol === "https:" ? https : http;

packages/fetch/src/getAgentOptions.ts

@@ -61,6 +61,19 @@ export function getAgentOptions(requestOptions?: RequestOptions): {
     keepAliveMsecs: timeout,
   };
 
+  if (process.env.VERBOSE_FETCH) {
+    console.log("=== CERTIFICATE CONFIGURATION ===");
+    console.log(`Number of CA certificates: ${ca.length}`);
+    console.log(
+      `Global agent CA certificates: ${globalAgent.options.ca ? "Present" : "Not present"}`,
+    );
+    console.log(
+      `Custom certificates provided: ${customCerts?.length ? "Yes" : "No"}`,
+    );
+    console.log(`rejectUnauthorized: ${agentOptions.rejectUnauthorized}`);
+    console.log("================================");
+  }
+
   // Handle ClientCertificateOptions
   if (requestOptions?.clientCertificate) {
     const { cert, key, passphrase } = requestOptions.clientCertificate;

packages/fetch/src/util.test.ts

@@ -1,5 +1,9 @@
 import { afterEach, expect, test, vi } from "vitest";
-import { getProxyFromEnv, shouldBypassProxy } from "./util.js";
+import {
+  getProxyFromEnv,
+  patternMatchesHostname,
+  shouldBypassProxy,
+} from "./util.js";
 
 // Reset environment variables after each test
 afterEach(() => {
@@ -52,51 +56,159 @@ test("getProxyFromEnv prefers HTTPS_PROXY over other env vars for https protocol
   expect(getProxyFromEnv("https:")).toBe("https://preferred.example.com");
 });
 
+// Tests for patternMatchesHostname
+test("patternMatchesHostname with exact hostname match", () => {
+  expect(patternMatchesHostname("example.com", "example.com")).toBe(true);
+  expect(patternMatchesHostname("example.com", "different.com")).toBe(false);
+});
+
+test("patternMatchesHostname with wildcard domains", () => {
+  expect(patternMatchesHostname("sub.example.com", "*.example.com")).toBe(true);
+  expect(patternMatchesHostname("sub.sub.example.com", "*.example.com")).toBe(
+    true,
+  );
+  expect(patternMatchesHostname("example.com", "*.example.com")).toBe(false);
+  expect(patternMatchesHostname("sub.different.com", "*.example.com")).toBe(
+    false,
+  );
+});
+
+test("patternMatchesHostname with domain suffix", () => {
+  expect(patternMatchesHostname("sub.example.com", ".example.com")).toBe(true);
+  expect(patternMatchesHostname("example.com", ".example.com")).toBe(true);
+  expect(patternMatchesHostname("different.com", ".example.com")).toBe(false);
+});
+
+test("patternMatchesHostname with case insensitivity", () => {
+  expect(patternMatchesHostname("EXAMPLE.com", "example.COM")).toBe(true);
+  expect(patternMatchesHostname("sub.EXAMPLE.com", "*.example.COM")).toBe(true);
+});
+
+// Port handling tests
+test("patternMatchesHostname with exact port match", () => {
+  expect(patternMatchesHostname("example.com:8080", "example.com:8080")).toBe(
+    true,
+  );
+  expect(patternMatchesHostname("example.com:8080", "example.com:9090")).toBe(
+    false,
+  );
+});
+
+test("patternMatchesHostname with port in pattern but not in hostname", () => {
+  expect(patternMatchesHostname("example.com", "example.com:8080")).toBe(false);
+});
+
+test("patternMatchesHostname with port in hostname but not in pattern", () => {
+  expect(patternMatchesHostname("example.com:8080", "example.com")).toBe(true);
+});
+
+test("patternMatchesHostname with wildcard domains and ports", () => {
+  expect(
+    patternMatchesHostname("sub.example.com:8080", "*.example.com:8080"),
+  ).toBe(true);
+  expect(
+    patternMatchesHostname("sub.example.com:9090", "*.example.com:8080"),
+  ).toBe(false);
+  expect(patternMatchesHostname("sub.example.com", "*.example.com:8080")).toBe(
+    false,
+  );
+});
+
+test("patternMatchesHostname with domain suffix and ports", () => {
+  expect(
+    patternMatchesHostname("sub.example.com:8080", ".example.com:8080"),
+  ).toBe(true);
+  expect(patternMatchesHostname("example.com:8080", ".example.com:8080")).toBe(
+    true,
+  );
+  expect(
+    patternMatchesHostname("sub.example.com:9090", ".example.com:8080"),
+  ).toBe(false);
+});
+
 // Tests for shouldBypassProxy
 test("shouldBypassProxy returns false when NO_PROXY is not set", () => {
-  expect(shouldBypassProxy("example.com")).toBe(false);
+  expect(shouldBypassProxy("example.com", undefined)).toBe(false);
 });
 
 test("shouldBypassProxy returns true for exact hostname match", () => {
   process.env.NO_PROXY = "example.com,another.com";
-  expect(shouldBypassProxy("example.com")).toBe(true);
+  expect(shouldBypassProxy("example.com", undefined)).toBe(true);
 });
 
 test("shouldBypassProxy returns false when hostname doesn't match any NO_PROXY entry", () => {
   process.env.NO_PROXY = "example.com,another.com";
-  expect(shouldBypassProxy("different.com")).toBe(false);
+  expect(shouldBypassProxy("different.com", undefined)).toBe(false);
 });
 
 test("shouldBypassProxy handles lowercase no_proxy", () => {
   process.env.no_proxy = "example.com";
-  expect(shouldBypassProxy("example.com")).toBe(true);
+  expect(shouldBypassProxy("example.com", undefined)).toBe(true);
 });
 
 test("shouldBypassProxy works with wildcard domains", () => {
   process.env.NO_PROXY = "*.example.com";
-  expect(shouldBypassProxy("sub.example.com")).toBe(true);
-  expect(shouldBypassProxy("example.com")).toBe(false);
-  expect(shouldBypassProxy("different.com")).toBe(false);
+  expect(shouldBypassProxy("sub.example.com", undefined)).toBe(true);
+  expect(shouldBypassProxy("example.com", undefined)).toBe(false);
+  expect(shouldBypassProxy("different.com", undefined)).toBe(false);
 });
 
 test("shouldBypassProxy works with domain suffix", () => {
   process.env.NO_PROXY = ".example.com";
-  expect(shouldBypassProxy("sub.example.com")).toBe(true);
-  expect(shouldBypassProxy("example.com")).toBe(true);
-  expect(shouldBypassProxy("different.com")).toBe(false);
+  expect(shouldBypassProxy("sub.example.com", undefined)).toBe(true);
+  expect(shouldBypassProxy("example.com", undefined)).toBe(true);
+  expect(shouldBypassProxy("different.com", undefined)).toBe(false);
 });
 
 test("shouldBypassProxy handles multiple entries with different patterns", () => {
   process.env.NO_PROXY = "internal.local,*.example.com,.test.com";
-  expect(shouldBypassProxy("internal.local")).toBe(true);
-  expect(shouldBypassProxy("sub.example.com")).toBe(true);
-  expect(shouldBypassProxy("sub.test.com")).toBe(true);
-  expect(shouldBypassProxy("test.com")).toBe(true);
-  expect(shouldBypassProxy("example.org")).toBe(false);
+  expect(shouldBypassProxy("internal.local", undefined)).toBe(true);
+  expect(shouldBypassProxy("sub.example.com", undefined)).toBe(true);
+  expect(shouldBypassProxy("sub.test.com", undefined)).toBe(true);
+  expect(shouldBypassProxy("test.com", undefined)).toBe(true);
+  expect(shouldBypassProxy("example.org", undefined)).toBe(false);
 });
 
 test("shouldBypassProxy ignores whitespace in NO_PROXY", () => {
   process.env.NO_PROXY = " example.com , *.test.org ";
-  expect(shouldBypassProxy("example.com")).toBe(true);
-  expect(shouldBypassProxy("subdomain.test.org")).toBe(true);
+  expect(shouldBypassProxy("example.com", undefined)).toBe(true);
+  expect(shouldBypassProxy("subdomain.test.org", undefined)).toBe(true);
+});
+
+test("shouldBypassProxy with ports in NO_PROXY", () => {
+  process.env.NO_PROXY = "example.com:8080,*.test.org:443,.internal.net:8443";
+  expect(shouldBypassProxy("example.com:8080", undefined)).toBe(true);
+  expect(shouldBypassProxy("example.com:9090", undefined)).toBe(false);
+  expect(shouldBypassProxy("sub.test.org:443", undefined)).toBe(true);
+  expect(shouldBypassProxy("sub.internal.net:8443", undefined)).toBe(true);
+  expect(shouldBypassProxy("internal.net:8443", undefined)).toBe(true);
+});
+
+test("shouldBypassProxy accepts options with noProxy patterns", () => {
+  const options = { noProxy: ["example.com:8080", "*.internal.net"] };
+  expect(shouldBypassProxy("example.com:8080", options)).toBe(true);
+  expect(shouldBypassProxy("example.com", options)).toBe(false);
+  expect(shouldBypassProxy("server.internal.net", options)).toBe(true);
+});
+
+test("shouldBypassProxy combines environment and options noProxy patterns", () => {
+  process.env.NO_PROXY = "example.org,*.test.com";
+  const options = { noProxy: ["example.com:8080", "*.internal.net"] };
+  expect(shouldBypassProxy("example.org", options)).toBe(true);
+  expect(shouldBypassProxy("sub.test.com", options)).toBe(true);
+  expect(shouldBypassProxy("example.com:8080", options)).toBe(true);
+  expect(shouldBypassProxy("server.internal.net", options)).toBe(true);
+  expect(shouldBypassProxy("other.domain", options)).toBe(false);
+});
+
+test("shouldBypassProxy handles empty noProxy array in options", () => {
+  process.env.NO_PROXY = "example.org";
+  const options = { noProxy: [] };
+  expect(shouldBypassProxy("example.org", options)).toBe(true);
+  expect(shouldBypassProxy("different.com", options)).toBe(false);
+});
+
+test("shouldBypassProxy handles undefined options", () => {
+  process.env.NO_PROXY = "example.org";
+  expect(shouldBypassProxy("example.org", undefined)).toBe(true);
 });

packages/fetch/src/util.ts

@@ -1,3 +1,5 @@
+import { RequestOptions } from "@continuedev/config-types";
+
 /**
  * Gets the proxy settings from environment variables
  * @param protocol The URL protocol (http: or https:)
@@ -16,28 +18,91 @@ export function getProxyFromEnv(protocol: string): string | undefined {
   }
 }
 
-/**
- * Checks if a hostname should bypass proxy based on NO_PROXY environment variable
- * @param hostname The hostname to check
- * @returns True if the hostname should bypass proxy
- */
-export function shouldBypassProxy(hostname: string): boolean {
-  const noProxy = process.env.NO_PROXY || process.env.no_proxy;
-  if (!noProxy) return false;
-
-  const noProxyItems = noProxy.split(",").map((item) => item.trim());
+// Note that request options proxy (per model) takes precedence over environment variables
+export function getProxy(
+  protocol: string,
+  requestOptions?: RequestOptions,
+): string | undefined {
+  if (requestOptions?.proxy) {
+    return requestOptions.proxy;
+  }
+  return getProxyFromEnv(protocol);
+}
 
-  return noProxyItems.some((item) => {
-    // Exact match
-    if (item === hostname) return true;
+export function getEnvNoProxyPatterns(): string[] {
+  const envValue = process.env.NO_PROXY || process.env.no_proxy;
+  if (envValue) {
+    return envValue
+      .split(",")
+      .map((item) => item.trim().toLowerCase())
+      .filter((i) => !!i);
+  } else {
+    return [];
+  }
+}
 
-    // Wildcard domain match (*.example.com)
-    if (item.startsWith("*.") && hostname.endsWith(item.substring(1)))
-      return true;
+export function getReqOptionsNoProxyPatterns(
+  options: RequestOptions | undefined,
+): string[] {
+  return (
+    options?.noProxy?.map((i) => i.trim().toLowerCase()).filter((i) => !!i) ??
+    []
+  );
+}
 
-    // Domain suffix match (.example.com)
-    if (item.startsWith(".") && hostname.endsWith(item.slice(1))) return true;
+export function patternMatchesHostname(hostname: string, pattern: string) {
+  // Split hostname and pattern to separate hostname and port
+  const [hostnameWithoutPort, hostnamePort] = hostname.toLowerCase().split(":");
+  const [patternWithoutPort, patternPort] = pattern.toLowerCase().split(":");
 
+  // If pattern specifies a port but hostname doesn't match it, no match
+  if (patternPort && (!hostnamePort || hostnamePort !== patternPort)) {
     return false;
-  });
+  }
+
+  // Now compare just the hostname parts
+
+  // exact match
+  if (patternWithoutPort === hostnameWithoutPort) {
+    return true;
+  }
+  // wildcard domain match (*.example.com)
+  if (
+    patternWithoutPort.startsWith("*.") &&
+    hostnameWithoutPort.endsWith(patternWithoutPort.substring(1))
+  ) {
+    return true;
+  }
+  // Domain suffix match (.example.com)
+  if (
+    patternWithoutPort.startsWith(".") &&
+    hostnameWithoutPort.endsWith(patternWithoutPort.slice(1))
+  ) {
+    return true;
+  }
+
+  // TODO IP address ranges
+
+  // TODO CIDR notation
+
+  return false;
+}
+
+/**
+ * Checks if a hostname should bypass proxy based on NO_PROXY environment variable
+ * @param hostname The hostname to check
+ * @returns True if the hostname should bypass proxy
+ */
+export function shouldBypassProxy(
+  hostname: string,
+  requestOptions: RequestOptions | undefined,
+): boolean {
+  const ignores = [
+    ...getEnvNoProxyPatterns(),
+    ...getReqOptionsNoProxyPatterns(requestOptions),
+  ];
+  const hostLowerCase = hostname.toLowerCase();
+  return ignores.some((ignore) =>
+    patternMatchesHostname(hostLowerCase, ignore),
+  );
 }