From 16e8756115db3073a0a49a45561c0d8d3ec713c3 Mon Sep 17 00:00:00 2001 From: Dallin Romney Date: Fri, 13 Jun 2025 02:07:49 -0700 Subject: [PATCH 1/5] support for ports, case insensitivity, and removing empty gs strings --- packages/fetch/src/fetch.ts | 4 +- packages/fetch/src/util.ts | 92 +++++++++++++++++++++++++++++-------- 2 files changed, 74 insertions(+), 22 deletions(-) diff --git a/packages/fetch/src/fetch.ts b/packages/fetch/src/fetch.ts index 3b145012f0..b5b4616479 100644 --- a/packages/fetch/src/fetch.ts +++ b/packages/fetch/src/fetch.ts @@ -98,9 +98,7 @@ export async function fetchwithRequestOptions( } // Check if should bypass proxy based on requestOptions or NO_PROXY env var - const shouldBypass = - requestOptions?.noProxy?.includes(url.hostname) || - shouldBypassProxy(url.hostname); + const shouldBypass = shouldBypassProxy(url.hostname, requestOptions); // Create agent const protocol = url.protocol === "https:" ? https : http; diff --git a/packages/fetch/src/util.ts b/packages/fetch/src/util.ts index 20e3c11819..19fba2ba31 100644 --- a/packages/fetch/src/util.ts +++ b/packages/fetch/src/util.ts @@ -1,3 +1,5 @@ +import { RequestOptions } from "@continuedev/config-types"; + /** * Gets the proxy settings from environment variables * @param protocol The URL protocol (http: or https:) @@ -16,28 +18,80 @@ export function getProxyFromEnv(protocol: string): string | undefined { } } -/** - * Checks if a hostname should bypass proxy based on NO_PROXY environment variable - * @param hostname The hostname to check - * @returns True if the hostname should bypass proxy - */ -export function shouldBypassProxy(hostname: string): boolean { - const noProxy = process.env.NO_PROXY || process.env.no_proxy; - if (!noProxy) return false; +export function getEnvNoProxyPatterns(): string[] { + const envValue = process.env.NO_PROXY || process.env.no_proxy; + if (envValue) { + return envValue + .split(",") + .map((item) => item.trim().toLowerCase()) + .filter((i) => !!i); + } else { + return []; + } +} - const noProxyItems = noProxy.split(",").map((item) => item.trim()); +export function getReqOptionsNoProxyPatterns( + options: RequestOptions | undefined, +): string[] { + return ( + options?.noProxy?.map((i) => i.trim().toLowerCase()).filter((i) => !!i) ?? + [] + ); +} - return noProxyItems.some((item) => { - // Exact match - if (item === hostname) return true; +export function patternMatchesHostname(hostname: string, pattern: string) { + // Split hostname and pattern to separate hostname and port + const [hostnameWithoutPort, hostnamePort] = hostname.split(":"); + const [patternWithoutPort, patternPort] = pattern.split(":"); - // Wildcard domain match (*.example.com) - if (item.startsWith("*.") && hostname.endsWith(item.substring(1))) - return true; + // If pattern specifies a port but hostname doesn't match it, no match + if (patternPort && (!hostnamePort || hostnamePort !== patternPort)) { + return false; + } - // Domain suffix match (.example.com) - if (item.startsWith(".") && hostname.endsWith(item.slice(1))) return true; + // Now compare just the hostname parts - return false; - }); + // exact match + if (patternWithoutPort === hostnameWithoutPort) { + return true; + } + // wildcard domain match (*.example.com) + if ( + patternWithoutPort.startsWith("*.") && + hostnameWithoutPort.endsWith(patternWithoutPort.substring(1)) + ) { + return true; + } + // Domain suffix match (.example.com) + if ( + patternWithoutPort.startsWith(".") && + hostnameWithoutPort.endsWith(patternWithoutPort.slice(1)) + ) { + return true; + } + + // TODO IP address ranges + + // TODO CIDR notation + + return false; +} + +/** + * Checks if a hostname should bypass proxy based on NO_PROXY environment variable + * @param hostname The hostname to check + * @returns True if the hostname should bypass proxy + */ +export function shouldBypassProxy( + hostname: string, + requestOptions: RequestOptions | undefined, +): boolean { + const ignores = [ + ...getEnvNoProxyPatterns(), + ...getReqOptionsNoProxyPatterns(requestOptions), + ]; + const hostLowerCase = hostname.toLowerCase(); + return ignores.some((ignore) => + patternMatchesHostname(hostLowerCase, ignore), + ); } From bfe8199fe33efa17d122fa3b7d9c7554956e85f0 Mon Sep 17 00:00:00 2001 From: Dallin Romney Date: Fri, 13 Jun 2025 02:07:54 -0700 Subject: [PATCH 2/5] unit tests --- packages/fetch/src/util.test.ts | 148 ++++++++++++++++++++++++++++---- 1 file changed, 130 insertions(+), 18 deletions(-) diff --git a/packages/fetch/src/util.test.ts b/packages/fetch/src/util.test.ts index 934f1a7357..35c079b017 100644 --- a/packages/fetch/src/util.test.ts +++ b/packages/fetch/src/util.test.ts @@ -1,5 +1,9 @@ import { afterEach, expect, test, vi } from "vitest"; -import { getProxyFromEnv, shouldBypassProxy } from "./util.js"; +import { + getProxyFromEnv, + patternMatchesHostname, + shouldBypassProxy, +} from "./util.js"; // Reset environment variables after each test afterEach(() => { @@ -52,51 +56,159 @@ test("getProxyFromEnv prefers HTTPS_PROXY over other env vars for https protocol expect(getProxyFromEnv("https:")).toBe("https://preferred.example.com"); }); +// Tests for patternMatchesHostname +test("patternMatchesHostname with exact hostname match", () => { + expect(patternMatchesHostname("example.com", "example.com")).toBe(true); + expect(patternMatchesHostname("example.com", "different.com")).toBe(false); +}); + +test("patternMatchesHostname with wildcard domains", () => { + expect(patternMatchesHostname("sub.example.com", "*.example.com")).toBe(true); + expect(patternMatchesHostname("sub.sub.example.com", "*.example.com")).toBe( + true, + ); + expect(patternMatchesHostname("example.com", "*.example.com")).toBe(false); + expect(patternMatchesHostname("sub.different.com", "*.example.com")).toBe( + false, + ); +}); + +test("patternMatchesHostname with domain suffix", () => { + expect(patternMatchesHostname("sub.example.com", ".example.com")).toBe(true); + expect(patternMatchesHostname("example.com", ".example.com")).toBe(true); + expect(patternMatchesHostname("different.com", ".example.com")).toBe(false); +}); + +test("patternMatchesHostname with case insensitivity", () => { + expect(patternMatchesHostname("EXAMPLE.com", "example.COM")).toBe(true); + expect(patternMatchesHostname("sub.EXAMPLE.com", "*.example.COM")).toBe(true); +}); + +// Port handling tests +test("patternMatchesHostname with exact port match", () => { + expect(patternMatchesHostname("example.com:8080", "example.com:8080")).toBe( + true, + ); + expect(patternMatchesHostname("example.com:8080", "example.com:9090")).toBe( + false, + ); +}); + +test("patternMatchesHostname with port in pattern but not in hostname", () => { + expect(patternMatchesHostname("example.com", "example.com:8080")).toBe(false); +}); + +test("patternMatchesHostname with port in hostname but not in pattern", () => { + expect(patternMatchesHostname("example.com:8080", "example.com")).toBe(true); +}); + +test("patternMatchesHostname with wildcard domains and ports", () => { + expect( + patternMatchesHostname("sub.example.com:8080", "*.example.com:8080"), + ).toBe(true); + expect( + patternMatchesHostname("sub.example.com:9090", "*.example.com:8080"), + ).toBe(false); + expect(patternMatchesHostname("sub.example.com", "*.example.com:8080")).toBe( + false, + ); +}); + +test("patternMatchesHostname with domain suffix and ports", () => { + expect( + patternMatchesHostname("sub.example.com:8080", ".example.com:8080"), + ).toBe(true); + expect(patternMatchesHostname("example.com:8080", ".example.com:8080")).toBe( + true, + ); + expect( + patternMatchesHostname("sub.example.com:9090", ".example.com:8080"), + ).toBe(false); +}); + // Tests for shouldBypassProxy test("shouldBypassProxy returns false when NO_PROXY is not set", () => { - expect(shouldBypassProxy("example.com")).toBe(false); + expect(shouldBypassProxy("example.com", undefined)).toBe(false); }); test("shouldBypassProxy returns true for exact hostname match", () => { process.env.NO_PROXY = "example.com,another.com"; - expect(shouldBypassProxy("example.com")).toBe(true); + expect(shouldBypassProxy("example.com", undefined)).toBe(true); }); test("shouldBypassProxy returns false when hostname doesn't match any NO_PROXY entry", () => { process.env.NO_PROXY = "example.com,another.com"; - expect(shouldBypassProxy("different.com")).toBe(false); + expect(shouldBypassProxy("different.com", undefined)).toBe(false); }); test("shouldBypassProxy handles lowercase no_proxy", () => { process.env.no_proxy = "example.com"; - expect(shouldBypassProxy("example.com")).toBe(true); + expect(shouldBypassProxy("example.com", undefined)).toBe(true); }); test("shouldBypassProxy works with wildcard domains", () => { process.env.NO_PROXY = "*.example.com"; - expect(shouldBypassProxy("sub.example.com")).toBe(true); - expect(shouldBypassProxy("example.com")).toBe(false); - expect(shouldBypassProxy("different.com")).toBe(false); + expect(shouldBypassProxy("sub.example.com", undefined)).toBe(true); + expect(shouldBypassProxy("example.com", undefined)).toBe(false); + expect(shouldBypassProxy("different.com", undefined)).toBe(false); }); test("shouldBypassProxy works with domain suffix", () => { process.env.NO_PROXY = ".example.com"; - expect(shouldBypassProxy("sub.example.com")).toBe(true); - expect(shouldBypassProxy("example.com")).toBe(true); - expect(shouldBypassProxy("different.com")).toBe(false); + expect(shouldBypassProxy("sub.example.com", undefined)).toBe(true); + expect(shouldBypassProxy("example.com", undefined)).toBe(true); + expect(shouldBypassProxy("different.com", undefined)).toBe(false); }); test("shouldBypassProxy handles multiple entries with different patterns", () => { process.env.NO_PROXY = "internal.local,*.example.com,.test.com"; - expect(shouldBypassProxy("internal.local")).toBe(true); - expect(shouldBypassProxy("sub.example.com")).toBe(true); - expect(shouldBypassProxy("sub.test.com")).toBe(true); - expect(shouldBypassProxy("test.com")).toBe(true); - expect(shouldBypassProxy("example.org")).toBe(false); + expect(shouldBypassProxy("internal.local", undefined)).toBe(true); + expect(shouldBypassProxy("sub.example.com", undefined)).toBe(true); + expect(shouldBypassProxy("sub.test.com", undefined)).toBe(true); + expect(shouldBypassProxy("test.com", undefined)).toBe(true); + expect(shouldBypassProxy("example.org", undefined)).toBe(false); }); test("shouldBypassProxy ignores whitespace in NO_PROXY", () => { process.env.NO_PROXY = " example.com , *.test.org "; - expect(shouldBypassProxy("example.com")).toBe(true); - expect(shouldBypassProxy("subdomain.test.org")).toBe(true); + expect(shouldBypassProxy("example.com", undefined)).toBe(true); + expect(shouldBypassProxy("subdomain.test.org", undefined)).toBe(true); +}); + +test("shouldBypassProxy with ports in NO_PROXY", () => { + process.env.NO_PROXY = "example.com:8080,*.test.org:443,.internal.net:8443"; + expect(shouldBypassProxy("example.com:8080", undefined)).toBe(true); + expect(shouldBypassProxy("example.com:9090", undefined)).toBe(false); + expect(shouldBypassProxy("sub.test.org:443", undefined)).toBe(true); + expect(shouldBypassProxy("sub.internal.net:8443", undefined)).toBe(true); + expect(shouldBypassProxy("internal.net:8443", undefined)).toBe(true); +}); + +test("shouldBypassProxy accepts options with noProxy patterns", () => { + const options = { noProxy: ["example.com:8080", "*.internal.net"] }; + expect(shouldBypassProxy("example.com:8080", options)).toBe(true); + expect(shouldBypassProxy("example.com", options)).toBe(false); + expect(shouldBypassProxy("server.internal.net", options)).toBe(true); +}); + +test("shouldBypassProxy combines environment and options noProxy patterns", () => { + process.env.NO_PROXY = "example.org,*.test.com"; + const options = { noProxy: ["example.com:8080", "*.internal.net"] }; + expect(shouldBypassProxy("example.org", options)).toBe(true); + expect(shouldBypassProxy("sub.test.com", options)).toBe(true); + expect(shouldBypassProxy("example.com:8080", options)).toBe(true); + expect(shouldBypassProxy("server.internal.net", options)).toBe(true); + expect(shouldBypassProxy("other.domain", options)).toBe(false); +}); + +test("shouldBypassProxy handles empty noProxy array in options", () => { + process.env.NO_PROXY = "example.org"; + const options = { noProxy: [] }; + expect(shouldBypassProxy("example.org", options)).toBe(true); + expect(shouldBypassProxy("different.com", options)).toBe(false); +}); + +test("shouldBypassProxy handles undefined options", () => { + process.env.NO_PROXY = "example.org"; + expect(shouldBypassProxy("example.org", undefined)).toBe(true); }); From b0cc911cf972923ee2c984a6a1c3cdb095adba3e Mon Sep 17 00:00:00 2001 From: Dallin Romney Date: Fri, 13 Jun 2025 11:33:57 -0700 Subject: [PATCH 3/5] clean up getProxy logic --- packages/fetch/src/fetch.ts | 7 ++----- packages/fetch/src/util.ts | 11 +++++++++++ 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/packages/fetch/src/fetch.ts b/packages/fetch/src/fetch.ts index b5b4616479..fbf51f1083 100644 --- a/packages/fetch/src/fetch.ts +++ b/packages/fetch/src/fetch.ts @@ -5,7 +5,7 @@ import { HttpsProxyAgent } from "https-proxy-agent"; import { BodyInit, RequestInit, Response } from "node-fetch"; import { getAgentOptions } from "./getAgentOptions.js"; import fetch from "./node-fetch-patch.js"; -import { getProxyFromEnv, shouldBypassProxy } from "./util.js"; +import { getProxy, shouldBypassProxy } from "./util.js"; const { http, https } = (followRedirects as any).default; @@ -92,10 +92,7 @@ export async function fetchwithRequestOptions( const agentOptions = getAgentOptions(requestOptions); // Get proxy from options or environment variables - let proxy = requestOptions?.proxy; - if (!proxy) { - proxy = getProxyFromEnv(url.protocol); - } + const proxy = getProxy(url.protocol, requestOptions); // Check if should bypass proxy based on requestOptions or NO_PROXY env var const shouldBypass = shouldBypassProxy(url.hostname, requestOptions); diff --git a/packages/fetch/src/util.ts b/packages/fetch/src/util.ts index 19fba2ba31..1a18d5b1ea 100644 --- a/packages/fetch/src/util.ts +++ b/packages/fetch/src/util.ts @@ -18,6 +18,17 @@ export function getProxyFromEnv(protocol: string): string | undefined { } } +// Note that request options proxy (per model) takes precedence over environment variables +export function getProxy( + protocol: string, + requestOptions?: RequestOptions, +): string | undefined { + if (requestOptions?.proxy) { + return requestOptions.proxy; + } + return getProxyFromEnv(protocol); +} + export function getEnvNoProxyPatterns(): string[] { const envValue = process.env.NO_PROXY || process.env.no_proxy; if (envValue) { From 05020b62a99f1416a2934904574a57583b43b72d Mon Sep 17 00:00:00 2001 From: Dallin Romney Date: Fri, 13 Jun 2025 12:50:32 -0700 Subject: [PATCH 4/5] fix lowercase logic --- packages/fetch/src/util.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/fetch/src/util.ts b/packages/fetch/src/util.ts index 1a18d5b1ea..dc302bf123 100644 --- a/packages/fetch/src/util.ts +++ b/packages/fetch/src/util.ts @@ -52,8 +52,8 @@ export function getReqOptionsNoProxyPatterns( export function patternMatchesHostname(hostname: string, pattern: string) { // Split hostname and pattern to separate hostname and port - const [hostnameWithoutPort, hostnamePort] = hostname.split(":"); - const [patternWithoutPort, patternPort] = pattern.split(":"); + const [hostnameWithoutPort, hostnamePort] = hostname.toLowerCase().split(":"); + const [patternWithoutPort, patternPort] = pattern.toLowerCase().split(":"); // If pattern specifies a port but hostname doesn't match it, no match if (patternPort && (!hostnamePort || hostnamePort !== patternPort)) { From 45b7069eb86dfbc6935c605f8490f2e6200f6fd0 Mon Sep 17 00:00:00 2001 From: Dallin Romney Date: Fri, 13 Jun 2025 14:31:25 -0700 Subject: [PATCH 5/5] get agent options verbose fetch logging --- packages/fetch/src/getAgentOptions.ts | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/packages/fetch/src/getAgentOptions.ts b/packages/fetch/src/getAgentOptions.ts index 20fc7641cd..fd7200205f 100644 --- a/packages/fetch/src/getAgentOptions.ts +++ b/packages/fetch/src/getAgentOptions.ts @@ -61,6 +61,19 @@ export function getAgentOptions(requestOptions?: RequestOptions): { keepAliveMsecs: timeout, }; + if (process.env.VERBOSE_FETCH) { + console.log("=== CERTIFICATE CONFIGURATION ==="); + console.log(`Number of CA certificates: ${ca.length}`); + console.log( + `Global agent CA certificates: ${globalAgent.options.ca ? "Present" : "Not present"}`, + ); + console.log( + `Custom certificates provided: ${customCerts?.length ? "Yes" : "No"}`, + ); + console.log(`rejectUnauthorized: ${agentOptions.rejectUnauthorized}`); + console.log("================================"); + } + // Handle ClientCertificateOptions if (requestOptions?.clientCertificate) { const { cert, key, passphrase } = requestOptions.clientCertificate;