diff --git a/packages/fetch/src/fetch.ts b/packages/fetch/src/fetch.ts index 3b145012f0..fbf51f1083 100644 --- a/packages/fetch/src/fetch.ts +++ b/packages/fetch/src/fetch.ts @@ -5,7 +5,7 @@ import { HttpsProxyAgent } from "https-proxy-agent"; import { BodyInit, RequestInit, Response } from "node-fetch"; import { getAgentOptions } from "./getAgentOptions.js"; import fetch from "./node-fetch-patch.js"; -import { getProxyFromEnv, shouldBypassProxy } from "./util.js"; +import { getProxy, shouldBypassProxy } from "./util.js"; const { http, https } = (followRedirects as any).default; @@ -92,15 +92,10 @@ export async function fetchwithRequestOptions( const agentOptions = getAgentOptions(requestOptions); // Get proxy from options or environment variables - let proxy = requestOptions?.proxy; - if (!proxy) { - proxy = getProxyFromEnv(url.protocol); - } + const proxy = getProxy(url.protocol, requestOptions); // Check if should bypass proxy based on requestOptions or NO_PROXY env var - const shouldBypass = - requestOptions?.noProxy?.includes(url.hostname) || - shouldBypassProxy(url.hostname); + const shouldBypass = shouldBypassProxy(url.hostname, requestOptions); // Create agent const protocol = url.protocol === "https:" ? https : http; diff --git a/packages/fetch/src/getAgentOptions.ts b/packages/fetch/src/getAgentOptions.ts index 20fc7641cd..fd7200205f 100644 --- a/packages/fetch/src/getAgentOptions.ts +++ b/packages/fetch/src/getAgentOptions.ts @@ -61,6 +61,19 @@ export function getAgentOptions(requestOptions?: RequestOptions): { keepAliveMsecs: timeout, }; + if (process.env.VERBOSE_FETCH) { + console.log("=== CERTIFICATE CONFIGURATION ==="); + console.log(`Number of CA certificates: ${ca.length}`); + console.log( + `Global agent CA certificates: ${globalAgent.options.ca ? "Present" : "Not present"}`, + ); + console.log( + `Custom certificates provided: ${customCerts?.length ? "Yes" : "No"}`, + ); + console.log(`rejectUnauthorized: ${agentOptions.rejectUnauthorized}`); + console.log("================================"); + } + // Handle ClientCertificateOptions if (requestOptions?.clientCertificate) { const { cert, key, passphrase } = requestOptions.clientCertificate; diff --git a/packages/fetch/src/util.test.ts b/packages/fetch/src/util.test.ts index 934f1a7357..35c079b017 100644 --- a/packages/fetch/src/util.test.ts +++ b/packages/fetch/src/util.test.ts @@ -1,5 +1,9 @@ import { afterEach, expect, test, vi } from "vitest"; -import { getProxyFromEnv, shouldBypassProxy } from "./util.js"; +import { + getProxyFromEnv, + patternMatchesHostname, + shouldBypassProxy, +} from "./util.js"; // Reset environment variables after each test afterEach(() => { @@ -52,51 +56,159 @@ test("getProxyFromEnv prefers HTTPS_PROXY over other env vars for https protocol expect(getProxyFromEnv("https:")).toBe("https://preferred.example.com"); }); +// Tests for patternMatchesHostname +test("patternMatchesHostname with exact hostname match", () => { + expect(patternMatchesHostname("example.com", "example.com")).toBe(true); + expect(patternMatchesHostname("example.com", "different.com")).toBe(false); +}); + +test("patternMatchesHostname with wildcard domains", () => { + expect(patternMatchesHostname("sub.example.com", "*.example.com")).toBe(true); + expect(patternMatchesHostname("sub.sub.example.com", "*.example.com")).toBe( + true, + ); + expect(patternMatchesHostname("example.com", "*.example.com")).toBe(false); + expect(patternMatchesHostname("sub.different.com", "*.example.com")).toBe( + false, + ); +}); + +test("patternMatchesHostname with domain suffix", () => { + expect(patternMatchesHostname("sub.example.com", ".example.com")).toBe(true); + expect(patternMatchesHostname("example.com", ".example.com")).toBe(true); + expect(patternMatchesHostname("different.com", ".example.com")).toBe(false); +}); + +test("patternMatchesHostname with case insensitivity", () => { + expect(patternMatchesHostname("EXAMPLE.com", "example.COM")).toBe(true); + expect(patternMatchesHostname("sub.EXAMPLE.com", "*.example.COM")).toBe(true); +}); + +// Port handling tests +test("patternMatchesHostname with exact port match", () => { + expect(patternMatchesHostname("example.com:8080", "example.com:8080")).toBe( + true, + ); + expect(patternMatchesHostname("example.com:8080", "example.com:9090")).toBe( + false, + ); +}); + +test("patternMatchesHostname with port in pattern but not in hostname", () => { + expect(patternMatchesHostname("example.com", "example.com:8080")).toBe(false); +}); + +test("patternMatchesHostname with port in hostname but not in pattern", () => { + expect(patternMatchesHostname("example.com:8080", "example.com")).toBe(true); +}); + +test("patternMatchesHostname with wildcard domains and ports", () => { + expect( + patternMatchesHostname("sub.example.com:8080", "*.example.com:8080"), + ).toBe(true); + expect( + patternMatchesHostname("sub.example.com:9090", "*.example.com:8080"), + ).toBe(false); + expect(patternMatchesHostname("sub.example.com", "*.example.com:8080")).toBe( + false, + ); +}); + +test("patternMatchesHostname with domain suffix and ports", () => { + expect( + patternMatchesHostname("sub.example.com:8080", ".example.com:8080"), + ).toBe(true); + expect(patternMatchesHostname("example.com:8080", ".example.com:8080")).toBe( + true, + ); + expect( + patternMatchesHostname("sub.example.com:9090", ".example.com:8080"), + ).toBe(false); +}); + // Tests for shouldBypassProxy test("shouldBypassProxy returns false when NO_PROXY is not set", () => { - expect(shouldBypassProxy("example.com")).toBe(false); + expect(shouldBypassProxy("example.com", undefined)).toBe(false); }); test("shouldBypassProxy returns true for exact hostname match", () => { process.env.NO_PROXY = "example.com,another.com"; - expect(shouldBypassProxy("example.com")).toBe(true); + expect(shouldBypassProxy("example.com", undefined)).toBe(true); }); test("shouldBypassProxy returns false when hostname doesn't match any NO_PROXY entry", () => { process.env.NO_PROXY = "example.com,another.com"; - expect(shouldBypassProxy("different.com")).toBe(false); + expect(shouldBypassProxy("different.com", undefined)).toBe(false); }); test("shouldBypassProxy handles lowercase no_proxy", () => { process.env.no_proxy = "example.com"; - expect(shouldBypassProxy("example.com")).toBe(true); + expect(shouldBypassProxy("example.com", undefined)).toBe(true); }); test("shouldBypassProxy works with wildcard domains", () => { process.env.NO_PROXY = "*.example.com"; - expect(shouldBypassProxy("sub.example.com")).toBe(true); - expect(shouldBypassProxy("example.com")).toBe(false); - expect(shouldBypassProxy("different.com")).toBe(false); + expect(shouldBypassProxy("sub.example.com", undefined)).toBe(true); + expect(shouldBypassProxy("example.com", undefined)).toBe(false); + expect(shouldBypassProxy("different.com", undefined)).toBe(false); }); test("shouldBypassProxy works with domain suffix", () => { process.env.NO_PROXY = ".example.com"; - expect(shouldBypassProxy("sub.example.com")).toBe(true); - expect(shouldBypassProxy("example.com")).toBe(true); - expect(shouldBypassProxy("different.com")).toBe(false); + expect(shouldBypassProxy("sub.example.com", undefined)).toBe(true); + expect(shouldBypassProxy("example.com", undefined)).toBe(true); + expect(shouldBypassProxy("different.com", undefined)).toBe(false); }); test("shouldBypassProxy handles multiple entries with different patterns", () => { process.env.NO_PROXY = "internal.local,*.example.com,.test.com"; - expect(shouldBypassProxy("internal.local")).toBe(true); - expect(shouldBypassProxy("sub.example.com")).toBe(true); - expect(shouldBypassProxy("sub.test.com")).toBe(true); - expect(shouldBypassProxy("test.com")).toBe(true); - expect(shouldBypassProxy("example.org")).toBe(false); + expect(shouldBypassProxy("internal.local", undefined)).toBe(true); + expect(shouldBypassProxy("sub.example.com", undefined)).toBe(true); + expect(shouldBypassProxy("sub.test.com", undefined)).toBe(true); + expect(shouldBypassProxy("test.com", undefined)).toBe(true); + expect(shouldBypassProxy("example.org", undefined)).toBe(false); }); test("shouldBypassProxy ignores whitespace in NO_PROXY", () => { process.env.NO_PROXY = " example.com , *.test.org "; - expect(shouldBypassProxy("example.com")).toBe(true); - expect(shouldBypassProxy("subdomain.test.org")).toBe(true); + expect(shouldBypassProxy("example.com", undefined)).toBe(true); + expect(shouldBypassProxy("subdomain.test.org", undefined)).toBe(true); +}); + +test("shouldBypassProxy with ports in NO_PROXY", () => { + process.env.NO_PROXY = "example.com:8080,*.test.org:443,.internal.net:8443"; + expect(shouldBypassProxy("example.com:8080", undefined)).toBe(true); + expect(shouldBypassProxy("example.com:9090", undefined)).toBe(false); + expect(shouldBypassProxy("sub.test.org:443", undefined)).toBe(true); + expect(shouldBypassProxy("sub.internal.net:8443", undefined)).toBe(true); + expect(shouldBypassProxy("internal.net:8443", undefined)).toBe(true); +}); + +test("shouldBypassProxy accepts options with noProxy patterns", () => { + const options = { noProxy: ["example.com:8080", "*.internal.net"] }; + expect(shouldBypassProxy("example.com:8080", options)).toBe(true); + expect(shouldBypassProxy("example.com", options)).toBe(false); + expect(shouldBypassProxy("server.internal.net", options)).toBe(true); +}); + +test("shouldBypassProxy combines environment and options noProxy patterns", () => { + process.env.NO_PROXY = "example.org,*.test.com"; + const options = { noProxy: ["example.com:8080", "*.internal.net"] }; + expect(shouldBypassProxy("example.org", options)).toBe(true); + expect(shouldBypassProxy("sub.test.com", options)).toBe(true); + expect(shouldBypassProxy("example.com:8080", options)).toBe(true); + expect(shouldBypassProxy("server.internal.net", options)).toBe(true); + expect(shouldBypassProxy("other.domain", options)).toBe(false); +}); + +test("shouldBypassProxy handles empty noProxy array in options", () => { + process.env.NO_PROXY = "example.org"; + const options = { noProxy: [] }; + expect(shouldBypassProxy("example.org", options)).toBe(true); + expect(shouldBypassProxy("different.com", options)).toBe(false); +}); + +test("shouldBypassProxy handles undefined options", () => { + process.env.NO_PROXY = "example.org"; + expect(shouldBypassProxy("example.org", undefined)).toBe(true); }); diff --git a/packages/fetch/src/util.ts b/packages/fetch/src/util.ts index 20e3c11819..dc302bf123 100644 --- a/packages/fetch/src/util.ts +++ b/packages/fetch/src/util.ts @@ -1,3 +1,5 @@ +import { RequestOptions } from "@continuedev/config-types"; + /** * Gets the proxy settings from environment variables * @param protocol The URL protocol (http: or https:) @@ -16,28 +18,91 @@ export function getProxyFromEnv(protocol: string): string | undefined { } } -/** - * Checks if a hostname should bypass proxy based on NO_PROXY environment variable - * @param hostname The hostname to check - * @returns True if the hostname should bypass proxy - */ -export function shouldBypassProxy(hostname: string): boolean { - const noProxy = process.env.NO_PROXY || process.env.no_proxy; - if (!noProxy) return false; - - const noProxyItems = noProxy.split(",").map((item) => item.trim()); +// Note that request options proxy (per model) takes precedence over environment variables +export function getProxy( + protocol: string, + requestOptions?: RequestOptions, +): string | undefined { + if (requestOptions?.proxy) { + return requestOptions.proxy; + } + return getProxyFromEnv(protocol); +} - return noProxyItems.some((item) => { - // Exact match - if (item === hostname) return true; +export function getEnvNoProxyPatterns(): string[] { + const envValue = process.env.NO_PROXY || process.env.no_proxy; + if (envValue) { + return envValue + .split(",") + .map((item) => item.trim().toLowerCase()) + .filter((i) => !!i); + } else { + return []; + } +} - // Wildcard domain match (*.example.com) - if (item.startsWith("*.") && hostname.endsWith(item.substring(1))) - return true; +export function getReqOptionsNoProxyPatterns( + options: RequestOptions | undefined, +): string[] { + return ( + options?.noProxy?.map((i) => i.trim().toLowerCase()).filter((i) => !!i) ?? + [] + ); +} - // Domain suffix match (.example.com) - if (item.startsWith(".") && hostname.endsWith(item.slice(1))) return true; +export function patternMatchesHostname(hostname: string, pattern: string) { + // Split hostname and pattern to separate hostname and port + const [hostnameWithoutPort, hostnamePort] = hostname.toLowerCase().split(":"); + const [patternWithoutPort, patternPort] = pattern.toLowerCase().split(":"); + // If pattern specifies a port but hostname doesn't match it, no match + if (patternPort && (!hostnamePort || hostnamePort !== patternPort)) { return false; - }); + } + + // Now compare just the hostname parts + + // exact match + if (patternWithoutPort === hostnameWithoutPort) { + return true; + } + // wildcard domain match (*.example.com) + if ( + patternWithoutPort.startsWith("*.") && + hostnameWithoutPort.endsWith(patternWithoutPort.substring(1)) + ) { + return true; + } + // Domain suffix match (.example.com) + if ( + patternWithoutPort.startsWith(".") && + hostnameWithoutPort.endsWith(patternWithoutPort.slice(1)) + ) { + return true; + } + + // TODO IP address ranges + + // TODO CIDR notation + + return false; +} + +/** + * Checks if a hostname should bypass proxy based on NO_PROXY environment variable + * @param hostname The hostname to check + * @returns True if the hostname should bypass proxy + */ +export function shouldBypassProxy( + hostname: string, + requestOptions: RequestOptions | undefined, +): boolean { + const ignores = [ + ...getEnvNoProxyPatterns(), + ...getReqOptionsNoProxyPatterns(requestOptions), + ]; + const hostLowerCase = hostname.toLowerCase(); + return ignores.some((ignore) => + patternMatchesHostname(hostLowerCase, ignore), + ); }