diff --git a/caddytest/integration/caddyfile_adapt/reverse_proxy_http_transport_forward_proxy_url.txt b/caddytest/integration/caddyfile_adapt/reverse_proxy_http_transport_forward_proxy_url.txt new file mode 100644 index 00000000000..9fc44528336 --- /dev/null +++ b/caddytest/integration/caddyfile_adapt/reverse_proxy_http_transport_forward_proxy_url.txt @@ -0,0 +1,41 @@ +:8884 +reverse_proxy 127.0.0.1:65535 { + transport http { + forward_proxy_url http://localhost:8080 + } +} +---------- +{ + "apps": { + "http": { + "servers": { + "srv0": { + "listen": [ + ":8884" + ], + "routes": [ + { + "handle": [ + { + "handler": "reverse_proxy", + "transport": { + "network_proxy": { + "from": "url", + "url": "http://localhost:8080" + }, + "protocol": "http" + }, + "upstreams": [ + { + "dial": "127.0.0.1:65535" + } + ] + } + ] + } + ] + } + } + } + } +} diff --git a/caddytest/integration/caddyfile_adapt/reverse_proxy_http_transport_none_proxy.txt b/caddytest/integration/caddyfile_adapt/reverse_proxy_http_transport_none_proxy.txt new file mode 100644 index 00000000000..3805448d922 --- /dev/null +++ b/caddytest/integration/caddyfile_adapt/reverse_proxy_http_transport_none_proxy.txt @@ -0,0 +1,40 @@ +:8884 +reverse_proxy 127.0.0.1:65535 { + transport http { + network_proxy none + } +} +---------- +{ + "apps": { + "http": { + "servers": { + "srv0": { + "listen": [ + ":8884" + ], + "routes": [ + { + "handle": [ + { + "handler": "reverse_proxy", + "transport": { + "network_proxy": { + "from": "none" + }, + "protocol": "http" + }, + "upstreams": [ + { + "dial": "127.0.0.1:65535" + } + ] + } + ] + } + ] + } + } + } + } +} diff --git a/caddytest/integration/caddyfile_adapt/reverse_proxy_http_transport_url_proxy.txt b/caddytest/integration/caddyfile_adapt/reverse_proxy_http_transport_url_proxy.txt new file mode 100644 index 00000000000..9397458e9e0 --- /dev/null +++ b/caddytest/integration/caddyfile_adapt/reverse_proxy_http_transport_url_proxy.txt @@ -0,0 +1,41 @@ +:8884 +reverse_proxy 127.0.0.1:65535 { + transport http { + network_proxy url http://localhost:8080 + } +} +---------- +{ + "apps": { + "http": { + "servers": { + "srv0": { + "listen": [ + ":8884" + ], + "routes": [ + { + "handle": [ + { + "handler": "reverse_proxy", + "transport": { + "network_proxy": { + "from": "url", + "url": "http://localhost:8080" + }, + "protocol": "http" + }, + "upstreams": [ + { + "dial": "127.0.0.1:65535" + } + ] + } + ] + } + ] + } + } + } + } +} diff --git a/modules/caddyhttp/reverseproxy/caddyfile.go b/modules/caddyhttp/reverseproxy/caddyfile.go index 12e2b9b97a4..ae35723ea4c 100644 --- a/modules/caddyhttp/reverseproxy/caddyfile.go +++ b/modules/caddyhttp/reverseproxy/caddyfile.go @@ -33,6 +33,7 @@ import ( "github.com/caddyserver/caddy/v2/modules/caddyhttp/headers" "github.com/caddyserver/caddy/v2/modules/caddyhttp/rewrite" "github.com/caddyserver/caddy/v2/modules/caddytls" + "github.com/caddyserver/caddy/v2/modules/internal/network" ) func init() { @@ -980,7 +981,9 @@ func (h *Handler) FinalizeUnmarshalCaddyfile(helper httpcaddyfile.Helper) error // read_buffer // write_buffer // max_response_header -// forward_proxy_url +// network_proxy { +// ... +// } // dial_timeout // dial_fallback_delay // response_header_timeout @@ -991,6 +994,9 @@ func (h *Handler) FinalizeUnmarshalCaddyfile(helper httpcaddyfile.Helper) error // tls_insecure_skip_verify // tls_timeout // tls_trusted_ca_certs +// tls_trust_pool { +// ... +// } // tls_server_name // tls_renegotiation // tls_except_ports @@ -1069,10 +1075,24 @@ func (h *HTTPTransport) UnmarshalCaddyfile(d *caddyfile.Dispenser) error { } case "forward_proxy_url": + caddy.Log().Warn("The 'forward_proxy_url' field is deprecated. Use 'network_proxy url ' instead.") if !d.NextArg() { return d.ArgErr() } - h.ForwardProxyURL = d.Val() + u := network.ProxyFromURL{URL: d.Val()} + h.NetworkProxyRaw = caddyconfig.JSONModuleObject(u, "from", "url", nil) + + case "network_proxy": + if !d.NextArg() { + return d.ArgErr() + } + modStem := d.Val() + modID := "caddy.network_proxy.source." + modStem + unm, err := caddyfile.UnmarshalModule(d, modID) + if err != nil { + return err + } + h.NetworkProxyRaw = caddyconfig.JSONModuleObject(unm, "from", modStem, nil) case "dial_timeout": if !d.NextArg() { diff --git a/modules/caddyhttp/reverseproxy/httptransport.go b/modules/caddyhttp/reverseproxy/httptransport.go index 9929ae5d17e..534eef469d1 100644 --- a/modules/caddyhttp/reverseproxy/httptransport.go +++ b/modules/caddyhttp/reverseproxy/httptransport.go @@ -24,7 +24,6 @@ import ( weakrand "math/rand" "net" "net/http" - "net/url" "os" "reflect" "strings" @@ -36,8 +35,10 @@ import ( "golang.org/x/net/http2" "github.com/caddyserver/caddy/v2" + "github.com/caddyserver/caddy/v2/caddyconfig" "github.com/caddyserver/caddy/v2/modules/caddyhttp" "github.com/caddyserver/caddy/v2/modules/caddytls" + "github.com/caddyserver/caddy/v2/modules/internal/network" ) func init() { @@ -88,6 +89,7 @@ type HTTPTransport struct { // forward_proxy_url -> upstream // // Default: http.ProxyFromEnvironment + // DEPRECATED: Use NetworkProxyRaw|`network_proxy` instead. Subject to removal. ForwardProxyURL string `json:"forward_proxy_url,omitempty"` // How long to wait before timing out trying to connect to @@ -139,6 +141,22 @@ type HTTPTransport struct { // The pre-configured underlying HTTP transport. Transport *http.Transport `json:"-"` + // The module that provides the network (forward) proxy + // URL that the HTTP transport will use to proxy + // requests to the upstream. See [http.Transport.Proxy](https://pkg.go.dev/net/http#Transport.Proxy) + // for information regarding supported protocols. + // + // Providing a value to this parameter results in requests + // flowing through the reverse_proxy in the following way: + // + // User Agent -> + // reverse_proxy -> + // [proxy provided by the module] -> upstream + // + // If nil, defaults to reading the `HTTP_PROXY`, + // `HTTPS_PROXY`, and `NO_PROXY` environment variables. + NetworkProxyRaw json.RawMessage `json:"network_proxy,omitempty" caddy:"namespace=caddy.network_proxy.source inline_key=from"` + h2cTransport *http2.Transport h3Transport *http3.RoundTripper // TODO: EXPERIMENTAL (May 2024) } @@ -326,16 +344,22 @@ func (h *HTTPTransport) NewTransport(caddyCtx caddy.Context) (*http.Transport, e } // negotiate any HTTP/SOCKS proxy for the HTTP transport - var proxy func(*http.Request) (*url.URL, error) + proxy := http.ProxyFromEnvironment if h.ForwardProxyURL != "" { - pUrl, err := url.Parse(h.ForwardProxyURL) + caddyCtx.Logger().Warn("forward_proxy_url is deprecated; use network_proxy instead") + u := network.ProxyFromURL{URL: h.ForwardProxyURL} + h.NetworkProxyRaw = caddyconfig.JSONModuleObject(u, "from", "url", nil) + } + if len(h.NetworkProxyRaw) != 0 { + proxyMod, err := caddyCtx.LoadModule(h, "ForwardProxyRaw") if err != nil { - return nil, fmt.Errorf("failed to parse transport proxy url: %v", err) + return nil, fmt.Errorf("failed to load network_proxy module: %v", err) + } + if m, ok := proxyMod.(caddy.ProxyFuncProducer); ok { + proxy = m.ProxyFunc() + } else { + return nil, fmt.Errorf("network_proxy module is not `(func(*http.Request) (*url.URL, error))``") } - caddyCtx.Logger().Info("setting transport proxy url", zap.String("url", h.ForwardProxyURL)) - proxy = http.ProxyURL(pUrl) - } else { - proxy = http.ProxyFromEnvironment } rt := &http.Transport{ diff --git a/modules/caddytls/acmeissuer.go b/modules/caddytls/acmeissuer.go index f46e296e3ea..38fae46ebcb 100644 --- a/modules/caddytls/acmeissuer.go +++ b/modules/caddytls/acmeissuer.go @@ -97,6 +97,9 @@ type ACMEIssuer struct { // be used. EXPERIMENTAL: Subject to change. CertificateLifetime caddy.Duration `json:"certificate_lifetime,omitempty"` + // Forward proxy module + NetworkProxyRaw json.RawMessage `json:"network_proxy,omitempty" caddy:"namespace=caddy.network_proxy.source inline_key=from"` + rootPool *x509.CertPool logger *zap.Logger @@ -170,7 +173,7 @@ func (iss *ACMEIssuer) Provision(ctx caddy.Context) error { } var err error - iss.template, err = iss.makeIssuerTemplate() + iss.template, err = iss.makeIssuerTemplate(ctx) if err != nil { return err } @@ -178,7 +181,7 @@ func (iss *ACMEIssuer) Provision(ctx caddy.Context) error { return nil } -func (iss *ACMEIssuer) makeIssuerTemplate() (certmagic.ACMEIssuer, error) { +func (iss *ACMEIssuer) makeIssuerTemplate(ctx caddy.Context) (certmagic.ACMEIssuer, error) { template := certmagic.ACMEIssuer{ CA: iss.CA, TestCA: iss.TestCA, @@ -191,6 +194,18 @@ func (iss *ACMEIssuer) makeIssuerTemplate() (certmagic.ACMEIssuer, error) { Logger: iss.logger, } + if len(iss.NetworkProxyRaw) != 0 { + proxyMod, err := ctx.LoadModule(iss, "ForwardProxyRaw") + if err != nil { + return template, fmt.Errorf("failed to load network_proxy module: %v", err) + } + if m, ok := proxyMod.(caddy.ProxyFuncProducer); ok { + template.HTTPProxy = m.ProxyFunc() + } else { + return template, fmt.Errorf("network_proxy module is not `(func(*http.Request) (*url.URL, error))``") + } + } + if iss.Challenges != nil { if iss.Challenges.HTTP != nil { template.DisableHTTPChallenge = iss.Challenges.HTTP.Disabled diff --git a/modules/caddytls/proxy.go b/modules/caddytls/proxy.go new file mode 100644 index 00000000000..7a5e9c5c711 --- /dev/null +++ b/modules/caddytls/proxy.go @@ -0,0 +1,3 @@ +package caddytls + +import _ "github.com/caddyserver/caddy/v2/modules/internal/network" diff --git a/modules/internal/network/networkproxy.go b/modules/internal/network/networkproxy.go new file mode 100644 index 00000000000..fa0f5009a2b --- /dev/null +++ b/modules/internal/network/networkproxy.go @@ -0,0 +1,144 @@ +package network + +import ( + "errors" + "net/http" + "net/url" + "strings" + + "go.uber.org/zap" + + "github.com/caddyserver/caddy/v2" + "github.com/caddyserver/caddy/v2/caddyconfig/caddyfile" +) + +func init() { + caddy.RegisterModule(ProxyFromURL{}) + caddy.RegisterModule(ProxyFromNone{}) +} + +// The "url" proxy source uses the defined URL as the proxy +type ProxyFromURL struct { + URL string `json:"url"` + + ctx caddy.Context + logger *zap.Logger +} + +// CaddyModule implements Module. +func (p ProxyFromURL) CaddyModule() caddy.ModuleInfo { + return caddy.ModuleInfo{ + ID: "caddy.network_proxy.source.url", + New: func() caddy.Module { + return &ProxyFromURL{} + }, + } +} + +func (p *ProxyFromURL) Provision(ctx caddy.Context) error { + p.ctx = ctx + p.logger = ctx.Logger() + return nil +} + +// Validate implements Validator. +func (p ProxyFromURL) Validate() error { + if _, err := url.Parse(p.URL); err != nil { + return err + } + return nil +} + +// ProxyFunc implements ProxyFuncProducer. +func (p ProxyFromURL) ProxyFunc() func(*http.Request) (*url.URL, error) { + if strings.Contains(p.URL, "{") && strings.Contains(p.URL, "}") { + // courtesy of @ImpostorKeanu: https://github.com/caddyserver/caddy/pull/6397 + return func(r *http.Request) (*url.URL, error) { + // retrieve the replacer from context. + repl, ok := r.Context().Value(caddy.ReplacerCtxKey).(*caddy.Replacer) + if !ok { + err := errors.New("failed to obtain replacer from request") + p.logger.Error(err.Error()) + return nil, err + } + + // apply placeholders to the value + // note: h.ForwardProxyURL should never be empty at this point + s := repl.ReplaceAll(p.URL, "") + if s == "" { + p.logger.Error("network_proxy URL was empty after applying placeholders", + zap.String("initial_value", p.URL), + zap.String("final_value", s), + zap.String("hint", "check for invalid placeholders")) + return nil, errors.New("empty value for network_proxy URL") + } + + // parse the url + pUrl, err := url.Parse(s) + if err != nil { + p.logger.Warn("failed to derive transport proxy from network_proxy URL") + pUrl = nil + } else if pUrl.Host == "" || strings.Split("", pUrl.Host)[0] == ":" { + // url.Parse does not return an error on these values: + // + // - http://:80 + // - pUrl.Host == ":80" + // - /some/path + // - pUrl.Host == "" + // + // Super edge cases, but humans are human. + err = errors.New("supplied network_proxy URL is missing a host value") + pUrl = nil + } else { + p.logger.Debug("setting transport proxy url", zap.String("url", s)) + } + + return pUrl, err + } + } + return func(r *http.Request) (*url.URL, error) { + return url.Parse(p.URL) + } +} + +// UnmarshalCaddyfile implements caddyfile.Unmarshaler. +func (p *ProxyFromURL) UnmarshalCaddyfile(d *caddyfile.Dispenser) error { + d.Next() + d.Next() + p.URL = d.Val() + return nil +} + +// The "none" proxy source module disables the use of network proxy. +type ProxyFromNone struct{} + +func (p ProxyFromNone) CaddyModule() caddy.ModuleInfo { + return caddy.ModuleInfo{ + ID: "caddy.network_proxy.source.none", + New: func() caddy.Module { + return &ProxyFromNone{} + }, + } +} + +// UnmarshalCaddyfile implements caddyfile.Unmarshaler. +func (p ProxyFromNone) UnmarshalCaddyfile(d *caddyfile.Dispenser) error { + return nil +} + +// ProxyFunc implements ProxyFuncProducer. +func (p ProxyFromNone) ProxyFunc() func(*http.Request) (*url.URL, error) { + return nil +} + +var ( + _ caddy.Module = ProxyFromURL{} + _ caddy.Provisioner = &ProxyFromURL{} + _ caddy.Validator = ProxyFromURL{} + _ caddy.ProxyFuncProducer = ProxyFromURL{} + _ caddyfile.Unmarshaler = &ProxyFromURL{} + + _ caddy.Module = ProxyFromNone{} + _ caddy.ProxyFuncProducer = ProxyFromNone{} + _ caddyfile.Unmarshaler = ProxyFromNone{} +) diff --git a/network.go b/network.go new file mode 100644 index 00000000000..c7f65ecbefa --- /dev/null +++ b/network.go @@ -0,0 +1,10 @@ +package caddy + +import ( + "net/http" + "net/url" +) + +type ProxyFuncProducer interface { + ProxyFunc() func(*http.Request) (*url.URL, error) +}