From 011e2ee784c99907414dceafac68925e9dbad5d7 Mon Sep 17 00:00:00 2001 From: "Svec, Michal" Date: Fri, 5 Jan 2018 13:44:43 +0100 Subject: [PATCH 1/8] Switch travis to openjdk7 for jdk7. Travis CI does no longer support oraclejdk7. https://github.com/travis-ci/travis-ci/issues/7884#issuecomment-308451879 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index fa369a5af..7d23d5c7d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,7 @@ sudo: false language: java jdk: - - oraclejdk7 + - openjdk7 - oraclejdk8 cache: From 7a9a560308bbd92d28e329e3aeb06d7d65fc160a Mon Sep 17 00:00:00 2001 From: "Svec, Michal" Date: Fri, 5 Jan 2018 14:22:15 +0100 Subject: [PATCH 2/8] No longer possible to use java 7 because of Guava 23 --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 7d23d5c7d..45201c819 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,6 @@ sudo: false language: java jdk: - - openjdk7 - oraclejdk8 cache: From 9ce69f7e2ec8bec0ee0835062ae956d9ab3543ec Mon Sep 17 00:00:00 2001 From: "Svec, Michal" Date: Fri, 5 Jan 2018 15:19:41 +0100 Subject: [PATCH 3/8] Versions update --- pom.xml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/pom.xml b/pom.xml index 351b006d3..fe27c87cc 100644 --- a/pom.xml +++ b/pom.xml @@ -14,9 +14,9 @@ UTF-8 UTF-8 github - 4.0.44.Final - 1.7.24 - 1.7 + 4.0.54.Final + 1.7.25 + 1.8 @@ -177,7 +177,7 @@ netty-4.1 - 4.1.8.Final + 4.1.19.Final @@ -186,13 +186,13 @@ com.google.guava guava - 23.0 + 23.6-jre commons-cli commons-cli - 1.3.1 + 1.4 true @@ -200,7 +200,7 @@ org.apache.commons commons-lang3 - 3.5 + 3.7 @@ -234,7 +234,7 @@ org.mockito mockito-core - 2.7.12 + 2.13.0 test @@ -274,7 +274,7 @@ org.apache.httpcomponents httpclient - 4.5.3 + 4.5.4 test From 1dd05f83fc4e66a08858c7a0c186517ec8e07c9e Mon Sep 17 00:00:00 2001 From: "Svec, Michal" Date: Wed, 20 Jun 2018 12:19:55 +0200 Subject: [PATCH 4/8] Added requestInFlight indicator. Used when request was terminated while still waiting for data --- .../org/littleshoot/proxy/HttpFilters.java | 4 ++ .../littleshoot/proxy/HttpFiltersAdapter.java | 4 ++ .../proxy/impl/ClientToProxyConnection.java | 41 +++++++++++++++---- .../org/littleshoot/proxy/HttpFilterTest.java | 10 ++++- 4 files changed, 49 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/littleshoot/proxy/HttpFilters.java b/src/main/java/org/littleshoot/proxy/HttpFilters.java index b102d1d4d..9600e769f 100644 --- a/src/main/java/org/littleshoot/proxy/HttpFilters.java +++ b/src/main/java/org/littleshoot/proxy/HttpFilters.java @@ -208,4 +208,8 @@ void proxyToServerResolutionSucceeded(String serverHostAndPort, */ void proxyToServerConnectionSucceeded(ChannelHandlerContext serverCtx); + /** + * Informs filter that server disconnected while request was still in flight + */ + void proxyToServerDisconnected(); } diff --git a/src/main/java/org/littleshoot/proxy/HttpFiltersAdapter.java b/src/main/java/org/littleshoot/proxy/HttpFiltersAdapter.java index 2871364ac..e93920b2a 100644 --- a/src/main/java/org/littleshoot/proxy/HttpFiltersAdapter.java +++ b/src/main/java/org/littleshoot/proxy/HttpFiltersAdapter.java @@ -103,4 +103,8 @@ public void proxyToServerConnectionFailed() { @Override public void proxyToServerConnectionSucceeded(ChannelHandlerContext serverCtx) { } + + @Override + public void proxyToServerDisconnected() { + } } diff --git a/src/main/java/org/littleshoot/proxy/impl/ClientToProxyConnection.java b/src/main/java/org/littleshoot/proxy/impl/ClientToProxyConnection.java index 964858fbf..7134fe8ff 100644 --- a/src/main/java/org/littleshoot/proxy/impl/ClientToProxyConnection.java +++ b/src/main/java/org/littleshoot/proxy/impl/ClientToProxyConnection.java @@ -132,7 +132,12 @@ public class ClientToProxyConnection extends ProxyConnection { */ private volatile boolean mitming = false; - private AtomicBoolean authenticated = new AtomicBoolean(); + /** + * Tracks if client is expecting data from proxy. + */ + private volatile boolean isRequestInFlight = false; + + private final AtomicBoolean authenticated = new AtomicBoolean(); private final GlobalTrafficShapingHandler globalTrafficShapingHandler; @@ -221,9 +226,6 @@ protected ConnectionState readHTTPInitial(HttpRequest httpRequest) { * Note - the "server" could be a chained proxy, not the final endpoint for * the request. *

- * - * @param httpRequest - * @return */ private ConnectionState doReadHTTPInitial(HttpRequest httpRequest) { // Make a copy of the original request @@ -237,6 +239,7 @@ private ConnectionState doReadHTTPInitial(HttpRequest httpRequest) { } else { currentFilters = HttpFiltersAdapter.NOOP_FILTER; } + isRequestInFlight = true; // Send the request through the clientToProxyRequest filter, and respond with the short-circuit response if required HttpResponse clientToProxyFilterResponse = currentFilters.clientToProxyRequest(httpRequest); @@ -659,6 +662,17 @@ private void resumeReadingIfNecessary() { protected void serverDisconnected(ProxyToServerConnection serverConnection) { numberOfCurrentlyConnectedServers.decrementAndGet(); + // if server sends content-length larger than content, then client hangs because it waits for next chunk, + // but the proxy2server connection was already closed. In this case it's necessary + // to close client2proxy connection because client would have timed out otherwise + if(isRequestInFlight() && currentServerConnection == serverConnection) { + LOG.warn(String.format("Server disconnected unexpectedly: %s", serverConnection.getServerHostAndPort()), new Exception("Server disconnected unexpectedly")); + writeEmptyBuffer(); + disconnect(); + + currentFilters.proxyToServerDisconnected(); + } + // for non-SSL connections, do not disconnect the client from the proxy, even if this was the last server connection. // this allows clients to continue to use the open connection to the proxy to make future requests. for SSL // connections, whether we are tunneling or MITMing, we need to disconnect the client because there is always @@ -673,7 +687,7 @@ protected void serverDisconnected(ProxyToServerConnection serverConnection) { * associated ProxyToServerConnections. */ @Override - synchronized protected void becameSaturated() { + protected synchronized void becameSaturated() { super.becameSaturated(); for (ProxyToServerConnection serverConnection : serverConnectionsByHostAndPort .values()) { @@ -690,7 +704,7 @@ synchronized protected void becameSaturated() { * associated ProxyToServerConnections. */ @Override - synchronized protected void becameWritable() { + protected synchronized void becameWritable() { super.becameWritable(); for (ProxyToServerConnection serverConnection : serverConnectionsByHostAndPort .values()) { @@ -707,7 +721,7 @@ synchronized protected void becameWritable() { * * @param serverConnection */ - synchronized protected void serverBecameSaturated( + protected synchronized void serverBecameSaturated( ProxyToServerConnection serverConnection) { if (serverConnection.isSaturated()) { LOG.info("Connection to server became saturated, stopping reading"); @@ -721,7 +735,7 @@ synchronized protected void serverBecameSaturated( * * @param serverConnection */ - synchronized protected void serverBecameWriteable( + protected synchronized void serverBecameWriteable( ProxyToServerConnection serverConnection) { boolean anyServersSaturated = false; for (ProxyToServerConnection otherServerConnection : serverConnectionsByHostAndPort @@ -1267,7 +1281,8 @@ private boolean writeGatewayTimeout(HttpRequest httpRequest) { */ private boolean respondWithShortCircuitResponse(HttpResponse httpResponse) { // we are sending a response to the client, so we are done handling this request - this.currentRequest = null; + currentRequest = null; + isRequestInFlight = false; HttpResponse filteredResponse = (HttpResponse) currentFilters.proxyToClientResponse(httpResponse); if (filteredResponse == null) { @@ -1338,6 +1353,7 @@ private String identifyHostAndPort(HttpRequest httpRequest) { */ private void writeEmptyBuffer() { write(Unpooled.EMPTY_BUFFER); + isRequestInFlight = false; } public boolean isMitming() { @@ -1452,4 +1468,11 @@ private FlowContext flowContext() { } } + /** + * @return true if client is expecting data from proxy + */ + public boolean isRequestInFlight() + { + return isRequestInFlight; + } } diff --git a/src/test/java/org/littleshoot/proxy/HttpFilterTest.java b/src/test/java/org/littleshoot/proxy/HttpFilterTest.java index 56e3a229e..a58036e1d 100644 --- a/src/test/java/org/littleshoot/proxy/HttpFilterTest.java +++ b/src/test/java/org/littleshoot/proxy/HttpFilterTest.java @@ -190,6 +190,7 @@ public void proxyToServerRequestSent() { proxyToServerRequestSentNanos.set(requestCount.get(), now()); } + @Override public HttpObject serverToProxyResponse( HttpObject httpObject) { if (originalRequest.getUri().contains("testing3")) { @@ -223,7 +224,8 @@ public void serverToProxyResponseReceived() { serverToProxyResponseReceivedNanos.set(requestCount.get(), now()); } - public HttpObject proxyToClientResponse( + @Override + public HttpObject proxyToClientResponse( HttpObject httpObject) { if (originalRequest.getUri().contains("testing4")) { return new DefaultFullHttpResponse( @@ -827,6 +829,7 @@ private static class HttpFiltersMethodInvokedAdapter implements HttpFilters { private final AtomicBoolean proxyToServerResolutionFailed = new AtomicBoolean(false); private final AtomicBoolean proxyToServerResolutionSucceeded = new AtomicBoolean(false); private final AtomicBoolean proxyToServerConnectionSSLHandshakeStarted = new AtomicBoolean(false); + private final AtomicBoolean proxyToServerDisconnected = new AtomicBoolean(false); private final AtomicBoolean serverToProxyResponseTimedOut = new AtomicBoolean(false); public boolean isProxyToServerConnectionFailedInvoked() { @@ -907,6 +910,11 @@ public void proxyToServerConnectionSucceeded(ChannelHandlerContext serverCtx) { proxyToServerConnectionSucceeded.set(true); } + @Override + public void proxyToServerDisconnected() { + proxyToServerDisconnected.set(true); + } + @Override public HttpResponse clientToProxyRequest(HttpObject httpObject) { clientToProxyRequest.set(true); From ab082802ebadd6671ef38a9491a672ad0baa85f7 Mon Sep 17 00:00:00 2001 From: "Svec, Michal" Date: Wed, 20 Jun 2018 12:27:24 +0200 Subject: [PATCH 5/8] Dropping CONNECT requests on previously used connections --- .../proxy/impl/ClientToProxyConnection.java | 23 ++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/littleshoot/proxy/impl/ClientToProxyConnection.java b/src/main/java/org/littleshoot/proxy/impl/ClientToProxyConnection.java index 7134fe8ff..825270585 100644 --- a/src/main/java/org/littleshoot/proxy/impl/ClientToProxyConnection.java +++ b/src/main/java/org/littleshoot/proxy/impl/ClientToProxyConnection.java @@ -288,9 +288,21 @@ private ConnectionState doReadHTTPInitial(HttpRequest httpRequest) { boolean newConnectionRequired = false; if (ProxyUtils.isCONNECT(httpRequest)) { - LOG.debug( - "Not reusing existing ProxyToServerConnection because request is a CONNECT for: {}", - serverHostAndPort); + // Connect request on a previously used connection may break the connection later + // when another HTTP server (which had connected before on this connection before it was upgraded to HTTPS) disconnects. + // That may cause disconnection of client2proxy in serverDisconnected() + if(isConnectionUsed()) { + LOG.error( + "Dropping request to {} as it's a CONNECT request on a previously used connection.", + serverHostAndPort + ); + writeBadGateway(httpRequest); + return DISCONNECT_REQUESTED; + } + + LOG.debug("Not reusing existing ProxyToServerConnection because request is a CONNECT for: {}", + serverHostAndPort + ); newConnectionRequired = true; } else if (currentServerConnection == null) { LOG.debug("Didn't find existing ProxyToServerConnection for: {}", @@ -363,6 +375,11 @@ private ConnectionState doReadHTTPInitial(HttpRequest httpRequest) { } } + private boolean isConnectionUsed() + { + return currentServerConnection != null || !serverConnectionsByHostAndPort.isEmpty(); + } + /** * Returns true if the specified request is a request to an origin server, rather than to a proxy server. If this * request is being MITM'd, this method always returns false. The format of requests to a proxy server are defined From 69e8df707e80784364716d8dd05ad34e1b6ffdb7 Mon Sep 17 00:00:00 2001 From: "Svec, Michal" Date: Wed, 20 Jun 2018 12:31:30 +0200 Subject: [PATCH 6/8] Logging of server disconnects with a stacktrace so it's visible which server disconnect caused client disconnect --- .../org/littleshoot/proxy/impl/ClientToProxyConnection.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/littleshoot/proxy/impl/ClientToProxyConnection.java b/src/main/java/org/littleshoot/proxy/impl/ClientToProxyConnection.java index 825270585..6bdc8bb8a 100644 --- a/src/main/java/org/littleshoot/proxy/impl/ClientToProxyConnection.java +++ b/src/main/java/org/littleshoot/proxy/impl/ClientToProxyConnection.java @@ -694,7 +694,8 @@ protected void serverDisconnected(ProxyToServerConnection serverConnection) { // this allows clients to continue to use the open connection to the proxy to make future requests. for SSL // connections, whether we are tunneling or MITMing, we need to disconnect the client because there is always // exactly one ClientToProxyConnection per ProxyToServerConnection, and vice versa. - if (isTunneling() || isMitming()) { + if (isMitming() || isTunneling()) { + LOG.warn(String.format("Server %s disconnected. Closing client connection", serverConnection.getServerHostAndPort()), new Exception("Server disconnected")); disconnect(); } } From 47495a116a6093ad44b070aeaf210ebb08fbb73e Mon Sep 17 00:00:00 2001 From: "Svec, Michal" Date: Wed, 20 Jun 2018 12:39:00 +0200 Subject: [PATCH 7/8] Correctly indicate connection close --- .../org/littleshoot/proxy/impl/ClientToProxyConnection.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/java/org/littleshoot/proxy/impl/ClientToProxyConnection.java b/src/main/java/org/littleshoot/proxy/impl/ClientToProxyConnection.java index 6bdc8bb8a..cc85e419b 100644 --- a/src/main/java/org/littleshoot/proxy/impl/ClientToProxyConnection.java +++ b/src/main/java/org/littleshoot/proxy/impl/ClientToProxyConnection.java @@ -1146,6 +1146,7 @@ private void modifyResponseHeadersToReflectProxying( HttpResponse httpResponse) { if (!proxyServer.isTransparent()) { HttpHeaders headers = httpResponse.headers(); + boolean isKeepAlive = HttpHeaders.isKeepAlive(httpResponse); stripConnectionTokens(headers); stripHopByHopHeaders(headers); @@ -1161,6 +1162,9 @@ private void modifyResponseHeadersToReflectProxying( if (!headers.contains(HttpHeaders.Names.DATE)) { HttpHeaders.setDate(httpResponse, new Date()); } + if (isMitming()) { + HttpHeaders.setKeepAlive(httpResponse, isKeepAlive); + } } } From 83ee1ef6c8a2bc622b0f8bf7dfe5ae6fdc046678 Mon Sep 17 00:00:00 2001 From: "Svec, Michal" Date: Wed, 20 Jun 2018 14:50:22 +0200 Subject: [PATCH 8/8] Updated travis badge to a current repository path --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d512855e6..50befbff3 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,11 @@ -[![Build Status](https://travis-ci.org/adamfisk/LittleProxy.png?branch=master)](https://travis-ci.org/adamfisk/LittleProxy) +[![Build Status](https://travis-ci.org/michalsvec/LittleProxy.svg?branch=master)](https://travis-ci.org/michalsvec/LittleProxy) LittleProxy is a high performance HTTP proxy written in Java atop Trustin Lee's excellent [Netty](http://netty.io) event-based networking library. It's quite stable, performs well, and is easy to integrate into your projects. One option is to clone LittleProxy and run it from the command line. This is as simple as: ``` -$ git clone git://github.com/adamfisk/LittleProxy.git +$ git clone git://github.com/michalsvec/LittleProxy.git $ cd LittleProxy $ ./run.bash ```