Add a test proxy (#2029)

* Added a proxy implementation and test cases
* Simplified servlet tests to implement all HTTP methods

CHANGES

@@ -29,6 +29,8 @@ Release 1.17.1 [PENDING]
   * Bugfix: in W3CDom, if the jsoup input document contained an empty doctype, the conversion would fail with a
     DOMException. Now, said doctype is discarded, and the conversion continues.
 
+  * Build Improvement: added a local test proxy implementation, for proxy integration tests.
+
 Release 1.16.2 [20-Oct-2023]
   * Improvement: optimized the performance of complex CSS selectors, by adding a cost-based query planner. Evaluators
     are sorted by their relative execution cost, and executed in order of lower to higher cost. This speeds the

src/test/java/org/jsoup/integration/ConnectTest.java

@@ -78,7 +78,7 @@ public void exceptOnUnsupportedProtocol() {
         assertTrue(threw);
     }
 
-    private static String ihVal(String key, Document doc) {
+    static String ihVal(String key, Document doc) {
         final Element first = doc.select("th:contains(" + key + ") + td").first();
         return first != null ? first.text() : null;
     }
@@ -403,7 +403,7 @@ public void multiCookieSet() throws IOException {
 
     @Test
     public void supportsDeflate() throws IOException {
-        Connection.Response res = Jsoup.connect(Deflateservlet.Url).execute();
+        Connection.Response res = Jsoup.connect(DeflateServlet.Url).execute();
         assertEquals("deflate", res.header("Content-Encoding"));
 
         Document doc = res.parse();

src/test/java/org/jsoup/integration/ProxyTest.java

@@ -0,0 +1,75 @@
+package org.jsoup.integration;
+
+import org.jsoup.Connection;
+import org.jsoup.Jsoup;
+import org.jsoup.integration.servlets.EchoServlet;
+import org.jsoup.integration.servlets.FileServlet;
+import org.jsoup.integration.servlets.HelloServlet;
+import org.jsoup.integration.servlets.ProxyServlet;
+import org.jsoup.integration.servlets.RedirectServlet;
+import org.jsoup.nodes.Document;
+import org.jsoup.nodes.Element;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+
+import static org.jsoup.integration.ConnectTest.ihVal;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+/**
+ Tests Jsoup.connect proxy support
+ */
+public class ProxyTest {
+    private static String echoUrl;
+    private static TestServer.ProxySettings proxy;
+
+    @BeforeAll
+    public static void setUp() {
+        echoUrl = EchoServlet.Url;
+        proxy = ProxyServlet.ProxySettings;
+    }
+
+    @Test void fetchViaProxy() throws IOException {
+        Connection con = Jsoup.connect(HelloServlet.Url)
+            .proxy(proxy.hostname, proxy.port);
+
+        Connection.Response res = con.execute();
+        assertVia(res);
+
+        Document doc = res.parse();
+        Element p = doc.expectFirst("p");
+        assertEquals("Hello, World!", p.text());
+    }
+
+    private static void assertVia(Connection.Response res) {
+        assertEquals(res.header("Via"), ProxyServlet.Via);
+    }
+
+    @Test void redirectViaProxy() throws IOException {
+        Connection.Response res = Jsoup
+            .connect(RedirectServlet.Url)
+            .data(RedirectServlet.LocationParam, echoUrl)
+            .header("Random-Header-name", "hello")
+            .proxy(proxy.hostname, proxy.port)
+            .execute();
+
+        assertVia(res);
+        Document doc = res.parse();
+        assertEquals(echoUrl, doc.location());
+        assertEquals("hello", ihVal("Random-Header-name", doc));
+        assertVia(res);
+    }
+
+    @Test void proxyForSession() throws IOException {
+        Connection session = Jsoup.newSession().proxy(proxy.hostname, proxy.port);
+
+        Connection.Response medRes = session.newRequest().url(FileServlet.urlTo("/htmltests/medium.html")).execute();
+        Connection.Response largeRes = session.newRequest().url(FileServlet.urlTo("/htmltests/large.html")).execute();
+
+        assertVia(medRes);
+        assertVia(largeRes);
+        assertEquals("Medium HTML", medRes.parse().title());
+        assertEquals("Large HTML", largeRes.parse().title());
+    }
+}

src/test/java/org/jsoup/integration/TestServer.java

@@ -4,24 +4,43 @@
 import org.eclipse.jetty.server.ServerConnector;
 import org.eclipse.jetty.servlet.ServletHandler;
 import org.jsoup.integration.servlets.BaseServlet;
+import org.jsoup.integration.servlets.ProxyServlet;
 
 import java.net.InetSocketAddress;
 
 public class TestServer {
-    private static final Server jetty = new Server(new InetSocketAddress("localhost", 0));
+    private static final String localhost = "localhost";
+    private static final Server jetty = newServer();
     private static final ServletHandler handler = new ServletHandler();
+    static int port;
+
+    private static final Server proxy = newServer();
+    private static final ServletHandler proxyHandler = new ServletHandler();
+    private static final ProxySettings proxySettings = new ProxySettings();
+
+    private static Server newServer() {
+        return new Server(new InetSocketAddress(localhost, 0));
+    }
 
     static {
         jetty.setHandler(handler);
+        proxy.setHandler(proxyHandler);
+        proxyHandler.addServletWithMapping(ProxyServlet.class, "/*");
     }
 
     private TestServer() {
     }
 
     public static void start() {
         synchronized (jetty) {
+            if (jetty.isStarted()) return;
+
             try {
                 jetty.start(); // jetty will safely no-op a start on an already running instance
+                port = ((ServerConnector) jetty.getConnectors()[0]).getLocalPort();
+
+                proxy.start();
+                proxySettings.port = ((ServerConnector) proxy.getConnectors()[0]).getLocalPort();
             } catch (Exception e) {
                 throw new IllegalStateException(e);
             }
@@ -35,8 +54,22 @@ public static String map(Class<? extends BaseServlet> servletClass) {
 
             String path = "/" + servletClass.getSimpleName();
             handler.addServletWithMapping(servletClass, path + "/*");
-            int port = ((ServerConnector) jetty.getConnectors()[0]).getLocalPort();
-            return "http://localhost:" + port + path;
+            return "http://" + localhost + ":" + port + path;
+        }
+    }
+
+    public static ProxySettings proxySettings(Class<? extends BaseServlet> servletClass) {
+        synchronized (jetty) {
+            if (!jetty.isStarted())
+                start(); // if running out of the test cases
+
+            return proxySettings;
         }
     }
+
+    //public static String proxy
+    public static class ProxySettings {
+        final String hostname = localhost;
+        int port;
+    }
 }

src/test/java/org/jsoup/integration/servlets/BaseServlet.java

@@ -9,24 +9,25 @@
 public abstract class BaseServlet extends HttpServlet {
     static final String TextHtml = "text/html; charset=UTF-8";
 
-    // these are overridden just to get the response name to be 'res' not 'resp'
+    abstract protected void doIt(HttpServletRequest req, HttpServletResponse res) throws IOException, ServletException;
+
     @Override
-    protected void doGet(HttpServletRequest req, HttpServletResponse res) throws ServletException, IOException {
-        super.doGet(req, res);
+    protected void doGet(HttpServletRequest req, HttpServletResponse res) throws IOException, ServletException {
+        doIt(req, res);
     }
 
     @Override
-    protected void doPost(HttpServletRequest req, HttpServletResponse res) throws ServletException, IOException {
-        super.doPost(req, res);
+    protected void doPost(HttpServletRequest req, HttpServletResponse res) throws IOException, ServletException {
+        doIt(req, res);
     }
 
     @Override
-    protected void doPut(HttpServletRequest req, HttpServletResponse res) throws ServletException, IOException {
-        super.doPut(req, res);
+    protected void doPut(HttpServletRequest req, HttpServletResponse res) throws IOException, ServletException {
+        doIt(req, res);
     }
 
     @Override
-    protected void doDelete(HttpServletRequest req, HttpServletResponse res) throws ServletException, IOException {
-        super.doPut(req, res);
+    protected void doDelete(HttpServletRequest req, HttpServletResponse res) throws IOException, ServletException {
+        doIt(req, res);
     }
 }

src/test/java/org/jsoup/integration/servlets/CookieServlet.java

@@ -2,35 +2,19 @@
 
 import org.jsoup.integration.TestServer;
 
-import javax.servlet.ServletException;
 import javax.servlet.http.Cookie;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 import java.io.IOException;
 import java.io.PrintWriter;
 
-public class CookieServlet extends BaseServlet{
+public class CookieServlet extends BaseServlet {
     public static final String Url = TestServer.map(CookieServlet.class);
     public static final String SetCookiesParam = "setCookies";
     public static final String LocationParam = "loc";
 
-
     @Override
-    protected void doGet(HttpServletRequest req, HttpServletResponse res) throws ServletException, IOException {
-        doIt(req, res);
-    }
-
-    @Override
-    protected void doPost(HttpServletRequest req, HttpServletResponse res) throws ServletException, IOException {
-        doIt(req, res);
-    }
-
-    @Override
-    protected void doPut(HttpServletRequest req, HttpServletResponse res) throws ServletException, IOException {
-        doIt(req, res);
-    }
-
-    private void doIt(HttpServletRequest req, HttpServletResponse res) throws IOException {
+    protected void doIt(HttpServletRequest req, HttpServletResponse res) throws IOException {
         // Do we want to set cookies?
         if (req.getParameter(SetCookiesParam) != null)
             setCookies(res);

src/test/java/org/jsoup/integration/servlets/Deflateservlet.java → src/test/java/org/jsoup/integration/servlets/DeflateServlet.java

@@ -9,11 +9,11 @@
 import java.util.zip.Deflater;
 import java.util.zip.DeflaterOutputStream;
 
-public class Deflateservlet extends BaseServlet {
-    public static final String Url = TestServer.map(Deflateservlet.class);
+public class DeflateServlet extends BaseServlet {
+    public static final String Url = TestServer.map(DeflateServlet.class);
 
     @Override
-    protected void doGet(HttpServletRequest req, HttpServletResponse res) throws IOException {
+    protected void doIt(HttpServletRequest req, HttpServletResponse res) throws IOException {
         res.setContentType(TextHtml);
         res.setStatus(HttpServletResponse.SC_OK);
         res.setHeader("Content-Encoding", "deflate");

src/test/java/org/jsoup/integration/servlets/EchoServlet.java

@@ -26,26 +26,7 @@ public class EchoServlet extends BaseServlet {
     private static final int DefaultCode = HttpServletResponse.SC_OK;
 
     @Override
-    protected void doGet(HttpServletRequest req, HttpServletResponse res) throws ServletException, IOException {
-        doIt(req, res);
-    }
-
-    @Override
-    protected void doPost(HttpServletRequest req, HttpServletResponse res) throws ServletException, IOException {
-        doIt(req, res);
-    }
-
-    @Override
-    protected void doPut(HttpServletRequest req, HttpServletResponse res) throws ServletException, IOException {
-        doIt(req, res);
-    }
-
-    @Override
-    protected void doDelete(HttpServletRequest req, HttpServletResponse res) throws ServletException, IOException {
-        doIt(req, res);
-    }
-
-    private void doIt(HttpServletRequest req, HttpServletResponse res) throws IOException, ServletException {
+    protected void doIt(HttpServletRequest req, HttpServletResponse res) throws IOException, ServletException {
         int intCode = DefaultCode;
         String code = req.getHeader(CodeParam);
         if (code != null)

src/test/java/org/jsoup/integration/servlets/FileServlet.java

@@ -16,7 +16,7 @@ public class FileServlet extends BaseServlet {
     public static final String DefaultType = "text/html";
 
     @Override
-    protected void doGet(HttpServletRequest req, HttpServletResponse res) throws IOException {
+    protected void doIt(HttpServletRequest req, HttpServletResponse res) throws IOException {
         String contentType = req.getParameter(ContentTypeParam);
         if (contentType == null)
             contentType = DefaultType;
@@ -33,16 +33,11 @@ protected void doGet(HttpServletRequest req, HttpServletResponse res) throws IOE
             Files.copy(file.toPath(), out);
             out.flush();
         } else {
-            res.setStatus(HttpServletResponse.SC_NOT_FOUND);
+            res.sendError(HttpServletResponse.SC_NOT_FOUND);
         }
     }
 
     public static String urlTo(String path) {
         return Url + path;
     }
-
-    @Override
-    protected void doPost(HttpServletRequest req, HttpServletResponse res) throws IOException {
-        doGet(req, res);
-    }
 }

src/test/java/org/jsoup/integration/servlets/HelloServlet.java

@@ -10,7 +10,7 @@ public class HelloServlet extends BaseServlet {
     public static final String Url = TestServer.map(HelloServlet.class);
 
     @Override
-    protected void doGet(HttpServletRequest req, HttpServletResponse res) throws IOException {
+    protected void doIt(HttpServletRequest req, HttpServletResponse res) throws IOException {
         res.setContentType(TextHtml);
         res.setStatus(HttpServletResponse.SC_OK);
 

src/test/java/org/jsoup/integration/servlets/InterruptedServlet.java

@@ -12,9 +12,8 @@ public class InterruptedServlet extends BaseServlet {
     public static final String Magnitude = "magnitude";
     public static final String Larger = "larger";
 
-
     @Override
-    protected void doGet(HttpServletRequest req, HttpServletResponse res) throws IOException {
+    protected void doIt(HttpServletRequest req, HttpServletResponse res) throws IOException {
         String magnitude = req.getParameter(Magnitude);
         magnitude  = magnitude == null ? "" : magnitude;
         res.setContentType(TextHtml);