diff --git a/hadoop-client-modules/hadoop-client-minicluster/pom.xml b/hadoop-client-modules/hadoop-client-minicluster/pom.xml index bc848adaf4..1251f47f95 100644 --- a/hadoop-client-modules/hadoop-client-minicluster/pom.xml +++ b/hadoop-client-modules/hadoop-client-minicluster/pom.xml @@ -793,6 +793,36 @@ ehcache-core.xsd + + + org.eclipse.jetty:jetty-client + + */** + + + + + org.eclipse.jetty:jetty-client + + */** + + + + org.eclipse.jetty:jetty-util-ajax + + */** + + + + org.eclipse.jetty:jetty-server + + jetty-dir.css + + @@ -924,6 +954,13 @@ **/pom.xml + + javax/websocket/ + ${shaded.dependency.prefix}.javax.websocket. + + **/pom.xml + + jersey/ ${shaded.dependency.prefix}.jersey. diff --git a/hadoop-client-modules/hadoop-client-runtime/pom.xml b/hadoop-client-modules/hadoop-client-runtime/pom.xml index 1a2ef9d50c..50fb4ccbe3 100644 --- a/hadoop-client-modules/hadoop-client-runtime/pom.xml +++ b/hadoop-client-modules/hadoop-client-runtime/pom.xml @@ -174,6 +174,15 @@ com.google.code.findbugs:jsr305 io.dropwizard.metrics:metrics-core + org.eclipse.jetty.websocket:* + org.eclipse.jetty:jetty-servlet + org.eclipse.jetty:jetty-security + org.eclipse.jetty:jetty-client + org.eclipse.jetty:jetty-http + org.eclipse.jetty:jetty-xml + org.ow2.asm:* + + org.bouncycastle:* diff --git a/hadoop-common-project/hadoop-auth/pom.xml b/hadoop-common-project/hadoop-auth/pom.xml index 792be4709c..36b1007d4f 100644 --- a/hadoop-common-project/hadoop-auth/pom.xml +++ b/hadoop-common-project/hadoop-auth/pom.xml @@ -188,6 +188,10 @@ guava compile + + org.eclipse.jetty + jetty-server + diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java index 5aeddac26d..8914f2f0c9 100644 --- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java +++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java @@ -19,6 +19,7 @@ import org.apache.hadoop.security.authentication.client.AuthenticationException; import org.apache.hadoop.security.authentication.client.KerberosAuthenticator; import org.apache.hadoop.security.authentication.util.*; +import org.eclipse.jetty.server.Response; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -619,11 +620,20 @@ && getMaxInactiveInterval() > 0) { KerberosAuthenticator.WWW_AUTHENTICATE))) { errCode = HttpServletResponse.SC_FORBIDDEN; } + // After Jetty 9.4.21, sendError() no longer allows a custom message. + // use setStatusWithReason() to set a custom message. + String reason; if (authenticationEx == null) { - httpResponse.sendError(errCode, "Authentication required"); + reason = "Authentication required"; } else { - httpResponse.sendError(errCode, authenticationEx.getMessage()); + reason = authenticationEx.getMessage(); } + + if (httpResponse instanceof Response) { + ((Response)httpResponse).setStatusWithReason(errCode, reason); + } + + httpResponse.sendError(errCode, reason); } } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpRequestLog.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpRequestLog.java index 05573a8de9..b2f18538b6 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpRequestLog.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpRequestLog.java @@ -24,7 +24,8 @@ import org.apache.commons.logging.LogConfigurationException; import org.apache.commons.logging.LogFactory; import org.apache.log4j.Appender; -import org.eclipse.jetty.server.NCSARequestLog; +import org.eclipse.jetty.server.AsyncRequestLogWriter; +import org.eclipse.jetty.server.CustomRequestLog; import org.eclipse.jetty.server.RequestLog; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -85,10 +86,11 @@ public static RequestLog getRequestLog(String name) { if (appender instanceof HttpRequestLogAppender) { HttpRequestLogAppender requestLogAppender = (HttpRequestLogAppender)appender; - NCSARequestLog requestLog = new NCSARequestLog(); - requestLog.setFilename(requestLogAppender.getFilename()); - requestLog.setRetainDays(requestLogAppender.getRetainDays()); - return requestLog; + AsyncRequestLogWriter logWriter = new AsyncRequestLogWriter(); + logWriter.setFilename(requestLogAppender.getFilename()); + logWriter.setRetainDays(requestLogAppender.getRetainDays()); + return new CustomRequestLog(logWriter, + CustomRequestLog.EXTENDED_NCSA_FORMAT); } else { LOG.warn("Jetty request log for {} was of the wrong class", loggerName); return null; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java index 4b67b63da5..bbf9a0b251 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java @@ -82,12 +82,10 @@ import org.eclipse.jetty.server.SecureRequestCustomizer; import org.eclipse.jetty.server.Server; import org.eclipse.jetty.server.ServerConnector; -import org.eclipse.jetty.server.SessionManager; import org.eclipse.jetty.server.SslConnectionFactory; import org.eclipse.jetty.server.handler.ContextHandlerCollection; import org.eclipse.jetty.server.handler.HandlerCollection; import org.eclipse.jetty.server.handler.RequestLogHandler; -import org.eclipse.jetty.server.session.AbstractSessionManager; import org.eclipse.jetty.server.session.SessionHandler; import org.eclipse.jetty.servlet.DefaultServlet; import org.eclipse.jetty.servlet.FilterHolder; @@ -506,7 +504,8 @@ private ServerConnector createHttpsChannelConnector( httpConfig.addCustomizer(new SecureRequestCustomizer()); ServerConnector conn = createHttpChannelConnector(server, httpConfig); - SslContextFactory sslContextFactory = new SslContextFactory(); + SslContextFactory.Server sslContextFactory = + new SslContextFactory.Server(); sslContextFactory.setNeedClientAuth(needsClientAuth); sslContextFactory.setKeyManagerPassword(keyPassword); if (keyStore != null) { @@ -574,12 +573,9 @@ private void initializeWebServer(String name, String hostName, threadPool.setMaxThreads(maxThreads); } - SessionManager sm = webAppContext.getSessionHandler().getSessionManager(); - if (sm instanceof AbstractSessionManager) { - AbstractSessionManager asm = (AbstractSessionManager)sm; - asm.setHttpOnly(true); - asm.getSessionCookieConfig().setSecure(true); - } + SessionHandler handler = webAppContext.getSessionHandler(); + handler.setHttpOnly(true); + handler.getSessionCookieConfig().setSecure(true); ContextHandlerCollection contexts = new ContextHandlerCollection(); RequestLog requestLog = HttpRequestLog.getRequestLog(name); @@ -718,12 +714,8 @@ protected void addDefaultApps(ContextHandlerCollection parent, } logContext.setDisplayName("logs"); SessionHandler handler = new SessionHandler(); - SessionManager sm = handler.getSessionManager(); - if (sm instanceof AbstractSessionManager) { - AbstractSessionManager asm = (AbstractSessionManager) sm; - asm.setHttpOnly(true); - asm.getSessionCookieConfig().setSecure(true); - } + handler.setHttpOnly(true); + handler.getSessionCookieConfig().setSecure(true); logContext.setSessionHandler(handler); setContextAttributes(logContext, conf); addNoCacheFilter(logContext); @@ -740,12 +732,8 @@ protected void addDefaultApps(ContextHandlerCollection parent, params.put("org.eclipse.jetty.servlet.Default.dirAllowed", "false"); params.put("org.eclipse.jetty.servlet.Default.gzip", "true"); SessionHandler handler = new SessionHandler(); - SessionManager sm = handler.getSessionManager(); - if (sm instanceof AbstractSessionManager) { - AbstractSessionManager asm = (AbstractSessionManager) sm; - asm.setHttpOnly(true); - asm.getSessionCookieConfig().setSecure(true); - } + handler.setHttpOnly(true); + handler.getSessionCookieConfig().setSecure(true); staticContext.setSessionHandler(handler); setContextAttributes(staticContext, conf); defaultContexts.put(staticContext, true); @@ -1198,7 +1186,7 @@ private static void bindListener(ServerConnector listener) throws Exception { * @return */ private static BindException constructBindException(ServerConnector listener, - BindException ex) { + IOException ex) { BindException be = new BindException("Port in use: " + listener.getHost() + ":" + listener.getPort()); if (ex != null) { @@ -1220,7 +1208,7 @@ private void bindForSinglePort(ServerConnector listener, int port) try { bindListener(listener); break; - } catch (BindException ex) { + } catch (IOException ex) { if (port == 0 || !findPort) { throw constructBindException(listener, ex); } @@ -1240,13 +1228,13 @@ private void bindForSinglePort(ServerConnector listener, int port) */ private void bindForPortRange(ServerConnector listener, int startPort) throws Exception { - BindException bindException = null; + IOException ioException = null; try { bindListener(listener); return; - } catch (BindException ex) { + } catch (IOException ex) { // Ignore exception. - bindException = ex; + ioException = ex; } for(Integer port : portRanges) { if (port == startPort) { @@ -1259,10 +1247,10 @@ private void bindForPortRange(ServerConnector listener, int startPort) return; } catch (BindException ex) { // Ignore exception. Move to next port. - bindException = ex; + ioException = ex; } } - throw constructBindException(listener, bindException); + throw constructBindException(listener, ioException); } /** diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/http/RestCsrfPreventionFilter.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/http/RestCsrfPreventionFilter.java index 59cb0d6599..b81ed8e901 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/http/RestCsrfPreventionFilter.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/http/RestCsrfPreventionFilter.java @@ -37,6 +37,7 @@ import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; +import org.eclipse.jetty.server.Response; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -271,6 +272,10 @@ public void proceed() throws IOException, ServletException { @Override public void sendError(int code, String message) throws IOException { + if (httpResponse instanceof Response) { + ((Response)httpResponse).setStatusWithReason(code, message); + } + httpResponse.sendError(code, message); } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpRequestLog.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpRequestLog.java index 212807f78e..d0123e3203 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpRequestLog.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpRequestLog.java @@ -18,7 +18,7 @@ package org.apache.hadoop.http; import org.apache.log4j.Logger; -import org.eclipse.jetty.server.NCSARequestLog; +import org.eclipse.jetty.server.CustomRequestLog; import org.eclipse.jetty.server.RequestLog; import org.junit.Test; @@ -42,6 +42,7 @@ public void testAppenderDefined() { RequestLog requestLog = HttpRequestLog.getRequestLog("test"); Logger.getLogger("http.requests.test").removeAppender(requestLogAppender); assertNotNull("RequestLog should not be null", requestLog); - assertEquals("Class mismatch", NCSARequestLog.class, requestLog.getClass()); + assertEquals("Class mismatch", + CustomRequestLog.class, requestLog.getClass()); } } diff --git a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java index 3e98a25232..3de97fdb0d 100644 --- a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java +++ b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java @@ -27,6 +27,7 @@ import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationHandler; import org.apache.hadoop.security.token.delegation.web.KerberosDelegationTokenAuthenticationHandler; import org.apache.hadoop.security.token.delegation.web.PseudoDelegationTokenAuthenticationHandler; +import org.eclipse.jetty.server.Response; import javax.servlet.FilterChain; import javax.servlet.FilterConfig; @@ -106,6 +107,18 @@ public void setStatus(int sc) { public void sendError(int sc, String msg) throws IOException { statusCode = sc; this.msg = msg; + + ServletResponse response = getResponse(); + + // After Jetty 9.4.21, sendError() no longer allows a custom message. + // use setStatusWithReason() to set a custom message. + if (response instanceof Response) { + ((Response) response).setStatusWithReason(sc, msg); + } else { + KMS.LOG.warn("The wrapped response object is instance of {}" + + ", not org.eclipse.jetty.server.Response. Can't set custom error " + + "message", response.getClass()); + } super.sendError(sc, HtmlQuoting.quoteHtmlChars(msg)); } diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/test/TestJettyHelper.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/test/TestJettyHelper.java index e1a55fa9c1..4f8e0913b5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/test/TestJettyHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/test/TestJettyHelper.java @@ -108,7 +108,8 @@ private Server createJettyServer() { conn.setHost(host); conn.setPort(port); if (ssl) { - SslContextFactory sslContextFactory = new SslContextFactory(); + SslContextFactory.Server sslContextFactory = + new SslContextFactory.Server(); sslContextFactory.setNeedClientAuth(false); sslContextFactory.setKeyStorePath(keyStore); sslContextFactory.setKeyStoreType(keyStoreType); diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 66960e9bf2..66957741e2 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -35,7 +35,7 @@ false true - 9.3.24.v20180605 + 9.4.20.v20190813 _ _ diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java index 1fadd42c3d..59f40e8ee0 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java @@ -91,7 +91,6 @@ import org.apache.hadoop.yarn.util.UTCClock; import org.apache.hadoop.yarn.util.resource.ResourceUtils; import org.apache.hadoop.yarn.util.resource.Resources; -import org.eclipse.jetty.util.ConcurrentHashSet; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -332,7 +331,7 @@ private void startNM() throws YarnException, IOException, // create NM simulators Random random = new Random(); - Set rackSet = new ConcurrentHashSet<>(); + Set rackSet = ConcurrentHashMap.newKeySet(); int threadPoolSize = Math.max(poolSize, SLSConfiguration.RUNNER_POOL_SIZE_DEFAULT); ExecutorService executorService = Executors.