diff --git a/bin/testfiles/HTMLParserTestFile_2.xml b/bin/testfiles/HTMLParserTestFile_2.xml index 3a47e4de8ab..36dc54f77f4 100644 --- a/bin/testfiles/HTMLParserTestFile_2.xml +++ b/bin/testfiles/HTMLParserTestFile_2.xml @@ -10,7 +10,7 @@ file:testfiles/HTMLParserTestFile_2.html - + @@ -29,7 +29,7 @@ file:testfiles/HTMLParserTestFile_2_files/halfbanner.htm - + @@ -46,7 +46,7 @@ file:testfiles/HTMLParserTestFile_2_files/halfbanner.htm - + @@ -55,7 +55,7 @@ file:testfiles/HTMLParserTestFile_2_files/jakarta-logo.gif - + @@ -64,7 +64,7 @@ file:testfiles/HTMLParserTestFile_2_files/logo.jpg - + @@ -73,7 +73,7 @@ file:testfiles/HTMLParserTestFile_2_files/http-config-example.png - + @@ -82,7 +82,7 @@ file:testfiles/HTMLParserTestFile_2_files/scoping1.png - + @@ -91,7 +91,7 @@ file:testfiles/HTMLParserTestFile_2_files/scoping2.png - + diff --git a/bin/testfiles/TEST_HTTP.jmx b/bin/testfiles/TEST_HTTP.jmx index 30e7ae16fd2..da93b79e0a8 100644 --- a/bin/testfiles/TEST_HTTP.jmx +++ b/bin/testfiles/TEST_HTTP.jmx @@ -932,7 +932,7 @@ mirrorServer.start(); true - String textToCheck = 'Content-Disposition: form-data; name="?_param"'; + String textToCheck = 'Content-Disposition: form-data; name="安_param"'; if(prev.getSamplerData().indexOf(textToCheck) < 0) { AssertionResult.setFailure(true); AssertionResult.setFailureMessage("Request does not contains '"+textToCheck+"'"); @@ -1005,7 +1005,7 @@ if(prev.getSamplerData().indexOf(textToCheck) < 0) { nv_contentType - text/plain + text/plain; charset=UTF-8 Assertion.response_data false diff --git a/src/core/src/main/java/org/apache/jmeter/samplers/SampleResult.java b/src/core/src/main/java/org/apache/jmeter/samplers/SampleResult.java index 2518f26c318..137755b8f47 100644 --- a/src/core/src/main/java/org/apache/jmeter/samplers/SampleResult.java +++ b/src/core/src/main/java/org/apache/jmeter/samplers/SampleResult.java @@ -56,7 +56,7 @@ public class SampleResult implements Serializable, Cloneable, Searchable { * The default encoding to be used if not overridden. * The value is ISO-8859-1. */ - public static final String DEFAULT_HTTP_ENCODING = StandardCharsets.ISO_8859_1.name(); + public static final String DEFAULT_HTTP_ENCODING = StandardCharsets.UTF_8.name(); private static final String OK_CODE = Integer.toString(HttpURLConnection.HTTP_OK); private static final String OK_MSG = "OK"; // $NON-NLS-1$ diff --git a/src/protocol/http/src/main/java/org/apache/jmeter/protocol/http/proxy/DefaultSamplerCreator.java b/src/protocol/http/src/main/java/org/apache/jmeter/protocol/http/proxy/DefaultSamplerCreator.java index a972399ca9b..75206a91080 100644 --- a/src/protocol/http/src/main/java/org/apache/jmeter/protocol/http/proxy/DefaultSamplerCreator.java +++ b/src/protocol/http/src/main/java/org/apache/jmeter/protocol/http/proxy/DefaultSamplerCreator.java @@ -43,7 +43,6 @@ import org.apache.jmeter.protocol.http.control.gui.HttpTestSampleGui; import org.apache.jmeter.protocol.http.sampler.HTTPSamplerBase; import org.apache.jmeter.protocol.http.sampler.HTTPSamplerFactory; -import org.apache.jmeter.protocol.http.sampler.PostWriter; import org.apache.jmeter.protocol.http.util.ConversionUtils; import org.apache.jmeter.protocol.http.util.GraphQLRequestParamUtils; import org.apache.jmeter.protocol.http.util.HTTPConstants; @@ -218,24 +217,10 @@ protected void computeFromPostBody(HTTPSamplerBase sampler, final String contentType = request.getContentType(); MultipartUrlConfig urlConfig = request.getMultipartConfig(contentType); String contentEncoding = sampler.getContentEncoding(); - // Get the post data using the content encoding of the request - String postData = null; - if (log.isDebugEnabled()) { - if(!StringUtils.isEmpty(contentEncoding)) { - log.debug("Using encoding {} for request body", contentEncoding); - } - else { - log.debug("No encoding found, using JRE default encoding for request body"); - } - } - + log.debug("Using encoding {} for request body", contentEncoding); - if (!StringUtils.isEmpty(contentEncoding)) { - postData = new String(request.getRawPostData(), contentEncoding); - } else { - // Use default encoding - postData = new String(request.getRawPostData(), PostWriter.ENCODING); - } + // Get the post data using the content encoding of the request + String postData = new String(request.getRawPostData(), contentEncoding); if (urlConfig != null) { urlConfig.parseArguments(postData); @@ -436,16 +421,7 @@ private static String getNumberedFormat(int httpSampleNameMode) { * @param request {@link HttpRequestHdr} */ protected void computePath(HTTPSamplerBase sampler, HttpRequestHdr request) { - if(sampler.getContentEncoding() != null) { - sampler.setPath(request.getPath(), sampler.getContentEncoding()); - } - else { - // Although the spec says UTF-8 should be used for encoding URL parameters, - // most browser use ISO-8859-1 for default if encoding is not known. - // We use null for contentEncoding, then the url parameters will be added - // with the value in the URL, and the "encode?" flag set to false - sampler.setPath(request.getPath(), null); - } + sampler.setPath(request.getPath(), sampler.getContentEncoding()); if (log.isDebugEnabled()) { log.debug("Proxy: finished setting path: {}", sampler.getPath()); } diff --git a/src/protocol/http/src/main/java/org/apache/jmeter/protocol/http/proxy/HttpRequestHdr.java b/src/protocol/http/src/main/java/org/apache/jmeter/protocol/http/proxy/HttpRequestHdr.java index 0532d5adcdb..b118ea659fe 100644 --- a/src/protocol/http/src/main/java/org/apache/jmeter/protocol/http/proxy/HttpRequestHdr.java +++ b/src/protocol/http/src/main/java/org/apache/jmeter/protocol/http/proxy/HttpRequestHdr.java @@ -165,7 +165,7 @@ public byte[] parse(InputStream in) throws IOException { inHeaders = false; firstLine = false; // cannot be first line either } - final String reqLine = line.toString(StandardCharsets.ISO_8859_1.name()); + final String reqLine = line.toString(StandardCharsets.UTF_8.name()); if (firstLine) { parseFirstLine(reqLine); firstLine = false; diff --git a/src/protocol/http/src/main/java/org/apache/jmeter/protocol/http/sampler/HTTPHC4Impl.java b/src/protocol/http/src/main/java/org/apache/jmeter/protocol/http/sampler/HTTPHC4Impl.java index 09c9d875f6f..bc4478eecb3 100644 --- a/src/protocol/http/src/main/java/org/apache/jmeter/protocol/http/sampler/HTTPHC4Impl.java +++ b/src/protocol/http/src/main/java/org/apache/jmeter/protocol/http/sampler/HTTPHC4Impl.java @@ -103,7 +103,7 @@ import org.apache.http.entity.StringEntity; import org.apache.http.entity.mime.FormBodyPart; import org.apache.http.entity.mime.FormBodyPartBuilder; -import org.apache.http.entity.mime.MIME; +import org.apache.http.entity.mime.HttpMultipartMode; import org.apache.http.entity.mime.MultipartEntityBuilder; import org.apache.http.entity.mime.content.FileBody; import org.apache.http.entity.mime.content.StringBody; @@ -151,7 +151,6 @@ import org.apache.jmeter.protocol.http.sampler.hc.LaxGZIPInputStream; import org.apache.jmeter.protocol.http.sampler.hc.LazyLayeredConnectionSocketFactory; import org.apache.jmeter.protocol.http.util.ConversionUtils; -import org.apache.jmeter.protocol.http.util.EncoderCache; import org.apache.jmeter.protocol.http.util.HTTPArgument; import org.apache.jmeter.protocol.http.util.HTTPConstants; import org.apache.jmeter.protocol.http.util.HTTPFileArg; @@ -1508,13 +1507,18 @@ private static class ViewableFileBody extends FileBody { "".getBytes(StandardCharsets.UTF_8); private boolean hideFileData; - public ViewableFileBody(File file, ContentType contentType) { - // Note: HttpClient4 does not support encoding the file name, so we explicitly encode it here + public ViewableFileBody(File file, ContentType contentType, Charset charset) { + // Note: HttpClient4 does not support encoding the file name, and it always encodes names in IS88 // See https://issues.apache.org/jira/browse/HTTPCLIENT-293 - super(file, contentType, ConversionUtils.percentEncode(file.getName())); + super(file, contentType, encodeFilename(file.getName(), charset)); hideFileData = false; } + private static String encodeFilename(String fileName, Charset charset) { + return ConversionUtils.percentEncode( + ConversionUtils.encodeWithEntities(fileName, charset)); + } + @Override public void writeTo(final OutputStream out) throws IOException { if (hideFileData) { @@ -1535,8 +1539,9 @@ protected String setupHttpEntityEnclosingRequestData(HttpEntityEnclosingRequestB StringBuilder postedBody = new StringBuilder(1000); HTTPFileArg[] files = getHTTPFiles(); - final String contentEncoding = getContentEncodingOrNull(); - final boolean haveContentEncoding = contentEncoding != null; + final String contentEncoding = getContentEncoding(); + Charset charset = Charset.forName(contentEncoding); + final boolean haveContentEncoding = true; // Check if we should do a multipart/form-data or an // application/x-www-form-urlencoded post request @@ -1547,25 +1552,22 @@ protected String setupHttpEntityEnclosingRequestData(HttpEntityEnclosingRequestB Arrays.asList(entityEnclosingRequest.getHeaders(HTTPConstants.HEADER_CONTENT_TYPE))); entityEnclosingRequest.removeHeaders(HTTPConstants.HEADER_CONTENT_TYPE); } - // If a content encoding is specified, we use that as the - // encoding of any parameter values - Charset charset; - if(haveContentEncoding) { - charset = Charset.forName(contentEncoding); - } else { - charset = MIME.DEFAULT_CHARSET; - } + // doBrowserCompatibleMultipart means "use charset for encoding MIME headers", + // while RFC6532 means "use UTF-8 for encoding MIME headers" + boolean doBrowserCompatibleMultipart = getDoBrowserCompatibleMultipart(); if(log.isDebugEnabled()) { log.debug("Building multipart with:getDoBrowserCompatibleMultipart(): {}, with charset:{}, haveContentEncoding:{}", - getDoBrowserCompatibleMultipart(), charset, haveContentEncoding); + doBrowserCompatibleMultipart, charset, haveContentEncoding); } // Write the request to our own stream MultipartEntityBuilder multipartEntityBuilder = MultipartEntityBuilder.create(); - if(getDoBrowserCompatibleMultipart()) { + multipartEntityBuilder.setCharset(charset); + if (doBrowserCompatibleMultipart) { multipartEntityBuilder.setLaxMode(); } else { - multipartEntityBuilder.setStrictMode(); + // Use UTF-8 for encoding header names and values + multipartEntityBuilder.setMode(HttpMultipartMode.RFC6532); } // Create the parts // Add any parameters @@ -1596,7 +1598,8 @@ protected String setupHttpEntityEnclosingRequestData(HttpEntityEnclosingRequestB HTTPFileArg file = files[i]; File reservedFile = FileServer.getFileServer().getResolvedFile(file.getPath()); - fileBodies[i] = new ViewableFileBody(reservedFile, ContentType.parse(file.getMimeType())); + Charset filenameCharset = doBrowserCompatibleMultipart ? charset : StandardCharsets.UTF_8; + fileBodies[i] = new ViewableFileBody(reservedFile, ContentType.parse(file.getMimeType()), filenameCharset); multipartEntityBuilder.addPart(file.getParamName(), fileBodies[i] ); } @@ -1652,12 +1655,7 @@ else if(ADD_CONTENT_TYPE_TO_POST_IF_MISSING) { StringBuilder postBody = new StringBuilder(); for (JMeterProperty jMeterProperty : getArguments()) { HTTPArgument arg = (HTTPArgument) jMeterProperty.getObjectValue(); - // Note: if "Encoded?" is not selected, arg.getEncodedValue is equivalent to arg.getValue - if (haveContentEncoding) { - postBody.append(arg.getEncodedValue(contentEncoding)); - } else { - postBody.append(arg.getEncodedValue()); - } + postBody.append(arg.getEncodedValue(contentEncoding)); } // Let StringEntity perform the encoding StringEntity requestEntity = new StringEntity(postBody.toString(), contentEncoding); @@ -1669,8 +1667,7 @@ else if(ADD_CONTENT_TYPE_TO_POST_IF_MISSING) { if(!hasContentTypeHeader && ADD_CONTENT_TYPE_TO_POST_IF_MISSING) { entityEnclosingRequest.setHeader(HTTPConstants.HEADER_CONTENT_TYPE, HTTPConstants.APPLICATION_X_WWW_FORM_URLENCODED); } - String urlContentEncoding = contentEncoding; - UrlEncodedFormEntity entity = createUrlEncodedFormEntity(urlContentEncoding); + UrlEncodedFormEntity entity = createUrlEncodedFormEntity(contentEncoding); entityEnclosingRequest.setEntity(entity); writeEntityToSB(postedBody, entity, EMPTY_FILE_BODIES, contentEncoding); } @@ -1742,7 +1739,7 @@ protected String sendEntityData( HttpEntityEnclosingRequestBase entity) throws I // Check for local contentEncoding (charset) override; fall back to default for content body // we do this here rather so we can use the same charset to retrieve the data - final String charset = getContentEncoding(HTTP.DEF_CONTENT_CHARSET.name()); + final String charset = getContentEncoding(); // Only create this if we are overriding whatever default there may be // If there are no arguments, we can send a file as the body of the request @@ -1775,7 +1772,7 @@ else if(getSendParameterValuesAsPostBody()) { entity.setEntity(requestEntity); } else if (hasArguments()) { hasEntityBody = true; - entity.setEntity(createUrlEncodedFormEntity(getContentEncodingOrNull())); + entity.setEntity(createUrlEncodedFormEntity(getContentEncoding())); } // Check if we have any content to send for body if(hasEntityBody) { @@ -1792,20 +1789,15 @@ else if(getSendParameterValuesAsPostBody()) { /** * Create UrlEncodedFormEntity from parameters - * @param contentEncoding Content encoding may be null or empty + * @param urlContentEncoding Content encoding may be null or empty * @return {@link UrlEncodedFormEntity} * @throws UnsupportedEncodingException */ - private UrlEncodedFormEntity createUrlEncodedFormEntity(final String contentEncoding) throws UnsupportedEncodingException { + private UrlEncodedFormEntity createUrlEncodedFormEntity(final String urlContentEncoding) throws UnsupportedEncodingException { // It is a normal request, with parameter names and values // Add the parameters PropertyIterator args = getArguments().iterator(); List nvps = new ArrayList<>(); - String urlContentEncoding = contentEncoding; - if (urlContentEncoding == null || urlContentEncoding.length() == 0) { - // Use the default encoding for urls - urlContentEncoding = EncoderCache.URL_ARGUMENT_ENCODING; - } while (args.hasNext()) { HTTPArgument arg = (HTTPArgument) args.next().getObjectValue(); // The HTTPClient always urlencodes both name and value, @@ -1830,27 +1822,6 @@ private UrlEncodedFormEntity createUrlEncodedFormEntity(final String contentEnco return new UrlEncodedFormEntity(nvps, urlContentEncoding); } - - /** - * @return the value of {@link #getContentEncoding()}; forced to null if empty - */ - private String getContentEncodingOrNull() { - return getContentEncoding(null); - } - - /** - * @param dflt the default to be used - * @return the value of {@link #getContentEncoding()}; default if null or empty - */ - private String getContentEncoding(String dflt) { - String ce = getContentEncoding(); - if (isNullOrEmptyTrimmed(ce)) { - return dflt; - } else { - return ce; - } - } - private static void saveConnectionCookies(HttpResponse method, URL u, CookieManager cookieManager) { if (cookieManager != null) { Header[] hdrs = method.getHeaders(HTTPConstants.HEADER_SET_COOKIE); diff --git a/src/protocol/http/src/main/java/org/apache/jmeter/protocol/http/sampler/HTTPSamplerBase.java b/src/protocol/http/src/main/java/org/apache/jmeter/protocol/http/sampler/HTTPSamplerBase.java index 7efb81e4e6c..3b3b37d0141 100644 --- a/src/protocol/http/src/main/java/org/apache/jmeter/protocol/http/sampler/HTTPSamplerBase.java +++ b/src/protocol/http/src/main/java/org/apache/jmeter/protocol/http/sampler/HTTPSamplerBase.java @@ -558,7 +558,11 @@ public void setContentEncoding(String charsetName) { * @return the encoding of the content, i.e. its charset name */ public String getContentEncoding() { - return get(getSchema().getContentEncoding()); + String encoding = get(getSchema().getContentEncoding()); + if (encoding.isEmpty()) { + return getSchema().getContentEncoding().getDefaultValue(); + } + return encoding; } public void setUseKeepAlive(boolean value) { diff --git a/src/protocol/http/src/main/java/org/apache/jmeter/protocol/http/sampler/PostWriter.java b/src/protocol/http/src/main/java/org/apache/jmeter/protocol/http/sampler/PostWriter.java index c604c760ef3..60d01217525 100644 --- a/src/protocol/http/src/main/java/org/apache/jmeter/protocol/http/sampler/PostWriter.java +++ b/src/protocol/http/src/main/java/org/apache/jmeter/protocol/http/sampler/PostWriter.java @@ -22,8 +22,10 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.io.UnsupportedEncodingException; +import java.io.OutputStreamWriter; +import java.io.Writer; import java.net.URLConnection; +import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Paths; @@ -32,7 +34,6 @@ import org.apache.jmeter.protocol.http.util.HTTPArgument; import org.apache.jmeter.protocol.http.util.HTTPConstants; import org.apache.jmeter.protocol.http.util.HTTPFileArg; -import org.apache.jmeter.samplers.SampleResult; import org.apache.jmeter.testelement.property.JMeterProperty; /** @@ -49,7 +50,9 @@ public class PostWriter { private static final byte[] CRLF = { 0x0d, 0x0A }; - public static final String ENCODING = StandardCharsets.ISO_8859_1.name(); + private static final String CRLF_STRING = "\r\n"; + + public static final String ENCODING = StandardCharsets.UTF_8.name(); /** The form data that is going to be sent as url encoded */ protected byte[] formDataUrlEncoded; @@ -58,6 +61,9 @@ public class PostWriter { /** The boundary string for multipart */ private final String boundary; + private final String multipartDivider; + private final byte[] multipartDividerBytes; + /** * Constructor for PostWriter. * Uses the PostWriter.BOUNDARY as the boundary string @@ -74,6 +80,8 @@ public PostWriter() { */ public PostWriter(String boundary) { this.boundary = boundary; + this.multipartDivider = DASH_DASH + boundary; + this.multipartDividerBytes = multipartDivider.getBytes(StandardCharsets.UTF_8); } /** @@ -94,9 +102,6 @@ public String sendPostData(URLConnection connection, HTTPSamplerBase sampler) th HTTPFileArg[] files = sampler.getHTTPFiles(); String contentEncoding = sampler.getContentEncoding(); - if(contentEncoding == null || contentEncoding.length() == 0) { - contentEncoding = ENCODING; - } // Check if we should do a multipart/form-data or an // application/x-www-form-urlencoded post request @@ -111,11 +116,16 @@ public String sendPostData(URLConnection connection, HTTPSamplerBase sampler) th postedBody.append(new String(formDataPostBody, contentEncoding)); // Add any files - for (int i=0; i < files.length; i++) { - HTTPFileArg file = files[i]; + for (HTTPFileArg file : files) { + out.write(multipartDividerBytes); + out.write(CRLF); + postedBody.append(multipartDivider); + postedBody.append("\r\n"); + // First write the start multipart file final String headerValue = file.getHeader(); - byte[] header = headerValue.getBytes(ENCODING); + // TODO: reuse the bytes prepared in org.apache.jmeter.protocol.http.sampler.PostWriter.setHeaders + byte[] header = headerValue.getBytes(contentEncoding); out.write(header); // Retrieve the formatted data using the same encoding used to create it postedBody.append(headerValue); @@ -123,22 +133,15 @@ public String sendPostData(URLConnection connection, HTTPSamplerBase sampler) th writeFileToStream(file.getPath(), out); // We just add placeholder text for file content postedBody.append(""); // $NON-NLS-1$ - // Write the end of multipart file - byte[] fileMultipartEndDivider = getFileMultipartEndDivider(); - out.write(fileMultipartEndDivider); - // Retrieve the formatted data using the same encoding used to create it - postedBody.append(new String(fileMultipartEndDivider, ENCODING)); - if(i + 1 < files.length) { - out.write(CRLF); - postedBody.append(new String(CRLF, SampleResult.DEFAULT_HTTP_ENCODING)); - } + out.write(CRLF); + postedBody.append(CRLF_STRING); } - // Write end of multipart - byte[] multipartEndDivider = getMultipartEndDivider(); - out.write(multipartEndDivider); - postedBody.append(new String(multipartEndDivider, ENCODING)); - - out.flush(); + // Write end of multipart: --, boundary, --, CRLF + out.write(multipartDividerBytes); + out.write(DASH_DASH_BYTES); + out.write(CRLF); + postedBody.append(multipartDivider); + postedBody.append("--\r\n"); out.close(); } else { @@ -172,9 +175,6 @@ else if (formDataUrlEncoded != null){ // may be null for PUT public void setHeaders(URLConnection connection, HTTPSamplerBase sampler) throws IOException { // Get the encoding to use for the request String contentEncoding = sampler.getContentEncoding(); - if(contentEncoding == null || contentEncoding.length() == 0) { - contentEncoding = ENCODING; - } long contentLength = 0L; HTTPFileArg[] files = sampler.getHTTPFiles(); @@ -188,9 +188,7 @@ public void setHeaders(URLConnection connection, HTTPSamplerBase sampler) throws // Write the form section ByteArrayOutputStream bos = new ByteArrayOutputStream(); - - // First the multipart start divider - bos.write(getMultipartDivider()); + OutputStreamWriter osw = new OutputStreamWriter(bos, contentEncoding); // Add any parameters for (JMeterProperty jMeterProperty : sampler.getArguments()) { HTTPArgument arg = (HTTPArgument) jMeterProperty.getObjectValue(); @@ -198,49 +196,40 @@ public void setHeaders(URLConnection connection, HTTPSamplerBase sampler) throws if (arg.isSkippable(parameterName)) { continue; } - // End the previous multipart - bos.write(CRLF); // Write multipart for parameter - writeFormMultipart(bos, parameterName, arg.getValue(), contentEncoding, sampler.getDoBrowserCompatibleMultipart()); + writeFormMultipart(osw, contentEncoding, parameterName, arg.getValue(), sampler.getDoBrowserCompatibleMultipart()); } - // If there are any files, we need to end the previous multipart - if(files.length > 0) { - // End the previous multipart - bos.write(CRLF); - } - bos.flush(); + osw.flush(); // Keep the content, will be sent later formDataPostBody = bos.toByteArray(); - bos.close(); contentLength = formDataPostBody.length; // Now we just construct any multipart for the files // We only construct the file multipart start, we do not write // the actual file content - for (int i=0; i < files.length; i++) { + for (int i = 0; i < files.length; i++) { + bos.reset(); + contentLength += multipartDividerBytes.length + CRLF.length; HTTPFileArg file = files[i]; // Write multipart for file - bos = new ByteArrayOutputStream(); - writeStartFileMultipart(bos, file.getPath(), file.getParamName(), file.getMimeType()); - bos.flush(); - String header = bos.toString(contentEncoding);// TODO is this correct? + writeStartFileMultipart(osw, contentEncoding, file.getPath(), file.getParamName(), file.getMimeType()); + osw.flush(); + // Technically speaking, we should refrain from decoding the header to string + // since we will have to encode it again when sending the request + // However, HTTPFileArg#setHeaer(byte[]) does not exist yet + String header = bos.toString(contentEncoding); // If this is not the first file we can't write its header now // for simplicity we always save it, even if there is only one file file.setHeader(header); - bos.close(); - contentLength += header.length(); + contentLength += bos.size(); // Add also the length of the file content File uploadFile = new File(file.getPath()); contentLength += uploadFile.length(); - // And the end of the file multipart - contentLength += getFileMultipartEndDivider().length; - if(i+1 < files.length) { - contentLength += CRLF.length; - } + contentLength += CRLF.length; } // Add the end of multipart - contentLength += getMultipartEndDivider().length; + contentLength += multipartDividerBytes.length + DASH_DASH_BYTES.length + CRLF.length; // Set the content length connection.setRequestProperty(HTTPConstants.HEADER_CONTENT_LENGTH, Long.toString(contentLength)); @@ -343,60 +332,33 @@ protected String getBoundary() { return boundary; } - /** - * Get the bytes used to separate multiparts - * Encoded using ENCODING - * - * @return the bytes used to separate multiparts - * @throws IOException - */ - private byte[] getMultipartDivider() throws IOException { - return (DASH_DASH + getBoundary()).getBytes(ENCODING); - } - - /** - * Get the bytes used to end a file multipart - * Encoded using ENCODING - * - * @return the bytes used to end a file multipart - * @throws IOException - */ - private byte[] getFileMultipartEndDivider() throws IOException{ - byte[] ending = getMultipartDivider(); - byte[] completeEnding = new byte[ending.length + CRLF.length]; - System.arraycopy(CRLF, 0, completeEnding, 0, CRLF.length); - System.arraycopy(ending, 0, completeEnding, CRLF.length, ending.length); - return completeEnding; - } - - /** - * Get the bytes used to end the multipart request - * - * @return the bytes used to end the multipart request - */ - private static byte[] getMultipartEndDivider(){ - byte[] ending = DASH_DASH_BYTES; - byte[] completeEnding = new byte[ending.length + CRLF.length]; - System.arraycopy(ending, 0, completeEnding, 0, ending.length); - System.arraycopy(CRLF, 0, completeEnding, ending.length, CRLF.length); - return completeEnding; - } - /** * Write the start of a file multipart, up to the point where the * actual file content should be written */ - private static void writeStartFileMultipart(OutputStream out, String filename, + private static void writeStartFileMultipart( + Writer out, + String contentEncoding, + String filePath, String nameField, String mimetype) throws IOException { write(out, "Content-Disposition: form-data; name=\""); // $NON-NLS-1$ - write(out, nameField); + // See quoting in (line is wrapped to avoid checkstyle warnings) + // https://source.chromium.org/chromium/chromium/src/+/main:third_party/blink/renderer/platform/network/form_data_encoder.cc + // ;l=142;drc=4cd749d0d82138ff31ed3a2bc5d925bb6d83fe16 + write(out, ConversionUtils.percentEncode(nameField)); write(out, "\"; filename=\"");// $NON-NLS-1$ - write(out, ConversionUtils.percentEncode(new File(filename).getName())); + String filename = new File(filePath).getName(); + // See quoting in + // https://source.chromium.org/chromium/chromium/src/+/main:third_party/blink/renderer/platform/network/form_data_encoder.cc + // ;l=190;drc=4cd749d0d82138ff31ed3a2bc5d925bb6d83fe16 + Charset charset = Charset.forName(contentEncoding); + write(out, ConversionUtils.percentEncode(ConversionUtils.encodeWithEntities(filename, charset))); writeln(out, "\""); // $NON-NLS-1$ - writeln(out, "Content-Type: " + mimetype); // $NON-NLS-1$ + write(out, "Content-Type: "); // $NON-NLS-1$ + writeln(out, mimetype); writeln(out, "Content-Transfer-Encoding: binary"); // $NON-NLS-1$ - out.write(CRLF); + out.write(CRLF_STRING); } /** @@ -423,32 +385,33 @@ private static void writeFileToStream(String filename, OutputStream out) throws /** * Writes form data in multipart format. */ - private void writeFormMultipart(OutputStream out, String name, String value, String charSet, + private void writeFormMultipart( + Writer out, + String contentEncoding, + String name, String value, boolean browserCompatibleMultipart) throws IOException { - writeln(out, "Content-Disposition: form-data; name=\"" + name + "\""); // $NON-NLS-1$ // $NON-NLS-2$ + writeln(out, multipartDivider); + write(out, "Content-Disposition: form-data; name=\""); + write(out, ConversionUtils.percentEncode(name)); + writeln(out, "\""); // $NON-NLS-1$ // $NON-NLS-2$ if (!browserCompatibleMultipart){ - writeln(out, "Content-Type: text/plain; charset=" + charSet); // $NON-NLS-1$ + write(out, "Content-Type: text/plain; charset="); // $NON-NLS-1$ + writeln(out, contentEncoding); writeln(out, "Content-Transfer-Encoding: 8bit"); // $NON-NLS-1$ } - out.write(CRLF); - out.write(value.getBytes(charSet)); - out.write(CRLF); - // Write boundary end marker - out.write(getMultipartDivider()); + out.write(CRLF_STRING); + out.write(value); + out.write(CRLF_STRING); } - private static void write(OutputStream out, String value) - throws UnsupportedEncodingException, IOException - { - out.write(value.getBytes(ENCODING)); + private static void write(Writer out, String value) throws IOException { + out.write(value); } - private static void writeln(OutputStream out, String value) - throws UnsupportedEncodingException, IOException - { - out.write(value.getBytes(ENCODING)); - out.write(CRLF); + private static void writeln(Writer out, String value) throws IOException { + out.write(value); + out.write(CRLF_STRING); } } diff --git a/src/protocol/http/src/main/java/org/apache/jmeter/protocol/http/sampler/PutWriter.java b/src/protocol/http/src/main/java/org/apache/jmeter/protocol/http/sampler/PutWriter.java index 1b46936df90..d6f5c87cd59 100644 --- a/src/protocol/http/src/main/java/org/apache/jmeter/protocol/http/sampler/PutWriter.java +++ b/src/protocol/http/src/main/java/org/apache/jmeter/protocol/http/sampler/PutWriter.java @@ -44,9 +44,6 @@ public PutWriter() { public void setHeaders(URLConnection connection, HTTPSamplerBase sampler) throws IOException { // Get the encoding to use for the request String contentEncoding = sampler.getContentEncoding(); - if(contentEncoding == null || contentEncoding.length() == 0) { - contentEncoding = ENCODING; - } long contentLength = 0L; boolean hasPutBody = false; diff --git a/src/protocol/http/src/main/java/org/apache/jmeter/protocol/http/util/ConversionUtils.java b/src/protocol/http/src/main/java/org/apache/jmeter/protocol/http/util/ConversionUtils.java index 2c71d8888af..add440fe693 100644 --- a/src/protocol/http/src/main/java/org/apache/jmeter/protocol/http/util/ConversionUtils.java +++ b/src/protocol/http/src/main/java/org/apache/jmeter/protocol/http/util/ConversionUtils.java @@ -22,7 +22,12 @@ import java.net.URISyntaxException; import java.net.URL; import java.net.URLDecoder; +import java.nio.ByteBuffer; +import java.nio.CharBuffer; import java.nio.charset.Charset; +import java.nio.charset.CharsetEncoder; +import java.nio.charset.CoderResult; +import java.nio.charset.CodingErrorAction; import java.nio.charset.IllegalCharsetNameException; import java.nio.charset.StandardCharsets; import java.util.ArrayList; @@ -97,20 +102,85 @@ public static String getEncodingFromContentType(String contentType){ } /** - * Encodes the string according to RFC7578 and RFC3986. - * The string is UTF-8 encoded, and non-ASCII bytes are represented as {@code %XX}. - * It is close to UrlEncode, however, {@code percentEncode} does not replace space with +. + * Encodes strings for {@code multipart/form-data} names and values. + * The encoding is {@code "} as {@code %22}, {@code CR} as {@code %0D}, and {@code LF} as {@code %0A}. + * Note: {@code %} is not encoded, so it creates ambiguity which might be resolved in a later specification version. + * @see Multipart form data specification + * @see Escaping % in multipart/form-data * @param value input value to convert * @return converted value * @since 5.6 */ @API(status = API.Status.MAINTAINED, since = "5.6") public static String percentEncode(String value) { - try { - return new URI(null, null, value, null).toASCIIString(); - } catch (URISyntaxException e) { - throw new IllegalStateException("Can't encode value " + value, e); + if (value.indexOf('"') == -1 && value.indexOf('\r') == -1 && value.indexOf('\n') == -1) { + return value; + } + StringBuilder sb = new StringBuilder(value.length() + 2); + for (int i = 0; i < value.length(); i++) { + char c = value.charAt(i); + switch (c) { + case '"': + sb.append("%22"); + break; + case 0x0A: + sb.append("%0A"); + break; + case 0x0D: + sb.append("%0D"); + break; + default: + sb.append(c); + break; + } + } + return sb.toString(); + } + + /** + * Encodes non-encodable characters as HTML entities like e.g. &#128514; for 😂. + * @param value value to encode + * @param charset charset that will be used for encoding, defaults to UTF-8 if null + * @return input value with non-encodable characters replaced with HTML entities + */ + @API(status = API.Status.EXPERIMENTAL, since = "5.6.1") + public static String encodeWithEntities(String value, Charset charset) { + // See the reason at + // https://source.chromium.org/chromium/chromium/src/+/main:third_party/blink/renderer/platform/network/form_data_encoder.cc; + // l=162-191;drc=4cd749d0d82138ff31ed3a2bc5d925bb6d83fe16 + + if (charset == null) { + charset = StandardCharsets.UTF_8; + } + CharsetEncoder encoder = charset.newEncoder(); + if (encoder.canEncode(value)) { + // When the strinc can be encoded, leave it intact + return value; + } + // Some of the characters can't be encoded, so replace them with HTML entities + StringBuilder sb = new StringBuilder(value.length() + 10); + encoder.onUnmappableCharacter(CodingErrorAction.REPORT); + CharBuffer input = CharBuffer.wrap(value); + ByteBuffer output = ByteBuffer.allocate(Math.min(1000, (int) (encoder.maxBytesPerChar() * value.length()))); + int lastPos = 0; + while (input.position() < input.limit()) { + output.clear(); + CoderResult cr = encoder.encode(input, output, true); + + // Append successfully encoded chars + if (input.position() > lastPos) { + sb.append(value, lastPos, input.position()); + lastPos = input.position(); + } + + if (cr.isUnmappable()) { + int codePoint = value.codePointAt(input.position()); + sb.append("&#").append(codePoint); + input.position(input.position() + Character.charCount(codePoint)); + lastPos = input.position(); + } } + return sb.toString(); } /** diff --git a/src/protocol/http/src/main/kotlin/org/apache/jmeter/protocol/http/sampler/HTTPSamplerBaseSchema.kt b/src/protocol/http/src/main/kotlin/org/apache/jmeter/protocol/http/sampler/HTTPSamplerBaseSchema.kt index ed67101b18a..587d9ed19f4 100644 --- a/src/protocol/http/src/main/kotlin/org/apache/jmeter/protocol/http/sampler/HTTPSamplerBaseSchema.kt +++ b/src/protocol/http/src/main/kotlin/org/apache/jmeter/protocol/http/sampler/HTTPSamplerBaseSchema.kt @@ -32,6 +32,7 @@ import org.apache.jmeter.testelement.schema.IntegerPropertyDescriptor import org.apache.jmeter.testelement.schema.StringPropertyDescriptor import org.apache.jmeter.testelement.schema.TestElementPropertyDescriptor import org.apiguardian.api.API +import java.nio.charset.StandardCharsets /** * Lists properties of a [HTTPSamplerBase]. @@ -81,7 +82,7 @@ public abstract class HTTPSamplerBaseSchema : TestElementSchema() { public val proxy: HTTPSamplerProxyParamsSchema by HTTPSamplerProxyParamsSchema() public val contentEncoding: StringPropertyDescriptor - by string("HTTPSampler.contentEncoding") + by string("HTTPSampler.contentEncoding", default = StandardCharsets.UTF_8.name()) public val implementation: StringPropertyDescriptor by string("HTTPSampler.implementation") diff --git a/src/protocol/http/src/test/java/org/apache/jmeter/protocol/http/proxy/TestHttpRequestHdr.java b/src/protocol/http/src/test/java/org/apache/jmeter/protocol/http/proxy/TestHttpRequestHdr.java index cc4e609cab0..c0b1326b76e 100644 --- a/src/protocol/http/src/test/java/org/apache/jmeter/protocol/http/proxy/TestHttpRequestHdr.java +++ b/src/protocol/http/src/test/java/org/apache/jmeter/protocol/http/proxy/TestHttpRequestHdr.java @@ -24,7 +24,7 @@ import java.io.ByteArrayInputStream; import java.io.IOException; import java.net.URLEncoder; -import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -215,16 +215,25 @@ private void testEncodedArguments(String url) throws Exception { // know the encoding for the page HTTPSamplerBase s = getSamplerForRequest(null, testGetRequest, null); assertEquals(HTTPConstants.GET, s.getMethod()); - assertEquals(queryString, s.getQueryString()); - assertEquals(contentEncoding, s.getContentEncoding()); + // %20 and + are interchangeable in the URL, so we should expect any of them + String actualQueryString = s.getQueryString(); + String alternativeExpectedQueryString = queryString.replaceAll("%20", "+"); + if (!queryString.equals(actualQueryString) && !alternativeExpectedQueryString.equals(actualQueryString)) { + assertEquals( + "%20 is the same as +, so expecting either " + + queryString + " or " + alternativeExpectedQueryString, + queryString, + actualQueryString); + } + assertEquals("UTF-8", s.getContentEncoding()); // Check arguments Arguments arguments = s.getArguments(); assertEquals(3, arguments.getArgumentCount()); // When the encoding is not known, the argument will get the encoded value, and the "encode?" set to false - checkArgument((HTTPArgument)arguments.getArgument(0), "abc%3FSPACE", "a+b", "a+b", contentEncoding, false); - checkArgument((HTTPArgument)arguments.getArgument(1), "space", "a%20b", "a%20b", contentEncoding, false); - checkArgument((HTTPArgument)arguments.getArgument(2), "query", "What%3F", "What%3F", contentEncoding, false); + checkArgument((HTTPArgument)arguments.getArgument(0), "abc?SPACE", "a b", "a+b", contentEncoding, true); + checkArgument((HTTPArgument)arguments.getArgument(1), "space", "a b", "a+b", contentEncoding, true); + checkArgument((HTTPArgument)arguments.getArgument(2), "query", "What?", "What%3F", contentEncoding, true); // A HTTP GET request, with UTF-8 encoding contentEncoding = "UTF-8"; @@ -258,17 +267,24 @@ private void testEncodedArguments(String url) throws Exception { // know the encoding for the page s = getSamplerForRequest(null, testPostRequest, null); assertEquals(HTTPConstants.POST, s.getMethod()); - assertEquals(queryString, s.getQueryString()); - assertEquals(contentEncoding, s.getContentEncoding()); + alternativeExpectedQueryString = expectedQueryString.replaceAll("%20", "+"); + if (!queryString.equals(actualQueryString) && !alternativeExpectedQueryString.equals(actualQueryString)) { + assertEquals( + "%20 is the same as +, so expecting either " + + queryString + " or " + alternativeExpectedQueryString, + queryString, + actualQueryString); + } + assertEquals("UTF-8", s.getContentEncoding()); assertFalse(s.getDoMultipart()); // Check arguments arguments = s.getArguments(); assertEquals(3, arguments.getArgumentCount()); // When the encoding is not known, the argument will get the encoded value, and the "encode?" set to false - checkArgument((HTTPArgument)arguments.getArgument(0), "abc%3FSPACE", "a+b", "a+b", contentEncoding, false); - checkArgument((HTTPArgument)arguments.getArgument(1), "space", "a%20b", "a%20b", contentEncoding, false); - checkArgument((HTTPArgument)arguments.getArgument(2), "query", "What%3F", "What%3F", contentEncoding, false); + checkArgument((HTTPArgument)arguments.getArgument(0), "abc?SPACE", "a b", "a+b", contentEncoding, true); + checkArgument((HTTPArgument)arguments.getArgument(1), "space", "a b", "a+b", contentEncoding, true); + checkArgument((HTTPArgument)arguments.getArgument(2), "query", "What?", "What%3F", contentEncoding, true); // A HTTP POST request, with UTF-8 encoding contentEncoding = "UTF-8"; @@ -318,13 +334,13 @@ private void testGetRequestEncodings(String url) throws Exception { // know the encoding for the page HTTPSamplerBase s = getSamplerForRequest(null, testGetRequest, null); assertEquals(HTTPConstants.GET, s.getMethod()); - assertEquals(contentEncoding, s.getContentEncoding()); + assertEquals("Default content encoding is UTF-8", "UTF-8", s.getContentEncoding()); // Check arguments Arguments arguments = s.getArguments(); assertEquals(2, arguments.getArgumentCount()); checkArgument((HTTPArgument)arguments.getArgument(0), "param1", param1Value, param1Value, contentEncoding, false); - // When the encoding is not known, the argument will get the encoded value, and the "encode?" set to false - checkArgument((HTTPArgument)arguments.getArgument(1), "param2", param2ValueEncoded, param2ValueEncoded, contentEncoding, false); + // When the encoding is not known, it should assume UTF-8 by default + checkArgument((HTTPArgument)arguments.getArgument(1), "param2", param2Value, param2ValueEncoded, contentEncoding, true); // A HTTP GET request, with UTF-8 encoding contentEncoding = "UTF-8"; @@ -384,13 +400,13 @@ public void testPostRequestEncodings() throws Exception { // know the encoding for the page HTTPSamplerBase s = getSamplerForRequest(null, testPostRequest, null); assertEquals(HTTPConstants.POST, s.getMethod()); - assertEquals(contentEncoding, s.getContentEncoding()); + assertEquals("Default content encoding is UTF-8", "UTF-8", s.getContentEncoding()); // Check arguments Arguments arguments = s.getArguments(); assertEquals(2, arguments.getArgumentCount()); checkArgument((HTTPArgument)arguments.getArgument(0), "param1", param1Value, param1Value, contentEncoding, false); - // When the encoding is not known, the argument will get the encoded value, and the "encode?" set to false - checkArgument((HTTPArgument)arguments.getArgument(1), "param2", param2ValueEncoded, param2ValueEncoded, contentEncoding, false); + // When the encoding is not known, we expect UTF-8 by default + checkArgument((HTTPArgument)arguments.getArgument(1), "param2", param2Value, param2ValueEncoded, contentEncoding, true); // A HTTP POST request, with UTF-8 encoding contentEncoding = "UTF-8"; @@ -638,11 +654,10 @@ private HTTPSamplerBase getSamplerForRequest(String url, String request, String ByteArrayInputStream bis = null; if(contentEncoding != null) { bis = new ByteArrayInputStream(request.getBytes(contentEncoding)); - } else { - // Most browsers use ISO-8859-1 as default encoding, even if spec says UTF-8 - bis = new ByteArrayInputStream(request.getBytes("ISO-8859-1")); + // Most browsers use UTF-8 by default + bis = new ByteArrayInputStream(request.getBytes(StandardCharsets.UTF_8)); } req.parse(bis); bis.close(); @@ -671,8 +686,8 @@ private void checkArgument( assertEquals(expectedEncodedValue, arg.getEncodedValue(contentEncoding)); } else { - // Most browsers use ISO-8859-1 as default encoding, even if spec says UTF-8 - assertEquals(expectedEncodedValue, arg.getEncodedValue("ISO-8859-1")); + // Most browsers use UTF-8 as default encoding + assertEquals(expectedEncodedValue, arg.getEncodedValue(StandardCharsets.UTF_8.name())); } assertPrimitiveEquals(expectedEncoded, arg.isAlwaysEncoded()); } @@ -682,8 +697,8 @@ private int getBodyLength(String postBody, String contentEncoding) throws IOExce return postBody.getBytes(contentEncoding).length; } else { - // Most browsers use ISO-8859-1 as default encoding, even if spec says UTF-8 - return postBody.getBytes(Charset.defaultCharset()).length; // TODO - charset? + // Most browsers use UTF-8 + return postBody.getBytes(StandardCharsets.UTF_8).length; // TODO - charset? } } } diff --git a/src/protocol/http/src/test/java/org/apache/jmeter/protocol/http/sampler/PostWriterTest.java b/src/protocol/http/src/test/java/org/apache/jmeter/protocol/http/sampler/PostWriterTest.java index 8de0d14c694..cae9635384e 100644 --- a/src/protocol/http/src/test/java/org/apache/jmeter/protocol/http/sampler/PostWriterTest.java +++ b/src/protocol/http/src/test/java/org/apache/jmeter/protocol/http/sampler/PostWriterTest.java @@ -114,8 +114,8 @@ public void testSendPostData() throws IOException { checkContentTypeMultipart(connection, PostWriter.BOUNDARY); byte[] expectedFormBody = createExpectedOutput(PostWriter.BOUNDARY, null, titleValue, descriptionValue, TEST_FILE_CONTENT); - checkContentLength(connection, expectedFormBody.length); checkArraysHaveSameContent(expectedFormBody, connection.getOutputStreamContent()); + checkContentLength(connection, expectedFormBody.length); connection.disconnect(); // Test sending data as ISO-8859-1 @@ -545,14 +545,14 @@ public void testSendFormData_Urlencoded() throws IOException { checkNoContentType(connection); StringBuilder sb = new StringBuilder(); - expectedUrl = sb.append("title=").append(titleValue.replaceAll("%20", "+").replaceAll("%C3%85", "%C5")) - .append("&description=").append(descriptionValue.replaceAll("%C3%85", "%C5")).toString().getBytes("US-ASCII"); - checkContentLength(connection, expectedUrl.length); + expectedUrl = sb.append("title=").append(titleValue.replaceAll("%20", "+")) + .append("&description=").append(descriptionValue).toString().getBytes(StandardCharsets.UTF_8); checkArraysHaveSameContent(expectedUrl, connection.getOutputStreamContent()); + checkContentLength(connection, expectedUrl.length); assertEquals( - // HTTPSampler uses ISO-8859-1 as default encoding - URLDecoder.decode(new String(expectedUrl, "US-ASCII"), "ISO-8859-1"), - URLDecoder.decode(new String(connection.getOutputStreamContent(), "US-ASCII"), "ISO-8859-1")); + // HTTPSampler uses UTF-8 as default encoding + URLDecoder.decode(new String(expectedUrl, StandardCharsets.UTF_8), "UTF-8"), + URLDecoder.decode(new String(connection.getOutputStreamContent(), StandardCharsets.UTF_8), "UTF-8")); connection.disconnect(); // Test sending data as ISO-8859-1 diff --git a/src/protocol/http/src/test/java/org/apache/jmeter/protocol/http/sampler/TestHTTPSamplersAgainstHttpMirrorServer.java b/src/protocol/http/src/test/java/org/apache/jmeter/protocol/http/sampler/TestHTTPSamplersAgainstHttpMirrorServer.java index 1ca676f0c6b..1467ce5affe 100644 --- a/src/protocol/http/src/test/java/org/apache/jmeter/protocol/http/sampler/TestHTTPSamplersAgainstHttpMirrorServer.java +++ b/src/protocol/http/src/test/java/org/apache/jmeter/protocol/http/sampler/TestHTTPSamplersAgainstHttpMirrorServer.java @@ -70,6 +70,8 @@ public class TestHTTPSamplersAgainstHttpMirrorServer extends JMeterTestCaseJUnit private static final String ISO_8859_1 = "ISO-8859-1"; // $NON-NLS-1$ private static final String US_ASCII = "US-ASCII"; // $NON-NLS-1$ + private static final String DEFAULT_HTTP_CONTENT_ENCODING = StandardCharsets.UTF_8.name(); + private static final String CONTENT_TYPE_TEXT_PLAIN = "text/plain"; private static final byte[] CRLF = {0x0d, 0x0A}; @@ -151,21 +153,21 @@ public void itemised_testPostRequest_UrlEncoded3() throws Exception { } public void testPostRequest_FormMultipart_0() throws Exception { - testPostRequest_FormMultipart(HTTP_SAMPLER, ISO_8859_1); + testPostRequest_FormMultipart(HTTP_SAMPLER); } public void testPostRequest_FormMultipart3() throws Exception { // see https://issues.apache.org/jira/browse/HTTPCLIENT-1665 - testPostRequest_FormMultipart(HTTP_SAMPLER3, US_ASCII); + testPostRequest_FormMultipart(HTTP_SAMPLER3); } public void testPostRequest_FileUpload() throws Exception { - testPostRequest_FileUpload(HTTP_SAMPLER, ISO_8859_1); + testPostRequest_FileUpload(HTTP_SAMPLER); } public void testPostRequest_FileUpload3() throws Exception { // see https://issues.apache.org/jira/browse/HTTPCLIENT-1665 - testPostRequest_FileUpload(HTTP_SAMPLER3, US_ASCII); + testPostRequest_FileUpload(HTTP_SAMPLER3); } public void testPostRequest_BodyFromParameterValues() throws Exception { @@ -352,7 +354,7 @@ private void testPostRequest_UrlEncoded(int samplerType, String samplerDefaultEn } } - private void testPostRequest_FormMultipart(int samplerType, String samplerDefaultEncoding) throws Exception { + private void testPostRequest_FormMultipart(int samplerType) throws Exception { String titleField = "title"; String titleValue = "mytitle"; String descriptionField = "description"; @@ -365,7 +367,7 @@ private void testPostRequest_FormMultipart(int samplerType, String samplerDefaul setupFormData(sampler, false, titleField, titleValue, descriptionField, descriptionValue); sampler.setDoMultipart(true); HTTPSampleResult res = executeSampler(sampler); - checkPostRequestFormMultipart(sampler, res, samplerDefaultEncoding, + checkPostRequestFormMultipart(sampler, res, contentEncoding, titleField, titleValue, descriptionField, descriptionValue); @@ -376,7 +378,7 @@ private void testPostRequest_FormMultipart(int samplerType, String samplerDefaul setupFormData(sampler, false, titleField, titleValue, descriptionField, descriptionValue); sampler.setDoMultipart(true); res = executeSampler(sampler); - checkPostRequestFormMultipart(sampler, res, samplerDefaultEncoding, + checkPostRequestFormMultipart(sampler, res, contentEncoding, titleField, titleValue, descriptionField, descriptionValue); @@ -389,7 +391,7 @@ private void testPostRequest_FormMultipart(int samplerType, String samplerDefaul setupFormData(sampler, false, titleField, titleValue, descriptionField, descriptionValue); sampler.setDoMultipart(true); res = executeSampler(sampler); - checkPostRequestFormMultipart(sampler, res, samplerDefaultEncoding, + checkPostRequestFormMultipart(sampler, res, contentEncoding, titleField, titleValue, descriptionField, descriptionValue); @@ -403,7 +405,7 @@ private void testPostRequest_FormMultipart(int samplerType, String samplerDefaul setupFormData(sampler, false, titleField, titleValue, descriptionField, descriptionValue); sampler.setDoMultipart(true); res = executeSampler(sampler); - checkPostRequestFormMultipart(sampler, res, samplerDefaultEncoding, + checkPostRequestFormMultipart(sampler, res, contentEncoding, titleField, titleValue, descriptionField, descriptionValue); @@ -418,7 +420,7 @@ private void testPostRequest_FormMultipart(int samplerType, String samplerDefaul res = executeSampler(sampler); String expectedTitleValue = "mytitle/="; String expectedDescriptionValue = "mydescription /\\"; - checkPostRequestFormMultipart(sampler, res, samplerDefaultEncoding, + checkPostRequestFormMultipart(sampler, res, contentEncoding, titleField, expectedTitleValue, descriptionField, expectedDescriptionValue); @@ -431,7 +433,7 @@ private void testPostRequest_FormMultipart(int samplerType, String samplerDefaul setupFormData(sampler, false, titleField, titleValue, descriptionField, descriptionValue); sampler.setDoMultipart(true); res = executeSampler(sampler); - checkPostRequestFormMultipart(sampler, res, samplerDefaultEncoding, + checkPostRequestFormMultipart(sampler, res, contentEncoding, titleField, titleValue, descriptionField, descriptionValue); @@ -459,12 +461,12 @@ private void testPostRequest_FormMultipart(int samplerType, String samplerDefaul res = executeSampler(sampler); expectedTitleValue = "a test\u00c5mytitle\u0153\u20a1\u0115\u00c5"; expectedDescriptionValue = "mydescription\u0153\u20a1\u0115\u00c5the_end"; - checkPostRequestFormMultipart(sampler, res, samplerDefaultEncoding, + checkPostRequestFormMultipart(sampler, res, contentEncoding, titleField, expectedTitleValue, descriptionField, expectedDescriptionValue); } - private void testPostRequest_FileUpload(int samplerType, String samplerDefaultEncoding) throws Exception { + private void testPostRequest_FileUpload(int samplerType) throws Exception { String titleField = "title"; String titleValue = "mytitle"; String descriptionField = "description"; @@ -480,7 +482,7 @@ private void testPostRequest_FileUpload(int samplerType, String samplerDefaultEn descriptionField, descriptionValue, fileField, temporaryFile, fileMimeType); HTTPSampleResult res = executeSampler(sampler); - checkPostRequestFileUpload(sampler, res, samplerDefaultEncoding, + checkPostRequestFileUpload(sampler, res, contentEncoding, titleField, titleValue, descriptionField, descriptionValue, fileField, temporaryFile, fileMimeType, TEST_FILE_CONTENT); @@ -493,7 +495,7 @@ private void testPostRequest_FileUpload(int samplerType, String samplerDefaultEn descriptionField, descriptionValue, fileField, temporaryFile, fileMimeType); res = executeSampler(sampler); - checkPostRequestFileUpload(sampler, res, samplerDefaultEncoding, + checkPostRequestFileUpload(sampler, res, contentEncoding, titleField, titleValue, descriptionField, descriptionValue, fileField, temporaryFile, fileMimeType, TEST_FILE_CONTENT); @@ -508,7 +510,7 @@ private void testPostRequest_FileUpload(int samplerType, String samplerDefaultEn descriptionField, descriptionValue, fileField, temporaryFile, fileMimeType); res = executeSampler(sampler); - checkPostRequestFileUpload(sampler, res, samplerDefaultEncoding, + checkPostRequestFileUpload(sampler, res, contentEncoding, titleField, titleValue, descriptionField, descriptionValue, fileField, temporaryFile, fileMimeType, TEST_FILE_CONTENT); @@ -868,14 +870,13 @@ private void checkPostRequestUrlEncoded( private void checkPostRequestFormMultipart( HTTPSamplerBase sampler, HTTPSampleResult res, - String samplerDefaultEncoding, String contentEncoding, String titleField, String titleValue, String descriptionField, String descriptionValue) throws IOException { if (contentEncoding == null || contentEncoding.isEmpty()) { - contentEncoding = samplerDefaultEncoding; + contentEncoding = DEFAULT_HTTP_CONTENT_ENCODING; } // Check URL assertEquals(sampler.getUrl(), res.getURL()); @@ -913,7 +914,6 @@ private void checkPostRequestFormMultipart( private void checkPostRequestFileUpload( HTTPSamplerBase sampler, HTTPSampleResult res, - String samplerDefaultEncoding, String contentEncoding, String titleField, String titleValue, @@ -924,7 +924,7 @@ private void checkPostRequestFileUpload( String fileMimeType, byte[] fileContent) throws IOException { if (contentEncoding == null || contentEncoding.isEmpty()) { - contentEncoding = samplerDefaultEncoding; + contentEncoding = DEFAULT_HTTP_CONTENT_ENCODING; } // Check URL assertEquals(sampler.getUrl(), res.getURL()); diff --git a/src/protocol/http/src/test/kotlin/org/apache/jmeter/protocol/http/sampler/HttpSamplerTest.kt b/src/protocol/http/src/test/kotlin/org/apache/jmeter/protocol/http/sampler/HttpSamplerTest.kt index 72b6eea1c7d..c3bd08825e0 100644 --- a/src/protocol/http/src/test/kotlin/org/apache/jmeter/protocol/http/sampler/HttpSamplerTest.kt +++ b/src/protocol/http/src/test/kotlin/org/apache/jmeter/protocol/http/sampler/HttpSamplerTest.kt @@ -21,16 +21,19 @@ import com.github.tomakehurst.wiremock.client.WireMock.aMultipart import com.github.tomakehurst.wiremock.client.WireMock.aResponse import com.github.tomakehurst.wiremock.client.WireMock.containing import com.github.tomakehurst.wiremock.client.WireMock.equalTo +import com.github.tomakehurst.wiremock.client.WireMock.matching import com.github.tomakehurst.wiremock.client.WireMock.post import com.github.tomakehurst.wiremock.client.WireMock.postRequestedFor import com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo import com.github.tomakehurst.wiremock.junit5.WireMockRuntimeInfo import com.github.tomakehurst.wiremock.junit5.WireMockTest +import com.github.tomakehurst.wiremock.matching.RequestPatternBuilder import org.apache.jmeter.control.LoopController import org.apache.jmeter.junit.JMeterTestCase import org.apache.jmeter.protocol.http.util.HTTPFileArg import org.apache.jmeter.test.assertions.executePlanAndCollectEvents import org.apache.jmeter.threads.ThreadGroup +import org.apache.jmeter.treebuilder.TreeBuilder import org.junit.jupiter.api.Assumptions.assumeTrue import org.junit.jupiter.api.io.TempDir import org.junit.jupiter.params.ParameterizedTest @@ -46,6 +49,30 @@ class HttpSamplerTest : JMeterTestCase() { @TempDir lateinit var dir: Path + fun TreeBuilder.oneRequest(body: ThreadGroup.() -> Unit) { + ThreadGroup::class { + numThreads = 1 + rampUp = 0 + setSamplerController( + LoopController().apply { + loops = 1 + } + ) + body() + } + } + + fun TreeBuilder.httpPost(body: HTTPSamplerProxy.() -> Unit) { + org.apache.jmeter.protocol.http.sampler.HTTPSamplerProxy::class { + name = "Upload file" + method = "POST" + domain = "localhost" + path = "/upload" + doMultipart = true + body() + } + } + @ParameterizedTest @ValueSource(strings = ["Java", "HttpClient4"]) fun `upload file uses percent encoding for filename`(httpImplementation: String, server: WireMockRuntimeInfo) { @@ -63,28 +90,17 @@ class HttpSamplerTest : JMeterTestCase() { dir.resolve("testfile привет %.txt") } catch (e: InvalidPathException) { assumeTrue(false) { - "Skipping the test as the filesystem does not suppport unicode filenames" + "Skipping the test as the filesystem does not support unicode filenames" } TODO("This is never reached as the assumption above throws error") } testFile.writeText("hello, привет") executePlanAndCollectEvents(10.seconds) { - ThreadGroup::class { - numThreads = 1 - rampUp = 0 - setSamplerController( - LoopController().apply { - loops = 1 - } - ) - org.apache.jmeter.protocol.http.sampler.HTTPSamplerProxy::class { - name = "Upload file" + oneRequest { + httpPost { implementation = httpImplementation - method = "POST" - domain = "localhost" port = server.httpPort - path = "/upload" httpFiles = arrayOf( HTTPFileArg(testFile.absolutePathString(), "file_parameter", "application/octet-stream") ) @@ -99,10 +115,253 @@ class HttpSamplerTest : JMeterTestCase() { aMultipart("file_parameter") .withHeader( "Content-Disposition", - containing("filename=\"testfile%20%D0%BF%D1%80%D0%B8%D0%B2%D0%B5%D1%82%20%25.txt\"") + // Only CR, LF, and % should be percent-encoded + containing("filename=\"testfile привет %.txt\"") ) .withBody(equalTo("hello, привет")) ) ) } + + fun RequestPatternBuilder.withRequestBody( + httpImplementation: String, + body: String + ) = apply { + // normalize line endings to CRLF + val normalizedBody = body.replace("\r\n", "\n").replace("\n", "\r\n") + withRequestBody( + if (httpImplementation == "Java") { + equalTo(normalizedBody) + } else { + matching( + normalizedBody + .replace(PostWriter.BOUNDARY, "[^ \\n\\r]{1,69}?") + ) + } + ) + } + + @ParameterizedTest + @ValueSource(strings = ["Java", "HttpClient4"]) + fun `one parameter`(httpImplementation: String, server: WireMockRuntimeInfo) { + server.wireMock.register( + post("/upload").willReturn(aResponse().withStatus(200)) + ) + + executePlanAndCollectEvents(10.seconds) { + oneRequest { + httpPost { + implementation = httpImplementation + port = server.httpPort + addArgument("hello", "world") + } + } + } + + server.wireMock.verifyThat( + 1, + postRequestedFor(urlEqualTo("/upload")) + .withRequestBodyPart( + aMultipart("hello") + .withBody(equalTo("world")) + .build() + ) + .withRequestBody( + httpImplementation, + """ + -----------------------------7d159c1302d0y0 + Content-Disposition: form-data; name="hello" + Content-Type: text/plain; charset=UTF-8 + Content-Transfer-Encoding: 8bit + + world + -----------------------------7d159c1302d0y0-- + + """.trimIndent() + ) + ) + } + + @ParameterizedTest + @ValueSource(strings = ["Java", "HttpClient4"]) + fun `two parameters`(httpImplementation: String, server: WireMockRuntimeInfo) { + server.wireMock.register( + post("/upload").willReturn(aResponse().withStatus(200)) + ) + + executePlanAndCollectEvents(10.seconds) { + oneRequest { + httpPost { + implementation = httpImplementation + port = server.httpPort + addArgument("hello", "world") + addArgument("name", "Tim") + } + } + } + + server.wireMock.verifyThat( + 1, + postRequestedFor(urlEqualTo("/upload")) + .withRequestBodyPart( + aMultipart("hello").withBody(equalTo("world")).build() + ) + .withRequestBodyPart( + aMultipart("name").withBody(equalTo("Tim")).build() + ) + .withRequestBody( + httpImplementation, + """ + -----------------------------7d159c1302d0y0 + Content-Disposition: form-data; name="hello" + Content-Type: text/plain; charset=UTF-8 + Content-Transfer-Encoding: 8bit + + world + -----------------------------7d159c1302d0y0 + Content-Disposition: form-data; name="name" + Content-Type: text/plain; charset=UTF-8 + Content-Transfer-Encoding: 8bit + + Tim + -----------------------------7d159c1302d0y0-- + + """.trimIndent() + ) + ) + } + + @ParameterizedTest + @ValueSource(strings = ["Java", "HttpClient4"]) + fun `two parameters and file`(httpImplementation: String, server: WireMockRuntimeInfo) { + server.wireMock.register( + post("/upload").willReturn(aResponse().withStatus(200)) + ) + + val testFile = dir.resolve("testfile.txt").apply { + writeText("file contents") + } + + executePlanAndCollectEvents(10.seconds) { + oneRequest { + httpPost { + implementation = httpImplementation + port = server.httpPort + addArgument("hello", "world") + addArgument("name", "Tim") + httpFiles = arrayOf( + HTTPFileArg(testFile.absolutePathString(), "file_parameter", "application/octet-stream") + ) + } + } + } + + server.wireMock.verifyThat( + 1, + postRequestedFor(urlEqualTo("/upload")) + .withRequestBodyPart( + aMultipart("hello").withBody(equalTo("world")).build() + ) + .withRequestBodyPart( + aMultipart("name").withBody(equalTo("Tim")).build() + ) + .withRequestBody( + httpImplementation, + """ + -----------------------------7d159c1302d0y0 + Content-Disposition: form-data; name="hello" + Content-Type: text/plain; charset=UTF-8 + Content-Transfer-Encoding: 8bit + + world + -----------------------------7d159c1302d0y0 + Content-Disposition: form-data; name="name" + Content-Type: text/plain; charset=UTF-8 + Content-Transfer-Encoding: 8bit + + Tim + -----------------------------7d159c1302d0y0 + Content-Disposition: form-data; name="file_parameter"; filename="testfile.txt" + Content-Type: application/octet-stream + Content-Transfer-Encoding: binary + + file contents + -----------------------------7d159c1302d0y0-- + + """.trimIndent() + ) + ) + } + + @ParameterizedTest + @ValueSource(strings = ["Java", "HttpClient4"]) + fun `two parameters and two files`(httpImplementation: String, server: WireMockRuntimeInfo) { + server.wireMock.register( + post("/upload").willReturn(aResponse().withStatus(200)) + ) + + val testFile1 = dir.resolve("testfile1.txt").apply { + writeText("file contents1") + } + val testFile2 = dir.resolve("testfile2.txt").apply { + writeText("file contents2") + } + + executePlanAndCollectEvents(10.seconds) { + oneRequest { + httpPost { + implementation = httpImplementation + port = server.httpPort + addArgument("hello", "world") + addArgument("name", "Tim") + httpFiles = arrayOf( + HTTPFileArg(testFile1.absolutePathString(), "file_parameter", "application/octet-stream"), + HTTPFileArg(testFile2.absolutePathString(), "file_parameter", "application/octet-stream"), + ) + } + } + } + + server.wireMock.verifyThat( + 1, + postRequestedFor(urlEqualTo("/upload")) + .withRequestBodyPart( + aMultipart("hello").withBody(equalTo("world")).build() + ) + .withRequestBodyPart( + aMultipart("name").withBody(equalTo("Tim")).build() + ) + .withRequestBody( + httpImplementation, + """ + -----------------------------7d159c1302d0y0 + Content-Disposition: form-data; name="hello" + Content-Type: text/plain; charset=UTF-8 + Content-Transfer-Encoding: 8bit + + world + -----------------------------7d159c1302d0y0 + Content-Disposition: form-data; name="name" + Content-Type: text/plain; charset=UTF-8 + Content-Transfer-Encoding: 8bit + + Tim + -----------------------------7d159c1302d0y0 + Content-Disposition: form-data; name="file_parameter"; filename="testfile1.txt" + Content-Type: application/octet-stream + Content-Transfer-Encoding: binary + + file contents1 + -----------------------------7d159c1302d0y0 + Content-Disposition: form-data; name="file_parameter"; filename="testfile2.txt" + Content-Type: application/octet-stream + Content-Transfer-Encoding: binary + + file contents2 + -----------------------------7d159c1302d0y0-- + + """.trimIndent() + ) + ) + } } diff --git a/src/protocol/http/src/test/kotlin/org/apache/jmeter/protocol/http/util/ConversionUtilsTest.kt b/src/protocol/http/src/test/kotlin/org/apache/jmeter/protocol/http/util/ConversionUtilsTest.kt index eed34b6ee70..d05a2544ee1 100644 --- a/src/protocol/http/src/test/kotlin/org/apache/jmeter/protocol/http/util/ConversionUtilsTest.kt +++ b/src/protocol/http/src/test/kotlin/org/apache/jmeter/protocol/http/util/ConversionUtilsTest.kt @@ -19,24 +19,49 @@ package org.apache.jmeter.protocol.http.util import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.params.ParameterizedTest -import org.junit.jupiter.params.provider.CsvSource +import org.junit.jupiter.params.provider.Arguments.arguments +import org.junit.jupiter.params.provider.MethodSource +import java.nio.charset.Charset +import java.nio.charset.StandardCharsets class ConversionUtilsTest { + companion object { + @JvmStatic + fun percentEncodeValues() = + listOf( + arguments("hello", "hello"), + arguments("%", "%"), + arguments("\"", "%22"), + arguments(" ", " "), + arguments("\r", "%0D"), + arguments("\n", "%0A"), + arguments("\r\r\n\n", "%0D%0D%0A%0A"), + arguments("😃", "😃"), + arguments("comment ça va?", "comment ça va?"), + arguments("quoted \"content\"", "quoted %22content%22"), + ) + + @JvmStatic + fun htmlEntityValues() = + listOf( + arguments("Hello, 😃, world", "Hello, 😃, world", StandardCharsets.UTF_8), + arguments("Hello, 😃, world", "Hello, 😃, world", StandardCharsets.ISO_8859_1), + arguments("丈, 😃, and नि", "丈, 😃, and नि", StandardCharsets.UTF_8), + arguments("丈, 😃, and नि", "丈, 😃, and नि", StandardCharsets.ISO_8859_1), + ) + } + @ParameterizedTest - @CsvSource( - ignoreLeadingAndTrailingWhitespace = false, - value = [ - "hello,hello", - "%,%25", - "\",%22", - " ,%20", - "😃,%F0%9F%98%83", - "comment ça va?,comment%20%C3%A7a%20va%3F", - ] - ) + @MethodSource("percentEncodeValues") fun percentEncode(input: String, output: String) { assertEquals(output, ConversionUtils.percentEncode(input)) { "ConversionUtils.percentEncode($input)" } } + + @ParameterizedTest + @MethodSource("htmlEntityValues") + fun htmlEntities(input: String, output: String, charset: Charset) { + assertEquals(output, ConversionUtils.encodeWithEntities(input, charset)) + } } diff --git a/xdocs/changes.xml b/xdocs/changes.xml index f4fa544022a..0d26e161dc1 100644 --- a/xdocs/changes.xml +++ b/xdocs/changes.xml @@ -73,6 +73,7 @@ Summary

HTTP Samplers and Test Script Recorder

    +
  • 6010Use UTF-8 as a default encoding in HTTP sampler. It enables sending parameter names, and filenames with unicode characters

Other samplers