diff --git a/src/core/build.gradle.kts b/src/core/build.gradle.kts index 98160b953a..78b3b26643 100644 --- a/src/core/build.gradle.kts +++ b/src/core/build.gradle.kts @@ -117,6 +117,7 @@ dependencies { isTransitive = false } implementation("org.apache.xmlgraphics:xmlgraphics-commons") + implementation("org.brotli:dec") implementation("org.freemarker:freemarker") implementation("org.jodd:jodd-core") implementation("org.jodd:jodd-props") diff --git a/src/core/src/main/java/org/apache/jmeter/samplers/SampleResult.java b/src/core/src/main/java/org/apache/jmeter/samplers/SampleResult.java index 6658916243..3b0a53f784 100644 --- a/src/core/src/main/java/org/apache/jmeter/samplers/SampleResult.java +++ b/src/core/src/main/java/org/apache/jmeter/samplers/SampleResult.java @@ -17,6 +17,10 @@ package org.apache.jmeter.samplers; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; import java.io.Serializable; import java.io.UnsupportedEncodingException; import java.net.HttpURLConnection; @@ -25,9 +29,13 @@ import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; +import java.util.Locale; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; +import java.util.zip.GZIPInputStream; +import java.util.zip.Inflater; +import java.util.zip.InflaterInputStream; import org.apache.jmeter.assertions.AssertionResult; import org.apache.jmeter.gui.Searchable; @@ -160,6 +168,8 @@ public class SampleResult implements Serializable, Cloneable, Searchable { private byte[] responseData = EMPTY_BA; + private String contentEncoding; // Stores gzip/deflate encoding if response is compressed + private String responseCode = "";// Never return null private String label = "";// Never return null @@ -217,7 +227,7 @@ public class SampleResult implements Serializable, Cloneable, Searchable { // TODO do contentType and/or dataEncoding belong in HTTPSampleResult instead? private String dataEncoding;// (is this really the character set?) e.g. - // ISO-8895-1, UTF-8 + // ISO-8895-1, UTF-8 private String contentType = ""; // e.g. text/html; charset=utf-8 @@ -791,6 +801,27 @@ public class SampleResult implements Serializable, Cloneable, Searchable { * @return the responseData value (cannot be null) */ public byte[] getResponseData() { + if (responseData == null) { + return EMPTY_BA; + } + if (contentEncoding != null && responseData.length > 0) { + try { + switch (contentEncoding.toLowerCase(Locale.ROOT)) { + case "gzip": + return decompressGzip(responseData); + case "x-gzip": + return decompressGzip(responseData); + case "deflate": + return decompressDeflate(responseData); + case "br": + return decompressBrotli(responseData); + default: + return responseData; + } + } catch (IOException e) { + log.warn("Failed to decompress response data", e); + } + } return responseData; } @@ -802,12 +833,12 @@ public class SampleResult implements Serializable, Cloneable, Searchable { public String getResponseDataAsString() { try { if(responseDataAsString == null) { - responseDataAsString= new String(responseData,getDataEncodingWithDefault()); + responseDataAsString= new String(getResponseData(),getDataEncodingWithDefault()); } return responseDataAsString; } catch (UnsupportedEncodingException e) { log.warn("Using platform default as {} caused {}", getDataEncodingWithDefault(), e.getLocalizedMessage()); - return new String(responseData,Charset.defaultCharset()); // N.B. default charset is used deliberately here + return new String(getResponseData(),Charset.defaultCharset()); // N.B. default charset is used deliberately here } } @@ -1665,4 +1696,63 @@ public class SampleResult implements Serializable, Cloneable, Searchable { public void setTestLogicalAction(TestLogicalAction testLogicalAction) { this.testLogicalAction = testLogicalAction; } + + /** + * Sets the response data and its compression encoding. + * @param data The response data + * @param encoding The content encoding (e.g. gzip, deflate) + */ + public void setResponseData(byte[] data, String encoding) { + responseData = data == null ? EMPTY_BA : data; + contentEncoding = encoding; + responseDataAsString = null; + } + + private static byte[] decompressGzip(byte[] in) throws IOException { + try (GZIPInputStream gis = new GZIPInputStream(new ByteArrayInputStream(in)); + ByteArrayOutputStream out = new ByteArrayOutputStream()) { + byte[] buf = new byte[8192]; + int len; + while ((len = gis.read(buf)) > 0) { + out.write(buf, 0, len); + } + return out.toByteArray(); + } + } + + private static byte[] decompressDeflate(byte[] in) throws IOException { + // Try with ZLIB wrapper first + try { + return decompressWithInflater(in, false); + } catch (IOException e) { + // If that fails, try with NO_WRAP for raw DEFLATE + return decompressWithInflater(in, true); + } + } + + private static byte[] decompressWithInflater(byte[] in, boolean nowrap) throws IOException { + try (InflaterInputStream iis = new InflaterInputStream( + new ByteArrayInputStream(in), + new Inflater(nowrap)); + ByteArrayOutputStream out = new ByteArrayOutputStream()) { + byte[] buf = new byte[8192]; + int len; + while ((len = iis.read(buf)) > 0) { + out.write(buf, 0, len); + } + return out.toByteArray(); + } + } + + private static byte[] decompressBrotli(byte[] in) throws IOException { + try (InputStream bis = new org.brotli.dec.BrotliInputStream(new ByteArrayInputStream(in)); + ByteArrayOutputStream out = new ByteArrayOutputStream()) { + byte[] buf = new byte[8192]; + int len; + while ((len = bis.read(buf)) > 0) { + out.write(buf, 0, len); + } + return out.toByteArray(); + } + } } diff --git a/src/protocol/http/build.gradle.kts b/src/protocol/http/build.gradle.kts index 2b1dd7f1ac..2dfe96039d 100644 --- a/src/protocol/http/build.gradle.kts +++ b/src/protocol/http/build.gradle.kts @@ -73,7 +73,6 @@ dependencies { implementation("dnsjava:dnsjava") implementation("org.apache.httpcomponents:httpmime") implementation("org.apache.httpcomponents:httpcore") - implementation("org.brotli:dec") implementation("com.miglayout:miglayout-swing") implementation("com.fasterxml.jackson.core:jackson-core") implementation("com.fasterxml.jackson.core:jackson-databind") diff --git a/src/protocol/http/src/main/java/org/apache/jmeter/protocol/http/sampler/HTTPHC4Impl.java b/src/protocol/http/src/main/java/org/apache/jmeter/protocol/http/sampler/HTTPHC4Impl.java index bc4478eecb..7cdf7f2482 100644 --- a/src/protocol/http/src/main/java/org/apache/jmeter/protocol/http/sampler/HTTPHC4Impl.java +++ b/src/protocol/http/src/main/java/org/apache/jmeter/protocol/http/sampler/HTTPHC4Impl.java @@ -57,7 +57,6 @@ import org.apache.http.HttpHost; import org.apache.http.HttpRequest; import org.apache.http.HttpRequestInterceptor; import org.apache.http.HttpResponse; -import org.apache.http.HttpResponseInterceptor; import org.apache.http.NameValuePair; import org.apache.http.StatusLine; import org.apache.http.auth.AuthSchemeProvider; @@ -72,7 +71,6 @@ import org.apache.http.client.CredentialsProvider; import org.apache.http.client.config.AuthSchemes; import org.apache.http.client.config.CookieSpecs; import org.apache.http.client.config.RequestConfig; -import org.apache.http.client.entity.InputStreamFactory; import org.apache.http.client.entity.UrlEncodedFormEntity; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpEntityEnclosingRequestBase; @@ -86,7 +84,6 @@ import org.apache.http.client.methods.HttpRequestBase; import org.apache.http.client.methods.HttpTrace; import org.apache.http.client.methods.HttpUriRequest; import org.apache.http.client.protocol.HttpClientContext; -import org.apache.http.client.protocol.ResponseContentEncoding; import org.apache.http.config.Lookup; import org.apache.http.config.Registry; import org.apache.http.config.RegistryBuilder; @@ -147,8 +144,6 @@ import org.apache.jmeter.protocol.http.control.CookieManager; import org.apache.jmeter.protocol.http.control.DynamicKerberosSchemeFactory; import org.apache.jmeter.protocol.http.control.DynamicSPNegoSchemeFactory; import org.apache.jmeter.protocol.http.control.HeaderManager; -import org.apache.jmeter.protocol.http.sampler.hc.LaxDeflateInputStream; -import org.apache.jmeter.protocol.http.sampler.hc.LaxGZIPInputStream; import org.apache.jmeter.protocol.http.sampler.hc.LazyLayeredConnectionSocketFactory; import org.apache.jmeter.protocol.http.util.ConversionUtils; import org.apache.jmeter.protocol.http.util.HTTPArgument; @@ -166,7 +161,6 @@ import org.apache.jmeter.util.JMeterUtils; import org.apache.jmeter.util.JsseSSLManager; import org.apache.jmeter.util.SSLManager; import org.apache.jorphan.util.JOrphanUtils; -import org.brotli.dec.BrotliInputStream; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -195,20 +189,8 @@ public class HTTPHC4Impl extends HTTPHCAbstractImpl { private static final boolean DISABLE_DEFAULT_UA = JMeterUtils.getPropDefault("httpclient4.default_user_agent_disabled", false); - private static final boolean GZIP_RELAX_MODE = JMeterUtils.getPropDefault("httpclient4.gzip_relax_mode", false); - - private static final boolean DEFLATE_RELAX_MODE = JMeterUtils.getPropDefault("httpclient4.deflate_relax_mode", false); - private static final Logger log = LoggerFactory.getLogger(HTTPHC4Impl.class); - private static final InputStreamFactory GZIP = - instream -> new LaxGZIPInputStream(instream, GZIP_RELAX_MODE); - - private static final InputStreamFactory DEFLATE = - instream -> new LaxDeflateInputStream(instream, DEFLATE_RELAX_MODE); - - private static final InputStreamFactory BROTLI = BrotliInputStream::new; - private static final class ManagedCredentialsProvider implements CredentialsProvider { private final AuthManager authManager; private final Credentials proxyCredentials; @@ -472,55 +454,6 @@ public class HTTPHC4Impl extends HTTPHCAbstractImpl { } }; - private static final String[] HEADERS_TO_SAVE = new String[]{ - "content-length", - "content-encoding", - "content-md5" - }; - - /** - * Custom implementation that backups headers related to Compressed responses - * that HC core {@link ResponseContentEncoding} removes after uncompressing - * See Bug 59401 - */ - @SuppressWarnings("UnnecessaryAnonymousClass") - private static final HttpResponseInterceptor RESPONSE_CONTENT_ENCODING = new ResponseContentEncoding(createLookupRegistry()) { - @Override - public void process(HttpResponse response, HttpContext context) - throws HttpException, IOException { - ArrayList headersToSave = null; - - final HttpEntity entity = response.getEntity(); - final HttpClientContext clientContext = HttpClientContext.adapt(context); - final RequestConfig requestConfig = clientContext.getRequestConfig(); - // store the headers if necessary - if (requestConfig.isContentCompressionEnabled() && entity != null && entity.getContentLength() != 0) { - final Header ceheader = entity.getContentEncoding(); - if (ceheader != null) { - headersToSave = new ArrayList<>(3); - for(String name : HEADERS_TO_SAVE) { - Header[] hdr = response.getHeaders(name); // empty if none - headersToSave.add(hdr); - } - } - } - - // Now invoke original parent code - super.process(response, clientContext); - // Should this be in a finally ? - if(headersToSave != null) { - for (Header[] headers : headersToSave) { - for (Header headerToRestore : headers) { - if (response.containsHeader(headerToRestore.getName())) { - break; - } - response.addHeader(headerToRestore); - } - } - } - } - }; - /** * 1 HttpClient instance per combination of (HttpClient,HttpClientKey) */ @@ -558,19 +491,6 @@ public class HTTPHC4Impl extends HTTPHCAbstractImpl { super(testElement); } - /** - * Customize to plug Brotli - * @return {@link Lookup} - */ - private static Lookup createLookupRegistry() { - return - RegistryBuilder.create() - .register("br", BROTLI) - .register("gzip", GZIP) - .register("x-gzip", GZIP) - .register("deflate", DEFLATE).build(); - } - /** * Implementation that allows GET method to have a body */ @@ -675,7 +595,12 @@ public class HTTPHC4Impl extends HTTPHCAbstractImpl { } HttpEntity entity = httpResponse.getEntity(); if (entity != null) { - res.setResponseData(readResponse(res, entity.getContent(), entity.getContentLength())); + Header contentEncodingHeader = entity.getContentEncoding(); + if (contentEncodingHeader != null) { + res.setResponseData(EntityUtils.toByteArray(entity), contentEncodingHeader.getValue()); + } else { + res.setResponseData(EntityUtils.toByteArray(entity)); + } } res.sampleEnd(); // Done with the sampling proper. @@ -1157,7 +1082,7 @@ public class HTTPHC4Impl extends HTTPHCAbstractImpl { } builder.setDefaultCredentialsProvider(credsProvider); } - builder.disableContentCompression().addInterceptorLast(RESPONSE_CONTENT_ENCODING); + builder.disableContentCompression(); // Disable automatic decompression if(BASIC_AUTH_PREEMPTIVE) { builder.addInterceptorFirst(PREEMPTIVE_AUTH_INTERCEPTOR); } else { diff --git a/src/protocol/http/src/main/java/org/apache/jmeter/protocol/http/sampler/HTTPJavaImpl.java b/src/protocol/http/src/main/java/org/apache/jmeter/protocol/http/sampler/HTTPJavaImpl.java index da3a7ebc8c..841bc897bc 100644 --- a/src/protocol/http/src/main/java/org/apache/jmeter/protocol/http/sampler/HTTPJavaImpl.java +++ b/src/protocol/http/src/main/java/org/apache/jmeter/protocol/http/sampler/HTTPJavaImpl.java @@ -31,7 +31,6 @@ import java.util.Collections; import java.util.List; import java.util.Map; import java.util.function.Predicate; -import java.util.zip.GZIPInputStream; import org.apache.commons.io.input.CountingInputStream; import org.apache.jmeter.protocol.http.control.AuthManager; @@ -240,15 +239,11 @@ public class HTTPJavaImpl extends HTTPAbstractImpl { } // works OK even if ContentEncoding is null - boolean gzipped = HTTPConstants.ENCODING_GZIP.equals(conn.getContentEncoding()); + String contentEncoding = conn.getContentEncoding(); CountingInputStream instream = null; try { instream = new CountingInputStream(conn.getInputStream()); - if (gzipped) { - in = new GZIPInputStream(instream); - } else { - in = instream; - } + in = instream; } catch (IOException e) { if (! (e.getCause() instanceof FileNotFoundException)) { @@ -276,21 +271,7 @@ public class HTTPJavaImpl extends HTTPAbstractImpl { log.info("Error Response Code: {}", conn.getResponseCode()); } - if (gzipped) { - in = new GZIPInputStream(errorStream); - } else { - in = errorStream; - } - } catch (Exception e) { - log.error("readResponse: {}", e.toString()); - Throwable cause = e.getCause(); - if (cause != null){ - log.error("Cause: {}", cause.toString()); - if(cause instanceof Error) { - throw (Error)cause; - } - } - in = conn.getErrorStream(); + in = errorStream; } // N.B. this closes 'in' byte[] responseData = readResponse(res, in, contentLength); @@ -298,6 +279,7 @@ public class HTTPJavaImpl extends HTTPAbstractImpl { res.setBodySize(instream.getByteCount()); instream.close(); } + res.setResponseData(responseData, contentEncoding); return responseData; }