diff --git a/httpclient5-testing/src/test/java/org/apache/hc/client5/testing/sync/TestRedirects.java b/httpclient5-testing/src/test/java/org/apache/hc/client5/testing/sync/TestRedirects.java index 8b67437cad..25c9e7f413 100644 --- a/httpclient5-testing/src/test/java/org/apache/hc/client5/testing/sync/TestRedirects.java +++ b/httpclient5-testing/src/test/java/org/apache/hc/client5/testing/sync/TestRedirects.java @@ -646,8 +646,8 @@ public void handle(final ClassicHttpRequest request, Assertions.assertEquals(new URIBuilder().setHttpHost(target).setPath("/random/100").build(), reqWrapper.getUri()); - assertThat(values.poll(), CoreMatchers.equalTo("gzip, x-gzip, deflate")); - assertThat(values.poll(), CoreMatchers.equalTo("gzip, x-gzip, deflate")); + assertThat(values.poll(), CoreMatchers.equalTo("gzip, deflate, lz4-framed, lz4-block, bzip2, pack200, deflate64, x-gzip")); + assertThat(values.poll(), CoreMatchers.equalTo("gzip, deflate, lz4-framed, lz4-block, bzip2, pack200, deflate64, x-gzip")); assertThat(values.poll(), CoreMatchers.nullValue()); } diff --git a/httpclient5/pom.xml b/httpclient5/pom.xml index 47777876b0..b66f028f0b 100644 --- a/httpclient5/pom.xml +++ b/httpclient5/pom.xml @@ -108,6 +108,11 @@ commons-io test + + org.apache.commons + commons-compress + true + diff --git a/httpclient5/src/main/java/org/apache/hc/client5/http/entity/BrotliInputStreamFactory.java b/httpclient5/src/main/java/org/apache/hc/client5/http/entity/BrotliInputStreamFactory.java index b5eac4fecd..68abf0aedc 100644 --- a/httpclient5/src/main/java/org/apache/hc/client5/http/entity/BrotliInputStreamFactory.java +++ b/httpclient5/src/main/java/org/apache/hc/client5/http/entity/BrotliInputStreamFactory.java @@ -41,6 +41,17 @@ @Contract(threading = ThreadingBehavior.STATELESS) public class BrotliInputStreamFactory implements InputStreamFactory { + /** + * Canonical token for the deflate content-coding. + * @since 5.6 + */ + public static final String ENCODING = "br"; + + @Override + public String getContentEncoding() { + return ENCODING; + } + /** * Default instance of {@link BrotliInputStreamFactory}. */ diff --git a/httpclient5/src/main/java/org/apache/hc/client5/http/entity/DeflateInputStreamFactory.java b/httpclient5/src/main/java/org/apache/hc/client5/http/entity/DeflateInputStreamFactory.java index cfd113a762..ab1a285ab2 100644 --- a/httpclient5/src/main/java/org/apache/hc/client5/http/entity/DeflateInputStreamFactory.java +++ b/httpclient5/src/main/java/org/apache/hc/client5/http/entity/DeflateInputStreamFactory.java @@ -41,6 +41,17 @@ @Contract(threading = ThreadingBehavior.STATELESS) public class DeflateInputStreamFactory implements InputStreamFactory { + /** + * Canonical token for the deflate content-coding. + * @since 5.6 + */ + public static final String ENCODING = "deflate"; + + @Override + public String getContentEncoding() { + return ENCODING; + } + /** * Default instance of {@link DeflateInputStreamFactory}. */ diff --git a/httpclient5/src/main/java/org/apache/hc/client5/http/entity/GZIPInputStreamFactory.java b/httpclient5/src/main/java/org/apache/hc/client5/http/entity/GZIPInputStreamFactory.java index a03e20dd9f..f59712afe0 100644 --- a/httpclient5/src/main/java/org/apache/hc/client5/http/entity/GZIPInputStreamFactory.java +++ b/httpclient5/src/main/java/org/apache/hc/client5/http/entity/GZIPInputStreamFactory.java @@ -42,6 +42,17 @@ @Contract(threading = ThreadingBehavior.STATELESS) public class GZIPInputStreamFactory implements InputStreamFactory { + /** + * Canonical token for the gzip content-coding. + * @since 5.6 + */ + public static final String ENCODING = "gzip"; + + @Override + public String getContentEncoding() { + return ENCODING; + } + /** * Default instance of {@link GZIPInputStreamFactory}. */ diff --git a/httpclient5/src/main/java/org/apache/hc/client5/http/entity/InputStreamFactory.java b/httpclient5/src/main/java/org/apache/hc/client5/http/entity/InputStreamFactory.java index a6689435ef..b2664648e3 100644 --- a/httpclient5/src/main/java/org/apache/hc/client5/http/entity/InputStreamFactory.java +++ b/httpclient5/src/main/java/org/apache/hc/client5/http/entity/InputStreamFactory.java @@ -38,4 +38,21 @@ public interface InputStreamFactory { InputStream create(InputStream inputStream) throws IOException; + /** + * Returns the canonical {@code Content-Encoding} token handled by this + * factory (for example {@code "gzip"}, {@code "deflate"}, {@code "br"}). + *

+ * Implementations that do not represent a HTTP + * content-decoder should simply inherit the default implementation, + * which returns an empty string. + * + * @return the lower-case encoding token, or an empty string when the + * factory is not intended for HTTP content-decoding + * + * @since 5.6 + */ + default String getContentEncoding() { + return ""; + } + } diff --git a/httpclient5/src/main/java/org/apache/hc/client5/http/entity/compress/CommonsCompressDecoderFactory.java b/httpclient5/src/main/java/org/apache/hc/client5/http/entity/compress/CommonsCompressDecoderFactory.java new file mode 100644 index 0000000000..412ba414b0 --- /dev/null +++ b/httpclient5/src/main/java/org/apache/hc/client5/http/entity/compress/CommonsCompressDecoderFactory.java @@ -0,0 +1,117 @@ +/* + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + * + * This software consists of voluntary contributions made by many + * individuals on behalf of the Apache Software Foundation. For more + * information on the Apache Software Foundation, please see + * . + * + */ + +package org.apache.hc.client5.http.entity.compress; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Collections; +import java.util.EnumMap; +import java.util.Locale; +import java.util.Map; + +import org.apache.commons.compress.compressors.CompressorException; +import org.apache.commons.compress.compressors.CompressorStreamFactory; +import org.apache.hc.client5.http.entity.InputStreamFactory; +import org.apache.hc.core5.annotation.Contract; +import org.apache.hc.core5.annotation.Internal; +import org.apache.hc.core5.annotation.ThreadingBehavior; + +/** + * A factory for creating InputStream instances, utilizing Apache Commons Compress. + * This class is compiled with Commons Compress as an optional dependency, loading + * only when the library is present at runtime, avoiding mandatory inclusion in + * downstream builds. + *

+ *

+ * Some encodings require native helper JARs; runtime availability is checked + * using a lightweight Class.forName probe to register codecs only when helpers + * are present. + * + * @since 5.6 + */ +@Internal +@Contract(threading = ThreadingBehavior.STATELESS) +final class CommonsCompressDecoderFactory implements InputStreamFactory { + + + /** + * Map of codings that need extra JARs → the fully‐qualified class we test for + */ + private static final Map REQUIRED_CLASS_NAME; + + static { + final Map m = new EnumMap<>(ContentCoding.class); + m.put(ContentCoding.BROTLI, "org.brotli.dec.BrotliInputStream"); + m.put(ContentCoding.ZSTD, "com.github.luben.zstd.ZstdInputStream"); + m.put(ContentCoding.XZ, "org.tukaani.xz.XZInputStream"); + m.put(ContentCoding.LZMA, "org.tukaani.xz.XZInputStream"); + REQUIRED_CLASS_NAME = Collections.unmodifiableMap(m); + } + + private final String encoding; + + CommonsCompressDecoderFactory(final String encoding) { + this.encoding = encoding.toLowerCase(Locale.ROOT); + } + + @Override + public String getContentEncoding() { + return encoding; + } + + @Override + public InputStream create(final InputStream source) throws IOException { + try { + return new CompressorStreamFactory() + .createCompressorInputStream(encoding, source); + } catch (final CompressorException | LinkageError ex) { + throw new IOException( + "Unable to decode Content-Encoding '" + encoding + '\'', ex); + } + } + + + static boolean runtimeAvailable(final String token) { + final ContentCoding coding = ContentCoding.fromToken(token); + if (coding == null) { + return true; + } + final String helper = REQUIRED_CLASS_NAME.get(coding); + if (helper == null) { + // no extra JAR needed + return true; + } + try { + Class.forName(helper, false, + CommonsCompressDecoderFactory.class.getClassLoader()); + return true; + } catch (final ClassNotFoundException | LinkageError ex) { + return false; + } + } +} \ No newline at end of file diff --git a/httpclient5/src/main/java/org/apache/hc/client5/http/entity/compress/ContentCoding.java b/httpclient5/src/main/java/org/apache/hc/client5/http/entity/compress/ContentCoding.java new file mode 100644 index 0000000000..3841a4ed01 --- /dev/null +++ b/httpclient5/src/main/java/org/apache/hc/client5/http/entity/compress/ContentCoding.java @@ -0,0 +1,136 @@ +/* + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + * + * This software consists of voluntary contributions made by many + * individuals on behalf of the Apache Software Foundation. For more + * information on the Apache Software Foundation, please see + * . + * + */ + +package org.apache.hc.client5.http.entity.compress; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; + +/** + * Enumeration of the canonical IANA content-coding tokens supported by HttpClient for + * HTTP request and response bodies. + *

+ * Each constant corresponds to the standard token used in the {@code Content-Encoding} + * and {@code Accept-Encoding} headers. Some codings (e.g. Brotli, Zstandard, XZ/LZMA) + * may require additional helper libraries at runtime. + * + * @since 5.6 + */ +public enum ContentCoding { + + /** + * GZIP compression format. + */ + GZIP("gzip"), + /** + * "deflate" compression format (zlib or raw). + */ + DEFLATE("deflate"), + /** + * Legacy alias for GZIP. + */ + X_GZIP("x-gzip"), + + // Optional codecs requiring Commons-Compress or native helpers + /** + * Brotli compression format. + */ + BROTLI("br"), + /** + * Zstandard compression format. + */ + ZSTD("zstd"), + /** + * XZ compression format. + */ + XZ("xz"), + /** + * LZMA compression format. + */ + LZMA("lzma"), + /** + * Framed LZ4 compression format. + */ + LZ4_FRAMED("lz4-framed"), + /** + * Block LZ4 compression format. + */ + LZ4_BLOCK("lz4-block"), + /** + * BZIP2 compression format. + */ + BZIP2("bzip2"), + /** + * Pack200 compression format. + */ + PACK200("pack200"), + /** + * Deflate64 compression format. + */ + DEFLATE64("deflate64"); + + private static final Map TOKEN_LOOKUP; + static { + final Map map = new HashMap<>(values().length, 1f); + for (final ContentCoding contentCoding : values()) { + map.put(contentCoding.token, contentCoding); + } + TOKEN_LOOKUP = Collections.unmodifiableMap(map); + } + + private final String token; + + ContentCoding(final String token) { + this.token = token; + } + + /** + * Returns the standard IANA token string for this content-coding. + * + * @return the lowercase token used in HTTP headers + */ + public String token() { + return token; + } + + /** + * Lookup an enum by its token (case‐insensitive), or {@code null} if none matches. + *

+ * This method is backed by a static, pre‐populated map so the lookup is O(1) + * instead of O(n).

+ * + * @param token the content‐coding token to look up + * @return the matching enum constant, or {@code null} if none + */ + public static ContentCoding fromToken(final String token) { + return TOKEN_LOOKUP.get( + token == null ? null : token.toLowerCase(Locale.ROOT) + ); + } +} diff --git a/httpclient5/src/main/java/org/apache/hc/client5/http/entity/compress/ContentDecoderRegistry.java b/httpclient5/src/main/java/org/apache/hc/client5/http/entity/compress/ContentDecoderRegistry.java new file mode 100644 index 0000000000..a019f705e6 --- /dev/null +++ b/httpclient5/src/main/java/org/apache/hc/client5/http/entity/compress/ContentDecoderRegistry.java @@ -0,0 +1,144 @@ +/* + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + * + * This software consists of voluntary contributions made by many + * individuals on behalf of the Apache Software Foundation. For more + * information on the Apache Software Foundation, please see + * . + * + */ + +package org.apache.hc.client5.http.entity.compress; + +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; + +import org.apache.hc.client5.http.entity.BrotliDecompressingEntity; +import org.apache.hc.client5.http.entity.BrotliInputStreamFactory; +import org.apache.hc.client5.http.entity.DeflateInputStreamFactory; +import org.apache.hc.client5.http.entity.GZIPInputStreamFactory; +import org.apache.hc.client5.http.entity.InputStreamFactory; +import org.apache.hc.core5.annotation.Contract; +import org.apache.hc.core5.annotation.Internal; +import org.apache.hc.core5.annotation.ThreadingBehavior; + +/** + * Immutable run-time catalogue of {@link InputStreamFactory} instances + * capable of decoding HTTP entity bodies. + * + *

The map is populated once during class initialisation:

+ *
    + *
  1. Built-ins: {@code gzip} and {@code deflate} are always present.
    2. + *
  2. If Commons-Compress is on the class-path we register a configurable + * list of codecs (br, zstd, xz, …) via + * {@link CommonsCompressDecoderFactory} – guarded by a cheap + * presence check.
    3. + *
  3. If Commons was absent or could not supply br, + * we fall back to the pure native singleton + * {@link BrotliInputStreamFactory} (when the org.brotli + * decoder JAR is available).
    4. + *
+ * + *

The resulting {@code Map} is wrapped in + * {@link Collections#unmodifiableMap(Map)} and published through + * {@link #getRegistry()} for safe, lock-free concurrent reads.

+ * + * @since 5.6 + */ +@Internal +@Contract(threading = ThreadingBehavior.STATELESS) +public final class ContentDecoderRegistry { + + private static final String CCSF = + "org.apache.commons.compress.compressors.CompressorStreamFactory"; + + + private static final Map REGISTRY = buildRegistry(); + + + /** + * Returns the unmodifiable codec map (key = canonical token, value = factory). + */ + public static Map getRegistry() { + return REGISTRY; + } + + + private static Map buildRegistry() { + final LinkedHashMap m = new LinkedHashMap<>(); + + // 1. Built-ins + register(m, ContentCoding.GZIP, new GZIPInputStreamFactory()); + register(m, ContentCoding.DEFLATE, new DeflateInputStreamFactory()); + + // 2. Commons-Compress (optional) + if (commonsCompressPresent()) { + for (final ContentCoding coding : Arrays.asList( + ContentCoding.BROTLI, + ContentCoding.ZSTD, + ContentCoding.XZ, + ContentCoding.LZMA, + ContentCoding.LZ4_FRAMED, + ContentCoding.LZ4_BLOCK, + ContentCoding.BZIP2, + ContentCoding.PACK200, + ContentCoding.DEFLATE64)) { + addCommons(m, coding); + } + } + + // 3. Native Brotli fallback if Commons did not register it + if (!m.containsKey(ContentCoding.BROTLI) + && BrotliDecompressingEntity.isAvailable()) { + register(m, ContentCoding.BROTLI, new BrotliInputStreamFactory()); + } + + return Collections.unmodifiableMap(m); + } + + private static void register(final Map map, + final ContentCoding coding, + final InputStreamFactory factory) { + map.put(coding, factory); + } + + private static void addCommons(final Map map, + final ContentCoding coding) { + if (CommonsCompressDecoderFactory.runtimeAvailable(coding.token())) { + register(map, coding, new CommonsCompressDecoderFactory(coding.token())); + } + } + + private static boolean commonsCompressPresent() { + try { + Class.forName( + CCSF, false, ContentDecoderRegistry.class.getClassLoader()); + return true; + } catch (final ClassNotFoundException | LinkageError ex) { + return false; + } + } + + private ContentDecoderRegistry() { + } +} + diff --git a/httpclient5/src/main/java/org/apache/hc/client5/http/impl/classic/ContentCompressionExec.java b/httpclient5/src/main/java/org/apache/hc/client5/http/impl/classic/ContentCompressionExec.java index ac3cd5a559..4b9d7289d4 100644 --- a/httpclient5/src/main/java/org/apache/hc/client5/http/impl/classic/ContentCompressionExec.java +++ b/httpclient5/src/main/java/org/apache/hc/client5/http/impl/classic/ContentCompressionExec.java @@ -29,20 +29,20 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Locale; +import java.util.Map; import java.util.zip.GZIPInputStream; import org.apache.hc.client5.http.classic.ExecChain; import org.apache.hc.client5.http.classic.ExecChainHandler; import org.apache.hc.client5.http.config.RequestConfig; -import org.apache.hc.client5.http.entity.BrotliDecompressingEntity; -import org.apache.hc.client5.http.entity.BrotliInputStreamFactory; import org.apache.hc.client5.http.entity.DecompressingEntity; import org.apache.hc.client5.http.entity.DeflateInputStream; -import org.apache.hc.client5.http.entity.DeflateInputStreamFactory; -import org.apache.hc.client5.http.entity.GZIPInputStreamFactory; import org.apache.hc.client5.http.entity.InputStreamFactory; +import org.apache.hc.client5.http.entity.compress.ContentCoding; +import org.apache.hc.client5.http.entity.compress.ContentDecoderRegistry; import org.apache.hc.client5.http.protocol.HttpClientContext; import org.apache.hc.core5.annotation.Contract; import org.apache.hc.core5.annotation.Internal; @@ -81,36 +81,47 @@ public final class ContentCompressionExec implements ExecChainHandler { private final Lookup decoderRegistry; private final boolean ignoreUnknown; + private static final Map DECODERS = ContentDecoderRegistry.getRegistry(); + + /** + * Pre-built list of all supported tokens (plus X-GZIP alias) for + * the Accept-Encoding header, to avoid reconstructing it every time. + */ + private static final List DEFAULT_ACCEPT_ENCODINGS; + static { + final List tmp = new ArrayList<>(DECODERS.size() + 1); + for (final ContentCoding coding : DECODERS.keySet()) { + tmp.add(coding.token()); + } + // add x-gzip alias if gzip is present + if (DECODERS.containsKey(ContentCoding.GZIP)) { + tmp.add(ContentCoding.X_GZIP.token()); + } + DEFAULT_ACCEPT_ENCODINGS = Collections.unmodifiableList(tmp); + } + public ContentCompressionExec( final List acceptEncoding, final Lookup decoderRegistry, final boolean ignoreUnknown) { - final boolean brotliSupported = decoderRegistry == null && BrotliDecompressingEntity.isAvailable(); - if (acceptEncoding != null) { - this.acceptEncoding = MessageSupport.headerOfTokens(HttpHeaders.ACCEPT_ENCODING, acceptEncoding); - } else { - final List encodings = new ArrayList<>(4); - encodings.add("gzip"); - encodings.add("x-gzip"); - encodings.add("deflate"); - if (brotliSupported) { - encodings.add("br"); - } - this.acceptEncoding = MessageSupport.headerOfTokens(HttpHeaders.ACCEPT_ENCODING, encodings); - } + final List encodingsHeader = acceptEncoding != null ? acceptEncoding : DEFAULT_ACCEPT_ENCODINGS; + + this.acceptEncoding = MessageSupport.headerOfTokens(HttpHeaders.ACCEPT_ENCODING, encodingsHeader); + if (decoderRegistry != null) { this.decoderRegistry = decoderRegistry; } else { - final RegistryBuilder builder = RegistryBuilder.create() - .register("gzip", GZIPInputStreamFactory.getInstance()) - .register("x-gzip", GZIPInputStreamFactory.getInstance()) - .register("deflate", DeflateInputStreamFactory.getInstance()); - if (brotliSupported) { - builder.register("br", BrotliInputStreamFactory.getInstance()); + final RegistryBuilder builder = RegistryBuilder.create(); + DECODERS.forEach((coding, factory) -> + builder.register(coding.token(), factory)); + // register the x-gzip alias again + if (DECODERS.containsKey(ContentCoding.GZIP)) { + builder.register(ContentCoding.X_GZIP.token(), DECODERS.get(ContentCoding.GZIP)); } this.decoderRegistry = builder.build(); } + this.ignoreUnknown = ignoreUnknown; } @@ -176,5 +187,4 @@ public ClassicHttpResponse execute( } return response; } - } diff --git a/pom.xml b/pom.xml index 5fd9b34422..7556ea0107 100644 --- a/pom.xml +++ b/pom.xml @@ -78,6 +78,7 @@ 2.10.1 5.3 javax.net.ssl.SSLEngine,javax.net.ssl.SSLParameters,java.nio.ByteBuffer,java.nio.CharBuffer + 1.27.1 @@ -204,6 +205,12 @@ commons-io 2.19.0 + + org.apache.commons + commons-compress + ${commons.compress.version} + +