From cf54f606836c43fb83b53e7b00fe770ca63819ad Mon Sep 17 00:00:00 2001 From: Arturo Bernal Date: Mon, 13 Oct 2025 08:44:21 +0200 Subject: [PATCH 1/2] Use reflection for Commons Compress codecs to avoid hard dep. (#737) Prevent EIIE when commons-compress is absent; wrap failures as IOException. Fix runtimeAvailable() to probe CC classes; decouple from helper JARs. --- httpclient5/pom.xml | 6 +- .../http/entity/BrotliInputStreamFactory.java | 3 +- .../compress/CommonsCompressCodecFactory.java | 177 ++++++++++++++++++ .../CommonsCompressDecoderFactory.java | 111 ----------- .../compress/CommonsCompressSupport.java | 16 +- ...singEntity.java => CompressingEntity.java} | 59 +++--- .../entity/compress/ContentCodecRegistry.java | 94 ++++++---- pom.xml | 5 - 8 files changed, 284 insertions(+), 187 deletions(-) create mode 100644 httpclient5/src/main/java/org/apache/hc/client5/http/entity/compress/CommonsCompressCodecFactory.java delete mode 100644 httpclient5/src/main/java/org/apache/hc/client5/http/entity/compress/CommonsCompressDecoderFactory.java rename httpclient5/src/main/java/org/apache/hc/client5/http/entity/compress/{CommonsCompressingEntity.java => CompressingEntity.java} (56%) diff --git a/httpclient5/pom.xml b/httpclient5/pom.xml index 4478777448..4003210b9d 100644 --- a/httpclient5/pom.xml +++ b/httpclient5/pom.xml @@ -77,11 +77,6 @@ log4j-core test - - org.brotli - dec - true - com.kohlschutter.junixsocket junixsocket-core @@ -121,6 +116,7 @@ com.aayushatharva.brotli4j brotli4j + true diff --git a/httpclient5/src/main/java/org/apache/hc/client5/http/entity/BrotliInputStreamFactory.java b/httpclient5/src/main/java/org/apache/hc/client5/http/entity/BrotliInputStreamFactory.java index 369f914fc5..2b06d30d90 100644 --- a/httpclient5/src/main/java/org/apache/hc/client5/http/entity/BrotliInputStreamFactory.java +++ b/httpclient5/src/main/java/org/apache/hc/client5/http/entity/BrotliInputStreamFactory.java @@ -29,9 +29,10 @@ import java.io.IOException; import java.io.InputStream; +import com.aayushatharva.brotli4j.decoder.BrotliInputStream; + import org.apache.hc.core5.annotation.Contract; import org.apache.hc.core5.annotation.ThreadingBehavior; -import org.brotli.dec.BrotliInputStream; /** * {@link InputStreamFactory} for handling Brotli Content Coded responses. diff --git a/httpclient5/src/main/java/org/apache/hc/client5/http/entity/compress/CommonsCompressCodecFactory.java b/httpclient5/src/main/java/org/apache/hc/client5/http/entity/compress/CommonsCompressCodecFactory.java new file mode 100644 index 0000000000..6d47f0a2f6 --- /dev/null +++ b/httpclient5/src/main/java/org/apache/hc/client5/http/entity/compress/CommonsCompressCodecFactory.java @@ -0,0 +1,177 @@ +/* + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + * + * This software consists of voluntary contributions made by many + * individuals on behalf of the Apache Software Foundation. For more + * information on the Apache Software Foundation, please see + * . + * + */ + +package org.apache.hc.client5.http.entity.compress; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.lang.reflect.Method; +import java.util.Locale; + +import org.apache.hc.core5.annotation.Contract; +import org.apache.hc.core5.annotation.Internal; +import org.apache.hc.core5.annotation.ThreadingBehavior; +import org.apache.hc.core5.io.IOFunction; + +/** + * Reflection-based codecs factory for Apache Commons Compress. + *

+ * This class never links Commons Compress at compile time. At runtime, it + * reflectively locates {@code CompressorStreamFactory} and creates encoder / + * decoder streams for IANA tokens (e.g. {@code "zstd"}, {@code "xz"}) only + * when the relevant classes (and any helper JARs) are present. + *

+ *

+ * Use {@link #runtimeAvailable(ContentCoding)} to probe whether a given coding + * can be provided by the current classpath configuration. Callers can then + * register codecs conditionally without hard dependencies. + *

+ * + * @since 5.6 + */ +@Internal +@Contract(threading = ThreadingBehavior.STATELESS) +final class CommonsCompressCodecFactory { + + private static final String FACTORY_CLASS = + "org.apache.commons.compress.compressors.CompressorStreamFactory"; + + // CC stream classes + private static final String CC_BROTLI = "org.apache.commons.compress.compressors.brotli.BrotliCompressorInputStream"; + private static final String CC_ZSTD = "org.apache.commons.compress.compressors.zstandard.ZstdCompressorInputStream"; + private static final String CC_XZ = "org.apache.commons.compress.compressors.xz.XZCompressorInputStream"; + private static final String CC_LZMA = "org.apache.commons.compress.compressors.lzma.LZMACompressorInputStream"; + private static final String CC_LZ4_F = "org.apache.commons.compress.compressors.lz4.FramedLZ4CompressorInputStream"; + private static final String CC_LZ4_B = "org.apache.commons.compress.compressors.lz4.BlockLZ4CompressorInputStream"; + private static final String CC_BZIP2 = "org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream"; + private static final String CC_PACK200 = "org.apache.commons.compress.compressors.pack200.Pack200CompressorInputStream"; + private static final String CC_DEFLATE64 = "org.apache.commons.compress.compressors.deflate64.Deflate64CompressorInputStream"; + + // Helper libs + private static final String H_BROTLI = "org.brotli.dec.BrotliInputStream"; + private static final String H_ZSTD = "com.github.luben.zstd.ZstdInputStream"; + private static final String H_XZ = "org.tukaani.xz.XZInputStream"; + + private CommonsCompressCodecFactory() { + } + + private static boolean isPresent(final String className) { + try { + Class.forName(className, false, CommonsCompressCodecFactory.class.getClassLoader()); + return true; + } catch (final ClassNotFoundException | LinkageError ex) { + return false; + } + } + + /** + * Creates a lazy decoder that instantiates the Commons Compress stream + * reflectively on first use. Throws {@link IOException} if Commons Compress + * is not available or the codec cannot be created. + */ + static IOFunction decoder(final String token) { + final String enc = token.toLowerCase(Locale.ROOT); + return in -> { + try { + final ClassLoader cl = CommonsCompressCodecFactory.class.getClassLoader(); + final Class factoryCls = Class.forName(FACTORY_CLASS, false, cl); + final Object factory = factoryCls.getConstructor().newInstance(); + final Method m = factoryCls.getMethod("createCompressorInputStream", String.class, InputStream.class); + final Object stream = m.invoke(factory, enc, in); + return (InputStream) stream; + } catch (final ClassNotFoundException e) { + throw new IOException("Apache Commons Compress is not on the classpath", e); + } catch (final ReflectiveOperationException | IllegalArgumentException | LinkageError e) { + throw new IOException("Unable to decode Content-Encoding '" + enc + '\'', e); + } + }; + } + + /** + * Creates a lazy encoder that instantiates the Commons Compress stream + * reflectively on first use. Throws {@link IOException} if Commons Compress + * is not available or the codec cannot be created. + */ + static IOFunction encoder(final String token) { + final String enc = token.toLowerCase(Locale.ROOT); + return out -> { + try { + final ClassLoader cl = CommonsCompressCodecFactory.class.getClassLoader(); + final Class factoryCls = Class.forName(FACTORY_CLASS, false, cl); + final Object factory = factoryCls.getConstructor().newInstance(); + final Method m = factoryCls.getMethod("createCompressorOutputStream", String.class, OutputStream.class); + final Object cos = m.invoke(factory, enc, out); + return (OutputStream) cos; + } catch (final ClassNotFoundException e) { + throw new IOException("Apache Commons Compress is not on the classpath", e); + } catch (final ReflectiveOperationException | IllegalArgumentException | LinkageError e) { + throw new IOException("Unable to encode using Content-Encoding '" + enc + '\'', e); + } + }; + } + + /** + * Best-effort availability probe for optional Commons-Compress codecs. + *

+ * Returns {@code true} only if the CC factory and the codec-specific + * implementation (and helper classes if required) are present on the + * classpath. Built-in gzip/deflate are handled elsewhere and are not + * probed here. + *

+ */ + static boolean runtimeAvailable(final ContentCoding coding) { + if (coding == null) { + return false; + } + if (!isPresent(FACTORY_CLASS)) { + return false; + } + switch (coding) { + case BROTLI: + return isPresent(CC_BROTLI) && isPresent(H_BROTLI); + case ZSTD: + return isPresent(CC_ZSTD) && isPresent(H_ZSTD); + case XZ: + return isPresent(CC_XZ) && isPresent(H_XZ); + case LZMA: + return isPresent(CC_LZMA) && isPresent(H_XZ); + case LZ4_FRAMED: + return isPresent(CC_LZ4_F); + case LZ4_BLOCK: + return isPresent(CC_LZ4_B); + case BZIP2: + return isPresent(CC_BZIP2); + case PACK200: + return isPresent(CC_PACK200) || isPresent("java.util.jar.Pack200"); + case DEFLATE64: + return isPresent(CC_DEFLATE64); + default: + return false; + } + } +} diff --git a/httpclient5/src/main/java/org/apache/hc/client5/http/entity/compress/CommonsCompressDecoderFactory.java b/httpclient5/src/main/java/org/apache/hc/client5/http/entity/compress/CommonsCompressDecoderFactory.java deleted file mode 100644 index 9bc7d77708..0000000000 --- a/httpclient5/src/main/java/org/apache/hc/client5/http/entity/compress/CommonsCompressDecoderFactory.java +++ /dev/null @@ -1,111 +0,0 @@ -/* - * ==================================================================== - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * ==================================================================== - * - * This software consists of voluntary contributions made by many - * individuals on behalf of the Apache Software Foundation. For more - * information on the Apache Software Foundation, please see - * . - * - */ - -package org.apache.hc.client5.http.entity.compress; - -import java.io.IOException; -import java.io.InputStream; -import java.util.Collections; -import java.util.EnumMap; -import java.util.Locale; -import java.util.Map; - -import org.apache.commons.compress.compressors.CompressorException; -import org.apache.commons.compress.compressors.CompressorStreamFactory; -import org.apache.hc.core5.annotation.Contract; -import org.apache.hc.core5.annotation.Internal; -import org.apache.hc.core5.annotation.ThreadingBehavior; -import org.apache.hc.core5.io.IOFunction; - -/** - * A factory for creating InputStream instances, utilizing Apache Commons Compress. - * This class is compiled with Commons Compress as an optional dependency, loading - * only when the library is present at runtime, avoiding mandatory inclusion in - * downstream builds. - *

- *

- * Some encodings require native helper JARs; runtime availability is checked - * using a lightweight Class.forName probe to register codecs only when helpers - * are present. - * - * @since 5.6 - */ -@Internal -@Contract(threading = ThreadingBehavior.STATELESS) -final class CommonsCompressDecoderFactory { - - /** - * Map of codings that need extra JARs → the fully‐qualified class we test for - */ - private static final Map REQUIRED_CLASS_NAME; - - static { - final Map m = new EnumMap<>(ContentCoding.class); - m.put(ContentCoding.BROTLI, "org.brotli.dec.BrotliInputStream"); - m.put(ContentCoding.ZSTD, "com.github.luben.zstd.ZstdInputStream"); - m.put(ContentCoding.XZ, "org.tukaani.xz.XZInputStream"); - m.put(ContentCoding.LZMA, "org.tukaani.xz.XZInputStream"); - REQUIRED_CLASS_NAME = Collections.unmodifiableMap(m); - } - - /** - * @return lazy decoder for the given IANA token (lower-case). - */ - static IOFunction decoder(final String token) { - final String enc = token.toLowerCase(Locale.ROOT); - final CompressorStreamFactory factory = new CompressorStreamFactory(); - return in -> { - try { - return factory.createCompressorInputStream(enc, in); - } catch (final CompressorException | LinkageError ex) { - throw new IOException("Unable to decode Content-Encoding '" + enc + '\'', ex); - } - }; - } - - /** - * Tests that required helper classes are present for a coding token. - */ - static boolean runtimeAvailable(final String token) { - final ContentCoding coding = ContentCoding.fromToken(token); - if (coding == null) { - return true; - } - final String helper = REQUIRED_CLASS_NAME.get(coding); - if (helper == null) { - // no extra JAR needed - return true; - } - try { - Class.forName(helper, false, - CommonsCompressDecoderFactory.class.getClassLoader()); - return true; - } catch (final ClassNotFoundException | LinkageError ex) { - return false; - } - } -} \ No newline at end of file diff --git a/httpclient5/src/main/java/org/apache/hc/client5/http/entity/compress/CommonsCompressSupport.java b/httpclient5/src/main/java/org/apache/hc/client5/http/entity/compress/CommonsCompressSupport.java index 986f5c5683..2053bd59d4 100644 --- a/httpclient5/src/main/java/org/apache/hc/client5/http/entity/compress/CommonsCompressSupport.java +++ b/httpclient5/src/main/java/org/apache/hc/client5/http/entity/compress/CommonsCompressSupport.java @@ -33,9 +33,12 @@ import org.apache.hc.core5.annotation.ThreadingBehavior; /** - * Utility that answers the question “Is Apache Commons Compress - * on the class-path and in a usable state?” Both the encoder and - * decoder registries rely on this information. + * Lightweight guard that checks whether the Commons Compress factory + * class is loadable with the current class loader. + *

+ * Used by the codec registry to decide if reflective wiring of optional + * codecs should even be attempted. + *

* * @since 5.6 */ @@ -46,8 +49,11 @@ final class CommonsCompressSupport { private static final String CCSF = "org.apache.commons.compress.compressors.CompressorStreamFactory"; - /** Non-instantiable. */ - private CommonsCompressSupport() { } + /** + * Non-instantiable. + */ + private CommonsCompressSupport() { + } /** * Returns {@code true} if the core Commons Compress class can be loaded diff --git a/httpclient5/src/main/java/org/apache/hc/client5/http/entity/compress/CommonsCompressingEntity.java b/httpclient5/src/main/java/org/apache/hc/client5/http/entity/compress/CompressingEntity.java similarity index 56% rename from httpclient5/src/main/java/org/apache/hc/client5/http/entity/compress/CommonsCompressingEntity.java rename to httpclient5/src/main/java/org/apache/hc/client5/http/entity/compress/CompressingEntity.java index 7580b1a34b..0b4ce098bd 100644 --- a/httpclient5/src/main/java/org/apache/hc/client5/http/entity/compress/CommonsCompressingEntity.java +++ b/httpclient5/src/main/java/org/apache/hc/client5/http/entity/compress/CompressingEntity.java @@ -30,36 +30,39 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.util.Locale; -import org.apache.commons.compress.compressors.CompressorException; -import org.apache.commons.compress.compressors.CompressorStreamFactory; -import org.apache.hc.core5.annotation.Contract; -import org.apache.hc.core5.annotation.Internal; -import org.apache.hc.core5.annotation.ThreadingBehavior; import org.apache.hc.core5.http.HttpEntity; import org.apache.hc.core5.http.io.entity.HttpEntityWrapper; +import org.apache.hc.core5.io.IOFunction; import org.apache.hc.core5.util.Args; + /** - * Compresses the wrapped entity on-the-fly using Apache Commons Compress. - * - *

The codec is chosen by its IANA token (for example {@code "br"} or - * {@code "zstd"}). The helper JAR must be present at run-time; otherwise - * {@link #writeTo(OutputStream)} will throw {@link IOException}.

+ * Streaming wrapper that compresses the enclosed {@link HttpEntity} on write. + *

+ * The actual compressor is supplied as an {@link IOFunction}<OutputStream,OutputStream> + * and is resolved by the caller (for example via reflective factories). This keeps + * compression back-ends fully optional and avoids hard classpath dependencies. + *

+ *

+ * The entity reports the configured {@code Content-Encoding} token and streams + * the content; length is unknown ({@code -1}), and the entity is chunked. + *

* * @since 5.6 */ -@Internal -@Contract(threading = ThreadingBehavior.STATELESS) -public final class CommonsCompressingEntity extends HttpEntityWrapper { +public final class CompressingEntity extends HttpEntityWrapper { - private final String coding; // lower-case - private final CompressorStreamFactory factory = new CompressorStreamFactory(); + private final IOFunction encoder; + private final String coding; // lower-case token for header reporting - CommonsCompressingEntity(final HttpEntity src, final String coding) { + public CompressingEntity( + final HttpEntity src, + final String coding, + final IOFunction encoder) { super(src); - this.coding = coding.toLowerCase(Locale.ROOT); + this.encoder = Args.notNull(encoder, "Stream encoder"); + this.coding = Args.notNull(coding, "Content coding").toLowerCase(java.util.Locale.ROOT); } @Override @@ -69,8 +72,8 @@ public String getContentEncoding() { @Override public long getContentLength() { - return -1; - } // streaming + return -1; // streaming + } @Override public boolean isChunked() { @@ -78,17 +81,23 @@ public boolean isChunked() { } @Override - public InputStream getContent() { // Pull-mode is not supported + public InputStream getContent() { throw new UnsupportedOperationException("Compressed entity is write-only"); } @Override public void writeTo(final OutputStream out) throws IOException { Args.notNull(out, "Output stream"); - try (OutputStream cos = factory.createCompressorOutputStream(coding, out)) { - super.writeTo(cos); - } catch (final CompressorException | LinkageError ex) { - throw new IOException("Unable to compress using coding '" + coding + '\'', ex); + final OutputStream wrapped = encoder.apply(out); + try { + super.writeTo(wrapped); + } finally { + // Close the wrapped stream to flush trailers/footers if any. + try { + wrapped.close(); + } catch (final IOException ignore) { + // best effort + } } } } \ No newline at end of file diff --git a/httpclient5/src/main/java/org/apache/hc/client5/http/entity/compress/ContentCodecRegistry.java b/httpclient5/src/main/java/org/apache/hc/client5/http/entity/compress/ContentCodecRegistry.java index 3d7ad9eaaa..76bceecbf5 100644 --- a/httpclient5/src/main/java/org/apache/hc/client5/http/entity/compress/ContentCodecRegistry.java +++ b/httpclient5/src/main/java/org/apache/hc/client5/http/entity/compress/ContentCodecRegistry.java @@ -27,6 +27,9 @@ package org.apache.hc.client5.http.entity.compress; +import java.io.IOException; +import java.io.InputStream; +import java.lang.reflect.Constructor; import java.util.Arrays; import java.util.Collections; import java.util.EnumMap; @@ -34,20 +37,30 @@ import java.util.function.UnaryOperator; import java.util.zip.GZIPInputStream; +import org.apache.hc.client5.http.entity.DeflateCompressingEntity; import org.apache.hc.client5.http.entity.DeflateInputStream; +import org.apache.hc.client5.http.entity.GzipCompressingEntity; import org.apache.hc.core5.annotation.Contract; import org.apache.hc.core5.annotation.Internal; import org.apache.hc.core5.annotation.ThreadingBehavior; import org.apache.hc.core5.http.HttpEntity; -import org.brotli.dec.BrotliInputStream; +import org.apache.hc.core5.io.IOFunction; /** - * Run-time catalogue of built-in and Commons-Compress - * {@linkplain java.util.function.UnaryOperator encoders} / {@linkplain java.util.function.UnaryOperator decoders}. - * - *

Entries are wired once at class-load time and published through an - * unmodifiable map, so lookups are lock-free and thread-safe.

+ * Registry of encode/decode transformations for HTTP content codings. + *

+ * Entries are wired once at class-load time and exposed via an unmodifiable map. + * Built-in gzip/deflate are always available. Additional codecs are discovered + * reflectively: + *

+ *
    + *
  • Commons Compress codecs, when the library and (if required) its helper JARs + * are present.
    • + *
  • Decode-only Brotli via brotli4j, when present on the classpath. This does + * not affect the advertised {@code Accept-Encoding} unless an encoder is also + * registered.
    • + *
* * @since 5.6 */ @@ -61,16 +74,10 @@ private static Map build() { final Map m = new EnumMap<>(ContentCoding.class); m.put(ContentCoding.GZIP, - new Codec( - // encoder - org.apache.hc.client5.http.entity.GzipCompressingEntity::new, - ent -> new DecompressingEntity(ent, GZIPInputStream::new))); - m.put(ContentCoding.DEFLATE, - new Codec( - org.apache.hc.client5.http.entity.DeflateCompressingEntity::new, - ent -> new DecompressingEntity(ent, DeflateInputStream::new))); - - /* 2. Commons-Compress extras ---------------------------------- */ + new Codec(GzipCompressingEntity::new, ent -> new DecompressingEntity(ent, GZIPInputStream::new))); + m.put(ContentCoding.DEFLATE, new Codec(DeflateCompressingEntity::new, ent -> new DecompressingEntity(ent, DeflateInputStream::new))); + + // 2) Commons-Compress (optional) — reflectively wired if (CommonsCompressSupport.isPresent()) { for (final ContentCoding c : Arrays.asList( ContentCoding.BROTLI, @@ -83,21 +90,21 @@ private static Map build() { ContentCoding.PACK200, ContentCoding.DEFLATE64)) { - if (CommonsCompressDecoderFactory.runtimeAvailable(c.token())) { + if (CommonsCompressCodecFactory.runtimeAvailable(c)) { m.put(c, new Codec( - e -> new CommonsCompressingEntity(e, c.token()), + e -> new CompressingEntity(e, c.token(), + CommonsCompressCodecFactory.encoder(c.token())), ent -> new DecompressingEntity(ent, - CommonsCompressDecoderFactory.decoder(c.token())))); + CommonsCompressCodecFactory.decoder(c.token())))); } } - } - /* 3. Native Brotli fallback (decode-only) ---------------------- */ - if (!m.containsKey(ContentCoding.BROTLI) - && CommonsCompressDecoderFactory.runtimeAvailable(ContentCoding.BROTLI.token())) { - m.put(ContentCoding.BROTLI, - Codec.decodeOnly(ent -> - new DecompressingEntity(ent, BrotliInputStream::new))); + } + // 3) Native Brotli fallback (decode-only), no compile-time dep + if (isPresent("com.aayushatharva.brotli4j.decoder.BrotliInputStream") + && isPresent("com.aayushatharva.brotli4j.Brotli4jLoader")) { + m.put(ContentCoding.BROTLI, Codec.decodeOnly(ent -> + new DecompressingEntity(ent, brotli4jDecoder()))); } return Collections.unmodifiableMap(m); @@ -113,20 +120,11 @@ public static HttpEntity unwrap(final ContentCoding c, final HttpEntity src) { return k != null && k.decoder != null ? k.decoder.apply(src) : null; } - private ContentCodecRegistry() { - } - - /** - * Returns the {@link java.util.function.UnaryOperator}<HttpEntity> for the given coding, or {@code null}. - */ public static UnaryOperator decoder(final ContentCoding coding) { final Codec c = REGISTRY.get(coding); return c != null ? c.decoder : null; } - /** - * Returns the {@link java.util.function.UnaryOperator}<HttpEntity> for the given coding, or {@code null}. - */ public static UnaryOperator encoder(final ContentCoding coding) { final Codec c = REGISTRY.get(coding); return c != null ? c.encoder : null; @@ -150,4 +148,30 @@ static Codec decodeOnly(final UnaryOperator d) { } } + private static boolean isPresent(final String className) { + try { + Class.forName(className, false, ContentCodecRegistry.class.getClassLoader()); + return true; + } catch (final ClassNotFoundException | LinkageError ex) { + return false; + } + } + + private static IOFunction brotli4jDecoder() { + return in -> { + try { + final ClassLoader cl = ContentCodecRegistry.class.getClassLoader(); + final Class loader = Class.forName("com.aayushatharva.brotli4j.Brotli4jLoader", false, cl); + loader.getMethod("ensureAvailability").invoke(null); + final Class cls = Class.forName("com.aayushatharva.brotli4j.decoder.BrotliInputStream", false, cl); + final Constructor ctor = cls.getConstructor(InputStream.class); + return (InputStream) ctor.newInstance(in); + } catch (final ReflectiveOperationException | LinkageError e) { + throw new IOException("Unable to decode brotli (brotli4j)", e); + } + }; + } + + private ContentCodecRegistry() { + } } \ No newline at end of file diff --git a/pom.xml b/pom.xml index 15f0627fe8..4e050498f0 100644 --- a/pom.xml +++ b/pom.xml @@ -145,11 +145,6 @@ log4j-core ${log4j.version} - - org.brotli - dec - ${brotli.version} - org.conscrypt conscrypt-openjdk-uber From be49950dd3cb017ff42927bfc7a90643e6c38a74 Mon Sep 17 00:00:00 2001 From: Arturo Bernal Date: Sun, 12 Oct 2025 19:19:28 +0200 Subject: [PATCH 2/2] Make DeflatingBrotliEntityProducer and InflatingBrotliDataConsumer load brotli4j via reflection. Removes hard dependencies and avoids linkage errors when Brotli is absent. --- .../http/async/methods/AsyncBrotli.java | 197 +++++++++++++++ .../DeflatingBrotliEntityProducer.java | 238 +++++++++--------- .../methods/InflatingBrotliDataConsumer.java | 154 +++++++----- .../entity/BrotliDecompressingEntity.java | 2 +- .../http/entity/BrotliInputStreamFactory.java | 12 +- .../compress/CommonsCompressCodecFactory.java | 3 +- .../entity/compress/ContentCodecRegistry.java | 1 - .../DeflatingBrotliEntityProducerTest.java | 146 ++++++++++- .../InflatingBrotliDataConsumerTest.java | 58 +++++ .../AsyncClientServerBrotliRoundTrip.java | 40 +-- pom.xml | 1 - 11 files changed, 621 insertions(+), 231 deletions(-) create mode 100644 httpclient5/src/main/java/org/apache/hc/client5/http/async/methods/AsyncBrotli.java diff --git a/httpclient5/src/main/java/org/apache/hc/client5/http/async/methods/AsyncBrotli.java b/httpclient5/src/main/java/org/apache/hc/client5/http/async/methods/AsyncBrotli.java new file mode 100644 index 0000000000..5639aa5554 --- /dev/null +++ b/httpclient5/src/main/java/org/apache/hc/client5/http/async/methods/AsyncBrotli.java @@ -0,0 +1,197 @@ +/* + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + * + * This software consists of voluntary contributions made by many + * individuals on behalf of the Apache Software Foundation. For more + * information on the Apache Software Foundation, please see + * . + * + */ +package org.apache.hc.client5.http.async.methods; + +import java.lang.reflect.Constructor; +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.nio.ByteBuffer; + +import org.apache.hc.core5.annotation.Internal; + +/** + * Internal helper that wraps all brotli4j reflection for async encode/decode. + * No public API exposure; avoids hard deps when brotli4j is absent. + */ +@Internal +final class AsyncBrotli { + + private static final String LOADER = "com.aayushatharva.brotli4j.Brotli4jLoader"; + private static final String DEC_WRAPPER = "com.aayushatharva.brotli4j.decoder.DecoderJNI$Wrapper"; + private static final String ENC_WRAPPER = "com.aayushatharva.brotli4j.encoder.EncoderJNI$Wrapper"; + private static final String ENC_MODE = "com.aayushatharva.brotli4j.encoder.Encoder$Mode"; + private static final String ENC_OPERATION = "com.aayushatharva.brotli4j.encoder.EncoderJNI$Operation"; + + private static final Class LOADER_C; + private static final Method ENSURE_M; + + // Decoder methods + private static final Class DEC_C; + private static final Constructor DEC_CTOR; + private static final Method DEC_GET_INPUT; // ByteBuffer getInputBuffer() + private static final Method DEC_PUSH; // void push(int) + private static final Method DEC_PULL; // ByteBuffer pull() + private static final Method DEC_STATUS; // Status getStatus() + private static final Method DEC_HAS_OUTPUT; // boolean hasOutput() + private static final Method DEC_DESTROY; // void destroy() + private static final Method DEC_STATUS_NAME; // via enum.name() + + // Encoder methods + private static final Class ENC_C; + private static final Constructor ENC_CTOR; // (int outBuf, int q, int lgwin, Mode) + private static final Method ENC_GET_INPUT; // ByteBuffer getInputBuffer() + private static final Method ENC_PUSH; // void push(Operation, int) + private static final Method ENC_PULL; // ByteBuffer pull() + private static final Method ENC_HAS_MORE; // boolean hasMoreOutput() + private static final Method ENC_DESTROY; // void destroy() + private static final Class MODE_C; + private static final Method MODE_VALUE_OF; // Mode valueOf(String) + private static final Class OP_C; + private static final Object OP_PROCESS; + private static final Object OP_FINISH; + + static { + try { + final ClassLoader cl = AsyncBrotli.class.getClassLoader(); + + LOADER_C = Class.forName(LOADER, false, cl); + ENSURE_M = LOADER_C.getMethod("ensureAvailability"); + + // Decoder + DEC_C = Class.forName(DEC_WRAPPER, false, cl); + DEC_CTOR = DEC_C.getConstructor(int.class); + DEC_GET_INPUT = DEC_C.getMethod("getInputBuffer"); + DEC_PUSH = DEC_C.getMethod("push", int.class); + DEC_PULL = DEC_C.getMethod("pull"); + DEC_STATUS = DEC_C.getMethod("getStatus"); + DEC_HAS_OUTPUT = DEC_C.getMethod("hasOutput"); + DEC_DESTROY = DEC_C.getMethod("destroy"); + DEC_STATUS_NAME = DEC_STATUS.getReturnType().getMethod("name"); + + // Encoder + ENC_C = Class.forName(ENC_WRAPPER, false, cl); + MODE_C = Class.forName(ENC_MODE, false, cl); + OP_C = Class.forName(ENC_OPERATION, false, cl); + ENC_CTOR = ENC_C.getConstructor(int.class, int.class, int.class, MODE_C); + ENC_GET_INPUT = ENC_C.getMethod("getInputBuffer"); + ENC_PUSH = ENC_C.getMethod("push", OP_C, int.class); + ENC_PULL = ENC_C.getMethod("pull"); + ENC_HAS_MORE = ENC_C.getMethod("hasMoreOutput"); + ENC_DESTROY = ENC_C.getMethod("destroy"); + MODE_VALUE_OF = MODE_C.getMethod("valueOf", String.class); + + final Field fProcess = OP_C.getField("PROCESS"); + final Field fFinish = OP_C.getField("FINISH"); + OP_PROCESS = fProcess.get(null); + OP_FINISH = fFinish.get(null); + } catch (final Throwable ex) { + throw new ExceptionInInitializerError(ex); + } + } + + private AsyncBrotli() { + } + + static void ensureAvailable() throws Exception { + ENSURE_M.invoke(null); + } + + // -------- Decoder -------- + + static Object newDecoder(final int outBuf) throws Exception { + ensureAvailable(); + return DEC_CTOR.newInstance(outBuf); + } + + static ByteBuffer decInput(final Object decoder) throws Exception { + return (ByteBuffer) DEC_GET_INPUT.invoke(decoder); + } + + static void decPush(final Object decoder, final int bytes) throws Exception { + DEC_PUSH.invoke(decoder, bytes); + } + + static ByteBuffer decPull(final Object decoder) throws Exception { + return (ByteBuffer) DEC_PULL.invoke(decoder); + } + + static String decStatusName(final Object decoder) throws Exception { + final Object status = DEC_STATUS.invoke(decoder); + return (String) DEC_STATUS_NAME.invoke(status); + } + + static boolean decHasOutput(final Object decoder) throws Exception { + return (Boolean) DEC_HAS_OUTPUT.invoke(decoder); + } + + static void decDestroy(final Object decoder) { + try { + ENC_DESTROY.invoke(decoder); + } catch (final Throwable ignore) { + } + try { + DEC_DESTROY.invoke(decoder); + } catch (final Throwable ignore) { + } + } + + // -------- Encoder -------- + + static Object newEncoder(final int outBuf, final int q, final int lgwin, final String modeName) throws Exception { + ensureAvailable(); + final String mn = modeName == null ? "GENERIC" : modeName.toUpperCase(java.util.Locale.ROOT); + final Object mode = MODE_VALUE_OF.invoke(null, mn); + return ENC_CTOR.newInstance(outBuf, q, lgwin, mode); + } + + static ByteBuffer encInput(final Object encoder) throws Exception { + return (ByteBuffer) ENC_GET_INPUT.invoke(encoder); + } + + static void encPushProcess(final Object encoder, final int bytes) throws Exception { + ENC_PUSH.invoke(encoder, OP_PROCESS, bytes); + } + + static void encPushFinish(final Object encoder) throws Exception { + ENC_PUSH.invoke(encoder, OP_FINISH, 0); + } + + static boolean encHasMoreOutput(final Object encoder) throws Exception { + return (Boolean) ENC_HAS_MORE.invoke(encoder); + } + + static ByteBuffer encPull(final Object encoder) throws Exception { + return (ByteBuffer) ENC_PULL.invoke(encoder); + } + + static void encDestroy(final Object encoder) { + try { + ENC_DESTROY.invoke(encoder); + } catch (final Throwable ignore) { + } + } +} \ No newline at end of file diff --git a/httpclient5/src/main/java/org/apache/hc/client5/http/async/methods/DeflatingBrotliEntityProducer.java b/httpclient5/src/main/java/org/apache/hc/client5/http/async/methods/DeflatingBrotliEntityProducer.java index 676c530a4d..c2099b33ee 100644 --- a/httpclient5/src/main/java/org/apache/hc/client5/http/async/methods/DeflatingBrotliEntityProducer.java +++ b/httpclient5/src/main/java/org/apache/hc/client5/http/async/methods/DeflatingBrotliEntityProducer.java @@ -32,50 +32,14 @@ import java.util.List; import java.util.Set; -import com.aayushatharva.brotli4j.encoder.Encoder; -import com.aayushatharva.brotli4j.encoder.EncoderJNI; - import org.apache.hc.core5.http.Header; import org.apache.hc.core5.http.nio.AsyncEntityProducer; import org.apache.hc.core5.http.nio.DataStreamChannel; import org.apache.hc.core5.util.Args; /** - * {@code AsyncEntityProducer} that Brotli-compresses bytes from an upstream producer - * on the fly and writes the compressed stream to the target {@link DataStreamChannel}. - *

- * Purely async/streaming: no {@code InputStream}/{@code OutputStream}. Back-pressure is - * honored via {@link #available()} and the I/O reactor’s calls into {@link #produce(DataStreamChannel)}. - * Trailers from the upstream producer are preserved and emitted once the compressed output - * has been fully drained. - *

- * - *

Content metadata

- * Returns {@code Content-Encoding: br}, {@code Content-Length: -1} and {@code chunked=true}. - * Repeatability matches the upstream producer. - * - *

Implementation notes

- * Uses Brotli4j’s {@code EncoderJNI.Wrapper}. JNI-owned output buffers are written directly - * when possible; if the channel applies back-pressure, the unwritten tail is copied into - * small pooled direct {@link java.nio.ByteBuffer}s to reduce allocation churn. Native - * resources are released in {@link #releaseResources()}. - *

- * Ensure {@link com.aayushatharva.brotli4j.Brotli4jLoader#ensureAvailability()} has been - * called once at startup; this class also invokes it in a static initializer as a safeguard. - *

- * - *

Usage

- * 
{@code
- * AsyncEntityProducer plain = new StringAsyncEntityProducer("hello", ContentType.TEXT_PLAIN);
- * AsyncEntityProducer br = new DeflatingBrotliEntityProducer(plain); // defaults q=5, lgwin=22
- * client.execute(new BasicRequestProducer(post, br),
- *                new BasicResponseConsumer<>(new StringAsyncEntityConsumer()),
- *                null);
- * }
+ * Async Brotli deflater (reflection-based brotli4j). No compile-time dep. * - * @see org.apache.hc.core5.http.nio.AsyncEntityProducer - * @see org.apache.hc.core5.http.nio.DataStreamChannel - * @see com.aayushatharva.brotli4j.encoder.EncoderJNI * @since 5.6 */ public final class DeflatingBrotliEntityProducer implements AsyncEntityProducer { @@ -83,56 +47,65 @@ public final class DeflatingBrotliEntityProducer implements AsyncEntityProducer private enum State { STREAMING, FINISHING, DONE } private final AsyncEntityProducer upstream; - private final EncoderJNI.Wrapper encoder; + // Reflective encoder instance (brotli4j EncoderJNI.Wrapper) + private final Object encoder; private ByteBuffer pendingOut; private List pendingTrailers; private State state = State.STREAMING; + public enum BrotliMode { + GENERIC, + TEXT, + FONT; + } + + /** + * Defaults: quality=5, lgwin=22, mode=GENERIC. + */ + public DeflatingBrotliEntityProducer(final AsyncEntityProducer upstream) throws IOException { + this(upstream, 5, 22, BrotliMode.GENERIC); + } + /** - * Create a producer with explicit Brotli params. - * - * @param upstream upstream entity producer whose bytes will be compressed - * @param quality Brotli quality level (see brotli4j documentation) - * @param lgwin Brotli window size log2 (see brotli4j documentation) - * @param mode Brotli mode hint (GENERIC/TEXT/FONT) - * @throws IOException if the native encoder cannot be created - * @since 5.6 + * Convenience: modeInt 0=GENERIC, 1=TEXT, 2=FONT. */ public DeflatingBrotliEntityProducer( final AsyncEntityProducer upstream, final int quality, final int lgwin, - final Encoder.Mode mode) throws IOException { - this.upstream = Args.notNull(upstream, "upstream"); - this.encoder = new EncoderJNI.Wrapper(256 * 1024, quality, lgwin, mode); + final int modeInt) throws IOException { + this(upstream, quality, lgwin, modeInt == 1 ? BrotliMode.TEXT.name() : (modeInt == 2 ? BrotliMode.FONT.name() : BrotliMode.GENERIC.name())); } /** - * Convenience constructor mapping {@code 0=GENERIC, 1=TEXT, 2=FONT}. - * - * @since 5.6 + * Kept for compatibility with existing code/tests. + * Uses reflection under the hood; no EncoderJNI references here. */ public DeflatingBrotliEntityProducer( final AsyncEntityProducer upstream, final int quality, final int lgwin, - final int modeInt) throws IOException { - this(upstream, quality, lgwin, - modeInt == 1 ? Encoder.Mode.TEXT : - modeInt == 2 ? Encoder.Mode.FONT : Encoder.Mode.GENERIC); + final BrotliMode mode) throws java.io.IOException { + this(upstream, quality, lgwin, mode != null ? mode.name() : "GENERIC"); } /** - * Create a producer with sensible defaults ({@code quality=5}, {@code lgwin=22}, {@code GENERIC}). - * - * @since 5.6 + * Fully reflective constructor used by the others. */ - public DeflatingBrotliEntityProducer(final AsyncEntityProducer upstream) throws IOException { - this(upstream, 5, 22, Encoder.Mode.GENERIC); + public DeflatingBrotliEntityProducer( + final AsyncEntityProducer upstream, + final int quality, + final int lgwin, + final String modeName) throws IOException { + this.upstream = Args.notNull(upstream, "upstream"); + try { + this.encoder = AsyncBrotli.newEncoder(256 * 1024, quality, lgwin, modeName); + } catch (final Exception e) { + throw new IOException("Brotli (brotli4j) not available", e); + } } - @Override public String getContentType() { return upstream.getContentType(); @@ -177,67 +150,13 @@ public int available() { @Override public void produce(final DataStreamChannel channel) throws IOException { - if (flushPending(channel)) { - return; - } - - if (state == State.FINISHING) { - encoder.push(EncoderJNI.Operation.FINISH, 0); - if (drainEncoder(channel)) { + try { + if (flushPending(channel)) { return; } - if (pendingTrailers == null) { - pendingTrailers = Collections.emptyList(); - } - channel.endStream(pendingTrailers); - pendingTrailers = null; - state = State.DONE; - return; - } - - upstream.produce(new DataStreamChannel() { - @Override - public void requestOutput() { - channel.requestOutput(); - } - - @Override - public int write(final ByteBuffer src) throws IOException { - int accepted = 0; - while (src.hasRemaining()) { - final ByteBuffer in = encoder.getInputBuffer(); - if (!in.hasRemaining()) { - encoder.push(EncoderJNI.Operation.PROCESS, 0); - if (drainEncoder(channel)) { - break; - } - continue; - } - final int xfer = Math.min(src.remaining(), in.remaining()); - final int lim = src.limit(); - src.limit(src.position() + xfer); - in.put(src); - src.limit(lim); - accepted += xfer; - - encoder.push(EncoderJNI.Operation.PROCESS, xfer); - if (drainEncoder(channel)) { - break; - } - } - return accepted; - } - - @Override - public void endStream() throws IOException { - endStream(Collections.emptyList()); - } - @Override - public void endStream(final List trailers) throws IOException { - pendingTrailers = trailers; - state = State.FINISHING; - encoder.push(EncoderJNI.Operation.FINISH, 0); + if (state == State.FINISHING) { + AsyncBrotli.encPushFinish(encoder); if (drainEncoder(channel)) { return; } @@ -247,8 +166,76 @@ public void endStream(final List trailers) throws IOException channel.endStream(pendingTrailers); pendingTrailers = null; state = State.DONE; + return; } - }); + + upstream.produce(new DataStreamChannel() { + @Override + public void requestOutput() { + channel.requestOutput(); + } + + @Override + public int write(final ByteBuffer src) throws IOException { + int accepted = 0; + try { + while (src.hasRemaining()) { + final ByteBuffer in = AsyncBrotli.encInput(encoder); + if (!in.hasRemaining()) { + AsyncBrotli.encPushProcess(encoder, 0); + if (drainEncoder(channel)) { + break; + } + continue; + } + final int xfer = Math.min(src.remaining(), in.remaining()); + final int lim = src.limit(); + src.limit(src.position() + xfer); + in.put(src); + src.limit(lim); + accepted += xfer; + + AsyncBrotli.encPushProcess(encoder, xfer); + if (drainEncoder(channel)) { + break; + } + } + return accepted; + } catch (final Exception ex) { + throw new IOException("Brotli encode failed", ex); + } + } + + @Override + public void endStream() throws IOException { + endStream(Collections.emptyList()); + } + + @Override + public void endStream(final List trailers) throws IOException { + try { + pendingTrailers = trailers; + state = State.FINISHING; + AsyncBrotli.encPushFinish(encoder); + if (drainEncoder(channel)) { + return; + } + if (pendingTrailers == null) { + pendingTrailers = Collections.
emptyList(); + } + channel.endStream(pendingTrailers); + pendingTrailers = null; + state = State.DONE; + } catch (final Exception ex) { + throw new IOException("Brotli finalize failed", ex); + } + } + }); + } catch (final IOException ioe) { + throw ioe; + } catch (final Exception ex) { + throw new IOException("Brotli encode failed", ex); + } } @Override @@ -259,7 +246,7 @@ public void failed(final Exception cause) { @Override public void releaseResources() { try { - encoder.destroy(); + AsyncBrotli.encDestroy(encoder); } catch (final Throwable ignore) { } upstream.releaseResources(); @@ -268,7 +255,6 @@ public void releaseResources() { state = State.DONE; } - private boolean flushPending(final DataStreamChannel channel) throws IOException { if (pendingOut != null && pendingOut.hasRemaining()) { channel.write(pendingOut); @@ -287,9 +273,9 @@ private boolean flushPending(final DataStreamChannel channel) throws IOException return false; } - private boolean drainEncoder(final DataStreamChannel channel) throws IOException { - while (encoder.hasMoreOutput()) { - final ByteBuffer buf = encoder.pull(); + private boolean drainEncoder(final DataStreamChannel channel) throws Exception { + while (AsyncBrotli.encHasMoreOutput(encoder)) { + final ByteBuffer buf = AsyncBrotli.encPull(encoder); if (buf == null || !buf.hasRemaining()) { continue; } diff --git a/httpclient5/src/main/java/org/apache/hc/client5/http/async/methods/InflatingBrotliDataConsumer.java b/httpclient5/src/main/java/org/apache/hc/client5/http/async/methods/InflatingBrotliDataConsumer.java index b2c3ae630f..cad1dfbc28 100644 --- a/httpclient5/src/main/java/org/apache/hc/client5/http/async/methods/InflatingBrotliDataConsumer.java +++ b/httpclient5/src/main/java/org/apache/hc/client5/http/async/methods/InflatingBrotliDataConsumer.java @@ -30,13 +30,10 @@ import java.nio.ByteBuffer; import java.util.List; -import com.aayushatharva.brotli4j.decoder.DecoderJNI; - import org.apache.hc.core5.http.Header; import org.apache.hc.core5.http.HttpException; import org.apache.hc.core5.http.nio.AsyncDataConsumer; import org.apache.hc.core5.http.nio.CapacityChannel; -import org.apache.hc.core5.util.Asserts; /** * {@code AsyncDataConsumer} that inflates a Brotli-compressed byte stream and forwards @@ -72,16 +69,15 @@ public final class InflatingBrotliDataConsumer implements AsyncDataConsumer { private final AsyncDataConsumer downstream; - private final DecoderJNI.Wrapper decoder; + private final Object decoder; // brotli4j DecoderJNI.Wrapper (reflective) private volatile CapacityChannel capacity; - public InflatingBrotliDataConsumer(final AsyncDataConsumer downstream) { this.downstream = downstream; try { - this.decoder = new DecoderJNI.Wrapper(8 * 1024); - } catch (final IOException e) { - throw new RuntimeException("Unable to initialize DecoderJNI", e); + this.decoder = AsyncBrotli.newDecoder(8 * 1024); + } catch (final Exception e) { + throw new RuntimeException("Brotli (brotli4j) not available", e); } } @@ -93,87 +89,113 @@ public void updateCapacity(final CapacityChannel capacityChannel) throws IOExcep @Override public void consume(final ByteBuffer src) throws IOException { - while (src.hasRemaining()) { - final ByteBuffer in = decoder.getInputBuffer(); - final int xfer = Math.min(src.remaining(), in.remaining()); - if (xfer == 0) { - decoder.push(0); + try { + while (src.hasRemaining()) { + final ByteBuffer in = AsyncBrotli.decInput(decoder); + final int xfer = Math.min(src.remaining(), in.remaining()); + if (xfer == 0) { + AsyncBrotli.decPush(decoder, 0); + pump(); + continue; + } + final int lim = src.limit(); + src.limit(src.position() + xfer); + in.put(src); + src.limit(lim); + + AsyncBrotli.decPush(decoder, xfer); pump(); - continue; } - final int lim = src.limit(); - src.limit(src.position() + xfer); - in.put(src); - src.limit(lim); - - decoder.push(xfer); - pump(); - } - final CapacityChannel ch = this.capacity; - if (ch != null) { - ch.update(Integer.MAX_VALUE); + final CapacityChannel ch = this.capacity; + if (ch != null) { + ch.update(Integer.MAX_VALUE); + } + } catch (final Exception ex) { + throw new IOException("Brotli decode failed", ex); } } @Override public void streamEnd(final List trailers) throws IOException, HttpException { - pump(); - Asserts.check(decoder.getStatus() == DecoderJNI.Status.DONE || !decoder.hasOutput(), - "Truncated brotli stream"); - downstream.streamEnd(trailers); + try { + // Bounded finish to avoid spins on truncated streams. + for (int i = 0; i < 8; i++) { + final String st = AsyncBrotli.decStatusName(decoder); + if ("DONE".equals(st)) { + downstream.streamEnd(trailers); + return; + } + if ("NEEDS_MORE_OUTPUT".equals(st)) { + pump(); // drain pending output + continue; + } + if ("OK".equals(st)) { + AsyncBrotli.decPush(decoder, 0); // advance without new input + pump(); + continue; + } + if ("NEEDS_MORE_INPUT".equals(st)) { + // End of input reached; decoder still wants bytes -> truncated + throw new IOException("Truncated brotli stream"); + } + throw new IOException("Brotli stream corrupted: " + st); + } + throw new IOException("Brotli stream did not reach DONE"); + } catch (final IOException | HttpException e) { + throw e; + } catch (final Exception ex) { + throw new IOException("Brotli stream end failed", ex); + } } + @Override public void releaseResources() { - try { - decoder.destroy(); - } catch (final Throwable ignore) { - } + AsyncBrotli.decDestroy(decoder); downstream.releaseResources(); } - private void pump() throws IOException { + private void pump() throws Exception { for (; ; ) { - switch (decoder.getStatus()) { - case OK: - decoder.push(0); - break; - case NEEDS_MORE_OUTPUT: { - // Pull a decoder-owned buffer; copy before handing off. - final ByteBuffer nativeBuf = decoder.pull(); + final String st = AsyncBrotli.decStatusName(decoder); + if ("OK".equals(st)) { + AsyncBrotli.decPush(decoder, 0); + continue; + } + if ("NEEDS_MORE_OUTPUT".equals(st)) { + final ByteBuffer nativeBuf = AsyncBrotli.decPull(decoder); + if (nativeBuf != null && nativeBuf.hasRemaining()) { + final ByteBuffer copy = ByteBuffer.allocateDirect(nativeBuf.remaining()); + copy.put(nativeBuf).flip(); + downstream.consume(copy); + } + continue; + } + if ("NEEDS_MORE_INPUT".equals(st)) { + if (AsyncBrotli.decHasOutput(decoder)) { + final ByteBuffer nativeBuf = AsyncBrotli.decPull(decoder); if (nativeBuf != null && nativeBuf.hasRemaining()) { final ByteBuffer copy = ByteBuffer.allocateDirect(nativeBuf.remaining()); copy.put(nativeBuf).flip(); downstream.consume(copy); + continue; } - break; } - case NEEDS_MORE_INPUT: - if (decoder.hasOutput()) { - final ByteBuffer nativeBuf = decoder.pull(); - if (nativeBuf != null && nativeBuf.hasRemaining()) { - final ByteBuffer copy = ByteBuffer.allocateDirect(nativeBuf.remaining()); - copy.put(nativeBuf).flip(); - downstream.consume(copy); - break; - } - } - return; // wait for more input - case DONE: - if (decoder.hasOutput()) { - final ByteBuffer nativeBuf = decoder.pull(); - if (nativeBuf != null && nativeBuf.hasRemaining()) { - final ByteBuffer copy = ByteBuffer.allocateDirect(nativeBuf.remaining()); - copy.put(nativeBuf).flip(); - downstream.consume(copy); - break; - } + return; // wait for more input + } + if ("DONE".equals(st)) { + if (AsyncBrotli.decHasOutput(decoder)) { + final ByteBuffer nativeBuf = AsyncBrotli.decPull(decoder); + if (nativeBuf != null && nativeBuf.hasRemaining()) { + final ByteBuffer copy = ByteBuffer.allocateDirect(nativeBuf.remaining()); + copy.put(nativeBuf).flip(); + downstream.consume(copy); + continue; } - return; - default: - // Corrupted stream - throw new IOException("Brotli stream corrupted"); + } + return; } + throw new IOException("Brotli stream corrupted"); } } } \ No newline at end of file diff --git a/httpclient5/src/main/java/org/apache/hc/client5/http/entity/BrotliDecompressingEntity.java b/httpclient5/src/main/java/org/apache/hc/client5/http/entity/BrotliDecompressingEntity.java index 216fd27c60..f638b1700b 100644 --- a/httpclient5/src/main/java/org/apache/hc/client5/http/entity/BrotliDecompressingEntity.java +++ b/httpclient5/src/main/java/org/apache/hc/client5/http/entity/BrotliDecompressingEntity.java @@ -49,7 +49,7 @@ public BrotliDecompressingEntity(final HttpEntity entity) { public static boolean isAvailable() { try { - Class.forName("org.brotli.dec.BrotliInputStream"); + Class.forName("com.aayushatharva.brotli4j.decoder.BrotliInputStream"); return true; } catch (final ClassNotFoundException | NoClassDefFoundError e) { return false; diff --git a/httpclient5/src/main/java/org/apache/hc/client5/http/entity/BrotliInputStreamFactory.java b/httpclient5/src/main/java/org/apache/hc/client5/http/entity/BrotliInputStreamFactory.java index 2b06d30d90..15984c01ed 100644 --- a/httpclient5/src/main/java/org/apache/hc/client5/http/entity/BrotliInputStreamFactory.java +++ b/httpclient5/src/main/java/org/apache/hc/client5/http/entity/BrotliInputStreamFactory.java @@ -28,8 +28,7 @@ import java.io.IOException; import java.io.InputStream; - -import com.aayushatharva.brotli4j.decoder.BrotliInputStream; +import java.lang.reflect.Constructor; import org.apache.hc.core5.annotation.Contract; import org.apache.hc.core5.annotation.ThreadingBehavior; @@ -60,6 +59,13 @@ public static BrotliInputStreamFactory getInstance() { @Override public InputStream create(final InputStream inputStream) throws IOException { - return new BrotliInputStream(inputStream); + try { + // Prefer brotli4j if present + final Class cls = Class.forName("com.aayushatharva.brotli4j.decoder.BrotliInputStream"); + final Constructor c = cls.getConstructor(InputStream.class); + return (InputStream) c.newInstance(inputStream); + } catch (final ReflectiveOperationException | NoClassDefFoundError e) { + throw new IOException("Brotli decoder not available", e); + } } } diff --git a/httpclient5/src/main/java/org/apache/hc/client5/http/entity/compress/CommonsCompressCodecFactory.java b/httpclient5/src/main/java/org/apache/hc/client5/http/entity/compress/CommonsCompressCodecFactory.java index 6d47f0a2f6..aab322ecef 100644 --- a/httpclient5/src/main/java/org/apache/hc/client5/http/entity/compress/CommonsCompressCodecFactory.java +++ b/httpclient5/src/main/java/org/apache/hc/client5/http/entity/compress/CommonsCompressCodecFactory.java @@ -73,7 +73,6 @@ final class CommonsCompressCodecFactory { private static final String CC_DEFLATE64 = "org.apache.commons.compress.compressors.deflate64.Deflate64CompressorInputStream"; // Helper libs - private static final String H_BROTLI = "org.brotli.dec.BrotliInputStream"; private static final String H_ZSTD = "com.github.luben.zstd.ZstdInputStream"; private static final String H_XZ = "org.tukaani.xz.XZInputStream"; @@ -153,7 +152,7 @@ static boolean runtimeAvailable(final ContentCoding coding) { } switch (coding) { case BROTLI: - return isPresent(CC_BROTLI) && isPresent(H_BROTLI); + return isPresent(CC_BROTLI); case ZSTD: return isPresent(CC_ZSTD) && isPresent(H_ZSTD); case XZ: diff --git a/httpclient5/src/main/java/org/apache/hc/client5/http/entity/compress/ContentCodecRegistry.java b/httpclient5/src/main/java/org/apache/hc/client5/http/entity/compress/ContentCodecRegistry.java index 76bceecbf5..1af114d38c 100644 --- a/httpclient5/src/main/java/org/apache/hc/client5/http/entity/compress/ContentCodecRegistry.java +++ b/httpclient5/src/main/java/org/apache/hc/client5/http/entity/compress/ContentCodecRegistry.java @@ -80,7 +80,6 @@ private static Map build() { // 2) Commons-Compress (optional) — reflectively wired if (CommonsCompressSupport.isPresent()) { for (final ContentCoding c : Arrays.asList( - ContentCoding.BROTLI, ContentCoding.ZSTD, ContentCoding.XZ, ContentCoding.LZMA, diff --git a/httpclient5/src/test/java/org/apache/hc/client5/http/async/methods/DeflatingBrotliEntityProducerTest.java b/httpclient5/src/test/java/org/apache/hc/client5/http/async/methods/DeflatingBrotliEntityProducerTest.java index 549f12cbcd..2c5ac85820 100644 --- a/httpclient5/src/test/java/org/apache/hc/client5/http/async/methods/DeflatingBrotliEntityProducerTest.java +++ b/httpclient5/src/test/java/org/apache/hc/client5/http/async/methods/DeflatingBrotliEntityProducerTest.java @@ -26,10 +26,12 @@ */ package org.apache.hc.client5.http.async.methods; +import static org.apache.hc.client5.http.async.methods.DeflatingBrotliEntityProducer.BrotliMode; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.ByteArrayOutputStream; +import java.io.IOException; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.util.Collections; @@ -38,11 +40,11 @@ import com.aayushatharva.brotli4j.Brotli4jLoader; import com.aayushatharva.brotli4j.decoder.Decoder; import com.aayushatharva.brotli4j.decoder.DirectDecompress; -import com.aayushatharva.brotli4j.encoder.Encoder; import org.apache.hc.core5.http.Header; import org.apache.hc.core5.http.nio.AsyncEntityProducer; import org.apache.hc.core5.http.nio.DataStreamChannel; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; @@ -75,7 +77,9 @@ public void requestOutput() { /* no-op for test */ } @Override public int write(final ByteBuffer src) { final int len = Math.min(src.remaining(), maxChunk); - if (len <= 0) return 0; + if (len <= 0) { + return 0; + } final byte[] tmp = new byte[len]; src.get(tmp); sink.write(tmp, 0, len); @@ -118,7 +122,7 @@ void roundTrip() throws Exception { org.apache.hc.core5.http.ContentType.TEXT_PLAIN.withCharset(java.nio.charset.StandardCharsets.UTF_8) ); final DeflatingBrotliEntityProducer br = - new DeflatingBrotliEntityProducer(raw, 5, 22, Encoder.Mode.TEXT); + new DeflatingBrotliEntityProducer(raw, 5, 22, BrotliMode.TEXT); final ThrottledChannel ch = new ThrottledChannel(1024); @@ -140,4 +144,140 @@ void roundTrip() throws Exception { assertEquals(-1, br.getContentLength()); assertTrue(ch.ended(), "stream was not ended"); } + + private static final class CloseOnFirstUpstream implements AsyncEntityProducer { + private final byte[] data; + private boolean done; + + CloseOnFirstUpstream(final byte[] data) { + this.data = data; + } + + @Override + public void releaseResources() { + } + + @Override + public String getContentType() { + return "text/plain"; + } + + @Override + public String getContentEncoding() { + return null; + } + + @Override + public long getContentLength() { + return data.length; + } + + @Override + public boolean isChunked() { + return false; + } + + @Override + public java.util.Set getTrailerNames() { + return java.util.Collections.emptySet(); + } + + @Override + public boolean isRepeatable() { + return true; + } + + @Override + public void failed(final Exception e) { + + } + + @Override + public int available() { + return done ? 0 : 1; + } + + @Override + public void produce(final DataStreamChannel ch) throws IOException { + if (!done) { + ch.write(ByteBuffer.wrap(data)); + ch.endStream(Collections.emptyList()); // finish immediately + done = true; + } + } + } + + private static final class BackpressuredChannel implements DataStreamChannel { + private final java.io.ByteArrayOutputStream sink = new ByteArrayOutputStream(); + private final int maxChunk; + private boolean ended; + private boolean blockFirstWrite = true; + + BackpressuredChannel(final int maxChunk) { + this.maxChunk = maxChunk; + } + + @Override + public void requestOutput() { /* no-op */ } + + @Override + public int write(final java.nio.ByteBuffer src) { + if (blockFirstWrite) { + blockFirstWrite = false; + return 0; + } // force pendingOut + final int len = Math.min(src.remaining(), maxChunk); + if (len <= 0) { + return 0; + } + final byte[] tmp = new byte[len]; + src.get(tmp); + sink.write(tmp, 0, len); + return len; + } + + @Override + public void endStream() { + endStream(Collections.
emptyList()); + } + + @Override + public void endStream(final List trailers) { + ended = true; + } + + boolean ended() { + return ended; + } + } + + @Test + void encoderProbePreventsStall() throws Exception { + final StringBuilder sb = new StringBuilder(128 * 1024); + for (int i = 0; i < 128 * 1024; i++) { + sb.append('x'); + } + final byte[] payload = sb.toString().getBytes(StandardCharsets.UTF_8); + + final AsyncEntityProducer upstream = new CloseOnFirstUpstream(payload); + final DeflatingBrotliEntityProducer br = + new DeflatingBrotliEntityProducer(upstream, 5, 22, + DeflatingBrotliEntityProducer.BrotliMode.TEXT); + + final BackpressuredChannel ch = new BackpressuredChannel(64); + + br.produce(ch); + + Assertions.assertTrue( + br.available() > 0, + "Producer has pending output but available()==0 (stall)"); + + // Drain to completion + while (br.available() > 0) { + br.produce(ch); + } + Assertions.assertTrue(ch.ended(), "Stream not ended"); + } + + } diff --git a/httpclient5/src/test/java/org/apache/hc/client5/http/async/methods/InflatingBrotliDataConsumerTest.java b/httpclient5/src/test/java/org/apache/hc/client5/http/async/methods/InflatingBrotliDataConsumerTest.java index 8446d28307..e9083a9e6b 100644 --- a/httpclient5/src/test/java/org/apache/hc/client5/http/async/methods/InflatingBrotliDataConsumerTest.java +++ b/httpclient5/src/test/java/org/apache/hc/client5/http/async/methods/InflatingBrotliDataConsumerTest.java @@ -198,4 +198,62 @@ void registerInExec() { final ContentCompressionAsyncExec exec = new ContentCompressionAsyncExec(map, false); assertNotNull(exec); } + + @Test + void inflateBrotli_truncated_throws_onStreamEnd_java8() throws Exception { + final String seed = "Hello ✈ brotli 🎈!"; + final StringBuilder sb = new StringBuilder(seed.length() * 5000); + for (int i = 0; i < 5000; i++) { + sb.append(seed); + } + final byte[] src = sb.toString().getBytes(StandardCharsets.UTF_8); + + final Encoder.Parameters p = new Encoder.Parameters() + .setQuality(6).setWindow(22).setMode(Encoder.Mode.TEXT); + final byte[] good = Encoder.compress(src, p); + + final int cut = Math.min(8, Math.max(1, good.length / 10)); + final byte[] truncated = java.util.Arrays.copyOf(good, good.length - cut); + + final InflatingBrotliDataConsumer inflating = + new InflatingBrotliDataConsumer(new AsyncEntityConsumer() { + public void streamStart(final EntityDetails ed, final FutureCallback cb) { + } + + public void updateCapacity(final CapacityChannel c) { + } + + public void consume(final ByteBuffer srcBuf) { + } + + public void streamEnd(final List t) { + } + + public void failed(final Exception cause) { + org.junit.jupiter.api.Assertions.fail(cause); + } + + public void releaseResources() { + } + + public Object getContent() { + return null; + } + }); + + for (int off = 0; off < truncated.length; off += 1024) { + final int n = Math.min(1024, truncated.length - off); + inflating.consume(ByteBuffer.wrap(truncated, off, n)); + } + + try { + inflating.streamEnd(java.util.Collections.emptyList()); + org.junit.jupiter.api.Assertions.fail("Expected truncation to be detected at streamEnd()"); + } catch (final IOException expected) { + // ok + } + } + + + } diff --git a/httpclient5/src/test/java/org/apache/hc/client5/http/examples/AsyncClientServerBrotliRoundTrip.java b/httpclient5/src/test/java/org/apache/hc/client5/http/examples/AsyncClientServerBrotliRoundTrip.java index 74f522f78b..33a08717f5 100644 --- a/httpclient5/src/test/java/org/apache/hc/client5/http/examples/AsyncClientServerBrotliRoundTrip.java +++ b/httpclient5/src/test/java/org/apache/hc/client5/http/examples/AsyncClientServerBrotliRoundTrip.java @@ -34,11 +34,9 @@ import java.util.concurrent.Future; import com.aayushatharva.brotli4j.Brotli4jLoader; +import com.aayushatharva.brotli4j.decoder.BrotliInputStream; import com.aayushatharva.brotli4j.encoder.BrotliOutputStream; -import org.apache.commons.compress.compressors.CompressorException; -import org.apache.commons.compress.compressors.CompressorInputStream; -import org.apache.commons.compress.compressors.CompressorStreamFactory; import org.apache.hc.client5.http.async.methods.SimpleHttpRequest; import org.apache.hc.client5.http.async.methods.SimpleRequestBuilder; import org.apache.hc.client5.http.impl.async.CloseableHttpAsyncClient; @@ -66,18 +64,14 @@ /** * Async client/server demo with Brotli in both directions: - *

* - Client sends a Brotli-compressed request body (Content-Encoding: br) - * - Server decompresses request, then responds with a Brotli-compressed body - * - Client checks the response Content-Encoding and decompresses if needed - *

- * Notes: - * - Encoding uses brotli4j (native JNI); make sure matching native dependency is on the runtime classpath. - * - Decoding here uses Commons Compress via CompressorStreamFactory("br"). + * - Server decompresses request using brotli4j, then responds with a Brotli-compressed body + * - Client checks the response Content-Encoding and decompresses with brotli4j if needed */ public final class AsyncClientServerBrotliRoundTrip { static { + // Ensure native libs are loaded once Brotli4jLoader.ensureAvailability(); } @@ -99,7 +93,6 @@ public static void main(final String[] args) throws Exception { final String requestBody = "Hello Brotli world (round-trip)!"; System.out.println("Request (plain): " + requestBody); - // --- client compresses request --- final byte[] reqCompressed = brotliCompress(requestBody.getBytes(StandardCharsets.UTF_8)); final SimpleHttpRequest post = SimpleRequestBuilder.post(url) @@ -131,15 +124,15 @@ public static void main(final String[] args) throws Exception { /** * Server handler: - * - If request has Content-Encoding: br, decompress it - * - Echo the text back, but re-encode the response with Brotli (Content-Encoding: br) + * - If request has Content-Encoding: br, decompress it with brotli4j + * - Echo the text back, re-encoded with Brotli */ private static final class EchoHandler implements HttpRequestHandler { @Override public void handle( final ClassicHttpRequest request, final ClassicHttpResponse response, - final HttpContext context) throws IOException { + final HttpContext context) { final HttpEntity entity = request.getEntity(); if (entity == null) { @@ -153,8 +146,7 @@ public void handle( final Header ce = request.getFirstHeader(HttpHeaders.CONTENT_ENCODING); if (ce != null && BR.equalsIgnoreCase(ce.getValue())) { try (final InputStream in = entity.getContent(); - final CompressorInputStream bin = - new CompressorStreamFactory().createCompressorInputStream(BR, in)) { + final BrotliInputStream bin = new BrotliInputStream(in)) { requestPlain = readAll(bin); } } else { @@ -172,19 +164,13 @@ public void handle( response.addHeader(HttpHeaders.CONTENT_ENCODING, BR); response.setEntity(new ByteArrayEntity(respCompressed, ContentType.APPLICATION_OCTET_STREAM)); - } catch (final CompressorException ex) { + } catch (final Exception ex) { response.setCode(HttpStatus.SC_BAD_REQUEST); response.setEntity(new StringEntity("Invalid Brotli payload", StandardCharsets.UTF_8)); - } catch (final Exception ex) { - response.setCode(HttpStatus.SC_INTERNAL_SERVER_ERROR); - response.setEntity(new StringEntity("Server error", StandardCharsets.UTF_8)); } } } - /** - * Utility: read entire stream into a byte[] (demo-only). - */ private static byte[] readAll(final InputStream in) throws IOException { final ByteArrayOutputStream bos = new ByteArrayOutputStream(); final byte[] buf = new byte[8192]; @@ -196,7 +182,7 @@ private static byte[] readAll(final InputStream in) throws IOException { } /** - * Compress a byte[] with Brotli using brotli4j. + * Compress using brotli4j. */ private static byte[] brotliCompress(final byte[] plain) throws IOException { final ByteArrayOutputStream baos = new ByteArrayOutputStream(); @@ -207,14 +193,12 @@ private static byte[] brotliCompress(final byte[] plain) throws IOException { } /** - * Decompress a Brotli-compressed byte[] using Commons Compress. + * Decompress using brotli4j. */ private static byte[] brotliDecompress(final byte[] compressed) throws IOException { try (final InputStream in = new ByteArrayInputStream(compressed); - final CompressorInputStream bin = new CompressorStreamFactory().createCompressorInputStream(BR, in)) { + final BrotliInputStream bin = new BrotliInputStream(in)) { return readAll(bin); - } catch (final CompressorException e) { - throw new IOException("Failed to decompress Brotli data", e); } } } diff --git a/pom.xml b/pom.xml index 4e050498f0..4d5bc79145 100644 --- a/pom.xml +++ b/pom.xml @@ -64,7 +64,6 @@ 1.8 5.4-alpha1 2.25.0 - 0.1.2 1.20.0 2.5.2 3.10.8