HTTPCLIENT-2375 Add first-class request-side compression support and pluggable encoders

httpclient5/pom.xml

@@ -113,6 +113,11 @@
       <artifactId>commons-compress</artifactId>
       <optional>true</optional>
     </dependency>
+    <dependency>
+      <groupId>com.github.luben</groupId>
+      <artifactId>zstd-jni</artifactId>
+      <scope>test</scope>
+    </dependency>
   </dependencies>
 
   <build>

httpclient5/src/main/java/org/apache/hc/client5/http/entity/DeflateCompressingEntity.java

@@ -0,0 +1,83 @@
+/*
+ * ====================================================================
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ * ====================================================================
+ *
+ * This software consists of voluntary contributions made by many
+ * individuals on behalf of the Apache Software Foundation.  For more
+ * information on the Apache Software Foundation, please see
+ * <http://www.apache.org/>.
+ *
+ */
+
+package org.apache.hc.client5.http.entity;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.zip.Deflater;
+import java.util.zip.DeflaterOutputStream;
+
+import org.apache.hc.core5.http.HttpEntity;
+import org.apache.hc.core5.http.io.entity.HttpEntityWrapper;
+import org.apache.hc.core5.util.Args;
+
+/**
+ * Entity wrapper that compresses the wrapped entity with
+ * <code>Content-Encoding: deflate</code> on write-out.
+ *
+ * @since 5.6
+ */
+public final class DeflateCompressingEntity extends HttpEntityWrapper {
+
+    private static final String DEFLATE_CODEC = "deflate";
+
+    public DeflateCompressingEntity(final HttpEntity entity) {
+        super(entity);
+    }
+
+    @Override
+    public String getContentEncoding() {
+        return DEFLATE_CODEC;
+    }
+
+    @Override
+    public long getContentLength() {
+        return -1;                // length unknown after compression
+    }
+
+    @Override
+    public boolean isChunked() {
+        return true;              // force chunked transfer-encoding
+    }
+
+    @Override
+    public InputStream getContent() throws IOException {
+        throw new UnsupportedOperationException("getContent() not supported");
+    }
+
+    @Override
+    public void writeTo(final OutputStream out) throws IOException {
+        Args.notNull(out, "Output stream");
+        // ‘false’ second arg = include zlib wrapper (= RFC 1950 = HTTP “deflate”)
+        try (DeflaterOutputStream deflater =
+                     new DeflaterOutputStream(out, new Deflater(Deflater.DEFAULT_COMPRESSION, /*nowrap*/ false))) {
+            super.writeTo(deflater);
+        }
+    }
+}

httpclient5/src/main/java/org/apache/hc/client5/http/entity/EntityBuilder.java

@@ -33,6 +33,8 @@
 import java.util.Arrays;
 import java.util.List;
 
+import org.apache.hc.client5.http.entity.compress.ContentCoding;
+import org.apache.hc.client5.http.entity.compress.ContentEncoderRegistry;
 import org.apache.hc.core5.http.ContentType;
 import org.apache.hc.core5.http.HttpEntity;
 import org.apache.hc.core5.http.NameValuePair;
@@ -72,8 +74,29 @@ public class EntityBuilder {
     private ContentType contentType;
     private String contentEncoding;
     private boolean chunked;
+    /**
+     * @deprecated use {@link #compressWith} instead
+     */
+    @Deprecated
     private boolean gzipCompressed;
 
+    /**
+     * The compression algorithm to apply when {@link #build() building}
+     * the final {@link HttpEntity}.
+     * <p>
+     * If {@code null} (default) the entity is sent as-is; otherwise the
+     * entity content is wrapped in the corresponding <i>compressing</i>
+     * wrapper (for example {@code GzipCompressingEntity},
+     * {@code DeflateCompressingEntity}, or a Commons-Compress based
+     * implementation) and the correct {@code Content-Encoding} header is
+     * added.
+     * </p>
+     *
+     * @since 5.6
+     */
+    private ContentCoding compressWith;
+
+
     EntityBuilder() {
         super();
     }
@@ -335,21 +358,56 @@ public EntityBuilder chunked() {
      * Tests if entities are to be GZIP compressed ({@code true}), or not ({@code false}).
      *
      * @return {@code true} if entity is to be GZIP compressed, {@code false} otherwise.
+     * @deprecated since 5.6 – use {@link #getCompressWith()} and
+     *             check for {@code ContentCoding.GZIP} instead.
      */
+    @Deprecated
     public boolean isGzipCompressed() {
-        return gzipCompressed;
+        return compressWith == ContentCoding.GZIP;
     }
 
     /**
      * Sets entities to be GZIP compressed.
      *
      * @return this instance.
+     * @deprecated since 5.6 – replace with
+     *             {@code compressed(ContentCoding.GZIP)}.
      */
+    @Deprecated
     public EntityBuilder gzipCompressed() {
-        this.gzipCompressed = true;
+        this.compressWith = ContentCoding.GZIP;
+        return this;
+    }
+
+    /**
+     * Requests that the entity produced by this builder be <em>compressed</em>
+     * with the supplied content-coding.
+     *
+     * @param coding the content-coding to use (never {@code null})
+     * @return this builder for method chaining
+     * @since 5.6
+     */
+    public EntityBuilder compressed(final ContentCoding coding) {
+        this.compressWith = coding;
         return this;
     }
 
+    /**
+     * Returns the content-coding that {@link #build()} will apply to the
+     * outgoing {@link org.apache.hc.core5.http.HttpEntity}, or {@code null}
+     * when no compression has been requested.
+     *
+     * @return the chosen {@link ContentCoding} — typically
+     * {@link ContentCoding#GZIP}, {@link ContentCoding#DEFLATE}, etc. —
+     * or {@code null} if the request body will be sent uncompressed.
+     * @since 5.6
+     */
+    public ContentCoding getCompressWith() {
+        return compressWith;
+    }
+
+
+
     private ContentType getContentOrDefault(final ContentType def) {
         return this.contentType != null ? this.contentType : def;
     }
@@ -380,8 +438,13 @@ public HttpEntity build() {
         } else {
             throw new IllegalStateException("No entity set");
         }
-        if (this.gzipCompressed) {
-            return new GzipCompressingEntity(e);
+        if (compressWith != null) {
+            final ContentEncoderRegistry.EncoderFactory f = ContentEncoderRegistry.lookup(compressWith);
+            if (f == null) {
+                throw new UnsupportedOperationException(
+                        "No encoder available for content-coding '" + compressWith.token() + '\'');
+            }
+            return f.wrap(e);
         }
         return e;
     }

httpclient5/src/main/java/org/apache/hc/client5/http/entity/compress/CommonsCompressSupport.java

@@ -0,0 +1,66 @@
+/*
+ * ====================================================================
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ * ====================================================================
+ *
+ * This software consists of voluntary contributions made by many
+ * individuals on behalf of the Apache Software Foundation.  For more
+ * information on the Apache Software Foundation, please see
+ * <http://www.apache.org/>.
+ *
+ */
+
+package org.apache.hc.client5.http.entity.compress;
+
+
+import org.apache.hc.core5.annotation.Contract;
+import org.apache.hc.core5.annotation.Internal;
+import org.apache.hc.core5.annotation.ThreadingBehavior;
+
+/**
+ * Utility that answers the question “Is Apache Commons Compress
+ * on the class-path and in a usable state?”  Both the encoder and
+ * decoder registries rely on this information.
+ *
+ * @since 5.6
+ */
+@Internal
+@Contract(threading = ThreadingBehavior.STATELESS)
+final class CommonsCompressSupport {
+
+    private static final String CCSF =
+            "org.apache.commons.compress.compressors.CompressorStreamFactory";
+
+    /** Non-instantiable. */
+    private CommonsCompressSupport() { }
+
+    /**
+     * Returns {@code true} if the core Commons Compress class can be loaded
+     * with the current class-loader, {@code false} otherwise.
+     */
+    static boolean isPresent() {
+        try {
+            Class.forName(CCSF, false,
+                    CommonsCompressSupport.class.getClassLoader());
+            return true;
+        } catch (ClassNotFoundException | LinkageError ex) {
+            return false;
+        }
+    }
+}
+

httpclient5/src/main/java/org/apache/hc/client5/http/entity/compress/CommonsCompressingEntity.java

@@ -0,0 +1,94 @@
+/*
+ * ====================================================================
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ * ====================================================================
+ *
+ * This software consists of voluntary contributions made by many
+ * individuals on behalf of the Apache Software Foundation.  For more
+ * information on the Apache Software Foundation, please see
+ * <http://www.apache.org/>.
+ *
+ */
+
+package org.apache.hc.client5.http.entity.compress;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.Locale;
+
+import org.apache.commons.compress.compressors.CompressorException;
+import org.apache.commons.compress.compressors.CompressorStreamFactory;
+import org.apache.hc.core5.annotation.Contract;
+import org.apache.hc.core5.annotation.Internal;
+import org.apache.hc.core5.annotation.ThreadingBehavior;
+import org.apache.hc.core5.http.HttpEntity;
+import org.apache.hc.core5.http.io.entity.HttpEntityWrapper;
+import org.apache.hc.core5.util.Args;
+
+/**
+ * Compresses the wrapped entity on-the-fly using Apache&nbsp;Commons Compress.
+ *
+ * <p>The codec is chosen by its IANA token (for example {@code "br"} or
+ * {@code "zstd"}).  The helper JAR must be present at run-time; otherwise
+ * {@link #writeTo(OutputStream)} will throw {@link IOException}.</p>
+ *
+ * @since 5.6
+ */
+@Internal
+@Contract(threading = ThreadingBehavior.STATELESS)
+public final class CommonsCompressingEntity extends HttpEntityWrapper {
+
+    private final String coding;                     // lower-case
+    private final CompressorStreamFactory factory = new CompressorStreamFactory();
+
+    CommonsCompressingEntity(final HttpEntity src, final String coding) {
+        super(src);
+        this.coding = coding.toLowerCase(Locale.ROOT);
+    }
+
+    @Override
+    public String getContentEncoding() {
+        return coding;
+    }
+
+    @Override
+    public long getContentLength() {
+        return -1;
+    }   // streaming
+
+    @Override
+    public boolean isChunked() {
+        return true;
+    }
+
+    @Override
+    public InputStream getContent() {          // Pull-mode is not supported
+        throw new UnsupportedOperationException("Compressed entity is write-only");
+    }
+
+    @Override
+    public void writeTo(final OutputStream out) throws IOException {
+        Args.notNull(out, "Output stream");
+        try (OutputStream cos = factory.createCompressorOutputStream(coding, out)) {
+            super.writeTo(cos);
+        } catch (final CompressorException | LinkageError ex) {
+            throw new IOException("Unable to compress using coding '" + coding + '\'', ex);
+        }
+    }
+}