debian/changelog

+wagon (3.3.1-1) unstable; urgency=medium
+
+  * Team upload.
+  * New upstream release
+  * Rebuild with libhttpclient-java (>= 4.5.6-2~) to restore the compatibility
+    with Java 8 (Closes: #918772)
+  * Standards-Version updated to 4.3.0
+
+ -- Emmanuel Bourg <ebourg@apache.org>  Fri, 11 Jan 2019 23:47:34 +0100
+
 wagon (3.2.0-2) unstable; urgency=medium
 
   * Team upload.

debian/control

@@ -11,7 +11,7 @@ Build-Depends:
  libcommons-io-java,
  libcommons-net-java,
  libeasymock-java (>= 3.2),
- libhttpclient-java (>= 4.3.1),
+ libhttpclient-java (>= 4.5.6-2~),
  libjetty9-extra-java (>= 9.4),
  libjetty9-java (>= 9.4),
  libjsch-agent-proxy-java,
@@ -26,7 +26,7 @@ Build-Depends:
  libplexus-utils2-java,
  libservlet3.1-java,
  maven-debian-helper
-Standards-Version: 4.2.1
+Standards-Version: 4.3.0
 Vcs-Git: https://salsa.debian.org/java-team/wagon.git
 Vcs-Browser: https://salsa.debian.org/java-team/wagon
 Homepage: http://maven.apache.org/wagon/

pom.xml

@@ -29,7 +29,7 @@ under the License.
 
   <groupId>org.apache.maven.wagon</groupId>
   <artifactId>wagon</artifactId>
-  <version>3.2.0</version>
+  <version>3.3.1</version>
   <packaging>pom</packaging>
 
   <name>Apache Maven Wagon</name>
@@ -200,7 +200,7 @@ under the License.
     <connection>scm:git:https://gitbox.apache.org/repos/asf/maven-wagon.git</connection>
     <developerConnection>scm:git:https://gitbox.apache.org/repos/asf/maven-wagon.git</developerConnection>
     <url>https://github.com/apache/maven-wagon/tree/${project.scm.tag}</url>
-    <tag>wagon-3.2.0</tag>
+    <tag>wagon-3.3.1</tag>
   </scm>
 
   <issueManagement>

wagon-provider-api/pom.xml

@@ -23,7 +23,7 @@ under the License.
   <parent>
     <groupId>org.apache.maven.wagon</groupId>
     <artifactId>wagon</artifactId>
-    <version>3.2.0</version>
+    <version>3.3.1</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 

wagon-provider-api/src/main/java/org/apache/maven/wagon/AbstractWagon.java

@@ -42,8 +42,14 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.nio.channels.Channels;
+import java.nio.channels.ReadableByteChannel;
 import java.util.List;
 
+import static java.lang.Math.max;
+import static java.lang.Math.min;
+
 /**
  * Implementation of common facilities for Wagon providers.
  *
@@ -52,7 +58,24 @@
 public abstract class AbstractWagon
     implements Wagon
 {
-    protected static final int DEFAULT_BUFFER_SIZE = 1024 * 4;
+    protected static final int DEFAULT_BUFFER_SIZE = 4 * 1024;
+    protected static final int MAXIMUM_BUFFER_SIZE = 512 * 1024;
+
+    /**
+     * To efficiently buffer data, use a multiple of 4 KiB as this is likely to match the hardware
+     * buffer size of certain storage devices.
+     */
+    protected static final int BUFFER_SEGMENT_SIZE = 4 * 1024;
+
+    /**
+     * The desired minimum amount of chunks in which a {@link Resource} shall be
+     * {@link #transfer(Resource, InputStream, OutputStream, int, long) transferred}.
+     * This corresponds to the minimum times {@link #fireTransferProgress(TransferEvent, byte[], int)}
+     * is executed. 100 notifications is a conservative value that will lead to small chunks for
+     * any artifact less that {@link #BUFFER_SEGMENT_SIZE} * {@link #MINIMUM_AMOUNT_OF_TRANSFER_CHUNKS}
+     * in size.
+     */
+    protected static final int MINIMUM_AMOUNT_OF_TRANSFER_CHUNKS = 100;
 
     protected Repository repository;
 
@@ -560,31 +583,72 @@ protected void transfer( Resource resource, InputStream input, OutputStream outp
     protected void transfer( Resource resource, InputStream input, OutputStream output, int requestType, long maxSize )
         throws IOException
     {
-        byte[] buffer = new byte[DEFAULT_BUFFER_SIZE];
+        ByteBuffer buffer = ByteBuffer.allocate( getBufferCapacityForTransfer( resource.getContentLength() ) );
+        int halfBufferCapacity = buffer.capacity() / 2;
 
         TransferEvent transferEvent = new TransferEvent( this, resource, TransferEvent.TRANSFER_PROGRESS, requestType );
         transferEvent.setTimestamp( System.currentTimeMillis() );
 
+        ReadableByteChannel in = Channels.newChannel( input );
+
         long remaining = maxSize;
-        while ( remaining > 0 )
+        while ( remaining > 0L )
         {
-            // let's safely cast to int because the min value will be lower than the buffer size.
-            int n = input.read( buffer, 0, (int) Math.min( buffer.length, remaining ) );
+            int read = in.read( buffer );
 
-            if ( n == -1 )
+            if ( read == -1 )
             {
-                break;
+                // EOF, but some data has not been written yet.
+                if ( buffer.position() != 0 )
+                {
+                    buffer.flip();
+                    fireTransferProgress( transferEvent, buffer.array(), buffer.limit() );
+                    output.write( buffer.array(), 0, buffer.limit() );
                 }
 
-            fireTransferProgress( transferEvent, buffer, n );
+                break;
+            }
 
-            output.write( buffer, 0, n );
+            // Prevent minichunking / fragmentation: when less than half the buffer is utilized,
+            // read some more bytes before writing and firing progress.
+            if ( buffer.position() < halfBufferCapacity )
+            {
+                continue;
+            }
 
-            remaining -= n;
+            buffer.flip();
+            fireTransferProgress( transferEvent, buffer.array(), buffer.limit() );
+            output.write( buffer.array(), 0, buffer.limit() );
+            remaining -= buffer.limit();
+            buffer.clear();
         }
         output.flush();
     }
 
+    /**
+     * Provides a buffer size for efficiently transferring the given amount of bytes such that
+     * it is not fragmented into too many chunks. For larger files larger buffers are provided such that downstream
+     * {@link #fireTransferProgress(TransferEvent, byte[], int) listeners} are not notified too frequently.
+     * For instance, transferring gigabyte-sized resources would result in millions of notifications when using
+     * only a few kibibytes of buffer, drastically slowing down transfer since transfer progress listeners and
+     * notifications are synchronous and may block, e.g., when writing download progress status to console.
+     *
+     * @param numberOfBytes can be 0 or less, in which case a default buffer size is used.
+     * @return a byte buffer suitable for transferring the given amount of bytes without too many chunks.
+     */
+    protected int getBufferCapacityForTransfer( long numberOfBytes )
+    {
+        if ( numberOfBytes <= 0L )
+        {
+            return DEFAULT_BUFFER_SIZE;
+        }
+
+        final int numberOfBufferSegments = (int)
+            numberOfBytes / ( BUFFER_SEGMENT_SIZE * MINIMUM_AMOUNT_OF_TRANSFER_CHUNKS );
+        final int potentialBufferSize = numberOfBufferSegments * BUFFER_SEGMENT_SIZE;
+        return min( MAXIMUM_BUFFER_SIZE, max( DEFAULT_BUFFER_SIZE, potentialBufferSize ) );
+    }
+
     // ----------------------------------------------------------------------
     //
     // ----------------------------------------------------------------------

wagon-provider-test/pom.xml

@@ -23,7 +23,7 @@ under the License.
   <parent>
     <groupId>org.apache.maven.wagon</groupId>
     <artifactId>wagon</artifactId>
-    <version>3.2.0</version>
+    <version>3.3.1</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 

wagon-providers/pom.xml

@@ -23,7 +23,7 @@ under the License.
   <parent>
     <groupId>org.apache.maven.wagon</groupId>
     <artifactId>wagon</artifactId>
-    <version>3.2.0</version>
+    <version>3.3.1</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 

wagon-providers/wagon-file/pom.xml

@@ -23,7 +23,7 @@ under the License.
   <parent>
     <groupId>org.apache.maven.wagon</groupId>
     <artifactId>wagon-providers</artifactId>
-    <version>3.2.0</version>
+    <version>3.3.1</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 

wagon-providers/wagon-ftp/pom.xml

@@ -23,7 +23,7 @@ under the License.
   <parent>
     <groupId>org.apache.maven.wagon</groupId>
     <artifactId>wagon-providers</artifactId>
-    <version>3.2.0</version>
+    <version>3.3.1</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 

wagon-providers/wagon-http-lightweight/pom.xml

@@ -23,7 +23,7 @@ under the License.
   <parent>
     <groupId>org.apache.maven.wagon</groupId>
     <artifactId>wagon-providers</artifactId>
-    <version>3.2.0</version>
+    <version>3.3.1</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 

wagon-providers/wagon-http-shared/pom.xml

@@ -23,7 +23,7 @@ under the License.
   <parent>
     <groupId>org.apache.maven.wagon</groupId>
     <artifactId>wagon-providers</artifactId>
-    <version>3.2.0</version>
+    <version>3.3.1</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 

wagon-providers/wagon-http-shared/src/main/java/org/apache/maven/wagon/shared/http/AbstractHttpClientWagon.java

@@ -25,6 +25,7 @@
 import org.apache.http.HttpHost;
 import org.apache.http.HttpResponse;
 import org.apache.http.HttpStatus;
+import org.apache.http.auth.AuthSchemeProvider;
 import org.apache.http.auth.AuthScope;
 import org.apache.http.auth.ChallengeState;
 import org.apache.http.auth.Credentials;
@@ -33,6 +34,7 @@
 import org.apache.http.client.AuthCache;
 import org.apache.http.client.CredentialsProvider;
 import org.apache.http.client.HttpRequestRetryHandler;
+import org.apache.http.client.config.AuthSchemes;
 import org.apache.http.client.config.CookieSpecs;
 import org.apache.http.client.config.RequestConfig;
 import org.apache.http.client.methods.CloseableHttpResponse;
@@ -52,6 +54,9 @@
 import org.apache.http.conn.ssl.SSLInitializationException;
 import org.apache.http.entity.AbstractHttpEntity;
 import org.apache.http.impl.auth.BasicScheme;
+import org.apache.http.impl.auth.BasicSchemeFactory;
+import org.apache.http.impl.auth.DigestSchemeFactory;
+import org.apache.http.impl.auth.NTLMSchemeFactory;
 import org.apache.http.impl.client.BasicAuthCache;
 import org.apache.http.impl.client.BasicCredentialsProvider;
 import org.apache.http.impl.client.CloseableHttpClient;
@@ -84,6 +89,11 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
+import java.io.RandomAccessFile;
+import java.nio.ByteBuffer;
+import java.nio.channels.Channels;
+import java.nio.channels.ReadableByteChannel;
+import java.nio.charset.StandardCharsets;
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.Collection;
@@ -105,9 +115,6 @@ public abstract class AbstractHttpClientWagon
     private final class RequestEntityImplementation
         extends AbstractHttpEntity
     {
-
-        private static final int BUFFER_SIZE = 2048;
-
         private final Resource resource;
 
         private final Wagon wagon;
@@ -160,52 +167,57 @@ public boolean isRepeatable()
             return repeatable;
         }
 
-        public void writeTo( final OutputStream outputStream )
+        public void writeTo( final OutputStream output )
             throws IOException
         {
-            if ( outputStream == null )
+            if ( output == null )
             {
-                throw new NullPointerException( "outputStream cannot be null" );
+                throw new NullPointerException( "output cannot be null" );
             }
             TransferEvent transferEvent =
                 new TransferEvent( wagon, resource, TransferEvent.TRANSFER_PROGRESS, TransferEvent.REQUEST_PUT );
             transferEvent.setTimestamp( System.currentTimeMillis() );
-            InputStream instream = ( this.source != null )
-                ? new FileInputStream( this.source )
-                : stream;
-            try
-            {
-                byte[] buffer = new byte[BUFFER_SIZE];
-                int l;
-                if ( this.length < 0 )
-                {
-                    // until EOF
-                    while ( ( l = instream.read( buffer ) ) != -1 )
+
+            try ( ReadableByteChannel input = ( this.source != null )
+                    ? new RandomAccessFile( this.source, "r" ).getChannel()
+                    : Channels.newChannel( stream ) )
             {
-                        fireTransferProgress( transferEvent, buffer, -1 );
-                        outputStream.write( buffer, 0, l );
-                    }
-                }
-                else
+                ByteBuffer buffer = ByteBuffer.allocate( getBufferCapacityForTransfer( this.length ) );
+                int halfBufferCapacity = buffer.capacity() / 2;
+
+                long remaining = this.length < 0L ? Long.MAX_VALUE : this.length;
+                while ( remaining > 0L )
                 {
-                    // no need to consume more than length
-                    long remaining = this.length;
-                    while ( remaining > 0 )
+                    int read = input.read( buffer );
+                    if ( read == -1 )
                     {
-                        l = instream.read( buffer, 0, (int) Math.min( BUFFER_SIZE, remaining ) );
-                        if ( l == -1 )
+                        // EOF, but some data has not been written yet.
+                        if ( buffer.position() != 0 )
                         {
-                            break;
+                            buffer.flip();
+                            fireTransferProgress( transferEvent, buffer.array(), buffer.limit() );
+                            output.write( buffer.array(), 0, buffer.limit() );
+                            buffer.clear();
                         }
-                        fireTransferProgress( transferEvent, buffer, (int) Math.min( BUFFER_SIZE, remaining ) );
-                        outputStream.write( buffer, 0, l );
-                        remaining -= l;
+
+                        break;
                     }
+
+                    // Prevent minichunking / fragmentation: when less than half the buffer is utilized,
+                    // read some more bytes before writing and firing progress.
+                    if ( buffer.position() < halfBufferCapacity )
+                    {
+                        continue;
                     }
+
+                    buffer.flip();
+                    fireTransferProgress( transferEvent, buffer.array(), buffer.limit() );
+                    output.write( buffer.array(), 0, buffer.limit() );
+                    remaining -= buffer.limit();
+                    buffer.clear();
+
                 }
-            finally
-            {
-                instream.close();
+                output.flush();
             }
         }
 
@@ -439,6 +451,15 @@ RETRY_HANDLER_COUNT, RETRY_HANDLER_REQUEST_SENT_ENABLED, getNonRetryableExceptio
         }
     }
 
+    private static Registry<AuthSchemeProvider> createAuthSchemeRegistry()
+    {
+        return RegistryBuilder.<AuthSchemeProvider>create()
+            .register( AuthSchemes.BASIC, new BasicSchemeFactory( StandardCharsets.UTF_8 ) )
+            .register( AuthSchemes.DIGEST, new DigestSchemeFactory( StandardCharsets.UTF_8 ) )
+            .register( AuthSchemes.NTLM, new NTLMSchemeFactory() )
+            .build();
+    }
+
     private static Collection<Class<? extends IOException>> getNonRetryableExceptions()
     {
         final List<Class<? extends IOException>> exceptions = new ArrayList<>();
@@ -466,6 +487,7 @@ private static CloseableHttpClient createClient()
             .disableConnectionState() //
             .setConnectionManager( httpClientConnectionManager ) //
             .setRetryHandler( createRetryHandler() )
+            .setDefaultAuthSchemeRegistry( createAuthSchemeRegistry() )
             .build();
     }
 

wagon-providers/wagon-http/pom.xml

@@ -23,7 +23,7 @@ under the License.
   <parent>
     <groupId>org.apache.maven.wagon</groupId>
     <artifactId>wagon-providers</artifactId>
-    <version>3.2.0</version>
+    <version>3.3.1</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 

wagon-providers/wagon-scm/pom.xml

@@ -23,7 +23,7 @@ under the License.
   <parent>
     <groupId>org.apache.maven.wagon</groupId>
     <artifactId>wagon-providers</artifactId>
-    <version>3.2.0</version>
+    <version>3.3.1</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 

wagon-providers/wagon-ssh-common-test/pom.xml

@@ -23,7 +23,7 @@ under the License.
   <parent>
     <groupId>org.apache.maven.wagon</groupId>
     <artifactId>wagon-providers</artifactId>
-    <version>3.2.0</version>
+    <version>3.3.1</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 

wagon-providers/wagon-ssh-common/pom.xml

@@ -23,7 +23,7 @@ under the License.
   <parent>
     <groupId>org.apache.maven.wagon</groupId>
     <artifactId>wagon-providers</artifactId>
-    <version>3.2.0</version>
+    <version>3.3.1</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 

wagon-providers/wagon-ssh-external/pom.xml

@@ -23,7 +23,7 @@ under the License.
   <parent>
     <groupId>org.apache.maven.wagon</groupId>
     <artifactId>wagon-providers</artifactId>
-    <version>3.2.0</version>
+    <version>3.3.1</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 

wagon-providers/wagon-ssh/pom.xml

@@ -23,7 +23,7 @@ under the License.
   <parent>
     <groupId>org.apache.maven.wagon</groupId>
     <artifactId>wagon-providers</artifactId>
-    <version>3.2.0</version>
+    <version>3.3.1</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 

wagon-providers/wagon-ssh/src/main/java/org/apache/maven/wagon/providers/ssh/jsch/AbstractJschWagon.java

@@ -24,7 +24,9 @@
 import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.IOException;
+import java.io.InputStream;
 import java.io.InputStreamReader;
+import java.io.OutputStream;
 import java.util.List;
 import java.util.Properties;
 
@@ -479,4 +481,35 @@ public void setStrictHostKeyChecking( String strictHostKeyChecking )
     {
         this.strictHostKeyChecking = strictHostKeyChecking;
     }
+
+    /** {@inheritDoc} */
+    // This method will be removed as soon as JSch issue #122 is resolved
+    @Override
+    protected void transfer( Resource resource, InputStream input, OutputStream output, int requestType, long maxSize )
+        throws IOException
+    {
+        byte[] buffer = new byte[DEFAULT_BUFFER_SIZE];
+
+        TransferEvent transferEvent = new TransferEvent( this, resource, TransferEvent.TRANSFER_PROGRESS, requestType );
+        transferEvent.setTimestamp( System.currentTimeMillis() );
+
+        long remaining = maxSize;
+        while ( remaining > 0L )
+        {
+            // let's safely cast to int because the min value will be lower than the buffer size.
+            int n = input.read( buffer, 0, (int) Math.min( buffer.length, remaining ) );
+
+            if ( n == -1 )
+            {
+                break;
+            }
+
+            fireTransferProgress( transferEvent, buffer, n );
+
+            output.write( buffer, 0, n );
+
+            remaining -= n;
+        }
+        output.flush();
+    }
 }

wagon-providers/wagon-webdav-jackrabbit/pom.xml

@@ -23,7 +23,7 @@ under the License.
   <parent>
     <groupId>org.apache.maven.wagon</groupId>
     <artifactId>wagon-providers</artifactId>
-    <version>3.2.0</version>
+    <version>3.3.1</version>
     <relativePath>../pom.xml</relativePath>
   </parent>