Exposed MaxListingConcurrency so the initial phase of the transfer can be sped up (#226)

* Exposed MaxListingConcurrency so the intial phase of the transfer can be sped up

* Made the config value optional so the existing values can be preserved

* PR feedback

* No need to duplicate values

Author: pawelpabich

lib/Resources.Designer.cs

@@ -1,17 +1,17 @@
 <?xml version="1.0" encoding="utf-8"?>
 <root>
-  <!-- 
-    Microsoft ResX Schema 
-    
+  <!--
+    Microsoft ResX Schema
+
     Version 2.0
-    
-    The primary goals of this format is to allow a simple XML format 
-    that is mostly human readable. The generation and parsing of the 
-    various data types are done through the TypeConverter classes 
+
+    The primary goals of this format is to allow a simple XML format
+    that is mostly human readable. The generation and parsing of the
+    various data types are done through the TypeConverter classes
     associated with the data types.
-    
+
     Example:
-    
+
     ... ado.net/XML headers & schema ...
     <resheader name="resmimetype">text/microsoft-resx</resheader>
     <resheader name="version">2.0</resheader>
@@ -26,36 +26,36 @@
         <value>[base64 mime encoded string representing a byte array form of the .NET Framework object]</value>
         <comment>This is a comment</comment>
     </data>
-                
-    There are any number of "resheader" rows that contain simple 
+
+    There are any number of "resheader" rows that contain simple
     name/value pairs.
-    
-    Each data row contains a name, and value. The row also contains a 
-    type or mimetype. Type corresponds to a .NET class that support 
-    text/value conversion through the TypeConverter architecture. 
-    Classes that don't support this are serialized and stored with the 
+
+    Each data row contains a name, and value. The row also contains a
+    type or mimetype. Type corresponds to a .NET class that support
+    text/value conversion through the TypeConverter architecture.
+    Classes that don't support this are serialized and stored with the
     mimetype set.
-    
-    The mimetype is used for serialized objects, and tells the 
-    ResXResourceReader how to depersist the object. This is currently not 
+
+    The mimetype is used for serialized objects, and tells the
+    ResXResourceReader how to depersist the object. This is currently not
     extensible. For a given mimetype the value must be set accordingly:
-    
-    Note - application/x-microsoft.net.object.binary.base64 is the format 
-    that the ResXResourceWriter will generate, however the reader can 
+
+    Note - application/x-microsoft.net.object.binary.base64 is the format
+    that the ResXResourceWriter will generate, however the reader can
     read any of the formats listed below.
-    
+
     mimetype: application/x-microsoft.net.object.binary.base64
-    value   : The object must be serialized with 
+    value   : The object must be serialized with
             : System.Runtime.Serialization.Formatters.Binary.BinaryFormatter
             : and then encoded with base64 encoding.
-    
+
     mimetype: application/x-microsoft.net.object.soap.base64
-    value   : The object must be serialized with 
+    value   : The object must be serialized with
             : System.Runtime.Serialization.Formatters.Soap.SoapFormatter
             : and then encoded with base64 encoding.
 
     mimetype: application/x-microsoft.net.object.bytearray.base64
-    value   : The object must be serialized into a byte array 
+    value   : The object must be serialized into a byte array
             : using a System.ComponentModel.TypeConverter
             : and then encoded with base64 encoding.
     -->
@@ -222,6 +222,9 @@ MD5 in property: {2}</value>
   <data name="ParallelCountNotPositiveException" xml:space="preserve">
     <value>Parallel operations count must be positive.</value>
   </data>
+  <data name="MaxListingConcurrencyNotPositiveException" xml:space="preserve">
+    <value>Max listing concurrency must be positive.</value>
+  </data>
   <data name="ParameterCannotBeNullException" xml:space="preserve">
     <value>{0} cannot be null.</value>
     <comment>{0} is the property or parameter name.</comment>

lib/Resources.resx

@@ -32,13 +32,18 @@ public class TransferConfigurations
         /// </summary>
         private long maximumCacheSize;
 
+        /// <summary>
+        /// How many listings can be performed in parallel.
+        /// </summary>
+        private int? maxListingConcurrency;
+
         /// <summary>
         /// Instance to call native methods to get current memory status.
         /// </summary>
         private GlobalMemoryStatusNativeMethods memStatus = new GlobalMemoryStatusNativeMethods();
 
         /// <summary>
-        /// Initializes a new instance of the 
+        /// Initializes a new instance of the
         /// <see cref="TransferConfigurations" /> class.
         /// </summary>
         public TransferConfigurations()
@@ -52,12 +57,12 @@ public TransferConfigurations()
         }
 
         /// <summary>
-        /// Gets or sets a value indicating how many work items to process 
-        /// concurrently. Downloading or uploading a single blob can consist 
+        /// Gets or sets a value indicating how many work items to process
+        /// concurrently. Downloading or uploading a single blob can consist
         /// of a large number of work items.
         /// </summary>
         /// <value>How many work items to process concurrently.</value>
-        public int ParallelOperations 
+        public int ParallelOperations
         {
             get
             {
@@ -78,10 +83,24 @@ public int ParallelOperations
             }
         }
 
+        public int? MaxListingConcurrency
+        {
+            get
+            {
+                return this.maxListingConcurrency;
+            }
+            set
+            {
+                if (value <= 0)
+                    throw new ArgumentException(string.Format((IFormatProvider) CultureInfo.CurrentCulture, Resources.MaxListingConcurrencyNotPositiveException));
+                this.maxListingConcurrency = value;
+            }
+        }
+
         /// <summary>
-        /// Gets or sets the BlockSize to use for Windows Azure Storage transfers to block blob(s). 
+        /// Gets or sets the BlockSize to use for Windows Azure Storage transfers to block blob(s).
         /// It must be between 4MB and 100MB and be multiple of 4MB.
-        /// 
+        ///
         /// Currently, the max block count of a block blob is limited to 50000.
         /// When transfering a big file and the BlockSize provided is smaller than the minimum value - (size/50000),
         /// it'll be reset to a value which is greater than the minimum value and multiple of 4MB for this file.
@@ -123,7 +142,7 @@ public int BlockSize
         public string UserAgentPrefix
         {
             get;
-            set; 
+            set;
         }
 
         /// <summary>

lib/TransferConfigurations.cs

@@ -24,8 +24,8 @@ namespace Microsoft.Azure.Storage.DataMovement
     /// <summary>
     /// Represents a hierarchy directory transfer operation.
     /// In a flat directory transfer, the enumeration only returns file entries and it only transfers files under the directory.
-    /// 
-    /// In a hierarchy directory transfer, the enumeration also returns directory entries, 
+    ///
+    /// In a hierarchy directory transfer, the enumeration also returns directory entries,
     /// it transfers files under the directory and also handles opertions on directories.
     /// </summary>
 #if BINARY_SERIALIZATION
@@ -63,6 +63,7 @@ private ConcurrentDictionary<SubDirectoryTransfer, object> ongoingSubDirTransfer
         private SemaphoreSlim maxConcurrencyControl = null;
 
         private int maxConcurrency = 0;
+        private int? maxListingConcurrency = 0;
 
         object continuationTokenLock = new object();
 
@@ -81,7 +82,7 @@ private ConcurrentDictionary<SubDirectoryTransfer, object> ongoingSubDirTransfer
 
         private DirectoryListingScheduler directoryListingScheduler = null;
 
-        private AzureFileDirectorySDDLCache azureFileDirectorySDDLCache = new AzureFileDirectorySDDLCache(); 
+        private AzureFileDirectorySDDLCache azureFileDirectorySDDLCache = new AzureFileDirectorySDDLCache();
 
 #if !BINARY_SERIALIZATION
         [DataMember]
@@ -283,6 +284,14 @@ public override int MaxTransferConcurrency
             }
         }
 
+        public int? MaxListingConcurrency
+        {
+            set
+            {
+                this.maxListingConcurrency = value;
+            }
+        }
+
         public AzureFileDirectorySDDLCache SDDLCache
         {
             get
@@ -405,7 +414,8 @@ private void ResetExecutionStatus()
 #endif
             }
 
-            directoryListingScheduler = new DirectoryListingScheduler(maxListingThreadCount);
+
+            directoryListingScheduler = new DirectoryListingScheduler(this.maxListingConcurrency ?? maxListingThreadCount);
         }
 
         public override async Task ExecuteInternalAsync(TransferScheduler scheduler, CancellationToken cancellationToken)
@@ -537,7 +547,7 @@ internal async void TransferFile(SingleObjectTransfer transferItem, TransferSche
                         TransferErrorCode.FailToEnumerateDirectory,
                         string.Format(CultureInfo.CurrentCulture,
                             Resources.EnumerateDirectoryException,
-                            this.Destination.Instance.ConvertToString()), 
+                            this.Destination.Instance.ConvertToString()),
                         ex.InnerException);
                 }
 
@@ -867,7 +877,7 @@ protected TransferEntry CreateDirectoryTransferEntry(string relativePath)
             else if (this.Source.Type == TransferLocationType.LocalDirectory)
             {
                 return new DirectoryEntry
-                    (relativePath, 
+                    (relativePath,
                     LongPath.Combine(this.Source.Instance as string, relativePath), null);
             }
             else

lib/TransferJobs/HierarchyDirectoryTransfer.cs

@@ -31,7 +31,7 @@ public static class TransferManager
         /// Transfer scheduler that schedules execution of transfer jobs
         /// </summary>
         private static TransferScheduler scheduler = new TransferScheduler(configurations);
-        
+
         /// <summary>
         /// Stores all running transfers
         /// </summary>
@@ -93,7 +93,7 @@ public static Task UploadAsync(string sourcePath, CloudBlob destBlob, UploadOpti
             if (options != null)
             {
                 destLocation.AccessCondition = options.DestinationAccessCondition;
-            }               
+            }
 
             return UploadInternalAsync(sourceLocation, destLocation, options, context, cancellationToken);
         }
@@ -571,7 +571,7 @@ public static Task DownloadAsync(CloudFile sourceFile, Stream destStream, Downlo
         {
             AzureFileLocation sourceLocation = new AzureFileLocation(sourceFile);
             StreamLocation destLocation = new StreamLocation(destStream);
-            
+
             // Set default request options
             SetDefaultRequestOptions(sourceLocation);
 
@@ -611,7 +611,7 @@ public static Task<TransferStatus> DownloadDirectoryAsync(CloudBlobDirectory sou
             AzureBlobDirectoryLocation sourceLocation = new AzureBlobDirectoryLocation(sourceBlobDir);
             DirectoryLocation destLocation = new DirectoryLocation(destPath);
             AzureBlobEnumerator sourceEnumerator = new AzureBlobEnumerator(sourceLocation);
-            
+
             // Set default request options
             SetDefaultRequestOptions(sourceLocation);
 
@@ -659,7 +659,7 @@ public static async Task<TransferStatus> DownloadDirectoryAsync(CloudFileDirecto
 
             if (options != null)
             {
-                TransferManager.CheckSearchPatternOfAzureFileSource(options);                                
+                TransferManager.CheckSearchPatternOfAzureFileSource(options);
                 sourceLocation.FileRequestOptions.DisableContentMD5Validation = options.DisableContentMD5Validation;
                 FileSecurityOperations.EnableRequiredPrivileges(options.PreserveSMBPermissions, true);
             }
@@ -832,11 +832,11 @@ public static Task CopyAsync(CloudBlob sourceBlob, CloudFile destFile, bool isSe
         /// <returns>A <see cref="Task"/> object that represents the asynchronous operation.</returns>
         [Obsolete("Replaced by overload that takes CopyMethod", false)]
         public static Task CopyAsync(
-            CloudBlob sourceBlob, 
-            CloudFile destFile, 
-            bool isServiceCopy, 
-            CopyOptions options, 
-            SingleTransferContext context, 
+            CloudBlob sourceBlob,
+            CloudFile destFile,
+            bool isServiceCopy,
+            CopyOptions options,
+            SingleTransferContext context,
             CancellationToken cancellationToken)
         {
             return CopyAsync(sourceBlob, destFile, isServiceCopy ? CopyMethod.ServiceSideAsyncCopy : CopyMethod.SyncCopy, options, context, cancellationToken);
@@ -1294,8 +1294,8 @@ public static Task<TransferStatus> CopyDirectoryAsync(CloudBlobDirectory sourceB
                 sourceBlobDir,
                 destBlobDir,
                 isServiceCopy ? CopyMethod.ServiceSideAsyncCopy : CopyMethod.SyncCopy,
-                options, 
-                context, 
+                options,
+                context,
                 cancellationToken);
         }
 
@@ -1673,7 +1673,7 @@ private static async Task<TransferStatus> DownloadDirectoryInternalAsync(Transfe
                     hierarchyDirectoryTransfer.Recursive = options.Recursive;
                 }
             }
-            
+
             if (transfer.SourceEnumerator == null || !AreSameTransferEnumerators(transfer.SourceEnumerator, sourceEnumerator))
             {
                 transfer.SourceEnumerator = sourceEnumerator;
@@ -1685,12 +1685,12 @@ private static async Task<TransferStatus> DownloadDirectoryInternalAsync(Transfe
         }
 
         private static async Task<TransferStatus> CopyDirectoryInternalAsync(
-            TransferLocation sourceLocation, 
-            TransferLocation destLocation, 
-            TransferMethod transferMethod, 
-            ITransferEnumerator sourceEnumerator, 
-            CopyDirectoryOptions options, 
-            DirectoryTransferContext context, 
+            TransferLocation sourceLocation,
+            TransferLocation destLocation,
+            TransferMethod transferMethod,
+            ITransferEnumerator sourceEnumerator,
+            CopyDirectoryOptions options,
+            DirectoryTransferContext context,
             CancellationToken cancellationToken)
         {
             DirectoryTransfer transfer = GetOrCreateDirectoryTransfer(sourceLocation, destLocation, transferMethod, context);
@@ -1782,7 +1782,10 @@ private static DirectoryTransfer GetOrCreateDirectoryTransfer(TransferLocation s
                 if ((sourceLocation.Type == TransferLocationType.AzureFileDirectory)
                     || ((sourceLocation.Type == TransferLocationType.LocalDirectory) && (destLocation.Type == TransferLocationType.AzureFileDirectory)))
                 {
-                    directoryTransfer = new HierarchyDirectoryTransfer(sourceLocation, destLocation, transferMethod);
+                    directoryTransfer = new HierarchyDirectoryTransfer(sourceLocation, destLocation, transferMethod)
+                    {
+                        MaxListingConcurrency = configurations.MaxListingConcurrency
+                    };
                 }
                 else
                 {