Add profile for uploading files in a given directory. Add support for maintaining relative directory structure for file uploads.

Author: brdeyo

VERSION

@@ -1 +1 @@
-2.0.28
+2.0.29

src/VirtualClient/Module.props

@@ -83,6 +83,9 @@
         <!-- Microsoft.Windows.Compatibility -->
         <Microsoft_Windows_Compatibility_PackageVersion>9.0.3</Microsoft_Windows_Compatibility_PackageVersion>
 
+		<!-- MimeMapping -->
+		<MimeMapping_PackageVersion>3.1.0</MimeMapping_PackageVersion>
+
         <!-- Moq -->
         <Moq_PackageVersion>4.18.2</Moq_PackageVersion>
 

src/VirtualClient/VirtualClient.Contracts.UnitTests/VirtualClientComponentExtensionsTests.cs

@@ -6,13 +6,15 @@ namespace VirtualClient.Contracts
     using System;
     using System.Collections.Generic;
     using System.Diagnostics;
+    using System.IO;
     using System.Linq;
     using System.Runtime.InteropServices;
     using System.Text;
     using System.Threading;
     using System.Threading.Tasks;
     using Microsoft.Extensions.DependencyInjection;
     using Microsoft.Extensions.DependencyInjection.Extensions;
+    using Moq;
     using NUnit.Framework;
     using VirtualClient.Common.Contracts;
     using VirtualClient.Common.Extensions;
@@ -249,6 +251,200 @@ public void CombineExtensionProducesTheExpectedPathOnUnixSystems()
             }
         }
 
+        [Test]
+        public void CreateCreateFileUploadDescriptorsExtensionCreatesTheExpectedDescriptorsOnUnixSystems_1()
+        {
+            this.fixture.Setup(PlatformID.Unix);
+
+            string directory = "/home/user/Logs";
+            string[] expectedFiles = new string[]
+            {
+                $"{directory}/log1.txt",
+                $"{directory}/log2.txt"
+            };
+
+            this.fixture.FileSystem
+                .Setup(fs => fs.Path.GetDirectoryName(It.IsAny<string>()))
+                .Returns<string>(file => directory);
+
+            this.fixture.FileSystem
+                .Setup(fs => fs.Directory.GetFiles(directory, "*.*", SearchOption.AllDirectories))
+                .Returns(expectedFiles);
+
+            using (var component = new TestVirtualClientComponent(this.fixture))
+            {
+                IEnumerable<FileUploadDescriptor> descriptors = component.CreateFileUploadDescriptors(directory, timestamped: false);
+
+                Assert.IsNotNull(descriptors);
+                Assert.IsTrue(descriptors.Count() == 2);
+
+                FileUploadDescriptor descriptor1 = descriptors.ElementAt(0);
+                Assert.AreEqual(expectedFiles[0], descriptor1.FilePath);
+                Assert.IsTrue(descriptor1.BlobName.EndsWith("log1.txt"));
+                Assert.IsTrue(descriptor1.BlobPath.EndsWith("/log1.txt"));
+                Assert.AreEqual(component.ExperimentId, descriptor1.ContainerName);
+                Assert.AreEqual(Encoding.UTF8.WebName, descriptor1.ContentEncoding);
+                Assert.IsNotNull(descriptor1.ContentType);
+
+                FileUploadDescriptor descriptor2 = descriptors.ElementAt(1);
+                Assert.AreEqual(expectedFiles[1], descriptor2.FilePath);
+                Assert.IsTrue(descriptor2.BlobName.EndsWith("log2.txt"));
+                Assert.IsTrue(descriptor2.BlobPath.EndsWith("/log2.txt"));
+                Assert.AreEqual(component.ExperimentId, descriptor2.ContainerName);
+                Assert.AreEqual(Encoding.UTF8.WebName, descriptor2.ContentEncoding);
+                Assert.IsNotNull(descriptor2.ContentType);
+            }
+        }
+
+        [Test]
+        public void CreateCreateFileUploadDescriptorsExtensionCreatesTheExpectedDescriptorsOnUnixSystems_2()
+        {
+            this.fixture.Setup(PlatformID.Unix);
+
+            string directory = "/home/user/Logs";
+            string[] expectedFiles = new string[]
+            {
+                $"{directory}/log1.txt",
+                $"{directory}/directory2/log2.txt",
+                $"{directory}/directory2/directory3/log3.txt"
+            };
+
+            this.fixture.FileSystem
+                .Setup(fs => fs.Path.GetDirectoryName(It.IsAny<string>()))
+                .Returns<string>(file => file.Replace(Path.GetFileName(file), string.Empty));
+
+            this.fixture.FileSystem
+                .Setup(fs => fs.Directory.GetFiles(directory, "*.*", SearchOption.AllDirectories))
+                .Returns(expectedFiles);
+
+            using (var component = new TestVirtualClientComponent(this.fixture))
+            {
+                IEnumerable<FileUploadDescriptor> descriptors = component.CreateFileUploadDescriptors(directory, timestamped: false);
+
+                Assert.IsNotNull(descriptors);
+                Assert.IsTrue(descriptors.Count() == 3);
+
+                FileUploadDescriptor descriptor1 = descriptors.ElementAt(0);
+                Assert.AreEqual(expectedFiles[0], descriptor1.FilePath);
+                Assert.IsTrue(descriptor1.BlobName.EndsWith("log1.txt"));
+                Assert.IsTrue(descriptor1.BlobPath.EndsWith("/log1.txt"));
+                Assert.AreEqual(component.ExperimentId, descriptor1.ContainerName);
+                Assert.AreEqual(Encoding.UTF8.WebName, descriptor1.ContentEncoding);
+                Assert.IsNotNull(descriptor1.ContentType);
+
+                FileUploadDescriptor descriptor2 = descriptors.ElementAt(1);
+                Assert.AreEqual(expectedFiles[1], descriptor2.FilePath);
+                Assert.IsTrue(descriptor2.BlobName.EndsWith("log2.txt"));
+                Assert.IsTrue(descriptor2.BlobPath.EndsWith("/directory2/log2.txt"));
+                Assert.AreEqual(component.ExperimentId, descriptor2.ContainerName);
+                Assert.AreEqual(Encoding.UTF8.WebName, descriptor2.ContentEncoding);
+                Assert.IsNotNull(descriptor2.ContentType);
+
+                FileUploadDescriptor descriptor3 = descriptors.ElementAt(2);
+                Assert.AreEqual(expectedFiles[2], descriptor3.FilePath);
+                Assert.IsTrue(descriptor3.BlobName.EndsWith("log3.txt"));
+                Assert.IsTrue(descriptor3.BlobPath.EndsWith("/directory2/directory3/log3.txt"));
+                Assert.AreEqual(component.ExperimentId, descriptor3.ContainerName);
+                Assert.AreEqual(Encoding.UTF8.WebName, descriptor3.ContentEncoding);
+                Assert.IsNotNull(descriptor3.ContentType);
+            }
+        }
+
+        [Test]
+        public void CreateCreateFileUploadDescriptorsExtensionCreatesTheExpectedDescriptorsOnWindowsSystems_1()
+        {
+            string directory = "C:\\Users\\User\\Logs";
+            string[] expectedFiles = new string[]
+            {
+                $"{directory}\\log1.txt",
+                $"{directory}\\log2.txt"
+            };
+
+            this.fixture.FileSystem
+                .Setup(fs => fs.Path.GetDirectoryName(It.IsAny<string>()))
+                .Returns<string>(file => directory);
+
+            this.fixture.FileSystem
+                .Setup(fs => fs.Directory.GetFiles(directory, "*.*", SearchOption.AllDirectories))
+                .Returns(expectedFiles);
+
+            using (var component = new TestVirtualClientComponent(this.fixture))
+            {
+                IEnumerable<FileUploadDescriptor> descriptors = component.CreateFileUploadDescriptors(directory, timestamped: false);
+
+                Assert.IsNotNull(descriptors);
+                Assert.IsTrue(descriptors.Count() == 2);
+
+                FileUploadDescriptor descriptor1 = descriptors.ElementAt(0);
+                Assert.AreEqual(expectedFiles[0], descriptor1.FilePath);
+                Assert.IsTrue(descriptor1.BlobName.EndsWith("log1.txt"));
+                Assert.IsTrue(descriptor1.BlobPath.EndsWith("/log1.txt"));
+                Assert.AreEqual(component.ExperimentId, descriptor1.ContainerName);
+                Assert.AreEqual(Encoding.UTF8.WebName, descriptor1.ContentEncoding);
+                Assert.IsNotNull(descriptor1.ContentType);
+
+                FileUploadDescriptor descriptor2 = descriptors.ElementAt(1);
+                Assert.AreEqual(expectedFiles[1], descriptor2.FilePath);
+                Assert.IsTrue(descriptor2.BlobName.EndsWith("log2.txt"));
+                Assert.IsTrue(descriptor2.BlobPath.EndsWith("/log2.txt"));
+                Assert.AreEqual(component.ExperimentId, descriptor2.ContainerName);
+                Assert.AreEqual(Encoding.UTF8.WebName, descriptor2.ContentEncoding);
+                Assert.IsNotNull(descriptor2.ContentType);
+            }
+        }
+
+        [Test]
+        public void CreateCreateFileUploadDescriptorsExtensionCreatesTheExpectedDescriptorsOnWindowsSystems_2()
+        {
+            string directory = "C:\\Users\\User\\Logs";
+            string[] expectedFiles = new string[]
+            {
+                $"{directory}\\log1.txt",
+                $"{directory}\\directory2\\log2.txt",
+                $"{directory}\\directory2\\directory3\\log3.txt"
+            };
+
+            this.fixture.FileSystem
+                .Setup(fs => fs.Path.GetDirectoryName(It.IsAny<string>()))
+                .Returns<string>(file => file.Replace(Path.GetFileName(file), string.Empty));
+
+            this.fixture.FileSystem
+                .Setup(fs => fs.Directory.GetFiles(directory, "*.*", SearchOption.AllDirectories))
+                .Returns(expectedFiles);
+
+            using (var component = new TestVirtualClientComponent(this.fixture))
+            {
+                IEnumerable<FileUploadDescriptor> descriptors = component.CreateFileUploadDescriptors(directory, timestamped: false);
+
+                Assert.IsNotNull(descriptors);
+                Assert.IsTrue(descriptors.Count() == 3);
+
+                FileUploadDescriptor descriptor1 = descriptors.ElementAt(0);
+                Assert.AreEqual(expectedFiles[0], descriptor1.FilePath);
+                Assert.IsTrue(descriptor1.BlobName.EndsWith("log1.txt"));
+                Assert.IsTrue(descriptor1.BlobPath.EndsWith("/log1.txt"));
+                Assert.AreEqual(component.ExperimentId, descriptor1.ContainerName);
+                Assert.AreEqual(Encoding.UTF8.WebName, descriptor1.ContentEncoding);
+                Assert.IsNotNull(descriptor1.ContentType);
+
+                FileUploadDescriptor descriptor2 = descriptors.ElementAt(1);
+                Assert.AreEqual(expectedFiles[1], descriptor2.FilePath);
+                Assert.IsTrue(descriptor2.BlobName.EndsWith("log2.txt"));
+                Assert.IsTrue(descriptor2.BlobPath.EndsWith("/directory2/log2.txt"));
+                Assert.AreEqual(component.ExperimentId, descriptor2.ContainerName);
+                Assert.AreEqual(Encoding.UTF8.WebName, descriptor2.ContentEncoding);
+                Assert.IsNotNull(descriptor2.ContentType);
+
+                FileUploadDescriptor descriptor3 = descriptors.ElementAt(2);
+                Assert.AreEqual(expectedFiles[2], descriptor3.FilePath);
+                Assert.IsTrue(descriptor3.BlobName.EndsWith("log3.txt"));
+                Assert.IsTrue(descriptor3.BlobPath.EndsWith("/directory2/directory3/log3.txt"));
+                Assert.AreEqual(component.ExperimentId, descriptor3.ContainerName);
+                Assert.AreEqual(Encoding.UTF8.WebName, descriptor3.ContentEncoding);
+                Assert.IsNotNull(descriptor3.ContentType);
+            }
+        }
+
         [Test]
         public void VerifyLayoutDefinedExtensionThrowsWhenVerifyingTheEnvironmentLayoutIfItDoesNotExist()
         {

src/VirtualClient/VirtualClient.Contracts/VirtualClient.Contracts.csproj

@@ -9,6 +9,7 @@
     <ItemGroup>
         <PackageReference Include="MathNet.Numerics.Signed" Version="$(MathNet_Numerics_Signed_PackageVersion)" />
         <PackageReference Include="Microsoft.Extensions.Http.Polly" Version="$(Microsoft_Extensions_Http_Polly_PackageVersion)" />
+		<PackageReference Include="MimeMapping " Version="[$(MimeMapping_PackageVersion)]" />
         <PackageReference Include="System.IO.Abstractions" Version="$(System_IO_Abstractions_PackageVersion)" />
         <PackageReference Include="System.IO.FileSystem.Primitives" Version="$(System_IO_FileSystem_Primitives_PackageVersion)" />
         <PackageReference Include="YamlDotNet" Version="$(YamlDotNet_PackageVersion)" />

src/VirtualClient/VirtualClient.Contracts/VirtualClientComponentExtensions.cs

@@ -9,9 +9,11 @@ namespace VirtualClient.Contracts
     using System.IO.Abstractions;
     using System.Linq;
     using System.Runtime.InteropServices;
+    using System.Text;
     using System.Text.RegularExpressions;
     using System.Threading;
     using System.Threading.Tasks;
+    using Microsoft.CodeAnalysis;
     using Microsoft.Extensions.DependencyInjection;
     using Microsoft.Extensions.Logging;
     using Polly;
@@ -128,6 +130,59 @@ public static FileUploadDescriptor CreateFileUploadDescriptor(this VirtualClient
             return descriptor;
         }
 
+        /// <summary>
+        /// Creates a descriptor that can be used to publish a request
+        /// </summary>
+        /// <param name="component">The component requesting the file upload descriptor.</param>
+        /// <param name="targetDirectory">A directory with files to upload.</param>
+        /// <param name="parameters">Parameters related to the component that produced the file (e.g. the parameters from the component).</param>
+        /// <param name="metadata">Additional information and metadata related to the blob/file to include in the descriptor alongside the default manifest information.</param>
+        /// <param name="timestamped">
+        /// True to to include the file creation time in the file name (e.g. 2023-05-21t09-23-30-23813z-file.log). This is explicit to allow for cases where modification of the 
+        /// file name is not desirable. Default = true (timestamped file names).
+        /// </param>
+        public static IEnumerable<FileUploadDescriptor> CreateFileUploadDescriptors(this VirtualClientComponent component, string targetDirectory, IDictionary<string, IConvertible> parameters = null, IDictionary<string, IConvertible> metadata = null, bool timestamped = true)
+        {
+            component.ThrowIfNull(nameof(component));
+            targetDirectory.ThrowIfNullOrWhiteSpace(nameof(targetDirectory));
+
+            IFileSystem fileSystem = component.Dependencies.GetService<IFileSystem>();
+            IEnumerable<string> filesToUpload = fileSystem.Directory.GetFiles(targetDirectory, "*.*", SearchOption.AllDirectories);
+            List<FileUploadDescriptor> descriptors = new List<FileUploadDescriptor>();
+
+            if (filesToUpload?.Any() == true)
+            {
+                foreach (string file in filesToUpload)
+                {
+                    string contentType = MimeMapping.MimeUtility.GetMimeMapping(file);
+                    string subDirectory = fileSystem.Path.GetDirectoryName(file).Substring(targetDirectory.Length)?.Trim('\\')?.Trim('/');
+                    string uploadDirectory = null;
+
+                    if (!string.IsNullOrWhiteSpace(subDirectory))
+                    {
+                        uploadDirectory = subDirectory.Trim('\\').Trim('/');
+                    }
+
+                    FileUploadDescriptor descriptor = VirtualClientComponentExtensions.CreateFileUploadDescriptor(
+                        component,
+                        new FileContext(
+                            fileSystem.FileInfo.New(file),
+                            contentType,
+                            Encoding.UTF8.WebName,
+                            component.ExperimentId,
+                            component.AgentId,
+                            null,
+                            uploadDirectory),
+                        parameters: parameters,
+                        timestamped: timestamped);
+
+                    descriptors.Add(descriptor);
+                }
+            }
+
+            return descriptors;
+        }
+
         /// <summary>
         /// Evaluates each of the parameters provided to the component to replace
         /// supported placeholder expressions (e.g. {PackagePath:anytool} -> replace with path to 'anytool' package).