Skip to content

Commit 4580f93

Browse files
EmmaZhuEmmaZhu
authored
New storage feature support (#237)
* Support encryption scope in DMLib. * Add support for preserving ACL in Azure File copying. * Fix test issue in cases for preserving SMB permission and attributes in copying. * Fix issue of there's chance of skipping file/blob in resuming of a service side sync copy. * Add comments on the MaxListingConcurrency and fix a maxListingConcurrency initialization issue. * Change SetAttributesCallbackAsync to also output source instance as reference. * Update DataMovement Library version * Update samples to reference to latest DMLib and add sample to store file properties and permissions in blob meta data in uploading. * Update test cases to do more test on SetAttributesCallback. * Fix illegal charactor issue caused by using UNC path in some environment. * Resolve comments to avoid duplicated codes. * Update to referece client library 11.2.2 * Update samples to reference to DMLib 2.0.0 * Update readme, change log and breaking change.
1 parent 6412711 commit 4580f93

File tree

92 files changed

+3222
-295
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

92 files changed

+3222
-295
lines changed

‎BreakingChanges.txt‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,9 @@
1+
Tracking Breaking Changes since 1.0.0
2+
- Added a parameter in SetAttributesCallbackAsync to represent source instance, changed the delegate definition from:
3+
public delegate Task SetAttributesCallbackAsync(object)
4+
to
5+
public delegate Task SetAttributesCallbackAsync(object, object)
6+
17
Tracking Breaking Changes since 0.12.0
28
- Changed parameter isServiceCopy of bool value to copyMethod which is an enumeration value in interfaces of CopyAsync and CopyDirectoryAsync.
39

‎DMLibTest_NetStandard.sln‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
2+
Microsoft Visual Studio Solution File, Format Version 12.00
3+
# Visual Studio Version 16
4+
VisualStudioVersion = 16.0.29613.14
5+
MinimumVisualStudioVersion = 10.0.40219.1
6+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DMLibTest", "test\DMLibTest_NetStandard\DMLibTest.csproj", "{2A4656A4-F744-4653-A9D6-15112E9AB352}"
7+
EndProject
8+
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Azure.Storage.DataMovement", "netcore\Microsoft.Azure.Storage.DataMovement\Microsoft.Azure.Storage.DataMovement.csproj", "{D52B1401-CF85-441F-9A19-D8A90010306A}"
9+
EndProject
10+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DMLibTestCodeGen", "test\DMLibTestCodeGen\DMLibTestCodeGen.csproj", "{7018EE4E-D389-424E-A8DD-F9B4FFDA5194}"
11+
EndProject
12+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DMTestLib", "test\DMTestLib\DMTestLib.csproj", "{2C79E154-EFD2-4FFD-8A73-5A62A61BC6E5}"
13+
EndProject
14+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MsTestLib", "test\MsTestLib\MsTestLib.csproj", "{AC39B50F-DC27-4411-9ED4-A4A137190ACB}"
15+
EndProject
16+
Global
17+
GlobalSection(SolutionConfigurationPlatforms) = preSolution
18+
Debug|Any CPU = Debug|Any CPU
19+
Release|Any CPU = Release|Any CPU
20+
EndGlobalSection
21+
GlobalSection(ProjectConfigurationPlatforms) = postSolution
22+
{2A4656A4-F744-4653-A9D6-15112E9AB352}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
23+
{2A4656A4-F744-4653-A9D6-15112E9AB352}.Debug|Any CPU.Build.0 = Debug|Any CPU
24+
{2A4656A4-F744-4653-A9D6-15112E9AB352}.Release|Any CPU.ActiveCfg = Release|Any CPU
25+
{2A4656A4-F744-4653-A9D6-15112E9AB352}.Release|Any CPU.Build.0 = Release|Any CPU
26+
{D52B1401-CF85-441F-9A19-D8A90010306A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
27+
{D52B1401-CF85-441F-9A19-D8A90010306A}.Debug|Any CPU.Build.0 = Debug|Any CPU
28+
{D52B1401-CF85-441F-9A19-D8A90010306A}.Release|Any CPU.ActiveCfg = Release|Any CPU
29+
{D52B1401-CF85-441F-9A19-D8A90010306A}.Release|Any CPU.Build.0 = Release|Any CPU
30+
{7018EE4E-D389-424E-A8DD-F9B4FFDA5194}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
31+
{7018EE4E-D389-424E-A8DD-F9B4FFDA5194}.Debug|Any CPU.Build.0 = Debug|Any CPU
32+
{7018EE4E-D389-424E-A8DD-F9B4FFDA5194}.Release|Any CPU.ActiveCfg = Release|Any CPU
33+
{7018EE4E-D389-424E-A8DD-F9B4FFDA5194}.Release|Any CPU.Build.0 = Release|Any CPU
34+
{2C79E154-EFD2-4FFD-8A73-5A62A61BC6E5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
35+
{2C79E154-EFD2-4FFD-8A73-5A62A61BC6E5}.Debug|Any CPU.Build.0 = Debug|Any CPU
36+
{2C79E154-EFD2-4FFD-8A73-5A62A61BC6E5}.Release|Any CPU.ActiveCfg = Release|Any CPU
37+
{2C79E154-EFD2-4FFD-8A73-5A62A61BC6E5}.Release|Any CPU.Build.0 = Release|Any CPU
38+
{AC39B50F-DC27-4411-9ED4-A4A137190ACB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
39+
{AC39B50F-DC27-4411-9ED4-A4A137190ACB}.Debug|Any CPU.Build.0 = Debug|Any CPU
40+
{AC39B50F-DC27-4411-9ED4-A4A137190ACB}.Release|Any CPU.ActiveCfg = Release|Any CPU
41+
{AC39B50F-DC27-4411-9ED4-A4A137190ACB}.Release|Any CPU.Build.0 = Release|Any CPU
42+
EndGlobalSection
43+
GlobalSection(SolutionProperties) = preSolution
44+
HideSolutionNode = FALSE
45+
EndGlobalSection
46+
GlobalSection(ExtensibilityGlobals) = postSolution
47+
SolutionGuid = {1AA31E1E-29C8-467B-B6FE-48F50C90FFCF}
48+
EndGlobalSection
49+
EndGlobal

‎README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# Microsoft Azure Storage Data Movement Library (1.3.0)
1+
# Microsoft Azure Storage Data Movement Library (2.0.0)
22

33
The Microsoft Azure Storage Data Movement Library designed for high-performance uploading, downloading and copying Azure Storage Blob and File. This library is based on the core data movement framework that powers [AzCopy](https://azure.microsoft.com/documentation/articles/storage-use-azcopy/).
44

‎changelog.txt‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,15 @@
1+
2020.08.31 Version 2.0.0
2+
* All Service on both .Net Framework and .Net Core
3+
- Upgraded Microsoft.Azure.Storage.Blob from 11.1.2 to 11.2.2
4+
- Upgraded Microsoft.Azure.Storage.File from 11.1.2 to 11.2.2
5+
- Added a parameter in SetAttributesCallbackAsync to represent source instance.
6+
* Blob Service on both .Net Framework and .Net Core
7+
- Added support to set encryption scope when destination is Azure Blob Service.
8+
* File Service on both .Net Framework and .Net Core
9+
- Added support to preserve preserve file permissions and SMB attributes when copying between Azure File Services.
10+
* All Service on .Net Core
11+
- Fix an issue of throwing out "illegal character" in some environment without UNC path support.
12+
113
2020.02.26 Version 1.3.0
214
* All Service on both .Net Framework and .Net Core
315
- Upgraded Microsoft.Azure.Storage.Blob from 11.1.1 to 11.1.2

‎lib/DataMovement.csproj‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -54,14 +54,14 @@
5454
<Reference Include="Microsoft.Azure.KeyVault.Core, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
5555
<HintPath>..\packages\Microsoft.Azure.KeyVault.Core.1.0.0\lib\net40\Microsoft.Azure.KeyVault.Core.dll</HintPath>
5656
</Reference>
57-
<Reference Include="Microsoft.Azure.Storage.Blob, Version=11.1.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
58-
<HintPath>..\packages\Microsoft.Azure.Storage.Blob.11.1.2\lib\net452\Microsoft.Azure.Storage.Blob.dll</HintPath>
57+
<Reference Include="Microsoft.Azure.Storage.Blob, Version=11.2.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
58+
<HintPath>..\packages\Microsoft.Azure.Storage.Blob.11.2.2\lib\net452\Microsoft.Azure.Storage.Blob.dll</HintPath>
5959
</Reference>
60-
<Reference Include="Microsoft.Azure.Storage.Common, Version=11.1.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
61-
<HintPath>..\packages\Microsoft.Azure.Storage.Common.11.1.2\lib\net452\Microsoft.Azure.Storage.Common.dll</HintPath>
60+
<Reference Include="Microsoft.Azure.Storage.Common, Version=11.2.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
61+
<HintPath>..\packages\Microsoft.Azure.Storage.Common.11.2.2\lib\net452\Microsoft.Azure.Storage.Common.dll</HintPath>
6262
</Reference>
63-
<Reference Include="Microsoft.Azure.Storage.File, Version=11.1.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
64-
<HintPath>..\packages\Microsoft.Azure.Storage.File.11.1.2\lib\net452\Microsoft.Azure.Storage.File.dll</HintPath>
63+
<Reference Include="Microsoft.Azure.Storage.File, Version=11.2.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
64+
<HintPath>..\packages\Microsoft.Azure.Storage.File.11.2.2\lib\net452\Microsoft.Azure.Storage.File.dll</HintPath>
6565
</Reference>
6666
<Reference Include="Microsoft.WindowsAzure.Configuration, Version=1.8.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
6767
<HintPath>..\packages\Microsoft.WindowsAzure.ConfigurationManager.1.8.0.0\lib\net35-full\Microsoft.WindowsAzure.Configuration.dll</HintPath>

‎lib/Extensions/StorageExtensions.cs‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -41,14 +41,15 @@ internal static bool Equals(
4141
}
4242

4343
internal static CloudFile GenerateCopySourceFile(
44-
this CloudFile file)
44+
this CloudFile file,
45+
bool preservePermission)
4546
{
4647
if (null == file)
4748
{
4849
throw new ArgumentNullException("file");
4950
}
5051

51-
string sasToken = GetFileSASToken(file);
52+
string sasToken = GetFileSASToken(file, preservePermission);
5253

5354
if (string.IsNullOrEmpty(sasToken))
5455
{
@@ -58,13 +59,13 @@ internal static CloudFile GenerateCopySourceFile(
5859
return new CloudFile(file.SnapshotQualifiedUri, new StorageCredentials(sasToken));
5960
}
6061

61-
internal static Uri GenerateCopySourceUri(this CloudFile file)
62+
internal static Uri GenerateCopySourceUri(this CloudFile file, bool preservePermission = false)
6263
{
63-
CloudFile fileForCopy = file.GenerateCopySourceFile();
64+
CloudFile fileForCopy = file.GenerateCopySourceFile(preservePermission);
6465
return fileForCopy.ServiceClient.Credentials.TransformUri(fileForCopy.SnapshotQualifiedUri);
6566
}
6667

67-
private static string GetFileSASToken(CloudFile file)
68+
private static string GetFileSASToken(CloudFile file, bool preservePermission)
6869
{
6970
if (null == file.ServiceClient.Credentials
7071
|| file.ServiceClient.Credentials.IsAnonymous)
@@ -85,7 +86,7 @@ private static string GetFileSASToken(CloudFile file)
8586
Permissions = SharedAccessFilePermissions.Read,
8687
};
8788

88-
return file.GetSharedAccessSignature(policy);
89+
return preservePermission ? file.Share.GetSharedAccessSignature(policy) : file.GetSharedAccessSignature(policy);
8990
}
9091

9192
/// <summary>

‎lib/FileSecurityOperations.cs‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -320,9 +320,6 @@ public static void SetFileSecurity(string filePath, string portableSDDL, Preserv
320320
return;
321321
}
322322
#endif
323-
324-
filePath = LongPath.ToUncPath(filePath);
325-
326323
if (PreserveSMBPermissions.None == preserveSMBPermissions) return;
327324

328325
if (string.IsNullOrEmpty(portableSDDL)) return;
@@ -621,8 +618,6 @@ private static string GetFileSDDL(
621618
string filePath,
622619
NativeMethods.SECURITY_INFORMATION securityInfo)
623620
{
624-
filePath = LongPath.ToUncPath(filePath);
625-
626621
// Note: to get the SACL, special permissions are needed. Refer https://docs.microsoft.com/en-us/windows/win32/api/aclapi/nf-aclapi-getsecurityinfo
627622
IntPtr pZero = IntPtr.Zero;
628623
IntPtr pSid = pZero;

‎lib/LongPathFile.cs‎

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -15,20 +15,14 @@ internal static partial class LongPathFile
1515
public static bool Exists(string path)
1616
{
1717
#if DOTNET5_4
18-
string longFilePath = path;
19-
if (Interop.CrossPlatformHelpers.IsWindows)
20-
{
21-
longFilePath = LongPath.ToUncPath(longFilePath);
22-
}
23-
return File.Exists(longFilePath);
18+
return File.Exists(path);
2419
#else
2520
try
2621
{
2722
if (String.IsNullOrEmpty(path))
2823
{
2924
return false;
3025
}
31-
path = LongPath.ToUncPath(path);
3226
bool success = NativeMethods.PathFileExistsW(path);
3327
if (!success)
3428
{

‎lib/LongPathFileStream.cs‎

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -313,7 +313,6 @@ public static bool Exists(string path)
313313

314314
if (String.IsNullOrEmpty(path))
315315
return false;
316-
path = LongPath.ToUncPath(path);
317316
bool success = NativeMethods.PathFileExistsW(path);
318317
if (!success)
319318
{
@@ -358,8 +357,6 @@ public static void CreateDirectory(string path)
358357
LongPathDirectory.CreateDirectory(dir);
359358
}
360359

361-
path = LongPath.ToUncPath(path);
362-
363360
if (!NativeMethods.CreateDirectoryW(path, IntPtr.Zero))
364361
NativeMethods.ThrowExceptionForLastWin32ErrorIfExists(new int[] {
365362
NativeMethods.ERROR_SUCCESS,
@@ -512,7 +509,6 @@ public static FileStream Open(string filePath, FileMode mode, FileAccess access,
512509
#if DOTNET5_4
513510
return new FileStream(filePath, mode, access, share);
514511
#else
515-
filePath = LongPath.ToUncPath(filePath);
516512
SafeFileHandle fileHandle = GetFileHandle(filePath, mode, access, share);
517513
return new FileStream(fileHandle, access);
518514
#endif
@@ -573,7 +569,6 @@ public static void SetAttributes(string path, FileAttributes fileAttributes)
573569
#if DOTNET5_4
574570
File.SetAttributes(path, fileAttributes);
575571
#else
576-
path = LongPath.ToUncPath(path);
577572
if (!NativeMethods.SetFileAttributesW(path, (uint)(fileAttributes)))
578573
{
579574
NativeMethods.ThrowExceptionForLastWin32ErrorIfExists();
@@ -609,8 +604,6 @@ public static void GetFileProperties(string path, out DateTimeOffset? creationTi
609604
)
610605
{
611606
#if !DOTNET5_4
612-
path = LongPath.ToUncPath(path);
613-
614607
if (path.EndsWith(Path.DirectorySeparatorChar.ToString(), StringComparison.Ordinal))
615608
{
616609
path = path.Substring(0, path.Length - 1);
@@ -666,7 +659,6 @@ public static void GetFileProperties(string path, out DateTimeOffset? creationTi
666659
public static void SetFileTime(string path, DateTimeOffset creationTimeUtc, DateTimeOffset lastWriteTimeUtc, bool isDirectory = false)
667660
{
668661
#if !DOTNET5_4
669-
path = LongPath.ToUncPath(path);
670662
SafeFileHandle fileHandle = GetFileHandle(path, FileMode.Open, FileAccess.Write, FileShare.None, isDirectory);
671663

672664
try

‎lib/TransferCallbacks.cs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ public delegate Task<bool> ShouldTransferCallbackAsync(
3232
/// Callback invoked to set destination's attributes in memory.
3333
/// The attributes set in this callback will be sent to azure storage service.
3434
/// </summary>
35+
/// <param name="source">Source instance in the transfer.</param>
3536
/// <param name="destination">Instance of destination to be overwritten.</param>
36-
public delegate Task SetAttributesCallbackAsync(object destination);
37+
public delegate Task SetAttributesCallbackAsync(object source, object destination);
3738
}

0 commit comments

Comments
 (0)