New storage feature support (#237)

* Support encryption scope in DMLib.

* Add support for preserving ACL in Azure File copying.

* Fix test issue in cases for preserving SMB permission and attributes in copying.

* Fix issue of there's chance of skipping file/blob in resuming of a service side sync copy.

* Add comments on the MaxListingConcurrency and fix a maxListingConcurrency initialization issue.

* Change SetAttributesCallbackAsync to also output source instance as reference.

* Update DataMovement Library version

* Update samples to reference to latest DMLib and add sample to store file properties and permissions in blob meta data in uploading.

* Update test cases to do more test on SetAttributesCallback.

* Fix illegal charactor issue caused by using UNC path in some environment.

* Resolve comments to avoid duplicated codes.

* Update to referece client library 11.2.2

* Update samples to reference to DMLib 2.0.0

* Update readme, change log and breaking change.

BreakingChanges.txt

@@ -1,3 +1,9 @@
+Tracking Breaking Changes since 1.0.0
+   - Added a parameter in SetAttributesCallbackAsync to represent source instance, changed the delegate definition from: 
+		public delegate Task SetAttributesCallbackAsync(object)
+		to 
+		public delegate Task SetAttributesCallbackAsync(object, object)
+
 Tracking Breaking Changes since 0.12.0
    - Changed parameter isServiceCopy of bool value to copyMethod which is an enumeration value in interfaces of CopyAsync and CopyDirectoryAsync.
 

DMLibTest_NetStandard.sln

@@ -0,0 +1,49 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio Version 16
+VisualStudioVersion = 16.0.29613.14
+MinimumVisualStudioVersion = 10.0.40219.1
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DMLibTest", "test\DMLibTest_NetStandard\DMLibTest.csproj", "{2A4656A4-F744-4653-A9D6-15112E9AB352}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Azure.Storage.DataMovement", "netcore\Microsoft.Azure.Storage.DataMovement\Microsoft.Azure.Storage.DataMovement.csproj", "{D52B1401-CF85-441F-9A19-D8A90010306A}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DMLibTestCodeGen", "test\DMLibTestCodeGen\DMLibTestCodeGen.csproj", "{7018EE4E-D389-424E-A8DD-F9B4FFDA5194}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DMTestLib", "test\DMTestLib\DMTestLib.csproj", "{2C79E154-EFD2-4FFD-8A73-5A62A61BC6E5}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MsTestLib", "test\MsTestLib\MsTestLib.csproj", "{AC39B50F-DC27-4411-9ED4-A4A137190ACB}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|Any CPU = Debug|Any CPU
+		Release|Any CPU = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{2A4656A4-F744-4653-A9D6-15112E9AB352}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{2A4656A4-F744-4653-A9D6-15112E9AB352}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{2A4656A4-F744-4653-A9D6-15112E9AB352}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{2A4656A4-F744-4653-A9D6-15112E9AB352}.Release|Any CPU.Build.0 = Release|Any CPU
+		{D52B1401-CF85-441F-9A19-D8A90010306A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{D52B1401-CF85-441F-9A19-D8A90010306A}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{D52B1401-CF85-441F-9A19-D8A90010306A}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{D52B1401-CF85-441F-9A19-D8A90010306A}.Release|Any CPU.Build.0 = Release|Any CPU
+		{7018EE4E-D389-424E-A8DD-F9B4FFDA5194}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{7018EE4E-D389-424E-A8DD-F9B4FFDA5194}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{7018EE4E-D389-424E-A8DD-F9B4FFDA5194}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{7018EE4E-D389-424E-A8DD-F9B4FFDA5194}.Release|Any CPU.Build.0 = Release|Any CPU
+		{2C79E154-EFD2-4FFD-8A73-5A62A61BC6E5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{2C79E154-EFD2-4FFD-8A73-5A62A61BC6E5}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{2C79E154-EFD2-4FFD-8A73-5A62A61BC6E5}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{2C79E154-EFD2-4FFD-8A73-5A62A61BC6E5}.Release|Any CPU.Build.0 = Release|Any CPU
+		{AC39B50F-DC27-4411-9ED4-A4A137190ACB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{AC39B50F-DC27-4411-9ED4-A4A137190ACB}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{AC39B50F-DC27-4411-9ED4-A4A137190ACB}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{AC39B50F-DC27-4411-9ED4-A4A137190ACB}.Release|Any CPU.Build.0 = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+	GlobalSection(ExtensibilityGlobals) = postSolution
+		SolutionGuid = {1AA31E1E-29C8-467B-B6FE-48F50C90FFCF}
+	EndGlobalSection
+EndGlobal

README.md

@@ -1,4 +1,4 @@
-# Microsoft Azure Storage Data Movement Library (1.3.0)
+# Microsoft Azure Storage Data Movement Library (2.0.0)
 
 The Microsoft Azure Storage Data Movement Library designed for high-performance uploading, downloading and copying Azure Storage Blob and File. This library is based on the core data movement framework that powers [AzCopy](https://azure.microsoft.com/documentation/articles/storage-use-azcopy/).
 

changelog.txt

@@ -1,3 +1,15 @@
+2020.08.31 Version 2.0.0
+ * All Service on both .Net Framework and .Net Core
+   - Upgraded Microsoft.Azure.Storage.Blob from 11.1.2 to 11.2.2
+   - Upgraded Microsoft.Azure.Storage.File from 11.1.2 to 11.2.2
+   - Added a parameter in SetAttributesCallbackAsync to represent source instance.
+ * Blob Service on both .Net Framework and .Net Core
+   - Added support to set encryption scope when destination is Azure Blob Service.
+ * File Service on both .Net Framework and .Net Core
+   - Added support to preserve preserve file permissions and SMB attributes when copying between Azure File Services.
+ * All Service on .Net Core
+   - Fix an issue of throwing out "illegal character" in some environment without UNC path support.
+
 2020.02.26 Version 1.3.0
  * All Service on both .Net Framework and .Net Core
    - Upgraded Microsoft.Azure.Storage.Blob from 11.1.1 to 11.1.2

lib/DataMovement.csproj

@@ -54,14 +54,14 @@
     <Reference Include="Microsoft.Azure.KeyVault.Core, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
       <HintPath>..\packages\Microsoft.Azure.KeyVault.Core.1.0.0\lib\net40\Microsoft.Azure.KeyVault.Core.dll</HintPath>
     </Reference>
-    <Reference Include="Microsoft.Azure.Storage.Blob, Version=11.1.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
-      <HintPath>..\packages\Microsoft.Azure.Storage.Blob.11.1.2\lib\net452\Microsoft.Azure.Storage.Blob.dll</HintPath>
+    <Reference Include="Microsoft.Azure.Storage.Blob, Version=11.2.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
+      <HintPath>..\packages\Microsoft.Azure.Storage.Blob.11.2.2\lib\net452\Microsoft.Azure.Storage.Blob.dll</HintPath>
     </Reference>
-    <Reference Include="Microsoft.Azure.Storage.Common, Version=11.1.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
-      <HintPath>..\packages\Microsoft.Azure.Storage.Common.11.1.2\lib\net452\Microsoft.Azure.Storage.Common.dll</HintPath>
+    <Reference Include="Microsoft.Azure.Storage.Common, Version=11.2.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
+      <HintPath>..\packages\Microsoft.Azure.Storage.Common.11.2.2\lib\net452\Microsoft.Azure.Storage.Common.dll</HintPath>
     </Reference>
-    <Reference Include="Microsoft.Azure.Storage.File, Version=11.1.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
-      <HintPath>..\packages\Microsoft.Azure.Storage.File.11.1.2\lib\net452\Microsoft.Azure.Storage.File.dll</HintPath>
+    <Reference Include="Microsoft.Azure.Storage.File, Version=11.2.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
+      <HintPath>..\packages\Microsoft.Azure.Storage.File.11.2.2\lib\net452\Microsoft.Azure.Storage.File.dll</HintPath>
     </Reference>
     <Reference Include="Microsoft.WindowsAzure.Configuration, Version=1.8.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
       <HintPath>..\packages\Microsoft.WindowsAzure.ConfigurationManager.1.8.0.0\lib\net35-full\Microsoft.WindowsAzure.Configuration.dll</HintPath>

lib/Extensions/StorageExtensions.cs

@@ -41,14 +41,15 @@ internal static bool Equals(
         }
 
         internal static CloudFile GenerateCopySourceFile(
-            this CloudFile file)
+            this CloudFile file,
+            bool preservePermission)
         {
             if (null == file)
             {
                 throw new ArgumentNullException("file");
             }
 
-            string sasToken = GetFileSASToken(file);
+            string sasToken = GetFileSASToken(file, preservePermission);
 
             if (string.IsNullOrEmpty(sasToken))
             {
@@ -58,13 +59,13 @@ internal static CloudFile GenerateCopySourceFile(
             return new CloudFile(file.SnapshotQualifiedUri, new StorageCredentials(sasToken));
         }
 
-        internal static Uri GenerateCopySourceUri(this CloudFile file)
+        internal static Uri GenerateCopySourceUri(this CloudFile file, bool preservePermission = false)
         {
-            CloudFile fileForCopy = file.GenerateCopySourceFile();
+            CloudFile fileForCopy = file.GenerateCopySourceFile(preservePermission);
             return fileForCopy.ServiceClient.Credentials.TransformUri(fileForCopy.SnapshotQualifiedUri);
         }
 
-        private static string GetFileSASToken(CloudFile file)
+        private static string GetFileSASToken(CloudFile file, bool preservePermission)
         {
             if (null == file.ServiceClient.Credentials
                 || file.ServiceClient.Credentials.IsAnonymous)
@@ -85,7 +86,7 @@ private static string GetFileSASToken(CloudFile file)
                 Permissions = SharedAccessFilePermissions.Read,
             };
 
-            return file.GetSharedAccessSignature(policy);
+            return preservePermission ? file.Share.GetSharedAccessSignature(policy) : file.GetSharedAccessSignature(policy);
         }
 
         /// <summary>

lib/FileSecurityOperations.cs

@@ -320,9 +320,6 @@ public static void SetFileSecurity(string filePath, string portableSDDL, Preserv
                 return;
             }
 #endif
-
-            filePath = LongPath.ToUncPath(filePath);
-
             if (PreserveSMBPermissions.None == preserveSMBPermissions) return;
 
             if (string.IsNullOrEmpty(portableSDDL)) return;
@@ -621,8 +618,6 @@ private static string GetFileSDDL(
             string filePath,
             NativeMethods.SECURITY_INFORMATION securityInfo)
         {
-            filePath = LongPath.ToUncPath(filePath);
-
             // Note: to get the SACL, special permissions are needed. Refer https://docs.microsoft.com/en-us/windows/win32/api/aclapi/nf-aclapi-getsecurityinfo
             IntPtr pZero = IntPtr.Zero;
             IntPtr pSid = pZero;

lib/LongPathFile.cs

@@ -15,20 +15,14 @@ internal static partial class LongPathFile
         public static bool Exists(string path)
         {
 #if DOTNET5_4
-            string longFilePath = path;
-            if (Interop.CrossPlatformHelpers.IsWindows)
-            {
-                longFilePath = LongPath.ToUncPath(longFilePath);
-            }
-            return File.Exists(longFilePath);
+            return File.Exists(path);
 #else
             try
             {
                 if (String.IsNullOrEmpty(path))
                 {
                     return false;
                 }
-                path = LongPath.ToUncPath(path);
                 bool success = NativeMethods.PathFileExistsW(path);
                 if (!success)
                 {

lib/LongPathFileStream.cs

@@ -313,7 +313,6 @@ public static bool Exists(string path)
 
                 if (String.IsNullOrEmpty(path))
                     return false;
-                path = LongPath.ToUncPath(path);
                 bool success = NativeMethods.PathFileExistsW(path);
                 if (!success)
                 {
@@ -358,8 +357,6 @@ public static void CreateDirectory(string path)
                 LongPathDirectory.CreateDirectory(dir);
             }
 
-            path = LongPath.ToUncPath(path);
-
             if (!NativeMethods.CreateDirectoryW(path, IntPtr.Zero))
                 NativeMethods.ThrowExceptionForLastWin32ErrorIfExists(new int[] {
                     NativeMethods.ERROR_SUCCESS,
@@ -512,7 +509,6 @@ public static FileStream Open(string filePath, FileMode mode, FileAccess access,
 #if DOTNET5_4
             return new FileStream(filePath, mode, access, share);
 #else
-            filePath = LongPath.ToUncPath(filePath);
             SafeFileHandle fileHandle = GetFileHandle(filePath, mode, access, share);
             return new FileStream(fileHandle, access);
 #endif
@@ -573,7 +569,6 @@ public static void SetAttributes(string path, FileAttributes fileAttributes)
 #if DOTNET5_4
             File.SetAttributes(path, fileAttributes);
 #else
-            path = LongPath.ToUncPath(path);
             if (!NativeMethods.SetFileAttributesW(path, (uint)(fileAttributes)))
             {
                 NativeMethods.ThrowExceptionForLastWin32ErrorIfExists();
@@ -609,8 +604,6 @@ public static void GetFileProperties(string path, out DateTimeOffset? creationTi
             )
         {
 #if !DOTNET5_4
-            path = LongPath.ToUncPath(path);
-
             if (path.EndsWith(Path.DirectorySeparatorChar.ToString(), StringComparison.Ordinal))
             {
                 path = path.Substring(0, path.Length - 1);
@@ -666,7 +659,6 @@ public static void GetFileProperties(string path, out DateTimeOffset? creationTi
         public static void SetFileTime(string path, DateTimeOffset creationTimeUtc, DateTimeOffset lastWriteTimeUtc, bool isDirectory = false)
         {
 #if !DOTNET5_4
-            path = LongPath.ToUncPath(path);
             SafeFileHandle fileHandle = GetFileHandle(path, FileMode.Open, FileAccess.Write, FileShare.None, isDirectory);
 
             try

lib/TransferCallbacks.cs

@@ -32,6 +32,7 @@ public delegate Task<bool> ShouldTransferCallbackAsync(
     /// Callback invoked to set destination's attributes in memory. 
     /// The attributes set in this callback will be sent to azure storage service. 
     /// </summary>
+    /// <param name="source">Source instance in the transfer.</param>
     /// <param name="destination">Instance of destination to be overwritten.</param>
-    public delegate Task SetAttributesCallbackAsync(object destination);
+    public delegate Task SetAttributesCallbackAsync(object source, object destination);
 }

lib/TransferConfigurations.cs

@@ -7,6 +7,7 @@ namespace Microsoft.Azure.Storage.DataMovement
 {
     using System;
     using System.Globalization;
+    using System.IO;
     using System.Reflection;
     using System.Runtime.InteropServices;
     using ClientLibraryConstants = Microsoft.Azure.Storage.Shared.Protocol.Constants;
@@ -46,6 +47,7 @@ public class TransferConfigurations
         /// Initializes a new instance of the
         /// <see cref="TransferConfigurations" /> class.
         /// </summary>
+        [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Design", "CA1031:DoNotCatchGeneralExceptionTypes")]
         public TransferConfigurations()
         {
             // setup default values.
@@ -54,6 +56,19 @@ public TransferConfigurations()
             this.MemoryChunkSize = Constants.DefaultMemoryChunkSize;
 
             this.UpdateMaximumCacheSize(this.blockSize);
+            this.SupportUncPath = false;
+
+            if (Interop.CrossPlatformHelpers.IsWindows)
+            {
+                try
+                {
+                    LongPath.GetFullPath("\\\\?\\F:");
+                    this.SupportUncPath = true;
+                }
+                catch (Exception)
+                {
+                }
+            }
         }
 
         /// <summary>
@@ -83,6 +98,12 @@ public int ParallelOperations
             }
         }
 
+        /// <summary>
+        /// Gets or sets a value indicating how many listing works to process concurrently.
+        /// When source is an Azure File directory or local file directory, DataMovement Library would list the directory in parallel.
+        /// This value is to indicate the maximum number of listing works to process in parallel.
+        /// </summary>
+        /// <value>How many listing works to process concurrently.</value>
         public int? MaxListingConcurrency
         {
             get
@@ -192,6 +213,16 @@ internal long MaximumCacheSize
             }
         }
 
+        /// <summary>
+        /// To indicate whether the process environment supports UNC path.
+        /// 
+        /// On Windows, it requires to use UNC path to access files/directories with long path. 
+        /// In some environment, the .Net Framework only supports legacy path without UNC path support.
+        /// DataMovement Library will detect whether .Net Framework supports UNC path in the process environment 
+        /// to determine whether to use UNC path in the following transfers.
+        /// </summary>
+        internal bool SupportUncPath { get; private set; }
+
         /// <summary>
         /// The size of memory chunk of memory pool
         /// </summary>

lib/TransferControllers/AsyncCopyControllers/BlobAsyncCopyController.cs

@@ -205,7 +205,7 @@ protected override async Task SetAttributesAsync(SetAttributesCallbackAsync setC
             var originalAttributes = Utils.GenerateAttributes(this.destBlob);
             var originalMetadata = new Dictionary<string, string>(this.destBlob.Metadata);
 
-            await setCustomAttributes(this.destBlob);
+            await setCustomAttributes(this.TransferJob.Source.Instance, this.destBlob);
 
             if (!Utils.CompareProperties(originalAttributes, Utils.GenerateAttributes(this.destBlob)))
             {

lib/TransferControllers/AsyncCopyControllers/FileAsyncCopyController.cs

@@ -89,6 +89,7 @@ await this.destFile.StartCopyAsync(
                     this.SourceUri,
                     null,
                     null,
+                    default(FileCopyOptions),
                     Utils.GenerateFileRequestOptions(this.destLocation.FileRequestOptions),
                     operationContext,
                     this.CancellationToken); 
@@ -101,6 +102,7 @@ await this.destFile.StartCopyAsync(
                     this.SourceBlob.GenerateCopySourceUri(),
                     null,
                     null,
+                    default(FileCopyOptions),
                     Utils.GenerateFileRequestOptions(this.destLocation.FileRequestOptions),
                     operationContext,
                     this.CancellationToken); 
@@ -116,10 +118,24 @@ await this.destFile.StartCopyAsync(
             }
             else
             {
+                var transfer = this.TransferJob.Transfer;
+                FileCopyOptions fileCopyOptions = new FileCopyOptions();
+
+                if (transfer.PreserveSMBAttributes)
+                {
+                    fileCopyOptions.PreserveCreationTime = transfer.PreserveSMBAttributes;
+                    fileCopyOptions.PreserveLastWriteTime = transfer.PreserveSMBAttributes;
+                    fileCopyOptions.PreserveNtfsAttributes = transfer.PreserveSMBAttributes;
+                    fileCopyOptions.SetArchive = false;
+                }
+
+                fileCopyOptions.PreservePermissions = (transfer.PreserveSMBPermissions != PreserveSMBPermissions.None);
+
                 await this.destFile.StartCopyAsync(
-                    this.SourceFile.GenerateCopySourceUri(),
+                    this.SourceFile.GenerateCopySourceUri(fileCopyOptions.PreservePermissions),
                     null,
                     null,
+                    fileCopyOptions,
                     Utils.GenerateFileRequestOptions(this.destLocation.FileRequestOptions),
                     operationContext,
                     this.CancellationToken); 
@@ -155,7 +171,7 @@ protected override async Task SetAttributesAsync(SetAttributesCallbackAsync setC
             var originalAttributes = Utils.GenerateAttributes(this.destFile, true);
             var originalMetadata = new Dictionary<string, string>(this.destFile.Metadata);
 
-            await setCustomAttributes(this.destFile);
+            await setCustomAttributes(this.TransferJob.Source.Instance, this.destFile);
 
             if (!Utils.CompareProperties(originalAttributes, Utils.GenerateAttributes(this.destFile, true)))
             {

lib/TransferControllers/DummyTransferController.cs

@@ -70,7 +70,8 @@ await Task.Run(
                         && this.TransferJob.Destination.Type == TransferLocationType.FilePath)
                     {
                         // Dummy transfer for downloading dummy blobs.
-                        var filePath = (this.TransferJob.Destination as FileLocation).FilePath;
+                        var filePath = (this.TransferJob.Destination as FileLocation).FilePath.ToLongPath();
+
                         if (LongPathFile.Exists(filePath))
                         {
                             string exceptionMessage = string.Format(