diff --git a/BreakingChanges.txt b/BreakingChanges.txt index 80fe1d1a..61661bba 100644 --- a/BreakingChanges.txt +++ b/BreakingChanges.txt @@ -1,3 +1,9 @@ +Tracking Breaking Changes since 1.0.0 + - Added a parameter in SetAttributesCallbackAsync to represent source instance, changed the delegate definition from: + public delegate Task SetAttributesCallbackAsync(object) + to + public delegate Task SetAttributesCallbackAsync(object, object) + Tracking Breaking Changes since 0.12.0 - Changed parameter isServiceCopy of bool value to copyMethod which is an enumeration value in interfaces of CopyAsync and CopyDirectoryAsync. diff --git a/DMLibTest_NetStandard.sln b/DMLibTest_NetStandard.sln new file mode 100644 index 00000000..4795393f --- /dev/null +++ b/DMLibTest_NetStandard.sln @@ -0,0 +1,49 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 16 +VisualStudioVersion = 16.0.29613.14 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DMLibTest", "test\DMLibTest_NetStandard\DMLibTest.csproj", "{2A4656A4-F744-4653-A9D6-15112E9AB352}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Azure.Storage.DataMovement", "netcore\Microsoft.Azure.Storage.DataMovement\Microsoft.Azure.Storage.DataMovement.csproj", "{D52B1401-CF85-441F-9A19-D8A90010306A}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DMLibTestCodeGen", "test\DMLibTestCodeGen\DMLibTestCodeGen.csproj", "{7018EE4E-D389-424E-A8DD-F9B4FFDA5194}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DMTestLib", "test\DMTestLib\DMTestLib.csproj", "{2C79E154-EFD2-4FFD-8A73-5A62A61BC6E5}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MsTestLib", "test\MsTestLib\MsTestLib.csproj", "{AC39B50F-DC27-4411-9ED4-A4A137190ACB}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {2A4656A4-F744-4653-A9D6-15112E9AB352}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {2A4656A4-F744-4653-A9D6-15112E9AB352}.Debug|Any CPU.Build.0 = Debug|Any CPU + {2A4656A4-F744-4653-A9D6-15112E9AB352}.Release|Any CPU.ActiveCfg = Release|Any CPU + {2A4656A4-F744-4653-A9D6-15112E9AB352}.Release|Any CPU.Build.0 = Release|Any CPU + {D52B1401-CF85-441F-9A19-D8A90010306A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {D52B1401-CF85-441F-9A19-D8A90010306A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {D52B1401-CF85-441F-9A19-D8A90010306A}.Release|Any CPU.ActiveCfg = Release|Any CPU + {D52B1401-CF85-441F-9A19-D8A90010306A}.Release|Any CPU.Build.0 = Release|Any CPU + {7018EE4E-D389-424E-A8DD-F9B4FFDA5194}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {7018EE4E-D389-424E-A8DD-F9B4FFDA5194}.Debug|Any CPU.Build.0 = Debug|Any CPU + {7018EE4E-D389-424E-A8DD-F9B4FFDA5194}.Release|Any CPU.ActiveCfg = Release|Any CPU + {7018EE4E-D389-424E-A8DD-F9B4FFDA5194}.Release|Any CPU.Build.0 = Release|Any CPU + {2C79E154-EFD2-4FFD-8A73-5A62A61BC6E5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {2C79E154-EFD2-4FFD-8A73-5A62A61BC6E5}.Debug|Any CPU.Build.0 = Debug|Any CPU + {2C79E154-EFD2-4FFD-8A73-5A62A61BC6E5}.Release|Any CPU.ActiveCfg = Release|Any CPU + {2C79E154-EFD2-4FFD-8A73-5A62A61BC6E5}.Release|Any CPU.Build.0 = Release|Any CPU + {AC39B50F-DC27-4411-9ED4-A4A137190ACB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {AC39B50F-DC27-4411-9ED4-A4A137190ACB}.Debug|Any CPU.Build.0 = Debug|Any CPU + {AC39B50F-DC27-4411-9ED4-A4A137190ACB}.Release|Any CPU.ActiveCfg = Release|Any CPU + {AC39B50F-DC27-4411-9ED4-A4A137190ACB}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {1AA31E1E-29C8-467B-B6FE-48F50C90FFCF} + EndGlobalSection +EndGlobal diff --git a/README.md b/README.md index c8db4220..060b0c43 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Microsoft Azure Storage Data Movement Library (1.3.0) +# Microsoft Azure Storage Data Movement Library (2.0.0) The Microsoft Azure Storage Data Movement Library designed for high-performance uploading, downloading and copying Azure Storage Blob and File. This library is based on the core data movement framework that powers [AzCopy](https://azure.microsoft.com/documentation/articles/storage-use-azcopy/). diff --git a/changelog.txt b/changelog.txt index 4a0a8033..19a495ac 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,3 +1,15 @@ +2020.08.31 Version 2.0.0 + * All Service on both .Net Framework and .Net Core + - Upgraded Microsoft.Azure.Storage.Blob from 11.1.2 to 11.2.2 + - Upgraded Microsoft.Azure.Storage.File from 11.1.2 to 11.2.2 + - Added a parameter in SetAttributesCallbackAsync to represent source instance. + * Blob Service on both .Net Framework and .Net Core + - Added support to set encryption scope when destination is Azure Blob Service. + * File Service on both .Net Framework and .Net Core + - Added support to preserve preserve file permissions and SMB attributes when copying between Azure File Services. + * All Service on .Net Core + - Fix an issue of throwing out "illegal character" in some environment without UNC path support. + 2020.02.26 Version 1.3.0 * All Service on both .Net Framework and .Net Core - Upgraded Microsoft.Azure.Storage.Blob from 11.1.1 to 11.1.2 diff --git a/lib/DataMovement.csproj b/lib/DataMovement.csproj index c7f6d3d7..7ee84cae 100644 --- a/lib/DataMovement.csproj +++ b/lib/DataMovement.csproj @@ -54,14 +54,14 @@ ..\packages\Microsoft.Azure.KeyVault.Core.1.0.0\lib\net40\Microsoft.Azure.KeyVault.Core.dll - - ..\packages\Microsoft.Azure.Storage.Blob.11.1.2\lib\net452\Microsoft.Azure.Storage.Blob.dll + + ..\packages\Microsoft.Azure.Storage.Blob.11.2.2\lib\net452\Microsoft.Azure.Storage.Blob.dll - - ..\packages\Microsoft.Azure.Storage.Common.11.1.2\lib\net452\Microsoft.Azure.Storage.Common.dll + + ..\packages\Microsoft.Azure.Storage.Common.11.2.2\lib\net452\Microsoft.Azure.Storage.Common.dll - - ..\packages\Microsoft.Azure.Storage.File.11.1.2\lib\net452\Microsoft.Azure.Storage.File.dll + + ..\packages\Microsoft.Azure.Storage.File.11.2.2\lib\net452\Microsoft.Azure.Storage.File.dll ..\packages\Microsoft.WindowsAzure.ConfigurationManager.1.8.0.0\lib\net35-full\Microsoft.WindowsAzure.Configuration.dll diff --git a/lib/Extensions/StorageExtensions.cs b/lib/Extensions/StorageExtensions.cs index fd2e3fad..e05f666f 100644 --- a/lib/Extensions/StorageExtensions.cs +++ b/lib/Extensions/StorageExtensions.cs @@ -41,14 +41,15 @@ internal static bool Equals( } internal static CloudFile GenerateCopySourceFile( - this CloudFile file) + this CloudFile file, + bool preservePermission) { if (null == file) { throw new ArgumentNullException("file"); } - string sasToken = GetFileSASToken(file); + string sasToken = GetFileSASToken(file, preservePermission); if (string.IsNullOrEmpty(sasToken)) { @@ -58,13 +59,13 @@ internal static CloudFile GenerateCopySourceFile( return new CloudFile(file.SnapshotQualifiedUri, new StorageCredentials(sasToken)); } - internal static Uri GenerateCopySourceUri(this CloudFile file) + internal static Uri GenerateCopySourceUri(this CloudFile file, bool preservePermission = false) { - CloudFile fileForCopy = file.GenerateCopySourceFile(); + CloudFile fileForCopy = file.GenerateCopySourceFile(preservePermission); return fileForCopy.ServiceClient.Credentials.TransformUri(fileForCopy.SnapshotQualifiedUri); } - private static string GetFileSASToken(CloudFile file) + private static string GetFileSASToken(CloudFile file, bool preservePermission) { if (null == file.ServiceClient.Credentials || file.ServiceClient.Credentials.IsAnonymous) @@ -85,7 +86,7 @@ private static string GetFileSASToken(CloudFile file) Permissions = SharedAccessFilePermissions.Read, }; - return file.GetSharedAccessSignature(policy); + return preservePermission ? file.Share.GetSharedAccessSignature(policy) : file.GetSharedAccessSignature(policy); } /// diff --git a/lib/FileSecurityOperations.cs b/lib/FileSecurityOperations.cs index 98238669..d18eb768 100644 --- a/lib/FileSecurityOperations.cs +++ b/lib/FileSecurityOperations.cs @@ -320,9 +320,6 @@ public static void SetFileSecurity(string filePath, string portableSDDL, Preserv return; } #endif - - filePath = LongPath.ToUncPath(filePath); - if (PreserveSMBPermissions.None == preserveSMBPermissions) return; if (string.IsNullOrEmpty(portableSDDL)) return; @@ -621,8 +618,6 @@ private static string GetFileSDDL( string filePath, NativeMethods.SECURITY_INFORMATION securityInfo) { - filePath = LongPath.ToUncPath(filePath); - // Note: to get the SACL, special permissions are needed. Refer https://docs.microsoft.com/en-us/windows/win32/api/aclapi/nf-aclapi-getsecurityinfo IntPtr pZero = IntPtr.Zero; IntPtr pSid = pZero; diff --git a/lib/LongPathFile.cs b/lib/LongPathFile.cs index 779a5127..b3f5db74 100644 --- a/lib/LongPathFile.cs +++ b/lib/LongPathFile.cs @@ -15,12 +15,7 @@ internal static partial class LongPathFile public static bool Exists(string path) { #if DOTNET5_4 - string longFilePath = path; - if (Interop.CrossPlatformHelpers.IsWindows) - { - longFilePath = LongPath.ToUncPath(longFilePath); - } - return File.Exists(longFilePath); + return File.Exists(path); #else try { @@ -28,7 +23,6 @@ public static bool Exists(string path) { return false; } - path = LongPath.ToUncPath(path); bool success = NativeMethods.PathFileExistsW(path); if (!success) { diff --git a/lib/LongPathFileStream.cs b/lib/LongPathFileStream.cs index 47464062..d3a0cdde 100644 --- a/lib/LongPathFileStream.cs +++ b/lib/LongPathFileStream.cs @@ -313,7 +313,6 @@ public static bool Exists(string path) if (String.IsNullOrEmpty(path)) return false; - path = LongPath.ToUncPath(path); bool success = NativeMethods.PathFileExistsW(path); if (!success) { @@ -358,8 +357,6 @@ public static void CreateDirectory(string path) LongPathDirectory.CreateDirectory(dir); } - path = LongPath.ToUncPath(path); - if (!NativeMethods.CreateDirectoryW(path, IntPtr.Zero)) NativeMethods.ThrowExceptionForLastWin32ErrorIfExists(new int[] { NativeMethods.ERROR_SUCCESS, @@ -512,7 +509,6 @@ public static FileStream Open(string filePath, FileMode mode, FileAccess access, #if DOTNET5_4 return new FileStream(filePath, mode, access, share); #else - filePath = LongPath.ToUncPath(filePath); SafeFileHandle fileHandle = GetFileHandle(filePath, mode, access, share); return new FileStream(fileHandle, access); #endif @@ -573,7 +569,6 @@ public static void SetAttributes(string path, FileAttributes fileAttributes) #if DOTNET5_4 File.SetAttributes(path, fileAttributes); #else - path = LongPath.ToUncPath(path); if (!NativeMethods.SetFileAttributesW(path, (uint)(fileAttributes))) { NativeMethods.ThrowExceptionForLastWin32ErrorIfExists(); @@ -609,8 +604,6 @@ public static void GetFileProperties(string path, out DateTimeOffset? creationTi ) { #if !DOTNET5_4 - path = LongPath.ToUncPath(path); - if (path.EndsWith(Path.DirectorySeparatorChar.ToString(), StringComparison.Ordinal)) { path = path.Substring(0, path.Length - 1); @@ -666,7 +659,6 @@ public static void GetFileProperties(string path, out DateTimeOffset? creationTi public static void SetFileTime(string path, DateTimeOffset creationTimeUtc, DateTimeOffset lastWriteTimeUtc, bool isDirectory = false) { #if !DOTNET5_4 - path = LongPath.ToUncPath(path); SafeFileHandle fileHandle = GetFileHandle(path, FileMode.Open, FileAccess.Write, FileShare.None, isDirectory); try diff --git a/lib/TransferCallbacks.cs b/lib/TransferCallbacks.cs index 6d4a5d85..9e21bc3e 100644 --- a/lib/TransferCallbacks.cs +++ b/lib/TransferCallbacks.cs @@ -32,6 +32,7 @@ public delegate Task ShouldTransferCallbackAsync( /// Callback invoked to set destination's attributes in memory. /// The attributes set in this callback will be sent to azure storage service. /// + /// Source instance in the transfer. /// Instance of destination to be overwritten. - public delegate Task SetAttributesCallbackAsync(object destination); + public delegate Task SetAttributesCallbackAsync(object source, object destination); } diff --git a/lib/TransferConfigurations.cs b/lib/TransferConfigurations.cs index 02f0f838..05b5804e 100644 --- a/lib/TransferConfigurations.cs +++ b/lib/TransferConfigurations.cs @@ -7,6 +7,7 @@ namespace Microsoft.Azure.Storage.DataMovement { using System; using System.Globalization; + using System.IO; using System.Reflection; using System.Runtime.InteropServices; using ClientLibraryConstants = Microsoft.Azure.Storage.Shared.Protocol.Constants; @@ -46,6 +47,7 @@ public class TransferConfigurations /// Initializes a new instance of the /// class. /// + [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Design", "CA1031:DoNotCatchGeneralExceptionTypes")] public TransferConfigurations() { // setup default values. @@ -54,6 +56,19 @@ public TransferConfigurations() this.MemoryChunkSize = Constants.DefaultMemoryChunkSize; this.UpdateMaximumCacheSize(this.blockSize); + this.SupportUncPath = false; + + if (Interop.CrossPlatformHelpers.IsWindows) + { + try + { + LongPath.GetFullPath("\\\\?\\F:"); + this.SupportUncPath = true; + } + catch (Exception) + { + } + } } /// @@ -83,6 +98,12 @@ public int ParallelOperations } } + /// + /// Gets or sets a value indicating how many listing works to process concurrently. + /// When source is an Azure File directory or local file directory, DataMovement Library would list the directory in parallel. + /// This value is to indicate the maximum number of listing works to process in parallel. + /// + /// How many listing works to process concurrently. public int? MaxListingConcurrency { get @@ -192,6 +213,16 @@ internal long MaximumCacheSize } } + /// + /// To indicate whether the process environment supports UNC path. + /// + /// On Windows, it requires to use UNC path to access files/directories with long path. + /// In some environment, the .Net Framework only supports legacy path without UNC path support. + /// DataMovement Library will detect whether .Net Framework supports UNC path in the process environment + /// to determine whether to use UNC path in the following transfers. + /// + internal bool SupportUncPath { get; private set; } + /// /// The size of memory chunk of memory pool /// diff --git a/lib/TransferControllers/AsyncCopyControllers/BlobAsyncCopyController.cs b/lib/TransferControllers/AsyncCopyControllers/BlobAsyncCopyController.cs index 04f5d646..5a91ff70 100644 --- a/lib/TransferControllers/AsyncCopyControllers/BlobAsyncCopyController.cs +++ b/lib/TransferControllers/AsyncCopyControllers/BlobAsyncCopyController.cs @@ -205,7 +205,7 @@ protected override async Task SetAttributesAsync(SetAttributesCallbackAsync setC var originalAttributes = Utils.GenerateAttributes(this.destBlob); var originalMetadata = new Dictionary(this.destBlob.Metadata); - await setCustomAttributes(this.destBlob); + await setCustomAttributes(this.TransferJob.Source.Instance, this.destBlob); if (!Utils.CompareProperties(originalAttributes, Utils.GenerateAttributes(this.destBlob))) { diff --git a/lib/TransferControllers/AsyncCopyControllers/FileAsyncCopyController.cs b/lib/TransferControllers/AsyncCopyControllers/FileAsyncCopyController.cs index f75f99a1..de8532a3 100644 --- a/lib/TransferControllers/AsyncCopyControllers/FileAsyncCopyController.cs +++ b/lib/TransferControllers/AsyncCopyControllers/FileAsyncCopyController.cs @@ -89,6 +89,7 @@ await this.destFile.StartCopyAsync( this.SourceUri, null, null, + default(FileCopyOptions), Utils.GenerateFileRequestOptions(this.destLocation.FileRequestOptions), operationContext, this.CancellationToken); @@ -101,6 +102,7 @@ await this.destFile.StartCopyAsync( this.SourceBlob.GenerateCopySourceUri(), null, null, + default(FileCopyOptions), Utils.GenerateFileRequestOptions(this.destLocation.FileRequestOptions), operationContext, this.CancellationToken); @@ -116,10 +118,24 @@ await this.destFile.StartCopyAsync( } else { + var transfer = this.TransferJob.Transfer; + FileCopyOptions fileCopyOptions = new FileCopyOptions(); + + if (transfer.PreserveSMBAttributes) + { + fileCopyOptions.PreserveCreationTime = transfer.PreserveSMBAttributes; + fileCopyOptions.PreserveLastWriteTime = transfer.PreserveSMBAttributes; + fileCopyOptions.PreserveNtfsAttributes = transfer.PreserveSMBAttributes; + fileCopyOptions.SetArchive = false; + } + + fileCopyOptions.PreservePermissions = (transfer.PreserveSMBPermissions != PreserveSMBPermissions.None); + await this.destFile.StartCopyAsync( - this.SourceFile.GenerateCopySourceUri(), + this.SourceFile.GenerateCopySourceUri(fileCopyOptions.PreservePermissions), null, null, + fileCopyOptions, Utils.GenerateFileRequestOptions(this.destLocation.FileRequestOptions), operationContext, this.CancellationToken); @@ -155,7 +171,7 @@ protected override async Task SetAttributesAsync(SetAttributesCallbackAsync setC var originalAttributes = Utils.GenerateAttributes(this.destFile, true); var originalMetadata = new Dictionary(this.destFile.Metadata); - await setCustomAttributes(this.destFile); + await setCustomAttributes(this.TransferJob.Source.Instance, this.destFile); if (!Utils.CompareProperties(originalAttributes, Utils.GenerateAttributes(this.destFile, true))) { diff --git a/lib/TransferControllers/DummyTransferController.cs b/lib/TransferControllers/DummyTransferController.cs index bb000a82..10fb89bd 100644 --- a/lib/TransferControllers/DummyTransferController.cs +++ b/lib/TransferControllers/DummyTransferController.cs @@ -70,7 +70,8 @@ await Task.Run( && this.TransferJob.Destination.Type == TransferLocationType.FilePath) { // Dummy transfer for downloading dummy blobs. - var filePath = (this.TransferJob.Destination as FileLocation).FilePath; + var filePath = (this.TransferJob.Destination as FileLocation).FilePath.ToLongPath(); + if (LongPathFile.Exists(filePath)) { string exceptionMessage = string.Format( diff --git a/lib/TransferControllers/ServiceSideSyncCopyControllers/BlockBlobServiceSideSyncCopyController.cs b/lib/TransferControllers/ServiceSideSyncCopyControllers/BlockBlobServiceSideSyncCopyController.cs index e9c9b9f7..6ed6637a 100644 --- a/lib/TransferControllers/ServiceSideSyncCopyControllers/BlockBlobServiceSideSyncCopyController.cs +++ b/lib/TransferControllers/ServiceSideSyncCopyControllers/BlockBlobServiceSideSyncCopyController.cs @@ -256,7 +256,7 @@ protected override async Task CommitAsync() OverWriteAll = true }); - await this.SetCustomAttributesAsync(this.destBlockBlob); + await this.SetCustomAttributesAsync(this.TransferJob.Source.Instance, this.destBlockBlob); BlobRequestOptions blobRequestOptions = Utils.GenerateBlobRequestOptions(this.destLocation.BlobRequestOptions); OperationContext operationContext = Utils.GenerateOperationContext(this.TransferContext); diff --git a/lib/TransferControllers/ServiceSideSyncCopyControllers/ServiceSideSyncCopyDest/BlobDestHandler.cs b/lib/TransferControllers/ServiceSideSyncCopyControllers/ServiceSideSyncCopyDest/BlobDestHandler.cs index 1e3fd8e3..e1b21cf2 100644 --- a/lib/TransferControllers/ServiceSideSyncCopyControllers/ServiceSideSyncCopyDest/BlobDestHandler.cs +++ b/lib/TransferControllers/ServiceSideSyncCopyControllers/ServiceSideSyncCopyDest/BlobDestHandler.cs @@ -60,37 +60,43 @@ public async Task CheckAndCreateDestinationAsync( { bool needCreateDestination = true; bool gotDestAttributes = false; - if (!isForceOverwrite) + + // Check access condition here no matter whether force overwrite is true. + if (!this.destLocation.CheckedAccessCondition && null != this.destLocation.AccessCondition) { - if (this.transferJob.Overwrite.HasValue) - { - await checkOverwrite(true); - } - else if (!this.destLocation.CheckedAccessCondition && null != this.destLocation.AccessCondition) + try { - try + await this.destBlob.FetchAttributesAsync( + Utils.GenerateConditionWithCustomerCondition(this.destLocation.AccessCondition, false), + Utils.GenerateBlobRequestOptions(this.destLocation.BlobRequestOptions), + Utils.GenerateOperationContext(this.transferContext), + cancellationToken); + + // Only try to send the blob creating request, when blob length is not as expected. Otherwise, only need to clear all pages. + needCreateDestination = true; + this.destLocation.CheckedAccessCondition = true; + gotDestAttributes = true; + + if (!isForceOverwrite) { - await this.destBlob.FetchAttributesAsync( - Utils.GenerateConditionWithCustomerCondition(this.destLocation.AccessCondition, false), - Utils.GenerateBlobRequestOptions(this.destLocation.BlobRequestOptions), - Utils.GenerateOperationContext(this.transferContext), - cancellationToken); - - // Only try to send the blob creating request, when blob length is not as expected. Otherwise, only need to clear all pages. - needCreateDestination = (this.destBlob.Properties.Length != totalLength); - this.destLocation.CheckedAccessCondition = true; - gotDestAttributes = true; - await checkOverwrite(true); } - catch (StorageException se) + } + catch (StorageException se) + { + if ((null == se.RequestInformation) || ((int)HttpStatusCode.NotFound != se.RequestInformation.HttpStatusCode)) { - if ((null == se.RequestInformation) || ((int)HttpStatusCode.NotFound != se.RequestInformation.HttpStatusCode)) - { - throw; - } + throw; } } + } + + if (!isForceOverwrite) + { + if (this.transferJob.Overwrite.HasValue) + { + await checkOverwrite(true); + } else { AccessCondition accessCondition = AccessCondition.GenerateIfNoneMatchCondition("*"); @@ -128,14 +134,13 @@ await this.destBlob.FetchAttributesAsync( if (needCreateDestination) { - await this.CreateDestinationAsync( - totalLength, - Utils.GenerateConditionWithCustomerCondition(this.destLocation.AccessCondition, this.destLocation.CheckedAccessCondition), - cancellationToken); - this.transferJob.Overwrite = true; - this.destLocation.CheckedAccessCondition = true; this.transferJob.Transfer.UpdateJournal(); + + await this.CreateDestinationAsync( + totalLength, + Utils.GenerateConditionWithCustomerCondition(this.destLocation.AccessCondition, true), + CancellationToken.None); } return gotDestAttributes; @@ -144,7 +149,7 @@ await this.CreateDestinationAsync( public virtual async Task CommitAsync( bool gotDestAttributes, Attributes sourceAttributes, - Func setCustomAttributes, + Func setCustomAttributes, CancellationToken cancellationToken) { BlobRequestOptions blobRequestOptions = Utils.GenerateBlobRequestOptions(this.destLocation.BlobRequestOptions); @@ -163,7 +168,7 @@ await this.destBlob.FetchAttributesAsync( Utils.SetAttributes(this.destBlob, sourceAttributes); - await setCustomAttributes(this.destBlob); + await setCustomAttributes(this.transferJob.Source.Instance, this.destBlob); await this.destBlob.SetPropertiesAsync( Utils.GenerateConditionWithCustomerCondition(this.destLocation.AccessCondition), diff --git a/lib/TransferControllers/ServiceSideSyncCopyControllers/ServiceSideSyncCopyDest/FileDestHandler.cs b/lib/TransferControllers/ServiceSideSyncCopyControllers/ServiceSideSyncCopyDest/FileDestHandler.cs index 56edde5f..2bd66ed3 100644 --- a/lib/TransferControllers/ServiceSideSyncCopyControllers/ServiceSideSyncCopyDest/FileDestHandler.cs +++ b/lib/TransferControllers/ServiceSideSyncCopyControllers/ServiceSideSyncCopyDest/FileDestHandler.cs @@ -82,13 +82,13 @@ await this.destFile.FetchAttributesAsync( if (needCreateDestination) { + this.transferJob.Overwrite = true; + this.transferJob.Transfer.UpdateJournal(); + await this.CreateDestinationAsync( totalLength, null, - cancellationToken); - - this.transferJob.Overwrite = true; - this.transferJob.Transfer.UpdateJournal(); + CancellationToken.None); } return gotDestAttributes; @@ -97,7 +97,7 @@ await this.CreateDestinationAsync( public virtual async Task CommitAsync( bool gotDestAttributes, Attributes sourceAttributes, - Func setCustomAttributes, + Func setCustomAttributes, CancellationToken cancellationToken) { FileRequestOptions fileRequestOptions = Utils.GenerateFileRequestOptions(this.destLocation.FileRequestOptions); @@ -114,9 +114,48 @@ await this.destFile.FetchAttributesAsync( var originalMetadata = new Dictionary(this.destFile.Metadata); - Utils.SetAttributes(this.destFile, sourceAttributes, false); + SingleObjectTransfer transferInstance = this.transferJob.Transfer; + + Utils.SetAttributes(this.destFile, sourceAttributes, transferInstance.PreserveSMBAttributes); + + if ((PreserveSMBPermissions.None != transferInstance.PreserveSMBPermissions) + && !string.IsNullOrEmpty(sourceAttributes.PortableSDDL)) + { + if (sourceAttributes.PortableSDDL.Length >= Constants.MaxSDDLLengthInProperties) + { + string permissionKey = null; + var sddlCache = transferInstance.SDDLCache; + if (null != sddlCache) + { + sddlCache.TryGetValue(sourceAttributes.PortableSDDL, out permissionKey); + + if (null == permissionKey) + { + permissionKey = await this.destFile.Share.CreateFilePermissionAsync(sourceAttributes.PortableSDDL, + fileRequestOptions, + operationContext, + cancellationToken).ConfigureAwait(false); + + sddlCache.TryAddValue(sourceAttributes.PortableSDDL, permissionKey); + } + } + else + { + permissionKey = await this.destFile.Share.CreateFilePermissionAsync(sourceAttributes.PortableSDDL, + fileRequestOptions, + operationContext, + cancellationToken).ConfigureAwait(false); + } + + this.destFile.Properties.FilePermissionKey = permissionKey; + } + else + { + this.destFile.FilePermission = sourceAttributes.PortableSDDL; + } + } - await setCustomAttributes(this.destFile); + await setCustomAttributes(this.transferJob.Source.Instance, this.destFile); await this.destFile.SetPropertiesAsync( Utils.GenerateConditionWithCustomerCondition(this.destLocation.AccessCondition), diff --git a/lib/TransferControllers/ServiceSideSyncCopyControllers/ServiceSideSyncCopyDest/IDestHandler.cs b/lib/TransferControllers/ServiceSideSyncCopyControllers/ServiceSideSyncCopyDest/IDestHandler.cs index bd88f785..d70d58c5 100644 --- a/lib/TransferControllers/ServiceSideSyncCopyControllers/ServiceSideSyncCopyDest/IDestHandler.cs +++ b/lib/TransferControllers/ServiceSideSyncCopyControllers/ServiceSideSyncCopyDest/IDestHandler.cs @@ -29,7 +29,7 @@ Task CheckAndCreateDestinationAsync( Task CommitAsync( bool gotDestAttributes, Attributes sourceAttributes, - Func setCustomAttributes, + Func setCustomAttributes, CancellationToken cancellationToken); } } diff --git a/lib/TransferControllers/ServiceSideSyncCopyControllers/ServiceSideSyncCopySource/FileSourceHandler.cs b/lib/TransferControllers/ServiceSideSyncCopyControllers/ServiceSideSyncCopySource/FileSourceHandler.cs index 7d59db66..d8f990e7 100644 --- a/lib/TransferControllers/ServiceSideSyncCopyControllers/ServiceSideSyncCopySource/FileSourceHandler.cs +++ b/lib/TransferControllers/ServiceSideSyncCopyControllers/ServiceSideSyncCopySource/FileSourceHandler.cs @@ -105,7 +105,50 @@ await this.sourceFile.FetchAttributesAsync( throw new InvalidOperationException(Resources.RestartableInfoCorruptedException); } - this.sourceAttributes = Utils.GenerateAttributes(this.sourceFile, true); + SingleObjectTransfer transferInstance = this.transferJob.Transfer; + + this.sourceAttributes = Utils.GenerateAttributes(this.sourceFile, transferInstance.PreserveSMBAttributes); + + if (transferInstance.PreserveSMBPermissions != PreserveSMBPermissions.None) + { + if (!string.IsNullOrEmpty(this.sourceFile.FilePermission)) + { + this.sourceAttributes.PortableSDDL = this.sourceFile.FilePermission; + } + else if (!string.IsNullOrEmpty(this.sourceFile.Properties.FilePermissionKey)) + { + var sddlCache = transferInstance.SDDLCache; + + if (null != sddlCache) + { + string portableSDDL = null; + sddlCache.TryGetValue(this.sourceFile.Properties.FilePermissionKey, out portableSDDL); + + if (null == portableSDDL) + { + portableSDDL = await this.sourceFile.Share.GetFilePermissionAsync(this.sourceFile.Properties.FilePermissionKey, + Utils.GenerateFileRequestOptions(this.sourceLocation.FileRequestOptions), + Utils.GenerateOperationContext(this.transferContext), + cancellationToken).ConfigureAwait(false); + + sddlCache.TryAddValue(this.sourceFile.Properties.FilePermissionKey, portableSDDL); + } + + this.sourceAttributes.PortableSDDL = portableSDDL; + } + else + { + this.sourceAttributes.PortableSDDL = await this.sourceFile.Share.GetFilePermissionAsync(this.sourceFile.Properties.FilePermissionKey, + Utils.GenerateFileRequestOptions(this.sourceLocation.FileRequestOptions), + Utils.GenerateOperationContext(this.transferContext), + cancellationToken).ConfigureAwait(false); + } + } + else + { + this.sourceAttributes.PortableSDDL = null; + } + } this.totalLength = this.sourceFile.Properties.Length; } diff --git a/lib/TransferControllers/ServiceSideSyncCopyControllers/ServiceSideSyncCopySource/ISourceHandler.cs b/lib/TransferControllers/ServiceSideSyncCopyControllers/ServiceSideSyncCopySource/ISourceHandler.cs index fe8ba7ce..ec7207f1 100644 --- a/lib/TransferControllers/ServiceSideSyncCopyControllers/ServiceSideSyncCopySource/ISourceHandler.cs +++ b/lib/TransferControllers/ServiceSideSyncCopyControllers/ServiceSideSyncCopySource/ISourceHandler.cs @@ -37,6 +37,7 @@ Task DownloadRangeToStreamAsync(Stream stream, long TotalLength { get; } string ETag { get; } + AccessCondition AccessCondition { get; } bool NeedToCheckAccessCondition { get; } diff --git a/lib/TransferControllers/TransferControllerBase.cs b/lib/TransferControllers/TransferControllerBase.cs index 91fdf3c6..1ff3c4a5 100644 --- a/lib/TransferControllers/TransferControllerBase.cs +++ b/lib/TransferControllers/TransferControllerBase.cs @@ -440,11 +440,11 @@ public async Task CheckOverwriteAsync( } } - public async Task SetCustomAttributesAsync(object dest) + public async Task SetCustomAttributesAsync(object source, object dest) { if (null != this.TransferContext && null != this.TransferContext.SetAttributesCallbackAsync) { - await this.TransferContext.SetAttributesCallbackAsync(dest); + await this.TransferContext.SetAttributesCallbackAsync(source, dest); } } } diff --git a/lib/TransferControllers/TransferReaders/StreamedReader.cs b/lib/TransferControllers/TransferReaders/StreamedReader.cs index cf4dd247..4d61be4d 100644 --- a/lib/TransferControllers/TransferReaders/StreamedReader.cs +++ b/lib/TransferControllers/TransferReaders/StreamedReader.cs @@ -176,12 +176,9 @@ private async Task OpenInputStreamAsync() throw new TransferException(TransferErrorCode.OpenFileFailed, errorMessage); } - this.filePath = fileLocation.FilePath; + this.filePath = fileLocation.FilePath.ToLongPath(); + #if DOTNET5_4 - if (Interop.CrossPlatformHelpers.IsWindows) - { - filePath = LongPath.ToUncPath(fileLocation.FilePath); - } // Attempt to open the file first so that we throw an exception before getting into the async work this.inputStream = new FileStream( filePath, diff --git a/lib/TransferControllers/TransferWriters/AppendBlobWriter.cs b/lib/TransferControllers/TransferWriters/AppendBlobWriter.cs index 449f4308..d0f0078c 100644 --- a/lib/TransferControllers/TransferWriters/AppendBlobWriter.cs +++ b/lib/TransferControllers/TransferWriters/AppendBlobWriter.cs @@ -427,7 +427,7 @@ await Utils.ExecuteXsclApiCallAsync( var originalMetadata = new Dictionary(this.appendBlob.Metadata); Utils.SetAttributes(this.appendBlob, this.SharedTransferData.Attributes); - await this.Controller.SetCustomAttributesAsync(this.appendBlob); + await this.Controller.SetCustomAttributesAsync(this.SharedTransferData.TransferJob.Source.Instance, this.appendBlob); await this.appendBlob.SetPropertiesAsync( Utils.GenerateConditionWithCustomerCondition(this.destLocation.AccessCondition), diff --git a/lib/TransferControllers/TransferWriters/BlockBlobWriter.cs b/lib/TransferControllers/TransferWriters/BlockBlobWriter.cs index 23b302d9..74f43581 100644 --- a/lib/TransferControllers/TransferWriters/BlockBlobWriter.cs +++ b/lib/TransferControllers/TransferWriters/BlockBlobWriter.cs @@ -392,7 +392,7 @@ private async Task CommitAsync() } Utils.SetAttributes(this.blockBlob, this.SharedTransferData.Attributes); - await this.Controller.SetCustomAttributesAsync(this.blockBlob); + await this.Controller.SetCustomAttributesAsync(this.SharedTransferData.TransferJob.Source.Instance, this.blockBlob); BlobRequestOptions blobRequestOptions = Utils.GenerateBlobRequestOptions(this.destLocation.BlobRequestOptions); OperationContext operationContext = Utils.GenerateOperationContext(this.Controller.TransferContext); @@ -448,7 +448,7 @@ private async Task UploadBlobAndSetAttributesAsync() Utils.SetAttributes(this.blockBlob, this.SharedTransferData.Attributes); - await this.Controller.SetCustomAttributesAsync(this.blockBlob); + await this.Controller.SetCustomAttributesAsync(this.SharedTransferData.TransferJob.Source.Instance, this.blockBlob); await this.DoUploadAndSetBlobAttributes(transferData.Stream); } diff --git a/lib/TransferControllers/TransferWriters/CloudFileWriter.cs b/lib/TransferControllers/TransferWriters/CloudFileWriter.cs index e2cd0326..8a367b02 100644 --- a/lib/TransferControllers/TransferWriters/CloudFileWriter.cs +++ b/lib/TransferControllers/TransferWriters/CloudFileWriter.cs @@ -170,7 +170,7 @@ await this.cloudFile.FetchAttributesAsync( } Utils.SetAttributes(this.cloudFile, this.SharedTransferData.Attributes, this.TransferJob.Transfer.PreserveSMBAttributes); - await this.Controller.SetCustomAttributesAsync(this.cloudFile).ConfigureAwait(false); + await this.Controller.SetCustomAttributesAsync(this.TransferJob.Source.Instance, this.cloudFile).ConfigureAwait(false); await this.cloudFile.SetPropertiesAsync( null, diff --git a/lib/TransferControllers/TransferWriters/PageBlobWriter.cs b/lib/TransferControllers/TransferWriters/PageBlobWriter.cs index 62477da8..feadc9da 100644 --- a/lib/TransferControllers/TransferWriters/PageBlobWriter.cs +++ b/lib/TransferControllers/TransferWriters/PageBlobWriter.cs @@ -139,7 +139,7 @@ await this.pageBlob.FetchAttributesAsync( var originalMetadata = new Dictionary(this.pageBlob.Metadata); Utils.SetAttributes(this.pageBlob, this.SharedTransferData.Attributes); - await this.Controller.SetCustomAttributesAsync(this.pageBlob); + await this.Controller.SetCustomAttributesAsync(this.TransferJob.Source.Instance, this.pageBlob); await this.pageBlob.SetPropertiesAsync( Utils.GenerateConditionWithCustomerCondition(this.destLocation.AccessCondition), diff --git a/lib/TransferControllers/TransferWriters/StreamedWriter.cs b/lib/TransferControllers/TransferWriters/StreamedWriter.cs index 7b8d8e13..f8bea959 100644 --- a/lib/TransferControllers/TransferWriters/StreamedWriter.cs +++ b/lib/TransferControllers/TransferWriters/StreamedWriter.cs @@ -42,6 +42,8 @@ internal sealed class StreamedWriter : TransferReaderWriterBase, IDisposable private string filePath = null; + private string longFilePath = null; + private volatile State state; private volatile bool isStateSwitchedInternal; @@ -166,11 +168,12 @@ await Task.Run(async () => else { this.filePath = (this.TransferJob.Destination as FileLocation).FilePath; + this.longFilePath = this.filePath.ToLongPath(); if (!this.Controller.IsForceOverwrite) { await this.Controller.CheckOverwriteAsync( - LongPathFile.Exists(this.filePath), + LongPathFile.Exists(this.longFilePath), this.TransferJob.Source.Instance, this.filePath); } @@ -182,21 +185,16 @@ await this.Controller.CheckOverwriteAsync( FileMode fileMode = 0 == this.expectOffset ? FileMode.OpenOrCreate : FileMode.Open; #if DOTNET5_4 - string longFilePath = filePath; - if (Interop.CrossPlatformHelpers.IsWindows) - { - longFilePath = LongPath.ToUncPath(longFilePath); - } // Attempt to open the file first so that we throw an exception before getting into the async work this.outputStream = new FileStream( - longFilePath, + this.longFilePath, fileMode, FileAccess.ReadWrite, FileShare.None); #else this.outputStream = LongPathFile.Open( - filePath, + this.longFilePath, fileMode, FileAccess.ReadWrite, FileShare.None); @@ -209,7 +207,7 @@ await this.Controller.CheckOverwriteAsync( string exceptionMessage = string.Format( CultureInfo.CurrentCulture, Resources.FailedToOpenFileException, - filePath, + this.filePath, ex.Message); throw new TransferException( @@ -383,20 +381,20 @@ private void SetHasWorkOrFinished() if (this.TransferJob.Transfer.PreserveSMBAttributes) { if (this.SharedTransferData.Attributes.CloudFileNtfsAttributes.HasValue - && !string.IsNullOrEmpty(this.filePath)) + && !string.IsNullOrEmpty(this.longFilePath)) { - LongPathFile.SetFileTime(this.filePath, this.SharedTransferData.Attributes.CreationTime.Value, this.SharedTransferData.Attributes.LastWriteTime.Value); - LongPathFile.SetAttributes(this.filePath, Utils.AzureFileNtfsAttributesToLocalAttributes(this.SharedTransferData.Attributes.CloudFileNtfsAttributes.Value)); + LongPathFile.SetFileTime(this.longFilePath, this.SharedTransferData.Attributes.CreationTime.Value, this.SharedTransferData.Attributes.LastWriteTime.Value); + LongPathFile.SetAttributes(this.longFilePath, Utils.AzureFileNtfsAttributesToLocalAttributes(this.SharedTransferData.Attributes.CloudFileNtfsAttributes.Value)); } } if ((PreserveSMBPermissions.None != this.TransferJob.Transfer.PreserveSMBPermissions) && (!string.IsNullOrEmpty(this.SharedTransferData.Attributes.PortableSDDL))) { - if (!string.IsNullOrEmpty(this.filePath)) + if (!string.IsNullOrEmpty(this.longFilePath)) { FileSecurityOperations.SetFileSecurity( - this.filePath, + this.longFilePath, this.SharedTransferData.Attributes.PortableSDDL, this.TransferJob.Transfer.PreserveSMBPermissions); } diff --git a/lib/TransferEnumerators/EnumerateDirectoryHelper.cs b/lib/TransferEnumerators/EnumerateDirectoryHelper.cs index 2a799090..4e0f17ab 100644 --- a/lib/TransferEnumerators/EnumerateDirectoryHelper.cs +++ b/lib/TransferEnumerators/EnumerateDirectoryHelper.cs @@ -101,7 +101,9 @@ public static IEnumerable EnumerateAllEntriesInDirectory( string fullPath = null; if (Interop.CrossPlatformHelpers.IsWindows) { - fullPath = LongPath.ToUncPath(path); + fullPath = TransferManager.Configurations.SupportUncPath ? + LongPath.ToUncPath(path) : + LongPath.GetFullPath(path); } else { @@ -235,7 +237,9 @@ public static IEnumerable EnumerateInDirectory( string fullPath = null; if(Interop.CrossPlatformHelpers.IsWindows) { - fullPath = LongPath.ToUncPath(path); + fullPath = TransferManager.Configurations.SupportUncPath ? + LongPath.ToUncPath(path) : + LongPath.GetFullPath(path); } else { @@ -441,8 +445,20 @@ private static IEnumerable InternalEnumerateInDirectory( } // Cross-plat file system accessibility settings may cause exceptions while // retrieving attributes from inaccessible paths. These paths shold be skipped. - catch (FileNotFoundException) { } - catch (IOException) { } + catch (FileNotFoundException ex) + { + if (!TransferManager.Configurations.SupportUncPath) + { + throw new TransferException(string.Format(CultureInfo.CurrentCulture, Resources.FailedToGetFileInfoException, filePath), ex); + } + } + catch (IOException ex) + { + if (!TransferManager.Configurations.SupportUncPath) + { + throw new TransferException(string.Format(CultureInfo.CurrentCulture, Resources.FailedToGetFileInfoException, filePath), ex); + } + } catch (UnauthorizedAccessException) { } catch (Exception ex) { diff --git a/lib/TransferEnumerators/FileEnumerator.cs b/lib/TransferEnumerators/FileEnumerator.cs index 88cdfcef..534717a2 100644 --- a/lib/TransferEnumerators/FileEnumerator.cs +++ b/lib/TransferEnumerators/FileEnumerator.cs @@ -71,19 +71,18 @@ public IEnumerable EnumerateLocation(CancellationToken cancellati Utils.CheckCancellation(cancellationToken); -#if DOTNET5_4 string fullPath = null; if (Interop.CrossPlatformHelpers.IsWindows) { - fullPath = LongPath.ToUncPath(this.location.DirectoryPath); + fullPath = TransferManager.Configurations.SupportUncPath ? + LongPath.ToUncPath(this.location.DirectoryPath) + : LongPath.GetFullPath(this.location.DirectoryPath); } else { fullPath = Path.GetFullPath(this.location.DirectoryPath); } -#else - string fullPath = LongPath.ToUncPath(this.location.DirectoryPath); -#endif + fullPath = AppendDirectorySeparator(fullPath); try diff --git a/lib/TransferEnumerators/FileHierarchyEnumerator.cs b/lib/TransferEnumerators/FileHierarchyEnumerator.cs index 89d9c180..6b4b32d2 100644 --- a/lib/TransferEnumerators/FileHierarchyEnumerator.cs +++ b/lib/TransferEnumerators/FileHierarchyEnumerator.cs @@ -75,23 +75,22 @@ public IEnumerable EnumerateLocation(CancellationToken cancellati Utils.CheckCancellation(cancellationToken); -#if DOTNET5_4 string fullPath = null; string baseFullPath = null; if (Interop.CrossPlatformHelpers.IsWindows) { - fullPath = LongPath.ToUncPath(this.location.DirectoryPath); - baseFullPath = LongPath.ToUncPath(this.baseDirectory); + fullPath = TransferManager.Configurations.SupportUncPath ? + LongPath.ToUncPath(this.location.DirectoryPath) : + LongPath.GetFullPath(this.location.DirectoryPath); + baseFullPath = TransferManager.Configurations.SupportUncPath ? + LongPath.ToUncPath(this.baseDirectory) : + LongPath.GetFullPath(this.baseDirectory); } else { fullPath = Path.GetFullPath(this.location.DirectoryPath); baseFullPath = Path.GetFullPath(this.baseDirectory); } -#else - string fullPath = LongPath.ToUncPath(this.location.DirectoryPath); - string baseFullPath = LongPath.ToUncPath(this.baseDirectory); -#endif fullPath = AppendDirectorySeparator(fullPath); baseFullPath = AppendDirectorySeparator(baseFullPath); diff --git a/lib/TransferJobs/DirectoryLocation.cs b/lib/TransferJobs/DirectoryLocation.cs index f9232e45..686eb0c7 100644 --- a/lib/TransferJobs/DirectoryLocation.cs +++ b/lib/TransferJobs/DirectoryLocation.cs @@ -120,9 +120,11 @@ public override void Validate() DirectoryInfo di = new DirectoryInfo(this.DirectoryPath); di.Create(); #else - if(!LongPathDirectory.Exists(this.DirectoryPath)) + string longDirectoryPath = this.DirectoryPath.ToLongPath(); + + if (!LongPathDirectory.Exists(longDirectoryPath)) { - LongPathDirectory.CreateDirectory(this.DirectoryPath); + LongPathDirectory.CreateDirectory(longDirectoryPath); } #endif } diff --git a/lib/TransferJobs/DirectoryTransfer.cs b/lib/TransferJobs/DirectoryTransfer.cs index cb091593..f287444d 100644 --- a/lib/TransferJobs/DirectoryTransfer.cs +++ b/lib/TransferJobs/DirectoryTransfer.cs @@ -392,6 +392,7 @@ public void CreateParentDirectory(SingleObjectTransfer transfer) case TransferLocationType.FilePath: var filePath = (transfer.Destination as FileLocation).FilePath; Utils.ValidateDestinationPath(transfer.Source.Instance.ConvertToString(), filePath); + filePath = filePath.ToLongPath(); Utils.CreateParentDirectoryIfNotExists(filePath); break; case TransferLocationType.AzureFile: diff --git a/lib/TransferJobs/HierarchyDirectoryTransfer.cs b/lib/TransferJobs/HierarchyDirectoryTransfer.cs index 21c4a73a..949e8095 100644 --- a/lib/TransferJobs/HierarchyDirectoryTransfer.cs +++ b/lib/TransferJobs/HierarchyDirectoryTransfer.cs @@ -63,7 +63,6 @@ private ConcurrentDictionary ongoingSubDirTransfer private SemaphoreSlim maxConcurrencyControl = null; private int maxConcurrency = 0; - private int? maxListingConcurrency = 0; object continuationTokenLock = new object(); @@ -284,14 +283,6 @@ public override int MaxTransferConcurrency } } - public int? MaxListingConcurrency - { - set - { - this.maxListingConcurrency = value; - } - } - public AzureFileDirectorySDDLCache SDDLCache { get @@ -399,23 +390,32 @@ private void ResetExecutionStatus() this.transfersCompleteSource = new TaskCompletionSource(); this.subDirTransfersCompleteSource = new TaskCompletionSource(); + int maxListingThreadCount = 0; + + if (TransferManager.Configurations.MaxListingConcurrency.HasValue) + { + maxListingThreadCount = TransferManager.Configurations.MaxListingConcurrency.Value; + } + else + { #if DOTNET5_4 - int maxListingThreadCount = 6; + maxListingThreadCount = 6; #else - int maxListingThreadCount = 2; + maxListingThreadCount = 2; #endif - if ((this.Destination.Type == TransferLocationType.LocalDirectory) || (this.Source.Type == TransferLocationType.LocalDirectory)) - { + if ((this.Destination.Type == TransferLocationType.LocalDirectory) || (this.Source.Type == TransferLocationType.LocalDirectory)) + { #if DOTNET5_4 - maxListingThreadCount = 4; + maxListingThreadCount = 4; #else - maxListingThreadCount = 2; + maxListingThreadCount = 2; #endif + } } - directoryListingScheduler = new DirectoryListingScheduler(this.maxListingConcurrency ?? maxListingThreadCount); + directoryListingScheduler = new DirectoryListingScheduler(maxListingThreadCount); } public override async Task ExecuteInternalAsync(TransferScheduler scheduler, CancellationToken cancellationToken) @@ -817,7 +817,7 @@ private async void WaitOnSubDirectoryListTask(Task directoryListTask, SubDirecto TransferErrorCode.FailToEnumerateDirectory, string.Format(CultureInfo.CurrentCulture, Resources.EnumerateDirectoryException, - this.Destination.Instance.ConvertToString()), + subDirTransfer.Source.Instance.ConvertToString()), ex); } diff --git a/lib/TransferJobs/SubDirectoryTransfer.cs b/lib/TransferJobs/SubDirectoryTransfer.cs index adac9c22..26b9b987 100644 --- a/lib/TransferJobs/SubDirectoryTransfer.cs +++ b/lib/TransferJobs/SubDirectoryTransfer.cs @@ -123,6 +123,14 @@ public SerializableListContinuationToken ListContinuationToken } } + public TransferLocation Source + { + get + { + return this.source; + } + } + public async Task ExecuteAsync(CancellationToken cancellationToken) { await Task.Yield(); @@ -203,7 +211,7 @@ public void CreateDestinationParentDirectoryRecursively(SingleObjectTransfer tra case TransferLocationType.FilePath: var filePath = (transferItem.Destination as FileLocation).FilePath; Utils.ValidateDestinationPath(transferItem.Source.Instance.ConvertToString(), filePath); - Utils.CreateParentDirectoryIfNotExists(filePath); + Utils.CreateParentDirectoryIfNotExists(filePath.ToLongPath()); break; case TransferLocationType.AzureFile: var parent = (transferItem.Destination as AzureFileLocation).AzureFile.Parent; @@ -273,21 +281,22 @@ private void CreateDestinationDirectory(CancellationToken cancellationToken) if (this.dest.Type == TransferLocationType.LocalDirectory) { - var localFileDestLocation = this.dest as DirectoryLocation; - if (!LongPathDirectory.Exists(localFileDestLocation.DirectoryPath)) + string directoryPath = (this.dest as DirectoryLocation).DirectoryPath.ToLongPath(); + + if (!LongPathDirectory.Exists(directoryPath)) { - LongPathDirectory.CreateDirectory(localFileDestLocation.DirectoryPath); + LongPathDirectory.CreateDirectory(directoryPath); } if (fileAttributes.HasValue) { - LongPathFile.SetFileTime(localFileDestLocation.DirectoryPath, creationTime.Value, lastWriteTime.Value, true); - LongPathFile.SetAttributes(localFileDestLocation.DirectoryPath, Utils.AzureFileNtfsAttributesToLocalAttributes(fileAttributes.Value)); + LongPathFile.SetFileTime(directoryPath, creationTime.Value, lastWriteTime.Value, true); + LongPathFile.SetAttributes(directoryPath, Utils.AzureFileNtfsAttributesToLocalAttributes(fileAttributes.Value)); } if (!string.IsNullOrEmpty(portableSDDL)) { - FileSecurityOperations.SetFileSecurity(localFileDestLocation.DirectoryPath, portableSDDL, this.baseDirectoryTransfer.PreserveSMBPermissions); + FileSecurityOperations.SetFileSecurity(directoryPath, portableSDDL, this.baseDirectoryTransfer.PreserveSMBPermissions); } } else if (this.dest.Type == TransferLocationType.AzureFileDirectory) @@ -349,7 +358,7 @@ private void GetSourceProperites(out CloudFileNtfsAttributes? fileAttributes, || (PreserveSMBPermissions.None != this.baseDirectoryTransfer.PreserveSMBPermissions)) { var sourceLocalDirLocation = this.source as DirectoryLocation; - string directoryPath = sourceLocalDirLocation.DirectoryPath; + string directoryPath = sourceLocalDirLocation.DirectoryPath.ToLongPath(); if (this.baseDirectoryTransfer.PreserveSMBAttributes) { diff --git a/lib/TransferManager.cs b/lib/TransferManager.cs index 7e6031b3..78076a8c 100644 --- a/lib/TransferManager.cs +++ b/lib/TransferManager.cs @@ -93,7 +93,8 @@ public static Task UploadAsync(string sourcePath, CloudBlob destBlob, UploadOpti if (options != null) { destLocation.AccessCondition = options.DestinationAccessCondition; - } + destLocation.BlobRequestOptions.EncryptionScope = options.EncryptionScope; + } return UploadInternalAsync(sourceLocation, destLocation, options, context, cancellationToken); } @@ -143,6 +144,7 @@ public static Task UploadAsync(Stream sourceStream, CloudBlob destBlob, UploadOp if (options != null) { destLocation.AccessCondition = options.DestinationAccessCondition; + destLocation.BlobRequestOptions.EncryptionScope = options.EncryptionScope; } return UploadInternalAsync(sourceLocation, destLocation, options, context, cancellationToken); @@ -329,6 +331,7 @@ public static Task UploadDirectoryAsync(string sourcePath, Cloud { sourceEnumerator.SearchPattern = options.SearchPattern; sourceEnumerator.Recursive = options.Recursive; + destLocation.BlobRequestOptions.EncryptionScope = options.EncryptionScope; } return UploadDirectoryInternalAsync(sourceLocation, destLocation, sourceEnumerator, options, context, cancellationToken); @@ -781,9 +784,10 @@ public static Task CopyAsync(CloudBlob sourceBlob, CloudBlob destBlob, CopyMetho { sourceLocation.AccessCondition = options.SourceAccessCondition; destLocation.AccessCondition = options.DestinationAccessCondition; + destLocation.BlobRequestOptions.EncryptionScope = options.EncryptionScope; } - return CopyInternalAsync(sourceLocation, destLocation, copyMethod, context, cancellationToken); + return CopyInternalAsync(sourceLocation, destLocation, copyMethod, options, context, cancellationToken); } /// @@ -897,7 +901,7 @@ public static Task CopyAsync(CloudBlob sourceBlob, CloudFile destFile, CopyMetho destLocation.AccessCondition = options.DestinationAccessCondition; } - return CopyInternalAsync(sourceLocation, destLocation, copyMethod, context, cancellationToken); + return CopyInternalAsync(sourceLocation, destLocation, copyMethod, options, context, cancellationToken); } /// @@ -1004,9 +1008,10 @@ public static Task CopyAsync(CloudFile sourceFile, CloudBlob destBlob, CopyMetho { sourceLocation.AccessCondition = options.SourceAccessCondition; destLocation.AccessCondition = options.DestinationAccessCondition; + destLocation.BlobRequestOptions.EncryptionScope = options.EncryptionScope; } - return CopyInternalAsync(sourceLocation, destLocation, copyMethod, context, cancellationToken); + return CopyInternalAsync(sourceLocation, destLocation, copyMethod, options, context, cancellationToken); } /// @@ -1115,7 +1120,7 @@ public static Task CopyAsync(CloudFile sourceFile, CloudFile destFile, CopyMetho destLocation.AccessCondition = options.DestinationAccessCondition; } - return CopyInternalAsync(sourceLocation, destLocation, copyMethod, context, cancellationToken); + return CopyInternalAsync(sourceLocation, destLocation, copyMethod, options, context, cancellationToken); } /// @@ -1182,7 +1187,7 @@ public static Task CopyAsync(Uri sourceUri, CloudBlob destBlob, bool isServiceCo destLocation.AccessCondition = options.DestinationAccessCondition; } - return CopyInternalAsync(sourceLocation, destLocation, CopyMethod.ServiceSideAsyncCopy, context, cancellationToken); + return CopyInternalAsync(sourceLocation, destLocation, CopyMethod.ServiceSideAsyncCopy, options, context, cancellationToken); } /// @@ -1249,7 +1254,7 @@ public static Task CopyAsync(Uri sourceUri, CloudFile destFile, bool isServiceCo destLocation.AccessCondition = options.DestinationAccessCondition; } - return CopyInternalAsync(sourceLocation, destLocation, CopyMethod.ServiceSideAsyncCopy, context, cancellationToken); + return CopyInternalAsync(sourceLocation, destLocation, CopyMethod.ServiceSideAsyncCopy, options, context, cancellationToken); } /// @@ -1340,6 +1345,7 @@ public static Task CopyDirectoryAsync(CloudBlobDirectory sourceB sourceEnumerator.SearchPattern = options.SearchPattern; sourceEnumerator.Recursive = options.Recursive; sourceEnumerator.IncludeSnapshots = options.IncludeSnapshots; + destLocation.BlobRequestOptions.EncryptionScope = options.EncryptionScope; } return CopyDirectoryInternalAsync(sourceLocation, destLocation, CopyMethodToTransferMethod(copyMethod), sourceEnumerator, options, context, cancellationToken); @@ -1574,6 +1580,11 @@ public static Task CopyDirectoryAsync(CloudFileDirectory sourceF SetDefaultRequestOptions(sourceLocation); SetDefaultRequestOptions(destLocation); + if (null != options) + { + destLocation.BlobRequestOptions.EncryptionScope = options.EncryptionScope; + } + return CopyDirectoryInternalAsync(sourceLocation, destLocation, CopyMethodToTransferMethod(copyMethod), null, options, context, cancellationToken); } @@ -1610,9 +1621,22 @@ private static Task DownloadInternalAsync(TransferLocation sourceLocation, Trans return DoTransfer(transfer, context, cancellationToken); } - private static Task CopyInternalAsync(TransferLocation sourceLocation, TransferLocation destLocation, CopyMethod copyMethod, TransferContext context, CancellationToken cancellationToken) + private static Task CopyInternalAsync(TransferLocation sourceLocation, TransferLocation destLocation, CopyMethod copyMethod, CopyOptions options, TransferContext context, CancellationToken cancellationToken) { Transfer transfer = GetOrCreateSingleObjectTransfer(sourceLocation, destLocation, CopyMethodToTransferMethod(copyMethod), context); + + if (null != options) + { + if ((sourceLocation.Type == TransferLocationType.AzureFile) + && (destLocation.Type == TransferLocationType.AzureFile)) + { + //By default, copy permissions and SMB attributes for Azure File copying. + transfer.PreserveSMBPermissions = options.PreserveSMBPermissions ? + PreserveSMBPermissions.Owner | PreserveSMBPermissions.Group | PreserveSMBPermissions.DACL | PreserveSMBPermissions.SACL : PreserveSMBPermissions.None; + transfer.PreserveSMBAttributes = options.PreserveSMBAttributes; + } + } + return DoTransfer(transfer, context, cancellationToken); } @@ -1712,6 +1736,16 @@ private static async Task CopyDirectoryInternalAsync( } transfer.BlobType = options.BlobType; transfer.Delimiter = options.Delimiter; + + if ((sourceLocation.Type == TransferLocationType.AzureFileDirectory) + && (destLocation.Type == TransferLocationType.AzureFileDirectory)) + { + //By default, copy permissions and SMB attributes for Azure File copying. + transfer.PreserveSMBPermissions = + options.PreserveSMBPermissions ? + PreserveSMBPermissions.Owner | PreserveSMBPermissions.Group | PreserveSMBPermissions.DACL | PreserveSMBPermissions.SACL : PreserveSMBPermissions.None; + transfer.PreserveSMBAttributes = options.PreserveSMBAttributes; + } } await DoTransfer(transfer, context, cancellationToken); @@ -1782,10 +1816,7 @@ private static DirectoryTransfer GetOrCreateDirectoryTransfer(TransferLocation s if ((sourceLocation.Type == TransferLocationType.AzureFileDirectory) || ((sourceLocation.Type == TransferLocationType.LocalDirectory) && (destLocation.Type == TransferLocationType.AzureFileDirectory))) { - directoryTransfer = new HierarchyDirectoryTransfer(sourceLocation, destLocation, transferMethod) - { - MaxListingConcurrency = configurations.MaxListingConcurrency - }; + directoryTransfer = new HierarchyDirectoryTransfer(sourceLocation, destLocation, transferMethod); } else { diff --git a/lib/TransferOptions/CopyDirectoryOptions.cs b/lib/TransferOptions/CopyDirectoryOptions.cs index d514ea69..70b86f8e 100644 --- a/lib/TransferOptions/CopyDirectoryOptions.cs +++ b/lib/TransferOptions/CopyDirectoryOptions.cs @@ -6,6 +6,7 @@ namespace Microsoft.Azure.Storage.DataMovement { using Microsoft.Azure.Storage.Blob; + using Microsoft.Azure.Storage.File; /// /// Represents a set of options that may be specified for copy directory operation @@ -14,6 +15,22 @@ public sealed class CopyDirectoryOptions : DirectoryOptions { private char delimiter = '/'; + /// + /// Gets or sets a flag that indicates whether to preserve SMB attributes during copying. + /// If set to true, destination Azure File's attributes will be set as source local file's attributes. + /// SMB attributes includes last write time, creation time and . + /// This flag only takes effect when copying from Azure File Service to Azure File Service. + /// + [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "SMB")] + public bool PreserveSMBAttributes { get; set; } + + /// + /// Gets or sets a flag that indicates whether to preserve SMB permissions during copying. + /// This flag only takes effect when copying from Azure File Service to Azure File Service. + /// + [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "SMB")] + public bool PreserveSMBPermissions { get; set; } + /// /// Gets or sets type of destination blob. This option takes effect only when copying from non Azure /// blob storage to Azure blob storage. If blob type is not specified, BlockBlob is used. @@ -45,5 +62,11 @@ public char Delimiter this.delimiter = value; } } + + /// + /// Gets or sets a value which specifies the name of the encryption scope to use to encrypt the data provided in the request. + /// This value only takes effect when destination is Azure Blob Service. + /// + public string EncryptionScope { get; set; } } } diff --git a/lib/TransferOptions/CopyOptions.cs b/lib/TransferOptions/CopyOptions.cs index bd9afd6d..dcb37d57 100644 --- a/lib/TransferOptions/CopyOptions.cs +++ b/lib/TransferOptions/CopyOptions.cs @@ -3,13 +3,31 @@ // Copyright (c) Microsoft Corporation // //------------------------------------------------------------------------------ + namespace Microsoft.Azure.Storage.DataMovement { + using Microsoft.Azure.Storage.File; /// /// Represents a set of options that may be specified for copy operation /// public sealed class CopyOptions { + /// + /// Gets or sets a flag that indicates whether to preserve SMB attributes during copying. + /// If set to true, destination Azure File's attributes will be set as source local file's attributes. + /// SMB attributes includes last write time, creation time and . + /// This flag only takes effect when copying from Azure File Service to Azure File Service. + /// + [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "SMB")] + public bool PreserveSMBAttributes { get; set; } + + /// + /// Gets or sets a flag that indicates whether to preserve SMB permissions during copying. + /// This flag only takes effect when copying from Azure File Service to Azure File Service. + /// + [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "SMB")] + public bool PreserveSMBPermissions { get; set; } + /// /// Gets or sets an object that represents the access conditions for the source object. If null, no condition is used. /// @@ -19,5 +37,11 @@ public sealed class CopyOptions /// Gets or sets an object that represents the access conditions for the destination object. If null, no condition is used. /// public AccessCondition DestinationAccessCondition { get; set; } + + /// + /// Gets or sets a value which specifies the name of the encryption scope to use to encrypt the data provided in the request. + /// This value only takes effect when destination is Azure Blob Service. + /// + public string EncryptionScope { get; set; } } } diff --git a/lib/TransferOptions/UploadDirectoryOptions.cs b/lib/TransferOptions/UploadDirectoryOptions.cs index b671c0fc..85da1320 100644 --- a/lib/TransferOptions/UploadDirectoryOptions.cs +++ b/lib/TransferOptions/UploadDirectoryOptions.cs @@ -104,5 +104,11 @@ public PreserveSMBPermissions PreserveSMBPermissions this.preserveSMBPermissions = value; } } + + /// + /// Gets or sets a value which specifies the name of the encryption scope to use to encrypt the data provided in the request. + /// This value only takes effect when destination is Azure Blob Service. + /// + public string EncryptionScope { get; set; } } } diff --git a/lib/TransferOptions/UploadOptions.cs b/lib/TransferOptions/UploadOptions.cs index 36325726..e74f6ba8 100644 --- a/lib/TransferOptions/UploadOptions.cs +++ b/lib/TransferOptions/UploadOptions.cs @@ -80,5 +80,11 @@ public PreserveSMBPermissions PreserveSMBPermissions this.preserveSMBPermissions = value; } } + + /// + /// Gets or sets a value which specifies the name of the encryption scope to use to encrypt the data provided in the request. + /// This value only takes effect when destination is Azure Blob Service. + /// + public string EncryptionScope { get; set; } } } diff --git a/lib/TransferStatusHelpers/Attributes.cs b/lib/TransferStatusHelpers/Attributes.cs index 6e32041d..841ca22e 100644 --- a/lib/TransferStatusHelpers/Attributes.cs +++ b/lib/TransferStatusHelpers/Attributes.cs @@ -67,10 +67,10 @@ internal class Attributes /// public string PortableSDDL { get; set; } - /// - /// Gets or sets a value to indicate whether to overwrite all attribute on destination, - /// or keep its original value if it's not set. - /// - public bool OverWriteAll { get; set; } + /// + /// Gets or sets a value to indicate whether to overwrite all attribute on destination, + /// or keep its original value if it's not set. + /// + public bool OverWriteAll { get; set; } } } diff --git a/lib/Utils.cs b/lib/Utils.cs index 95ab75bd..cd7a1f30 100644 --- a/lib/Utils.cs +++ b/lib/Utils.cs @@ -356,6 +356,8 @@ public static BlobRequestOptions GenerateBlobRequestOptions( } requestOptions.DisableContentMD5Validation = customRequestOptions.DisableContentMD5Validation; + + requestOptions.EncryptionScope = customRequestOptions.EncryptionScope; } return requestOptions; @@ -874,6 +876,11 @@ public static CloudFileNtfsAttributes LocalAttributesToAzureFileNtfsAttributes(F return cloudFileNtfsAttributes; } + public static string ToLongPath(this string originPath) + { + return TransferManager.Configurations.SupportUncPath ? LongPath.ToUncPath(originPath) : originPath; + } + private static bool IsValidWindowsFileName(string fileName) { string fileNameNoExt = LongPath.GetFileNameWithoutExtension(fileName); diff --git a/lib/packages.config b/lib/packages.config index 8ec1615d..3cf9614d 100644 --- a/lib/packages.config +++ b/lib/packages.config @@ -1,9 +1,9 @@  - - - + + + diff --git a/netcore/DMLibTest/DMLibTest.csproj b/netcore/DMLibTest/DMLibTest.csproj index ad928312..7d3a870f 100644 --- a/netcore/DMLibTest/DMLibTest.csproj +++ b/netcore/DMLibTest/DMLibTest.csproj @@ -33,8 +33,8 @@ - - + + diff --git a/netcore/DMLibTest/TestClassFixtures.cs b/netcore/DMLibTest/TestClassFixtures.cs index 79f2f5e6..d00098a1 100644 --- a/netcore/DMLibTest/TestClassFixtures.cs +++ b/netcore/DMLibTest/TestClassFixtures.cs @@ -108,6 +108,19 @@ public void Dispose() } } + public class EncryptionScopeTestFixture : IDisposable + { + public EncryptionScopeTestFixture() + { + EncryptionScopeTest.MyClassInitialize(null); + } + + public void Dispose() + { + EncryptionScopeTest.MyClassCleanup(); + } + } + public class FollowSymlinkTestFixture : IDisposable { public FollowSymlinkTestFixture() diff --git a/netcore/Microsoft.Azure.Storage.DataMovement/Microsoft.Azure.Storage.DataMovement.csproj b/netcore/Microsoft.Azure.Storage.DataMovement/Microsoft.Azure.Storage.DataMovement.csproj index 2aec7d19..71e04f76 100644 --- a/netcore/Microsoft.Azure.Storage.DataMovement/Microsoft.Azure.Storage.DataMovement.csproj +++ b/netcore/Microsoft.Azure.Storage.DataMovement/Microsoft.Azure.Storage.DataMovement.csproj @@ -2,7 +2,7 @@ Microsoft.WindowsAzure.Storage.DataMovement Class Library - 1.3.0.0 + 2.0.0.0 Microsoft netstandard2.0 true @@ -37,8 +37,8 @@ - - + + diff --git a/samples/DataMovementSamples/DataMovementSamples/DataMovementSamples.csproj b/samples/DataMovementSamples/DataMovementSamples/DataMovementSamples.csproj index 442446ab..f28bd223 100644 --- a/samples/DataMovementSamples/DataMovementSamples/DataMovementSamples.csproj +++ b/samples/DataMovementSamples/DataMovementSamples/DataMovementSamples.csproj @@ -36,17 +36,17 @@ ..\packages\Microsoft.Azure.KeyVault.Core.1.0.0\lib\net40\Microsoft.Azure.KeyVault.Core.dll - - ..\packages\Microsoft.Azure.Storage.Blob.11.1.2\lib\net452\Microsoft.Azure.Storage.Blob.dll + + ..\packages\Microsoft.Azure.Storage.Blob.11.2.2\lib\net452\Microsoft.Azure.Storage.Blob.dll - - ..\packages\Microsoft.Azure.Storage.Common.11.1.2\lib\net452\Microsoft.Azure.Storage.Common.dll + + ..\packages\Microsoft.Azure.Storage.Common.11.2.2\lib\net452\Microsoft.Azure.Storage.Common.dll - - ..\packages\Microsoft.Azure.Storage.DataMovement.1.3.0\lib\net452\Microsoft.Azure.Storage.DataMovement.dll + + ..\packages\Microsoft.Azure.Storage.DataMovement.2.0.0\lib\net452\Microsoft.Azure.Storage.DataMovement.dll - - ..\packages\Microsoft.Azure.Storage.File.11.1.2\lib\net452\Microsoft.Azure.Storage.File.dll + + ..\packages\Microsoft.Azure.Storage.File.11.2.2\lib\net452\Microsoft.Azure.Storage.File.dll ..\packages\Microsoft.WindowsAzure.ConfigurationManager.1.8.0.0\lib\net35-full\Microsoft.WindowsAzure.Configuration.dll diff --git a/samples/DataMovementSamples/DataMovementSamples/Samples.cs b/samples/DataMovementSamples/DataMovementSamples/Samples.cs index 8eaf2af0..80f52d75 100644 --- a/samples/DataMovementSamples/DataMovementSamples/Samples.cs +++ b/samples/DataMovementSamples/DataMovementSamples/Samples.cs @@ -98,7 +98,7 @@ private static async Task BlobUploadSample() UploadOptions options = new UploadOptions(); SingleTransferContext context = new SingleTransferContext(); - context.SetAttributesCallbackAsync = async (destination) => + context.SetAttributesCallbackAsync = async (source, destination) => { CloudBlob destBlob = destination as CloudBlob; destBlob.Properties.ContentType = "image/png"; @@ -377,7 +377,7 @@ private static async Task BlobDirectoryUploadSample() context.FileFailed += FileFailedCallback; context.FileSkipped += FileSkippedCallback; - context.SetAttributesCallbackAsync = async (destination) => + context.SetAttributesCallbackAsync = async (source, destination) => { CloudBlob destBlob = destination as CloudBlob; destBlob.Properties.ContentType = "image/png"; @@ -420,7 +420,7 @@ private static async Task BlobDirectoryUploadSample() resumeContext.FileFailed += FileFailedCallback; resumeContext.FileSkipped += FileSkippedCallback; - resumeContext.SetAttributesCallbackAsync = async (destination) => + resumeContext.SetAttributesCallbackAsync = async (source, destination) => { CloudBlob destBlob = destination as CloudBlob; destBlob.Properties.ContentType = "image/png"; diff --git a/samples/DataMovementSamples/DataMovementSamples/packages.config b/samples/DataMovementSamples/DataMovementSamples/packages.config index fb88746d..17253a78 100644 --- a/samples/DataMovementSamples/DataMovementSamples/packages.config +++ b/samples/DataMovementSamples/DataMovementSamples/packages.config @@ -1,10 +1,10 @@  - - - - + + + + diff --git a/samples/DataMovementSamples/netcore/DataMovementSamples/DataMovementSamples.csproj b/samples/DataMovementSamples/netcore/DataMovementSamples/DataMovementSamples.csproj index 9803b084..7bcf9e5f 100644 --- a/samples/DataMovementSamples/netcore/DataMovementSamples/DataMovementSamples.csproj +++ b/samples/DataMovementSamples/netcore/DataMovementSamples/DataMovementSamples.csproj @@ -25,7 +25,7 @@ - + diff --git a/samples/PreserveFilePropertiesSamples/PreserveFilePropertiesSamples.sln b/samples/PreserveFilePropertiesSamples/PreserveFilePropertiesSamples.sln new file mode 100644 index 00000000..331873af --- /dev/null +++ b/samples/PreserveFilePropertiesSamples/PreserveFilePropertiesSamples.sln @@ -0,0 +1,25 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 16 +VisualStudioVersion = 16.0.29613.14 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PreserveFilePropertiesSamples", "PreserveFilePropertiesSamples\PreserveFilePropertiesSamples.csproj", "{BEE26B27-C556-4811-B445-979C5395AA74}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {BEE26B27-C556-4811-B445-979C5395AA74}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {BEE26B27-C556-4811-B445-979C5395AA74}.Debug|Any CPU.Build.0 = Debug|Any CPU + {BEE26B27-C556-4811-B445-979C5395AA74}.Release|Any CPU.ActiveCfg = Release|Any CPU + {BEE26B27-C556-4811-B445-979C5395AA74}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {EA7938F8-3BA4-45E2-8B44-42938FB55E22} + EndGlobalSection +EndGlobal diff --git a/samples/PreserveFilePropertiesSamples/PreserveFilePropertiesSamples/App.config b/samples/PreserveFilePropertiesSamples/PreserveFilePropertiesSamples/App.config new file mode 100644 index 00000000..c5c58280 --- /dev/null +++ b/samples/PreserveFilePropertiesSamples/PreserveFilePropertiesSamples/App.config @@ -0,0 +1,13 @@ + + + + + + + + + + + diff --git a/samples/PreserveFilePropertiesSamples/PreserveFilePropertiesSamples/FileNativeMethods.cs b/samples/PreserveFilePropertiesSamples/PreserveFilePropertiesSamples/FileNativeMethods.cs new file mode 100644 index 00000000..5db48c78 --- /dev/null +++ b/samples/PreserveFilePropertiesSamples/PreserveFilePropertiesSamples/FileNativeMethods.cs @@ -0,0 +1,110 @@ +//------------------------------------------------------------------------------ +// +// Copyright (c) Microsoft Corporation +// +//------------------------------------------------------------------------------ + +namespace PreserveFilePropertiesSamples +{ + using System; + using System.IO; + using System.Linq; + using System.Runtime.InteropServices; + using Microsoft.Win32.SafeHandles; + + static class FileNativeMethods + { +#if !DOTNET5_4 + public const int ERROR_SUCCESS = 0; + public const int ERROR_FILE_NOT_FOUND = 2; + public const int ERROR_DIRECTORY_NOT_FOUND = 3; + public const int ERROR_NO_MORE_FILES = 18; + public const int ERROR_ALREADY_EXISTS = 183; + public const int ERROR_HANDLE_EOF = 38; + public const uint FILE_BEGIN = 0; + public const uint INVALID_SET_FILE_POINTER = 0xFFFFFFFF; + + public const uint GENERIC_READ = 0x80000000; + public const uint GENERIC_WRITE = 0x40000000; + public const uint GENERIC_READ_WRITE = GENERIC_READ | GENERIC_WRITE; + + // This flag must be set to obtain a handle to a directory. + public const uint FILE_FLAG_BACKUP_SEMANTICS = 0x02000000; + + // Open or create file + [DllImport("kernel32.dll", EntryPoint = "CreateFileW", SetLastError = true, CharSet = CharSet.Unicode)] + public static extern SafeFileHandle GetFileHandleW( + [MarshalAs(UnmanagedType.LPWStr)] string filename, + [MarshalAs(UnmanagedType.U4)] uint access, + [MarshalAs(UnmanagedType.U4)] FileShare share, + IntPtr securityAttributes, + [MarshalAs(UnmanagedType.U4)] FileMode creationDisposition, + [MarshalAs(UnmanagedType.U4)] uint flagsAndAttributes, + IntPtr templateFile); + + [DllImport("kernel32.dll", EntryPoint = "GetFileAttributesW", SetLastError = true, CharSet = CharSet.Unicode)] + public static extern uint GetFileAttributesW(string lpFileName); + + [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Usage", "CA2205:UseManagedEquivalentsOfWin32Api")] + [DllImport("kernel32.dll", EntryPoint = "SetFileAttributesW", SetLastError = true, CharSet = CharSet.Unicode)] + [return: MarshalAs(UnmanagedType.Bool)] + public static extern bool SetFileAttributesW(string lpFileName, uint dwFileAttributes); + + [StructLayout(LayoutKind.Sequential)] + public struct FILETIME + { + public uint dwLowDateTime; + public uint dwHighDateTime; + }; + + [DllImport("kernel32.dll", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + public static extern bool SetFileTime( + SafeFileHandle hFile, + ref FILETIME lpCreationTime, + ref FILETIME lpLastAccessTime, + ref FILETIME lpLastWriteTime); + + /// + /// Throw exception if last Win32 error is not zero. + /// + public static void ThrowExceptionForLastWin32ErrorIfExists() + { + ThrowExceptionForLastWin32ErrorIfExists(new int[] { + ERROR_SUCCESS + }); + } + + /// + /// Throw exception if last Win32 error is not expected. + /// + /// Error codes that are expected. + public static void ThrowExceptionForLastWin32ErrorIfExists(int[] expectErrorCodes) + { + int errorCode = Marshal.GetLastWin32Error(); + + if (expectErrorCodes != null + && expectErrorCodes.Contains(errorCode)) + { + return; + } + throw Marshal.GetExceptionForHR(Marshal.GetHRForLastWin32Error()); + } + + /// + /// Throw exception if the Win32 error given is not expected. + /// + /// Win32 error code want to check. + /// Error codes that are expected. + public static void ThrowExceptionForLastWin32ErrorIfExists(int errorCode, int[] expectErrorCodes) + { + if (expectErrorCodes != null + && expectErrorCodes.Contains(errorCode)) + { + return; + } + throw Marshal.GetExceptionForHR(Marshal.GetHRForLastWin32Error()); + } +#endif + } +} diff --git a/samples/PreserveFilePropertiesSamples/PreserveFilePropertiesSamples/FileOperations.cs b/samples/PreserveFilePropertiesSamples/PreserveFilePropertiesSamples/FileOperations.cs new file mode 100644 index 00000000..1e9f0912 --- /dev/null +++ b/samples/PreserveFilePropertiesSamples/PreserveFilePropertiesSamples/FileOperations.cs @@ -0,0 +1,55 @@ +//------------------------------------------------------------------------------ +// +// Copyright (c) Microsoft Corporation +// +//------------------------------------------------------------------------------ + +namespace PreserveFilePropertiesSamples +{ + using System; + using System.IO; + + static class FileOperations + { + public static void GetFileProperties( + string path, + out DateTimeOffset? creationTime, + out DateTimeOffset? lastWriteTime, + out FileAttributes? fileAttributes, + bool isDirectory = false) + { + fileAttributes = File.GetAttributes(path); + + if (isDirectory) + { + creationTime = Directory.GetCreationTimeUtc(path); + lastWriteTime = Directory.GetLastWriteTimeUtc(path); + } + else + { + creationTime = File.GetCreationTimeUtc(path); + lastWriteTime = File.GetLastWriteTimeUtc(path); + } + } + + public static void SetFileProperties(string path, + DateTimeOffset creationTime, + DateTimeOffset lastWriteTime, + FileAttributes fileAttributes, + bool isDirectory = false) + { + File.SetAttributes(path, fileAttributes); + + if (isDirectory) + { + Directory.SetCreationTimeUtc(path, creationTime.UtcDateTime); + Directory.SetLastWriteTimeUtc(path, lastWriteTime.UtcDateTime); + } + else + { + File.SetCreationTimeUtc(path, creationTime.UtcDateTime); + File.SetLastWriteTimeUtc(path, lastWriteTime.UtcDateTime); + } + } + } +} diff --git a/samples/PreserveFilePropertiesSamples/PreserveFilePropertiesSamples/FileSecurityNativeMethods.cs b/samples/PreserveFilePropertiesSamples/PreserveFilePropertiesSamples/FileSecurityNativeMethods.cs new file mode 100644 index 00000000..a3083827 --- /dev/null +++ b/samples/PreserveFilePropertiesSamples/PreserveFilePropertiesSamples/FileSecurityNativeMethods.cs @@ -0,0 +1,172 @@ +//------------------------------------------------------------------------------ +// +// Copyright (c) Microsoft Corporation +// +//------------------------------------------------------------------------------ + +namespace PreserveFilePropertiesSamples +{ + using System; + using System.Runtime.InteropServices; + using System.Text; + + static class FileSecurityNativeMethods + { + public const int SE_PRIVILEGE_ENABLED = 0x00000002; + public const int SE_PRIVILEGE_DISABLED = 0x00000000; + + public const int ERROR_ACCESS_DENIED = 5; + public const int ERROR_NOT_ALL_ASSIGNED = 1300; + public const int ERROR_INVALID_OWNER = 1307; + public const int ERROR_PRIVILEGE_NOT_HELD = 1314; + public const int ERROR_INSUFFICIENT_BUFFER = 122; + public const int SDDL_REVISION_1 = 1; + + public const string SACLPrivilegeName = "SeSecurityPrivilege"; + public const string OwnerPrivilegeName = "SeRestorePrivilege"; + + [Flags] + public enum SECURITY_INFORMATION : uint + { + OWNER_SECURITY_INFORMATION = 0x00000001, + GROUP_SECURITY_INFORMATION = 0x00000002, + DACL_SECURITY_INFORMATION = 0x00000004, + SACL_SECURITY_INFORMATION = 0x00000008, + UNPROTECTED_SACL_SECURITY_INFORMATION = 0x10000000, + UNPROTECTED_DACL_SECURITY_INFORMATION = 0x20000000, + PROTECTED_SACL_SECURITY_INFORMATION = 0x40000000, + PROTECTED_DACL_SECURITY_INFORMATION = 0x80000000 + } + + public enum SE_OBJECT_TYPE + { + SE_UNKNOWN_OBJECT_TYPE = 0, + SE_FILE_OBJECT, + SE_SERVICE, + SE_PRINTER, + SE_REGISTRY_KEY, + SE_LMSHARE, + SE_KERNEL_OBJECT, + SE_WINDOW_OBJECT, + SE_DS_OBJECT, + SE_DS_OBJECT_ALL, + SE_PROVIDER_DEFINED_OBJECT, + SE_WMIGUID_OBJECT, + SE_REGISTRY_WOW64_32KEY + } + + public enum SID_NAME_USE + { + SidTypeUser, + SidTypeGroup, + SidTypeDomain, + SidTypeAlias, + SidTypeWellKnownGroup, + SidTypeDeletedAccount, + SidTypeInvalid, + SidTypeUnknown, + SidTypeComputer, + SidTypeLabel, + SidTypeLogonSession + }; + + [StructLayout(LayoutKind.Sequential)] + public struct LUID + { + public UInt32 LowPart; + public Int32 HighPart; + } + + [StructLayout(LayoutKind.Sequential, Pack = 4)] + public struct LUID_AND_ATTRIBUTES + { + public LUID Luid; + public UInt32 Attributes; + } + + public struct TOKEN_PRIVILEGES + { + public int PrivilegeCount; + [MarshalAs(UnmanagedType.ByValArray, SizeConst = 1)] + public LUID_AND_ATTRIBUTES[] Privileges; + } + + #region Get/Set FileDDL + [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Usage", "CA2205:UseManagedEquivalentsOfWin32Api")] + [DllImport("advapi32.dll", CharSet = CharSet.Unicode, SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + public static extern bool GetUserName(StringBuilder sb, ref Int32 length); + + [DllImport("advapi32.dll", CharSet = CharSet.Unicode, SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + public static extern bool LookupAccountName( + string lpSystemName, + string lpAccountName, + [MarshalAs(UnmanagedType.LPArray)] byte[] Sid, + ref uint cbSid, + StringBuilder ReferencedDomainName, + ref uint cchReferencedDomainName, + out SID_NAME_USE peUse); + + [DllImport("advapi32", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + public static extern bool GetWindowsAccountDomainSid( + [MarshalAs(UnmanagedType.LPArray)] byte[] sid, + [MarshalAs(UnmanagedType.LPArray)] byte[] domainSid, + ref uint length); + + [DllImport("advapi32.dll", EntryPoint = "GetNamedSecurityInfoW", SetLastError = true, CharSet = CharSet.Unicode)] + public static extern uint GetNamedSecurityInfoW( + string pObjectName, + SE_OBJECT_TYPE ObjectType, + SECURITY_INFORMATION SecurityInfo, + out IntPtr pSidOwner, + out IntPtr pSidGroup, + out IntPtr pDacl, + out IntPtr pSacl, + out IntPtr pSecurityDescriptor); + + [DllImport("Advapi32.dll", CharSet = CharSet.Unicode, SetLastError = true)] + public static extern void SetFileSecurity(string path, int type_of_sd, IntPtr sd); + + [DllImport("kernel32.dll", EntryPoint = "LocalFree", SetLastError = true)] + public static extern IntPtr LocalFree(IntPtr handle); + + [DllImport("advapi32", CharSet = CharSet.Unicode, SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + public static extern bool ConvertSidToStringSid([MarshalAs(UnmanagedType.LPArray)] byte[] sid, out IntPtr ptrSid); + + // Call LocalFree to free SecurityDescriptor pointer. + [DllImport("advapi32.dll", SetLastError = true, CharSet = CharSet.Unicode)] + [return: MarshalAs(UnmanagedType.Bool)] + public static extern bool ConvertStringSecurityDescriptorToSecurityDescriptor( + string StringSecurityDescriptor, + uint StringSDRevision, + out IntPtr SecurityDescriptor, + out UIntPtr SecurityDescriptorSize); + + [DllImport("advapi32.dll", SetLastError = true, CharSet = CharSet.Unicode)] + [return: MarshalAs(UnmanagedType.Bool)] + public static extern bool ConvertSecurityDescriptorToStringSecurityDescriptor( + IntPtr SecurityDescriptor, + uint StringSDRevision, + SECURITY_INFORMATION SecurityInformation, + out IntPtr StringSecurityDescriptor, + out UIntPtr StringSecurityDescriptorLen); + #endregion + + [DllImport("advapi32.dll", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + public static extern bool AdjustTokenPrivileges(IntPtr TokenHandle, + [MarshalAs(UnmanagedType.Bool)]bool DisableAllPrivileges, + ref TOKEN_PRIVILEGES NewState, + UInt32 BufferLengthInBytes, + ref TOKEN_PRIVILEGES PreviousState, + out UInt32 ReturnLengthInBytes); + + [DllImport("advapi32.dll", CharSet = CharSet.Unicode, SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + public static extern bool LookupPrivilegeValue(string lpSystemName, string lpName, + ref LUID lpLuid); + } +} diff --git a/samples/PreserveFilePropertiesSamples/PreserveFilePropertiesSamples/FileSecurityOperations.cs b/samples/PreserveFilePropertiesSamples/PreserveFilePropertiesSamples/FileSecurityOperations.cs new file mode 100644 index 00000000..61667a38 --- /dev/null +++ b/samples/PreserveFilePropertiesSamples/PreserveFilePropertiesSamples/FileSecurityOperations.cs @@ -0,0 +1,582 @@ +//------------------------------------------------------------------------------ +// +// Copyright (c) Microsoft Corporation +// +//------------------------------------------------------------------------------ + +namespace PreserveFilePropertiesSamples +{ + using System; + using System.Collections.Generic; + using System.ComponentModel; + using System.Linq; + using System.Runtime.InteropServices; + using System.Security.Principal; + using System.Text; + using Microsoft.Azure.Storage.DataMovement; + + class FileSecurityOperations + { + private static bool OwnerPrivilegeEnabledOriginal = false; + private static bool SACLPrivilegeEnabledOriginal = false; + + /// + /// Enable privileges required for getting/setting premissions from/to local file. + /// + /// Permissions types to be preserved. + /// Indicating whether to set permission to local file. + /// Setting owner info to local file would need specific privilege, while getting owner info doesn't require this privilege. + public static void EnableRequiredPrivileges(PreserveSMBPermissions preserveSMBPermissions, bool setLocalFilePermission) + { + if (PreserveSMBPermissions.SACL == (preserveSMBPermissions & PreserveSMBPermissions.SACL)) + { + if (!SACLPrivilegeEnabledOriginal) + { + try + { + SACLPrivilegeEnabledOriginal = SetPrivilege(FileSecurityNativeMethods.SACLPrivilegeName, true); + } + catch (Exception) + { + throw; + } + } + } + + if (!setLocalFilePermission) + { + return; + } + + if (PreserveSMBPermissions.Owner == (preserveSMBPermissions & PreserveSMBPermissions.Owner)) + { + if (!OwnerPrivilegeEnabledOriginal) + { + + try + { + OwnerPrivilegeEnabledOriginal = SetPrivilege(FileSecurityNativeMethods.OwnerPrivilegeName, true); + } + catch (COMException) + { + // Ignore exception here + // Under some condition, setting owner info to local file will still success even when there's no SeRestorePrivilege privilege. + // So here only try to enable the privilege. Later, if it fails to set owner info to local file, reports error for the specific file. + return; + } + catch (TransferException) + { + // Ignore exception here + // Under some condition, setting owner info to local file will still success even when there's no SeRestorePrivilege privilege. + // So here only try to enable the privilege. Later, if it fails to set owner info to local file, reports error for the specific file. + return; + } + } + } + } + + /// + /// Restore privileges enabled before for getting/setting local file permissions. + /// + /// Permissions types to be preserved. + /// Indicating whether to set permission to local file. + /// Setting owner info to local file would need specific privilege, while getting owner info doesn't require this privilege. + public static void RestorePrivileges(PreserveSMBPermissions preserveSMBPermissions, bool setLocalFilePermission) + { + if (PreserveSMBPermissions.SACL == (preserveSMBPermissions & PreserveSMBPermissions.SACL)) + { + if (!SACLPrivilegeEnabledOriginal) + { + try + { + SetPrivilege(FileSecurityNativeMethods.SACLPrivilegeName, false); + } + catch (Win32Exception) + { + // Ignore the exception + // Here just try to clear up the privileges which may have been enabled at the very beginning of a tranfer job. + // Failure on disabling the privilege won't impact transfer result. + } + catch (COMException) + { + // Ignore the exception + // Here just try to clear up the privileges which may have been enabled at the very beginning of a tranfer job. + // Failure on disabling the privilege won't impact transfer result. + } + } + } + + if (!setLocalFilePermission) + { + return; + } + + if (PreserveSMBPermissions.Owner == (preserveSMBPermissions & PreserveSMBPermissions.Owner)) + { + if (!OwnerPrivilegeEnabledOriginal) + { + try + { + SetPrivilege(FileSecurityNativeMethods.OwnerPrivilegeName, false); + } + catch (Win32Exception) + { + // Ignore the exception + // Here just try to clear up the privileges which may have been enabled at the very beginning of a tranfer job. + // Failure on disabling the privilege won't impact transfer result. + } + catch (COMException) + { + // Ignore the exception + // Here just try to clear up the privileges which may have been enabled at the very beginning of a tranfer job. + // Failure on disabling the privilege won't impact transfer result. + } + } + } + } + + [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Interoperability", "CA1404:CallGetLastErrorImmediatelyAfterPInvoke")] + private static bool SetPrivilege( + string securityPrivilege, + bool bEnablePrivilege // to enable or disable privilege + ) + { + var identity = WindowsIdentity.GetCurrent(TokenAccessLevels.AdjustPrivileges | TokenAccessLevels.Query); + var accessToken = identity.Token; + + FileSecurityNativeMethods.TOKEN_PRIVILEGES tp; + FileSecurityNativeMethods.LUID luid = new FileSecurityNativeMethods.LUID(); + + if (!FileSecurityNativeMethods.LookupPrivilegeValue( + null, // lookup privilege on local system + securityPrivilege, // privilege to lookup + ref luid)) // receives LUID of privilege + { + throw Marshal.GetExceptionForHR(Marshal.GetHRForLastWin32Error()); + } + + tp.PrivilegeCount = 1; + tp.Privileges = new FileSecurityNativeMethods.LUID_AND_ATTRIBUTES[1]; + tp.Privileges[0].Luid = luid; + if (bEnablePrivilege) + tp.Privileges[0].Attributes = FileSecurityNativeMethods.SE_PRIVILEGE_ENABLED; + else + tp.Privileges[0].Attributes = FileSecurityNativeMethods.SE_PRIVILEGE_DISABLED; + + FileSecurityNativeMethods.TOKEN_PRIVILEGES previousPrivileges = new FileSecurityNativeMethods.TOKEN_PRIVILEGES(); + uint previousPrivilegesLength = 0; + + // Enable the privilege or disable all privileges. + bool succeeded = FileSecurityNativeMethods.AdjustTokenPrivileges( + accessToken, + false, + ref tp, + 1024, + ref previousPrivileges, + out previousPrivilegesLength); + + if (!succeeded) + { + throw Marshal.GetExceptionForHR(Marshal.GetHRForLastWin32Error()); + } + else if (Marshal.GetLastWin32Error() == FileSecurityNativeMethods.ERROR_NOT_ALL_ASSIGNED) + { + throw new Win32Exception(Marshal.GetLastWin32Error()); + } + + if ((previousPrivileges.Privileges.Count() == 1) + && (previousPrivileges.Privileges[0].Luid.LowPart == luid.LowPart) + && (previousPrivileges.Privileges[0].Luid.HighPart == luid.HighPart) + && (previousPrivileges.Privileges[0].Attributes == FileSecurityNativeMethods.SE_PRIVILEGE_ENABLED)) + { + return true; + } + else + { + return false; + } + } + + public static void SetFileSecurity(string filePath, string portableSDDL, PreserveSMBPermissions preserveSMBPermissions) + { + if (PreserveSMBPermissions.None == preserveSMBPermissions) return; + + if (string.IsNullOrEmpty(portableSDDL)) return; + + IntPtr pSecurityDescriptor = new IntPtr(); + UIntPtr securityDescriptorLength = new UIntPtr(); + + if (!FileSecurityNativeMethods.ConvertStringSecurityDescriptorToSecurityDescriptor(portableSDDL, + FileSecurityNativeMethods.SDDL_REVISION_1, + out pSecurityDescriptor, + out securityDescriptorLength)) + { + throw Marshal.GetExceptionForHR(Marshal.GetHRForLastWin32Error()); + } + + try + { + FileSecurityNativeMethods.SetFileSecurity(filePath, + (int)(ToSecurityInfo(preserveSMBPermissions)), + pSecurityDescriptor); + + int errorCode = Marshal.GetLastWin32Error(); + int hresult = Marshal.GetHRForLastWin32Error(); + + if ((errorCode == FileSecurityNativeMethods.ERROR_PRIVILEGE_NOT_HELD) + || (errorCode == FileSecurityNativeMethods.ERROR_ACCESS_DENIED) + || (errorCode == FileSecurityNativeMethods.ERROR_INVALID_OWNER)) + { + throw new Win32Exception(errorCode); + } + + if (errorCode != 0) + { + Marshal.ThrowExceptionForHR(hresult); + } + } + finally + { + FileSecurityNativeMethods.LocalFree(pSecurityDescriptor); + } + } + + public static string GetFilePortableSDDL( + string filePath, + PreserveSMBPermissions preserveSMBPermissions) + { + if (preserveSMBPermissions == PreserveSMBPermissions.None) + { + return null; + } + + string sddl = GetFileSDDL(filePath, ToSecurityInfo(preserveSMBPermissions)); + string userName = GetUserName(); + string domainSid = GetDomainSid(userName); + return GetPortableSDDL(sddl, domainSid); + } + + private static FileSecurityNativeMethods.SECURITY_INFORMATION ToSecurityInfo(PreserveSMBPermissions preserveSMBPermissions) + { + uint securityInfo = 0; + + if (PreserveSMBPermissions.Owner == (preserveSMBPermissions & PreserveSMBPermissions.Owner)) + securityInfo |= (uint)FileSecurityNativeMethods.SECURITY_INFORMATION.OWNER_SECURITY_INFORMATION; + + if (PreserveSMBPermissions.Group == (preserveSMBPermissions & PreserveSMBPermissions.Group)) + securityInfo |= (uint)FileSecurityNativeMethods.SECURITY_INFORMATION.GROUP_SECURITY_INFORMATION; + + if (PreserveSMBPermissions.DACL == (preserveSMBPermissions & PreserveSMBPermissions.DACL)) + securityInfo |= (uint)FileSecurityNativeMethods.SECURITY_INFORMATION.DACL_SECURITY_INFORMATION; + + if (PreserveSMBPermissions.SACL == (preserveSMBPermissions & PreserveSMBPermissions.SACL)) + securityInfo |= (uint)FileSecurityNativeMethods.SECURITY_INFORMATION.SACL_SECURITY_INFORMATION; + + return (FileSecurityNativeMethods.SECURITY_INFORMATION)(securityInfo); + } + + private static string GetUserName() + { + StringBuilder userNameBuffer = new StringBuilder(64); + int nSize = 64; + if (!FileSecurityNativeMethods.GetUserName(userNameBuffer, ref nSize)) + { + if (Marshal.GetLastWin32Error() == FileSecurityNativeMethods.ERROR_INSUFFICIENT_BUFFER) + { + userNameBuffer.EnsureCapacity(nSize); + + if (!FileSecurityNativeMethods.GetUserName(userNameBuffer, ref nSize)) + { + throw Marshal.GetExceptionForHR(Marshal.GetHRForLastWin32Error()); + } + } + else + { + throw Marshal.GetExceptionForHR(Marshal.GetHRForLastWin32Error()); + } + } + + return userNameBuffer.ToString(); + } + + private static string GetPortableSDDL(string sddl, string domainSid) + { + StringBuilder portableSddl = new StringBuilder(); + + // Map of well known domain relative idenifiers to their full SID + Dictionary domainRelativeIdentifiersMap = new Dictionary + { + { "LA", domainSid + "-500" }, + { "LG", domainSid + "-501" }, + { "CA", domainSid + "-517" }, + { "DA", domainSid + "-512" }, + { "DD", domainSid + "-516" }, + { "DU", domainSid + "-513" }, + { "DG", domainSid + "-514" }, + { "DC", domainSid + "-515" }, + { "SA", domainSid + "-518" }, + { "EA", domainSid + "-519" }, + { "PA", domainSid + "-520" }, + { "RS", domainSid + "-553" }, + { "ED", domainSid + "-498" }, + { "RO", domainSid + "-521" } + }; + + // Parse the sddl and convert into portable format i.e. replace each detected RID into full SID + { + int ownerIndex = sddl.IndexOf("O:", StringComparison.Ordinal); + int groupIndex = sddl.IndexOf("G:", StringComparison.Ordinal); + int daclIndex = sddl.IndexOf("D:", StringComparison.Ordinal); + int saclIndex = sddl.IndexOf("S:", StringComparison.Ordinal); + + List indexes = new List(); + + if (ownerIndex != -1) + { + indexes.Add(ownerIndex); + } + + if (groupIndex != -1) + { + indexes.Add(groupIndex); + } + + if (daclIndex != -1) + { + indexes.Add(daclIndex); + } + + if (saclIndex != -1) + { + indexes.Add(saclIndex); + } + + indexes.Add(sddl.Length); + + indexes.Sort(); + + for (int index = 0; index < indexes.Count - 1; index++) + { + int beginIndex = indexes[index]; + int endIndex = indexes[index + 1]; + + if ((beginIndex == ownerIndex) || + (beginIndex == groupIndex)) + { + // Fix owner / group + // Owner and Group have the same format + bool sddlCopied = false; + + foreach (var keypair in domainRelativeIdentifiersMap) + { + int ridIndex = sddl.IndexOf(keypair.Key, beginIndex, StringComparison.Ordinal); + + if ((ridIndex != -1) && (ridIndex < endIndex)) + { + sddlCopied = true; + portableSddl.Append(sddl, beginIndex, ridIndex - beginIndex); + portableSddl.Append(keypair.Value); + + beginIndex = ridIndex + keypair.Key.Length; + } + } + + if (!sddlCopied) + { + portableSddl.Append(sddl, beginIndex, endIndex - beginIndex); + } + } + else if ((beginIndex == daclIndex) || + (beginIndex == saclIndex)) + { + // Fix dacl / sacl + // DACL and SACL have the same format + // Each acl format is: ace_type;ace_flags;rights;object_guid;inherit_object_guid;account_sid;(resource_attribute) + int aclStartIndex = sddl.IndexOf('(', beginIndex); + bool sddlCopied = false; + + if ((aclStartIndex != -1) && (aclStartIndex < endIndex)) + { + portableSddl.Append(sddl, beginIndex, aclStartIndex - beginIndex); + } + + while ((aclStartIndex != -1) && (aclStartIndex < endIndex)) + { + sddlCopied = false; + + int aclEndIndex = sddl.IndexOf(')', aclStartIndex + 1); + + if (aclStartIndex == -1) + { + Console.Error.WriteLine("Invalid SDDL"); + return ""; + } + + // Fix the account Sid if it have domain RID + // format is: ace_type;ace_flags;rights;object_guid;inherit_object_guid;account_sid;(resource_attribute) + int accountSidStart = sddl.IndexOf(';', aclStartIndex + 1); + accountSidStart = sddl.IndexOf(';', accountSidStart + 1); + accountSidStart = sddl.IndexOf(';', accountSidStart + 1); + accountSidStart = sddl.IndexOf(';', accountSidStart + 1); + accountSidStart = sddl.IndexOf(';', accountSidStart + 1); + + portableSddl.Append(sddl, aclStartIndex, accountSidStart - aclStartIndex); + + foreach (var keypair in domainRelativeIdentifiersMap) + { + int ridIndex = sddl.IndexOf(keypair.Key, accountSidStart, StringComparison.Ordinal); + + if ((ridIndex != -1) && (ridIndex < aclEndIndex)) + { + sddlCopied = true; + portableSddl.Append(sddl, accountSidStart, ridIndex - accountSidStart); + portableSddl.Append(keypair.Value); + portableSddl.Append(sddl, ridIndex + keypair.Key.Length, aclEndIndex - (ridIndex + keypair.Key.Length) + 1); + } + } + + if (!sddlCopied) + { + sddlCopied = true; + portableSddl.Append(sddl, accountSidStart, (aclEndIndex - accountSidStart) + 1); + } + + aclStartIndex = sddl.IndexOf('(', aclEndIndex); + } + + if (!sddlCopied) + { + portableSddl.Append(sddl, beginIndex, endIndex - beginIndex); + } + } + } + } + + return portableSddl.ToString(); + } + + private static string GetFileSDDL( + string filePath, + FileSecurityNativeMethods.SECURITY_INFORMATION securityInfo) + { + // Note: to get the SACL, special permissions are needed. Refer https://docs.microsoft.com/en-us/windows/win32/api/aclapi/nf-aclapi-getsecurityinfo + IntPtr pZero = IntPtr.Zero; + IntPtr pSid = pZero; + IntPtr psd = pZero; + + uint errorReturn = FileSecurityNativeMethods.GetNamedSecurityInfoW( + filePath, + FileSecurityNativeMethods.SE_OBJECT_TYPE.SE_FILE_OBJECT, + securityInfo, + out pSid, out pZero, out pZero, out pZero, out psd); + + if (errorReturn != 0) + { + throw new Win32Exception((int)errorReturn); + } + + try + { + IntPtr sdString = IntPtr.Zero; + UIntPtr sd_string_size_ptr = new UIntPtr(); + bool success = FileSecurityNativeMethods.ConvertSecurityDescriptorToStringSecurityDescriptor(psd, FileSecurityNativeMethods.SDDL_REVISION_1, securityInfo, out sdString, out sd_string_size_ptr); + try + { + return Marshal.PtrToStringAuto(sdString); + } + finally + { + Marshal.FreeHGlobal(sdString); + } + } + finally + { + FileSecurityNativeMethods.LocalFree(psd); + } + } + + private static string GetDomainSid(string accountName) + { + uint sidLength = 0; + uint domainNameLength = 0; + byte[] sid = null; + StringBuilder domainName = new StringBuilder(); + FileSecurityNativeMethods.SID_NAME_USE peUse; + + // Query buffer size requirement by passing in NULL for Sid and ReferencedDomainName + if (!FileSecurityNativeMethods.LookupAccountName(null, // lpSystemName + accountName, + null, // Sid + ref sidLength, + domainName, // ReferencedDomainName + ref domainNameLength, + out peUse)) + { + int errorCode = Marshal.GetLastWin32Error(); + + if (errorCode != FileSecurityNativeMethods.ERROR_INSUFFICIENT_BUFFER) + { + throw Marshal.GetExceptionForHR(Marshal.GetHRForLastWin32Error()); + } + + sid = new byte[sidLength]; + domainName.EnsureCapacity((int)domainNameLength); + + if (!FileSecurityNativeMethods.LookupAccountName(null, + accountName, + sid, + ref sidLength, + domainName, + ref domainNameLength, + out peUse)) + { + throw Marshal.GetExceptionForHR(Marshal.GetHRForLastWin32Error()); + } + } + else + { + throw new TransferException("Unexpected LookupAccountName success"); + } + + // Get domain Sid from User Sid + uint domainSidLength = 0; + byte[] domainSid = null; + if (!FileSecurityNativeMethods.GetWindowsAccountDomainSid(sid, domainSid, ref domainSidLength)) + { + int errorCode = Marshal.GetLastWin32Error(); + + if (errorCode != FileSecurityNativeMethods.ERROR_INSUFFICIENT_BUFFER) + { + throw Marshal.GetExceptionForHR(Marshal.GetHRForLastWin32Error()); + } + + domainSid = new byte[domainSidLength]; + + if (!FileSecurityNativeMethods.GetWindowsAccountDomainSid(sid, domainSid, ref domainSidLength)) + { + throw Marshal.GetExceptionForHR(Marshal.GetHRForLastWin32Error()); + } + } + else + { + throw new TransferException("Unexpected GetWindowsAccountDomainSid success"); + } + + // Get string corresponding to SID + IntPtr domainSidPtr; + if (!FileSecurityNativeMethods.ConvertSidToStringSid(domainSid, out domainSidPtr)) + { + throw Marshal.GetExceptionForHR(Marshal.GetHRForLastWin32Error()); + } + + try + { + return Marshal.PtrToStringAuto(domainSidPtr); + } + finally + { + FileSecurityNativeMethods.LocalFree(domainSidPtr); + } + } + } +} diff --git a/samples/PreserveFilePropertiesSamples/PreserveFilePropertiesSamples/PreserveFilePropertiesSamples.csproj b/samples/PreserveFilePropertiesSamples/PreserveFilePropertiesSamples/PreserveFilePropertiesSamples.csproj new file mode 100644 index 00000000..57ccd17a --- /dev/null +++ b/samples/PreserveFilePropertiesSamples/PreserveFilePropertiesSamples/PreserveFilePropertiesSamples.csproj @@ -0,0 +1,86 @@ + + + + + Debug + AnyCPU + {BEE26B27-C556-4811-B445-979C5395AA74} + Exe + PreserveFilePropertiesSamples + PreserveFilePropertiesSamples + v4.5.2 + 512 + true + true + + + AnyCPU + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 4 + + + AnyCPU + pdbonly + true + bin\Release\ + TRACE + prompt + 4 + + + + ..\packages\Microsoft.Azure.KeyVault.Core.1.0.0\lib\net40\Microsoft.Azure.KeyVault.Core.dll + + + ..\packages\Microsoft.Azure.Storage.Blob.11.2.2\lib\net452\Microsoft.Azure.Storage.Blob.dll + + + ..\packages\Microsoft.Azure.Storage.Common.11.2.2\lib\net452\Microsoft.Azure.Storage.Common.dll + + + ..\packages\Microsoft.Azure.Storage.DataMovement.2.0.0\lib\net452\Microsoft.Azure.Storage.DataMovement.dll + + + ..\packages\Microsoft.Azure.Storage.File.11.2.2\lib\net452\Microsoft.Azure.Storage.File.dll + + + ..\packages\Microsoft.WindowsAzure.ConfigurationManager.1.8.0.0\lib\net35-full\Microsoft.WindowsAzure.Configuration.dll + + + ..\packages\Newtonsoft.Json.10.0.2\lib\net45\Newtonsoft.Json.dll + + + + + + + + + + + + + + + + + + + + + + + + + + + PreserveNewest + + + + \ No newline at end of file diff --git a/samples/PreserveFilePropertiesSamples/PreserveFilePropertiesSamples/Program.cs b/samples/PreserveFilePropertiesSamples/PreserveFilePropertiesSamples/Program.cs new file mode 100644 index 00000000..19ac261e --- /dev/null +++ b/samples/PreserveFilePropertiesSamples/PreserveFilePropertiesSamples/Program.cs @@ -0,0 +1,242 @@ +//------------------------------------------------------------------------------ +// +// Copyright (c) Microsoft Corporation +// +//------------------------------------------------------------------------------ + +namespace PreserveFilePropertiesSamples +{ + using System; + using System.IO; + using System.Threading.Tasks; + using Microsoft.Azure.Storage.Blob; + using Microsoft.Azure.Storage.DataMovement; + + static class Program + { + static void Main(string[] args) + { + try + { + Console.WriteLine("The sample should be executed with elevated privileges."); + Console.WriteLine(""); + Console.WriteLine("Samples to preserving file properties and permissions in blob uploading or downloading."); + BlobDirectoryPreserveFilePropertiesSampleAsync().GetAwaiter().GetResult(); + } + finally + { + Util.DeleteContainerAsync(ContainerName).GetAwaiter().GetResult(); + } + } + + /// + /// Container name used in this sample. + /// + private const string ContainerName = "samplecontainer"; + + /// + /// Name of meta data to store file's creation time. + /// + private const string CreationTimeName = "CreationTime"; + + /// + /// Name of meta data to store file's last write time. + /// + private const string LastWriteTimeName = "LastWriteTime"; + + /// + /// Name of meta data to store file's attributes. + /// + private const string FileAttributesName = "FileAttributes"; + + /// + /// Name of meta data to store file's permissions. + /// + private const string FilePermissionsName = "FilePermissions"; + + /// + /// Upload local pictures to azure storage. + /// 1. Upload png files starting with "azure" in the source directory as block blobs, not including the sub-directory. + /// 2. Store source file's file attributes and permissions into destination blob's meta data. + /// 3. Download png files starting with "azure" in the source directory to a local directory, not including the sub-directory. + /// 4. Restore file attributes and permissions to destination local file. + /// + private static async Task BlobDirectoryPreserveFilePropertiesSampleAsync() + { + //Enable required privileges before getting/setting permissions from/to local file system. + FileSecurityOperations.EnableRequiredPrivileges(PreserveSMBPermissions.Owner | PreserveSMBPermissions.Group | PreserveSMBPermissions.DACL | PreserveSMBPermissions.SACL, true); + + try + { + + string sourceDirPath = "."; + CloudBlobDirectory destDir = await Util.GetCloudBlobDirectoryAsync(ContainerName, "blobdir"); + + // SearchPattern and Recuresive can be used to determine the files to be transferred from the source directory. The behavior of + // SearchPattern and Recuresive varies for different source directory types. + // See https://azure.github.io/azure-storage-net-data-movement for more details. + // + // When source is local directory, data movement library matches source files against the SearchPattern as standard wildcards. If + // recuresive is set to false, only files directly under the source directory will be matched. Otherwise, all files in the + // sub-directory will be matched as well. + // + // In the following case, data movement library will upload png files starting with "azure" in the source directory as block blobs, + // not including the sub-directory. + UploadDirectoryOptions options = new UploadDirectoryOptions() + { + SearchPattern = "azure*.png", + Recursive = false, + BlobType = BlobType.BlockBlob + }; + + DirectoryTransferContext context = new DirectoryTransferContext(); + + // Register for transfer event. + context.FileTransferred += FileTransferredCallback; + context.FileFailed += FileFailedCallback; + context.FileSkipped += FileSkippedCallback; + + context.SetAttributesCallbackAsync = async (sourceObj, destination) => + { + string sourcePath = sourceObj as string; + DateTimeOffset? creationTime = null; + DateTimeOffset? lastWriteTime = null; + FileAttributes? fileAttributes = null; + + FileOperations.GetFileProperties(sourcePath, out creationTime, out lastWriteTime, out fileAttributes); + + string sourceFileSDDL = FileSecurityOperations.GetFilePortableSDDL(sourcePath, + PreserveSMBPermissions.Owner | PreserveSMBPermissions.Group | PreserveSMBPermissions.DACL | PreserveSMBPermissions.SACL); + + CloudBlob destBlob = destination as CloudBlob; + + // Blob's original meta data has already been gotten from azure storage by DataMovement Library, + // Here only need to add new meta data key-value pairs, DataMovement Library will set these value to destination blob later. + destBlob.Metadata.Add(CreationTimeName, creationTime.Value.ToString()); + destBlob.Metadata.Add(LastWriteTimeName, lastWriteTime.Value.ToString()); + destBlob.Metadata.Add(FileAttributesName, fileAttributes.Value.ToString()); + destBlob.Metadata.Add(FilePermissionsName, sourceFileSDDL); + }; + + context.ShouldTransferCallbackAsync = async (source, destination) => + { + // Can add more logic here to evaluate whether really need to transfer the target. + return true; + }; + + TransferStatus trasferStatus = + await TransferManager.UploadDirectoryAsync(sourceDirPath, destDir, options, context); + + + Console.WriteLine("Final transfer state: {0}", TransferStatusToString(trasferStatus)); + Console.WriteLine("Files in directory {0} uploading to {1} is finished.", sourceDirPath, destDir.Uri.ToString()); + + //Next the sample will show how to download a directory and restore file attributes to local file. + string destDirPath = "."; + CloudBlobDirectory sourceDir = await Util.GetCloudBlobDirectoryAsync(ContainerName, "blobdir"); + + // In the following case, data movement library will download file named "azure.png" in the source directory, + // not including the sub-directory. + DownloadDirectoryOptions downloadDirectoryOptions = new DownloadDirectoryOptions() + { + SearchPattern = "azure.png", + Recursive = false + }; + + DirectoryTransferContext directoryTransferContext = new DirectoryTransferContext(); + // Register for transfer event. + directoryTransferContext.FileFailed += FileFailedCallback; + directoryTransferContext.FileSkipped += FileSkippedCallback; + + //Get stored file properties from source blob meta data and set to local file. + directoryTransferContext.FileTransferred += (object sender, TransferEventArgs e) => + { + CloudBlob sourceBlob = e.Source as CloudBlob; + string destFilePath = e.Destination as string; + + string metadataValue = null; + DateTimeOffset creationTime = default(DateTimeOffset); + DateTimeOffset lastWriteTime = default(DateTimeOffset); + FileAttributes fileAttributes = default(FileAttributes); + + bool gotCreationTime = false; + bool gotLastWriteTime = false; + bool gotFileAttributes = false; + + if (sourceBlob.Metadata.TryGetValue(CreationTimeName, out metadataValue) + && !string.IsNullOrEmpty(metadataValue)) + { + gotCreationTime = DateTimeOffset.TryParse(metadataValue, out creationTime); + } + + if (sourceBlob.Metadata.TryGetValue(LastWriteTimeName, out metadataValue) + && !string.IsNullOrEmpty(metadataValue)) + { + gotLastWriteTime = DateTimeOffset.TryParse(metadataValue, out lastWriteTime); + } + + if (sourceBlob.Metadata.TryGetValue(FileAttributesName, out metadataValue) + && !string.IsNullOrEmpty(metadataValue)) + { + gotFileAttributes = Enum.TryParse(metadataValue, out fileAttributes); + } + + if (gotCreationTime && gotLastWriteTime && gotFileAttributes) + { + FileOperations.SetFileProperties(destFilePath, creationTime, lastWriteTime, fileAttributes); + } + + if (sourceBlob.Metadata.TryGetValue(FilePermissionsName, out metadataValue) + && !string.IsNullOrEmpty(metadataValue)) + { + FileSecurityOperations.SetFileSecurity(destFilePath, metadataValue, + PreserveSMBPermissions.Owner | PreserveSMBPermissions.Group | PreserveSMBPermissions.DACL | PreserveSMBPermissions.SACL); + } + + }; + + // Always writes to destination no matter it exists or not. + directoryTransferContext.ShouldOverwriteCallbackAsync = TransferContext.ForceOverwrite; + + trasferStatus = + await TransferManager.DownloadDirectoryAsync(sourceDir, destDirPath, downloadDirectoryOptions, directoryTransferContext); + + + Console.WriteLine("Final transfer state: {0}", TransferStatusToString(trasferStatus)); + Console.WriteLine("Files in directory {0} downloading to {1} is finished.", sourceDir.Uri.ToString(), destDirPath); + } + finally + { + //Restore privileges after getting/setting permissions from/to local file system. + FileSecurityOperations.RestorePrivileges(PreserveSMBPermissions.Owner | PreserveSMBPermissions.Group | PreserveSMBPermissions.DACL | PreserveSMBPermissions.SACL, true); + } + } + + private static void FileTransferredCallback(object sender, TransferEventArgs e) + { + Console.WriteLine("Transfer Succeeds. {0} -> {1}.", e.Source, e.Destination); + } + + private static void FileFailedCallback(object sender, TransferEventArgs e) + { + Console.WriteLine("Transfer fails. {0} -> {1}. Error message:{2}", e.Source, e.Destination, e.Exception.Message); + } + + private static void FileSkippedCallback(object sender, TransferEventArgs e) + { + Console.WriteLine("Transfer skips. {0} -> {1}.", e.Source, e.Destination); + } + + /// + /// Format the TransferStatus of DMlib to printable string + /// + public static string TransferStatusToString(TransferStatus status) + { + return string.Format("Transferred bytes: {0}; Transfered: {1}; Skipped: {2}, Failed: {3}", + status.BytesTransferred, + status.NumberOfFilesTransferred, + status.NumberOfFilesSkipped, + status.NumberOfFilesFailed); + } + } +} diff --git a/samples/PreserveFilePropertiesSamples/PreserveFilePropertiesSamples/Properties/AssemblyInfo.cs b/samples/PreserveFilePropertiesSamples/PreserveFilePropertiesSamples/Properties/AssemblyInfo.cs new file mode 100644 index 00000000..87bd8fd3 --- /dev/null +++ b/samples/PreserveFilePropertiesSamples/PreserveFilePropertiesSamples/Properties/AssemblyInfo.cs @@ -0,0 +1,36 @@ +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. +[assembly: AssemblyTitle("PreserveFilePropertiesSamples1")] +[assembly: AssemblyDescription("")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("")] +[assembly: AssemblyProduct("PreserveFilePropertiesSamples1")] +[assembly: AssemblyCopyright("Copyright © 2020")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] + +// Setting ComVisible to false makes the types in this assembly not visible +// to COM components. If you need to access a type in this assembly from +// COM, set the ComVisible attribute to true on that type. +[assembly: ComVisible(false)] + +// The following GUID is for the ID of the typelib if this project is exposed to COM +[assembly: Guid("bee26b27-c556-4811-b445-979c5395aa74")] + +// Version information for an assembly consists of the following four values: +// +// Major Version +// Minor Version +// Build Number +// Revision +// +// You can specify all the values or you can default the Build and Revision Numbers +// by using the '*' as shown below: +// [assembly: AssemblyVersion("1.0.*")] +[assembly: AssemblyVersion("1.0.0.0")] +[assembly: AssemblyFileVersion("1.0.0.0")] diff --git a/samples/PreserveFilePropertiesSamples/PreserveFilePropertiesSamples/Util.cs b/samples/PreserveFilePropertiesSamples/PreserveFilePropertiesSamples/Util.cs new file mode 100644 index 00000000..c9ab6b0a --- /dev/null +++ b/samples/PreserveFilePropertiesSamples/PreserveFilePropertiesSamples/Util.cs @@ -0,0 +1,81 @@ +//------------------------------------------------------------------------------ +// +// Copyright (c) Microsoft Corporation +// +//------------------------------------------------------------------------------ +namespace PreserveFilePropertiesSamples +{ + using System.IO; + using System.Threading.Tasks; + using Microsoft.Azure.Storage; + using Microsoft.Azure.Storage.Blob; + using Newtonsoft.Json.Linq; + + /// + /// A helper class provides convenient operations against storage account configured in the App.config. + /// + public class Util + { + private static CloudStorageAccount storageAccount; + private static CloudBlobClient blobClient; + + /// + /// Get a CloudBlobDirectory instance with the specified name in the given container. + /// + /// Container name. + /// Blob directory name. + /// A object of type that represents the asynchronous operation. + public static async Task GetCloudBlobDirectoryAsync(string containerName, string directoryName) + { + CloudBlobClient client = GetCloudBlobClient(); + CloudBlobContainer container = client.GetContainerReference(containerName); + await container.CreateIfNotExistsAsync(); + + return container.GetDirectoryReference(directoryName); + } + + /// + /// Delete the container with the specified name if it exists. + /// + /// Name of container to delete. + public static async Task DeleteContainerAsync(string containerName) + { + CloudBlobClient client = GetCloudBlobClient(); + CloudBlobContainer container = client.GetContainerReference(containerName); + await container.DeleteIfExistsAsync(); + } + + private static CloudBlobClient GetCloudBlobClient() + { + if (Util.blobClient == null) + { + Util.blobClient = GetStorageAccount().CreateCloudBlobClient(); + } + + return Util.blobClient; + } + + private static string LoadConnectionStringFromConfigration() + { + // How to create a storage connection string: http://msdn.microsoft.com/en-us/library/azure/ee758697.aspx +#if DOTNET5_4 + //For .Net Core, will get Storage Connection string from Config.json file + return JObject.Parse(File.ReadAllText("Config.json"))["StorageConnectionString"].ToString(); +#else + //For .net, will get Storage Connection string from App.Config file + return System.Configuration.ConfigurationManager.AppSettings["StorageConnectionString"]; +#endif + } + + private static CloudStorageAccount GetStorageAccount() + { + if (Util.storageAccount == null) + { + string connectionString = LoadConnectionStringFromConfigration(); + Util.storageAccount = CloudStorageAccount.Parse(connectionString); + } + + return Util.storageAccount; + } + } +} diff --git a/samples/PreserveFilePropertiesSamples/PreserveFilePropertiesSamples/azure.png b/samples/PreserveFilePropertiesSamples/PreserveFilePropertiesSamples/azure.png new file mode 100644 index 00000000..f50ba5b7 Binary files /dev/null and b/samples/PreserveFilePropertiesSamples/PreserveFilePropertiesSamples/azure.png differ diff --git a/samples/PreserveFilePropertiesSamples/PreserveFilePropertiesSamples/packages.config b/samples/PreserveFilePropertiesSamples/PreserveFilePropertiesSamples/packages.config new file mode 100644 index 00000000..1757f2ef --- /dev/null +++ b/samples/PreserveFilePropertiesSamples/PreserveFilePropertiesSamples/packages.config @@ -0,0 +1,10 @@ + + + + + + + + + + \ No newline at end of file diff --git a/samples/PreserveFilePropertiesSamples/PreserveFilePropertiesSamples_k.sln b/samples/PreserveFilePropertiesSamples/PreserveFilePropertiesSamples_k.sln new file mode 100644 index 00000000..13aef11b --- /dev/null +++ b/samples/PreserveFilePropertiesSamples/PreserveFilePropertiesSamples_k.sln @@ -0,0 +1,25 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 16 +VisualStudioVersion = 16.0.29613.14 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "PreserveFilePropertiesSamples", "netcore\PreserveFilePropertiesSamples.csproj", "{2FCFA585-A875-47B5-840D-DE8C8A42F3C8}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {2FCFA585-A875-47B5-840D-DE8C8A42F3C8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {2FCFA585-A875-47B5-840D-DE8C8A42F3C8}.Debug|Any CPU.Build.0 = Debug|Any CPU + {2FCFA585-A875-47B5-840D-DE8C8A42F3C8}.Release|Any CPU.ActiveCfg = Release|Any CPU + {2FCFA585-A875-47B5-840D-DE8C8A42F3C8}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {8E2AF0E8-AEC7-4F8E-8811-714510119944} + EndGlobalSection +EndGlobal diff --git a/samples/PreserveFilePropertiesSamples/netcore/Config.json b/samples/PreserveFilePropertiesSamples/netcore/Config.json new file mode 100644 index 00000000..dbf6d3d4 --- /dev/null +++ b/samples/PreserveFilePropertiesSamples/netcore/Config.json @@ -0,0 +1,3 @@ +{ + "StorageConnectionString": "DefaultEndpointsProtocol=http;AccountName=[AccountName];AccountKey=[AccountKey]" +} \ No newline at end of file diff --git a/samples/PreserveFilePropertiesSamples/netcore/PreserveFilePropertiesSamples.csproj b/samples/PreserveFilePropertiesSamples/netcore/PreserveFilePropertiesSamples.csproj new file mode 100644 index 00000000..34318089 --- /dev/null +++ b/samples/PreserveFilePropertiesSamples/netcore/PreserveFilePropertiesSamples.csproj @@ -0,0 +1,39 @@ + + + + Exe + netcoreapp3.1 + + + + TRACE;DOTNET5_4 + + + + TRACE;DOTNET5_4 + + + + + + + + + + + + + + + + + + + PreserveNewest + + + PreserveNewest + + + + diff --git a/samples/PreserveFilePropertiesSamples/netcore/azure.png b/samples/PreserveFilePropertiesSamples/netcore/azure.png new file mode 100644 index 00000000..f50ba5b7 Binary files /dev/null and b/samples/PreserveFilePropertiesSamples/netcore/azure.png differ diff --git a/samples/S3ToAzureSample/S3ToAzureSample/S3ToAzureSample.csproj b/samples/S3ToAzureSample/S3ToAzureSample/S3ToAzureSample.csproj index 11c29b6e..59865000 100644 --- a/samples/S3ToAzureSample/S3ToAzureSample/S3ToAzureSample.csproj +++ b/samples/S3ToAzureSample/S3ToAzureSample/S3ToAzureSample.csproj @@ -44,17 +44,17 @@ ..\packages\Microsoft.Azure.KeyVault.Core.1.0.0\lib\net40\Microsoft.Azure.KeyVault.Core.dll - - ..\packages\Microsoft.Azure.Storage.Blob.11.1.2\lib\net452\Microsoft.Azure.Storage.Blob.dll + + ..\packages\Microsoft.Azure.Storage.Blob.11.2.2\lib\net452\Microsoft.Azure.Storage.Blob.dll - - ..\packages\Microsoft.Azure.Storage.Common.11.1.2\lib\net452\Microsoft.Azure.Storage.Common.dll + + ..\packages\Microsoft.Azure.Storage.Common.11.2.2\lib\net452\Microsoft.Azure.Storage.Common.dll - - ..\packages\Microsoft.Azure.Storage.DataMovement.1.3.0\lib\net452\Microsoft.Azure.Storage.DataMovement.dll + + ..\packages\Microsoft.Azure.Storage.DataMovement.2.0.0\lib\net452\Microsoft.Azure.Storage.DataMovement.dll - - ..\packages\Microsoft.Azure.Storage.File.11.1.2\lib\net452\Microsoft.Azure.Storage.File.dll + + ..\packages\Microsoft.Azure.Storage.File.11.2.2\lib\net452\Microsoft.Azure.Storage.File.dll ..\packages\Microsoft.WindowsAzure.ConfigurationManager.1.8.0.0\lib\net35-full\Microsoft.WindowsAzure.Configuration.dll diff --git a/samples/S3ToAzureSample/S3ToAzureSample/packages.config b/samples/S3ToAzureSample/S3ToAzureSample/packages.config index 00186366..7678cf29 100644 --- a/samples/S3ToAzureSample/S3ToAzureSample/packages.config +++ b/samples/S3ToAzureSample/S3ToAzureSample/packages.config @@ -3,10 +3,10 @@ - - - - + + + + diff --git a/test/DMLibTest/Cases/AccessConditionTest.cs b/test/DMLibTest/Cases/AccessConditionTest.cs index dba02262..2fd0bb83 100644 --- a/test/DMLibTest/Cases/AccessConditionTest.cs +++ b/test/DMLibTest/Cases/AccessConditionTest.cs @@ -145,7 +145,7 @@ private void TestAccessCondition(SourceOrDest sourceOrDest) } Exception exception = result.Exceptions[0]; -#if DNXCORE50 +#if DOTNET5_4 VerificationHelper.VerifyTransferException(exception, TransferErrorCode.Unknown); // Verify innner StorageException diff --git a/test/DMLibTest/Cases/AllTransferDirectionTest.cs b/test/DMLibTest/Cases/AllTransferDirectionTest.cs index b12c815e..44fb2ae7 100644 --- a/test/DMLibTest/Cases/AllTransferDirectionTest.cs +++ b/test/DMLibTest/Cases/AllTransferDirectionTest.cs @@ -245,6 +245,11 @@ private static List GetTransformItemsForAllDirTransferDirections(b List allItems = new List(); foreach (DMLibTransferDirection direction in GetAllDirectoryValidDirections()) { + if ((direction.SourceType != DMLibDataType.CloudFile) + || (direction.DestType != DMLibDataType.Local)) + { + continue; + } string dirName = GetTransferDirName(direction); DataAdaptor sourceAdaptor = GetSourceAdaptor(direction.SourceType); DataAdaptor destAdaptor = GetDestAdaptor(direction.DestType); @@ -493,7 +498,7 @@ private void ResumeInAllDirectionsHelper(bool directoryTransfer) } } - private static async Task SetAttributesCallbackMethodAsync(object destObj) + private static async Task SetAttributesCallbackMethodAsync(object sourceObj, object destObj) { await Task.Run(() => { diff --git a/test/DMLibTest/Cases/EncryptionScopeTest.cs b/test/DMLibTest/Cases/EncryptionScopeTest.cs new file mode 100644 index 00000000..075dd822 --- /dev/null +++ b/test/DMLibTest/Cases/EncryptionScopeTest.cs @@ -0,0 +1,129 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using DMLibTestCodeGen; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using MS.Test.Common.MsTestLib; + +namespace DMLibTest.Cases +{ + [MultiDirectionTestClass] + public class EncryptionScopeTest : DMLibTestBase +#if DNXCORE50 + , IDisposable +#endif + { + #region Initialization and cleanup methods + +#if DNXCORE50 + public EncryptionScopeTest() + { + MyTestInitialize(); + } + + public void Dispose() + { + Dispose(true); + } + + protected virtual void Dispose(bool disposing) + { + MyTestCleanup(); + } +#endif + + [ClassInitialize()] + public static void MyClassInitialize(TestContext testContext) + { + Test.Info("Class Initialize: EncryptionScopeTest"); + DMLibTestBase.BaseClassInitialize(testContext); + } + + [ClassCleanup()] + public static void MyClassCleanup() + { + DMLibTestBase.BaseClassCleanup(); + } + + [TestInitialize()] + public void MyTestInitialize() + { + base.BaseTestInitialize(); + } + + [TestCleanup()] + public void MyTestCleanup() + { + base.BaseTestCleanup(); + } + #endregion + + [TestCategory(Tag.Function)] + [DMLibTestMethod(DMLibDataType.CloudBlob, DMLibCopyMethod.ServiceSideSyncCopy)] + [DMLibTestMethod(DMLibDataType.CloudFile, DMLibDataType.CloudBlob, DMLibCopyMethod.ServiceSideSyncCopy)] + [DMLibTestMethod(DMLibDataType.CloudBlob, DMLibCopyMethod.SyncCopy)] + [DMLibTestMethod(DMLibDataType.CloudFile, DMLibDataType.CloudBlob, DMLibCopyMethod.SyncCopy)] + [DMLibTestMethod(DMLibDataType.Local, DMLibDataType.CloudBlob)] + [DMLibTestMethod(DMLibDataType.Stream, DMLibDataType.CloudBlob)] + public void TestSingleBlobWithEncryptionScope() + { + DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty); + DMLibDataHelper.AddOneFileInBytes(sourceDataInfo.RootNode, DMLibTestBase.FileName, 1024); + + var options = new TestExecutionOptions(); + + options.TransferItemModifier = (fileNode, transferItem) => + { + dynamic transferOptions = DefaultTransferOptions; + transferOptions.EncryptionScope = Test.Data.Get(DMLibTestConstants.DestEncryptionScope); + + transferItem.Options = transferOptions; + }; + + var result = this.ExecuteTestCase(sourceDataInfo, options); + + VerificationHelper.VerifyTransferSucceed(result, sourceDataInfo); + + string destEncryptionScope = result.DataInfo.RootNode.FileNodes.First().EncryptionScope; + Test.Assert(string.Equals(destEncryptionScope, Test.Data.Get(DMLibTestConstants.DestEncryptionScope)), + "Encryption scope name in destination should be expected {0} == {1}", destEncryptionScope, Test.Data.Get(DMLibTestConstants.DestEncryptionScope)); + } + + [TestCategory(Tag.Function)] + [DMLibTestMethod(DMLibDataType.CloudBlob, DMLibCopyMethod.ServiceSideSyncCopy)] + [DMLibTestMethod(DMLibDataType.CloudFile, DMLibDataType.CloudBlob, DMLibCopyMethod.ServiceSideSyncCopy)] + [DMLibTestMethod(DMLibDataType.CloudBlob, DMLibCopyMethod.SyncCopy)] + [DMLibTestMethod(DMLibDataType.CloudFile, DMLibDataType.CloudBlob, DMLibCopyMethod.SyncCopy)] + [DMLibTestMethod(DMLibDataType.Local, DMLibDataType.CloudBlob)] + public void TestBlobDirectoryWithEncryptionScope() + { + DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty); + DMLibDataHelper.AddMultipleFilesNormalSize(sourceDataInfo.RootNode, DMLibTestBase.FileName); + + var options = new TestExecutionOptions(); + options.IsDirectoryTransfer = true; + + options.TransferItemModifier = (fileNode, transferItem) => + { + dynamic transferOptions = DefaultTransferDirectoryOptions; + transferOptions.EncryptionScope = Test.Data.Get(DMLibTestConstants.DestEncryptionScope); + transferOptions.Recursive = true; + + transferItem.Options = transferOptions; + }; + + var result = this.ExecuteTestCase(sourceDataInfo, options); + + VerificationHelper.VerifyTransferSucceed(result, sourceDataInfo); + + foreach (var fileNode in result.DataInfo.RootNode.EnumerateFileNodesRecursively()) + { + string destEncryptionScope = result.DataInfo.RootNode.FileNodes.First().EncryptionScope; + Test.Assert(string.Equals(destEncryptionScope, Test.Data.Get(DMLibTestConstants.DestEncryptionScope)), + "Encryption scope name in destination should be expected {0} == {1}", destEncryptionScope, Test.Data.Get(DMLibTestConstants.DestEncryptionScope)); + } + } + } +} diff --git a/test/DMLibTest/Cases/LongFilePathTest.cs b/test/DMLibTest/Cases/LongFilePathTest.cs index 06a05a2b..569bfa73 100644 --- a/test/DMLibTest/Cases/LongFilePathTest.cs +++ b/test/DMLibTest/Cases/LongFilePathTest.cs @@ -20,6 +20,7 @@ namespace DMLibTest.Cases using MS.Test.Common.MsTestLib; using DMLibTest.Framework; using System.Threading.Tasks; + using System.Linq.Expressions; #if DNXCORE50 using Xunit; using System.Threading.Tasks; @@ -1029,6 +1030,16 @@ public void LongFilePathResumeDirectoryDownload() resumeItem.TransferContext = resumeContext; + resumeContext.FileSkipped += (sender, eventArgs) => + { + Test.Error(eventArgs.Exception.ToString()); + }; + + resumeContext.FileFailed += (sender, eventArgs) => + { + Test.Error(eventArgs.Exception.ToString()); + }; + result = this.RunTransferItems(new List() { resumeItem }, new TestExecutionOptions()); VerificationHelper.VerifyFinalProgress(progressChecker, totalFileNum, 0, 0); diff --git a/test/DMLibTest/Cases/OverwriteTest.cs b/test/DMLibTest/Cases/OverwriteTest.cs index 1002812e..37930a80 100644 --- a/test/DMLibTest/Cases/OverwriteTest.cs +++ b/test/DMLibTest/Cases/OverwriteTest.cs @@ -106,6 +106,17 @@ public void OverwriteDestination() } transferItem.TransferContext = transferContext; + + dynamic transferOptions = DefaultTransferOptions; + + if (DMLibTestContext.SourceType == DMLibDataType.CloudFile + && DMLibTestContext.DestType == DMLibDataType.CloudFile) + { + transferOptions.PreserveSMBAttributes = true; + transferOptions.PreserveSMBPermissions = true; + } + + transferItem.Options = transferOptions; }; var result = this.ExecuteTestCase(sourceDataInfo, options); @@ -135,6 +146,16 @@ public void OverwriteDestination() VerificationHelper.VerifyTransferException(transferException, TransferErrorCode.NotOverwriteExistingDestination, "Skipped file", destExistNName); } + + if (DMLibTestContext.SourceType == DMLibDataType.CloudFile + && DMLibTestContext.DestType == DMLibDataType.CloudFile) + { + Helper.CompareSMBProperties(expectedDataInfo.RootNode, result.DataInfo.RootNode, true); + Helper.CompareSMBPermissions( + expectedDataInfo.RootNode, + result.DataInfo.RootNode, + PreserveSMBPermissions.Owner | PreserveSMBPermissions.Group | PreserveSMBPermissions.DACL | PreserveSMBPermissions.SACL); + } } [TestCategory(Tag.Function)] @@ -197,7 +218,16 @@ public void DirectoryOverwriteDestination() dynamic transferOptions = DefaultTransferDirectoryOptions; transferOptions.Recursive = true; + + if (DMLibTestContext.SourceType == DMLibDataType.CloudFile + && DMLibTestContext.DestType == DMLibDataType.CloudFile) + { + transferOptions.PreserveSMBAttributes = true; + transferOptions.PreserveSMBPermissions = true; + } + transferItem.Options = transferOptions; + transferItem.TransferContext = transferContext; }; var result = this.ExecuteTestCase(sourceDataInfo, options); @@ -230,6 +260,16 @@ public void DirectoryOverwriteDestination() Test.Assert(successCount == 3, "Very all transfers are success"); Test.Assert(skipCount == 0, "Very no transfer is skipped"); } + + if (DMLibTestContext.SourceType == DMLibDataType.CloudFile + && DMLibTestContext.DestType == DMLibDataType.CloudFile) + { + Helper.CompareSMBProperties(expectedDataInfo.RootNode, result.DataInfo.RootNode, true); + Helper.CompareSMBPermissions( + expectedDataInfo.RootNode, + result.DataInfo.RootNode, + PreserveSMBPermissions.Owner | PreserveSMBPermissions.Group | PreserveSMBPermissions.DACL | PreserveSMBPermissions.SACL); + } } [TestCategory(Tag.Function)] @@ -287,6 +327,16 @@ public void ForceOverwriteTest() options.TransferItemModifier = (fileNode, transferItem) => { transferItem.TransferContext = transferContext; + + dynamic transferOptions = DefaultTransferOptions; + if (DMLibTestContext.SourceType == DMLibDataType.CloudFile + && DMLibTestContext.DestType == DMLibDataType.CloudFile) + { + transferOptions.PreserveSMBAttributes = true; + transferOptions.PreserveSMBPermissions = true; + } + + transferItem.Options = transferOptions; }; var result = this.ExecuteTestCase(sourceDataInfo, options); @@ -295,6 +345,16 @@ public void ForceOverwriteTest() Test.Assert(DMLibDataHelper.Equals(sourceDataInfo, result.DataInfo), "Verify transfer result."); Test.Assert(successCount == 2, "Verify success transfers"); Test.Assert(skipCount == 0, "Verify skipped transfer"); + + if (DMLibTestContext.SourceType == DMLibDataType.CloudFile + && DMLibTestContext.DestType == DMLibDataType.CloudFile) + { + Helper.CompareSMBProperties(sourceDataInfo.RootNode, result.DataInfo.RootNode, true); + Helper.CompareSMBPermissions( + sourceDataInfo.RootNode, + result.DataInfo.RootNode, + PreserveSMBPermissions.Owner | PreserveSMBPermissions.Group | PreserveSMBPermissions.DACL | PreserveSMBPermissions.SACL); + } } [TestCategory(Tag.Function)] @@ -321,6 +381,11 @@ public void DirectoryForceOverwriteTest() Interlocked.Increment(ref skipCount); }; + transferContext.FileFailed += (object sender, TransferEventArgs args) => + { + Test.Error("Transfer failed {0}", args.Exception); + }; + transferContext.FileTransferred += (object sender, TransferEventArgs args) => { Interlocked.Increment(ref successCount); @@ -356,6 +421,13 @@ public void DirectoryForceOverwriteTest() dynamic transferOptions = DefaultTransferDirectoryOptions; transferOptions.Recursive = true; + if (DMLibTestContext.SourceType == DMLibDataType.CloudFile + && DMLibTestContext.DestType == DMLibDataType.CloudFile) + { + transferOptions.PreserveSMBAttributes = true; + transferOptions.PreserveSMBPermissions = true; + } + transferItem.Options = transferOptions; }; @@ -366,6 +438,16 @@ public void DirectoryForceOverwriteTest() VerificationHelper.VerifySingleTransferStatus(result, 2, 0, 0, 1024 * 2); Test.Assert(successCount == 2, "Verify success transfers"); Test.Assert(skipCount == 0, "Verify skipped transfer"); + + if (DMLibTestContext.SourceType == DMLibDataType.CloudFile + && DMLibTestContext.DestType == DMLibDataType.CloudFile) + { + Helper.CompareSMBProperties(sourceDataInfo.RootNode, result.DataInfo.RootNode, true); + Helper.CompareSMBPermissions( + sourceDataInfo.RootNode, + result.DataInfo.RootNode, + PreserveSMBPermissions.Owner | PreserveSMBPermissions.Group | PreserveSMBPermissions.DACL | PreserveSMBPermissions.SACL); + } } } } diff --git a/test/DMLibTest/Cases/ResumeTest.cs b/test/DMLibTest/Cases/ResumeTest.cs index d90ee026..b391a543 100644 --- a/test/DMLibTest/Cases/ResumeTest.cs +++ b/test/DMLibTest/Cases/ResumeTest.cs @@ -88,10 +88,20 @@ public void TestResume() transferContext.ProgressHandler = progressChecker.GetProgressHandler(); options.TransferItemModifier = (fileName, item) => { + dynamic transferOptions = DefaultTransferOptions; + + if (DMLibTestContext.SourceType == DMLibDataType.CloudFile + && DMLibTestContext.DestType == DMLibDataType.CloudFile) + { + transferOptions.PreserveSMBAttributes = true; + transferOptions.PreserveSMBPermissions = true; + } + item.CancellationToken = tokenSource.Token; item.TransferContext = transferContext; transferItem = item; item.DisableStreamDispose = true; + item.Options = transferOptions; }; TransferCheckpoint firstCheckpoint = null, secondCheckpoint = null; @@ -294,6 +304,16 @@ public void TestResume() VerificationHelper.VerifySingleObjectResumeResult(result, sourceDataInfo); } } + + if (DMLibTestContext.SourceType == DMLibDataType.CloudFile + && DMLibTestContext.DestType == DMLibDataType.CloudFile) + { + Helper.CompareSMBProperties(sourceDataInfo.RootNode, result.DataInfo.RootNode, true); + Helper.CompareSMBPermissions( + sourceDataInfo.RootNode, + result.DataInfo.RootNode, + PreserveSMBPermissions.Owner | PreserveSMBPermissions.Group | PreserveSMBPermissions.DACL | PreserveSMBPermissions.SACL); + } } } @@ -344,6 +364,13 @@ public void TestDirectoryResume() dynamic dirOptions = DefaultTransferDirectoryOptions; dirOptions.Recursive = true; + if (DMLibTestContext.SourceType == DMLibDataType.CloudFile + && DMLibTestContext.DestType == DMLibDataType.CloudFile) + { + dirOptions.PreserveSMBAttributes = true; + dirOptions.PreserveSMBPermissions = true; + } + item.Options = dirOptions; item.CancellationToken = tokenSource.Token; item.TransferContext = transferContext; @@ -444,6 +471,16 @@ public void TestDirectoryResume() VerificationHelper.VerifySingleTransferStatus(result, totalFileNum, 0, 0, totalSizeInBytes); VerificationHelper.VerifyTransferSucceed(result, sourceDataInfo); } + + if (DMLibTestContext.SourceType == DMLibDataType.CloudFile + && DMLibTestContext.DestType == DMLibDataType.CloudFile) + { + Helper.CompareSMBProperties(sourceDataInfo.RootNode, result.DataInfo.RootNode, true); + Helper.CompareSMBPermissions( + sourceDataInfo.RootNode, + result.DataInfo.RootNode, + PreserveSMBPermissions.Owner | PreserveSMBPermissions.Group | PreserveSMBPermissions.DACL | PreserveSMBPermissions.SACL); + } } } diff --git a/test/DMLibTest/Cases/SMBPropertiesTest.cs b/test/DMLibTest/Cases/SMBPropertiesTest.cs index 7d07576c..0a2b971b 100644 --- a/test/DMLibTest/Cases/SMBPropertiesTest.cs +++ b/test/DMLibTest/Cases/SMBPropertiesTest.cs @@ -97,7 +97,7 @@ public void TestDirectoryPreserveSMBProperties() Test.Info("Testing setting attributes {0} to directories", SMBFileAttributes[i]); // Prepare data DMLibDataInfo sourceDataInfo = new DMLibDataInfo(""); - GenerateDirNodeWithAttributes(sourceDataInfo.RootNode, 2, SMBFileAttributes[i]); + GenerateDirNodeWithAttributes(sourceDataInfo.RootNode, 2, SMBFileAttributes[i], null); DirectoryTransferContext dirTransferContext = new DirectoryTransferContext(); @@ -262,7 +262,7 @@ public void TestDirectoryPreserveSMBPropertiesResume() Test.Info("Testing setting attributes {0} to directories", SMBFileAttributes[i]); // Prepare data DMLibDataInfo sourceDataInfo = new DMLibDataInfo(""); - GenerateDirNodeWithAttributes(sourceDataInfo.RootNode, 2, SMBFileAttributes[i]); + GenerateDirNodeWithAttributes(sourceDataInfo.RootNode, 2, SMBFileAttributes[i], null); DirectoryTransferContext dirTransferContext = null; @@ -298,7 +298,9 @@ public void TestDirectoryPreserveSMBPropertiesResume() options.AfterAllItemAdded = () => { // Wait until there are data transferred - progressChecker.DataTransferred.WaitOne(); + bool gotProgress = progressChecker.DataTransferred.WaitOne(60000); + + Test.Assert(gotProgress, "Should got progress"); if (!IsStreamJournal) { @@ -561,8 +563,251 @@ public void TestPreserveSMBProperties() } [TestCategory(Tag.Function)] - [DMLibTestMethod(DMLibDataType.CloudFile, DMLibDataType.CloudFile)] - [DMLibTestMethod(DMLibDataType.CloudFile, DMLibDataType.CloudFile, DMLibCopyMethod.ServiceSideSyncCopy)] + [DMLibTestMethod(DMLibDataType.CloudFile)] + [DMLibTestMethod(DMLibDataType.CloudFile, DMLibCopyMethod.ServiceSideSyncCopy)] + [DMLibTestMethod(DMLibDataType.CloudFile, DMLibCopyMethod.ServiceSideAsyncCopy)] + public void TestCopySMBPropertiesACL() + { + CloudFileNtfsAttributes[] SMBFileAttributes = { + CloudFileNtfsAttributes.ReadOnly, + CloudFileNtfsAttributes.Hidden, + + CloudFileNtfsAttributes.System, + CloudFileNtfsAttributes.Archive, + CloudFileNtfsAttributes.Normal, + CloudFileNtfsAttributes.Temporary, + CloudFileNtfsAttributes.Offline, + CloudFileNtfsAttributes.NotContentIndexed, + CloudFileNtfsAttributes.NoScrubData, + + CloudFileNtfsAttributes.ReadOnly | CloudFileNtfsAttributes.Hidden, + CloudFileNtfsAttributes.System | CloudFileNtfsAttributes.Archive, + CloudFileNtfsAttributes.Temporary | CloudFileNtfsAttributes.Offline, + CloudFileNtfsAttributes.NotContentIndexed | CloudFileNtfsAttributes.NoScrubData, + + CloudFileNtfsAttributes.ReadOnly | + CloudFileNtfsAttributes.Hidden | + CloudFileNtfsAttributes.System | + CloudFileNtfsAttributes.Archive | + CloudFileNtfsAttributes.Temporary | + CloudFileNtfsAttributes.NotContentIndexed | + CloudFileNtfsAttributes.NoScrubData + }; + + string sampleSDDL = "O:S-1-5-21-2146773085-903363285-719344707-1375029G:S-1-5-21-2146773085-903363285-719344707-513D:(A;ID;FA;;;BA)(A;OICIIOID;GA;;;BA)(A;ID;FA;;;SY)(A;OICIIOID;GA;;;SY)(A;ID;0x1301bf;;;AU)(A;OICIIOID;SDGXGWGR;;;AU)(A;ID;0x1200a9;;;BU)(A;OICIIOID;GXGR;;;BU)"; + + for (int i = 0; i < SMBFileAttributes.Length; ++i) + { + // Prepare data + DMLibDataInfo sourceDataInfo = new DMLibDataInfo(""); + FileNode fileNode = new FileNode(DMLibTestBase.FileName); + fileNode.SizeInByte = 1024; + fileNode.SMBAttributes = SMBFileAttributes[i]; + fileNode.PortableSDDL = sampleSDDL; + sourceDataInfo.RootNode.AddFileNode(fileNode); + + SingleTransferContext transferContext = new SingleTransferContext(); + + var options = new TestExecutionOptions(); + + options.TransferItemModifier = (fileNodeVar, transferItem) => + { + transferItem.TransferContext = transferContext; + + dynamic transferOptions = DefaultTransferOptions; + transferOptions.PreserveSMBAttributes = true; + transferOptions.PreserveSMBPermissions = true; + transferItem.Options = transferOptions; + }; + + // Execute test case + var result = this.ExecuteTestCase(sourceDataInfo, options); + + VerificationHelper.VerifyTransferSucceed(result, sourceDataInfo); + + Helper.CompareSMBProperties(sourceDataInfo.RootNode, result.DataInfo.RootNode, true); + Helper.CompareSMBPermissions( + sourceDataInfo.RootNode, + result.DataInfo.RootNode, + PreserveSMBPermissions.Owner | PreserveSMBPermissions.Group | PreserveSMBPermissions.DACL | PreserveSMBPermissions.SACL); + } + } + + + [TestCategory(Tag.Function)] + [DMLibTestMethod(DMLibDataType.CloudFile)] + [DMLibTestMethod(DMLibDataType.CloudFile, DMLibCopyMethod.ServiceSideSyncCopy)] + [DMLibTestMethod(DMLibDataType.CloudFile, DMLibCopyMethod.ServiceSideAsyncCopy)] + public void TestCopyDirectorySMBPropertiesACL() + { + CloudFileNtfsAttributes[] SMBFileAttributes = { + CloudFileNtfsAttributes.ReadOnly | CloudFileNtfsAttributes.Hidden, + CloudFileNtfsAttributes.System | CloudFileNtfsAttributes.Archive, + CloudFileNtfsAttributes.Offline |CloudFileNtfsAttributes.NotContentIndexed | CloudFileNtfsAttributes.NoScrubData, + + CloudFileNtfsAttributes.ReadOnly | + CloudFileNtfsAttributes.Hidden | + CloudFileNtfsAttributes.System | + CloudFileNtfsAttributes.Archive | + CloudFileNtfsAttributes.NotContentIndexed | + CloudFileNtfsAttributes.NoScrubData + }; + + string sampleSDDL = "O:S-1-5-21-2146773085-903363285-719344707-1375029G:S-1-5-21-2146773085-903363285-719344707-513D:(A;ID;FA;;;BA)(A;OICIIOID;GA;;;BA)(A;ID;FA;;;SY)(A;OICIIOID;GA;;;SY)(A;ID;0x1301bf;;;AU)(A;OICIIOID;SDGXGWGR;;;AU)(A;ID;0x1200a9;;;BU)(A;OICIIOID;GXGR;;;BU)"; + + for (int i = 0; i < SMBFileAttributes.Length; ++i) + { + Test.Info("Testing setting attributes {0} to directories", SMBFileAttributes[i]); + // Prepare data + DMLibDataInfo sourceDataInfo = new DMLibDataInfo(""); + GenerateDirNodeWithAttributes(sourceDataInfo.RootNode, 2, SMBFileAttributes[i], sampleSDDL); + + DirectoryTransferContext dirTransferContext = new DirectoryTransferContext(); + + var options = new TestExecutionOptions(); + options.IsDirectoryTransfer = true; + + options.TransferItemModifier = (fileNode, transferItem) => + { + transferItem.TransferContext = dirTransferContext; + + dynamic transferOptions = DefaultTransferDirectoryOptions; + transferOptions.Recursive = true; + transferOptions.PreserveSMBAttributes = true; + transferOptions.PreserveSMBPermissions = true; + transferItem.Options = transferOptions; + }; + + // Execute test case + var result = this.ExecuteTestCase(sourceDataInfo, options); + + VerificationHelper.VerifyTransferSucceed(result, sourceDataInfo); + + Helper.CompareSMBProperties(sourceDataInfo.RootNode, result.DataInfo.RootNode, true); + Helper.CompareSMBPermissions( + sourceDataInfo.RootNode, + result.DataInfo.RootNode, + PreserveSMBPermissions.Owner | PreserveSMBPermissions.Group | PreserveSMBPermissions.DACL | PreserveSMBPermissions.SACL); + } + } + + [TestCategory(Tag.Function)] + [DMLibTestMethod(DMLibDataType.CloudFile)] + [DMLibTestMethod(DMLibDataType.CloudFile, DMLibCopyMethod.ServiceSideSyncCopy)] + [DMLibTestMethod(DMLibDataType.CloudFile, DMLibCopyMethod.ServiceSideAsyncCopy)] + public void TestCopyDirectorySMBPropertiesACLResume() + { + CloudFileNtfsAttributes[] SMBFileAttributes = { + CloudFileNtfsAttributes.ReadOnly | CloudFileNtfsAttributes.Hidden, + CloudFileNtfsAttributes.System | CloudFileNtfsAttributes.Archive, + CloudFileNtfsAttributes.Offline |CloudFileNtfsAttributes.NotContentIndexed | CloudFileNtfsAttributes.NoScrubData, + + CloudFileNtfsAttributes.ReadOnly | + CloudFileNtfsAttributes.Hidden | + CloudFileNtfsAttributes.System | + CloudFileNtfsAttributes.Archive | + CloudFileNtfsAttributes.NotContentIndexed | + CloudFileNtfsAttributes.NoScrubData + }; + + string sampleSDDL = "O:S-1-5-21-2146773085-903363285-719344707-1375029G:S-1-5-21-2146773085-903363285-719344707-513D:(A;ID;FA;;;BA)(A;OICIIOID;GA;;;BA)(A;ID;FA;;;SY)(A;OICIIOID;GA;;;SY)(A;ID;0x1301bf;;;AU)(A;OICIIOID;SDGXGWGR;;;AU)(A;ID;0x1200a9;;;BU)(A;OICIIOID;GXGR;;;BU)"; + + for (int i = 0; i < SMBFileAttributes.Length; ++i) + { + Test.Info("Testing setting attributes {0} to directories", SMBFileAttributes[i]); + // Prepare data + DMLibDataInfo sourceDataInfo = new DMLibDataInfo(""); + GenerateDirNodeWithAttributes(sourceDataInfo.RootNode, 2, SMBFileAttributes[i], sampleSDDL); + + CancellationTokenSource tokenSource = new CancellationTokenSource(); + TransferItem transferItem = null; + + bool IsStreamJournal = random.Next(0, 2) == 0; + using (Stream journalStream = new MemoryStream()) + { + DirectoryTransferContext dirTransferContext = null; + + if (IsStreamJournal) + { + dirTransferContext = new DirectoryTransferContext(journalStream); + } + else + { + dirTransferContext = new DirectoryTransferContext(); + } + + var progressChecker = new ProgressChecker(14, 14 * 1024); + dirTransferContext.ProgressHandler = progressChecker.GetProgressHandler(); + + var options = new TestExecutionOptions(); + options.IsDirectoryTransfer = true; + + options.TransferItemModifier = (fileNode, item) => + { + item.TransferContext = dirTransferContext; + + dynamic transferOptions = DefaultTransferDirectoryOptions; + transferOptions.Recursive = true; + transferOptions.PreserveSMBPermissions = true; + transferOptions.PreserveSMBAttributes = true; + item.Options = transferOptions; + item.CancellationToken = tokenSource.Token; + transferItem = item; + }; + + TransferCheckpoint checkpoint = null; + + options.AfterAllItemAdded = () => + { + // Wait until there are data transferred + progressChecker.DataTransferred.WaitOne(); + + if (!IsStreamJournal) + { + checkpoint = dirTransferContext.LastCheckpoint; + } + + // Cancel the transfer and store the second checkpoint + tokenSource.Cancel(); + }; + + // Execute test case + var result = this.ExecuteTestCase(sourceDataInfo, options); + + Test.Assert(result.Exceptions.Count == 1, "Verify job is cancelled"); + Exception exception = result.Exceptions[0]; + Helper.VerifyCancelException(exception); + + TransferItem resumeItem = transferItem.Clone(); + DirectoryTransferContext resumeContext = null; + journalStream.Position = 0; + if (IsStreamJournal) + { + resumeContext = new DirectoryTransferContext(journalStream); + } + else + { + resumeContext = new DirectoryTransferContext(DMLibTestHelper.RandomReloadCheckpoint(checkpoint)); + } + + resumeItem.TransferContext = resumeContext; + + result = this.RunTransferItems(new List() { resumeItem }, new TestExecutionOptions()); + VerificationHelper.VerifyTransferSucceed(result, sourceDataInfo); + + Helper.CompareSMBProperties(sourceDataInfo.RootNode, result.DataInfo.RootNode, true); + Helper.CompareSMBPermissions( + sourceDataInfo.RootNode, + result.DataInfo.RootNode, + PreserveSMBPermissions.Owner | PreserveSMBPermissions.Group | PreserveSMBPermissions.DACL | PreserveSMBPermissions.SACL); + } + } + } + + [TestCategory(Tag.Function)] + [DMLibTestMethod(DMLibDataType.CloudFile)] + [DMLibTestMethod(DMLibDataType.CloudFile, DMLibCopyMethod.ServiceSideSyncCopy)] + [DMLibTestMethod(DMLibDataType.CloudFile, DMLibCopyMethod.ServiceSideAsyncCopy)] public void TestDirectoryMeta() { // Prepare data @@ -720,7 +965,7 @@ public void TestDirectoryPreserveSMBPermissions() // Prepare data DMLibDataInfo sourceDataInfo = new DMLibDataInfo(""); string sampleSDDL = "O:S-1-5-21-2146773085-903363285-719344707-1375029G:S-1-5-21-2146773085-903363285-719344707-513D:(A;ID;FA;;;BA)(A;OICIIOID;GA;;;BA)(A;ID;FA;;;SY)(A;OICIIOID;GA;;;SY)(A;ID;0x1301bf;;;AU)(A;OICIIOID;SDGXGWGR;;;AU)(A;ID;0x1200a9;;;BU)(A;OICIIOID;GXGR;;;BU)"; - GenerateDirNodeWithPermissions(sourceDataInfo.RootNode, 2, sampleSDDL); + GenerateDirNodeWithAttributes(sourceDataInfo.RootNode, 2, null, sampleSDDL); DirectoryTransferContext dirTransferContext = new DirectoryTransferContext(); @@ -872,7 +1117,7 @@ public void TestDirectoryPreserveSMBPermissionsResume() { // Prepare data DMLibDataInfo sourceDataInfo = new DMLibDataInfo(""); - GenerateDirNodeWithPermissions(sourceDataInfo.RootNode, 2, sampleSDDL); + GenerateDirNodeWithAttributes(sourceDataInfo.RootNode, 2, null, sampleSDDL); DirectoryTransferContext dirTransferContext = null; @@ -908,7 +1153,9 @@ public void TestDirectoryPreserveSMBPermissionsResume() options.AfterAllItemAdded = () => { // Wait until there are data transferred - progressChecker.DataTransferred.WaitOne(); + bool gotProgress = progressChecker.DataTransferred.WaitOne(60000); + + Test.Assert(gotProgress, "Should got progress"); if (!IsStreamJournal) { @@ -1008,51 +1255,22 @@ private void GenerateDirNodeWithMetadata( parent.AddDirNode(dirNode); } - private void GenerateDirNodeWithPermissions( - DirNode parent, - int dirLevel, - string permissions) - { - FileNode fileNode = new FileNode(DMLibTestBase.FileName + "_0"); - fileNode.SizeInByte = 1024; - fileNode.PortableSDDL = permissions; - parent.AddFileNode(fileNode); - - fileNode = new FileNode(DMLibTestBase.FileName + "_1"); - fileNode.SizeInByte = 1024; - fileNode.PortableSDDL = permissions; - parent.AddFileNode(fileNode); - - if (dirLevel <= 0) - { - return; - } - - --dirLevel; - DirNode dirNode = new DirNode(DMLibTestBase.DirName + "_0"); - dirNode.PortableSDDL = permissions; - this.GenerateDirNodeWithPermissions(dirNode, dirLevel, permissions); - parent.AddDirNode(dirNode); - - dirNode = new DirNode(DMLibTestBase.DirName + "_1"); - dirNode.PortableSDDL = permissions; - this.GenerateDirNodeWithPermissions(dirNode, dirLevel, permissions); - parent.AddDirNode(dirNode); - } - private void GenerateDirNodeWithAttributes( DirNode parent, int dirLevel, - CloudFileNtfsAttributes smbAttributes) + CloudFileNtfsAttributes? smbAttributes, + string permissions) { FileNode fileNode = new FileNode(DMLibTestBase.FileName + "_0"); fileNode.SizeInByte = 1024; fileNode.SMBAttributes = smbAttributes; + fileNode.PortableSDDL = permissions; parent.AddFileNode(fileNode); fileNode = new FileNode(DMLibTestBase.FileName + "_1"); fileNode.SizeInByte = 1024; fileNode.SMBAttributes = smbAttributes; + fileNode.PortableSDDL = permissions; parent.AddFileNode(fileNode); if (dirLevel <= 0) @@ -1062,7 +1280,7 @@ private void GenerateDirNodeWithAttributes( --dirLevel; DirNode dirNode = new DirNode(DMLibTestBase.DirName + "_0"); - this.GenerateDirNodeWithAttributes(dirNode, dirLevel, smbAttributes); + this.GenerateDirNodeWithAttributes(dirNode, dirLevel, smbAttributes, permissions); if (smbAttributes == CloudFileNtfsAttributes.Normal) { dirNode.SMBAttributes = CloudFileNtfsAttributes.Directory; @@ -1071,10 +1289,11 @@ private void GenerateDirNodeWithAttributes( { dirNode.SMBAttributes = CloudFileNtfsAttributes.Directory | smbAttributes; } + dirNode.PortableSDDL = permissions; parent.AddDirNode(dirNode); dirNode = new DirNode(DMLibTestBase.DirName + "_1"); - this.GenerateDirNodeWithAttributes(dirNode, dirLevel, smbAttributes); + this.GenerateDirNodeWithAttributes(dirNode, dirLevel, smbAttributes, permissions); if (smbAttributes == CloudFileNtfsAttributes.Normal) { dirNode.SMBAttributes = CloudFileNtfsAttributes.Directory; @@ -1083,6 +1302,7 @@ private void GenerateDirNodeWithAttributes( { dirNode.SMBAttributes = CloudFileNtfsAttributes.Directory | smbAttributes; } + dirNode.PortableSDDL = permissions; parent.AddDirNode(dirNode); } } diff --git a/test/DMLibTest/Cases/SetAttributesTest.cs b/test/DMLibTest/Cases/SetAttributesTest.cs index d359aa37..f9f452b8 100644 --- a/test/DMLibTest/Cases/SetAttributesTest.cs +++ b/test/DMLibTest/Cases/SetAttributesTest.cs @@ -17,6 +17,9 @@ namespace DMLibTest.Cases using Microsoft.Azure.Storage.DataMovement; using MS.Test.Common.MsTestLib; + using Microsoft.Azure.Storage.Blob; + using Microsoft.Azure.Storage.File; + using System.Threading.Tasks; [MultiDirectionTestClass] public class SetAttributesTest : DMLibTestBase @@ -118,7 +121,7 @@ private void TestSetAttributesToLocal(bool IsDirectoryTransfer) { context = new DirectoryTransferContext() { - SetAttributesCallbackAsync = async (destObj) => + SetAttributesCallbackAsync = async (sourceObj, destObj) => { Test.Error("SetAttributes callback should not be invoked when destination is local"); } @@ -128,7 +131,7 @@ private void TestSetAttributesToLocal(bool IsDirectoryTransfer) { context = new SingleTransferContext() { - SetAttributesCallbackAsync = async (destObj) => + SetAttributesCallbackAsync = async (sourceObj, destObj) => { Test.Error("SetAttributes callback should not be invoked when destination is local"); } @@ -177,20 +180,20 @@ public void TestSetAttributes() TransferContext context = new SingleTransferContext() { - SetAttributesCallbackAsync = async (destObj) => - { - dynamic destCloudObj = destObj; - - destCloudObj.Properties.ContentType = SetAttributesTest.TestContentType; - - destCloudObj.Metadata.Add("aa", "bb"); - } + SetAttributesCallbackAsync = SetAttributesCallbackTest }; var options = new TestExecutionOptions(); options.TransferItemModifier = (node, transferItem) => { dynamic transferOptions = DefaultTransferOptions; + if ((DMLibTestContext.SourceType == DMLibDataType.CloudFile) + && (DMLibTestContext.DestType == DMLibDataType.CloudFile)) + { + transferOptions.PreserveSMBPermissions = true; + transferOptions.PreserveSMBAttributes = true; + } + transferItem.Options = transferOptions; transferItem.TransferContext = context; }; @@ -198,6 +201,7 @@ public void TestSetAttributes() var result = this.ExecuteTestCase(sourceDataInfo, options); fileNode.Metadata.Add("aa", "bb"); + fileNode.Metadata.Add("filelength", fileNode.SizeInByte.ToString()); VerificationHelper.VerifyTransferSucceed(result, sourceDataInfo); @@ -208,6 +212,16 @@ public void TestSetAttributes() { Test.Assert(SetAttributesTest.TestContentLanguage.Equals(destFileNode.ContentLanguage), "Verify ContentLanguage: {0}, expected {1}", destFileNode.ContentLanguage, SetAttributesTest.TestContentLanguage); } + + if (DMLibTestContext.SourceType == DMLibDataType.CloudFile + && DMLibTestContext.DestType == DMLibDataType.CloudFile) + { + Helper.CompareSMBProperties(sourceDataInfo.RootNode, result.DataInfo.RootNode, true); + Helper.CompareSMBPermissions( + sourceDataInfo.RootNode, + result.DataInfo.RootNode, + PreserveSMBPermissions.Owner | PreserveSMBPermissions.Group | PreserveSMBPermissions.DACL | PreserveSMBPermissions.SACL); + } } [TestCategory(Tag.Function)] @@ -235,14 +249,7 @@ public void TestDirectorySetAttributes() DirectoryTransferContext context = new DirectoryTransferContext() { - SetAttributesCallbackAsync = async (destObj) => - { - dynamic destCloudObj = destObj; - - destCloudObj.Properties.ContentType = SetAttributesTest.TestContentType; - - destCloudObj.Metadata.Add("aa", "bb"); - } + SetAttributesCallbackAsync = SetAttributesCallbackTest }; var options = new TestExecutionOptions(); @@ -250,6 +257,7 @@ public void TestDirectorySetAttributes() { dynamic transferOptions = DefaultTransferDirectoryOptions; transferOptions.Recursive = true; + transferItem.Options = transferOptions; transferItem.TransferContext = context; }; @@ -260,6 +268,7 @@ public void TestDirectorySetAttributes() foreach (FileNode fileNode in sourceDataInfo.EnumerateFileNodes()) { fileNode.Metadata.Add("aa", "bb"); + fileNode.Metadata.Add("filelength", fileNode.SizeInByte.ToString()); } VerificationHelper.VerifyTransferSucceed(result, sourceDataInfo); @@ -280,6 +289,8 @@ public void TestDirectorySetAttributes() [DMLibTestMethod(DMLibDataType.CloudBlob, DMLibDataType.CloudFile)] [DMLibTestMethod(DMLibDataType.CloudBlob)] [DMLibTestMethod(DMLibDataType.CloudBlob, DMLibCopyMethod.ServiceSideSyncCopy)] + [DMLibTestMethod(DMLibDataType.CloudFile)] + [DMLibTestMethod(DMLibDataType.CloudFile, DMLibCopyMethod.ServiceSideSyncCopy)] public void TestDirectorySetAttribute_Restart_Copy() { int bigFileSizeInKB = 5 * 1024; // 5 MB @@ -363,8 +374,19 @@ public void TestDirectorySetAttribute_Restart_Upload() smallFileNum, smallFileSizeInKB); }, - async (destObj) => + async (sourceObj, destObj) => { + long sourceLength = 0; + string path = sourceObj as string; + if (null == path) + { + Test.Error("Source should be a local file path."); + } + else + { + sourceLength = new System.IO.FileInfo(path).Length; + } + dynamic destCloudObj = destObj; destCloudObj.Properties.ContentType = RandomLongString; @@ -377,6 +399,7 @@ public void TestDirectorySetAttribute_Restart_Upload() destCloudObj.Metadata.Remove(MetadataKey1); destCloudObj.Metadata.Add(MetadataKey1, RandomLongString); + destCloudObj.Metadata["filelength"] = sourceLength.ToString(); }, sourceDataInfo => { @@ -391,11 +414,98 @@ public void TestDirectorySetAttribute_Restart_Upload() fileNode.MD5 = invalidMD5; fileNode.Metadata = new Dictionary { { MetadataKey1, RandomLongString } }; + fileNode.Metadata["filelength"] = fileNode.SizeInByte.ToString(); } } ); } + private async Task SetAttributesCallbackTest(object sourceObj, object destObj) + { + await Task.Yield(); + long sourceLength = 0; + switch (DMLibTestContext.SourceType) + { + case DMLibDataType.Local: + string path = sourceObj as string; + if (null == path) + { + Test.Error("Source should be a local file path."); + } + else + { + sourceLength = new System.IO.FileInfo(path).Length; + } + break; + case DMLibDataType.Stream: + Stream stream = sourceObj as Stream; + if (null == stream) + { + Test.Error("Source should be a local stream."); + } + else + { + sourceLength = stream.Length; + } + break; + case DMLibDataType.BlockBlob: + CloudBlockBlob blockBlob = sourceObj as CloudBlockBlob; + + if (null == blockBlob) + { + Test.Error("Source should be a CloudBlockBlob."); + } + else + { + sourceLength = blockBlob.Properties.Length; + } + break; + case DMLibDataType.PageBlob: + CloudPageBlob pageBlob = sourceObj as CloudPageBlob; + + if (null == pageBlob) + { + Test.Error("Source should be a CloudPageBlob."); + } + else + { + sourceLength = pageBlob.Properties.Length; + } + break; + case DMLibDataType.AppendBlob: + CloudAppendBlob appendBlob = sourceObj as CloudAppendBlob; + + if (null == appendBlob) + { + Test.Error("Source should be a CloudAppendBlob."); + } + else + { + sourceLength = appendBlob.Properties.Length; + } + break; + case DMLibDataType.CloudFile: + CloudFile cloudFile = sourceObj as CloudFile; + + if (null == cloudFile) + { + Test.Error("Source should be a CloudFile."); + } + else + { + sourceLength = cloudFile.Properties.Length; + } + break; + default: + break; + } + + dynamic destCloudObj = destObj; + destCloudObj.Properties.ContentType = SetAttributesTest.TestContentType; + destCloudObj.Metadata["aa"] = "bb"; + destCloudObj.Metadata["filelength"] = sourceLength.ToString(); + } + private void TestDirectorySetAttribute_Restart( int bigFileSizeInKB, int smallFileSizeInKB, diff --git a/test/DMLibTest/Cases/ShouldTransferTest.cs b/test/DMLibTest/Cases/ShouldTransferTest.cs index 4e1484eb..0085f582 100644 --- a/test/DMLibTest/Cases/ShouldTransferTest.cs +++ b/test/DMLibTest/Cases/ShouldTransferTest.cs @@ -121,6 +121,13 @@ public void DirectoryShouldTransfer() dynamic transferOptions = DefaultTransferDirectoryOptions; transferOptions.Recursive = true; + if (DMLibTestContext.SourceType == DMLibDataType.CloudFile + && DMLibTestContext.DestType == DMLibDataType.CloudFile) + { + transferOptions.PreserveSMBAttributes = true; + transferOptions.PreserveSMBPermissions = true; + } + transferItem.Options = transferOptions; }; @@ -139,6 +146,16 @@ public void DirectoryShouldTransfer() Test.Assert(expectedTransferred == transferred, string.Format("Verify transferred number. Expected: {0}, Actual: {1}", expectedTransferred, transferred)); Test.Assert(expectedSkipped == skipped, string.Format("Verify skipped number. Expected: {0}, Actual: {1}", expectedSkipped, skipped)); Test.Assert(expectedFailed == failed, string.Format("Verify failed number. Expected: {0}, Actual: {1}", expectedFailed, failed)); + + if (DMLibTestContext.SourceType == DMLibDataType.CloudFile + && DMLibTestContext.DestType == DMLibDataType.CloudFile) + { + Helper.CompareSMBProperties(expectedDataInfo.RootNode, result.DataInfo.RootNode, true); + Helper.CompareSMBPermissions( + expectedDataInfo.RootNode, + result.DataInfo.RootNode, + PreserveSMBPermissions.Owner | PreserveSMBPermissions.Group | PreserveSMBPermissions.DACL | PreserveSMBPermissions.SACL); + } } } } diff --git a/test/DMLibTest/Cases/SnapshotTest.cs b/test/DMLibTest/Cases/SnapshotTest.cs index 5c9889d0..9f6f23b9 100644 --- a/test/DMLibTest/Cases/SnapshotTest.cs +++ b/test/DMLibTest/Cases/SnapshotTest.cs @@ -170,7 +170,7 @@ public void TestDirectoryNotIncludeSnapshots() [DMLibTestMethod(DMLibDataType.CloudBlob, DMLibCopyMethod.ServiceSideSyncCopy)] public void TestDirectoryWithSpecialCharNamedBlobs() { -#if DNXCORE50 +#if DOTNET5_4 // TODO: There's a known issue that signature for URI with '[' or ']' doesn't work. string specialChars = "~`!@#$%()-_+={};?.^&"; #else diff --git a/test/DMLibTest/DMLibTest.csproj b/test/DMLibTest/DMLibTest.csproj index 5f5716f7..d92b9a4d 100644 --- a/test/DMLibTest/DMLibTest.csproj +++ b/test/DMLibTest/DMLibTest.csproj @@ -56,14 +56,14 @@ ..\..\packages\Microsoft.Azure.KeyVault.Core.1.0.0\lib\net40\Microsoft.Azure.KeyVault.Core.dll True - - ..\..\packages\Microsoft.Azure.Storage.Blob.11.1.2\lib\net452\Microsoft.Azure.Storage.Blob.dll + + ..\..\packages\Microsoft.Azure.Storage.Blob.11.2.2\lib\net452\Microsoft.Azure.Storage.Blob.dll - - ..\..\packages\Microsoft.Azure.Storage.Common.11.1.2\lib\net452\Microsoft.Azure.Storage.Common.dll + + ..\..\packages\Microsoft.Azure.Storage.Common.11.2.2\lib\net452\Microsoft.Azure.Storage.Common.dll - - ..\..\packages\Microsoft.Azure.Storage.File.11.1.2\lib\net452\Microsoft.Azure.Storage.File.dll + + ..\..\packages\Microsoft.Azure.Storage.File.11.2.2\lib\net452\Microsoft.Azure.Storage.File.dll @@ -119,6 +119,7 @@ + diff --git a/test/DMLibTest/Framework/CloudBlobDataAdaptor.cs b/test/DMLibTest/Framework/CloudBlobDataAdaptor.cs index e7599f55..0f02a250 100644 --- a/test/DMLibTest/Framework/CloudBlobDataAdaptor.cs +++ b/test/DMLibTest/Framework/CloudBlobDataAdaptor.cs @@ -278,6 +278,7 @@ private void BuildFileNode(CloudBlob cloudBlob, FileNode fileNode) fileNode.ContentDisposition = cloudBlob.Properties.ContentDisposition; fileNode.ContentEncoding = cloudBlob.Properties.ContentEncoding; fileNode.ContentLanguage = cloudBlob.Properties.ContentLanguage; + fileNode.EncryptionScope = cloudBlob.Properties.EncryptionScope; fileNode.Metadata = cloudBlob.Metadata; DateTimeOffset dateTimeOffset = (DateTimeOffset)cloudBlob.Properties.LastModified; diff --git a/test/DMLibTest/Framework/DMLibDataInfo.cs b/test/DMLibTest/Framework/DMLibDataInfo.cs index 39648137..d248164b 100644 --- a/test/DMLibTest/Framework/DMLibDataInfo.cs +++ b/test/DMLibTest/Framework/DMLibDataInfo.cs @@ -145,6 +145,12 @@ public string ContentLanguage set; } + public string EncryptionScope + { + get; + set; + } + public CloudFileNtfsAttributes? SMBAttributes { get; @@ -240,7 +246,8 @@ public FileNode Clone(string name = null) AbsolutePath = this.AbsolutePath, SMBAttributes = this.SMBAttributes, CreationTime = this.CreationTime, - LastWriteTime = this.LastWriteTime + LastWriteTime = this.LastWriteTime, + PortableSDDL = this.PortableSDDL, }; } } @@ -502,6 +509,7 @@ public DirNode Clone() newDirNode.SMBAttributes = this.SMBAttributes; newDirNode.CreationTime = this.CreationTime; newDirNode.LastWriteTime = this.LastWriteTime; + newDirNode.PortableSDDL = this.PortableSDDL; if (null != this.Metadata) { diff --git a/test/DMLibTest/Framework/LongPathFileStreamExtension.cs b/test/DMLibTest/Framework/LongPathFileStreamExtension.cs index 8f363bcb..b3388717 100644 --- a/test/DMLibTest/Framework/LongPathFileStreamExtension.cs +++ b/test/DMLibTest/Framework/LongPathFileStreamExtension.cs @@ -188,6 +188,10 @@ public static bool Exists(string path) #if DOTNET5_4 return Directory.Exists(path); #else + if (DMLibTestConstants.SupportUNCPath) + { + path = LongPath.ToUncPath(path); + } return LongPathDirectory.Exists(path); #endif } @@ -216,6 +220,15 @@ public static string[] GetDirectories(string path, SearchOption searchOption = S /// The directory to create. public static void CreateDirectory(string path) { + try + { + path = DMLibTestConstants.SupportUNCPath ? + LongPath.GetFullPath(LongPath.ToUncPath(path)) : + LongPath.GetFullPath(path); + } + catch (Exception) + { } + #if DOTNET5_4 Directory.CreateDirectory(path); #else @@ -346,17 +359,42 @@ public static FileStream Create(string path) public static FileStream Open(string path, FileMode mode, FileAccess access, FileShare share) { + try + { + path = DMLibTestConstants.SupportUNCPath ? + LongPath.GetFullPath(LongPath.ToUncPath(path)) : + LongPath.GetFullPath(path); + } + catch (Exception) + { } + return LongPathFile.Open(path, mode, access, share); } public static FileStream Open(string path, FileMode mode) { + try + { + path = DMLibTestConstants.SupportUNCPath ? + LongPath.GetFullPath(LongPath.ToUncPath(path)) : + LongPath.GetFullPath(path); + } + catch (Exception) + { } + return LongPathFile.Open(path, mode, FileAccess.ReadWrite, FileShare.ReadWrite); } public static void SetAttributes(string path, FileAttributes fileAttributes) { - path = LongPath.ToUncPath(path); + try + { + path = DMLibTestConstants.SupportUNCPath ? + LongPath.GetFullPath(LongPath.ToUncPath(path)) : + LongPath.GetFullPath(path); + } + catch (Exception) + { } LongPathFile.SetAttributes(path, fileAttributes); } @@ -366,21 +404,45 @@ public static void GetFileProperties(string path, out DateTimeOffset? creationTi #endif ) { + try + { + path = DMLibTestConstants.SupportUNCPath ? + LongPath.GetFullPath(LongPath.ToUncPath(path)) : + LongPath.GetFullPath(path); + } + catch (Exception) + { } + #if DOTNET5_4 LongPathFile.GetFileProperties(path, out creationTime, out lastWriteTime, out fileAttributes, isDirectory); #else - path = LongPath.ToUncPath(path); LongPathFile.GetFileProperties(path, out creationTime, out lastWriteTime, out fileAttributes); #endif } public static string GetFilePortableSDDL(string path, PreserveSMBPermissions preserveSMBPermissions) { + try + { + path = DMLibTestConstants.SupportUNCPath ? + LongPath.GetFullPath(LongPath.ToUncPath(path)) : + LongPath.GetFullPath(path); + } + catch (Exception) + { } return FileSecurityOperations.GetFilePortableSDDL(path, preserveSMBPermissions); } public static void SetFileSecurityInfo(string path, string portableSDDL, PreserveSMBPermissions preserveSMBPermissions) { + try + { + path = DMLibTestConstants.SupportUNCPath ? + LongPath.GetFullPath(LongPath.ToUncPath(path)) : + LongPath.GetFullPath(path); + } + catch (Exception) + { } FileSecurityOperations.SetFileSecurity(path, portableSDDL, preserveSMBPermissions); } } diff --git a/test/DMLibTest/Util/DMLibTestConstants.cs b/test/DMLibTest/Util/DMLibTestConstants.cs index 8031df2f..96e37bc4 100644 --- a/test/DMLibTest/Util/DMLibTestConstants.cs +++ b/test/DMLibTest/Util/DMLibTestConstants.cs @@ -11,6 +11,7 @@ namespace DMLibTest using Microsoft.Azure.Storage.RetryPolicies; using Microsoft.Azure.Storage.Blob; using Microsoft.Azure.Storage.File; + using System.IO; public static class Tag { @@ -49,12 +50,15 @@ public static class DMLibTestConstants { public const string ConnStr = "StorageConnectionString"; public const string ConnStr2 = "StorageConnectionString2"; + public const string DestEncryptionScope = "DestinationEncryptionScope"; public static readonly int DefaultNC = TransferManager.Configurations.ParallelOperations; public static readonly int DefaultBlockSize = 4 * 1024 * 1024; //4MB public static readonly int LimitedSpeedNC = 4; public static readonly TimeSpan DefaultExecutionTimeOut = TimeSpan.FromMinutes(15); private static Random random = new Random(); + + public static bool SupportUNCPath = true; public static IRetryPolicy DefaultRetryPolicy { @@ -120,5 +124,20 @@ public static int RecursiveFolderDepth return recursiveFolderDepth; } } + + static DMLibTestConstants() + { + SupportUNCPath = false; + if (CrossPlatformHelpers.IsWindows) + { + try + { + LongPath.GetFullPath(LongPath.ToUncPath("F:\\")); + SupportUNCPath = true; + } + catch (Exception) + { } + } + } } } diff --git a/test/DMLibTest/Util/Helpers.cs b/test/DMLibTest/Util/Helpers.cs index 81f99326..72b253cd 100644 --- a/test/DMLibTest/Util/Helpers.cs +++ b/test/DMLibTest/Util/Helpers.cs @@ -159,7 +159,21 @@ private static void ComparePortableSDDLString(string sourceSDDL, string destSDDL if ((preserveSMBPermissions & PreserveSMBPermissions.SACL) == PreserveSMBPermissions.SACL) { - Test.Assert(string.Equals(sourcesddlStrings[3], destsddlStrings[3]), "SDDL Value should be expected."); + bool sddlExpected = false; + if (string.IsNullOrEmpty(sourcesddlStrings[3])) + { + sddlExpected = string.IsNullOrEmpty(destsddlStrings[3]) || string.Equals(destsddlStrings[3], "S:NO_ACCESS_CONTROL"); + } + else if (string.IsNullOrEmpty(destsddlStrings[3])) + { + sddlExpected = string.IsNullOrEmpty(sourcesddlStrings[3]) || string.Equals(sourcesddlStrings[3], "S:NO_ACCESS_CONTROL"); + } + else + { + sddlExpected = string.Equals(sourcesddlStrings[3], destsddlStrings[3]); + } + + Test.Assert(sddlExpected, "SDDL Value should be expected."); } } @@ -274,13 +288,13 @@ public static void CompareSMBProperties(DirNode dirNode0, DirNode dirNode1, bool if ((fileNode.LastWriteTime.Value != fileNode1.LastWriteTime.Value) || (fileNode.CreationTime.Value != fileNode1.CreationTime.Value)) { - Test.Error("File node mismatch"); + Test.Error($"File node mismatch {fileNode.Name} {fileNode.LastWriteTime.Value} == {fileNode1.LastWriteTime.Value} {fileNode.CreationTime.Value} == {fileNode1.CreationTime.Value}"); } if (compareFileAttributes && fileNode.SMBAttributes.Value != fileNode1.SMBAttributes.Value) { - Test.Error("File node mismatch"); + Test.Error($"File node mismatch {fileNode.SMBAttributes.Value} == {fileNode1.SMBAttributes.Value}"); } } diff --git a/test/DMLibTest/app.config b/test/DMLibTest/app.config index 9cdc7765..c5e37f1b 100644 --- a/test/DMLibTest/app.config +++ b/test/DMLibTest/app.config @@ -12,7 +12,7 @@ - + diff --git a/test/DMLibTest/packages.config b/test/DMLibTest/packages.config index 24dfba2c..6a392612 100644 --- a/test/DMLibTest/packages.config +++ b/test/DMLibTest/packages.config @@ -1,9 +1,9 @@  - - - + + + diff --git a/test/DMLibTest_NetStandard/DMLibTest.csproj b/test/DMLibTest_NetStandard/DMLibTest.csproj new file mode 100644 index 00000000..defd5ddf --- /dev/null +++ b/test/DMLibTest_NetStandard/DMLibTest.csproj @@ -0,0 +1,368 @@ + + + + Debug + AnyCPU + {2A4656A4-F744-4653-A9D6-15112E9AB352} + Library + Properties + DMLibTest + DMLibTest + v4.7.2 + 512 + {3AC096D0-A1C2-E12C-1390-A8335801FDAB};{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC} + 10.0 + $(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion) + $(ProgramFiles)\Common Files\microsoft shared\VSTT\$(VisualStudioVersion)\UITestExtensionPackages + False + UnitTest + ..\..\ + true + + + + true + full + false + bin\Debug\ + TRACE;DEBUG;DMLIB_TEST + prompt + 4 + + + pdbonly + true + bin\Release\ + TRACE;BINARY_SERIALIZATION;DMLIB_TEST + prompt + 4 + + + false + + + true + + + ..\DMLibTestCodeGen\ + $(CodeGenPath)DMLibTestCodeGen.csproj + Generated + + + ..\..\tools\strongnamekeys\fake\windows.snk + + + + ..\..\packages\Microsoft.Azure.KeyVault.Core.1.0.0\lib\net40\Microsoft.Azure.KeyVault.Core.dll + True + + + ..\..\packages\Microsoft.Azure.Storage.Blob.11.2.2\lib\net452\Microsoft.Azure.Storage.Blob.dll + + + ..\..\packages\Microsoft.Azure.Storage.Common.11.2.2\lib\net452\Microsoft.Azure.Storage.Common.dll + + + ..\..\packages\Microsoft.Azure.Storage.File.11.2.2\lib\net452\Microsoft.Azure.Storage.File.dll + + + + ..\..\packages\Microsoft.WindowsAzure.ConfigurationManager.1.8.0.0\lib\net35-full\Microsoft.WindowsAzure.Configuration.dll + True + + + ..\..\packages\Newtonsoft.Json.10.0.2\lib\net45\Newtonsoft.Json.dll + + + + + + + + + + + + + + + + + + + + + + + + + Framework\FileNativeMethods.cs + + + Framework\FileSecurityNativeMethods.cs + + + Framework\FileSecurityOperations.cs + + + Framework\Interop.Windows.cs + + + Framework\LongPathFileStream.cs + + + DataContractSerializerExtensions.cs + + + SharedAssemblyInfo.cs + + + Cases\AccessConditionTest.cs + + + Cases\AllTransferDirectionTest.cs + + + Cases\BigFileTest.cs + + + Cases\BlockSizeTest.cs + + + Cases\BVT.cs + + + Cases\CheckContentMD5Test.cs + + + Cases\ChunkedMemoryStreamTests.cs + + + Cases\DelimiterTest.cs + + + Cases\DummyTransferTest.cs + + + Cases\EncryptionScopeTest.cs + + + Cases\FollowSymlinkTest.cs + + + Cases\LongFilePathTest.cs + + + Cases\MetadataTest.cs + + + Cases\OverwriteTest.cs + + + Cases\ProgressHandlerTest.cs + + + Cases\ResumeTest.cs + + + Cases\SASTokenVersionTest.cs + + + Cases\SearchPatternTest.cs + + + Cases\SetAttributesTest.cs + + + Cases\ShouldTransferTest.cs + + + Cases\SMBPropertiesTest.cs + + + Cases\SnapshotTest.cs + + + Cases\StreamTest.cs + + + Cases\UnsupportedDirectionTest.cs + + + Framework\AssemblyInitCleanup.cs + + + Framework\BlobDataAdaptorBase.cs + + + Framework\CloudBlobDataAdaptor.cs + + + Framework\CloudFileDataAdaptor.cs + + + Framework\CloudObjectExtensions.cs + + + Framework\CopyWrapper.cs + + + Framework\DataAdaptor.cs + + + Framework\DMLibDataHelper.cs + + + Framework\DMLibDataInfo.cs + + + Framework\DMLibDataPreparedTestBase.cs + + + Framework\DMLibInputHelper.cs + + + Framework\DMLibTestBase.cs + + + Framework\DMLibWrapper.cs + + + Framework\DownloadWrapper.cs + + + Framework\FileNativeMethodsExtension.cs + + + Framework\IDataInfo.cs + + + Framework\LocalDataAdaptor.cs + + + Framework\LocalDataAdaptorBase.cs + + + Framework\LongPathFileStreamExtension.cs + + + Framework\MultiDirectionTestBase.cs + + + Framework\MultiDirectionTestHelper.cs + + + Framework\MultiDirectionTestInfo.cs + + + Framework\ProgressChecker.cs + + + Framework\SharedAccessPermissions.cs + + + Framework\TestExecutionOptions.cs + + + Framework\TestResult.cs + + + Framework\TransferEventChecker.cs + + + Framework\TransferItem.cs + + + Framework\UploadWrapper.cs + + + Framework\URIBlobDataAdaptor.cs + + + Framework\VerificationHelper.cs + + + Util\DMLibTestConstants.cs + + + Util\DMLibTestHelper.cs + + + Util\DMLibTestStream.cs + + + Util\Helpers.cs + + + Util\SASGenerator.cs + + + Util\TestAccounts.cs + + + + + + + {ec084ea8-ea44-45ae-9c75-2183ade14a41} + Microsoft.Azure.Storage.DataMovement + + + {7018ee4e-d389-424e-a8dd-f9b4ffda5194} + DMLibTestCodeGen + + + {2c79e154-efd2-4ffd-8a73-5a62a61bc6e5} + DMTestLib + + + {ac39b50f-dc27-4411-9ed4-a4a137190acb} + MsTestLib + + + + + + + + + TestData.xml + + + + + + + False + + + False + + + False + + + False + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/test/DMLibTest_NetStandard/Properties/AssemblyInfo.cs b/test/DMLibTest_NetStandard/Properties/AssemblyInfo.cs new file mode 100644 index 00000000..88a70dcc --- /dev/null +++ b/test/DMLibTest_NetStandard/Properties/AssemblyInfo.cs @@ -0,0 +1,14 @@ +//------------------------------------------------------------------------------ +// +// Copyright (c) Microsoft Corporation +// +//------------------------------------------------------------------------------ +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. +// [assembly: AssemblyTitle("DMLibTest")] +// [assembly: AssemblyDescription("")] diff --git a/test/DMLibTest_NetStandard/app.config b/test/DMLibTest_NetStandard/app.config new file mode 100644 index 00000000..365e1758 --- /dev/null +++ b/test/DMLibTest_NetStandard/app.config @@ -0,0 +1,20 @@ + + + + + + + + + + + + + + + + + + + + diff --git a/test/DMLibTest_NetStandard/packages.config b/test/DMLibTest_NetStandard/packages.config new file mode 100644 index 00000000..71400379 --- /dev/null +++ b/test/DMLibTest_NetStandard/packages.config @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/tools/AssemblyInfo/SharedAssemblyInfo.cs b/tools/AssemblyInfo/SharedAssemblyInfo.cs index 2d7fe68a..b08bc89b 100644 --- a/tools/AssemblyInfo/SharedAssemblyInfo.cs +++ b/tools/AssemblyInfo/SharedAssemblyInfo.cs @@ -12,8 +12,8 @@ using System.Resources; using System.Runtime.InteropServices; -[assembly: AssemblyVersion("1.3.0.0")] -[assembly: AssemblyFileVersion("1.3.0.0")] +[assembly: AssemblyVersion("2.0.0.0")] +[assembly: AssemblyFileVersion("2.0.0.0")] [assembly: AssemblyCompany("Microsoft")] [assembly: AssemblyProduct("Microsoft Azure Storage")] diff --git a/tools/nupkg/Microsoft.Azure.Storage.DataMovement.nuspec b/tools/nupkg/Microsoft.Azure.Storage.DataMovement.nuspec index 3d7fc43e..9d5edab7 100644 --- a/tools/nupkg/Microsoft.Azure.Storage.DataMovement.nuspec +++ b/tools/nupkg/Microsoft.Azure.Storage.DataMovement.nuspec @@ -2,7 +2,7 @@ Microsoft.Azure.Storage.DataMovement - 1.3.0 + 2.0.0 Microsoft Azure Storage Data Movement Library Microsoft Microsoft @@ -19,14 +19,14 @@ - - + + - - + + diff --git a/tools/nupkg/buildNupkg.cmd b/tools/nupkg/buildNupkg.cmd index becafe1a..8ae23cd3 100644 --- a/tools/nupkg/buildNupkg.cmd +++ b/tools/nupkg/buildNupkg.cmd @@ -6,12 +6,12 @@ mkdir .\workspace\lib\net452 mkdir .\workspace\lib\netstandard2.0 pushd ..\.. del /q /f *.nupkg -copy D:\Work\DMLib\1.3.0\Binary\.Net\Microsoft.Azure.Storage.DataMovement.dll .\tools\nupkg\workspace\lib\net452 -copy D:\Work\DMLib\1.3.0\Binary\.Net\Microsoft.Azure.Storage.DataMovement.pdb .\tools\nupkg\workspace\lib\net452 -copy D:\Work\DMLib\1.3.0\Binary\.Net\Microsoft.Azure.Storage.DataMovement.XML .\tools\nupkg\workspace\lib\net452 -copy D:\Work\DMLib\1.3.0\Binary\NetCore\Microsoft.Azure.Storage.DataMovement.dll .\tools\nupkg\workspace\lib\netstandard2.0 -copy D:\Work\DMLib\1.3.0\Binary\NetCore\Microsoft.Azure.Storage.DataMovement.pdb .\tools\nupkg\workspace\lib\netstandard2.0 -copy D:\Work\DMLib\1.3.0\Binary\NetCore\Microsoft.Azure.Storage.DataMovement.xml .\tools\nupkg\workspace\lib\netstandard2.0 +copy .\lib\bin\Release\Microsoft.Azure.Storage.DataMovement.dll .\tools\nupkg\workspace\lib\net452 +copy .\lib\bin\Release\Microsoft.Azure.Storage.DataMovement.pdb .\tools\nupkg\workspace\lib\net452 +copy .\lib\bin\Release\Microsoft.Azure.Storage.DataMovement.XML .\tools\nupkg\workspace\lib\net452 +copy .\netcore\Microsoft.Azure.Storage.DataMovement\bin\Release\netstandard2.0\Microsoft.Azure.Storage.DataMovement.dll .\tools\nupkg\workspace\lib\netstandard2.0 +copy .\netcore\Microsoft.Azure.Storage.DataMovement\bin\Release\netstandard2.0\Microsoft.Azure.Storage.DataMovement.pdb .\tools\nupkg\workspace\lib\netstandard2.0 +copy .\netcore\Microsoft.Azure.Storage.DataMovement\bin\Release\netstandard2.0\Microsoft.Azure.Storage.DataMovement.xml .\tools\nupkg\workspace\lib\netstandard2.0 .\.nuget\nuget.exe pack .\tools\nupkg\workspace\Microsoft.Azure.Storage.DataMovement.nuspec popd rmdir /s /q .\workspace