Skip to content

Commit 4ec8c05

Browse files
amnguyeamnguye
authored
[Storage] [DataMovement] Fixing live upload directory tests (Azure#39537)
* Small fix to one of the upload directory tests * fixes to upload directory tests * Fix to download directory tests * Creating subdirectories before creating files * Fix for net462 long path bug
1 parent 0ffa73e commit 4ec8c05

File tree

3 files changed

+40
-40
lines changed

3 files changed

+40
-40
lines changed

‎sdk/storage/Azure.Storage.DataMovement/tests/Shared/TransferValidator.Local.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ Task<List<IResourceEnumerationItem>> ListFiles(CancellationToken cancellationTok
4444
}
4545
foreach (string filePath in Directory.GetFiles(workingDir))
4646
{
47-
result.Add(new LocalFileResourceEnumerationItem(filePath, filePath.Substring(workingDir.Length)));
47+
result.Add(new LocalFileResourceEnumerationItem(filePath, filePath.Substring(directoryPath.Length)));
4848
}
4949
}
5050
return Task.FromResult(result);

‎sdk/storage/Azure.Storage.DataMovement/tests/StartTransferDownloadDirectoryTests.cs‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -102,7 +102,7 @@ public async Task DownloadDirectoryAsync_Small(int size, int waitInSec)
102102
};
103103

104104
CancellationTokenSource cts = new();
105-
cts.CancelAfter(waitInSec);
105+
cts.CancelAfter(TimeSpan.FromSeconds(waitInSec));
106106
await DownloadBlobDirectoryAndVerify(
107107
test.Container,
108108
sourceBlobDirectoryName,
@@ -201,9 +201,9 @@ public async Task DownloadDirectoryAsync_ManySubDirectories()
201201
string fullSourceFolderPath = CreateRandomDirectory(tempFolder, blobDirectoryName);
202202
List<string> blobNames = new()
203203
{
204-
Path.Combine(fullSourceFolderPath, "bar", GetNewBlobName()),
205-
Path.Combine(fullSourceFolderPath, "rul", GetNewBlobName()),
206-
Path.Combine(fullSourceFolderPath, "pik", GetNewBlobName()),
204+
Path.Combine(blobDirectoryName, "bar", GetNewBlobName()),
205+
Path.Combine(blobDirectoryName, "rul", GetNewBlobName()),
206+
Path.Combine(blobDirectoryName, "pik", GetNewBlobName()),
207207
};
208208

209209
await DownloadBlobDirectoryAndVerify(

‎sdk/storage/Azure.Storage.DataMovement/tests/StartTransferUploadDirectoryTests.cs‎

Lines changed: 35 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,17 @@ public StartTransferUploadDirectoryTests(bool async, BlobClientOptions.ServiceVe
2626
: base(async, serviceVersion, null /* RecordedTestMode.Record /* to re-record */)
2727
{ }
2828

29+
private List<string> GetTestDirectoryTree(string parentDirectoryPath)
30+
{
31+
return new List<string>()
32+
{
33+
GetNewBlobName(),
34+
GetNewBlobName(),
35+
Path.Combine(GetNewBlobDirectoryName(), GetNewBlobName()),
36+
Path.Combine(GetNewBlobDirectoryName(), GetNewBlobName()),
37+
};
38+
}
39+
2940
#region Directory Block Blob
3041
private async Task SetupDirectory(
3142
string directoryPath,
@@ -38,7 +49,19 @@ private async Task SetupDirectory(
3849
{
3950
return;
4051
}
41-
using FileStream fs = File.OpenWrite(Path.Combine(directoryPath, filePath));
52+
53+
// Check if the parent subdirectory is already created,
54+
// if not create it before making the file
55+
string fullPath = Path.Combine(directoryPath, filePath);
56+
string subDirectory = Path.GetDirectoryName(fullPath);
57+
58+
if (!Directory.Exists(subDirectory))
59+
{
60+
Directory.CreateDirectory(subDirectory);
61+
}
62+
63+
// Check if it's a directory or not
64+
using FileStream fs = File.OpenWrite(fullPath);
4265
using Stream data = await CreateLimitedMemoryStream(size);
4366
await data.CopyToAsync(fs, bufferSize: 4 * Constants.KB, cancellationToken);
4467
}
@@ -61,6 +84,7 @@ private async Task UploadBlobDirectoryAndVerify(
6184
{
6285
// Set transfer options
6386
options ??= new DataTransferOptions();
87+
destinationPrefix ??= GetNewBlobDirectoryName();
6488
TestEventsRaised testEventsRaised = new TestEventsRaised(options);
6589

6690
transferManagerOptions ??= new TransferManagerOptions()
@@ -99,13 +123,7 @@ public async Task LocalToBlockBlobDirectory_SmallSize(long blobSize, int waitTim
99123
string localDirectory = CreateRandomDirectory(testDirectory.DirectoryPath);
100124
await using DisposingContainer test = await GetTestContainerAsync();
101125

102-
List<string> files = new()
103-
{
104-
GetNewBlobName(),
105-
GetNewBlobName(),
106-
$"{GetNewBlobName()}/{GetNewBlobName()}",
107-
$"{GetNewBlobName()}/{GetNewBlobName()}",
108-
};
126+
List<string> files = GetTestDirectoryTree(localDirectory);
109127

110128
CancellationToken cancellationToken = TestHelper.GetTimeoutToken(waitTimeInSec);
111129
await SetupDirectory(
@@ -170,13 +188,7 @@ public async Task LocalToBlockBlobDirectory_SmallChunks()
170188
string localDirectory = CreateRandomDirectory(testDirectory.DirectoryPath);
171189
await using DisposingContainer test = await GetTestContainerAsync();
172190

173-
List<string> files = new()
174-
{
175-
GetNewBlobName(),
176-
GetNewBlobName(),
177-
$"{GetNewBlobName()}/{GetNewBlobName()}",
178-
$"{GetNewBlobName()}/{GetNewBlobName()}",
179-
};
191+
List<string> files = GetTestDirectoryTree(localDirectory);
180192

181193
CancellationToken cancellationToken = TestHelper.GetTimeoutToken(waitTimeInSec);
182194
await SetupDirectory(
@@ -213,8 +225,8 @@ public async Task LocalToBlockBlobDirectory_SmallChunks_ManyFiles()
213225
string localDirectory = CreateRandomDirectory(testDirectory.DirectoryPath);
214226
await using DisposingContainer test = await GetTestContainerAsync();
215227

216-
string folder1 = GetNewBlobName();
217-
string folder2 = GetNewBlobName();
228+
string folder1 = GetNewBlobDirectoryName();
229+
string folder2 = GetNewBlobDirectoryName();
218230
List<string> files = new()
219231
{
220232
GetNewBlobName(),
@@ -357,14 +369,14 @@ public async Task DirectoryUpload_SubDirectoriesLevels(int level)
357369

358370
List<string> files = new List<string>();
359371

360-
string subfolderName = localDirectory;
372+
string subfolderName = "";
361373
for (int i = 0; i < level; i++)
362374
{
363-
subfolderName = Path.Combine(subfolderName, GetNewBlobName());
375+
subfolderName = Path.Combine(subfolderName, $"folder{i}");
364376
files.Add(Path.Combine(subfolderName, GetNewBlobName()));
365377
}
366378

367-
CancellationToken cancellationToken = TestHelper.GetTimeoutToken(10);
379+
CancellationToken cancellationToken = TestHelper.GetTimeoutToken(30);
368380
await SetupDirectory(
369381
localDirectory,
370382
files.Select(name => (name, (long)Constants.KB)).ToList(),
@@ -403,7 +415,7 @@ public async Task DirectoryUpload_EmptySubDirectories()
403415
await UploadBlobDirectoryAndVerify(
404416
localDirectory,
405417
test.Container,
406-
expectedTransfers: 0,
418+
expectedTransfers: 6,
407419
destinationPrefix: dirName,
408420
cancellationToken: TestHelper.GetTimeoutToken(10));
409421
}
@@ -422,13 +434,7 @@ public async Task DirectoryUpload_OverwriteTrue()
422434
using DisposingLocalDirectory testDirectory = DisposingLocalDirectory.GetTestDirectory();
423435
string localDirectory = CreateRandomDirectory(testDirectory.DirectoryPath);
424436

425-
List<string> files = new()
426-
{
427-
GetNewBlobName(),
428-
GetNewBlobName(),
429-
$"{GetNewBlobName()}/{GetNewBlobName()}",
430-
$"{GetNewBlobName()}/{GetNewBlobName()}",
431-
};
437+
List<string> files = GetTestDirectoryTree(localDirectory);
432438

433439
DataTransferOptions options = new DataTransferOptions()
434440
{
@@ -461,13 +467,7 @@ public async Task DirectoryUpload_OverwriteFalse()
461467
string localDirectory = CreateRandomDirectory(testDirectory.DirectoryPath);
462468
string dirName = GetNewBlobName();
463469

464-
List<string> files = new()
465-
{
466-
GetNewBlobName(),
467-
GetNewBlobName(),
468-
$"{GetNewBlobName()}/{GetNewBlobName()}",
469-
$"{GetNewBlobName()}/{GetNewBlobName()}",
470-
};
470+
List<string> files = GetTestDirectoryTree(localDirectory);
471471

472472
DataTransferOptions options = new DataTransferOptions()
473473
{

0 commit comments

Comments
 (0)