From 13bc26f0ea3fc639dd65c0fe471f5e1136e37e77 Mon Sep 17 00:00:00 2001 From: ksemenenko Date: Sun, 5 Oct 2025 11:36:45 +0200 Subject: [PATCH 1/8] extend vfs unicode coverage --- .../VirtualFileSystemTests.cs | 58 +++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/Tests/ManagedCode.Storage.Tests/VirtualFileSystem/VirtualFileSystemTests.cs b/Tests/ManagedCode.Storage.Tests/VirtualFileSystem/VirtualFileSystemTests.cs index 1f6fdae..b83f2c2 100644 --- a/Tests/ManagedCode.Storage.Tests/VirtualFileSystem/VirtualFileSystemTests.cs +++ b/Tests/ManagedCode.Storage.Tests/VirtualFileSystem/VirtualFileSystemTests.cs @@ -29,6 +29,15 @@ protected VirtualFileSystemTests(TFixture fixture) private Task CreateContextAsync() => _fixture.CreateContextAsync(); private VirtualFileSystemCapabilities Capabilities => _fixture.Capabilities; + public static IEnumerable UnicodeFolderTestCases => new[] + { + new object[] { "Українська-папка", "лист-привіт", "Привіт з Києва!" }, + new object[] { "中文目錄", "測試文件", "雲端中的內容" }, + new object[] { "日本語ディレクトリ", "テストファイル", "東京からこんにちは" }, + new object[] { "한국어_폴더", "테스트-파일", "부산에서 안녕하세요" }, + new object[] { "emoji📁", "😀-файл", "multi🌐lingual content" } + }; + [Fact] public async Task WriteAndReadFile_ShouldRoundtrip() { @@ -178,6 +187,55 @@ await file.SetMetadataAsync(new Dictionary metadataManager.CustomMetadataRequests.ShouldBe(0); } + [Theory] + [MemberData(nameof(UnicodeFolderTestCases))] + public async Task WriteAndReadFile_WithUnicodeDirectories_ShouldRoundtrip( + string directoryName, + string fileName, + string content) + { + if (!Capabilities.Enabled) + { + return; + } + + await using var context = await CreateContextAsync(); + var vfs = context.FileSystem; + + var path = new VfsPath($"/international/{directoryName}/{fileName}.txt"); + var file = await vfs.GetFileAsync(path); + + await file.WriteAllTextAsync(content); + + (await file.ReadAllTextAsync()).ShouldBe(content); + file.Path.GetFileName().ShouldBe($"{fileName}.txt"); + file.Path.GetFileNameWithoutExtension().ShouldBe(fileName); + file.Path.GetExtension().ShouldBe(".txt"); + file.Path.GetParent().Value.ShouldBe($"/international/{directoryName}"); + file.Path.ToBlobKey().ShouldBe($"international/{directoryName}/{fileName}.txt"); + + (await vfs.FileExistsAsync(path)).ShouldBeTrue(); + + if (Capabilities.SupportsListing) + { + var entries = new List(); + await foreach (var entry in vfs.ListAsync(new VfsPath($"/international/{directoryName}"), new ListOptions + { + IncludeFiles = true, + IncludeDirectories = false, + Recursive = false + })) + { + entries.Add(entry); + } + + entries.ShouldContain(e => e.Path.Value == path.Value); + } + + (await file.DeleteAsync()).ShouldBeTrue(); + (await vfs.FileExistsAsync(path)).ShouldBeFalse(); + } + [Fact] public async Task DeleteDirectoryAsync_NonRecursive_ShouldPreserveNestedContent() { From 4fc503d33a2dec7ef156a1684c1ead189d7a9da0 Mon Sep 17 00:00:00 2001 From: ksemenenko Date: Sun, 5 Oct 2025 11:47:15 +0200 Subject: [PATCH 2/8] add filesystem unicode mount regression test --- ...ystemVirtualFileSystemUnicodeMountTests.cs | 84 +++++++++++++++++++ .../VirtualFileSystem/UnicodeVfsTestCases.cs | 15 ++++ .../VirtualFileSystemTests.cs | 11 +-- 3 files changed, 100 insertions(+), 10 deletions(-) create mode 100644 Tests/ManagedCode.Storage.Tests/VirtualFileSystem/FileSystemVirtualFileSystemUnicodeMountTests.cs create mode 100644 Tests/ManagedCode.Storage.Tests/VirtualFileSystem/UnicodeVfsTestCases.cs diff --git a/Tests/ManagedCode.Storage.Tests/VirtualFileSystem/FileSystemVirtualFileSystemUnicodeMountTests.cs b/Tests/ManagedCode.Storage.Tests/VirtualFileSystem/FileSystemVirtualFileSystemUnicodeMountTests.cs new file mode 100644 index 0000000..f7f6627 --- /dev/null +++ b/Tests/ManagedCode.Storage.Tests/VirtualFileSystem/FileSystemVirtualFileSystemUnicodeMountTests.cs @@ -0,0 +1,84 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Threading.Tasks; +using ManagedCode.Storage.FileSystem; +using ManagedCode.Storage.FileSystem.Options; +using ManagedCode.Storage.Tests.VirtualFileSystem.Fixtures; +using ManagedCode.Storage.VirtualFileSystem.Core; +using ManagedCode.Storage.VirtualFileSystem.Options; +using Shouldly; +using Xunit; + +namespace ManagedCode.Storage.Tests.VirtualFileSystem; + +[Collection(VirtualFileSystemCollection.Name)] +public sealed class FileSystemVirtualFileSystemUnicodeMountTests +{ + [Theory] + [MemberData(nameof(UnicodeVfsTestCases.FolderScenarios), MemberType = typeof(UnicodeVfsTestCases))] + public async Task MountingExistingUnicodeDirectories_ShouldExposeFiles( + string directoryName, + string fileName, + string content) + { + var rootFolder = Path.Combine(Path.GetTempPath(), "managedcode-vfs-existing", Guid.NewGuid().ToString("N")); + var internationalFolder = Path.Combine(rootFolder, "international", directoryName); + Directory.CreateDirectory(internationalFolder); + + var seededFilePath = Path.Combine(internationalFolder, $"{fileName}.txt"); + await File.WriteAllTextAsync(seededFilePath, content); + + var options = new FileSystemStorageOptions + { + BaseFolder = rootFolder, + CreateContainerIfNotExists = true + }; + + var storage = new FileSystemStorage(options); + + async ValueTask Cleanup() + { + try + { + await storage.RemoveContainerAsync(); + } + finally + { + if (Directory.Exists(rootFolder)) + { + Directory.Delete(rootFolder, recursive: true); + } + } + } + + await using var context = await VirtualFileSystemTestContext.CreateAsync( + storage, + containerName: string.Empty, + ownsStorage: true, + serviceProvider: null, + cleanup: Cleanup); + + var vfs = context.FileSystem; + var expectedPath = new VfsPath($"/international/{directoryName}/{fileName}.txt"); + + (await vfs.FileExistsAsync(expectedPath)).ShouldBeTrue(); + + var file = await vfs.GetFileAsync(expectedPath); + var actualContent = await file.ReadAllTextAsync(); + actualContent.ShouldBe(content); + + var entries = new List(); + await foreach (var entry in vfs.ListAsync(new VfsPath($"/international/{directoryName}"), new ListOptions + { + IncludeDirectories = false, + IncludeFiles = true, + Recursive = false + })) + { + entries.Add(entry); + } + + entries.ShouldContain(e => e.Path.Value == expectedPath.Value); + } +} diff --git a/Tests/ManagedCode.Storage.Tests/VirtualFileSystem/UnicodeVfsTestCases.cs b/Tests/ManagedCode.Storage.Tests/VirtualFileSystem/UnicodeVfsTestCases.cs new file mode 100644 index 0000000..3892f74 --- /dev/null +++ b/Tests/ManagedCode.Storage.Tests/VirtualFileSystem/UnicodeVfsTestCases.cs @@ -0,0 +1,15 @@ +using System.Collections.Generic; + +namespace ManagedCode.Storage.Tests.VirtualFileSystem; + +public static class UnicodeVfsTestCases +{ + public static IEnumerable FolderScenarios => new[] + { + new object[] { "Українська-папка", "лист-привіт", "Привіт з Києва!" }, + new object[] { "中文目錄", "測試文件", "雲端中的內容" }, + new object[] { "日本語ディレクトリ", "テストファイル", "東京からこんにちは" }, + new object[] { "한국어_폴더", "테스트-파일", "부산에서 안녕하세요" }, + new object[] { "emoji📁", "😀-файл", "multi🌐lingual content" } + }; +} diff --git a/Tests/ManagedCode.Storage.Tests/VirtualFileSystem/VirtualFileSystemTests.cs b/Tests/ManagedCode.Storage.Tests/VirtualFileSystem/VirtualFileSystemTests.cs index b83f2c2..3d8079f 100644 --- a/Tests/ManagedCode.Storage.Tests/VirtualFileSystem/VirtualFileSystemTests.cs +++ b/Tests/ManagedCode.Storage.Tests/VirtualFileSystem/VirtualFileSystemTests.cs @@ -29,15 +29,6 @@ protected VirtualFileSystemTests(TFixture fixture) private Task CreateContextAsync() => _fixture.CreateContextAsync(); private VirtualFileSystemCapabilities Capabilities => _fixture.Capabilities; - public static IEnumerable UnicodeFolderTestCases => new[] - { - new object[] { "Українська-папка", "лист-привіт", "Привіт з Києва!" }, - new object[] { "中文目錄", "測試文件", "雲端中的內容" }, - new object[] { "日本語ディレクトリ", "テストファイル", "東京からこんにちは" }, - new object[] { "한국어_폴더", "테스트-파일", "부산에서 안녕하세요" }, - new object[] { "emoji📁", "😀-файл", "multi🌐lingual content" } - }; - [Fact] public async Task WriteAndReadFile_ShouldRoundtrip() { @@ -188,7 +179,7 @@ await file.SetMetadataAsync(new Dictionary } [Theory] - [MemberData(nameof(UnicodeFolderTestCases))] + [MemberData(nameof(UnicodeVfsTestCases.FolderScenarios), MemberType = typeof(UnicodeVfsTestCases))] public async Task WriteAndReadFile_WithUnicodeDirectories_ShouldRoundtrip( string directoryName, string fileName, From 831bebf1eeaca76eb66234ac2e3ca8c584a00038 Mon Sep 17 00:00:00 2001 From: ksemenenko Date: Sun, 5 Oct 2025 11:52:03 +0200 Subject: [PATCH 3/8] remove capability guards from vfs tests --- .../VirtualFileSystemTests.cs | 49 +++---------------- 1 file changed, 7 insertions(+), 42 deletions(-) diff --git a/Tests/ManagedCode.Storage.Tests/VirtualFileSystem/VirtualFileSystemTests.cs b/Tests/ManagedCode.Storage.Tests/VirtualFileSystem/VirtualFileSystemTests.cs index 3d8079f..2fdb55e 100644 --- a/Tests/ManagedCode.Storage.Tests/VirtualFileSystem/VirtualFileSystemTests.cs +++ b/Tests/ManagedCode.Storage.Tests/VirtualFileSystem/VirtualFileSystemTests.cs @@ -32,11 +32,6 @@ protected VirtualFileSystemTests(TFixture fixture) [Fact] public async Task WriteAndReadFile_ShouldRoundtrip() { - if (!Capabilities.Enabled) - { - return; - } - await using var context = await CreateContextAsync(); var vfs = context.FileSystem; @@ -52,11 +47,6 @@ public async Task WriteAndReadFile_ShouldRoundtrip() [Fact] public async Task FileExistsAsync_ShouldCacheResults() { - if (!Capabilities.Enabled) - { - return; - } - await using var context = await CreateContextAsync(); var vfs = context.FileSystem; var metadataManager = context.MetadataManager; @@ -79,7 +69,7 @@ public async Task FileExistsAsync_ShouldCacheResults() [Fact] public async Task ListAsync_ShouldEnumerateAllEntries() { - if (!Capabilities.Enabled || !Capabilities.SupportsListing) + if (!Capabilities.SupportsListing) { return; } @@ -116,11 +106,6 @@ public async Task ListAsync_ShouldEnumerateAllEntries() [Fact] public async Task DeleteFile_ShouldRemoveFromUnderlyingStorage() { - if (!Capabilities.Enabled) - { - return; - } - await using var context = await CreateContextAsync(); var vfs = context.FileSystem; var metadataManager = context.MetadataManager; @@ -149,11 +134,6 @@ public async Task DeleteFile_ShouldRemoveFromUnderlyingStorage() [Fact] public async Task GetMetadataAsync_ShouldCacheCustomMetadata() { - if (!Capabilities.Enabled) - { - return; - } - await using var context = await CreateContextAsync(); var vfs = context.FileSystem; var metadataManager = context.MetadataManager; @@ -185,11 +165,6 @@ public async Task WriteAndReadFile_WithUnicodeDirectories_ShouldRoundtrip( string fileName, string content) { - if (!Capabilities.Enabled) - { - return; - } - await using var context = await CreateContextAsync(); var vfs = context.FileSystem; @@ -230,7 +205,7 @@ public async Task WriteAndReadFile_WithUnicodeDirectories_ShouldRoundtrip( [Fact] public async Task DeleteDirectoryAsync_NonRecursive_ShouldPreserveNestedContent() { - if (!Capabilities.Enabled || !Capabilities.SupportsDirectoryDelete) + if (!Capabilities.SupportsDirectoryDelete) { return; } @@ -251,7 +226,7 @@ public async Task DeleteDirectoryAsync_NonRecursive_ShouldPreserveNestedContent( [Fact] public async Task DeleteDirectoryAsync_Recursive_ShouldRemoveAllContent() { - if (!Capabilities.Enabled || !Capabilities.SupportsDirectoryDelete) + if (!Capabilities.SupportsDirectoryDelete) { return; } @@ -272,7 +247,7 @@ public async Task DeleteDirectoryAsync_Recursive_ShouldRemoveAllContent() [Fact] public async Task MoveAsync_ShouldRelocateFile() { - if (!Capabilities.Enabled || !Capabilities.SupportsMove) + if (!Capabilities.SupportsMove) { return; } @@ -298,7 +273,7 @@ public async Task MoveAsync_ShouldRelocateFile() [Fact] public async Task CopyAsync_ShouldCopyDirectoryRecursively() { - if (!Capabilities.Enabled || !Capabilities.SupportsDirectoryCopy) + if (!Capabilities.SupportsDirectoryCopy) { return; } @@ -331,11 +306,6 @@ public async Task CopyAsync_ShouldCopyDirectoryRecursively() [Fact] public async Task ReadRangeAsync_ShouldReturnSlice() { - if (!Capabilities.Enabled) - { - return; - } - await using var context = await CreateContextAsync(); var vfs = context.FileSystem; @@ -349,7 +319,7 @@ public async Task ReadRangeAsync_ShouldReturnSlice() [Fact] public async Task ListAsync_WithDirectoryFilter_ShouldExcludeDirectoriesWhenRequested() { - if (!Capabilities.Enabled || !Capabilities.SupportsListing) + if (!Capabilities.SupportsListing) { return; } @@ -381,7 +351,7 @@ public async Task ListAsync_WithDirectoryFilter_ShouldExcludeDirectoriesWhenRequ [Fact] public async Task DirectoryStats_ShouldAggregateInformation() { - if (!Capabilities.Enabled || !Capabilities.SupportsDirectoryStats) + if (!Capabilities.SupportsDirectoryStats) { return; } @@ -407,11 +377,6 @@ public async Task DirectoryStats_ShouldAggregateInformation() [InlineData(5)] public async Task LargeFile_ShouldRoundTripViaStreams(int gigabytes) { - if (!Capabilities.Enabled) - { - return; - } - var sizeBytes = LargeFileTestHelper.ResolveSizeBytes(gigabytes); await using var context = await CreateContextAsync(); From 2889d80cf8c5ac5176694529036b48e3f08e9ec9 Mon Sep 17 00:00:00 2001 From: ksemenenko Date: Sun, 5 Oct 2025 11:55:07 +0200 Subject: [PATCH 4/8] Use repo-local folders for VFS tests --- .../FileSystemVirtualFileSystemUnicodeMountTests.cs | 2 +- .../Fixtures/FileSystemVirtualFileSystemFixture.cs | 2 +- .../VirtualFileSystem/VirtualFileSystemManagerTests.cs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Tests/ManagedCode.Storage.Tests/VirtualFileSystem/FileSystemVirtualFileSystemUnicodeMountTests.cs b/Tests/ManagedCode.Storage.Tests/VirtualFileSystem/FileSystemVirtualFileSystemUnicodeMountTests.cs index f7f6627..4112181 100644 --- a/Tests/ManagedCode.Storage.Tests/VirtualFileSystem/FileSystemVirtualFileSystemUnicodeMountTests.cs +++ b/Tests/ManagedCode.Storage.Tests/VirtualFileSystem/FileSystemVirtualFileSystemUnicodeMountTests.cs @@ -22,7 +22,7 @@ public async Task MountingExistingUnicodeDirectories_ShouldExposeFiles( string fileName, string content) { - var rootFolder = Path.Combine(Path.GetTempPath(), "managedcode-vfs-existing", Guid.NewGuid().ToString("N")); + var rootFolder = Path.Combine(Directory.GetCurrentDirectory(), "managedcode-vfs-existing", Guid.NewGuid().ToString("N")); var internationalFolder = Path.Combine(rootFolder, "international", directoryName); Directory.CreateDirectory(internationalFolder); diff --git a/Tests/ManagedCode.Storage.Tests/VirtualFileSystem/Fixtures/FileSystemVirtualFileSystemFixture.cs b/Tests/ManagedCode.Storage.Tests/VirtualFileSystem/Fixtures/FileSystemVirtualFileSystemFixture.cs index bbd2529..e14fac7 100644 --- a/Tests/ManagedCode.Storage.Tests/VirtualFileSystem/Fixtures/FileSystemVirtualFileSystemFixture.cs +++ b/Tests/ManagedCode.Storage.Tests/VirtualFileSystem/Fixtures/FileSystemVirtualFileSystemFixture.cs @@ -9,7 +9,7 @@ namespace ManagedCode.Storage.Tests.VirtualFileSystem.Fixtures; public sealed class FileSystemVirtualFileSystemFixture : IVirtualFileSystemFixture, IAsyncLifetime { - private readonly string _rootPath = Path.Combine(Path.GetTempPath(), "managedcode-vfs-matrix", Guid.NewGuid().ToString("N")); + private readonly string _rootPath = Path.Combine(Directory.GetCurrentDirectory(), "managedcode-vfs-matrix", Guid.NewGuid().ToString("N")); public VirtualFileSystemCapabilities Capabilities { get; } = new(); diff --git a/Tests/ManagedCode.Storage.Tests/VirtualFileSystem/VirtualFileSystemManagerTests.cs b/Tests/ManagedCode.Storage.Tests/VirtualFileSystem/VirtualFileSystemManagerTests.cs index 7deae71..8fd88f7 100644 --- a/Tests/ManagedCode.Storage.Tests/VirtualFileSystem/VirtualFileSystemManagerTests.cs +++ b/Tests/ManagedCode.Storage.Tests/VirtualFileSystem/VirtualFileSystemManagerTests.cs @@ -15,7 +15,7 @@ namespace ManagedCode.Storage.Tests.VirtualFileSystem; public class VirtualFileSystemManagerTests : IAsyncLifetime { - private readonly string _basePath = Path.Combine(Path.GetTempPath(), "managedcode-vfs-manager", Guid.NewGuid().ToString()); + private readonly string _basePath = Path.Combine(Directory.GetCurrentDirectory(), "managedcode-vfs-manager", Guid.NewGuid().ToString()); private ServiceProvider _serviceProvider = null!; private IStorage _storage = null!; From 513b58b2409b6e4d760dd8ffcb2411bf18beeedc Mon Sep 17 00:00:00 2001 From: ksemenenko Date: Sun, 5 Oct 2025 14:13:45 +0200 Subject: [PATCH 5/8] fixes --- ManagedCode.Storage.Core/Models/LocalFile.cs | 146 +++++++++++++++++- .../ManagedCode.Storage.Google/GCPStorage.cs | 36 ++++- 2 files changed, 174 insertions(+), 8 deletions(-) diff --git a/ManagedCode.Storage.Core/Models/LocalFile.cs b/ManagedCode.Storage.Core/Models/LocalFile.cs index 8e7d982..62a6e77 100644 --- a/ManagedCode.Storage.Core/Models/LocalFile.cs +++ b/ManagedCode.Storage.Core/Models/LocalFile.cs @@ -131,8 +131,11 @@ public void Close() public async Task CopyFromStreamAsync(Stream stream, CancellationToken cancellationToken = default) { - var fs = FileStream; - await stream.CopyToAsync(fs, cancellationToken); + await using var fs = FileStream; + fs.SetLength(0); + fs.Position = 0; + await stream.CopyToAsync(fs, cancellationToken).ConfigureAwait(false); + await fs.FlushAsync(cancellationToken).ConfigureAwait(false); return this; } @@ -180,6 +183,11 @@ public static LocalFile FromTempFile() return new LocalFile(); } + public Stream OpenReadStream(bool disposeOwner = true) + { + return new LocalFileReadStream(this, disposeOwner); + } + #region Read public string ReadAllText() @@ -304,4 +312,136 @@ public Task WriteAllBytesAsync(byte[] bytes, CancellationToken cancellationToken } #endregion -} \ No newline at end of file + + private sealed class LocalFileReadStream : Stream + { + private readonly LocalFile _owner; + private readonly FileStream _stream; + private readonly bool _disposeOwner; + private bool _disposed; + + public LocalFileReadStream(LocalFile owner, bool disposeOwner) + { + _owner = owner; + _disposeOwner = disposeOwner; + _stream = new FileStream(owner.FilePath, new FileStreamOptions + { + Mode = FileMode.Open, + Access = FileAccess.Read, + Share = FileShare.Read, + Options = FileOptions.Asynchronous + }); + } + + public override bool CanRead => _stream.CanRead; + + public override bool CanSeek => _stream.CanSeek; + + public override bool CanWrite => _stream.CanWrite; + + public override long Length => _stream.Length; + + public override long Position + { + get => _stream.Position; + set => _stream.Position = value; + } + + public override void Flush() => _stream.Flush(); + + public override Task FlushAsync(CancellationToken cancellationToken) + { + return _stream.FlushAsync(cancellationToken); + } + + public override int Read(byte[] buffer, int offset, int count) + { + return _stream.Read(buffer, offset, count); + } + + public override int Read(Span buffer) + { + return _stream.Read(buffer); + } + + public override ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = default) + { + return _stream.ReadAsync(buffer, cancellationToken); + } + + public override Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + return _stream.ReadAsync(buffer, offset, count, cancellationToken); + } + + public override long Seek(long offset, SeekOrigin origin) + { + return _stream.Seek(offset, origin); + } + + public override void SetLength(long value) + { + _stream.SetLength(value); + } + + public override void Write(byte[] buffer, int offset, int count) + { + _stream.Write(buffer, offset, count); + } + + public override void Write(ReadOnlySpan buffer) + { + _stream.Write(buffer); + } + + public override Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + return _stream.WriteAsync(buffer, offset, count, cancellationToken); + } + + public override ValueTask WriteAsync(ReadOnlyMemory buffer, CancellationToken cancellationToken = default) + { + return _stream.WriteAsync(buffer, cancellationToken); + } + + protected override void Dispose(bool disposing) + { + if (_disposed) + { + base.Dispose(disposing); + return; + } + + if (disposing) + { + _stream.Dispose(); + + if (_disposeOwner) + { + _owner.Dispose(); + } + } + + _disposed = true; + base.Dispose(disposing); + } + + public override async ValueTask DisposeAsync() + { + if (_disposed) + { + await ValueTask.CompletedTask; + return; + } + + await _stream.DisposeAsync(); + + if (_disposeOwner) + { + await _owner.DisposeAsync(); + } + + _disposed = true; + } + } +} diff --git a/Storages/ManagedCode.Storage.Google/GCPStorage.cs b/Storages/ManagedCode.Storage.Google/GCPStorage.cs index c02efc2..b880901 100644 --- a/Storages/ManagedCode.Storage.Google/GCPStorage.cs +++ b/Storages/ManagedCode.Storage.Google/GCPStorage.cs @@ -98,7 +98,33 @@ public override async Task> GetStreamAsync(string fileName, Cance await EnsureContainerExist(cancellationToken); if (urlSigner == null) - return Result.Fail("Google credentials are required to get stream"); + { + var tempFile = LocalFile.FromTempFile(); + + try + { + await using (var writableStream = tempFile.FileStream) + { + writableStream.SetLength(0); + + await StorageClient.DownloadObjectAsync( + StorageOptions.BucketOptions.Bucket, + fileName, + writableStream, + cancellationToken: cancellationToken); + + cancellationToken.ThrowIfCancellationRequested(); + } + + var downloadStream = tempFile.OpenReadStream(); + return Result.Succeed(downloadStream); + } + catch + { + await tempFile.DisposeAsync(); + throw; + } + } var signedUrl = urlSigner.Sign(StorageOptions.BucketOptions.Bucket, fileName, TimeSpan.FromHours(1), HttpMethod.Get); cancellationToken.ThrowIfCancellationRequested(); @@ -289,7 +315,8 @@ protected override async Task> GetBlobMetadataInternalAsync return Result.Succeed(new BlobMetadata { - Name = obj.Name, + FullName = obj.Name, + Name = Path.GetFileName(obj.Name), Uri = string.IsNullOrEmpty(obj.MediaLink) ? null : new Uri(obj.MediaLink), Container = obj.Bucket, CreatedOn = GetFirstSuccessfulValue(DateTimeOffset.UtcNow, () => obj.TimeCreatedDateTimeOffset, () => obj.TimeCreated), @@ -364,8 +391,7 @@ public static T GetFirstSuccessfulValue(T defaultValue, params Func[] get continue; } } - + return defaultValue; } - -} \ No newline at end of file +} From 1bb64745047b10e02640b0768895e95d3efa869a Mon Sep 17 00:00:00 2001 From: ksemenenko Date: Sun, 5 Oct 2025 15:08:00 +0200 Subject: [PATCH 6/8] tests and fixes --- Directory.Build.props | 4 +- .../Core/LocalFileTests.cs | 104 ++++++++++++++++++ 2 files changed, 106 insertions(+), 2 deletions(-) create mode 100644 Tests/ManagedCode.Storage.Tests/Core/LocalFileTests.cs diff --git a/Directory.Build.props b/Directory.Build.props index db9d9c2..21b97cd 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -27,8 +27,8 @@ https://github.com/managedcode/Storage https://github.com/managedcode/Storage Managed Code - Storage - 9.2.0 - 9.2.0 + 9.2.1 + 9.2.1 diff --git a/Tests/ManagedCode.Storage.Tests/Core/LocalFileTests.cs b/Tests/ManagedCode.Storage.Tests/Core/LocalFileTests.cs new file mode 100644 index 0000000..68c4c2c --- /dev/null +++ b/Tests/ManagedCode.Storage.Tests/Core/LocalFileTests.cs @@ -0,0 +1,104 @@ +using System; +using System.IO; +using System.Text; +using System.Threading.Tasks; +using ManagedCode.Storage.Core.Models; +using Shouldly; +using Xunit; + +namespace ManagedCode.Storage.Tests.Core; + +public class LocalFileTests +{ + [Fact] + public async Task OpenReadStream_DisposeOwnerByDefault_DeletesBackingFile() + { + var localFile = LocalFile.FromTempFile(); + var filePath = localFile.FilePath; + + var payload = Encoding.UTF8.GetBytes("ping"); + localFile.WriteAllBytes(payload); + File.Exists(filePath).ShouldBeTrue(); + + await using (var stream = localFile.OpenReadStream()) + { + var buffer = new byte[payload.Length]; + var read = await stream.ReadAsync(buffer, 0, buffer.Length); + read.ShouldBe(buffer.Length); + buffer.ShouldBe(payload); + } + + File.Exists(filePath).ShouldBeFalse(); + } + + [Fact] + public async Task OpenReadStream_DisposeOwnerFalse_PreservesBackingFile() + { + await using var localFile = LocalFile.FromTempFile(); + var filePath = localFile.FilePath; + + localFile.WriteAllText("pong"); + File.Exists(filePath).ShouldBeTrue(); + + await using (var stream = localFile.OpenReadStream(disposeOwner: false)) + { + var reader = new StreamReader(stream, leaveOpen: false); + var text = await reader.ReadToEndAsync(); + text.ShouldBe("pong"); + } + + File.Exists(filePath).ShouldBeTrue(); + } + + [Fact] + public async Task LocalFile_Finalizer_RemovesFile_WhenNotDisposed() + { + string filePath; + var weakReference = CreateUntrackedLocalFile(out filePath); + + File.Exists(filePath).ShouldBeTrue(); + + var deleted = await WaitForFileDeletionAsync(filePath, weakReference); + + deleted.ShouldBeTrue($"File '{filePath}' should be deleted once LocalFile is finalized."); + File.Exists(filePath).ShouldBeFalse(); + } + + private static WeakReference CreateUntrackedLocalFile(out string filePath) + { + var file = LocalFile.FromTempFile(); + filePath = file.FilePath; + + file.WriteAllText("ghost"); + + using (var stream = file.OpenReadStream(disposeOwner: false)) + { + var buffer = new byte[5]; + _ = stream.Read(buffer, 0, buffer.Length); + } + + var weakReference = new WeakReference(file); + file = null!; + return weakReference; + } + + private static async Task WaitForFileDeletionAsync(string filePath, WeakReference weakReference) + { + const int maxAttempts = 20; + for (var attempt = 0; attempt < maxAttempts; attempt++) + { + GC.Collect(); + GC.WaitForPendingFinalizers(); + GC.Collect(); + + if (!weakReference.IsAlive && !File.Exists(filePath)) + { + return true; + } + + await Task.Delay(100); + } + + return !weakReference.IsAlive && !File.Exists(filePath); + } +} From 7c298412e10143736264eb2d84e93a58171ab0db Mon Sep 17 00:00:00 2001 From: ksemenenko Date: Sun, 5 Oct 2025 15:08:12 +0200 Subject: [PATCH 7/8] sftp security fix --- .../ManagedCode.Storage.Sftp/Options/SftpStorageOptions.cs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Storages/ManagedCode.Storage.Sftp/Options/SftpStorageOptions.cs b/Storages/ManagedCode.Storage.Sftp/Options/SftpStorageOptions.cs index 37dd9e8..4202b83 100644 --- a/Storages/ManagedCode.Storage.Sftp/Options/SftpStorageOptions.cs +++ b/Storages/ManagedCode.Storage.Sftp/Options/SftpStorageOptions.cs @@ -68,9 +68,11 @@ public class SftpStorageOptions : IStorageOptions public string? PrivateKeyContent { get; set; } /// - /// Accept any host key presented by the server (not recommended for production). + /// Accept any host key presented by the server. + /// WARNING: Setting this to true is INSECURE and should only be used for development/testing. + /// In production, always set this to false and provide a valid HostKeyFingerprint. /// - public bool AcceptAnyHostKey { get; set; } = true; + public bool AcceptAnyHostKey { get; set; } = false; /// /// Expected host key fingerprint when is false. From b5eb4d68485ba51a24e86a9e0969fbbad2a5c6a6 Mon Sep 17 00:00:00 2001 From: ksemenenko Date: Sun, 5 Oct 2025 16:17:28 +0200 Subject: [PATCH 8/8] sanitiation --- .../ChunkUpload/ChunkUploadService.cs | 28 ++- .../Controllers/StorageControllerBase.cs | 42 ++++ .../Core/VfsPath.cs | 9 + .../FileSystemStorage.cs | 182 +++++++++++++++++- .../Abstracts/BaseStreamControllerTests.cs | 2 +- .../Azure/AzureSignalRStorageTests.cs | 2 +- .../Common/FileHelper.cs | 4 +- .../Core/Crc32HelperTests.cs | 2 +- .../Server/ChunkUploadServiceTests.cs | 2 +- .../FileSystem/FileSystemSecurityTests.cs | 132 +++++++++++++ .../FileSystemUnicodeSanitizerTests.cs | 47 +++++ .../FileSystem/FileSystemUploadTests.cs | 2 +- ...ystemVirtualFileSystemUnicodeMountTests.cs | 3 +- 13 files changed, 439 insertions(+), 18 deletions(-) create mode 100644 Tests/ManagedCode.Storage.Tests/Storages/FileSystem/FileSystemSecurityTests.cs create mode 100644 Tests/ManagedCode.Storage.Tests/Storages/FileSystem/FileSystemUnicodeSanitizerTests.cs diff --git a/Integraions/ManagedCode.Storage.Server/ChunkUpload/ChunkUploadService.cs b/Integraions/ManagedCode.Storage.Server/ChunkUpload/ChunkUploadService.cs index e444acc..86001fa 100644 --- a/Integraions/ManagedCode.Storage.Server/ChunkUpload/ChunkUploadService.cs +++ b/Integraions/ManagedCode.Storage.Server/ChunkUpload/ChunkUploadService.cs @@ -47,7 +47,10 @@ public async Task AppendChunkAsync(FileUploadPayload payload, Cancellati var descriptor = payload.Payload; var uploadId = ChunkUploadDescriptor.ResolveUploadId(descriptor); - var session = _sessions.GetOrAdd(uploadId, static (key, state) => + // Sanitize upload ID to prevent path traversal + var sanitizedUploadId = SanitizeUploadId(uploadId); + + var session = _sessions.GetOrAdd(sanitizedUploadId, static (key, state) => { var descriptor = state.Payload; var workingDirectory = Path.Combine(state.Options.TempPath, key); @@ -151,6 +154,29 @@ public void Abort(string uploadId) } } + private static string SanitizeUploadId(string uploadId) + { + if (string.IsNullOrWhiteSpace(uploadId)) + throw new ArgumentException("Upload ID cannot be null or empty", nameof(uploadId)); + + // Remove any path traversal sequences + uploadId = uploadId.Replace("..", string.Empty, StringComparison.Ordinal); + uploadId = uploadId.Replace("/", string.Empty, StringComparison.Ordinal); + uploadId = uploadId.Replace("\\", string.Empty, StringComparison.Ordinal); + + // Only allow alphanumeric characters, hyphens, and underscores + var allowedChars = uploadId.Where(c => char.IsLetterOrDigit(c) || c == '-' || c == '_').ToArray(); + var sanitized = new string(allowedChars); + + if (string.IsNullOrWhiteSpace(sanitized)) + throw new ArgumentException("Upload ID contains only invalid characters", nameof(uploadId)); + + if (sanitized.Length > 128) + throw new ArgumentException("Upload ID is too long", nameof(uploadId)); + + return sanitized; + } + private static async Task MergeChunksAsync(string destinationFile, IReadOnlyCollection chunkFiles, CancellationToken cancellationToken) { await using var destination = new FileStream(destinationFile, FileMode.Create, FileAccess.Write, FileShare.None, bufferSize: MergeBufferSize, useAsync: true); diff --git a/Integraions/ManagedCode.Storage.Server/Controllers/StorageControllerBase.cs b/Integraions/ManagedCode.Storage.Server/Controllers/StorageControllerBase.cs index 10180f5..483846b 100644 --- a/Integraions/ManagedCode.Storage.Server/Controllers/StorageControllerBase.cs +++ b/Integraions/ManagedCode.Storage.Server/Controllers/StorageControllerBase.cs @@ -57,6 +57,13 @@ public virtual async Task> UploadAsync([FromForm] IFormFile return Result.Fail(HttpStatusCode.BadRequest, "File payload is missing"); } + // Validate file size if enabled + if (_options.EnableFileSizeValidation && _options.MaxFileSize > 0 && file.Length > _options.MaxFileSize) + { + return Result.Fail(HttpStatusCode.RequestEntityTooLarge, + $"File size {file.Length} bytes exceeds maximum allowed size of {_options.MaxFileSize} bytes"); + } + try { return await Result.From(() => this.UploadFormFileAsync(Storage, file, cancellationToken: cancellationToken), cancellationToken); @@ -164,6 +171,13 @@ public virtual async Task UploadChunkAsync([FromForm] FileUploadPayload return Result.Fail(HttpStatusCode.BadRequest, "UploadId is required"); } + // Validate chunk size if enabled + if (_options.EnableFileSizeValidation && _options.MaxChunkSize > 0 && payload.File.Length > _options.MaxChunkSize) + { + return Result.Fail(HttpStatusCode.RequestEntityTooLarge, + $"Chunk size {payload.File.Length} bytes exceeds maximum allowed chunk size of {_options.MaxChunkSize} bytes"); + } + return await ChunkUploadService.AppendChunkAsync(payload, cancellationToken); } @@ -228,6 +242,16 @@ public class StorageServerOptions /// public const int DefaultMultipartBoundaryLengthLimit = 70; + /// + /// Default maximum file size: 100 MB. + /// + public const long DefaultMaxFileSize = 100 * 1024 * 1024; + + /// + /// Default maximum chunk size: 10 MB. + /// + public const long DefaultMaxChunkSize = 10 * 1024 * 1024; + /// /// Gets or sets a value indicating whether range processing is enabled for streaming responses. /// @@ -242,4 +266,22 @@ public class StorageServerOptions /// Gets or sets the maximum allowed length for multipart boundaries when parsing raw upload streams. /// public int MultipartBoundaryLengthLimit { get; set; } = DefaultMultipartBoundaryLengthLimit; + + /// + /// Gets or sets the maximum file size in bytes that can be uploaded. Set to 0 to disable the limit. + /// Default is 100 MB. + /// + public long MaxFileSize { get; set; } = DefaultMaxFileSize; + + /// + /// Gets or sets the maximum chunk size in bytes for chunk uploads. Set to 0 to disable the limit. + /// Default is 10 MB. + /// + public long MaxChunkSize { get; set; } = DefaultMaxChunkSize; + + /// + /// Gets or sets whether file size validation is enabled. + /// Default is true. + /// + public bool EnableFileSizeValidation { get; set; } = true; } diff --git a/ManagedCode.Storage.VirtualFileSystem/Core/VfsPath.cs b/ManagedCode.Storage.VirtualFileSystem/Core/VfsPath.cs index ffc4c75..6884d45 100644 --- a/ManagedCode.Storage.VirtualFileSystem/Core/VfsPath.cs +++ b/ManagedCode.Storage.VirtualFileSystem/Core/VfsPath.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.Linq; namespace ManagedCode.Storage.VirtualFileSystem.Core; @@ -95,6 +96,14 @@ public string ToBlobKey() /// private static string NormalizePath(string path) { + // Security: Check for null bytes (potential security issue) + if (path.Contains('\0')) + throw new ArgumentException("Path contains null bytes", nameof(path)); + + // Security: Check for control characters + if (path.Any(c => char.IsControl(c) && c != '\t' && c != '\r' && c != '\n')) + throw new ArgumentException("Path contains control characters", nameof(path)); + // 1. Replace backslashes with forward slashes path = path.Replace('\\', '/'); diff --git a/Storages/ManagedCode.Storage.FileSystem/FileSystemStorage.cs b/Storages/ManagedCode.Storage.FileSystem/FileSystemStorage.cs index d79727c..71676e9 100644 --- a/Storages/ManagedCode.Storage.FileSystem/FileSystemStorage.cs +++ b/Storages/ManagedCode.Storage.FileSystem/FileSystemStorage.cs @@ -55,14 +55,50 @@ public override async IAsyncEnumerable GetBlobMetadataListAsync(st if (cancellationToken.IsCancellationRequested) yield break; - var blobMetadata = await GetBlobMetadataAsync(file, cancellationToken); + var relativePath = Path.GetRelativePath(StorageClient, file) + .Replace('\\', '/'); + + var (relativeDirectory, relativeFileName) = SplitRelativePath(relativePath); + MetadataOptions options = new() + { + FileName = relativeFileName, + Directory = relativeDirectory + }; + + var blobMetadata = await GetBlobMetadataAsync(options, cancellationToken); cancellationToken.ThrowIfCancellationRequested(); if (blobMetadata.IsSuccess) + { yield return blobMetadata.Value; + continue; + } } } + private static (string? Directory, string FileName) SplitRelativePath(string relativePath) + { + if (string.IsNullOrWhiteSpace(relativePath)) + throw new ArgumentException("Relative path cannot be null or empty", nameof(relativePath)); + + var normalizedPath = relativePath.Replace('\\', '/'); + var separatorIndex = normalizedPath.LastIndexOf('/'); + + if (separatorIndex < 0) + return (null, normalizedPath); + + var directory = separatorIndex == 0 + ? null + : normalizedPath[..separatorIndex]; + + var fileName = normalizedPath[(separatorIndex + 1)..]; + + if (string.IsNullOrWhiteSpace(fileName)) + throw new InvalidOperationException($"Invalid relative path: '{relativePath}'"); + + return (string.IsNullOrWhiteSpace(directory) ? null : directory, fileName); + } + public override async Task> GetStreamAsync(string fileName, CancellationToken cancellationToken = default) { try @@ -315,19 +351,147 @@ protected override async Task> HasLegalHoldInternalAsync(LegalHoldO private string GetPathFromOptions(BaseOptions options) { - string filePath; - if (options.Directory is not null) + if (string.IsNullOrWhiteSpace(options.FileName)) + throw new ArgumentException("File name cannot be null or empty", nameof(options.FileName)); + + var (directoryFromFileName, fileNameOnly) = SplitDirectoryFromFileName(options.FileName); + + // Sanitize and validate components + var sanitizedFileName = SanitizeFileName(fileNameOnly); + + var combinedDirectory = CombineDirectoryParts(options.Directory, directoryFromFileName); + var sanitizedDirectory = combinedDirectory is not null + ? SanitizeDirectory(combinedDirectory) + : null; + + if (sanitizedDirectory is not null) { - EnsureDirectoryExist(options.Directory); - filePath = Path.Combine(StorageClient, options.Directory, options.FileName); + EnsureDirectoryExist(sanitizedDirectory); } - else + + string filePath = sanitizedDirectory is not null + ? Path.Combine(StorageClient, sanitizedDirectory, sanitizedFileName) + : Path.Combine(StorageClient, sanitizedFileName); + + // Get full paths for comparison + var fullPath = Path.GetFullPath(filePath); + var baseFullPath = Path.GetFullPath(StorageClient); + + // Verify the final path is within StorageClient directory + if (!fullPath.StartsWith(baseFullPath, StringComparison.OrdinalIgnoreCase)) { - filePath = Path.Combine(StorageClient, options.FileName); + throw new UnauthorizedAccessException($"Access to path '{options.FileName}' is denied. Path traversal detected."); + } + + EnsureDirectoryExist(Path.GetDirectoryName(fullPath)!); + return fullPath; + } + + private static (string? Directory, string FileName) SplitDirectoryFromFileName(string fileName) + { + var normalized = fileName.Replace('\\', '/'); + var lastSlash = normalized.LastIndexOf('/'); + + if (lastSlash < 0) + return (null, normalized); + + var directory = normalized[..lastSlash]; + var name = normalized[(lastSlash + 1)..]; + return (directory, name); + } + + private static string? CombineDirectoryParts(string? primary, string? secondary) + { + var parts = new List(); + + void AddPart(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + return; + + var normalized = value.Replace('\\', '/'); + foreach (var segment in normalized.Split('/', StringSplitOptions.RemoveEmptyEntries)) + { + parts.Add(segment); + } + } + + AddPart(primary); + AddPart(secondary); + + if (parts.Count == 0) + return null; + + return string.Join('/', parts); + } + + private static string SanitizeFileName(string fileName) + { + if (string.IsNullOrWhiteSpace(fileName)) + throw new ArgumentException("File name cannot be null or empty", nameof(fileName)); + + var originalFileName = fileName; + + // Check for path traversal attempts - throw exception if detected + if (fileName.Contains("..", StringComparison.Ordinal)) + throw new UnauthorizedAccessException($"Access to path '{originalFileName}' is denied. Path traversal detected."); + + // If there are path separators, extract only the filename part + // This handles cases like /tmp/file.txt -> file.txt + if (fileName.Contains('/') || fileName.Contains('\\')) + { + fileName = Path.GetFileName(fileName); + } + + if (string.IsNullOrWhiteSpace(fileName)) + throw new ArgumentException("Invalid file name", nameof(fileName)); + + // Remove any invalid filename characters + var invalidChars = Path.GetInvalidFileNameChars(); + var sanitized = fileName; + foreach (var c in invalidChars) + { + sanitized = sanitized.Replace(c.ToString(), string.Empty); + } + + if (string.IsNullOrWhiteSpace(sanitized)) + throw new ArgumentException("File name contains only invalid characters", nameof(fileName)); + + return sanitized; + } + + private static string SanitizeDirectory(string directory) + { + if (string.IsNullOrWhiteSpace(directory)) + return string.Empty; + + var originalDirectory = directory; + + // Check for path traversal attempts - throw exception if detected + if (directory.Contains("..", StringComparison.Ordinal)) + throw new UnauthorizedAccessException($"Access to path '{originalDirectory}' is denied. Path traversal detected."); + + // Normalize path separators + directory = directory.Replace('\\', '/'); + + // Remove leading and trailing slashes + directory = directory.Trim('/'); + + // Validate each directory segment + var segments = directory.Split('/', StringSplitOptions.RemoveEmptyEntries); + var invalidChars = Path.GetInvalidFileNameChars(); + + foreach (var segment in segments) + { + if (segment.IndexOfAny(invalidChars) >= 0) + throw new ArgumentException($"Directory path contains invalid characters: {segment}", nameof(directory)); + + // Additional check for suspicious segments + if (segment == ".." || segment == ".") + throw new UnauthorizedAccessException($"Access to path '{originalDirectory}' is denied. Path traversal detected."); } - EnsureDirectoryExist(Path.GetDirectoryName(filePath)!); - return filePath; + return string.Join(Path.DirectorySeparatorChar, segments); } private void EnsureDirectoryExist(string directory) diff --git a/Tests/ManagedCode.Storage.Tests/AspNetTests/Abstracts/BaseStreamControllerTests.cs b/Tests/ManagedCode.Storage.Tests/AspNetTests/Abstracts/BaseStreamControllerTests.cs index eb49c4c..f65f4d6 100644 --- a/Tests/ManagedCode.Storage.Tests/AspNetTests/Abstracts/BaseStreamControllerTests.cs +++ b/Tests/ManagedCode.Storage.Tests/AspNetTests/Abstracts/BaseStreamControllerTests.cs @@ -46,7 +46,7 @@ public async Task StreamFile_WhenFileExists_SaveToTempStorage_ReturnSuccess() var streamedValue = streamFileResult.Value ?? throw new InvalidOperationException("Stream result does not contain a stream"); await using var stream = streamedValue; - await using var newLocalFile = await LocalFile.FromStreamAsync(stream, Path.GetTempPath(), Guid.NewGuid() + await using var newLocalFile = await LocalFile.FromStreamAsync(stream, Environment.CurrentDirectory, Guid.NewGuid() .ToString("N") + extension); var streamedFileCRC = Crc32Helper.CalculateFileCrc(newLocalFile.FilePath); diff --git a/Tests/ManagedCode.Storage.Tests/AspNetTests/Azure/AzureSignalRStorageTests.cs b/Tests/ManagedCode.Storage.Tests/AspNetTests/Azure/AzureSignalRStorageTests.cs index 14213b4..8181332 100644 --- a/Tests/ManagedCode.Storage.Tests/AspNetTests/Azure/AzureSignalRStorageTests.cs +++ b/Tests/ManagedCode.Storage.Tests/AspNetTests/Azure/AzureSignalRStorageTests.cs @@ -120,7 +120,7 @@ public async Task DownloadStreamAsync_WhenBlobExists_ShouldDownloadContent() var expectedCrc = Crc32Helper.CalculateFileCrc(localFile.FilePath); memory.Position = 0; - await using var downloadedFile = await LocalFile.FromStreamAsync(memory, Path.GetTempPath(), Guid.NewGuid().ToString("N") + localFile.FileInfo.Extension); + await using var downloadedFile = await LocalFile.FromStreamAsync(memory, Environment.CurrentDirectory, Guid.NewGuid().ToString("N") + localFile.FileInfo.Extension); var downloadedCrc = Crc32Helper.CalculateFileCrc(downloadedFile.FilePath); downloadedCrc.ShouldBe(expectedCrc); diff --git a/Tests/ManagedCode.Storage.Tests/Common/FileHelper.cs b/Tests/ManagedCode.Storage.Tests/Common/FileHelper.cs index d093183..7621087 100644 --- a/Tests/ManagedCode.Storage.Tests/Common/FileHelper.cs +++ b/Tests/ManagedCode.Storage.Tests/Common/FileHelper.cs @@ -24,7 +24,7 @@ public static LocalFile GenerateLocalFile(LocalFile localFile, int byteSize) public static LocalFile GenerateLocalFile(string fileName, int byteSize) { - var path = Path.Combine(Path.GetTempPath(), fileName); + var path = Path.Combine(Environment.CurrentDirectory, fileName); var localFile = new LocalFile(path); var fs = localFile.FileStream; @@ -94,4 +94,4 @@ public static string GenerateRandomFileContent(int charCount = 250_000) .Select(s => s[Random.Next(s.Length)]) .ToArray()); } -} \ No newline at end of file +} diff --git a/Tests/ManagedCode.Storage.Tests/Core/Crc32HelperTests.cs b/Tests/ManagedCode.Storage.Tests/Core/Crc32HelperTests.cs index c42308a..1bacc55 100644 --- a/Tests/ManagedCode.Storage.Tests/Core/Crc32HelperTests.cs +++ b/Tests/ManagedCode.Storage.Tests/Core/Crc32HelperTests.cs @@ -12,7 +12,7 @@ public class Crc32HelperTests [Fact] public void CalculateFileCrc_ShouldMatchInMemoryCalculation() { - var tempPath = Path.Combine(Path.GetTempPath(), $"crc-test-{Guid.NewGuid():N}.bin"); + var tempPath = Path.Combine(Environment.CurrentDirectory, $"crc-test-{Guid.NewGuid():N}.bin"); try { var payload = new byte[4096 + 123]; diff --git a/Tests/ManagedCode.Storage.Tests/Server/ChunkUploadServiceTests.cs b/Tests/ManagedCode.Storage.Tests/Server/ChunkUploadServiceTests.cs index 0307b8a..e5117ec 100644 --- a/Tests/ManagedCode.Storage.Tests/Server/ChunkUploadServiceTests.cs +++ b/Tests/ManagedCode.Storage.Tests/Server/ChunkUploadServiceTests.cs @@ -15,7 +15,7 @@ namespace ManagedCode.Storage.Tests.Server; public class ChunkUploadServiceTests : IAsyncLifetime { - private readonly string _root = Path.Combine(Path.GetTempPath(), "managedcode-chunk-tests", Guid.NewGuid().ToString()); + private readonly string _root = Path.Combine(Environment.CurrentDirectory, "managedcode-chunk-tests", Guid.NewGuid().ToString()); private ChunkUploadOptions _options = null!; public Task InitializeAsync() diff --git a/Tests/ManagedCode.Storage.Tests/Storages/FileSystem/FileSystemSecurityTests.cs b/Tests/ManagedCode.Storage.Tests/Storages/FileSystem/FileSystemSecurityTests.cs new file mode 100644 index 0000000..c800ffa --- /dev/null +++ b/Tests/ManagedCode.Storage.Tests/Storages/FileSystem/FileSystemSecurityTests.cs @@ -0,0 +1,132 @@ +using System; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using ManagedCode.Storage.Core.Models; +using ManagedCode.Storage.FileSystem; +using ManagedCode.Storage.FileSystem.Options; +using Shouldly; +using Xunit; + +namespace ManagedCode.Storage.Tests.Storages.FileSystem; + +/// +/// Security tests for FileSystemStorage - verifying path traversal protection. +/// +public class FileSystemSecurityTests : IDisposable +{ + private readonly string _testBasePath; + private readonly FileSystemStorage _storage; + + public FileSystemSecurityTests() + { + _testBasePath = Path.Combine(Environment.CurrentDirectory, "FileSystemSecurityTests", Guid.NewGuid().ToString()); + Directory.CreateDirectory(_testBasePath); + + var options = new FileSystemStorageOptions + { + BaseFolder = _testBasePath + }; + + _storage = new FileSystemStorage(options); + } + + [Theory] + [InlineData("../../../etc/passwd")] + [InlineData("..\\..\\..\\Windows\\System32\\config\\SAM")] + [InlineData("../../../../secret.txt")] + [InlineData("..\\..\\sensitive.dat")] + public async Task UploadAsync_WithPathTraversal_ShouldFail(string maliciousFileName) + { + // Arrange + var stream = new MemoryStream(new byte[] { 1, 2, 3 }); + var options = new UploadOptions + { + FileName = maliciousFileName + }; + + // Act + var result = await _storage.UploadAsync(stream, options); + + // Assert - security validation should reject path traversal + result.IsFailed.ShouldBeTrue(); + result.Problem.Title.ShouldBe("UnauthorizedAccessException"); + } + + [Fact] + public async Task UploadAsync_WithValidFileName_ShouldSucceed() + { + // Arrange + var stream = new MemoryStream(new byte[] { 1, 2, 3 }); + var options = new UploadOptions + { + FileName = "legitimate-file.txt" + }; + + // Act + var result = await _storage.UploadAsync(stream, options); + + // Assert + result.IsSuccess.ShouldBeTrue(); + result.Value.Name.ShouldBe("legitimate-file.txt"); + } + + [Theory] + [InlineData("../../../malicious")] + [InlineData("../../outside")] + public async Task UploadAsync_WithPathTraversalInDirectory_ShouldFail( + string maliciousDirectory) + { + // Arrange + var stream = new MemoryStream(new byte[] { 1, 2, 3 }); + var options = new UploadOptions + { + FileName = "file.txt", + Directory = maliciousDirectory + }; + + // Act + var result = await _storage.UploadAsync(stream, options); + + // Assert - security validation should reject path traversal + result.IsFailed.ShouldBeTrue(); + result.Problem.Title.ShouldBe("UnauthorizedAccessException"); + } + + [Fact] + public async Task UploadAsync_WithValidDirectory_ShouldSucceed() + { + // Arrange + var stream = new MemoryStream(new byte[] { 1, 2, 3 }); + var options = new UploadOptions + { + FileName = "file.txt", + Directory = "subfolder/nested" + }; + + // Act + var result = await _storage.UploadAsync(stream, options); + + // Assert + result.IsSuccess.ShouldBeTrue(); + + // Verify file is in correct location + var expectedPath = Path.Combine(_testBasePath, "subfolder", "nested", "file.txt"); + File.Exists(expectedPath).ShouldBeTrue(); + } + + public void Dispose() + { + try + { + if (Directory.Exists(_testBasePath)) + { + Directory.Delete(_testBasePath, true); + } + } + catch + { + // Ignore cleanup errors + } + } +} diff --git a/Tests/ManagedCode.Storage.Tests/Storages/FileSystem/FileSystemUnicodeSanitizerTests.cs b/Tests/ManagedCode.Storage.Tests/Storages/FileSystem/FileSystemUnicodeSanitizerTests.cs new file mode 100644 index 0000000..9e0b14e --- /dev/null +++ b/Tests/ManagedCode.Storage.Tests/Storages/FileSystem/FileSystemUnicodeSanitizerTests.cs @@ -0,0 +1,47 @@ +using System; +using System.IO; +using System.Threading.Tasks; +using ManagedCode.Storage.FileSystem; +using ManagedCode.Storage.FileSystem.Options; +using Shouldly; +using Xunit; + +namespace ManagedCode.Storage.Tests.Storages.FileSystem; + +public class FileSystemUnicodeSanitizerTests +{ + [Fact] + public async Task ShouldResolveExistingUnicodeFileByPathOnly() + { + var root = Path.Combine(Environment.CurrentDirectory, "managedcode-vfs-existing", Guid.NewGuid().ToString("N")); + var directory = Path.Combine(root, "international", "Українська-папка"); + Directory.CreateDirectory(directory); + + var expectedFilePath = Path.Combine(directory, "лист-привіт.txt"); + await File.WriteAllTextAsync(expectedFilePath, "Привіт"); + + var storage = new FileSystemStorage(new FileSystemStorageOptions + { + BaseFolder = root, + CreateContainerIfNotExists = true + }); + + try + { + var exists = await storage.ExistsAsync("international/Українська-папка/лист-привіт.txt"); + exists.IsSuccess.ShouldBeTrue(); + exists.Value.ShouldBeTrue(); + + var metadata = await storage.GetBlobMetadataAsync("international/Українська-папка/лист-привіт.txt"); + metadata.IsSuccess.ShouldBeTrue(metadata.Problem?.ToString()); + metadata.Value!.FullName.ShouldBe("international/Українська-папка/лист-привіт.txt"); + } + finally + { + if (Directory.Exists(root)) + { + Directory.Delete(root, recursive: true); + } + } + } +} diff --git a/Tests/ManagedCode.Storage.Tests/Storages/FileSystem/FileSystemUploadTests.cs b/Tests/ManagedCode.Storage.Tests/Storages/FileSystem/FileSystemUploadTests.cs index 92efbe7..03d5817 100644 --- a/Tests/ManagedCode.Storage.Tests/Storages/FileSystem/FileSystemUploadTests.cs +++ b/Tests/ManagedCode.Storage.Tests/Storages/FileSystem/FileSystemUploadTests.cs @@ -38,7 +38,7 @@ public async Task UploadAsync_AsStream_CorrectlyOverwritesFiles() uploadStream2.Write(zeroByteBuffer); var filenameToUse = "UploadAsync_AsStream_CorrectlyOverwritesFiles.bin"; - var temporaryDirectory = Path.GetTempPath(); + var temporaryDirectory = Environment.CurrentDirectory; // Act var firstResult = await Storage.UploadAsync(uploadStream1, options => diff --git a/Tests/ManagedCode.Storage.Tests/VirtualFileSystem/FileSystemVirtualFileSystemUnicodeMountTests.cs b/Tests/ManagedCode.Storage.Tests/VirtualFileSystem/FileSystemVirtualFileSystemUnicodeMountTests.cs index 4112181..86d6e15 100644 --- a/Tests/ManagedCode.Storage.Tests/VirtualFileSystem/FileSystemVirtualFileSystemUnicodeMountTests.cs +++ b/Tests/ManagedCode.Storage.Tests/VirtualFileSystem/FileSystemVirtualFileSystemUnicodeMountTests.cs @@ -79,6 +79,7 @@ async ValueTask Cleanup() entries.Add(entry); } - entries.ShouldContain(e => e.Path.Value == expectedPath.Value); + var entryPaths = entries.ConvertAll(e => e.Path.Value); + entries.ShouldContain(e => e.Path.Value == expectedPath.Value, $"Entries: {string.Join(", ", entryPaths)}"); } }