From 1757165c4812fa45e955f64250846d12dff266bd Mon Sep 17 00:00:00 2001 From: Victor Chang Date: Thu, 6 Apr 2023 08:00:53 -0700 Subject: [PATCH] Throw any error from MinIO with ListObject APIs (#214) * Throw any error from MinIO when listing objects * Capture ListObjectsAsync exceptions in VerifyObjectExistsAsync * Configure minio client timeout * Throw VerifyObjectsException on error * Convert MinIO exception with custom exceptions * Update API doc Signed-off-by: Victor Chang --- src/Plugins/MinIO/ConfigurationKeys.cs | 3 +- src/Plugins/MinIO/LoggerMethods.cs | 9 +- src/Plugins/MinIO/MinIoClientFactory.cs | 12 +- src/Plugins/MinIO/MinIoStorageService.cs | 205 +++++++++++++----- .../MinIO/Tests/Unit/MinIoHealthCheckTest.cs | 3 +- .../Tests/Unit/MinIoStorageServiceTest.cs | 165 ++++++++++++++ src/Storage/API/IStorageService.cs | 4 + src/Storage/API/ListObjectException.cs | 48 ++++ src/Storage/API/ListObjectTimeoutException.cs | 39 ++++ src/Storage/API/StorageConnectionException.cs | 45 ++++ .../API/StorageObjectNotFoundException.cs | 39 ++++ src/Storage/API/StorageServiceException.cs | 39 ++++ src/Storage/API/VerifyObjectsException.cs | 53 +++++ 13 files changed, 600 insertions(+), 64 deletions(-) create mode 100644 src/Plugins/MinIO/Tests/Unit/MinIoStorageServiceTest.cs create mode 100644 src/Storage/API/ListObjectException.cs create mode 100644 src/Storage/API/ListObjectTimeoutException.cs create mode 100644 src/Storage/API/StorageConnectionException.cs create mode 100644 src/Storage/API/StorageObjectNotFoundException.cs create mode 100644 src/Storage/API/StorageServiceException.cs create mode 100644 src/Storage/API/VerifyObjectsException.cs diff --git a/src/Plugins/MinIO/ConfigurationKeys.cs b/src/Plugins/MinIO/ConfigurationKeys.cs index 150ae19..560aa9d 100644 --- a/src/Plugins/MinIO/ConfigurationKeys.cs +++ b/src/Plugins/MinIO/ConfigurationKeys.cs @@ -1,5 +1,5 @@ /* - * Copyright 2021-2022 MONAI Consortium + * Copyright 2021-2023 MONAI Consortium * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -29,6 +29,7 @@ internal static class ConfigurationKeys public static readonly string McExecutablePath = "executableLocation"; public static readonly string McServiceName = "serviceName"; public static readonly string CreateBuckets = "createBuckets"; + public static readonly string ApiCallTimeout = "timeout"; public static readonly string[] RequiredKeys = new[] { EndPoint, AccessKey, AccessToken, SecuredConnection, Region }; public static readonly string[] McRequiredKeys = new[] { EndPoint, AccessKey, AccessToken, McExecutablePath, McServiceName }; diff --git a/src/Plugins/MinIO/LoggerMethods.cs b/src/Plugins/MinIO/LoggerMethods.cs index edf31ff..3d92048 100644 --- a/src/Plugins/MinIO/LoggerMethods.cs +++ b/src/Plugins/MinIO/LoggerMethods.cs @@ -1,5 +1,5 @@ /* - * Copyright 2022 MONAI Consortium + * Copyright 2023 MONAI Consortium * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,6 +15,7 @@ */ using Microsoft.Extensions.Logging; +using Minio.Exceptions; namespace Monai.Deploy.Storage.MinIO { @@ -43,5 +44,11 @@ public static partial class LoggerMethods [LoggerMessage(EventId = 20007, Level = LogLevel.Information, Message = "Bucket {bucket} created in region {region}.")] public static partial void BucketCreated(this ILogger logger, string bucket, string region); + + [LoggerMessage(EventId = 20008, Level = LogLevel.Error, Message = "Error connecting to MinIO.")] + public static partial void ConnectionError(this ILogger logger, ConnectionException ex); + + [LoggerMessage(EventId = 20009, Level = LogLevel.Error, Message = "Storage service error.")] + public static partial void StorageServiceError(this ILogger logger, Exception ex); } } diff --git a/src/Plugins/MinIO/MinIoClientFactory.cs b/src/Plugins/MinIO/MinIoClientFactory.cs index 628e940..427906d 100644 --- a/src/Plugins/MinIO/MinIoClientFactory.cs +++ b/src/Plugins/MinIO/MinIoClientFactory.cs @@ -1,5 +1,5 @@ /* - * Copyright 2021-2022 MONAI Consortium + * Copyright 2021-2023 MONAI Consortium * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -26,6 +26,7 @@ namespace Monai.Deploy.Storage.MinIO public class MinIoClientFactory : IMinIoClientFactory { private static readonly string DefaultClient = "_DEFAULT_"; + internal static readonly int DefaultTimeout = 2500; private readonly ConcurrentDictionary _clients; private StorageServiceConfiguration Options { get; } @@ -112,10 +113,17 @@ private MinioClient CreateClient(string accessKey, string accessToken) { var endpoint = Options.Settings[ConfigurationKeys.EndPoint]; var securedConnection = Options.Settings[ConfigurationKeys.SecuredConnection]; + var timeout = DefaultTimeout; + + if (Options.Settings.ContainsKey(ConfigurationKeys.ApiCallTimeout) && !int.TryParse(Options.Settings[ConfigurationKeys.ApiCallTimeout], out timeout)) + { + throw new ConfigurationException($"Invalid value specified for {ConfigurationKeys.ApiCallTimeout}: {Options.Settings[ConfigurationKeys.ApiCallTimeout]}"); + } var client = new MinioClient() .WithEndpoint(endpoint) - .WithCredentials(accessKey, accessToken); + .WithCredentials(accessKey, accessToken) + .WithTimeout(timeout); if (bool.Parse(securedConnection)) { diff --git a/src/Plugins/MinIO/MinIoStorageService.cs b/src/Plugins/MinIO/MinIoStorageService.cs index cfe92e2..d44a15f 100644 --- a/src/Plugins/MinIO/MinIoStorageService.cs +++ b/src/Plugins/MinIO/MinIoStorageService.cs @@ -1,5 +1,5 @@ /* - * Copyright 2021-2022 MONAI Consortium + * Copyright 2021-2023 MONAI Consortium * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,10 +20,12 @@ using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Minio; +using Minio.Exceptions; using Monai.Deploy.Storage.API; using Monai.Deploy.Storage.Configuration; using Monai.Deploy.Storage.S3Policy; using Newtonsoft.Json; +using ObjectNotFoundException = Minio.Exceptions.ObjectNotFoundException; namespace Monai.Deploy.Storage.MinIO { @@ -39,7 +41,7 @@ public class MinIoStorageService : IStorageService public MinIoStorageService(IMinIoClientFactory minioClientFactory, IAmazonSecurityTokenServiceClientFactory amazonSecurityTokenServiceClientFactory, IOptions options, ILogger logger) { Guard.Against.Null(options); - _minioClientFactory = minioClientFactory ?? throw new ArgumentNullException(nameof(IMinIoClientFactory)); + _minioClientFactory = minioClientFactory ?? throw new ArgumentNullException(nameof(minioClientFactory)); _amazonSecurityTokenServiceClientFactory = amazonSecurityTokenServiceClientFactory ?? throw new ArgumentNullException(nameof(amazonSecurityTokenServiceClientFactory)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); @@ -101,13 +103,14 @@ public async Task> VerifyObjectsExistAsync(string bucke Guard.Against.Null(artifactList); var existingObjectsDict = new Dictionary(); + var exceptions = new List(); foreach (var artifact in artifactList) { try { - var fileObjects = await ListObjectsAsync(bucketName, artifact).ConfigureAwait(false); - var folderObjects = await ListObjectsAsync(bucketName, artifact.EndsWith("/") ? artifact : $"{artifact}/", true).ConfigureAwait(false); + var fileObjects = await ListObjectsAsync(bucketName, artifact, cancellationToken: cancellationToken).ConfigureAwait(false); + var folderObjects = await ListObjectsAsync(bucketName, artifact.EndsWith("/") ? artifact : $"{artifact}/", true, cancellationToken).ConfigureAwait(false); if (!folderObjects.Any() && !fileObjects.Any()) { @@ -122,10 +125,14 @@ public async Task> VerifyObjectsExistAsync(string bucke { _logger.VerifyObjectError(bucketName, e); existingObjectsDict.Add(artifact, false); + exceptions.Add(e); } - } + if (exceptions.Any()) + { + throw new VerifyObjectsException(exceptions, existingObjectsDict); + } return existingObjectsDict; } @@ -134,17 +141,25 @@ public async Task VerifyObjectExistsAsync(string bucketName, string artifa Guard.Against.NullOrWhiteSpace(bucketName); Guard.Against.NullOrWhiteSpace(artifactName); - var fileObjects = await ListObjectsAsync(bucketName, artifactName).ConfigureAwait(false); - var folderObjects = await ListObjectsAsync(bucketName, artifactName.EndsWith("/") ? artifactName : $"{artifactName}/", true).ConfigureAwait(false); - - if (folderObjects.Any() || fileObjects.Any()) + try { - return true; - } + var fileObjects = await ListObjectsAsync(bucketName, artifactName, cancellationToken: cancellationToken).ConfigureAwait(false); + var folderObjects = await ListObjectsAsync(bucketName, artifactName.EndsWith("/") ? artifactName : $"{artifactName}/", true, cancellationToken).ConfigureAwait(false); + + if (folderObjects.Any() || fileObjects.Any()) + { + return true; + } - _logger.FileNotFoundError(bucketName, $"{artifactName}"); + _logger.FileNotFoundError(bucketName, $"{artifactName}"); - return false; + return false; + } + catch (Exception ex) + { + _logger.VerifyObjectError(bucketName, ex); + throw new VerifyObjectsException(ex.Message, ex); + } } public async Task PutObjectAsync(string bucketName, string objectName, Stream data, long size, string contentType, Dictionary? metadata, CancellationToken cancellationToken = default) @@ -295,36 +310,51 @@ public async Task CreateFolderWithCredentialsAsync(string bucketName, string fol #region Internal Helper Methods - private static async Task CopyObjectUsingClient(IObjectOperations client, string sourceBucketName, string sourceObjectName, string destinationBucketName, string destinationObjectName, CancellationToken cancellationToken) + private async Task CopyObjectUsingClient(IObjectOperations client, string sourceBucketName, string sourceObjectName, string destinationBucketName, string destinationObjectName, CancellationToken cancellationToken) { - var copySourceObjectArgs = new CopySourceObjectArgs() - .WithBucket(sourceBucketName) - .WithObject(sourceObjectName); - var copyObjectArgs = new CopyObjectArgs() - .WithBucket(destinationBucketName) - .WithObject(destinationObjectName) - .WithCopyObjectSource(copySourceObjectArgs); - await client.CopyObjectAsync(copyObjectArgs, cancellationToken).ConfigureAwait(false); + await CallApi(async () => + { + try + { + var copySourceObjectArgs = new CopySourceObjectArgs() + .WithBucket(sourceBucketName) + .WithObject(sourceObjectName); + var copyObjectArgs = new CopyObjectArgs() + .WithBucket(destinationBucketName) + .WithObject(destinationObjectName) + .WithCopyObjectSource(copySourceObjectArgs); + await client.CopyObjectAsync(copyObjectArgs, cancellationToken).ConfigureAwait(false); + } + catch (ObjectNotFoundException ex) when (ex.ServerMessage.Contains("Not found", StringComparison.OrdinalIgnoreCase)) + { + throw new API.StorageObjectNotFoundException(ex.ServerMessage); + } + }).ConfigureAwait(false); } - private static async Task GetObjectUsingClient(IObjectOperations client, string bucketName, string objectName, Action callback, CancellationToken cancellationToken) + private async Task GetObjectUsingClient(IObjectOperations client, string bucketName, string objectName, Action callback, CancellationToken cancellationToken) { - var args = new GetObjectArgs() - .WithBucket(bucketName) - .WithObject(objectName) - .WithCallbackStream(callback); - await client.GetObjectAsync(args, cancellationToken).ConfigureAwait(false); + await CallApi(async () => + { + var args = new GetObjectArgs() + .WithBucket(bucketName) + .WithObject(objectName) + .WithCallbackStream(callback); + await client.GetObjectAsync(args, cancellationToken).ConfigureAwait(false); + }).ConfigureAwait(false); } - private async Task> ListObjectsUsingClient(IBucketOperations client, string bucketName, string? prefix, bool recursive, CancellationToken cancellationToken) + private Task> ListObjectsUsingClient(IBucketOperations client, string bucketName, string? prefix, bool recursive, CancellationToken cancellationToken) { - return await Task.Run(() => + var files = new List(); + var listArgs = new ListObjectsArgs() + .WithBucket(bucketName) + .WithPrefix(prefix) + .WithRecursive(recursive); + + try { - var files = new List(); - var listArgs = new ListObjectsArgs() - .WithBucket(bucketName) - .WithPrefix(prefix) - .WithRecursive(recursive); + var done = new TaskCompletionSource>(); var objservable = client.ListObjectsAsync(listArgs, cancellationToken); var completedEvent = new ManualResetEventSlim(false); @@ -341,44 +371,103 @@ private async Task> ListObjectsUsingClient(IBucketOperati error => { _logger.ListObjectError(bucketName, error.Message); + if (error is OperationCanceledException) + done.SetException(error); + else + done.SetException(new ListObjectException(error.ToString())); }, - () => completedEvent.Set(), cancellationToken); + () => + { + done.SetResult(files); + if (cancellationToken.IsCancellationRequested) + { + throw new ListObjectTimeoutException("Timed out waiting for results."); + } + }, cancellationToken); - completedEvent.Wait(cancellationToken); - return files; - }).ConfigureAwait(false); + return done.Task; + } + catch (ConnectionException ex) + { + _logger.ConnectionError(ex); + var iex = new StorageConnectionException(ex.Message); + iex.Errors.Add(ex.ServerMessage); + if (ex.ServerResponse is not null && !string.IsNullOrWhiteSpace(ex.ServerResponse.ErrorMessage)) + { + iex.Errors.Add(ex.ServerResponse.ErrorMessage); + } + throw iex; + } + catch (Exception ex) when (ex is not ListObjectTimeoutException && ex is not ListObjectException) + { + _logger.StorageServiceError(ex); + throw new StorageServiceException(ex.ToString()); + } } - private static async Task RemoveObjectUsingClient(IObjectOperations client, string bucketName, string objectName, CancellationToken cancellationToken) + private async Task RemoveObjectUsingClient(IObjectOperations client, string bucketName, string objectName, CancellationToken cancellationToken) { - var args = new RemoveObjectArgs() + await CallApi(async () => + { + var args = new RemoveObjectArgs() .WithBucket(bucketName) .WithObject(objectName); - await client.RemoveObjectAsync(args, cancellationToken).ConfigureAwait(false); + await client.RemoveObjectAsync(args, cancellationToken).ConfigureAwait(false); + }).ConfigureAwait(false); } - private static async Task PutObjectUsingClient(IObjectOperations client, string bucketName, string objectName, Stream data, long size, string contentType, Dictionary? metadata, CancellationToken cancellationToken) + private async Task PutObjectUsingClient(IObjectOperations client, string bucketName, string objectName, Stream data, long size, string contentType, Dictionary? metadata, CancellationToken cancellationToken) { - var args = new PutObjectArgs() - .WithBucket(bucketName) - .WithObject(objectName) - .WithStreamData(data) - .WithObjectSize(size) - .WithContentType(contentType); - if (metadata is not null) + await CallApi(async () => { - args.WithHeaders(metadata); - } + var args = new PutObjectArgs() + .WithBucket(bucketName) + .WithObject(objectName) + .WithStreamData(data) + .WithObjectSize(size) + .WithContentType(contentType); + if (metadata is not null) + { + args.WithHeaders(metadata); + } + + await client.PutObjectAsync(args, cancellationToken).ConfigureAwait(false); + }).ConfigureAwait(false); + } - await client.PutObjectAsync(args, cancellationToken).ConfigureAwait(false); + private async Task RemoveObjectsUsingClient(IObjectOperations client, string bucketName, IEnumerable objectNames, CancellationToken cancellationToken) + { + await CallApi(async () => + { + var args = new RemoveObjectsArgs() + .WithBucket(bucketName) + .WithObjects(objectNames.ToList()); + await client.RemoveObjectsAsync(args, cancellationToken).ConfigureAwait(false); + }).ConfigureAwait(false); } - private static async Task RemoveObjectsUsingClient(IObjectOperations client, string bucketName, IEnumerable objectNames, CancellationToken cancellationToken) + private async Task CallApi(Func func) { - var args = new RemoveObjectsArgs() - .WithBucket(bucketName) - .WithObjects(objectNames.ToList()); - await client.RemoveObjectsAsync(args, cancellationToken).ConfigureAwait(false); + try + { + await func().ConfigureAwait(false); + } + catch (ConnectionException ex) + { + _logger.ConnectionError(ex); + var iex = new StorageConnectionException(ex.Message); + iex.Errors.Add(ex.ServerMessage); + if (ex.ServerResponse is not null && !string.IsNullOrWhiteSpace(ex.ServerResponse.ErrorMessage)) + { + iex.Errors.Add(ex.ServerResponse.ErrorMessage); + } + throw iex; + } + catch (Exception ex) + { + _logger.StorageServiceError(ex); + throw new StorageServiceException(ex.ToString()); + } } #endregion Internal Helper Methods diff --git a/src/Plugins/MinIO/Tests/Unit/MinIoHealthCheckTest.cs b/src/Plugins/MinIO/Tests/Unit/MinIoHealthCheckTest.cs index 9584d5d..0d0b632 100644 --- a/src/Plugins/MinIO/Tests/Unit/MinIoHealthCheckTest.cs +++ b/src/Plugins/MinIO/Tests/Unit/MinIoHealthCheckTest.cs @@ -1,5 +1,5 @@ /* - * Copyright 2022 MONAI Consortium + * Copyright 2022-2023 MONAI Consortium * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,7 +16,6 @@ using Microsoft.Extensions.Diagnostics.HealthChecks; using Microsoft.Extensions.Logging; -using Minio; using Moq; using Xunit; diff --git a/src/Plugins/MinIO/Tests/Unit/MinIoStorageServiceTest.cs b/src/Plugins/MinIO/Tests/Unit/MinIoStorageServiceTest.cs new file mode 100644 index 0000000..83e85a6 --- /dev/null +++ b/src/Plugins/MinIO/Tests/Unit/MinIoStorageServiceTest.cs @@ -0,0 +1,165 @@ +/* + * Copyright 2023 MONAI Consortium + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Reactive.Disposables; +using System.Reactive.Linq; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Minio; +using Minio.DataModel; +using Minio.Exceptions; +using Monai.Deploy.Storage.API; +using Monai.Deploy.Storage.Configuration; +using Moq; +using Xunit; + +namespace Monai.Deploy.Storage.MinIO.Tests.Unit +{ + public class MinIoStorageServiceTest + { + private readonly Mock _minIoClientFactory; + private readonly Mock _amazonStsClient; + private readonly Mock> _logger; + private readonly Mock _objectOperations; + private readonly Mock _bucketOperations; + private readonly IOptions _options; + + public MinIoStorageServiceTest() + { + _minIoClientFactory = new Mock(); + _amazonStsClient = new Mock(); + _logger = new Mock>(); + _objectOperations = new Mock(); + _bucketOperations = new Mock(); + _options = Options.Create(new StorageServiceConfiguration()); + + _minIoClientFactory.Setup(p => p.GetObjectOperationsClient()).Returns(_objectOperations.Object); + _minIoClientFactory.Setup(p => p.GetBucketOperationsClient()).Returns(_bucketOperations.Object); + + _options.Value.Settings.Add(ConfigurationKeys.EndPoint, "endpoint"); + _options.Value.Settings.Add(ConfigurationKeys.AccessKey, "key"); + _options.Value.Settings.Add(ConfigurationKeys.AccessToken, "token"); + _options.Value.Settings.Add(ConfigurationKeys.SecuredConnection, "false"); + _options.Value.Settings.Add(ConfigurationKeys.Region, "region"); + } + + [Fact] + public async Task GivenAMinIoConnectionException_WhenCopyObjectIsCalled_ExpectExceptionToBeWrappedInStorageConnectionExceptionAsync() + { + await Assert.ThrowsAsync(async () => + { + var service = new MinIoStorageService(_minIoClientFactory.Object, _amazonStsClient.Object, _options, _logger.Object); + _objectOperations.Setup(p => p.CopyObjectAsync(It.IsAny(), It.IsAny())) + .ThrowsAsync(new ConnectionException("error", new ResponseResult(new HttpRequestMessage(), new Exception("inner exception")))); + + await service.CopyObjectAsync("sourceBucket", "sourceFile", "destinationBucket", "destinationFile").ConfigureAwait(false); + }).ConfigureAwait(false); + } + + [Fact] + public async Task GivenAnyException_WhenCopyObjectIsCalled_ExpectExceptionToBeWrappedInStorageServiceExceptionAsync() + { + await Assert.ThrowsAsync(async () => + { + var service = new MinIoStorageService(_minIoClientFactory.Object, _amazonStsClient.Object, _options, _logger.Object); + _objectOperations.Setup(p => p.CopyObjectAsync(It.IsAny(), It.IsAny())) + .ThrowsAsync(new Exception("inner exception")); + + await service.CopyObjectAsync("sourceBucket", "sourceFile", "destinationBucket", "destinationFile").ConfigureAwait(false); + }).ConfigureAwait(false); + } + + [Fact] + public async Task GivenAListObjectCall_WhenCancellationIsRequested_ExpectTimeoutWithListObjectTimeoutException() + { + var cancellationTokenSource = new CancellationTokenSource(); + await Assert.ThrowsAsync(async () => + { + var observable = Observable.Create(async (obs) => + { + while (true) + { + if (cancellationTokenSource.IsCancellationRequested) + { + obs.OnError(new OperationCanceledException()); + break; + } + } + return Disposable.Empty; + }); + var service = new MinIoStorageService(_minIoClientFactory.Object, _amazonStsClient.Object, _options, _logger.Object); + _bucketOperations.Setup(p => p.ListObjectsAsync(It.IsAny(), It.IsAny())) + .Returns(observable); + + cancellationTokenSource.CancelAfter(5000); + await service.ListObjectsAsync("bucket", cancellationToken: cancellationTokenSource.Token).ConfigureAwait(false); + + }).ConfigureAwait(false); + } + + [Fact] + public async Task GivenAListObjectCall_WhenErrorIsReceived_ExpectListObjectException() + { + var manualResetEvent = new ManualResetEvent(false); + await Assert.ThrowsAsync(async () => + { + var observable = Observable.Create(async (obs) => + { + obs.OnNext(new Item { Key = "key", ETag = "etag", Size = 1, IsDir = false }); + obs.OnError(new Exception("error")); + obs.OnCompleted(); + return Disposable.Empty; + }); + var service = new MinIoStorageService(_minIoClientFactory.Object, _amazonStsClient.Object, _options, _logger.Object); + _bucketOperations.Setup(p => p.ListObjectsAsync(It.IsAny(), It.IsAny())) + .Returns(observable); + + var listObjectTask = service.ListObjectsAsync("bucket"); + await Task.Delay(3000).ConfigureAwait(false); + + await listObjectTask.ConfigureAwait(false); + }).ConfigureAwait(false); + } + + [Fact] + public async Task GivenAListObjectCall_WhenConnectionExceptionIsThrown_ExpectTheExceptionToBeWrappedInStorageConnectionException() + { + var manualResetEvent = new ManualResetEvent(false); + await Assert.ThrowsAsync(async () => + { + var service = new MinIoStorageService(_minIoClientFactory.Object, _amazonStsClient.Object, _options, _logger.Object); + _bucketOperations.Setup(p => p.ListObjectsAsync(It.IsAny(), It.IsAny())) + .Throws(new ConnectionException("error", new ResponseResult(new HttpRequestMessage(), new Exception("inner exception")))); + + await service.ListObjectsAsync("bucket").ConfigureAwait(false); + }).ConfigureAwait(false); + } + + [Fact] + public async Task GivenAListObjectCall_WhenAnyMinIoExceptionIsThrown_ExpectTheExceptionToBeWrappedInStorageServiceException() + { + var manualResetEvent = new ManualResetEvent(false); + await Assert.ThrowsAsync(async () => + { + var service = new MinIoStorageService(_minIoClientFactory.Object, _amazonStsClient.Object, _options, _logger.Object); + _bucketOperations.Setup(p => p.ListObjectsAsync(It.IsAny(), It.IsAny())) + .Throws(new InvalidBucketNameException("bucket", "bad")); + + await service.ListObjectsAsync("bucket").ConfigureAwait(false); + }).ConfigureAwait(false); + } + } +} diff --git a/src/Storage/API/IStorageService.cs b/src/Storage/API/IStorageService.cs index 3d44ea8..7c5bb4f 100644 --- a/src/Storage/API/IStorageService.cs +++ b/src/Storage/API/IStorageService.cs @@ -27,6 +27,7 @@ public interface IStorageService /// /// Lists objects in a bucket. + /// Caution: use of this API requires user to manually manage timeout via CancellationToken in case storage service stops responding. /// /// Name of the bucket /// Objects with name starts with prefix @@ -97,6 +98,7 @@ public interface IStorageService /// /// Verifies a list of artifacts to ensure that they exist. + /// Caution: use of this API requires user to manually manage timeout via CancellationToken in case storage service stops responding. /// /// Name of the bucket /// Artifacts to verify @@ -105,6 +107,7 @@ public interface IStorageService /// /// Verifies the existence of an artifact to ensure that they exist. + /// Caution: use of this API requires user to manually manage timeout via CancellationToken in case storage service stops responding. /// /// Name of the bucket /// Artifact to verify @@ -148,6 +151,7 @@ public interface IStorageService /// /// Lists objects in a bucket using temporary credentials. + /// Caution: use of this API requires user to manually manage timeout via CancellationToken in case storage service stops responding. /// /// Name of the bucket /// Temporary credentials used to connect diff --git a/src/Storage/API/ListObjectException.cs b/src/Storage/API/ListObjectException.cs new file mode 100644 index 0000000..7f49360 --- /dev/null +++ b/src/Storage/API/ListObjectException.cs @@ -0,0 +1,48 @@ +/* + * Copyright 2023 MONAI Consortium + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace Monai.Deploy.Storage.API +{ + public class ListObjectException : Exception + { + private readonly List _files; + + public Exception? Exception { get; } + public IReadOnlyList Files + { get { return _files; } } + + public ListObjectException() + { + _files = new List(); + } + + public ListObjectException(string? message) : base(message) + { + _files = new List(); + } + + public ListObjectException(string? message, Exception? innerException) : base(message, innerException) + { + _files = new List(); + } + + public ListObjectException(Exception exception, List files) + { + Exception = exception; + _files = files; + } + } +} diff --git a/src/Storage/API/ListObjectTimeoutException.cs b/src/Storage/API/ListObjectTimeoutException.cs new file mode 100644 index 0000000..4cb0063 --- /dev/null +++ b/src/Storage/API/ListObjectTimeoutException.cs @@ -0,0 +1,39 @@ +/* + * Copyright 2023 MONAI Consortium + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Runtime.Serialization; + +namespace Monai.Deploy.Storage.API +{ + public class ListObjectTimeoutException : Exception + { + public ListObjectTimeoutException() + { + } + + public ListObjectTimeoutException(string? message) : base(message) + { + } + + public ListObjectTimeoutException(string? message, Exception? innerException) : base(message, innerException) + { + } + + protected ListObjectTimeoutException(SerializationInfo info, StreamingContext context) : base(info, context) + { + } + } +} diff --git a/src/Storage/API/StorageConnectionException.cs b/src/Storage/API/StorageConnectionException.cs new file mode 100644 index 0000000..786bfe9 --- /dev/null +++ b/src/Storage/API/StorageConnectionException.cs @@ -0,0 +1,45 @@ +/* + * Copyright 2023 MONAI Consortium + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Runtime.Serialization; + +namespace Monai.Deploy.Storage.API +{ + public class StorageConnectionException : Exception + { + public string ServerMessage { get; set; } + public List Errors { get; set; } + + public StorageConnectionException() + { + Errors = new List(); + } + + public StorageConnectionException(string message) : base(message) + { + Errors = new List(); + } + + public StorageConnectionException(string message, Exception innerException) : base(message, innerException) + { + Errors = new List(); + } + + protected StorageConnectionException(SerializationInfo info, StreamingContext context) : base(info, context) + { + } + } +} diff --git a/src/Storage/API/StorageObjectNotFoundException.cs b/src/Storage/API/StorageObjectNotFoundException.cs new file mode 100644 index 0000000..63cde89 --- /dev/null +++ b/src/Storage/API/StorageObjectNotFoundException.cs @@ -0,0 +1,39 @@ +/* + * Copyright 2023 MONAI Consortium + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Runtime.Serialization; + +namespace Monai.Deploy.Storage.API +{ + public class StorageObjectNotFoundException : Exception + { + public StorageObjectNotFoundException() + { + } + + public StorageObjectNotFoundException(string message) : base(message) + { + } + + public StorageObjectNotFoundException(string message, Exception innerException) : base(message, innerException) + { + } + + protected StorageObjectNotFoundException(SerializationInfo info, StreamingContext context) : base(info, context) + { + } + } +} diff --git a/src/Storage/API/StorageServiceException.cs b/src/Storage/API/StorageServiceException.cs new file mode 100644 index 0000000..fb4b1b1 --- /dev/null +++ b/src/Storage/API/StorageServiceException.cs @@ -0,0 +1,39 @@ +/* + * Copyright 2023 MONAI Consortium + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Runtime.Serialization; + +namespace Monai.Deploy.Storage.API +{ + public class StorageServiceException : Exception + { + public StorageServiceException() + { + } + + public StorageServiceException(string message) : base(message) + { + } + + public StorageServiceException(string message, Exception innerException) : base(message, innerException) + { + } + + protected StorageServiceException(SerializationInfo info, StreamingContext context) : base(info, context) + { + } + } +} diff --git a/src/Storage/API/VerifyObjectsException.cs b/src/Storage/API/VerifyObjectsException.cs new file mode 100644 index 0000000..8bf2fa1 --- /dev/null +++ b/src/Storage/API/VerifyObjectsException.cs @@ -0,0 +1,53 @@ +/* + * Copyright 2023 MONAI Consortium + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace Monai.Deploy.Storage.API +{ + public class VerifyObjectsException : Exception + { + private readonly List _errors; + private readonly Dictionary _results; + + public IReadOnlyList Exceptions + { get { return _errors; } } + public IReadOnlyDictionary Results + { get { return _results; } } + + public VerifyObjectsException() + { + _errors = new List(); + _results = new Dictionary(); + } + + public VerifyObjectsException(string? message) : base(message) + { + _errors = new List(); + _results = new Dictionary(); + } + + public VerifyObjectsException(string? message, Exception? innerException) : base(message, innerException) + { + _errors = new List(); + _results = new Dictionary(); + } + + public VerifyObjectsException(List errors, Dictionary files) + { + _errors = errors; + _results = files; + } + } +}