From 846944b2d8eb1cd13806c939a6343685544c54a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marco=20K=C3=B6pcke?= Date: Tue, 4 Jan 2022 14:35:57 +0100 Subject: [PATCH 1/2] Support for buckets that don't support lifecycle configurations --- .../blobstore/s3/internal/BucketManager.java | 62 ++++++++++++------- .../s3/internal/S3BlobStoreException.java | 2 + 2 files changed, 42 insertions(+), 22 deletions(-) diff --git a/plugins/nexus-blobstore-s3/src/main/java/org/sonatype/nexus/blobstore/s3/internal/BucketManager.java b/plugins/nexus-blobstore-s3/src/main/java/org/sonatype/nexus/blobstore/s3/internal/BucketManager.java index d477a16fce..a81b2dfa84 100644 --- a/plugins/nexus-blobstore-s3/src/main/java/org/sonatype/nexus/blobstore/s3/internal/BucketManager.java +++ b/plugins/nexus-blobstore-s3/src/main/java/org/sonatype/nexus/blobstore/s3/internal/BucketManager.java @@ -39,13 +39,7 @@ import static org.sonatype.nexus.blobstore.s3.internal.S3BlobStoreConfigurationHelper.getBucketPrefix; import static org.sonatype.nexus.blobstore.s3.internal.S3BlobStoreConfigurationHelper.getConfiguredBucket; import static org.sonatype.nexus.blobstore.s3.internal.S3BlobStoreConfigurationHelper.getConfiguredExpirationInDays; -import static org.sonatype.nexus.blobstore.s3.internal.S3BlobStoreException.ACCESS_DENIED_CODE; -import static org.sonatype.nexus.blobstore.s3.internal.S3BlobStoreException.INVALID_ACCESS_KEY_ID_CODE; -import static org.sonatype.nexus.blobstore.s3.internal.S3BlobStoreException.SIGNATURE_DOES_NOT_MATCH_CODE; -import static org.sonatype.nexus.blobstore.s3.internal.S3BlobStoreException.bucketOwnershipError; -import static org.sonatype.nexus.blobstore.s3.internal.S3BlobStoreException.buildException; -import static org.sonatype.nexus.blobstore.s3.internal.S3BlobStoreException.insufficientCreatePermissionsError; -import static org.sonatype.nexus.blobstore.s3.internal.S3BlobStoreException.unexpectedError; +import static org.sonatype.nexus.blobstore.s3.internal.S3BlobStoreException.*; /** * Creates and deletes buckets for the {@link S3BlobStore}. @@ -87,9 +81,17 @@ public void prepareStorageLocation(final BlobStoreConfiguration blobStoreConfigu } else { // bucket exists, we should test that the correct lifecycle config is present - BucketLifecycleConfiguration lifecycleConfiguration = s3.getBucketLifecycleConfiguration(bucket); - if (!isExpirationLifecycleConfigurationPresent(lifecycleConfiguration, blobStoreConfiguration)) { - setBucketLifecycleConfiguration(s3, blobStoreConfiguration, lifecycleConfiguration); + try { + BucketLifecycleConfiguration lifecycleConfiguration = s3.getBucketLifecycleConfiguration(bucket); + if (!isExpirationLifecycleConfigurationPresent(lifecycleConfiguration, blobStoreConfiguration)) { + setBucketLifecycleConfiguration(s3, blobStoreConfiguration, lifecycleConfiguration); + } + } catch (AmazonS3Exception e) { + if (NOT_IMPLEMENTED_CODE.equals(e.getErrorCode())) { + log.warn("Bucket {} does not support managing lifecycle config.", bucket, e); + } else { + throw e; + } } } } @@ -103,13 +105,21 @@ public void deleteStorageLocation(final BlobStoreConfiguration blobStoreConfigur } else { log.info("Not removing S3 bucket {} because it is not empty", bucket); - BucketLifecycleConfiguration lifecycleConfiguration = s3.getBucketLifecycleConfiguration(bucket); - List nonBlobstoreRules = nonBlobstoreRules(lifecycleConfiguration, blobStoreConfiguration.getName()); - if(!isEmpty(nonBlobstoreRules)) { - lifecycleConfiguration.setRules(nonBlobstoreRules); - s3.setBucketLifecycleConfiguration(bucket, lifecycleConfiguration); - } else { - s3.deleteBucketLifecycleConfiguration(bucket); + try { + BucketLifecycleConfiguration lifecycleConfiguration = s3.getBucketLifecycleConfiguration(bucket); + List nonBlobstoreRules = nonBlobstoreRules(lifecycleConfiguration, blobStoreConfiguration.getName()); + if(!isEmpty(nonBlobstoreRules)) { + lifecycleConfiguration.setRules(nonBlobstoreRules); + s3.setBucketLifecycleConfiguration(bucket, lifecycleConfiguration); + } else { + s3.deleteBucketLifecycleConfiguration(bucket); + } + } catch (AmazonS3Exception e) { + if (NOT_IMPLEMENTED_CODE.equals(e.getErrorCode())) { + log.warn("Bucket {} does not support managing lifecycle config.", bucket, e); + } else { + throw e; + } } } } @@ -181,11 +191,19 @@ private void setBucketLifecycleConfiguration(final AmazonS3 s3, String bucket = getConfiguredBucket(blobStoreConfiguration); BucketLifecycleConfiguration newLifecycleConfiguration = makeLifecycleConfiguration(lifecycleConfiguration, blobStoreConfiguration); - if (newLifecycleConfiguration != null) { - s3.setBucketLifecycleConfiguration(bucket, newLifecycleConfiguration); - } - else if (lifecycleConfiguration != null && !lifecycleConfiguration.getRules().isEmpty()) { - s3.deleteBucketLifecycleConfiguration(bucket); + try { + if (newLifecycleConfiguration != null) { + s3.setBucketLifecycleConfiguration(bucket, newLifecycleConfiguration); + } + else if (lifecycleConfiguration != null && !lifecycleConfiguration.getRules().isEmpty()) { + s3.deleteBucketLifecycleConfiguration(bucket); + } + } catch (AmazonS3Exception e) { + if (NOT_IMPLEMENTED_CODE.equals(e.getErrorCode())) { + log.warn("Bucket {} does not support managing lifecycle config.", bucket, e); + } else { + throw e; + } } } diff --git a/plugins/nexus-blobstore-s3/src/main/java/org/sonatype/nexus/blobstore/s3/internal/S3BlobStoreException.java b/plugins/nexus-blobstore-s3/src/main/java/org/sonatype/nexus/blobstore/s3/internal/S3BlobStoreException.java index aeb61f120f..e18daa9dbb 100644 --- a/plugins/nexus-blobstore-s3/src/main/java/org/sonatype/nexus/blobstore/s3/internal/S3BlobStoreException.java +++ b/plugins/nexus-blobstore-s3/src/main/java/org/sonatype/nexus/blobstore/s3/internal/S3BlobStoreException.java @@ -45,6 +45,8 @@ public class S3BlobStoreException public static final String SIGNATURE_DOES_NOT_MATCH_CODE = "SignatureDoesNotMatch"; + public static final String NOT_IMPLEMENTED_CODE = "NotImplemented"; + public static Map ERROR_CODE_MESSAGES = ImmutableMap.of( INVALID_ACCESS_KEY_ID_CODE, "The Access Key ID provided was invalid.", ACCESS_DENIED_CODE, "Access denied. Please check the credentials provided have proper permissions.", From 4e55b279fdc88f8d17c7904d94ce45a58f690f77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marco=20K=C3=B6pcke?= Date: Tue, 4 Jan 2022 15:22:57 +0100 Subject: [PATCH 2/2] Support for buckets that don't support setObjectTagging --- .../blobstore/s3/internal/S3BlobStore.java | 31 ++++++++++++++----- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/plugins/nexus-blobstore-s3/src/main/java/org/sonatype/nexus/blobstore/s3/internal/S3BlobStore.java b/plugins/nexus-blobstore-s3/src/main/java/org/sonatype/nexus/blobstore/s3/internal/S3BlobStore.java index a3d17acdbf..6a5ee0b945 100644 --- a/plugins/nexus-blobstore-s3/src/main/java/org/sonatype/nexus/blobstore/s3/internal/S3BlobStore.java +++ b/plugins/nexus-blobstore-s3/src/main/java/org/sonatype/nexus/blobstore/s3/internal/S3BlobStore.java @@ -85,6 +85,7 @@ import static org.sonatype.nexus.blobstore.api.OperationType.DOWNLOAD; import static org.sonatype.nexus.blobstore.api.OperationType.UPLOAD; import static org.sonatype.nexus.blobstore.s3.internal.S3BlobStoreConfigurationHelper.getConfiguredExpirationInDays; +import static org.sonatype.nexus.blobstore.s3.internal.S3BlobStoreException.NOT_IMPLEMENTED_CODE; import static org.sonatype.nexus.blobstore.s3.internal.S3BlobStoreException.buildException; import static org.sonatype.nexus.common.stateguard.StateGuardLifecycleSupport.State.FAILED; import static org.sonatype.nexus.common.stateguard.StateGuardLifecycleSupport.State.NEW; @@ -497,11 +498,19 @@ else if (blobAttributes.isDeleted()) { blobAttributes.setDeletedDateTime(new DateTime()); blobAttributes.store(); - // soft delete is implemented using an S3 lifecycle that sets expiration on objects with DELETED_TAG - // tag the bytes - s3.setObjectTagging(tagAsDeleted(contentPath(blobId))); - // tag the attributes - s3.setObjectTagging(tagAsDeleted(attributePath(blobId))); + try { + // soft delete is implemented using an S3 lifecycle that sets expiration on objects with DELETED_TAG + // tag the bytes + s3.setObjectTagging(tagAsDeleted(contentPath(blobId))); + // tag the attributes + s3.setObjectTagging(tagAsDeleted(attributePath(blobId))); + } catch (AmazonS3Exception e) { + if (NOT_IMPLEMENTED_CODE.equals(e.getErrorCode())) { + log.warn("Bucket does not support object tagging.", e); + } else { + throw e; + } + } blob.markStale(); Long contentSize = getContentSizeForDeletion(blobAttributes); @@ -797,8 +806,16 @@ public void setBlobAttributes(BlobId blobId, BlobAttributes blobAttributes) { @Override @Timed protected void doUndelete(final BlobId blobId, final BlobAttributes attributes) { - s3.setObjectTagging(untagAsDeleted(contentPath(blobId))); - s3.setObjectTagging(untagAsDeleted(attributePath(blobId))); + try { + s3.setObjectTagging(untagAsDeleted(contentPath(blobId))); + s3.setObjectTagging(untagAsDeleted(attributePath(blobId))); + } catch (AmazonS3Exception e) { + if (NOT_IMPLEMENTED_CODE.equals(e.getErrorCode())) { + log.warn("Bucket does not support object tagging.", e); + } else { + throw e; + } + } storeMetrics.recordAddition(attributes.getMetrics().getContentSize()); }