From e458e885f464e734b98b64602edc53e5400f7452 Mon Sep 17 00:00:00 2001 From: psainics Date: Tue, 21 Jan 2025 09:27:55 +0530 Subject: [PATCH] Catch known exceptions in ProgramFailureException --- .../io/cdap/plugin/gcp/gcs/StorageClient.java | 134 ++++++++++++++---- 1 file changed, 103 insertions(+), 31 deletions(-) diff --git a/src/main/java/io/cdap/plugin/gcp/gcs/StorageClient.java b/src/main/java/io/cdap/plugin/gcp/gcs/StorageClient.java index e124a4e73..5cb2c9e8a 100644 --- a/src/main/java/io/cdap/plugin/gcp/gcs/StorageClient.java +++ b/src/main/java/io/cdap/plugin/gcp/gcs/StorageClient.java @@ -28,7 +28,11 @@ import com.google.cloud.storage.StorageException; import com.google.cloud.storage.StorageOptions; import com.google.common.annotations.VisibleForTesting; +import io.cdap.cdap.api.exception.ErrorCategory; +import io.cdap.cdap.api.exception.ErrorType; +import io.cdap.cdap.api.exception.ErrorUtils; import io.cdap.plugin.gcp.common.GCPConnectorConfig; +import io.cdap.plugin.gcp.common.GCPErrorDetailsProviderUtil; import io.cdap.plugin.gcp.common.GCPUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -68,7 +72,14 @@ public Blob pickABlob(String path) { return null; } GCSPath gcsPath = GCSPath.from(path); - Page blobPage = storage.list(gcsPath.getBucket(), Storage.BlobListOption.prefix(gcsPath.getName())); + Page blobPage; + try { + blobPage = storage.list(gcsPath.getBucket(), Storage.BlobListOption.prefix(gcsPath.getName())); + } catch (Exception e) { + String errorReason = String.format("Unable to list objects in bucket %s.", gcsPath.getBucket()); + throw GCPErrorDetailsProviderUtil.getHttpResponseExceptionDetailsFromChain(e, errorReason, ErrorType.UNKNOWN, + true, GCPUtils.GCS_SUPPORTED_DOC_URL); + } Iterator iterator = blobPage.getValues().iterator(); while (iterator.hasNext()) { Blob blob = iterator.next(); @@ -89,7 +100,13 @@ public void setMetaData(Blob blob, Map metaData) { if (blob == null || metaData == null || metaData.isEmpty()) { return; } - storage.update(BlobInfo.newBuilder(blob.getBlobId()).setMetadata(metaData).build()); + try { + storage.update(BlobInfo.newBuilder(blob.getBlobId()).setMetadata(metaData).build()); + } catch (Exception e) { + String errorReason = String.format("Unable to update metadata for blob %s.", blob.getName()); + throw GCPErrorDetailsProviderUtil.getHttpResponseExceptionDetailsFromChain(e, errorReason, ErrorType.UNKNOWN, + true, GCPUtils.GCS_SUPPORTED_DOC_URL); + } } /** @@ -102,7 +119,14 @@ public void mapMetaDataForAllBlobs(String path, Consumer> fu return; } GCSPath gcsPath = GCSPath.from(path); - Page blobPage = storage.list(gcsPath.getBucket(), Storage.BlobListOption.prefix(gcsPath.getName())); + Page blobPage; + try { + blobPage = storage.list(gcsPath.getBucket(), Storage.BlobListOption.prefix(gcsPath.getName())); + } catch (Exception e) { + String errorReason = String.format("Unable to list objects in bucket %s.", gcsPath.getBucket()); + throw GCPErrorDetailsProviderUtil.getHttpResponseExceptionDetailsFromChain(e, errorReason, ErrorType.UNKNOWN, + true, GCPUtils.GCS_SUPPORTED_DOC_URL); + } Iterator blobIterator = blobPage.iterateAll().iterator(); while (blobIterator.hasNext()) { Blob blob = blobIterator.next(); @@ -132,9 +156,11 @@ public void createBucketIfNotExists(GCSPath path, @Nullable String location, @Nu LOG.warn("Getting 409 Conflict: {} Bucket at destination path {} may already exist.", e.getMessage(), path.getUri()); } else { - throw new RuntimeException( + String errorReason = String.format("Unable to create bucket %s. Ensure you entered the correct bucket path and " + - "have permissions for it.", path.getBucket()), e); + "have permissions for it.", path.getBucket()); + throw GCPErrorDetailsProviderUtil.getHttpResponseExceptionDetailsFromChain(e, errorReason, ErrorType.UNKNOWN, + true, GCPUtils.GCS_SUPPORTED_DOC_URL); } } } @@ -173,9 +199,16 @@ public void move(GCSPath sourcePath, GCSPath destPath, boolean recursive, boolea * Get all the matching wildcard paths given the regex input. */ public List getMatchedPaths(GCSPath sourcePath, boolean recursive, Pattern wildcardRegex) { - Page blobPage = storage.list(sourcePath.getBucket(), Storage.BlobListOption.prefix( + Page blobPage; + try { + blobPage = storage.list(sourcePath.getBucket(), Storage.BlobListOption.prefix( getWildcardPathPrefix(sourcePath, wildcardRegex) - )); + )); + } catch (Exception e) { + String errorReason = String.format("Unable to list objects in bucket %s.", sourcePath.getBucket()); + throw GCPErrorDetailsProviderUtil.getHttpResponseExceptionDetailsFromChain(e, errorReason, ErrorType.UNKNOWN, + true, GCPUtils.GCS_SUPPORTED_DOC_URL); + } List blobPageNames = new ArrayList<>(); blobPage.getValues().forEach(blob -> blobPageNames.add(blob.getName())); return getFilterMatchedPaths(sourcePath, blobPageNames, recursive); @@ -212,36 +245,47 @@ static List getFilterMatchedPaths(GCSPath sourcePath, List blob private void pairTraverse(GCSPath sourcePath, GCSPath destPath, boolean recursive, boolean overwrite, Consumer consumer) { - Bucket sourceBucket = null; + Bucket sourceBucket; try { sourceBucket = storage.get(sourcePath.getBucket()); - } catch (StorageException e) { + } catch (Exception e) { // Add more descriptive error message - throw new RuntimeException( - String.format("Unable to access source bucket %s. ", sourcePath.getBucket()) - + "Ensure you entered the correct bucket path.", e); + String errorReason = String.format("Unable to access GCS bucket '%s'", sourcePath.getBucket()); + throw GCPErrorDetailsProviderUtil.getHttpResponseExceptionDetailsFromChain(e, errorReason, ErrorType.UNKNOWN, + true, GCPUtils.GCS_SUPPORTED_DOC_URL); } if (sourceBucket == null) { - throw new IllegalArgumentException( - String.format("Source bucket '%s' does not exist.", sourcePath.getBucket())); + String errorReason = String.format("Source bucket '%s' does not exist.", sourcePath.getBucket()); + throw ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategory.ErrorCategoryEnum.PLUGIN), + errorReason, errorReason, ErrorType.USER, true, null); } - Bucket destBucket = null; + Bucket destBucket; try { destBucket = storage.get(destPath.getBucket()); - } catch (StorageException e) { + } catch (Exception e) { // Add more descriptive error message - throw new RuntimeException( - String.format("Unable to access destination bucket %s. ", destPath.getBucket()) - + "Ensure you entered the correct bucket path.", e); + String errorReason = String.format("Unable to access GCS bucket '%s'", destPath.getBucket()); + throw GCPErrorDetailsProviderUtil.getHttpResponseExceptionDetailsFromChain(e, errorReason, ErrorType.UNKNOWN, + true, GCPUtils.GCS_SUPPORTED_DOC_URL); } if (destBucket == null) { - throw new IllegalArgumentException( - String.format("Destination bucket '%s' does not exist. Please create it first.", destPath.getBucket())); + String errorReason = + String.format("Destination bucket '%s' does not exist. Please create it first.", destPath.getBucket()); + throw ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategory.ErrorCategoryEnum.PLUGIN), + errorReason, errorReason, ErrorType.USER, true, null); } boolean destinationBaseExists; String baseDestName = destPath.getName(); - if (destPath.isBucket() || storage.get(BlobId.of(destPath.getBucket(), baseDestName)) != null) { + boolean destinationBlobExists; + try { + destinationBlobExists = destPath.isBucket() || storage.get(BlobId.of(destPath.getBucket(), baseDestName)) != null; + } catch (Exception e) { + String errorReason = String.format("Unable to access GCS bucket '%s'", destPath.getBucket()); + throw GCPErrorDetailsProviderUtil.getHttpResponseExceptionDetailsFromChain(e, errorReason, ErrorType.UNKNOWN, + true, GCPUtils.GCS_SUPPORTED_DOC_URL); + } + if (destinationBlobExists) { destinationBaseExists = true; } else { // if gs://bucket2/subdir doesn't exist, check if gs://bucket2/subdir/ exists @@ -249,7 +293,13 @@ private void pairTraverse(GCSPath sourcePath, GCSPath destPath, boolean recursiv // this is because "cp dir0 subdir" and "cp dir0 subdir/" are equivalent if the 'subdir' directory exists String modifiedName = baseDestName.endsWith("/") ? baseDestName.substring(0, baseDestName.length() - 1) : baseDestName + "/"; - destinationBaseExists = storage.get(BlobId.of(destPath.getBucket(), modifiedName)) != null; + try { + destinationBaseExists = storage.get(BlobId.of(destPath.getBucket(), modifiedName)) != null; + } catch (Exception e) { + String errorReason = String.format("Unable to access GCS bucket '%s'", destPath.getBucket()); + throw GCPErrorDetailsProviderUtil.getHttpResponseExceptionDetailsFromChain(e, errorReason, ErrorType.UNKNOWN, + true, GCPUtils.GCS_SUPPORTED_DOC_URL); + } } List copyList = new ArrayList<>(); @@ -257,13 +307,22 @@ private void pairTraverse(GCSPath sourcePath, GCSPath destPath, boolean recursiv BlobId destBlobID = resolve(sourcePath.getName(), sourceBlob.getBlobId().getName(), destPath, destinationBaseExists); if (!overwrite) { - Blob destBlob = storage.get(destBlobID); + Blob destBlob; + try { + destBlob = storage.get(destBlobID); + } catch (Exception e) { + String errorReason = String.format("Unable to access GCS bucket '%s'", destPath.getBucket()); + throw GCPErrorDetailsProviderUtil.getHttpResponseExceptionDetailsFromChain(e, errorReason, ErrorType.UNKNOWN, + true, GCPUtils.GCS_SUPPORTED_DOC_URL); + } // we can't just use Blob's isDirectory() because the cloud console will create a 'directory' by creating // a 0 size placeholder blob that ends with '/'. This placeholder blob's isDirectory() method returns false, // but we don't want the overwrite check to fail on it. So we explicitly ignore the check for these 0 size // placeholder blobs. if (destBlob != null && !destBlob.getName().endsWith("/") && destBlob.getSize() != 0) { - throw new IllegalArgumentException(String.format("%s already exists.", toPath(destBlobID))); + String errorReason = String.format("%s already exists.", toPath(destBlobID)); + throw ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategory.ErrorCategoryEnum.PLUGIN), + errorReason, errorReason, ErrorType.USER, true, null); } } copyList.add(new BlobPair(sourceBlob, destBlobID)); @@ -347,8 +406,15 @@ static String append(String base, String part) { * @param consumer the blob consumer */ private void traverse(BlobId blobId, boolean recursive, Consumer consumer) { - Page blobList = storage.list(blobId.getBucket(), Storage.BlobListOption.currentDirectory(), - Storage.BlobListOption.prefix(blobId.getName())); + Page blobList; + try { + blobList = storage.list(blobId.getBucket(), Storage.BlobListOption.currentDirectory(), + Storage.BlobListOption.prefix(blobId.getName())); + } catch (Exception e) { + String errorReason = String.format(""); + throw GCPErrorDetailsProviderUtil.getHttpResponseExceptionDetailsFromChain(e, errorReason, ErrorType.UNKNOWN, + true, GCPUtils.GCS_SUPPORTED_DOC_URL); + } for (Blob blob : blobList.iterateAll()) { if (!blob.isDirectory()) { consumer.accept(blob); @@ -363,11 +429,17 @@ private static String toPath(BlobId blobId) { } public static StorageClient create(String project, @Nullable String serviceAccount, - Boolean isServiceAccountFilePath, @Nullable Integer readTimeout) - throws IOException { + Boolean isServiceAccountFilePath, @Nullable Integer readTimeout) { StorageOptions.Builder builder = StorageOptions.newBuilder().setProjectId(project); if (serviceAccount != null) { - builder.setCredentials(GCPUtils.loadServiceAccountCredentials(serviceAccount, isServiceAccountFilePath)); + try { + builder.setCredentials(GCPUtils.loadServiceAccountCredentials(serviceAccount, isServiceAccountFilePath)); + } catch (IOException e) { + String errorReason = "Unable to load service account credentials."; + throw ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategory.ErrorCategoryEnum.PLUGIN), + errorReason, String.format("%s %s: %s", errorReason, e.getClass().getName(), e.getMessage()), + ErrorType.UNKNOWN, false, null); + } } if (readTimeout != null) { builder.setTransportOptions(HttpTransportOptions.newBuilder().setReadTimeout(readTimeout * 1000).build()); @@ -376,7 +448,7 @@ public static StorageClient create(String project, @Nullable String serviceAccou return new StorageClient(storage); } - public static StorageClient create(GCPConnectorConfig config) throws IOException { + public static StorageClient create(GCPConnectorConfig config) { return create(config.getProject(), config.getServiceAccount(), config.isServiceAccountFilePath(), null); }