diff --git a/src/main/java/org/sagebionetworks/web/client/PortalGinInjector.java b/src/main/java/org/sagebionetworks/web/client/PortalGinInjector.java index 7f077b5ccb..288f722075 100644 --- a/src/main/java/org/sagebionetworks/web/client/PortalGinInjector.java +++ b/src/main/java/org/sagebionetworks/web/client/PortalGinInjector.java @@ -257,7 +257,6 @@ import org.sagebionetworks.web.client.widget.upload.FileHandleLink; import org.sagebionetworks.web.client.widget.upload.FileHandleUploadWidget; import org.sagebionetworks.web.client.widget.upload.ImageUploadView; -import org.sagebionetworks.web.client.widget.upload.MultipartUploaderImpl; import org.sagebionetworks.web.client.widget.user.UserBadge; import org.sagebionetworks.web.client.widget.verification.VerificationSubmissionModalViewImpl; import org.sagebionetworks.web.client.widget.verification.VerificationSubmissionRowViewImpl; @@ -863,6 +862,4 @@ public interface PortalGinInjector extends Ginjector { FollowingPagePresenter getFollowingPagePresenter(); ColumnModelsEditorWidget getColumnModelsEditorWidget(); - - MultipartUploaderImpl getLegacyMultipartUploader(); } diff --git a/src/main/java/org/sagebionetworks/web/client/widget/entity/download/Uploader.java b/src/main/java/org/sagebionetworks/web/client/widget/entity/download/Uploader.java index cb9f8d0f95..5d203d154e 100644 --- a/src/main/java/org/sagebionetworks/web/client/widget/entity/download/Uploader.java +++ b/src/main/java/org/sagebionetworks/web/client/widget/entity/download/Uploader.java @@ -77,8 +77,7 @@ public class Uploader private SynapseJSNIUtils synapseJsniUtils; private GlobalApplicationState globalAppState; private GWTWrapper gwt; - MultipartUploader multiPartUploader; - MultipartUploader legacyMultiPartUploader; + private MultipartUploader multiPartUploader; AuthenticationController authenticationController; private String[] fileNames; @@ -125,7 +124,6 @@ public Uploader( this.jsClient = jsClient; this.synapseProperties = synapseProperties; this.eventBus = eventBus; - legacyMultiPartUploader = ginInjector.getLegacyMultipartUploader(); view.setPresenter(this); } @@ -671,11 +669,8 @@ public void directUploadStep2(String fileName) { view ); } else { - // TODO: PLFM-8252: If Google Cloud platform Synapse solution supports parallel upload, then remove legacyMultiPartUploader (and all associated code) - MultipartUploader currentUploader = currentUploadType == UploadType.S3 - ? multiPartUploader - : legacyMultiPartUploader; - currentUploader.uploadFile( + // SWC-6765: Uses react implementation for uploading a file in all cases + multiPartUploader.uploadFile( fileName, contentType, currentFile, diff --git a/src/main/java/org/sagebionetworks/web/client/widget/upload/MultipartUploaderImpl.java b/src/main/java/org/sagebionetworks/web/client/widget/upload/MultipartUploaderImpl.java deleted file mode 100755 index 546647c9c3..0000000000 --- a/src/main/java/org/sagebionetworks/web/client/widget/upload/MultipartUploaderImpl.java +++ /dev/null @@ -1,542 +0,0 @@ -package org.sagebionetworks.web.client.widget.upload; - -import com.google.gwt.core.client.JavaScriptObject; -import com.google.gwt.event.logical.shared.HasAttachHandlers; -import com.google.gwt.i18n.client.NumberFormat; -import com.google.gwt.user.client.rpc.AsyncCallback; -import com.google.gwt.xhr.client.ReadyStateChangeHandler; -import com.google.gwt.xhr.client.XMLHttpRequest; -import com.google.inject.Inject; -import java.util.Collections; -import java.util.Date; -import org.sagebionetworks.repo.model.file.AddPartResponse; -import org.sagebionetworks.repo.model.file.AddPartState; -import org.sagebionetworks.repo.model.file.BatchPresignedUploadUrlRequest; -import org.sagebionetworks.repo.model.file.BatchPresignedUploadUrlResponse; -import org.sagebionetworks.repo.model.file.MultipartUploadRequest; -import org.sagebionetworks.repo.model.file.MultipartUploadStatus; -import org.sagebionetworks.repo.model.file.PartPresignedUrl; -import org.sagebionetworks.repo.model.file.PartUtils; -import org.sagebionetworks.web.client.DateTimeUtils; -import org.sagebionetworks.web.client.DisplayConstants; -import org.sagebionetworks.web.client.DisplayUtils; -import org.sagebionetworks.web.client.GWTWrapper; -import org.sagebionetworks.web.client.ProgressCallback; -import org.sagebionetworks.web.client.SynapseJSNIUtils; -import org.sagebionetworks.web.client.SynapseJavascriptClient; -import org.sagebionetworks.web.client.cookie.CookieProvider; -import org.sagebionetworks.web.client.utils.Callback; - -/** - * This was extracted from the uploader. - * - * @author Jay - * - */ -public class MultipartUploaderImpl implements MultipartUploader { - - public static final String PLEASE_SELECT_A_FILE = "Please select a file."; - public static final String BINARY_CONTENT_TYPE = "application/octet-stream"; - public static final String EMPTY_FILE_ERROR_MESSAGE = - "The selected file is empty: "; - // if any parts fail to upload, then it will restart the upload from the beginning up to 10 times, - // with a 3 second delay between attempts. - public static final int RETRY_DELAY = 3000; - - private GWTWrapper gwt; - private SynapseJavascriptClient jsClient; - private SynapseJSNIUtils synapseJsniUtils; - private NumberFormat percentFormat; - private CookieProvider cookies; - - // This class will create a multipart upload request (containing information specific to the file - // that the user wants to upload). - private MultipartUploadRequest request; - // Get the upload status of all parts from the backend. Will be refreshed from the server on each - // attempt. - private MultipartUploadStatus currentStatus; - // For convenient reference, remember how many parts this file upload has. - private int totalPartCount; - // Report success/failure/progress to the given ProgressingFileUploadHandler. - private ProgressingFileUploadHandler handler; - // Will retry the entire file upload if any part fails to upload. Use this variable to flag the - // necessity to retry after going through all parts. - private boolean retryRequired; - // Keep track of the part number (1-based index) that we are currently trying to upload. - private int currentPartNumber; - private int completedPartCount; - private long startTime, nextProgressPoint; - private String uploadSpeed; - // in alpha mode, upload log is sent to the js console - private boolean isDebugLevelLogging = false; - JavaScriptObject blob; - HasAttachHandlers view; - boolean isCanceled; - private long fileLastModifiedMs = -1; - DateTimeUtils dateTimeUtils; - - @Inject - public MultipartUploaderImpl( - GWTWrapper gwt, - SynapseJSNIUtils synapseJsniUtils, - SynapseJavascriptClient jsClient, - CookieProvider cookies, - DateTimeUtils dateTimeUtils - ) { - super(); - this.gwt = gwt; - this.synapseJsniUtils = synapseJsniUtils; - this.jsClient = jsClient; - this.percentFormat = gwt.getNumberFormat("##"); - this.cookies = cookies; - this.dateTimeUtils = dateTimeUtils; - } - - @Override - public void uploadFile( - final String fileName, - final String contentType, - final JavaScriptObject blob, - ProgressingFileUploadHandler handler, - final Long storageLocationId, - HasAttachHandlers view - ) { - // initialize attempt count. - this.request = null; - this.totalPartCount = 0; - this.handler = handler; - this.blob = blob; - this.view = view; - isCanceled = false; - isDebugLevelLogging = DisplayUtils.isInTestWebsite(cookies); - - long fileSize = (long) synapseJsniUtils.getFileSize(blob); - if (fileSize <= 0) { - handler.uploadSuccess(null); - return; - } - - log( - gwt.getUserAgent() + - "\n" + - gwt.getAppVersion() + - "\nDirectly uploading " + - fileName + - "\n" - ); - long partSizeBytes = PartUtils.choosePartSize(fileSize); - // create request - request = new MultipartUploadRequest(); - request.setContentType(contentType); - request.setFileName(fileName); - request.setFileSizeBytes(fileSize); - request.setPartSizeBytes(partSizeBytes); - request.setStorageLocationId(storageLocationId); - startMultipartUpload(); - } - - /** - * Start uploading the file - */ - public void startMultipartUpload() { - if (isStillUploading()) { - retryRequired = false; - fileLastModifiedMs = synapseJsniUtils.getLastModified(blob); - synapseJsniUtils.getFileMd5( - blob, - md5 -> { - if (md5 == null) { - handler.uploadFailed(DisplayConstants.MD5_CALCULATION_ERROR); - return; - } - if ( - request.getContentMD5Hex() != null && - !md5.equals(request.getContentMD5Hex()) - ) { - uploadFailedDueToFileModification(request.getContentMD5Hex(), md5); - } else { - startMultipartUpload(md5); - } - } - ); - } - } - - private void startMultipartUpload(String md5) { - request.setContentMD5Hex(md5); - String fileStats = - "fileName=" + - request.getFileName() + - " MD5=" + - request.getContentMD5Hex() + - " contentType=" + - request.getContentType() + - " fileSize=" + - request.getFileSizeBytes() + - " partSizeBytes=" + - request.getPartSizeBytes() + - "\n"; - log(fileStats); - - // update the status and process - jsClient.startMultipartUpload( - request, - false, - new AsyncCallback() { - @Override - public void onFailure(Throwable t) { - logError(t.getMessage()); - handler.uploadFailed(t.getMessage()); - } - - @Override - public void onSuccess(MultipartUploadStatus status) { - currentStatus = status; - currentPartNumber = 0; - startTime = new Date().getTime(); - nextProgressPoint = 2000; - uploadSpeed = ""; - totalPartCount = currentStatus.getPartsState().length(); - completedPartCount = - getCompletedPartCount(currentStatus.getPartsState()); - attemptToUploadNextPart(); - } - } - ); - } - - public int getCompletedPartCount(String partState) { - int successCount = 0; - for (int i = 0; i < partState.length(); i++) { - if (partState.charAt(i) == '1') { - successCount++; - } - } - - return successCount; - } - - /** - * Increment the current part that we are processing. Look at the MultipartUploadStatus part state - * to determine if we should try to upload the part, or skip it. - */ - public void attemptToUploadNextPart() { - // for each chunk that still needs to be uploaded, get the presigned url and upload to it - currentPartNumber++; - - // sanity check - if last modified has changed, then fail. - long lastModifiedOn = synapseJsniUtils.getLastModified(blob); - if (lastModifiedOn != fileLastModifiedMs) { - uploadFailedDueToFileModification(new Date(lastModifiedOn)); - return; - } - if (currentStatus.getPartsState().charAt(currentPartNumber - 1) == '0') { - attemptUploadCurrentPart(); - } else { - // this part has already been uploaded, skip it - log( - "attemptChunkUpload: skipping part number = " + currentPartNumber + "\n" - ); - partSuccess(); - } - } - - /** - * Get a presigned URL for the current part number, upload the part to it, and add the part to the - * upload. - */ - public void attemptUploadCurrentPart() { - log( - "attemptChunkUpload: attempting to upload part number = " + - currentPartNumber + - "\n" - ); - BatchPresignedUploadUrlRequest batchPresignedUploadUrlRequest = - new BatchPresignedUploadUrlRequest(); - batchPresignedUploadUrlRequest.setContentType(BINARY_CONTENT_TYPE); - batchPresignedUploadUrlRequest.setPartNumbers( - Collections.singletonList(new Long(currentPartNumber)) - ); - batchPresignedUploadUrlRequest.setUploadId(currentStatus.getUploadId()); - jsClient.getMultipartPresignedUrlBatch( - batchPresignedUploadUrlRequest, - new AsyncCallback() { - @Override - public void onFailure(Throwable caught) { - partFailure(caught.getMessage()); - } - - @Override - public void onSuccess( - BatchPresignedUploadUrlResponse batchPresignedUploadUrlResponse - ) { - if (isStillUploading()) { - PartPresignedUrl url = batchPresignedUploadUrlResponse - .getPartPresignedUrls() - .get(0); - String urlString = url.getUploadPresignedUrl(); - final XMLHttpRequest xhr = gwt.createXMLHttpRequest(); - if (xhr != null) { - xhr.setOnReadyStateChange( - new ReadyStateChangeHandler() { - @Override - public void onReadyStateChange(XMLHttpRequest xhr) { - log( - "XMLHttpRequest.setOnReadyStateChange: readyState=" + - xhr.getReadyState() + - " status=" + - xhr.getStatus() + - "\n" - ); - if (xhr.getReadyState() == 4) { // XMLHttpRequest.DONE=4, posts suggest this value is not resolved in some browsers - xhr.clearOnReadyStateChange(); - if (xhr.getStatus() == 200) { // OK - log("XMLHttpRequest.setOnReadyStateChange: OK\n"); - // add part number to the upload (and potentially complete) - addCurrentPartToMultipartUpload(); - } else { - log( - "XMLHttpRequest.setOnReadyStateChange: Failure\n" + - xhr.getStatusText() - ); - partFailure( - xhr.getStatus() + ": " + xhr.getStatusText() - ); - } - } - } - } - ); - } - ByteRange range = new ByteRange( - currentPartNumber, - request.getFileSizeBytes(), - request.getPartSizeBytes() - ); - log( - "attemptChunkUpload: uploading file chunk. ByteRange=" + - range.getStart() + - "-" + - range.getEnd() + - " \n" - ); - ProgressCallback progressCallback = new ProgressCallback() { - @Override - public void updateProgress(double loaded, double total) { - // 0 < currentPartProgress < 1. We need to add this to the chunks that have already been uploaded. - // And divide by the total chunk count. - if (!isStillUploading()) { - if (xhr != null) { - xhr.abort(); - } - } else { - double currentPartProgress = loaded / total; - double currentProgress = - (((double) (completedPartCount)) + currentPartProgress) / - ((double) totalPartCount); - String progressText = - percentFormat.format(currentProgress * 100.0) + "%"; - // update uploadSpeed every couple of seconds - long msElapsed = (new Date().getTime() - startTime); - if (msElapsed > 0 && (msElapsed > nextProgressPoint)) { - double totalBytesTransfered = - (request.getPartSizeBytes() * completedPartCount) + - loaded; - uploadSpeed = - "(" + - DisplayUtils.getFriendlySize( - totalBytesTransfered / (msElapsed / 1000), - true - ) + - "/s)"; - nextProgressPoint += 2000; - } - handler.updateProgress( - currentProgress, - progressText, - uploadSpeed - ); - } - } - }; - synapseJsniUtils.uploadFileChunk( - BINARY_CONTENT_TYPE, - blob, - range.getStart(), - range.getEnd(), - urlString, - xhr, - progressCallback - ); - } - } - } - ); - } - - /** - * @return True if user is still looking at the upload UI. - */ - public boolean isStillUploading() { - return view.isAttached() && !isCanceled; - } - - /** - * Called if the current part was successfully uploaded. Will continue on to process the next file - * part (if there is one). - */ - public void partSuccess() { - checkAllPartsProcessed(); - } - - /** - * Called if the current part failed to upload. Will continue on to process the next file part (if - * there is one). - */ - public void partFailure(String message) { - logError("Upload error on part " + currentPartNumber + ": \n" + message); - retryRequired = true; - checkAllPartsProcessed(); - } - - /** - * If all parts have been processed, it will either restart (if there were any problems during - * upload that were detected), or attempt to complete the upload. If parts are left, then it will - * continue on to process the next file part. - */ - public void checkAllPartsProcessed() { - if (currentPartNumber >= totalPartCount) { - if (retryRequired) { - // wait a couple of seconds and start over :( - gwt.scheduleExecution( - new Callback() { - @Override - public void invoke() { - startMultipartUpload(); - } - }, - RETRY_DELAY - ); - } else { - // complete upload and return file handle - completeMultipartUpload(); - } - } else { - attemptToUploadNextPart(); - } - } - - public void completeMultipartUpload() { - // before finishing, verify that the file checksum has not changed during the upload. - synapseJsniUtils.getFileMd5( - blob, - md5 -> { - if (!request.getContentMD5Hex().equals(md5)) { - uploadFailedDueToFileModification(request.getContentMD5Hex(), md5); - } else { - completeMultipartUploadAfterMd5Verification(); - } - } - ); - } - - private void uploadFailedDueToFileModification( - String startMd5, - String newMd5 - ) { - handler.uploadFailed( - "Unable to upload the file \"" + - request.getFileName() + - "\" because it's been modified during the upload. \n\nThe starting md5 of the file (" + - startMd5 + - ") differs from the current md5 (" + - newMd5 + - ")." - ); - } - - private void uploadFailedDueToFileModification(Date lastModifiedDate) { - handler.uploadFailed( - "Unable to upload the file \"" + - request.getFileName() + - "\" because it's been modified during the upload. \n\nThe file was last modified " + - dateTimeUtils.getRelativeTime(lastModifiedDate) + - "." - ); - } - - public void completeMultipartUploadAfterMd5Verification() { - // combine - jsClient.completeMultipartUpload( - currentStatus.getUploadId(), - new AsyncCallback() { - @Override - public void onFailure(Throwable caught) { - // failed to complete multipart upload. log it and start over. - logError(caught.getMessage()); - retryRequired = true; - checkAllPartsProcessed(); - } - - @Override - public void onSuccess(MultipartUploadStatus status) { - handler.uploadSuccess(status.getResultFileHandleId()); - } - } - ); - } - - public void addCurrentPartToMultipartUpload() { - // calculate the md5 of this file part - if (isStillUploading()) { - synapseJsniUtils.getFilePartMd5( - blob, - currentPartNumber - 1, - request.getPartSizeBytes(), - partMd5 -> { - log("partNumber=" + currentPartNumber + " partNumberMd5=" + partMd5); - jsClient.addPartToMultipartUpload( - currentStatus.getUploadId(), - currentPartNumber, - partMd5, - new AsyncCallback() { - @Override - public void onFailure(Throwable caught) { - partFailure(caught.getMessage()); - } - - public void onSuccess(AddPartResponse addPartResponse) { - if ( - addPartResponse - .getAddPartState() - .equals(AddPartState.ADD_SUCCESS) - ) { - completedPartCount++; - partSuccess(); - } else { - partFailure(addPartResponse.getErrorMessage()); - } - } - } - ); - } - ); - } - } - - public void log(String message) { - if (isDebugLevelLogging) { - synapseJsniUtils.consoleLog(message); - } - } - - public void logError(String message) { - // to the console - synapseJsniUtils.consoleError(message); - } - - @Override - public void cancelUpload() { - isCanceled = true; - } -} diff --git a/src/test/java/org/sagebionetworks/web/unitclient/widget/entity/download/UploaderTest.java b/src/test/java/org/sagebionetworks/web/unitclient/widget/entity/download/UploaderTest.java index 512abfbc13..9d4d038f74 100644 --- a/src/test/java/org/sagebionetworks/web/unitclient/widget/entity/download/UploaderTest.java +++ b/src/test/java/org/sagebionetworks/web/unitclient/widget/entity/download/UploaderTest.java @@ -70,8 +70,6 @@ import org.sagebionetworks.web.client.widget.entity.download.S3DirectUploader; import org.sagebionetworks.web.client.widget.entity.download.Uploader; import org.sagebionetworks.web.client.widget.entity.download.UploaderView; -import org.sagebionetworks.web.client.widget.upload.MultipartUploader; -import org.sagebionetworks.web.client.widget.upload.MultipartUploaderImpl; import org.sagebionetworks.web.shared.WebConstants; import org.sagebionetworks.web.shared.exceptions.NotFoundException; import org.sagebionetworks.web.shared.exceptions.RestServiceException; @@ -86,9 +84,6 @@ public class UploaderTest { MultipartUploaderStub multipartUploader; - @Mock - MultipartUploaderImpl mockLegacyMultipartUploader; - @Mock UploaderView mockView; @@ -174,8 +169,6 @@ public class UploaderTest { @Before public void before() throws Exception { multipartUploader = new MultipartUploaderStub(); - when(mockGinInjector.getLegacyMultipartUploader()) - .thenReturn(mockLegacyMultipartUploader); testEntity = new FileEntity(); testEntity.setName("test file"); testEntity.setId("syn99"); @@ -896,39 +889,6 @@ public void testQueryForUploadDestinationsWithUploadToExternalObjectStore() { ); } - @Test - public void testUploadToGoogleBucket() { - String banner = "banner"; - String endpoint = "endpointUrl"; - String bucket = "mr.h"; - String keyPrefixUUID = "keyPrefixUUID"; - UploadType uploadType = UploadType.GOOGLECLOUDSTORAGE; - String fileName = "f.txt"; - when(mockExternalGoogleCloudUploadDestination.getBanner()) - .thenReturn(banner); - when(mockExternalGoogleCloudUploadDestination.getStorageLocationId()) - .thenReturn(storageLocationId); - when(mockExternalGoogleCloudUploadDestination.getUploadType()) - .thenReturn(uploadType); - when(mockExternalGoogleCloudUploadDestination.getBucket()) - .thenReturn(bucket); - - AsyncMockStubber - .callSuccessWith( - Collections.singletonList(mockExternalGoogleCloudUploadDestination) - ) - .when(mockSynapseJavascriptClient) - .getUploadDestinations(anyString(), any(AsyncCallback.class)); - uploader.queryForUploadDestination(); - assertEquals(uploader.getStorageLocationId(), storageLocationId); - assertEquals(uploader.getCurrentUploadType(), uploadType); - - uploader.directUploadStep2(fileName); - - verify(mockLegacyMultipartUploader) - .uploadFile(anyString(), anyString(), any(), any(), anyLong(), any()); - } - @Test public void testDragAndDrop() throws RestServiceException { // widget configured in @Before diff --git a/src/test/java/org/sagebionetworks/web/unitclient/widget/upload/MultipartUploaderTest.java b/src/test/java/org/sagebionetworks/web/unitclient/widget/upload/MultipartUploaderTest.java deleted file mode 100755 index 7d00279955..0000000000 --- a/src/test/java/org/sagebionetworks/web/unitclient/widget/upload/MultipartUploaderTest.java +++ /dev/null @@ -1,808 +0,0 @@ -package org.sagebionetworks.web.unitclient.widget.upload; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.mockito.Matchers.any; -import static org.mockito.Matchers.anyBoolean; -import static org.mockito.Matchers.anyInt; -import static org.mockito.Matchers.anyLong; -import static org.mockito.Matchers.anyString; -import static org.mockito.Matchers.eq; -import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.reset; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyZeroInteractions; -import static org.mockito.Mockito.when; -import static org.sagebionetworks.web.client.ContentTypeUtils.fixDefaultContentType; -import static org.sagebionetworks.web.client.widget.upload.MultipartUploaderImpl.EMPTY_FILE_ERROR_MESSAGE; - -import com.google.gwt.core.client.JavaScriptObject; -import com.google.gwt.event.logical.shared.HasAttachHandlers; -import com.google.gwt.user.client.rpc.AsyncCallback; -import com.google.gwt.xhr.client.XMLHttpRequest; -import java.util.ArrayList; -import java.util.List; -import org.junit.Before; -import org.junit.Test; -import org.mockito.ArgumentCaptor; -import org.mockito.Captor; -import org.mockito.Mock; -import org.mockito.MockitoAnnotations; -import org.mockito.invocation.InvocationOnMock; -import org.mockito.stubbing.Answer; -import org.sagebionetworks.repo.model.file.AddPartResponse; -import org.sagebionetworks.repo.model.file.AddPartState; -import org.sagebionetworks.repo.model.file.BatchPresignedUploadUrlRequest; -import org.sagebionetworks.repo.model.file.BatchPresignedUploadUrlResponse; -import org.sagebionetworks.repo.model.file.MultipartUploadRequest; -import org.sagebionetworks.repo.model.file.MultipartUploadStatus; -import org.sagebionetworks.repo.model.file.PartPresignedUrl; -import org.sagebionetworks.repo.model.file.PartUtils; -import org.sagebionetworks.repo.model.util.ContentTypeUtils; -import org.sagebionetworks.web.client.ClientProperties; -import org.sagebionetworks.web.client.DateTimeUtils; -import org.sagebionetworks.web.client.DisplayConstants; -import org.sagebionetworks.web.client.GWTWrapper; -import org.sagebionetworks.web.client.ProgressCallback; -import org.sagebionetworks.web.client.SynapseJSNIUtils; -import org.sagebionetworks.web.client.SynapseJavascriptClient; -import org.sagebionetworks.web.client.callback.MD5Callback; -import org.sagebionetworks.web.client.cookie.CookieProvider; -import org.sagebionetworks.web.client.utils.Callback; -import org.sagebionetworks.web.client.widget.upload.ByteRange; -import org.sagebionetworks.web.client.widget.upload.MultipartUploaderImpl; -import org.sagebionetworks.web.client.widget.upload.ProgressingFileUploadHandler; -import org.sagebionetworks.web.shared.WebConstants; -import org.sagebionetworks.web.shared.exceptions.RestServiceException; -import org.sagebionetworks.web.test.helper.AsyncMockStubber; -import org.sagebionetworks.web.test.helper.CallbackMockStubber; - -public class MultipartUploaderTest { - - @Mock - ProgressingFileUploadHandler mockHandler; - - @Mock - SynapseJSNIUtils synapseJsniUtils; - - @Mock - GWTWrapper gwt; - - MultipartUploaderImpl uploader; - String MD5; - String partMd5; - - String[] fileNames; - Long storageLocationId = 9090L; - - @Mock - SynapseJavascriptClient mockJsClient; - - @Mock - CookieProvider mockCookies; - - @Mock - MultipartUploadStatus mockMultipartUploadStatus; - - @Mock - BatchPresignedUploadUrlResponse mockBatchPresignedUploadUrlResponse; - - @Mock - PartPresignedUrl mockPartPresignedUrl; - - @Mock - AddPartResponse mockAddPartResponse; - - @Mock - HasAttachHandlers mockView; - - @Mock - DateTimeUtils mockDateTimeUtils; - - @Captor - ArgumentCaptor progressCaptor; - - @Mock - JavaScriptObject mockFileBlob; - - public static final String UPLOAD_ID = "39282"; - public static final String RESULT_FILE_HANDLE_ID = "999999"; - public static final double FILE_SIZE = 9281; - public static final String FILE_NAME = "file.txt"; - public static final String CONTENT_TYPE = "text/plain"; - - @Before - public void before() throws Exception { - MockitoAnnotations.initMocks(this); - - // direct upload - // by default, do not support direct upload (direct upload tests will turn on) - when(synapseJsniUtils.getContentType(any(JavaScriptObject.class), anyInt())) - .thenReturn("image/png"); - AsyncMockStubber - .callSuccessWith(mockMultipartUploadStatus) - .when(mockJsClient) - .startMultipartUpload( - any(MultipartUploadRequest.class), - anyBoolean(), - any(AsyncCallback.class) - ); - when(mockMultipartUploadStatus.getUploadId()).thenReturn(UPLOAD_ID); - when(mockMultipartUploadStatus.getResultFileHandleId()) - .thenReturn(RESULT_FILE_HANDLE_ID); - List presignedUrlList = new ArrayList(); - when(mockBatchPresignedUploadUrlResponse.getPartPresignedUrls()) - .thenReturn(presignedUrlList); - presignedUrlList.add(mockPartPresignedUrl); - when(mockPartPresignedUrl.getUploadPresignedUrl()) - .thenReturn("http://fakepresignedurl.uploader.test"); - AsyncMockStubber - .callSuccessWith(mockBatchPresignedUploadUrlResponse) - .when(mockJsClient) - .getMultipartPresignedUrlBatch( - any(BatchPresignedUploadUrlRequest.class), - any(AsyncCallback.class) - ); - when(mockAddPartResponse.getAddPartState()) - .thenReturn(AddPartState.ADD_SUCCESS); - AsyncMockStubber - .callSuccessWith(mockAddPartResponse) - .when(mockJsClient) - .addPartToMultipartUpload( - anyString(), - anyInt(), - anyString(), - any(AsyncCallback.class) - ); - AsyncMockStubber - .callSuccessWith(mockMultipartUploadStatus) - .when(mockJsClient) - .completeMultipartUpload(anyString(), any(AsyncCallback.class)); - // Stub the generation of a MD5. - MD5 = "some md5"; - doAnswer( - new Answer() { - @Override - public Void answer(InvocationOnMock invocation) throws Throwable { - final Object[] args = invocation.getArguments(); - ((MD5Callback) args[args.length - 1]).setMD5(MD5); - return null; - } - } - ) - .when(synapseJsniUtils) - .getFileMd5(any(JavaScriptObject.class), any(MD5Callback.class)); - - partMd5 = "another md5"; - doAnswer( - new Answer() { - @Override - public Void answer(InvocationOnMock invocation) throws Throwable { - final Object[] args = invocation.getArguments(); - ((MD5Callback) args[args.length - 1]).setMD5(partMd5); - return null; - } - } - ) - .when(synapseJsniUtils) - .getFilePartMd5( - any(JavaScriptObject.class), - anyInt(), - anyLong(), - any(MD5Callback.class) - ); - - when(synapseJsniUtils.getFileSize(any(JavaScriptObject.class))) - .thenReturn(FILE_SIZE); - // fire the timer - CallbackMockStubber - .invokeCallback() - .when(gwt) - .scheduleExecution(any(Callback.class), anyInt()); - when(gwt.createXMLHttpRequest()).thenReturn(null); - - String[] fileNames = { "newFile.txt" }; - when( - synapseJsniUtils.getMultipleUploadFileNames(any(JavaScriptObject.class)) - ) - .thenReturn(fileNames); - - String file1 = "file1.txt"; - fileNames = new String[] { file1 }; - when( - synapseJsniUtils.getMultipleUploadFileNames(any(JavaScriptObject.class)) - ) - .thenReturn(fileNames); - - uploader = - new MultipartUploaderImpl( - gwt, - synapseJsniUtils, - mockJsClient, - mockCookies, - mockDateTimeUtils - ); - - when(mockView.isAttached()).thenReturn(true); - } - - private void setPartsState(String partsState) { - when(mockMultipartUploadStatus.getPartsState()).thenReturn(partsState); - } - - @Test - public void testDirectUploadFolder() throws Exception { - setPartsState("0"); - doAnswer( - new Answer() { - @Override - public Void answer(InvocationOnMock invocation) throws Throwable { - final Object[] args = invocation.getArguments(); - ((MD5Callback) args[args.length - 1]).setMD5(null); - return null; - } - } - ) - .when(synapseJsniUtils) - .getFileMd5(any(JavaScriptObject.class), any(MD5Callback.class)); - - uploader.uploadFile( - FILE_NAME, - CONTENT_TYPE, - mockFileBlob, - mockHandler, - storageLocationId, - mockView - ); - verify(synapseJsniUtils) - .getFileMd5(any(JavaScriptObject.class), any(MD5Callback.class)); - verify(mockHandler).uploadFailed(DisplayConstants.MD5_CALCULATION_ERROR); - } - - @Test - public void testDirectUploadAllPartsExist() throws Exception { - setPartsState("11"); - uploader.uploadFile( - FILE_NAME, - CONTENT_TYPE, - mockFileBlob, - mockHandler, - storageLocationId, - mockView - ); - verify(mockJsClient) - .startMultipartUpload( - any(MultipartUploadRequest.class), - anyBoolean(), - any(AsyncCallback.class) - ); - - // never tries to get a presigned url, since all parts are uploaded. - verify(mockJsClient, never()) - .getMultipartPresignedUrlBatch( - any(BatchPresignedUploadUrlRequest.class), - any(AsyncCallback.class) - ); - verify(synapseJsniUtils, never()) - .uploadFileChunk( - anyString(), - any(JavaScriptObject.class), - anyLong(), - anyLong(), - anyString(), - any(XMLHttpRequest.class), - any(ProgressCallback.class) - ); - // combine parts - verify(mockJsClient) - .completeMultipartUpload(anyString(), any(AsyncCallback.class)); - // the handler should get the id. - verify(mockHandler).uploadSuccess(RESULT_FILE_HANDLE_ID); - } - - @Test - public void testDirectUploadEmptyFile() throws Exception { - when(synapseJsniUtils.getFileSize(any(JavaScriptObject.class))) - .thenReturn(0.0); - setPartsState("0"); - uploader.uploadFile( - FILE_NAME, - CONTENT_TYPE, - mockFileBlob, - mockHandler, - storageLocationId, - mockView - ); - verify(mockJsClient, never()) - .startMultipartUpload( - any(MultipartUploadRequest.class), - anyBoolean(), - any(AsyncCallback.class) - ); - verify(mockHandler).uploadSuccess(null); - } - - @Test - public void testDirectUploadSinglePart() throws Exception { - setPartsState("0"); - uploader.uploadFile( - FILE_NAME, - CONTENT_TYPE, - mockFileBlob, - mockHandler, - storageLocationId, - mockView - ); - verify(mockJsClient) - .startMultipartUpload( - any(MultipartUploadRequest.class), - anyBoolean(), - any(AsyncCallback.class) - ); - ArgumentCaptor captor = - ArgumentCaptor.forClass(BatchPresignedUploadUrlRequest.class); - verify(mockJsClient) - .getMultipartPresignedUrlBatch( - captor.capture(), - any(AsyncCallback.class) - ); - assertEquals( - MultipartUploaderImpl.BINARY_CONTENT_TYPE, - captor.getValue().getContentType() - ); - verify(synapseJsniUtils) - .uploadFileChunk( - eq(MultipartUploaderImpl.BINARY_CONTENT_TYPE), - any(JavaScriptObject.class), - anyLong(), - anyLong(), - anyString(), - any(XMLHttpRequest.class), - any(ProgressCallback.class) - ); - // manually call the method that's invoked with a successful xhr put (upload) - uploader.addCurrentPartToMultipartUpload(); - verify(mockJsClient) - .addPartToMultipartUpload( - anyString(), - anyInt(), - anyString(), - any(AsyncCallback.class) - ); - verify(mockJsClient) - .completeMultipartUpload(anyString(), any(AsyncCallback.class)); - // the handler should get the id. - verify(mockHandler).uploadSuccess(RESULT_FILE_HANDLE_ID); - } - - @Test - public void testDirectUploadSingleSecondPart() throws Exception { - setPartsState("10"); - uploader.uploadFile( - FILE_NAME, - CONTENT_TYPE, - mockFileBlob, - mockHandler, - storageLocationId, - mockView - ); - verify(mockJsClient) - .startMultipartUpload( - any(MultipartUploadRequest.class), - anyBoolean(), - any(AsyncCallback.class) - ); - verify(mockJsClient) - .getMultipartPresignedUrlBatch( - any(BatchPresignedUploadUrlRequest.class), - any(AsyncCallback.class) - ); - verify(synapseJsniUtils) - .uploadFileChunk( - anyString(), - any(JavaScriptObject.class), - anyLong(), - anyLong(), - anyString(), - any(XMLHttpRequest.class), - any(ProgressCallback.class) - ); - // manually call the method that's invoked with a successful xhr put (upload) - uploader.addCurrentPartToMultipartUpload(); - verify(mockJsClient) - .addPartToMultipartUpload( - anyString(), - anyInt(), - anyString(), - any(AsyncCallback.class) - ); - verify(mockJsClient) - .completeMultipartUpload(anyString(), any(AsyncCallback.class)); - // the handler should get the id. - verify(mockHandler).uploadSuccess(RESULT_FILE_HANDLE_ID); - } - - @Test - public void testDirectUploadMultiPart() throws Exception { - setPartsState("00"); - uploader.uploadFile( - FILE_NAME, - CONTENT_TYPE, - mockFileBlob, - mockHandler, - storageLocationId, - mockView - ); - verify(mockJsClient) - .startMultipartUpload( - any(MultipartUploadRequest.class), - anyBoolean(), - any(AsyncCallback.class) - ); - verify(mockJsClient) - .getMultipartPresignedUrlBatch( - any(BatchPresignedUploadUrlRequest.class), - any(AsyncCallback.class) - ); - verify(synapseJsniUtils) - .uploadFileChunk( - anyString(), - any(JavaScriptObject.class), - anyLong(), - anyLong(), - anyString(), - any(XMLHttpRequest.class), - any(ProgressCallback.class) - ); - // manually call the method that's invoked with a successful xhr put (upload) - uploader.addCurrentPartToMultipartUpload(); - verify(mockJsClient) - .addPartToMultipartUpload( - anyString(), - anyInt(), - anyString(), - any(AsyncCallback.class) - ); - // should not have completed, since there's another part - - // SWC-4262: check the file md5 once in the beginning - verify(synapseJsniUtils) - .getFileMd5(any(JavaScriptObject.class), any(MD5Callback.class)); - verify(mockJsClient, never()) - .completeMultipartUpload(anyString(), any(AsyncCallback.class)); - - uploader.addCurrentPartToMultipartUpload(); - verify(mockJsClient) - .completeMultipartUpload(anyString(), any(AsyncCallback.class)); - // SWC-4262: check the file md5 once in the beginning, and once on complete. - verify(synapseJsniUtils, times(2)) - .getFileMd5(any(JavaScriptObject.class), any(MD5Callback.class)); - - // the handler should get the id. - verify(mockHandler).uploadSuccess(RESULT_FILE_HANDLE_ID); - } - - @Test - public void testDirectUploadMultiPartFileChanges() throws Exception { - setPartsState("00"); - uploader.uploadFile( - FILE_NAME, - CONTENT_TYPE, - mockFileBlob, - mockHandler, - storageLocationId, - mockView - ); - - verify(mockJsClient) - .startMultipartUpload( - any(MultipartUploadRequest.class), - anyBoolean(), - any(AsyncCallback.class) - ); - - // manually call the method that's invoked with a successful xhr put (upload) - uploader.addCurrentPartToMultipartUpload(); - - // SWC-4262: check the file md5 once in the beginning - verify(synapseJsniUtils) - .getFileMd5(any(JavaScriptObject.class), any(MD5Callback.class)); - verify(mockJsClient, never()) - .completeMultipartUpload(anyString(), any(AsyncCallback.class)); - - doAnswer( - new Answer() { - @Override - public Void answer(InvocationOnMock invocation) throws Throwable { - final Object[] args = invocation.getArguments(); - ((MD5Callback) args[args.length - 1]).setMD5("different md5!"); - return null; - } - } - ) - .when(synapseJsniUtils) - .getFileMd5(any(JavaScriptObject.class), any(MD5Callback.class)); - - uploader.addCurrentPartToMultipartUpload(); - verify(mockJsClient, never()) - .completeMultipartUpload(anyString(), any(AsyncCallback.class)); - - // SWC-4262: check the file md5 once in the beginning, once on complete (to verify) - verify(synapseJsniUtils, times(2)) - .getFileMd5(any(JavaScriptObject.class), any(MD5Callback.class)); - - // SWC-4262: the md5 check on complete caused upload to start from the beginning (recreating the - // request) - verify(mockHandler).uploadFailed(anyString()); - } - - // failure cases - - @Test - public void testNoLongerUploading() throws Exception { - // if file input element is no longer on the page, quitely shut the upload down. - when(mockView.isAttached()).thenReturn(false); - setPartsState("10"); - uploader.uploadFile( - FILE_NAME, - CONTENT_TYPE, - mockFileBlob, - mockHandler, - storageLocationId, - mockView - ); - verify(mockJsClient, never()) - .completeMultipartUpload(anyString(), any(AsyncCallback.class)); - } - - @Test - public void testUploadCanceled() throws Exception { - // single part - setPartsState("0"); - uploader.uploadFile( - FILE_NAME, - CONTENT_TYPE, - mockFileBlob, - mockHandler, - storageLocationId, - mockView - ); - - // user cancels the upload - uploader.cancelUpload(); - verify(synapseJsniUtils) - .uploadFileChunk( - eq(MultipartUploaderImpl.BINARY_CONTENT_TYPE), - any(JavaScriptObject.class), - anyLong(), - anyLong(), - anyString(), - any(XMLHttpRequest.class), - progressCaptor.capture() - ); - ProgressCallback progressCallback = progressCaptor.getValue(); - progressCallback.updateProgress(0, 1); - // never updated the handler because the upload has been canceled (can't verify the abort(), since - // xhr is a js object). - verifyZeroInteractions(mockHandler); - } - - @Test - public void testStartMultipartUploadFailure() throws Exception { - String error = "failed"; - AsyncMockStubber - .callFailureWith(new IllegalArgumentException(error)) - .when(mockJsClient) - .startMultipartUpload( - any(MultipartUploadRequest.class), - anyBoolean(), - any(AsyncCallback.class) - ); - setPartsState("00"); - uploader.uploadFile( - FILE_NAME, - CONTENT_TYPE, - mockFileBlob, - mockHandler, - storageLocationId, - mockView - ); - - verify(mockHandler).uploadFailed(error); - } - - @Test - public void testStartMultipartUploadAttemptsExceeded() throws Exception { - setPartsState("0"); - uploader.uploadFile( - FILE_NAME, - CONTENT_TYPE, - mockFileBlob, - mockHandler, - storageLocationId, - mockView - ); - // simulate the single part fails to upload MAX_RETRY times - for (int i = 0; i < 11; i++) { - uploader.partFailure("part failed"); - } - // should have retried 11 times. plus the initial attempt, so 12 calls to start the upload... - verify(mockJsClient, times(12)) - .startMultipartUpload( - any(MultipartUploadRequest.class), - anyBoolean(), - any(AsyncCallback.class) - ); - // close dialog, and retry once more - when(mockView.isAttached()).thenReturn(false); - reset(mockJsClient); - uploader.partFailure("part failed"); - verifyZeroInteractions(mockJsClient); - } - - @Test - public void testCompleteMultipartUploadFailure() throws Exception { - String error = "failed"; - AsyncMockStubber - .callFailureWith(new IllegalArgumentException(error)) - .when(mockJsClient) - .completeMultipartUpload(anyString(), any(AsyncCallback.class)); - setPartsState("0"); - uploader.uploadFile( - FILE_NAME, - CONTENT_TYPE, - mockFileBlob, - mockHandler, - storageLocationId, - mockView - ); - - // manually call the method that's invoked with a successful xhr put (upload) - uploader.addCurrentPartToMultipartUpload(); - // should have logged the error locally, and attempted to retry the upload from the beginning - verify(synapseJsniUtils).consoleError(error); - verify(mockJsClient, times(2)) - .startMultipartUpload( - any(MultipartUploadRequest.class), - anyBoolean(), - any(AsyncCallback.class) - ); - } - - @Test - public void testByteRange() { - // test chunk sizes - ByteRange range; - - // case when total file size is less than chunk size - range = - new ByteRange( - 1, - PartUtils.MAX_PART_SIZE_BYTES - 1024, - PartUtils.MAX_PART_SIZE_BYTES - ); - assertEquals(0, range.getStart()); - assertEquals(PartUtils.MAX_PART_SIZE_BYTES - 1024 - 1, range.getEnd()); - - // case when total file size is equal to chunk size - range = - new ByteRange( - 1, - PartUtils.MAX_PART_SIZE_BYTES, - PartUtils.MAX_PART_SIZE_BYTES - ); - assertEquals(0, range.getStart()); - assertEquals(PartUtils.MAX_PART_SIZE_BYTES - 1, range.getEnd()); - - // case when total file size is greater than chunk size - range = - new ByteRange( - 1, - PartUtils.MAX_PART_SIZE_BYTES + 1024, - PartUtils.MAX_PART_SIZE_BYTES - ); - assertEquals(0, range.getStart()); - assertEquals(PartUtils.MAX_PART_SIZE_BYTES - 1, range.getEnd()); - // also verify second chunk has the expected range - range = - new ByteRange( - 2, - PartUtils.MAX_PART_SIZE_BYTES + 1024, - PartUtils.MAX_PART_SIZE_BYTES - ); - assertEquals(PartUtils.MAX_PART_SIZE_BYTES, range.getStart()); - assertEquals(PartUtils.MAX_PART_SIZE_BYTES + 1024 - 1, range.getEnd()); - - // verify byte range is valid in later chunk in large file - range = - new ByteRange( - 430, - (long) (4 * ClientProperties.GB), - PartUtils.MAX_PART_SIZE_BYTES - ); - assertTrue(range.getStart() > -1); - assertTrue(range.getEnd() > -1); - } - - @Test - public void testFixingDefaultContentType() throws RestServiceException { - String inputFilename = "file.R"; - String inputContentType = "foo/bar"; - - // if the content type coming from the browser field is set, - // then this method should never override it - assertEquals( - inputContentType, - fixDefaultContentType(inputContentType, inputFilename) - ); - - // but if the field reports a null or empty content type, then this method should fix it - inputContentType = ""; - assertEquals( - ContentTypeUtils.PLAIN_TEXT, - fixDefaultContentType(inputContentType, inputFilename) - ); - - inputContentType = null; - assertEquals( - ContentTypeUtils.PLAIN_TEXT, - fixDefaultContentType(inputContentType, inputFilename) - ); - - // should fix tab delimited files too - inputFilename = "file.tab"; - assertEquals( - WebConstants.TEXT_TAB_SEPARATED_VALUES, - fixDefaultContentType(inputContentType, inputFilename) - ); - - inputFilename = "file.tsv"; - assertEquals( - WebConstants.TEXT_TAB_SEPARATED_VALUES, - fixDefaultContentType(inputContentType, inputFilename) - ); - - // should fix CSV files as well - inputFilename = "file.CSV"; - assertEquals( - WebConstants.TEXT_COMMA_SEPARATED_VALUES, - fixDefaultContentType(inputContentType, inputFilename) - ); - - // should fix text files as well - inputFilename = "file.TXT"; - assertEquals( - ContentTypeUtils.PLAIN_TEXT, - fixDefaultContentType(inputContentType, inputFilename) - ); - - // should workflow files - assertEquals( - ContentTypeUtils.PLAIN_TEXT, - fixDefaultContentType(inputContentType, "test.wDL") - ); - assertEquals( - ContentTypeUtils.PLAIN_TEXT, - fixDefaultContentType(inputContentType, "test.Cwl") - ); - - // test default - inputContentType = null; - inputFilename = ""; - assertEquals( - ContentTypeUtils.APPLICATION_OCTET_STREAM, - fixDefaultContentType(inputContentType, inputFilename) - ); - } - - @Test - public void testCompletedPartCount() { - assertEquals(0, uploader.getCompletedPartCount("")); - assertEquals(0, uploader.getCompletedPartCount("0")); - assertEquals(0, uploader.getCompletedPartCount("0000")); - assertEquals(2, uploader.getCompletedPartCount("0101")); - assertEquals(4, uploader.getCompletedPartCount("1111")); - } -}