Skip to content

Commit

Permalink
Merge pull request #1373 from blackducksoftware/dev/dterry/IDTECT-463…
Browse files Browse the repository at this point in the history 
…6-SCASS-fallback

SCASS failures due to inaccessible URLs should fallback to previous scanning approaches
  • Loading branch information
@dterrybd
dterrybd authored Feb 27, 2025
2 parents 72090bc + 3806ea5 commit 509c792
Show file tree
Hide file tree
Showing 3 changed files with 101 additions and 30 deletions.
30 changes: 18 additions & 12 deletions src/main/java/com/blackduck/integration/detect/lifecycle/run/operation/OperationRunner.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -203,6 +203,7 @@
import com.blackduck.integration.exception.IntegrationException;
import com.blackduck.integration.log.IntLogger;
import com.blackduck.integration.log.Slf4jIntLogger;
import com.blackduck.integration.rest.HttpMethod;
import com.blackduck.integration.rest.HttpUrl;
import com.blackduck.integration.rest.body.FileBodyContent;
import com.blackduck.integration.rest.response.Response;
Expand Down Expand Up @@ -536,22 +537,27 @@ public Response uploadJsonToStorageService(BlackDuckRunData blackDuckRunData, St

public UUID uploadBdioHeaderToInitiateScan(BlackDuckRunData blackDuckRunData, File bdioHeaderFile, String operationName) throws OperationException {
return auditLog.namedInternal(operationName, () -> {
BlackDuckServicesFactory blackDuckServicesFactory = blackDuckRunData.getBlackDuckServicesFactory();
BlackDuckApiClient blackDuckApiClient = blackDuckServicesFactory.getBlackDuckApiClient();
return initiatePreScassScan(blackDuckRunData, bdioHeaderFile);
});
}

String scanServicePostEndpoint = getScanServicePostEndpoint();
HttpUrl postUrl = blackDuckRunData.getBlackDuckServerConfig().getBlackDuckUrl().appendRelativeUrl(scanServicePostEndpoint);
public UUID initiatePreScassScan(BlackDuckRunData blackDuckRunData, File bdioHeaderFile)
throws IntegrationException {
BlackDuckServicesFactory blackDuckServicesFactory = blackDuckRunData.getBlackDuckServicesFactory();
BlackDuckApiClient blackDuckApiClient = blackDuckServicesFactory.getBlackDuckApiClient();

String scanServicePostContentType = getScanServicePostContentType();
BlackDuckResponseRequest buildBlackDuckResponseRequest = new BlackDuckRequestBuilder()
.postFile(bdioHeaderFile, ContentType.create(scanServicePostContentType))
.buildBlackDuckResponseRequest(postUrl);
String scanServicePostEndpoint = getScanServicePostEndpoint();
HttpUrl postUrl = blackDuckRunData.getBlackDuckServerConfig().getBlackDuckUrl().appendRelativeUrl(scanServicePostEndpoint);

HttpUrl responseUrl = blackDuckApiClient.executePostRequestAndRetrieveURL(buildBlackDuckResponseRequest);
String path = responseUrl.uri().getPath();
String scanServicePostContentType = getScanServicePostContentType();
BlackDuckResponseRequest buildBlackDuckResponseRequest = new BlackDuckRequestBuilder()
.postFile(bdioHeaderFile, ContentType.create(scanServicePostContentType))
.buildBlackDuckResponseRequest(postUrl);

return UUID.fromString(path.substring(path.lastIndexOf('/') + 1));
});
HttpUrl responseUrl = blackDuckApiClient.executePostRequestAndRetrieveURL(buildBlackDuckResponseRequest);
String path = responseUrl.uri().getPath();

return UUID.fromString(path.substring(path.lastIndexOf('/') + 1));
}

public ScanCreationResponse uploadBdioHeaderToInitiateScassScan(BlackDuckRunData blackDuckRunData, File bdioHeaderFile, String operationName, Gson gson, String computedMd5) throws OperationException {
Expand Down
83 changes: 71 additions & 12 deletions src/main/java/com/blackduck/integration/detect/lifecycle/run/step/CommonScanStepRunner.java
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,15 @@
package com.blackduck.integration.detect.lifecycle.run.step;

import java.io.File;
import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.Optional;
import java.util.UUID;

import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.blackduck.integration.blackduck.version.BlackDuckVersion;
import com.blackduck.integration.detect.lifecycle.OperationException;
Expand All @@ -13,6 +18,7 @@
import com.blackduck.integration.detect.lifecycle.run.data.ScanCreationResponse;
import com.blackduck.integration.detect.lifecycle.run.operation.OperationRunner;
import com.blackduck.integration.detect.lifecycle.run.operation.blackduck.ScassScanInitiationResult;
import com.blackduck.integration.detect.util.bdio.protobuf.DetectProtobufBdioHeaderUtil;
import com.blackduck.integration.detect.workflow.codelocation.CodeLocationNameManager;
import com.blackduck.integration.exception.IntegrationException;
import com.blackduck.integration.util.NameVersion;
Expand All @@ -27,6 +33,8 @@
public class CommonScanStepRunner {
private static final BlackDuckVersion MIN_SCASS_SCAN_VERSION = new BlackDuckVersion(2025, 1, 1);

private final Logger logger = LoggerFactory.getLogger(this.getClass());

// Supported scan types
public static final String BINARY = "BINARY";
public static final String CONTAINER = "CONTAINER";
Expand Down Expand Up @@ -56,31 +64,82 @@ public CommonScanResult performCommonScan(NameVersion projectNameVersion, BlackD

return operationRunner.getAuditLog().namedPublic(operationName, () -> {
UUID scanId = performCommonUpload(projectNameVersion, blackDuckRunData, scanFile, operationRunner, scanType,
initResult);
initResult, codeLocationName);

return new CommonScanResult(scanId, codeLocationName);
});
}

public UUID performCommonUpload(NameVersion projectNameVersion, BlackDuckRunData blackDuckRunData,
Optional<File> scanFile, OperationRunner operationRunner, String scanType,
ScassScanInitiationResult initResult) throws IntegrationException, OperationException {
ScassScanInitiationResult initResult, String codeLocationName) throws IntegrationException, OperationException, IOException {
ScanCreationResponse scanCreationResponse = initResult.getScanCreationResponse();

if (StringUtils.isNotEmpty(scanCreationResponse.getUploadUrl())) {
// This is a SCASS capable server server and SCASS is enabled.
ScassScanStepRunner scassScanStepRunner = createScassScanStepRunner(blackDuckRunData);
scassScanStepRunner.runScassScan(Optional.of(initResult.getFileToUpload()), scanCreationResponse);
} else {
// This is a SCASS capable server server but SCASS is not enabled.
BdbaScanStepRunner bdbaScanStepRunner = createBdbaScanStepRunner(operationRunner);

bdbaScanStepRunner.runBdbaScan(projectNameVersion, blackDuckRunData, scanFile, scanCreationResponse.getScanId(), scanType);
String uploadUrl = scanCreationResponse.getUploadUrl();
UUID scanId = UUID.fromString(scanCreationResponse.getScanId());

if (StringUtils.isNotEmpty(uploadUrl)) {
if (isAccessible(uploadUrl)) {
// This is a SCASS capable server server, SCASS is enabled, and we can access the upload URL.
ScassScanStepRunner scassScanStepRunner = createScassScanStepRunner(blackDuckRunData);
scassScanStepRunner.runScassScan(Optional.of(initResult.getFileToUpload()), scanCreationResponse);

return scanId;
} else {
// If we can't access the SCASS uplaod URL, we create a new scanId so we can try the BDBA flow.
// Note: as of 2025.1.1 there is no endpoint to cancel a SCASS scan.
scanId = createFallbackScanId(operationRunner, scanType, projectNameVersion, codeLocationName, scanFile.get().length(), blackDuckRunData);
}
}

// This is a SCASS capable server server but SCASS is not enabled or the GCP URL is inaccessible.
BdbaScanStepRunner bdbaScanStepRunner = createBdbaScanStepRunner(operationRunner);

return UUID.fromString(scanCreationResponse.getScanId());
bdbaScanStepRunner.runBdbaScan(projectNameVersion, blackDuckRunData, scanFile, scanId.toString(), scanType);

return scanId;
}

private UUID createFallbackScanId(OperationRunner operationRunner, String scanType, NameVersion projectNameVersion, String codeLocationName, long fileLength, BlackDuckRunData blackDuckRunData) throws IntegrationException {
String projectGroupName = operationRunner.calculateProjectGroupOptions().getProjectGroup();

DetectProtobufBdioHeaderUtil detectProtobufBdioHeaderUtil = new DetectProtobufBdioHeaderUtil(
UUID.randomUUID().toString(),
scanType,
projectNameVersion,
projectGroupName,
codeLocationName,
fileLength);

File bdioHeaderFile;

try {
bdioHeaderFile = detectProtobufBdioHeaderUtil.createProtobufBdioHeader(getOutputDirectory(operationRunner, scanType));
} catch (IOException e) {
throw new IntegrationException("Unable to create new scan. Ensure the file and output directory are accessible.");
}

return operationRunner.initiatePreScassScan(blackDuckRunData, bdioHeaderFile);
}

public boolean isAccessible(String uploadUrl) {
try {
URL url = new URL(uploadUrl);
HttpURLConnection connection = (HttpURLConnection) url.openConnection();
connection.setRequestMethod("HEAD");
int responseCode = connection.getResponseCode();
if (responseCode == HttpURLConnection.HTTP_OK) {
return true;
} else {
logger.debug("Attempted to access SCASS URL but failed. Response code: {}", responseCode);
return false;
}
} catch (IOException e) {
logger.debug("Error checking SCASS URL: " + e.getMessage());
return false;
}
}

private File getOutputDirectory(OperationRunner operationRunner, String scanType) throws IntegrationException {
switch (scanType) {
case BINARY:
Expand Down
18 changes: 12 additions & 6 deletions ...st/java/com/blackduck/integration/detect/lifecycle/run/step/CommonScanStepRunnerTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
import static org.mockito.Mockito.when;

import java.io.File;
import java.io.IOException;
import java.util.Optional;
import java.util.UUID;

Expand All @@ -30,6 +31,8 @@
import com.google.gson.Gson;

public class CommonScanStepRunnerTest {
private static final String CODE_LOCATION_NAME = "codeLocationName";

private static final String VERSION = "version";

private static final String PROJECT_NAME = "projectName";
Expand Down Expand Up @@ -87,18 +90,20 @@ public void testPerformBlackduckInteractionsBinaryScass() throws Exception {
CommonScanStepRunner commonScanStepRunner = spy(new CommonScanStepRunner());

when(scanCreationResponse.getUploadUrl()).thenReturn(uploadUrl);
when(commonScanStepRunner.isAccessible(uploadUrl)).thenReturn(true);

doNothing().when(scassScanStepRunner).runScassScan(any(), any());

doReturn(scassScanStepRunner).when(commonScanStepRunner).createScassScanStepRunner(nullable(BlackDuckRunData.class));

commonScanStepRunner.performCommonUpload(projectNameVersion,
blackDuckRunData, Optional.of(binaryScanFile), operationRunner, CommonScanStepRunner.BINARY, initResult);
blackDuckRunData, Optional.of(binaryScanFile), operationRunner, CommonScanStepRunner.BINARY, initResult, CODE_LOCATION_NAME);

verify(scassScanStepRunner).runScassScan(Optional.of(initResult.getFileToUpload()), scanCreationResponse);
}

@Test
public void testPerformBlackduckInteractionsBinaryBdba() throws OperationException, IntegrationException {
public void testPerformBlackduckInteractionsBinaryBdba() throws OperationException, IntegrationException, IOException {
NameVersion projectNameVersion = new NameVersion(PROJECT_NAME, VERSION);
File binaryScanFile = mock(File.class);

Expand All @@ -110,7 +115,7 @@ public void testPerformBlackduckInteractionsBinaryBdba() throws OperationExcepti
doReturn(bdbaScanStepRunner).when(commonScanStepRunner).createBdbaScanStepRunner(nullable(OperationRunner.class));

commonScanStepRunner.performCommonUpload(projectNameVersion,
blackDuckRunData, Optional.of(binaryScanFile), operationRunner, CommonScanStepRunner.BINARY, initResult);
blackDuckRunData, Optional.of(binaryScanFile), operationRunner, CommonScanStepRunner.BINARY, initResult, CODE_LOCATION_NAME);

verify(bdbaScanStepRunner).runBdbaScan(projectNameVersion, blackDuckRunData, Optional.of(binaryScanFile), scanCreationResponse.getScanId(), CommonScanStepRunner.BINARY);
}
Expand All @@ -124,18 +129,19 @@ public void testPerformBlackduckInteractionsContainerScass() throws Exception {
CommonScanStepRunner commonScanStepRunner = spy(new CommonScanStepRunner());

when(scanCreationResponse.getUploadUrl()).thenReturn(uploadUrl);
when(commonScanStepRunner.isAccessible(uploadUrl)).thenReturn(true);
doNothing().when(scassScanStepRunner).runScassScan(any(), any());

doReturn(scassScanStepRunner).when(commonScanStepRunner).createScassScanStepRunner(nullable(BlackDuckRunData.class));

commonScanStepRunner.performCommonUpload(projectNameVersion,
blackDuckRunData, Optional.of(containerScanFile), operationRunner, CommonScanStepRunner.CONTAINER, initResult);
blackDuckRunData, Optional.of(containerScanFile), operationRunner, CommonScanStepRunner.CONTAINER, initResult, CODE_LOCATION_NAME);

verify(scassScanStepRunner).runScassScan(Optional.of(initResult.getFileToUpload()), scanCreationResponse);
}

@Test
public void testPerformBlackduckInteractionsContainerBdba() throws OperationException, IntegrationException {
public void testPerformBlackduckInteractionsContainerBdba() throws OperationException, IntegrationException, IOException {
NameVersion projectNameVersion = new NameVersion(PROJECT_NAME, VERSION);
File containerScanFile = mock(File.class);

Expand All @@ -147,7 +153,7 @@ public void testPerformBlackduckInteractionsContainerBdba() throws OperationExce
doReturn(bdbaScanStepRunner).when(commonScanStepRunner).createBdbaScanStepRunner(nullable(OperationRunner.class));

commonScanStepRunner.performCommonUpload(projectNameVersion,
blackDuckRunData, Optional.of(containerScanFile), operationRunner, CommonScanStepRunner.CONTAINER, initResult);
blackDuckRunData, Optional.of(containerScanFile), operationRunner, CommonScanStepRunner.CONTAINER, initResult, CODE_LOCATION_NAME);

verify(bdbaScanStepRunner).runBdbaScan(projectNameVersion, blackDuckRunData, Optional.of(containerScanFile), scanCreationResponse.getScanId(), CommonScanStepRunner.CONTAINER);
}
Expand Down

0 comments on commit 509c792

Please sign in to comment.