Skip to content

Commit

Permalink
feat(analyzer): Use all available credentials for repository download
Browse files Browse the repository at this point in the history 
Instead of using a single `InfrastructureService` that matches the
repository, obtain all known services to generate the `.netrc` file
for checking out the repository. This is needed if the repository has
submodules that require additional credentials.

Signed-off-by: Oliver Heger <[email protected]>
  • Loading branch information
@oheger-bosch
oheger-bosch committed Jan 7, 2025
1 parent 1285cc3 commit 274721f
Show file tree
Hide file tree
Showing 5 changed files with 136 additions and 152 deletions.
15 changes: 7 additions & 8 deletions workers/analyzer/src/main/kotlin/analyzer/AnalyzerWorker.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ package org.eclipse.apoapsis.ortserver.workers.analyzer

import org.eclipse.apoapsis.ortserver.dao.dbQuery
import org.eclipse.apoapsis.ortserver.model.AnalyzerJob
import org.eclipse.apoapsis.ortserver.model.InfrastructureService
import org.eclipse.apoapsis.ortserver.model.JobStatus
import org.eclipse.apoapsis.ortserver.workers.common.JobIgnoredException
import org.eclipse.apoapsis.ortserver.workers.common.OrtRunService
Expand Down Expand Up @@ -61,17 +62,15 @@ internal class AnalyzerWorker(
val context = contextFactory.createContext(job.ortRunId)
val envConfigFromJob = job.configuration.environmentConfig

val repositoryService = envConfigFromJob?.let {
environmentService.findInfrastructureServiceForRepository(context, it)
} ?: environmentService.findInfrastructureServiceForRepository(context)
repositoryService?.also { serviceForRepo ->
val repositoryServices = environmentService.findInfrastructureServicesForRepository(context, envConfigFromJob)
if (repositoryServices.isNotEmpty()) {
logger.info(
"Generating a .netrc file with credentials from infrastructure service '{}' to download the " +
"Generating a .netrc file with credentials from infrastructure services '{}' to download the " +
"repository.",
serviceForRepo
repositoryServices.map(InfrastructureService::name)
)

environmentService.generateNetRcFile(context, listOf(serviceForRepo))
environmentService.generateNetRcFile(context, repositoryServices)
}

val sourcesDir = downloader.downloadRepository(
Expand All @@ -85,7 +84,7 @@ internal class AnalyzerWorker(
context,
sourcesDir,
envConfigFromJob,
repositoryService
repositoryServices
)
val ortResult = runner.run(context, sourcesDir, job.configuration, resolvedEnvConfig)

Expand Down
2 changes: 1 addition & 1 deletion workers/analyzer/src/test/kotlin/AnalyzerEndpointTest.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -366,7 +366,7 @@ class AnalyzerEndpointTest : KoinTest, StringSpec() {
val environmentService by inject<EnvironmentService>()

withSystemProperties(properties, mode = OverrideMode.SetOrOverride) {
environmentService.setUpEnvironment(context, repositoryFolder, null, null)
environmentService.setUpEnvironment(context, repositoryFolder, null, emptyList())
}

block(homeFolder)
Expand Down
41 changes: 20 additions & 21 deletions workers/analyzer/src/test/kotlin/AnalyzerWorkerTest.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -139,12 +139,12 @@ class AnalyzerWorkerTest : StringSpec({
every { createContext(analyzerJob.ortRunId) } returns context
}

val infrastructureService = mockk<InfrastructureService>()
val infrastructureServices = listOf<InfrastructureService>(mockk(relaxed = true), mockk(relaxed = true))
val envService = mockk<EnvironmentService> {
every { findInfrastructureServiceForRepository(context) } returns infrastructureService
coEvery { generateNetRcFile(context, listOf(infrastructureService)) } just runs
every { findInfrastructureServicesForRepository(context, null) } returns infrastructureServices
coEvery { generateNetRcFile(context, infrastructureServices) } just runs
coEvery {
setUpEnvironment(context, projectDir, null, infrastructureService)
setUpEnvironment(context, projectDir, null, infrastructureServices)
} returns ResolvedEnvironmentConfig()
}

Expand All @@ -168,9 +168,9 @@ class AnalyzerWorkerTest : StringSpec({
}

coVerifyOrder {
envService.generateNetRcFile(context, listOf(infrastructureService))
envService.generateNetRcFile(context, infrastructureServices)
downloader.downloadRepository(repository.url, ortRun.revision)
envService.setUpEnvironment(context, projectDir, null, infrastructureService)
envService.setUpEnvironment(context, projectDir, null, infrastructureServices)
}
}
}
Expand Down Expand Up @@ -201,8 +201,8 @@ class AnalyzerWorkerTest : StringSpec({
}

val envService = mockk<EnvironmentService> {
every { findInfrastructureServiceForRepository(context) } returns null
coEvery { setUpEnvironment(context, projectDir, null, null) } returns ResolvedEnvironmentConfig()
every { findInfrastructureServicesForRepository(context, null) } returns emptyList()
coEvery { setUpEnvironment(context, projectDir, null, emptyList()) } returns ResolvedEnvironmentConfig()
}

val worker = AnalyzerWorker(
Expand All @@ -228,7 +228,7 @@ class AnalyzerWorkerTest : StringSpec({
}

coVerify {
envService.setUpEnvironment(context, projectDir, null, null)
envService.setUpEnvironment(context, projectDir, null, emptyList())
}
}
}
Expand Down Expand Up @@ -261,9 +261,10 @@ class AnalyzerWorkerTest : StringSpec({
}

val envService = mockk<EnvironmentService> {
every { findInfrastructureServiceForRepository(context) } returns null
every { findInfrastructureServiceForRepository(context, envConfig) } returns null
coEvery { setUpEnvironment(context, projectDir, envConfig, null) } returns ResolvedEnvironmentConfig()
every { findInfrastructureServicesForRepository(context, envConfig) } returns emptyList()
coEvery {
setUpEnvironment(context, projectDir, envConfig, emptyList())
} returns ResolvedEnvironmentConfig()
}

val worker = AnalyzerWorker(
Expand All @@ -285,7 +286,7 @@ class AnalyzerWorkerTest : StringSpec({
}

coVerify {
envService.setUpEnvironment(context, projectDir, envConfig, null)
envService.setUpEnvironment(context, projectDir, envConfig, emptyList())
}
}
}
Expand Down Expand Up @@ -316,9 +317,8 @@ class AnalyzerWorkerTest : StringSpec({

val resolvedEnvConfig = mockk<ResolvedEnvironmentConfig>()
val envService = mockk<EnvironmentService> {
every { findInfrastructureServiceForRepository(context) } returns null
every { findInfrastructureServiceForRepository(context, envConfig) } returns null
coEvery { setUpEnvironment(context, projectDir, envConfig, null) } returns resolvedEnvConfig
every { findInfrastructureServicesForRepository(context, envConfig) } returns emptyList()
coEvery { setUpEnvironment(context, projectDir, envConfig, emptyList()) } returns resolvedEnvConfig
}

val testException = IllegalStateException("AnalyzerRunner test exception")
Expand Down Expand Up @@ -365,9 +365,8 @@ class AnalyzerWorkerTest : StringSpec({

val resolvedEnvConfig = mockk<ResolvedEnvironmentConfig>()
val envService = mockk<EnvironmentService> {
every { findInfrastructureServiceForRepository(context) } returns null
every { findInfrastructureServiceForRepository(context, any()) } returns null
coEvery { setUpEnvironment(context, projectDir, null, null) } returns resolvedEnvConfig
every { findInfrastructureServicesForRepository(context, any()) } returns emptyList()
coEvery { setUpEnvironment(context, projectDir, null, emptyList()) } returns resolvedEnvConfig
}

val testException = IllegalStateException("AnalyzerRunner test exception")
Expand Down Expand Up @@ -463,8 +462,8 @@ class AnalyzerWorkerTest : StringSpec({
}

val envService = mockk<EnvironmentService> {
every { findInfrastructureServiceForRepository(context) } returns null
coEvery { setUpEnvironment(context, projectDir, null, null) } returns ResolvedEnvironmentConfig()
every { findInfrastructureServicesForRepository(context, null) } returns emptyList()
coEvery { setUpEnvironment(context, projectDir, null, emptyList()) } returns ResolvedEnvironmentConfig()
}

val runnerMock = spyk(AnalyzerRunner(ConfigFactory.empty())) {
Expand Down
65 changes: 34 additions & 31 deletions workers/common/src/main/kotlin/common/env/EnvironmentService.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,66 +59,61 @@ class EnvironmentService(
/** The helper object for loading the environment configuration file. */
private val configLoader: EnvironmentConfigLoader
) {
/**
* Try to find the [InfrastructureService] that matches the current repository stored in the given [context]. This
* function is used to find the credentials for downloading the repository. The match is done based on the URL
* prefix. In case there are multiple matches, the longest match wins.
*/
fun findInfrastructureServiceForRepository(context: WorkerContext): InfrastructureService? =
with(context.hierarchy) {
infrastructureServiceRepository.listForRepositoryUrl(repository.url, organization.id, product.id)
.filter { repository.url.startsWith(it.url) }
.maxByOrNull { it.url.length }
}

/**
* Try to find the [InfrastructureService] that matches the current repository stored in the given [context],
* using the provided [config]. This function is used to find the credentials for downloading the repository.
* The match is done based on the URL prefix, and only services that support the [CredentialsType.NETRC_FILE]
* credential type are considered. In case of multiple matches, the longest match wins.
* Return a list of all [InfrastructureService]s that may be relevant for checking out the current repository
* defined by the given [context]. If specified, merge this list with the services from the given [config], so
* that the services from the [config] take priority over the ones from the database. The resulting services can
* then be added to the _.netrc_ file to make sure that all credentials are available, even if the repository
* contains submodules or other dependencies.
*/
fun findInfrastructureServiceForRepository(
fun findInfrastructureServicesForRepository(
context: WorkerContext,
config: EnvironmentConfig
): InfrastructureService? =
with(context.hierarchy) {
configLoader.resolve(config, context.hierarchy).infrastructureServices
.filter { CredentialsType.NETRC_FILE in it.credentialsTypes }
.filter { repository.url.startsWith(it.url) }
.maxByOrNull { it.url.length }
}
config: EnvironmentConfig?
): List<InfrastructureService> {
val hierarchyServices = infrastructureServiceRepository.listForHierarchy(
context.hierarchy.organization.id,
context.hierarchy.product.id
).netrcServices()

val configServices = config?.let {
configLoader.resolve(it, context.hierarchy).infrastructureServices
}.orEmpty().netrcServices()

return (hierarchyServices + configServices).values.toList()
}

/**
* Set up the analysis environment for the current repository defined by the given [context] that has been
* checked out to the given [repositoryFolder]. The credentials of this repository - if any - are defined by the
* given [repositoryService]. If an optional [config] is provided, it will be merged with the parsed configuration.
* given [repositoryServices]. If an optional [config] is provided, it will be merged with the parsed configuration.
* In case of overlapping entries, the provided [config] will take priority over the parsed configuration.
*/
suspend fun setUpEnvironment(
context: WorkerContext,
repositoryFolder: File,
config: EnvironmentConfig?,
repositoryService: InfrastructureService?
repositoryServices: List<InfrastructureService>
): ResolvedEnvironmentConfig {
val environmentConfigPath = context.ortRun.environmentConfigPath
val mergedConfig = configLoader.resolveAndParse(repositoryFolder, environmentConfigPath).merge(config)
val resolvedConfig = configLoader.resolve(mergedConfig, context.hierarchy)

return setUpEnvironmentForConfig(context, resolvedConfig, repositoryService)
return setUpEnvironmentForConfig(context, resolvedConfig, repositoryServices)
}

/**
* Set up the analysis environment based on the given resolved [config]. Use the given [context]. If the repository
* has credentials, as defined by the given optional [repositoryService], take them into account as well.
* Set up the analysis environment based on the given resolved [config]. Use the given [context]. Also take the
* given [repositoryServices] into account that have been used to download the repository.
*/
private suspend fun setUpEnvironmentForConfig(
context: WorkerContext,
config: ResolvedEnvironmentConfig,
repositoryService: InfrastructureService?
repositoryServices: List<InfrastructureService>
): ResolvedEnvironmentConfig {
val environmentServices = config.environmentDefinitions.map { it.service }
val infraServices = config.infrastructureServices.toMutableSet()
repositoryService?.let { infraServices += it }
infraServices += repositoryServices

val unreferencedServices = infraServices.filterNot { it in environmentServices }
val allEnvironmentDefinitions = config.environmentDefinitions +
Expand Down Expand Up @@ -211,3 +206,11 @@ internal fun EnvironmentConfig.merge(other: EnvironmentConfig?): EnvironmentConf

return EnvironmentConfig(mergedInfrastructureService, mergedEnvironmentDefinitions, mergedEnvironmentVariables)
}

/**
* Filter this collection of [InfrastructureService]s for services to be added to the _.netrc_ file and return a
* [Map] with the services' URLs as keys that can be used to merge services from different sources.
*/
private fun Collection<InfrastructureService>.netrcServices(): Map<String, InfrastructureService> =
filter { CredentialsType.NETRC_FILE in it.credentialsTypes }
.associateBy(InfrastructureService::url)
Loading

0 comments on commit 274721f

Please sign in to comment.