diff --git a/src/backend/analysis-executor/biz-analysis-executor/src/main/kotlin/com/tencent/bkrepo/analysis/executor/component/FileLoader.kt b/src/backend/analysis-executor/biz-analysis-executor/src/main/kotlin/com/tencent/bkrepo/analysis/executor/component/FileLoader.kt index be3b06fc41..9ee80dd589 100644 --- a/src/backend/analysis-executor/biz-analysis-executor/src/main/kotlin/com/tencent/bkrepo/analysis/executor/component/FileLoader.kt +++ b/src/backend/analysis-executor/biz-analysis-executor/src/main/kotlin/com/tencent/bkrepo/analysis/executor/component/FileLoader.kt @@ -39,9 +39,8 @@ import com.tencent.bkrepo.common.api.exception.SystemErrorException import com.tencent.bkrepo.common.api.message.CommonMessageCode import com.tencent.bkrepo.common.api.util.readJsonString import com.tencent.bkrepo.common.api.util.toJsonString +import com.tencent.bkrepo.common.artifact.manager.StorageManager import com.tencent.bkrepo.common.artifact.pojo.RepositoryType -import com.tencent.bkrepo.common.artifact.stream.Range -import com.tencent.bkrepo.common.storage.core.StorageService import com.tencent.bkrepo.common.storage.credentials.StorageCredentials import com.tencent.bkrepo.repository.api.NodeClient import com.tencent.bkrepo.repository.api.StorageCredentialsClient @@ -66,7 +65,7 @@ import java.security.MessageDigest class FileLoader( private val executorProperties: ScannerExecutorProperties, private val nodeClient: NodeClient, - private val storageService: StorageService, + private val storageManager: StorageManager, private val storageCredentialsClient: StorageCredentialsClient, ) { /** @@ -84,11 +83,11 @@ class FileLoader( // 获取存储凭据 val storageCredentials = credentialsKey?.let { storageCredentialsClient.findByKey(it).data!! } - + val node = nodeClient.getNodeDetail(projectId, repoName, fullPath).data // 获取文件 val file = File(tempDir, fileName(taskId, fileName(), repoType)) val fos = DigestOutputStream(file.outputStream(), MessageDigest.getInstance("SHA-256")) - storageService.load(sha256, Range.full(size), storageCredentials)?.use { artifactInputStream -> + storageManager.loadFullArtifactInputStream(node, storageCredentials)?.use { artifactInputStream -> fos.use { if (repoType == RepositoryType.DOCKER.name) { // 加载镜像文件 @@ -158,11 +157,11 @@ class FileLoader( logger.info("subtask[${task.taskId}] loading layer [$filePath]") // 加载layer - val size = getNodeSize(task.projectId, task.repoName, sha256) - storageService - .load(sha256, Range.full(size), storageCredentials) - ?.use { putArchiveEntry(filePath, size, it, tos) } - ?: throw SystemErrorException(CommonMessageCode.RESOURCE_NOT_FOUND, "layer not found sha256[$sha256]") + getNode(task.projectId, task.repoName, sha256)?.let { layerNode -> + storageManager + .loadFullArtifactInputStream(layerNode, storageCredentials) + ?.use { putArchiveEntry(filePath, layerNode.size, it, tos) } + } ?: throw SystemErrorException(CommonMessageCode.RESOURCE_NOT_FOUND, "layer not found sha256[$sha256]") } private fun putArchiveEntry(name: String, size: Long, inputStream: InputStream?, tos: TarArchiveOutputStream) { @@ -173,20 +172,21 @@ class FileLoader( tos.closeArchiveEntry() } - private fun getNodeSize(projectId: String, repoName: String, sha256: String): Long { + private fun getNode(projectId: String, repoName: String, sha256: String): NodeDetail? { val nodes = nodeClient.queryWithoutCount( NodeQueryBuilder() .projectId(projectId) .repoName(repoName) .sha256(sha256) - .select(NodeDetail::size.name) + .select(NodeDetail::fullPath.name) .page(1, 1) .build() ) if (nodes.isNotOk() || nodes.data!!.records.isEmpty()) { throw SystemErrorException(CommonMessageCode.RESOURCE_NOT_FOUND, sha256) } - return (nodes.data!!.records[0][NodeDetail::size.name] as Number).toLong() + val fullPath = nodes.data!!.records[0][NodeDetail::fullPath.name].toString() + return nodeClient.getNodeDetail(projectId, repoName, fullPath).data } /** diff --git a/src/backend/analysis-executor/biz-analysis-executor/src/main/kotlin/com/tencent/bkrepo/analysis/executor/trivy/TrivyScanExecutor.kt b/src/backend/analysis-executor/biz-analysis-executor/src/main/kotlin/com/tencent/bkrepo/analysis/executor/trivy/TrivyScanExecutor.kt index b9faf0e126..a04fe3d2f8 100644 --- a/src/backend/analysis-executor/biz-analysis-executor/src/main/kotlin/com/tencent/bkrepo/analysis/executor/trivy/TrivyScanExecutor.kt +++ b/src/backend/analysis-executor/biz-analysis-executor/src/main/kotlin/com/tencent/bkrepo/analysis/executor/trivy/TrivyScanExecutor.kt @@ -51,13 +51,14 @@ import com.tencent.bkrepo.common.api.exception.SystemErrorException import com.tencent.bkrepo.common.api.message.CommonMessageCode.RESOURCE_NOT_FOUND import com.tencent.bkrepo.common.api.message.CommonMessageCode.SYSTEM_ERROR import com.tencent.bkrepo.common.artifact.hash.md5 +import com.tencent.bkrepo.common.artifact.manager.StorageManager import com.tencent.bkrepo.common.artifact.stream.ArtifactInputStream -import com.tencent.bkrepo.common.artifact.stream.Range import com.tencent.bkrepo.common.query.model.Sort -import com.tencent.bkrepo.common.storage.core.StorageService import com.tencent.bkrepo.repository.api.NodeClient import com.tencent.bkrepo.repository.api.RepositoryClient -import com.tencent.bkrepo.repository.pojo.search.NodeQueryBuilder +import com.tencent.bkrepo.repository.pojo.node.NodeDetail +import com.tencent.bkrepo.repository.pojo.node.NodeInfo +import com.tencent.bkrepo.repository.pojo.node.NodeListOption import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty @@ -71,7 +72,7 @@ class TrivyScanExecutor @Autowired constructor( dockerClient: DockerClient, private val scannerExecutorProperties: ScannerExecutorProperties, private val repositoryClient: RepositoryClient, - private val storageService: StorageService, + private val storageManager: StorageManager, private val nodeClient: NodeClient ) : CommonScanExecutor() { @@ -153,7 +154,7 @@ class TrivyScanExecutor @Autowired constructor( if (!dbFile.parentFile.exists()) { dbFile.parentFile.mkdirs() } - if (!dbFile.exists() || dbFile.md5() != newestNode["md5"]) { + if (!dbFile.exists() || dbFile.md5() != newestNode.md5) { logger.info(buildLogMsg(task, "updating trivy.db")) dbFile.delete() dbFile.createNewFile() @@ -164,7 +165,7 @@ class TrivyScanExecutor @Autowired constructor( } } - private fun getTrivyDBInputStream(dbNode: Map, task: ScanExecutorTask): ArtifactInputStream { + private fun getTrivyDBInputStream(dbNode: NodeInfo, task: ScanExecutorTask): ArtifactInputStream { val scanner = task.scanner require(scanner is TrivyScanner) // 获取trivy默认仓库信息 @@ -172,32 +173,31 @@ class TrivyScanExecutor @Autowired constructor( if (repoRes.isNotOk()) { logger.error( "Get repo info failed: code[${repoRes.code}], message[${repoRes.message}]," + - " projectId[${scanner.vulDbConfig.projectId}], repoName[${scanner.vulDbConfig.repo}]" + " projectId[${scanner.vulDbConfig.projectId}], repoName[${scanner.vulDbConfig.repo}]" ) throw SystemErrorException(SYSTEM_ERROR, repoRes.message ?: "") } val repositoryDetail = repoRes.data ?: throw NotFoundException(RESOURCE_NOT_FOUND, scanner.vulDbConfig.repo) - val sha256 = dbNode["sha256"] as String - val size = dbNode["size"].toString().toLong() - return storageService.load(sha256, Range.full(size), repositoryDetail.storageCredentials) + return storageManager.loadFullArtifactInputStream(NodeDetail(dbNode), repositoryDetail.storageCredentials) ?: throw SystemErrorException(SYSTEM_ERROR, "load trivy.db file failed: res: ${repoRes.message}") } - private fun getNewestNode(projectId: String, repo: String): Map { + private fun getNewestNode(projectId: String, repo: String): NodeInfo { // 按修改时间 创建时间倒序排序，第一位则为最新的trivy.db文件 - val queryModel = NodeQueryBuilder().projectId(projectId).repoName(repo) - .path("/trivy/") - .page(1, 1) - .sort(Sort.Direction.DESC, "lastModifiedDate", "createdDate") - .select("fullPath", "size", "sha256", "md5") - .build() - val nodeRes = nodeClient.queryWithoutCount(queryModel) + val option = NodeListOption( + pageSize = 1, + includeFolder = false, + includeMetadata = true, + sortProperty = listOf("lastModifiedDate", "createdDate"), + direction = listOf(Sort.Direction.DESC.name, Sort.Direction.DESC.name) + ) + val nodeRes = nodeClient.listNodePage(projectId, repo, "/trivy", option) if (nodeRes.isNotOk()) { logger.error( "Get node info failed: code[${nodeRes.code}], message[${nodeRes.message}]," + - " projectId[$projectId], repoName[$repo]" + " projectId[$projectId], repoName[$repo]" ) throw SystemErrorException(SYSTEM_ERROR, nodeRes.message ?: "") } @@ -325,6 +325,6 @@ class TrivyScanExecutor @Autowired constructor( */ private const val METADATA_JSON_FILE_CONTENT = "{\"Version\":2,\"NextUpdate\":\"2022-07-15T12:06:50.078024068Z\"," + - "\"UpdatedAt\":\"2022-07-15T06:06:50.078024668Z\",\"DownloadedAt\":\"0001-01-01T00:00:00Z\"}" + "\"UpdatedAt\":\"2022-07-15T06:06:50.078024668Z\",\"DownloadedAt\":\"0001-01-01T00:00:00Z\"}" } } diff --git a/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/service/impl/SpdxLicenseServiceImpl.kt b/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/service/impl/SpdxLicenseServiceImpl.kt index 4b2657f865..7d5df29232 100644 --- a/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/service/impl/SpdxLicenseServiceImpl.kt +++ b/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/service/impl/SpdxLicenseServiceImpl.kt @@ -27,24 +27,23 @@ package com.tencent.bkrepo.analyst.service.impl +import com.tencent.bkrepo.analyst.dao.SpdxLicenseDao +import com.tencent.bkrepo.analyst.exception.LicenseNotFoundException +import com.tencent.bkrepo.analyst.model.TSpdxLicense +import com.tencent.bkrepo.analyst.pojo.license.SpdxLicenseInfo +import com.tencent.bkrepo.analyst.pojo.license.SpdxLicenseJsonInfo +import com.tencent.bkrepo.analyst.service.SpdxLicenseService import com.tencent.bkrepo.common.api.exception.NotFoundException import com.tencent.bkrepo.common.api.message.CommonMessageCode import com.tencent.bkrepo.common.api.pojo.Page import com.tencent.bkrepo.common.api.util.JsonUtils import com.tencent.bkrepo.common.api.util.readJsonString -import com.tencent.bkrepo.common.artifact.stream.Range +import com.tencent.bkrepo.common.artifact.manager.StorageManager import com.tencent.bkrepo.common.mongo.dao.util.Pages import com.tencent.bkrepo.common.security.util.SecurityUtils -import com.tencent.bkrepo.common.storage.core.StorageService import com.tencent.bkrepo.repository.api.NodeClient import com.tencent.bkrepo.repository.api.RepositoryClient import com.tencent.bkrepo.repository.api.StorageCredentialsClient -import com.tencent.bkrepo.analyst.dao.SpdxLicenseDao -import com.tencent.bkrepo.analyst.exception.LicenseNotFoundException -import com.tencent.bkrepo.analyst.model.TSpdxLicense -import com.tencent.bkrepo.analyst.pojo.license.SpdxLicenseInfo -import com.tencent.bkrepo.analyst.pojo.license.SpdxLicenseJsonInfo -import com.tencent.bkrepo.analyst.service.SpdxLicenseService import org.slf4j.LoggerFactory import org.springframework.data.domain.Sort import org.springframework.data.mongodb.core.query.Criteria @@ -62,7 +61,7 @@ class SpdxLicenseServiceImpl( private val nodeClient: NodeClient, private val repositoryClient: RepositoryClient, private val storageCredentialsClient: StorageCredentialsClient, - private val storageService: StorageService + private val storageManager: StorageManager, ) : SpdxLicenseService { override fun importLicense(path: String): Boolean { val licenseJsonInfo = try { @@ -84,7 +83,7 @@ class SpdxLicenseServiceImpl( val storageCredentials = repo.storageCredentialsKey?.let { storageCredentialsClient.findByKey(it).data } val node = nodeClient.getNodeDetail(projectId, repoName, fullPath).data ?: throw NotFoundException(CommonMessageCode.RESOURCE_NOT_FOUND, projectId, repoName, fullPath) - storageService.load(node.sha256!!, Range.full(node.size), storageCredentials)?.use { + storageManager.loadFullArtifactInputStream(node, storageCredentials)?.use { importLicense(it.readJsonString()) } ?: throw NotFoundException(CommonMessageCode.RESOURCE_NOT_FOUND, projectId, repoName, fullPath) return true diff --git a/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/service/impl/TemporaryScanTokenServiceImpl.kt b/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/service/impl/TemporaryScanTokenServiceImpl.kt index 0b705d49a4..468e2b5877 100644 --- a/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/service/impl/TemporaryScanTokenServiceImpl.kt +++ b/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/service/impl/TemporaryScanTokenServiceImpl.kt @@ -50,12 +50,11 @@ import com.tencent.bkrepo.common.api.exception.SystemErrorException import com.tencent.bkrepo.common.api.message.CommonMessageCode.RESOURCE_NOT_FOUND import com.tencent.bkrepo.common.api.message.CommonMessageCode.SYSTEM_ERROR import com.tencent.bkrepo.common.api.util.StreamUtils.readText +import com.tencent.bkrepo.common.artifact.manager.StorageManager import com.tencent.bkrepo.common.artifact.pojo.RepositoryType -import com.tencent.bkrepo.common.artifact.stream.Range import com.tencent.bkrepo.common.query.enums.OperationType import com.tencent.bkrepo.common.security.exception.AuthenticationException import com.tencent.bkrepo.common.service.util.HttpContextHolder -import com.tencent.bkrepo.common.storage.core.StorageService import com.tencent.bkrepo.oci.util.OciUtils import com.tencent.bkrepo.repository.api.NodeClient import com.tencent.bkrepo.repository.api.StorageCredentialsClient @@ -75,7 +74,7 @@ class TemporaryScanTokenServiceImpl( private val temporaryTokenClient: ServiceTemporaryTokenClient, private val redisTemplate: RedisTemplate, private val scannerProperties: ScannerProperties, - private val storageService: StorageService, + private val storageManager: StorageManager, private val storageCredentialsClient: StorageCredentialsClient, private val nodeClient: NodeClient ) : TemporaryScanTokenService { @@ -189,9 +188,7 @@ class TemporaryScanTokenServiceImpl( private fun getFullPaths(subtask: SubScanTask): Map = with(subtask) { return if (repoType == RepositoryType.DOCKER.name) { - val storageCredentials = credentialsKey?.let { storageCredentialsClient.findByKey(it).data!! } - val manifestContent = storageService.load(sha256, Range.full(size), storageCredentials)?.readText() - ?: throw ErrorCodeException(RESOURCE_NOT_FOUND, "file [$projectId:$repoName:$fullPath] not found") + val manifestContent = readManifest(projectId, repoName, sha256, credentialsKey) val schemeVersion = OciUtils.schemeVersion(manifestContent) val fullPaths = LinkedHashMap() // 将manifest下载链接加入fullPaths列表，需要保证map第一项是manifest文件 @@ -222,6 +219,26 @@ class TemporaryScanTokenServiceImpl( } } + private fun readManifest(projectId: String, repoName: String, sha256: String, credentialsKey: String?): String { + val storageCredentials = credentialsKey?.let { storageCredentialsClient.findByKey(it).data!! } + val nodes = nodeClient.queryWithoutCount( + NodeQueryBuilder() + .projectId(projectId) + .repoName(repoName) + .sha256(sha256) + .select(NodeDetail::fullPath.name) + .page(1, 1) + .build() + ) + if (nodes.isNotOk() || nodes.data!!.records.isEmpty()) { + throw SystemErrorException(RESOURCE_NOT_FOUND, sha256) + } + val fullPath = nodes.data!!.records[0][NodeDetail::fullPath.name].toString() + return nodeClient.getNodeDetail(projectId, repoName, fullPath).data?.let { node -> + storageManager.loadFullArtifactInputStream(node, storageCredentials)?.readText() + } ?: throw ErrorCodeException(RESOURCE_NOT_FOUND, "file [$projectId:$repoName:$fullPath] not found") + } + private fun tokenKey(subtaskId: String) = "scanner:token:$subtaskId" private fun getNodes(projectId: String, repoName: String, sha256: List): List> { diff --git a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/manager/StorageManager.kt b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/manager/StorageManager.kt index 1d329a986a..3dea1c22a4 100644 --- a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/manager/StorageManager.kt +++ b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/manager/StorageManager.kt @@ -152,6 +152,18 @@ class StorageManager( return loadArtifactInputStream(load.nodeInfo, storageCredentials) } + /** + * 加载[node]对应的完整ArtifactInputStream + */ + fun loadFullArtifactInputStream(node: NodeDetail?, storageCredentials: StorageCredentials?): ArtifactInputStream? { + if (node == null || node.folder) { + return null + } + val range = Range.full(node.size) + val nodeResource = nodeResourceFactory.getNodeResource(node.nodeInfo, range, storageCredentials) + return nodeResource.getArtifactInputStream() + } + companion object { private val logger = LoggerFactory.getLogger(StorageManager::class.java) } diff --git a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/remote/RemoteRepository.kt b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/remote/RemoteRepository.kt index 2dd1795cc3..54b4844a99 100644 --- a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/remote/RemoteRepository.kt +++ b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/remote/RemoteRepository.kt @@ -119,7 +119,7 @@ abstract class RemoteRepository : AbstractArtifactRepository() { * 加载要返回的资源 */ open fun loadArtifactResource(cacheNode: NodeDetail, context: ArtifactDownloadContext): ArtifactResource? { - return storageService.load(cacheNode.sha256!!, Range.full(cacheNode.size), context.storageCredentials)?.run { + return storageManager.loadFullArtifactInputStream(cacheNode, context.storageCredentials)?.run { if (logger.isDebugEnabled) { logger.debug("Cached remote artifact[${context.artifactInfo}] is hit.") } diff --git a/src/backend/common/common-storage/storage-service/src/main/kotlin/com/tencent/bkrepo/common/storage/core/StorageService.kt b/src/backend/common/common-storage/storage-service/src/main/kotlin/com/tencent/bkrepo/common/storage/core/StorageService.kt index 75cad09373..3411b98884 100644 --- a/src/backend/common/common-storage/storage-service/src/main/kotlin/com/tencent/bkrepo/common/storage/core/StorageService.kt +++ b/src/backend/common/common-storage/storage-service/src/main/kotlin/com/tencent/bkrepo/common/storage/core/StorageService.kt @@ -66,6 +66,9 @@ interface StorageService : /** * 在存储实例[storageCredentials]上加载摘要为[digest]的文件 * 当文件未找到时，会尝试去默认存储实例上查找文件 + * + * 注意：该方法只会从指定存储[storageCredentials]中加载文件，如果文件正在迁移中还在旧存储或者存在于其他集群该方法会加载失败， + * 此时需要考虑使用[com.tencent.bkrepo.common.artifact.manager.StorageManager]中加载方法 */ fun load(digest: String, range: Range, storageCredentials: StorageCredentials?): ArtifactInputStream? diff --git a/src/backend/composer/biz-composer/src/main/kotlin/com/tencent/bkrepo/composer/artifact/repository/ComposerLocalRepository.kt b/src/backend/composer/biz-composer/src/main/kotlin/com/tencent/bkrepo/composer/artifact/repository/ComposerLocalRepository.kt index 920f0778fe..ed00f7747a 100644 --- a/src/backend/composer/biz-composer/src/main/kotlin/com/tencent/bkrepo/composer/artifact/repository/ComposerLocalRepository.kt +++ b/src/backend/composer/biz-composer/src/main/kotlin/com/tencent/bkrepo/composer/artifact/repository/ComposerLocalRepository.kt @@ -40,7 +40,6 @@ import com.tencent.bkrepo.common.artifact.repository.local.LocalRepository import com.tencent.bkrepo.common.artifact.resolve.file.ArtifactFileFactory import com.tencent.bkrepo.common.artifact.resolve.response.ArtifactChannel import com.tencent.bkrepo.common.artifact.resolve.response.ArtifactResource -import com.tencent.bkrepo.common.artifact.stream.Range import com.tencent.bkrepo.common.artifact.stream.closeQuietly import com.tencent.bkrepo.common.artifact.util.PackageKeys import com.tencent.bkrepo.common.service.util.HttpContextHolder @@ -440,11 +439,10 @@ class ComposerLocalRepository(private val stageClient: StageClient) : LocalRepos } private fun nodeToJson(node: NodeDetail): String { - val inputStream = storageService.load( - node.sha256!!, - Range.full(node.size), - null - ) ?: throw RuntimeException("load ${node.projectId} | ${node.repoName} | ${node.fullPath} error") + val repoId = ArtifactContextHolder.RepositoryId(node.projectId, node.repoName) + val storageCredentials = ArtifactContextHolder.getRepoDetail(repoId).storageCredentials + val inputStream = storageManager.loadFullArtifactInputStream(node, storageCredentials) + ?: throw RuntimeException("load ${node.projectId} | ${node.repoName} | ${node.fullPath} error") val stringBuilder = StringBuilder() var line: String? try { diff --git a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/artifact/GenericRemoteRepository.kt b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/artifact/GenericRemoteRepository.kt index 5e70ce19d8..ecfef0b193 100644 --- a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/artifact/GenericRemoteRepository.kt +++ b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/artifact/GenericRemoteRepository.kt @@ -62,7 +62,6 @@ import com.tencent.bkrepo.common.artifact.stream.EmptyInputStream import com.tencent.bkrepo.common.artifact.stream.Range import com.tencent.bkrepo.common.artifact.stream.artifactStream import com.tencent.bkrepo.common.artifact.util.http.HttpRangeUtils.resolveContentRange -import com.tencent.bkrepo.common.artifact.util.http.HttpRangeUtils.resolveRange import com.tencent.bkrepo.common.query.enums.OperationType import com.tencent.bkrepo.common.query.model.QueryModel import com.tencent.bkrepo.common.query.model.Rule @@ -129,17 +128,7 @@ class GenericRemoteRepository( } override fun loadArtifactResource(cacheNode: NodeDetail, context: ArtifactDownloadContext): ArtifactResource? { - val range = HttpContextHolder.getRequestOrNull() - ?.let { resolveRange(it, cacheNode.size) } - ?: Range.full(cacheNode.size) - - val artifactInputStream = if (shouldReturnEmptyStream(range)) { - ArtifactInputStream(EmptyInputStream.INSTANCE, range) - } else { - storageService.load(cacheNode.sha256!!, range, context.repositoryDetail.storageCredentials) - } - - return artifactInputStream?.run { + return storageManager.loadArtifactInputStream(cacheNode, context.repositoryDetail.storageCredentials)?.run { if (logger.isDebugEnabled) { logger.debug("Cached remote artifact[${context.artifactInfo}] is hit.") } @@ -319,11 +308,6 @@ class GenericRemoteRepository( headers[X_CHECKSUM_SHA256]?.let { response.setHeader(X_CHECKSUM_SHA256, it) } } - private fun shouldReturnEmptyStream(range: Range? = null): Boolean { - val rangeToTest = range ?: HttpContextHolder.getRequestOrNull()?.let { resolveRange(it, Long.MAX_VALUE) } - return HttpContextHolder.getRequestOrNull()?.method == HEAD.name || rangeToTest?.isEmpty() == true - } - private inline fun request(remoteConfiguration: RemoteConfiguration, request: Request): T { val httpClient = createGenericHttpClient(remoteConfiguration) val response = httpClient.newCall(request).execute() diff --git a/src/backend/helm/biz-helm/src/main/kotlin/com/tencent/bkrepo/helm/artifact/repository/HelmLocalRepository.kt b/src/backend/helm/biz-helm/src/main/kotlin/com/tencent/bkrepo/helm/artifact/repository/HelmLocalRepository.kt index af3cfd384e..c0de152d46 100644 --- a/src/backend/helm/biz-helm/src/main/kotlin/com/tencent/bkrepo/helm/artifact/repository/HelmLocalRepository.kt +++ b/src/backend/helm/biz-helm/src/main/kotlin/com/tencent/bkrepo/helm/artifact/repository/HelmLocalRepository.kt @@ -40,7 +40,6 @@ import com.tencent.bkrepo.common.artifact.repository.local.LocalRepository import com.tencent.bkrepo.common.artifact.resolve.response.ArtifactChannel import com.tencent.bkrepo.common.artifact.resolve.response.ArtifactResource import com.tencent.bkrepo.common.artifact.stream.ArtifactInputStream -import com.tencent.bkrepo.common.artifact.stream.Range import com.tencent.bkrepo.common.artifact.util.FileNameParser import com.tencent.bkrepo.common.artifact.util.PackageKeys import com.tencent.bkrepo.common.service.util.SpringContextUtils.Companion.publishEvent @@ -216,9 +215,7 @@ class HelmLocalRepository( val fullPath = context.getStringAttribute(FULL_PATH)!! val node = nodeClient.getNodeDetail(projectId, repoName, fullPath).data if (node == null || node.folder) return null - return storageService.load( - node.sha256!!, Range.full(node.size), context.storageCredentials - ) + return storageManager.loadFullArtifactInputStream(node, context.storageCredentials) } /** diff --git a/src/backend/helm/biz-helm/src/main/kotlin/com/tencent/bkrepo/helm/service/impl/FixToolServiceImpl.kt b/src/backend/helm/biz-helm/src/main/kotlin/com/tencent/bkrepo/helm/service/impl/FixToolServiceImpl.kt index c59d79e537..c81c52b540 100644 --- a/src/backend/helm/biz-helm/src/main/kotlin/com/tencent/bkrepo/helm/service/impl/FixToolServiceImpl.kt +++ b/src/backend/helm/biz-helm/src/main/kotlin/com/tencent/bkrepo/helm/service/impl/FixToolServiceImpl.kt @@ -39,7 +39,6 @@ import com.tencent.bkrepo.common.api.util.readYamlString import com.tencent.bkrepo.common.artifact.api.ArtifactInfo import com.tencent.bkrepo.common.artifact.constant.ARTIFACT_INFO_KEY import com.tencent.bkrepo.common.artifact.pojo.RepositoryCategory -import com.tencent.bkrepo.common.artifact.stream.Range import com.tencent.bkrepo.common.artifact.util.PackageKeys import com.tencent.bkrepo.common.query.enums.OperationType import com.tencent.bkrepo.common.query.model.PageLimit @@ -49,7 +48,6 @@ import com.tencent.bkrepo.common.query.model.Sort import com.tencent.bkrepo.common.security.permission.Permission import com.tencent.bkrepo.common.service.exception.RemoteErrorCodeException import com.tencent.bkrepo.common.service.util.HttpContextHolder -import com.tencent.bkrepo.common.storage.core.StorageService import com.tencent.bkrepo.helm.constants.CHART_PACKAGE_FILE_EXTENSION import com.tencent.bkrepo.helm.constants.HelmMessageCode import com.tencent.bkrepo.helm.constants.NODE_FULL_PATH @@ -69,6 +67,7 @@ import com.tencent.bkrepo.helm.utils.HelmUtils import com.tencent.bkrepo.helm.utils.TimeFormatUtil import com.tencent.bkrepo.repository.pojo.metadata.MetadataSaveRequest import com.tencent.bkrepo.repository.pojo.metadata.packages.PackageMetadataSaveRequest +import com.tencent.bkrepo.repository.pojo.node.NodeDetail import com.tencent.bkrepo.repository.pojo.node.NodeInfo import com.tencent.bkrepo.repository.pojo.packages.request.PopulatedPackageVersion import com.tencent.bkrepo.repository.pojo.repo.RepoUpdateRequest @@ -81,9 +80,7 @@ import java.time.format.DateTimeFormatter import java.util.SortedSet @Service -class FixToolServiceImpl( - private val storageService: StorageService -) : FixToolService, AbstractChartService() { +class FixToolServiceImpl : FixToolService, AbstractChartService() { override fun repairPackageCreatedDate(): List { val repairResponse = mutableListOf() @@ -181,7 +178,7 @@ class FixToolServiceImpl( HelmMessageCode.HELM_FILE_NOT_FOUND, "index.yaml", "$projectId|$repoName" ) } - val inputStream = storageService.load(nodeDetail.sha256!!, Range.full(nodeDetail.size), null) ?: run { + val inputStream = storageManager.loadFullArtifactInputStream(nodeDetail, null) ?: run { logger.error("load index-cache.yaml file stream is null in repo [$projectId/$repoName]") throw HelmFileNotFoundException( HelmMessageCode.HELM_FILE_NOT_FOUND, "index.yaml", "$projectId|$repoName" @@ -229,7 +226,7 @@ class FixToolServiceImpl( } // sleep 0.1s Thread.sleep(100) - val inputStream = storageService.load(nodeDetail.sha256!!, Range.full(nodeDetail.size), null) ?: run { + val inputStream = storageManager.loadFullArtifactInputStream(nodeDetail, null) ?: run { logger.error("load index-cache.yaml file stream is null in repo [$projectId/$repoName]") throw HelmFileNotFoundException( HelmMessageCode.HELM_FILE_NOT_FOUND, "index.yaml", "$projectId|$repoName" @@ -320,7 +317,7 @@ class FixToolServiceImpl( with(it) { // sleep 0.1s Thread.sleep(100) - val helmChartMetadata = storageService.load(sha256!!, Range.full(size), null) + val helmChartMetadata = storageManager.loadFullArtifactInputStream(NodeDetail(it), null) ?.use { it.getArchivesContent(CHART_PACKAGE_FILE_EXTENSION).byteInputStream() .readYamlString() diff --git a/src/backend/maven/biz-maven/src/main/kotlin/com/tencent/bkrepo/maven/artifact/repository/MavenLocalRepository.kt b/src/backend/maven/biz-maven/src/main/kotlin/com/tencent/bkrepo/maven/artifact/repository/MavenLocalRepository.kt index 737c5e680a..3f8c71f78b 100644 --- a/src/backend/maven/biz-maven/src/main/kotlin/com/tencent/bkrepo/maven/artifact/repository/MavenLocalRepository.kt +++ b/src/backend/maven/biz-maven/src/main/kotlin/com/tencent/bkrepo/maven/artifact/repository/MavenLocalRepository.kt @@ -48,7 +48,6 @@ import com.tencent.bkrepo.common.artifact.repository.local.LocalRepository import com.tencent.bkrepo.common.artifact.resolve.file.ArtifactFileFactory import com.tencent.bkrepo.common.artifact.resolve.response.ArtifactChannel import com.tencent.bkrepo.common.artifact.resolve.response.ArtifactResource -import com.tencent.bkrepo.common.artifact.stream.Range import com.tencent.bkrepo.common.artifact.util.PackageKeys import com.tencent.bkrepo.common.security.util.SecurityUtils import com.tencent.bkrepo.common.service.util.HeaderUtils @@ -1178,11 +1177,7 @@ class MavenLocalRepository( // 查找 `/groupId/artifactId/maven-metadata.xml` with(artifactInfo) { val node = nodeClient.getNodeDetail(projectId, repoName, artifactInfo.getArtifactFullPath()).data ?: return - storageService.load( - node.sha256!!, - Range.full(node.size), - storageCredentials - ).use { artifactInputStream -> + storageManager.loadFullArtifactInputStream(node, storageCredentials).use { artifactInputStream -> // 更新 `/groupId/artifactId/maven-metadata.xml` val mavenMetadata = MetadataXpp3Reader().read(artifactInputStream) mavenMetadata.versioning.versions.remove(version) diff --git a/src/backend/npm/biz-npm/src/main/kotlin/com/tencent/bkrepo/npm/artifact/repository/NpmLocalRepository.kt b/src/backend/npm/biz-npm/src/main/kotlin/com/tencent/bkrepo/npm/artifact/repository/NpmLocalRepository.kt index 18b39f8463..1485ab9b7d 100644 --- a/src/backend/npm/biz-npm/src/main/kotlin/com/tencent/bkrepo/npm/artifact/repository/NpmLocalRepository.kt +++ b/src/backend/npm/biz-npm/src/main/kotlin/com/tencent/bkrepo/npm/artifact/repository/NpmLocalRepository.kt @@ -45,7 +45,6 @@ import com.tencent.bkrepo.common.artifact.repository.migration.MigrateDetail import com.tencent.bkrepo.common.artifact.repository.migration.PackageMigrateDetail import com.tencent.bkrepo.common.artifact.resolve.file.ArtifactFileFactory import com.tencent.bkrepo.common.artifact.resolve.response.ArtifactResource -import com.tencent.bkrepo.common.artifact.stream.Range import com.tencent.bkrepo.common.artifact.util.PackageKeys import com.tencent.bkrepo.common.query.enums.OperationType import com.tencent.bkrepo.npm.constants.ATTRIBUTE_OCTET_STREAM_SHA1 @@ -137,7 +136,7 @@ class NpmLocalRepository( val fullPath = context.getStringAttribute(NPM_FILE_FULL_PATH)!! val node = nodeClient.getNodeDetail(projectId, repoName, fullPath).data if (node == null || node.folder) return null - return storageService.load(node.sha256!!, Range.full(node.size), context.storageCredentials) + return storageManager.loadFullArtifactInputStream(node, context.storageCredentials) .also { logger.info("search artifact [$fullPath] success in repo [${context.artifactInfo.getRepoIdentify()}]") } @@ -367,7 +366,7 @@ class NpmLocalRepository( private fun ArtifactMigrateContext.npmPackageMetaData(fullPath: String): NpmPackageMetaData? { val node = nodeClient.getNodeDetail(projectId, repoName, fullPath).data return node?.let { - val inputStream = storageService.load(it.sha256!!, Range.full(it.size), storageCredentials) + val inputStream = storageManager.loadFullArtifactInputStream(it, storageCredentials) JsonUtils.objectMapper.readValue(inputStream, NpmPackageMetaData::class.java) } } diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryRemoteRepository.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryRemoteRepository.kt index 670fe2be0a..703fbc565d 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryRemoteRepository.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryRemoteRepository.kt @@ -463,7 +463,7 @@ class OciRegistryRemoteRepository( * 加载要返回的资源: oci协议需要返回特定的请求头和资源类型 */ override fun loadArtifactResource(cacheNode: NodeDetail, context: ArtifactDownloadContext): ArtifactResource? { - return storageService.load(cacheNode.sha256!!, Range.full(cacheNode.size), context.storageCredentials)?.run { + return storageManager.loadArtifactInputStream(cacheNode, context.storageCredentials)?.run { if (logger.isDebugEnabled) { logger.debug("Cached remote artifact[${context.artifactInfo}] is hit.") } diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt index a90f900f17..9f742bb596 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt @@ -44,12 +44,10 @@ import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContextHold import com.tencent.bkrepo.common.artifact.repository.context.ArtifactQueryContext import com.tencent.bkrepo.common.artifact.resolve.response.ArtifactChannel import com.tencent.bkrepo.common.artifact.stream.ArtifactInputStream -import com.tencent.bkrepo.common.artifact.stream.Range import com.tencent.bkrepo.common.artifact.util.PackageKeys import com.tencent.bkrepo.common.query.enums.OperationType import com.tencent.bkrepo.common.security.util.SecurityUtils import com.tencent.bkrepo.common.service.util.HeaderUtils -import com.tencent.bkrepo.common.storage.core.StorageService import com.tencent.bkrepo.common.storage.credentials.StorageCredentials import com.tencent.bkrepo.common.storage.pojo.FileInfo import com.tencent.bkrepo.oci.config.OciProperties @@ -134,14 +132,13 @@ class OciOperationServiceImpl( private val metadataClient: MetadataClient, private val packageMetadataClient: PackageMetadataClient, private val packageClient: PackageClient, - private val storageService: StorageService, private val storageManager: StorageManager, private val repositoryClient: RepositoryClient, private val ociProperties: OciProperties, private val ociReplicationRecordDao: OciReplicationRecordDao, private val storageCredentialsClient: StorageCredentialsClient, private val pluginManager: PluginManager - ) : OciOperationService { +) : OciOperationService { /** * 检查package 对应的version是否存在 @@ -429,7 +426,7 @@ class OciOperationServiceImpl( // https://github.com/docker/docker-ce/blob/master/components/engine/distribution/push_v2.go // docker 客户端上传manifest时先按照schema2的格式上传， // 如失败则按照schema1格式上传，但是非docker客户端不兼容schema1版本manifest - val manifest = loadManifest(nodeDetail.sha256!!, nodeDetail.size, storageCredentials) + val manifest = loadManifest(nodeDetail, storageCredentials) ?: throw OciBadRequestException(OciMessageCode.OCI_MANIFEST_SCHEMA1_NOT_SUPPORT) // 更新manifest文件的metadata val mediaType = if (manifest.mediaType.isNullOrEmpty()) { @@ -461,19 +458,12 @@ class OciOperationServiceImpl( ) } - private fun loadManifest( - sha256: String, - size: Long, + node: NodeDetail, storageCredentials: StorageCredentials? ): ManifestSchema2? { return try { - val manifestBytes = storageService.load( - sha256, - Range.full(size), - storageCredentials - )!!.readText() - + val manifestBytes = storageManager.loadFullArtifactInputStream(node, storageCredentials)!!.readText() OciUtils.stringToManifestV2(manifestBytes) } catch (e: Exception) { null @@ -487,9 +477,7 @@ class OciOperationServiceImpl( with(ociArtifactInfo) { val repositoryDetail = repositoryClient.getRepoDetail(projectId, repoName).data ?: return false val nodeDetail = nodeClient.getNodeDetail(projectId, repoName, manifestPath).data ?: return false - val manifest = loadManifest( - nodeDetail.sha256!!, nodeDetail.size, repositoryDetail.storageCredentials - ) ?: return false + val manifest = loadManifest(nodeDetail, repositoryDetail.storageCredentials) ?: return false return syncBlobInfo( ociArtifactInfo = ociArtifactInfo, manifest = manifest, @@ -1040,9 +1028,7 @@ class OciOperationServiceImpl( return true } val storageCredentials = repoInfo.storageCredentialsKey?.let { storageCredentialsClient.findByKey(it).data } - val manifest = loadManifest( - manifestNode.sha256!!, manifestNode.size, storageCredentials - ) ?: run { + val manifest = loadManifest(manifestNode, storageCredentials) ?: run { logger.warn("The content of manifest.json ${manifestNode.fullPath} is null, check the mediaType.") return false } diff --git a/src/backend/pypi/biz-pypi/src/main/kotlin/com/tencent/bkrepo/pypi/artifact/repository/PypiRemoteRepository.kt b/src/backend/pypi/biz-pypi/src/main/kotlin/com/tencent/bkrepo/pypi/artifact/repository/PypiRemoteRepository.kt index 39aa10ce74..47e09261eb 100644 --- a/src/backend/pypi/biz-pypi/src/main/kotlin/com/tencent/bkrepo/pypi/artifact/repository/PypiRemoteRepository.kt +++ b/src/backend/pypi/biz-pypi/src/main/kotlin/com/tencent/bkrepo/pypi/artifact/repository/PypiRemoteRepository.kt @@ -34,12 +34,11 @@ package com.tencent.bkrepo.pypi.artifact.repository import com.tencent.bkrepo.common.api.exception.BadRequestException import com.tencent.bkrepo.common.api.message.CommonMessageCode import com.tencent.bkrepo.common.artifact.api.ArtifactFile -import com.tencent.bkrepo.common.artifact.repository.context.ArtifactSearchContext import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContext import com.tencent.bkrepo.common.artifact.repository.context.ArtifactQueryContext +import com.tencent.bkrepo.common.artifact.repository.context.ArtifactSearchContext import com.tencent.bkrepo.common.artifact.repository.remote.RemoteRepository import com.tencent.bkrepo.common.artifact.resolve.file.ArtifactFileFactory -import com.tencent.bkrepo.common.artifact.stream.Range import com.tencent.bkrepo.common.storage.credentials.StorageCredentials import com.tencent.bkrepo.pypi.artifact.PypiSimpleArtifactInfo import com.tencent.bkrepo.pypi.artifact.xml.XmlConvertUtil @@ -129,8 +128,7 @@ class PypiRemoteRepository : RemoteRepository() { } }.start() val stringBuilder = StringBuilder() - storageService.load(node.sha256!!, Range.full(node.size), context.storageCredentials)?.use { - artifactInputStream -> + storageManager.loadFullArtifactInputStream(node, context.storageCredentials)?.use { artifactInputStream -> var line: String? val br = BufferedReader(InputStreamReader(artifactInputStream)) while (br.readLine().also { line = it } != null) { diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/repository/internal/type/DockerPackageNodeMapper.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/repository/internal/type/DockerPackageNodeMapper.kt index a55bbb2ff7..2542bf9d68 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/repository/internal/type/DockerPackageNodeMapper.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/repository/internal/type/DockerPackageNodeMapper.kt @@ -28,9 +28,8 @@ package com.tencent.bkrepo.replication.replica.repository.internal.type import com.tencent.bkrepo.common.artifact.exception.ArtifactNotFoundException +import com.tencent.bkrepo.common.artifact.manager.StorageManager import com.tencent.bkrepo.common.artifact.pojo.RepositoryType -import com.tencent.bkrepo.common.artifact.stream.Range -import com.tencent.bkrepo.common.storage.core.StorageService import com.tencent.bkrepo.replication.constant.BLOB_PATH_REFRESHED_KEY import com.tencent.bkrepo.replication.constant.DOCKER_LAYER_FULL_PATH import com.tencent.bkrepo.replication.constant.DOCKER_MANIFEST_JSON_FULL_PATH @@ -51,7 +50,7 @@ import org.springframework.stereotype.Component @Component class DockerPackageNodeMapper( private val nodeClient: NodeClient, - private val storageService: StorageService, + private val storageManager: StorageManager, private val repositoryClient: RepositoryClient ) : PackageNodeMapper { @@ -79,11 +78,7 @@ class DockerPackageNodeMapper( nodeClient.getNodeDetail(projectId, repoName, manifestFullPath).data!! } if (nodeDetail.sha256.isNullOrEmpty()) throw ArtifactNotFoundException(manifestFullPath) - val inputStream = storageService.load( - nodeDetail.sha256!!, - Range.full(nodeDetail.size), - repository.storageCredentials - )!! + val inputStream = storageManager.loadFullArtifactInputStream(nodeDetail, repository.storageCredentials)!! val manifestInfo = try { ManifestParser.parseManifest(inputStream) } catch (e: Exception) { diff --git a/src/backend/rpm/biz-rpm/src/main/kotlin/com/tencent/bkrepo/rpm/artifact/repository/RpmLocalRepository.kt b/src/backend/rpm/biz-rpm/src/main/kotlin/com/tencent/bkrepo/rpm/artifact/repository/RpmLocalRepository.kt index 14ff8d3703..e9fa9dc295 100644 --- a/src/backend/rpm/biz-rpm/src/main/kotlin/com/tencent/bkrepo/rpm/artifact/repository/RpmLocalRepository.kt +++ b/src/backend/rpm/biz-rpm/src/main/kotlin/com/tencent/bkrepo/rpm/artifact/repository/RpmLocalRepository.kt @@ -52,7 +52,6 @@ import com.tencent.bkrepo.common.artifact.repository.local.LocalRepository import com.tencent.bkrepo.common.artifact.resolve.file.ArtifactFileFactory import com.tencent.bkrepo.common.artifact.resolve.response.ArtifactChannel import com.tencent.bkrepo.common.artifact.resolve.response.ArtifactResource -import com.tencent.bkrepo.common.artifact.stream.Range import com.tencent.bkrepo.common.artifact.util.PackageKeys import com.tencent.bkrepo.common.query.enums.OperationType import com.tencent.bkrepo.common.query.model.PageLimit @@ -878,7 +877,9 @@ class RpmLocalRepository( return } logger.info("find primary index: ${indexNode.fullPath}") - val originXmlFile = storageService.load(indexNode.sha256!!, Range.full(indexNode.size), null)!! + + val originXmlFile = storageManager + .loadFullArtifactInputStream(NodeDetail(indexNode), repoDetail.storageCredentials)!! .use { it.unGzipInputStream() } logger.info("originIndexMd5: ${originXmlFile.md5()}") try {