Skip to content

Commit

Permalink
feat: 替换StorageService.load为StorageManager.load TencentBlueKing#2065
Browse files Browse the repository at this point in the history
* feat: 修改远程Generic仓库为使用StorageManager加载缓存 TencentBlueKing#2065

* feat: 替换rpm为使用storageManager.load TencentBlueKing#2065

* feat: 替换oci为使用storageManager.load TencentBlueKing#2065

* feat: 增加loadFullArtifactInputStream方法 TencentBlueKing#2065

* feat: 替换replication的storageService为storageManager TencentBlueKing#2065

* feat: 替换maven的storageService为storageManager TencentBlueKing#2065

* feat: 替换maven的storageService为storageManager TencentBlueKing#2065

* feat: 替换pypi的storageService为storageManager TencentBlueKing#2065

* feat: 替换npm的storageService为storageManager TencentBlueKing#2065

* feat: 替换helm的storageService为storageManager TencentBlueKing#2065

* feat: 替换helm的storageService为storageManager TencentBlueKing#2065

* feat: 替换composer的storageService为storageManager TencentBlueKing#2065

* feat: 替换analyst的storageService为storageManager TencentBlueKing#2065

* feat: 替换remote-repository的storageService为storageManager TencentBlueKing#2065

* feat: 替换pypi的storageService为storageManager TencentBlueKing#2065

* feat: 合并master代码 TencentBlueKing#2065

* feat: 修复代码检查错误 TencentBlueKing#2065

* feat: 增加注释 TencentBlueKing#2065
cnlkl authored Jun 3, 2024
1 parent 20cf255 commit 8c1e5c5
Showing 18 changed files with 110 additions and 129 deletions.
Original file line number Diff line number Diff line change
@@ -39,9 +39,8 @@ import com.tencent.bkrepo.common.api.exception.SystemErrorException
import com.tencent.bkrepo.common.api.message.CommonMessageCode
import com.tencent.bkrepo.common.api.util.readJsonString
import com.tencent.bkrepo.common.api.util.toJsonString
import com.tencent.bkrepo.common.artifact.manager.StorageManager
import com.tencent.bkrepo.common.artifact.pojo.RepositoryType
import com.tencent.bkrepo.common.artifact.stream.Range
import com.tencent.bkrepo.common.storage.core.StorageService
import com.tencent.bkrepo.common.storage.credentials.StorageCredentials
import com.tencent.bkrepo.repository.api.NodeClient
import com.tencent.bkrepo.repository.api.StorageCredentialsClient
@@ -66,7 +65,7 @@ import java.security.MessageDigest
class FileLoader(
private val executorProperties: ScannerExecutorProperties,
private val nodeClient: NodeClient,
private val storageService: StorageService,
private val storageManager: StorageManager,
private val storageCredentialsClient: StorageCredentialsClient,
) {
/**
@@ -84,11 +83,11 @@ class FileLoader(

// 获取存储凭据
val storageCredentials = credentialsKey?.let { storageCredentialsClient.findByKey(it).data!! }

val node = nodeClient.getNodeDetail(projectId, repoName, fullPath).data
// 获取文件
val file = File(tempDir, fileName(taskId, fileName(), repoType))
val fos = DigestOutputStream(file.outputStream(), MessageDigest.getInstance("SHA-256"))
storageService.load(sha256, Range.full(size), storageCredentials)?.use { artifactInputStream ->
storageManager.loadFullArtifactInputStream(node, storageCredentials)?.use { artifactInputStream ->
fos.use {
if (repoType == RepositoryType.DOCKER.name) {
// 加载镜像文件
@@ -158,11 +157,11 @@ class FileLoader(
logger.info("subtask[${task.taskId}] loading layer [$filePath]")

// 加载layer
val size = getNodeSize(task.projectId, task.repoName, sha256)
storageService
.load(sha256, Range.full(size), storageCredentials)
?.use { putArchiveEntry(filePath, size, it, tos) }
?: throw SystemErrorException(CommonMessageCode.RESOURCE_NOT_FOUND, "layer not found sha256[$sha256]")
getNode(task.projectId, task.repoName, sha256)?.let { layerNode ->
storageManager
.loadFullArtifactInputStream(layerNode, storageCredentials)
?.use { putArchiveEntry(filePath, layerNode.size, it, tos) }
} ?: throw SystemErrorException(CommonMessageCode.RESOURCE_NOT_FOUND, "layer not found sha256[$sha256]")
}

private fun putArchiveEntry(name: String, size: Long, inputStream: InputStream?, tos: TarArchiveOutputStream) {
@@ -173,20 +172,21 @@ class FileLoader(
tos.closeArchiveEntry()
}

private fun getNodeSize(projectId: String, repoName: String, sha256: String): Long {
private fun getNode(projectId: String, repoName: String, sha256: String): NodeDetail? {
val nodes = nodeClient.queryWithoutCount(
NodeQueryBuilder()
.projectId(projectId)
.repoName(repoName)
.sha256(sha256)
.select(NodeDetail::size.name)
.select(NodeDetail::fullPath.name)
.page(1, 1)
.build()
)
if (nodes.isNotOk() || nodes.data!!.records.isEmpty()) {
throw SystemErrorException(CommonMessageCode.RESOURCE_NOT_FOUND, sha256)
}
return (nodes.data!!.records[0][NodeDetail::size.name] as Number).toLong()
val fullPath = nodes.data!!.records[0][NodeDetail::fullPath.name].toString()
return nodeClient.getNodeDetail(projectId, repoName, fullPath).data
}

/**
Original file line number Diff line number Diff line change
@@ -51,13 +51,14 @@ import com.tencent.bkrepo.common.api.exception.SystemErrorException
import com.tencent.bkrepo.common.api.message.CommonMessageCode.RESOURCE_NOT_FOUND
import com.tencent.bkrepo.common.api.message.CommonMessageCode.SYSTEM_ERROR
import com.tencent.bkrepo.common.artifact.hash.md5
import com.tencent.bkrepo.common.artifact.manager.StorageManager
import com.tencent.bkrepo.common.artifact.stream.ArtifactInputStream
import com.tencent.bkrepo.common.artifact.stream.Range
import com.tencent.bkrepo.common.query.model.Sort
import com.tencent.bkrepo.common.storage.core.StorageService
import com.tencent.bkrepo.repository.api.NodeClient
import com.tencent.bkrepo.repository.api.RepositoryClient
import com.tencent.bkrepo.repository.pojo.search.NodeQueryBuilder
import com.tencent.bkrepo.repository.pojo.node.NodeDetail
import com.tencent.bkrepo.repository.pojo.node.NodeInfo
import com.tencent.bkrepo.repository.pojo.node.NodeListOption
import org.slf4j.LoggerFactory
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty
@@ -71,7 +72,7 @@ class TrivyScanExecutor @Autowired constructor(
dockerClient: DockerClient,
private val scannerExecutorProperties: ScannerExecutorProperties,
private val repositoryClient: RepositoryClient,
private val storageService: StorageService,
private val storageManager: StorageManager,
private val nodeClient: NodeClient
) : CommonScanExecutor() {

@@ -153,7 +154,7 @@ class TrivyScanExecutor @Autowired constructor(
if (!dbFile.parentFile.exists()) {
dbFile.parentFile.mkdirs()
}
if (!dbFile.exists() || dbFile.md5() != newestNode["md5"]) {
if (!dbFile.exists() || dbFile.md5() != newestNode.md5) {
logger.info(buildLogMsg(task, "updating trivy.db"))
dbFile.delete()
dbFile.createNewFile()
@@ -164,40 +165,39 @@ class TrivyScanExecutor @Autowired constructor(
}
}

private fun getTrivyDBInputStream(dbNode: Map<String, Any?>, task: ScanExecutorTask): ArtifactInputStream {
private fun getTrivyDBInputStream(dbNode: NodeInfo, task: ScanExecutorTask): ArtifactInputStream {
val scanner = task.scanner
require(scanner is TrivyScanner)
// 获取trivy默认仓库信息
val repoRes = repositoryClient.getRepoDetail(scanner.vulDbConfig.projectId, scanner.vulDbConfig.repo)
if (repoRes.isNotOk()) {
logger.error(
"Get repo info failed: code[${repoRes.code}], message[${repoRes.message}]," +
" projectId[${scanner.vulDbConfig.projectId}], repoName[${scanner.vulDbConfig.repo}]"
" projectId[${scanner.vulDbConfig.projectId}], repoName[${scanner.vulDbConfig.repo}]"
)
throw SystemErrorException(SYSTEM_ERROR, repoRes.message ?: "")
}
val repositoryDetail = repoRes.data
?: throw NotFoundException(RESOURCE_NOT_FOUND, scanner.vulDbConfig.repo)

val sha256 = dbNode["sha256"] as String
val size = dbNode["size"].toString().toLong()
return storageService.load(sha256, Range.full(size), repositoryDetail.storageCredentials)
return storageManager.loadFullArtifactInputStream(NodeDetail(dbNode), repositoryDetail.storageCredentials)
?: throw SystemErrorException(SYSTEM_ERROR, "load trivy.db file failed: res: ${repoRes.message}")
}

private fun getNewestNode(projectId: String, repo: String): Map<String, Any?> {
private fun getNewestNode(projectId: String, repo: String): NodeInfo {
// 按修改时间 创建时间倒序排序,第一位则为最新的trivy.db文件
val queryModel = NodeQueryBuilder().projectId(projectId).repoName(repo)
.path("/trivy/")
.page(1, 1)
.sort(Sort.Direction.DESC, "lastModifiedDate", "createdDate")
.select("fullPath", "size", "sha256", "md5")
.build()
val nodeRes = nodeClient.queryWithoutCount(queryModel)
val option = NodeListOption(
pageSize = 1,
includeFolder = false,
includeMetadata = true,
sortProperty = listOf("lastModifiedDate", "createdDate"),
direction = listOf(Sort.Direction.DESC.name, Sort.Direction.DESC.name)
)
val nodeRes = nodeClient.listNodePage(projectId, repo, "/trivy", option)
if (nodeRes.isNotOk()) {
logger.error(
"Get node info failed: code[${nodeRes.code}], message[${nodeRes.message}]," +
" projectId[$projectId], repoName[$repo]"
" projectId[$projectId], repoName[$repo]"
)
throw SystemErrorException(SYSTEM_ERROR, nodeRes.message ?: "")
}
@@ -325,6 +325,6 @@ class TrivyScanExecutor @Autowired constructor(
*/
private const val METADATA_JSON_FILE_CONTENT =
"{\"Version\":2,\"NextUpdate\":\"2022-07-15T12:06:50.078024068Z\"," +
"\"UpdatedAt\":\"2022-07-15T06:06:50.078024668Z\",\"DownloadedAt\":\"0001-01-01T00:00:00Z\"}"
"\"UpdatedAt\":\"2022-07-15T06:06:50.078024668Z\",\"DownloadedAt\":\"0001-01-01T00:00:00Z\"}"
}
}
Original file line number Diff line number Diff line change
@@ -27,24 +27,23 @@

package com.tencent.bkrepo.analyst.service.impl

import com.tencent.bkrepo.analyst.dao.SpdxLicenseDao
import com.tencent.bkrepo.analyst.exception.LicenseNotFoundException
import com.tencent.bkrepo.analyst.model.TSpdxLicense
import com.tencent.bkrepo.analyst.pojo.license.SpdxLicenseInfo
import com.tencent.bkrepo.analyst.pojo.license.SpdxLicenseJsonInfo
import com.tencent.bkrepo.analyst.service.SpdxLicenseService
import com.tencent.bkrepo.common.api.exception.NotFoundException
import com.tencent.bkrepo.common.api.message.CommonMessageCode
import com.tencent.bkrepo.common.api.pojo.Page
import com.tencent.bkrepo.common.api.util.JsonUtils
import com.tencent.bkrepo.common.api.util.readJsonString
import com.tencent.bkrepo.common.artifact.stream.Range
import com.tencent.bkrepo.common.artifact.manager.StorageManager
import com.tencent.bkrepo.common.mongo.dao.util.Pages
import com.tencent.bkrepo.common.security.util.SecurityUtils
import com.tencent.bkrepo.common.storage.core.StorageService
import com.tencent.bkrepo.repository.api.NodeClient
import com.tencent.bkrepo.repository.api.RepositoryClient
import com.tencent.bkrepo.repository.api.StorageCredentialsClient
import com.tencent.bkrepo.analyst.dao.SpdxLicenseDao
import com.tencent.bkrepo.analyst.exception.LicenseNotFoundException
import com.tencent.bkrepo.analyst.model.TSpdxLicense
import com.tencent.bkrepo.analyst.pojo.license.SpdxLicenseInfo
import com.tencent.bkrepo.analyst.pojo.license.SpdxLicenseJsonInfo
import com.tencent.bkrepo.analyst.service.SpdxLicenseService
import org.slf4j.LoggerFactory
import org.springframework.data.domain.Sort
import org.springframework.data.mongodb.core.query.Criteria
@@ -62,7 +61,7 @@ class SpdxLicenseServiceImpl(
private val nodeClient: NodeClient,
private val repositoryClient: RepositoryClient,
private val storageCredentialsClient: StorageCredentialsClient,
private val storageService: StorageService
private val storageManager: StorageManager,
) : SpdxLicenseService {
override fun importLicense(path: String): Boolean {
val licenseJsonInfo = try {
@@ -84,7 +83,7 @@ class SpdxLicenseServiceImpl(
val storageCredentials = repo.storageCredentialsKey?.let { storageCredentialsClient.findByKey(it).data }
val node = nodeClient.getNodeDetail(projectId, repoName, fullPath).data
?: throw NotFoundException(CommonMessageCode.RESOURCE_NOT_FOUND, projectId, repoName, fullPath)
storageService.load(node.sha256!!, Range.full(node.size), storageCredentials)?.use {
storageManager.loadFullArtifactInputStream(node, storageCredentials)?.use {
importLicense(it.readJsonString<SpdxLicenseJsonInfo>())
} ?: throw NotFoundException(CommonMessageCode.RESOURCE_NOT_FOUND, projectId, repoName, fullPath)
return true
Original file line number Diff line number Diff line change
@@ -50,12 +50,11 @@ import com.tencent.bkrepo.common.api.exception.SystemErrorException
import com.tencent.bkrepo.common.api.message.CommonMessageCode.RESOURCE_NOT_FOUND
import com.tencent.bkrepo.common.api.message.CommonMessageCode.SYSTEM_ERROR
import com.tencent.bkrepo.common.api.util.StreamUtils.readText
import com.tencent.bkrepo.common.artifact.manager.StorageManager
import com.tencent.bkrepo.common.artifact.pojo.RepositoryType
import com.tencent.bkrepo.common.artifact.stream.Range
import com.tencent.bkrepo.common.query.enums.OperationType
import com.tencent.bkrepo.common.security.exception.AuthenticationException
import com.tencent.bkrepo.common.service.util.HttpContextHolder
import com.tencent.bkrepo.common.storage.core.StorageService
import com.tencent.bkrepo.oci.util.OciUtils
import com.tencent.bkrepo.repository.api.NodeClient
import com.tencent.bkrepo.repository.api.StorageCredentialsClient
@@ -75,7 +74,7 @@ class TemporaryScanTokenServiceImpl(
private val temporaryTokenClient: ServiceTemporaryTokenClient,
private val redisTemplate: RedisTemplate<String, String>,
private val scannerProperties: ScannerProperties,
private val storageService: StorageService,
private val storageManager: StorageManager,
private val storageCredentialsClient: StorageCredentialsClient,
private val nodeClient: NodeClient
) : TemporaryScanTokenService {
@@ -189,9 +188,7 @@ class TemporaryScanTokenServiceImpl(

private fun getFullPaths(subtask: SubScanTask): Map<String, FileUrl> = with(subtask) {
return if (repoType == RepositoryType.DOCKER.name) {
val storageCredentials = credentialsKey?.let { storageCredentialsClient.findByKey(it).data!! }
val manifestContent = storageService.load(sha256, Range.full(size), storageCredentials)?.readText()
?: throw ErrorCodeException(RESOURCE_NOT_FOUND, "file [$projectId:$repoName:$fullPath] not found")
val manifestContent = readManifest(projectId, repoName, sha256, credentialsKey)
val schemeVersion = OciUtils.schemeVersion(manifestContent)
val fullPaths = LinkedHashMap<String, FileUrl>()
// 将manifest下载链接加入fullPaths列表,需要保证map第一项是manifest文件
@@ -222,6 +219,26 @@ class TemporaryScanTokenServiceImpl(
}
}

private fun readManifest(projectId: String, repoName: String, sha256: String, credentialsKey: String?): String {
val storageCredentials = credentialsKey?.let { storageCredentialsClient.findByKey(it).data!! }
val nodes = nodeClient.queryWithoutCount(
NodeQueryBuilder()
.projectId(projectId)
.repoName(repoName)
.sha256(sha256)
.select(NodeDetail::fullPath.name)
.page(1, 1)
.build()
)
if (nodes.isNotOk() || nodes.data!!.records.isEmpty()) {
throw SystemErrorException(RESOURCE_NOT_FOUND, sha256)
}
val fullPath = nodes.data!!.records[0][NodeDetail::fullPath.name].toString()
return nodeClient.getNodeDetail(projectId, repoName, fullPath).data?.let { node ->
storageManager.loadFullArtifactInputStream(node, storageCredentials)?.readText()
} ?: throw ErrorCodeException(RESOURCE_NOT_FOUND, "file [$projectId:$repoName:$fullPath] not found")
}

private fun tokenKey(subtaskId: String) = "scanner:token:$subtaskId"

private fun getNodes(projectId: String, repoName: String, sha256: List<String>): List<Map<String, Any?>> {
Original file line number Diff line number Diff line change
@@ -152,6 +152,18 @@ class StorageManager(
return loadArtifactInputStream(load.nodeInfo, storageCredentials)
}

/**
* 加载[node]对应的完整ArtifactInputStream
*/
fun loadFullArtifactInputStream(node: NodeDetail?, storageCredentials: StorageCredentials?): ArtifactInputStream? {
if (node == null || node.folder) {
return null
}
val range = Range.full(node.size)
val nodeResource = nodeResourceFactory.getNodeResource(node.nodeInfo, range, storageCredentials)
return nodeResource.getArtifactInputStream()
}

companion object {
private val logger = LoggerFactory.getLogger(StorageManager::class.java)
}
Original file line number Diff line number Diff line change
@@ -119,7 +119,7 @@ abstract class RemoteRepository : AbstractArtifactRepository() {
* 加载要返回的资源
*/
open fun loadArtifactResource(cacheNode: NodeDetail, context: ArtifactDownloadContext): ArtifactResource? {
return storageService.load(cacheNode.sha256!!, Range.full(cacheNode.size), context.storageCredentials)?.run {
return storageManager.loadFullArtifactInputStream(cacheNode, context.storageCredentials)?.run {
if (logger.isDebugEnabled) {
logger.debug("Cached remote artifact[${context.artifactInfo}] is hit.")
}
Original file line number Diff line number Diff line change
@@ -66,6 +66,9 @@ interface StorageService :
/**
* 在存储实例[storageCredentials]上加载摘要为[digest]的文件
* 当文件未找到时,会尝试去默认存储实例上查找文件
*
* 注意:该方法只会从指定存储[storageCredentials]中加载文件,如果文件正在迁移中还在旧存储或者存在于其他集群该方法会加载失败,
* 此时需要考虑使用[com.tencent.bkrepo.common.artifact.manager.StorageManager]中加载方法
*/
fun load(digest: String, range: Range, storageCredentials: StorageCredentials?): ArtifactInputStream?

Original file line number Diff line number Diff line change
@@ -40,7 +40,6 @@ import com.tencent.bkrepo.common.artifact.repository.local.LocalRepository
import com.tencent.bkrepo.common.artifact.resolve.file.ArtifactFileFactory
import com.tencent.bkrepo.common.artifact.resolve.response.ArtifactChannel
import com.tencent.bkrepo.common.artifact.resolve.response.ArtifactResource
import com.tencent.bkrepo.common.artifact.stream.Range
import com.tencent.bkrepo.common.artifact.stream.closeQuietly
import com.tencent.bkrepo.common.artifact.util.PackageKeys
import com.tencent.bkrepo.common.service.util.HttpContextHolder
@@ -440,11 +439,10 @@ class ComposerLocalRepository(private val stageClient: StageClient) : LocalRepos
}

private fun nodeToJson(node: NodeDetail): String {
val inputStream = storageService.load(
node.sha256!!,
Range.full(node.size),
null
) ?: throw RuntimeException("load ${node.projectId} | ${node.repoName} | ${node.fullPath} error")
val repoId = ArtifactContextHolder.RepositoryId(node.projectId, node.repoName)
val storageCredentials = ArtifactContextHolder.getRepoDetail(repoId).storageCredentials
val inputStream = storageManager.loadFullArtifactInputStream(node, storageCredentials)
?: throw RuntimeException("load ${node.projectId} | ${node.repoName} | ${node.fullPath} error")
val stringBuilder = StringBuilder()
var line: String?
try {
Original file line number Diff line number Diff line change
@@ -62,7 +62,6 @@ import com.tencent.bkrepo.common.artifact.stream.EmptyInputStream
import com.tencent.bkrepo.common.artifact.stream.Range
import com.tencent.bkrepo.common.artifact.stream.artifactStream
import com.tencent.bkrepo.common.artifact.util.http.HttpRangeUtils.resolveContentRange
import com.tencent.bkrepo.common.artifact.util.http.HttpRangeUtils.resolveRange
import com.tencent.bkrepo.common.query.enums.OperationType
import com.tencent.bkrepo.common.query.model.QueryModel
import com.tencent.bkrepo.common.query.model.Rule
@@ -129,17 +128,7 @@ class GenericRemoteRepository(
}

override fun loadArtifactResource(cacheNode: NodeDetail, context: ArtifactDownloadContext): ArtifactResource? {
val range = HttpContextHolder.getRequestOrNull()
?.let { resolveRange(it, cacheNode.size) }
?: Range.full(cacheNode.size)

val artifactInputStream = if (shouldReturnEmptyStream(range)) {
ArtifactInputStream(EmptyInputStream.INSTANCE, range)
} else {
storageService.load(cacheNode.sha256!!, range, context.repositoryDetail.storageCredentials)
}

return artifactInputStream?.run {
return storageManager.loadArtifactInputStream(cacheNode, context.repositoryDetail.storageCredentials)?.run {
if (logger.isDebugEnabled) {
logger.debug("Cached remote artifact[${context.artifactInfo}] is hit.")
}
@@ -319,11 +308,6 @@ class GenericRemoteRepository(
headers[X_CHECKSUM_SHA256]?.let { response.setHeader(X_CHECKSUM_SHA256, it) }
}

private fun shouldReturnEmptyStream(range: Range? = null): Boolean {
val rangeToTest = range ?: HttpContextHolder.getRequestOrNull()?.let { resolveRange(it, Long.MAX_VALUE) }
return HttpContextHolder.getRequestOrNull()?.method == HEAD.name || rangeToTest?.isEmpty() == true
}

private inline fun <reified T> request(remoteConfiguration: RemoteConfiguration, request: Request): T {
val httpClient = createGenericHttpClient(remoteConfiguration)
val response = httpClient.newCall(request).execute()
Original file line number Diff line number Diff line change
@@ -40,7 +40,6 @@ import com.tencent.bkrepo.common.artifact.repository.local.LocalRepository
import com.tencent.bkrepo.common.artifact.resolve.response.ArtifactChannel
import com.tencent.bkrepo.common.artifact.resolve.response.ArtifactResource
import com.tencent.bkrepo.common.artifact.stream.ArtifactInputStream
import com.tencent.bkrepo.common.artifact.stream.Range
import com.tencent.bkrepo.common.artifact.util.FileNameParser
import com.tencent.bkrepo.common.artifact.util.PackageKeys
import com.tencent.bkrepo.common.service.util.SpringContextUtils.Companion.publishEvent
@@ -216,9 +215,7 @@ class HelmLocalRepository(
val fullPath = context.getStringAttribute(FULL_PATH)!!
val node = nodeClient.getNodeDetail(projectId, repoName, fullPath).data
if (node == null || node.folder) return null
return storageService.load(
node.sha256!!, Range.full(node.size), context.storageCredentials
)
return storageManager.loadFullArtifactInputStream(node, context.storageCredentials)
}

/**
Original file line number Diff line number Diff line change
@@ -39,7 +39,6 @@ import com.tencent.bkrepo.common.api.util.readYamlString
import com.tencent.bkrepo.common.artifact.api.ArtifactInfo
import com.tencent.bkrepo.common.artifact.constant.ARTIFACT_INFO_KEY
import com.tencent.bkrepo.common.artifact.pojo.RepositoryCategory
import com.tencent.bkrepo.common.artifact.stream.Range
import com.tencent.bkrepo.common.artifact.util.PackageKeys
import com.tencent.bkrepo.common.query.enums.OperationType
import com.tencent.bkrepo.common.query.model.PageLimit
@@ -49,7 +48,6 @@ import com.tencent.bkrepo.common.query.model.Sort
import com.tencent.bkrepo.common.security.permission.Permission
import com.tencent.bkrepo.common.service.exception.RemoteErrorCodeException
import com.tencent.bkrepo.common.service.util.HttpContextHolder
import com.tencent.bkrepo.common.storage.core.StorageService
import com.tencent.bkrepo.helm.constants.CHART_PACKAGE_FILE_EXTENSION
import com.tencent.bkrepo.helm.constants.HelmMessageCode
import com.tencent.bkrepo.helm.constants.NODE_FULL_PATH
@@ -69,6 +67,7 @@ import com.tencent.bkrepo.helm.utils.HelmUtils
import com.tencent.bkrepo.helm.utils.TimeFormatUtil
import com.tencent.bkrepo.repository.pojo.metadata.MetadataSaveRequest
import com.tencent.bkrepo.repository.pojo.metadata.packages.PackageMetadataSaveRequest
import com.tencent.bkrepo.repository.pojo.node.NodeDetail
import com.tencent.bkrepo.repository.pojo.node.NodeInfo
import com.tencent.bkrepo.repository.pojo.packages.request.PopulatedPackageVersion
import com.tencent.bkrepo.repository.pojo.repo.RepoUpdateRequest
@@ -81,9 +80,7 @@ import java.time.format.DateTimeFormatter
import java.util.SortedSet

@Service
class FixToolServiceImpl(
private val storageService: StorageService
) : FixToolService, AbstractChartService() {
class FixToolServiceImpl : FixToolService, AbstractChartService() {

override fun repairPackageCreatedDate(): List<DateTimeRepairResponse> {
val repairResponse = mutableListOf<DateTimeRepairResponse>()
@@ -181,7 +178,7 @@ class FixToolServiceImpl(
HelmMessageCode.HELM_FILE_NOT_FOUND, "index.yaml", "$projectId|$repoName"
)
}
val inputStream = storageService.load(nodeDetail.sha256!!, Range.full(nodeDetail.size), null) ?: run {
val inputStream = storageManager.loadFullArtifactInputStream(nodeDetail, null) ?: run {
logger.error("load index-cache.yaml file stream is null in repo [$projectId/$repoName]")
throw HelmFileNotFoundException(
HelmMessageCode.HELM_FILE_NOT_FOUND, "index.yaml", "$projectId|$repoName"
@@ -229,7 +226,7 @@ class FixToolServiceImpl(
}
// sleep 0.1s
Thread.sleep(100)
val inputStream = storageService.load(nodeDetail.sha256!!, Range.full(nodeDetail.size), null) ?: run {
val inputStream = storageManager.loadFullArtifactInputStream(nodeDetail, null) ?: run {
logger.error("load index-cache.yaml file stream is null in repo [$projectId/$repoName]")
throw HelmFileNotFoundException(
HelmMessageCode.HELM_FILE_NOT_FOUND, "index.yaml", "$projectId|$repoName"
@@ -320,7 +317,7 @@ class FixToolServiceImpl(
with(it) {
// sleep 0.1s
Thread.sleep(100)
val helmChartMetadata = storageService.load(sha256!!, Range.full(size), null)
val helmChartMetadata = storageManager.loadFullArtifactInputStream(NodeDetail(it), null)
?.use {
it.getArchivesContent(CHART_PACKAGE_FILE_EXTENSION).byteInputStream()
.readYamlString<HelmChartMetadata>()
Original file line number Diff line number Diff line change
@@ -48,7 +48,6 @@ import com.tencent.bkrepo.common.artifact.repository.local.LocalRepository
import com.tencent.bkrepo.common.artifact.resolve.file.ArtifactFileFactory
import com.tencent.bkrepo.common.artifact.resolve.response.ArtifactChannel
import com.tencent.bkrepo.common.artifact.resolve.response.ArtifactResource
import com.tencent.bkrepo.common.artifact.stream.Range
import com.tencent.bkrepo.common.artifact.util.PackageKeys
import com.tencent.bkrepo.common.security.util.SecurityUtils
import com.tencent.bkrepo.common.service.util.HeaderUtils
@@ -1178,11 +1177,7 @@ class MavenLocalRepository(
// 查找 `/groupId/artifactId/maven-metadata.xml`
with(artifactInfo) {
val node = nodeClient.getNodeDetail(projectId, repoName, artifactInfo.getArtifactFullPath()).data ?: return
storageService.load(
node.sha256!!,
Range.full(node.size),
storageCredentials
).use { artifactInputStream ->
storageManager.loadFullArtifactInputStream(node, storageCredentials).use { artifactInputStream ->
// 更新 `/groupId/artifactId/maven-metadata.xml`
val mavenMetadata = MetadataXpp3Reader().read(artifactInputStream)
mavenMetadata.versioning.versions.remove(version)
Original file line number Diff line number Diff line change
@@ -45,7 +45,6 @@ import com.tencent.bkrepo.common.artifact.repository.migration.MigrateDetail
import com.tencent.bkrepo.common.artifact.repository.migration.PackageMigrateDetail
import com.tencent.bkrepo.common.artifact.resolve.file.ArtifactFileFactory
import com.tencent.bkrepo.common.artifact.resolve.response.ArtifactResource
import com.tencent.bkrepo.common.artifact.stream.Range
import com.tencent.bkrepo.common.artifact.util.PackageKeys
import com.tencent.bkrepo.common.query.enums.OperationType
import com.tencent.bkrepo.npm.constants.ATTRIBUTE_OCTET_STREAM_SHA1
@@ -137,7 +136,7 @@ class NpmLocalRepository(
val fullPath = context.getStringAttribute(NPM_FILE_FULL_PATH)!!
val node = nodeClient.getNodeDetail(projectId, repoName, fullPath).data
if (node == null || node.folder) return null
return storageService.load(node.sha256!!, Range.full(node.size), context.storageCredentials)
return storageManager.loadFullArtifactInputStream(node, context.storageCredentials)
.also {
logger.info("search artifact [$fullPath] success in repo [${context.artifactInfo.getRepoIdentify()}]")
}
@@ -367,7 +366,7 @@ class NpmLocalRepository(
private fun ArtifactMigrateContext.npmPackageMetaData(fullPath: String): NpmPackageMetaData? {
val node = nodeClient.getNodeDetail(projectId, repoName, fullPath).data
return node?.let {
val inputStream = storageService.load(it.sha256!!, Range.full(it.size), storageCredentials)
val inputStream = storageManager.loadFullArtifactInputStream(it, storageCredentials)
JsonUtils.objectMapper.readValue(inputStream, NpmPackageMetaData::class.java)
}
}
Original file line number Diff line number Diff line change
@@ -463,7 +463,7 @@ class OciRegistryRemoteRepository(
* 加载要返回的资源: oci协议需要返回特定的请求头和资源类型
*/
override fun loadArtifactResource(cacheNode: NodeDetail, context: ArtifactDownloadContext): ArtifactResource? {
return storageService.load(cacheNode.sha256!!, Range.full(cacheNode.size), context.storageCredentials)?.run {
return storageManager.loadArtifactInputStream(cacheNode, context.storageCredentials)?.run {
if (logger.isDebugEnabled) {
logger.debug("Cached remote artifact[${context.artifactInfo}] is hit.")
}
Original file line number Diff line number Diff line change
@@ -44,12 +44,10 @@ import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContextHold
import com.tencent.bkrepo.common.artifact.repository.context.ArtifactQueryContext
import com.tencent.bkrepo.common.artifact.resolve.response.ArtifactChannel
import com.tencent.bkrepo.common.artifact.stream.ArtifactInputStream
import com.tencent.bkrepo.common.artifact.stream.Range
import com.tencent.bkrepo.common.artifact.util.PackageKeys
import com.tencent.bkrepo.common.query.enums.OperationType
import com.tencent.bkrepo.common.security.util.SecurityUtils
import com.tencent.bkrepo.common.service.util.HeaderUtils
import com.tencent.bkrepo.common.storage.core.StorageService
import com.tencent.bkrepo.common.storage.credentials.StorageCredentials
import com.tencent.bkrepo.common.storage.pojo.FileInfo
import com.tencent.bkrepo.oci.config.OciProperties
@@ -134,14 +132,13 @@ class OciOperationServiceImpl(
private val metadataClient: MetadataClient,
private val packageMetadataClient: PackageMetadataClient,
private val packageClient: PackageClient,
private val storageService: StorageService,
private val storageManager: StorageManager,
private val repositoryClient: RepositoryClient,
private val ociProperties: OciProperties,
private val ociReplicationRecordDao: OciReplicationRecordDao,
private val storageCredentialsClient: StorageCredentialsClient,
private val pluginManager: PluginManager
) : OciOperationService {
) : OciOperationService {

/**
* 检查package 对应的version是否存在
@@ -429,7 +426,7 @@ class OciOperationServiceImpl(
// https://github.com/docker/docker-ce/blob/master/components/engine/distribution/push_v2.go
// docker 客户端上传manifest时先按照schema2的格式上传,
// 如失败则按照schema1格式上传,但是非docker客户端不兼容schema1版本manifest
val manifest = loadManifest(nodeDetail.sha256!!, nodeDetail.size, storageCredentials)
val manifest = loadManifest(nodeDetail, storageCredentials)
?: throw OciBadRequestException(OciMessageCode.OCI_MANIFEST_SCHEMA1_NOT_SUPPORT)
// 更新manifest文件的metadata
val mediaType = if (manifest.mediaType.isNullOrEmpty()) {
@@ -461,19 +458,12 @@ class OciOperationServiceImpl(
)
}


private fun loadManifest(
sha256: String,
size: Long,
node: NodeDetail,
storageCredentials: StorageCredentials?
): ManifestSchema2? {
return try {
val manifestBytes = storageService.load(
sha256,
Range.full(size),
storageCredentials
)!!.readText()

val manifestBytes = storageManager.loadFullArtifactInputStream(node, storageCredentials)!!.readText()
OciUtils.stringToManifestV2(manifestBytes)
} catch (e: Exception) {
null
@@ -487,9 +477,7 @@ class OciOperationServiceImpl(
with(ociArtifactInfo) {
val repositoryDetail = repositoryClient.getRepoDetail(projectId, repoName).data ?: return false
val nodeDetail = nodeClient.getNodeDetail(projectId, repoName, manifestPath).data ?: return false
val manifest = loadManifest(
nodeDetail.sha256!!, nodeDetail.size, repositoryDetail.storageCredentials
) ?: return false
val manifest = loadManifest(nodeDetail, repositoryDetail.storageCredentials) ?: return false
return syncBlobInfo(
ociArtifactInfo = ociArtifactInfo,
manifest = manifest,
@@ -1040,9 +1028,7 @@ class OciOperationServiceImpl(
return true
}
val storageCredentials = repoInfo.storageCredentialsKey?.let { storageCredentialsClient.findByKey(it).data }
val manifest = loadManifest(
manifestNode.sha256!!, manifestNode.size, storageCredentials
) ?: run {
val manifest = loadManifest(manifestNode, storageCredentials) ?: run {
logger.warn("The content of manifest.json ${manifestNode.fullPath} is null, check the mediaType.")
return false
}
Original file line number Diff line number Diff line change
@@ -34,12 +34,11 @@ package com.tencent.bkrepo.pypi.artifact.repository
import com.tencent.bkrepo.common.api.exception.BadRequestException
import com.tencent.bkrepo.common.api.message.CommonMessageCode
import com.tencent.bkrepo.common.artifact.api.ArtifactFile
import com.tencent.bkrepo.common.artifact.repository.context.ArtifactSearchContext
import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContext
import com.tencent.bkrepo.common.artifact.repository.context.ArtifactQueryContext
import com.tencent.bkrepo.common.artifact.repository.context.ArtifactSearchContext
import com.tencent.bkrepo.common.artifact.repository.remote.RemoteRepository
import com.tencent.bkrepo.common.artifact.resolve.file.ArtifactFileFactory
import com.tencent.bkrepo.common.artifact.stream.Range
import com.tencent.bkrepo.common.storage.credentials.StorageCredentials
import com.tencent.bkrepo.pypi.artifact.PypiSimpleArtifactInfo
import com.tencent.bkrepo.pypi.artifact.xml.XmlConvertUtil
@@ -129,8 +128,7 @@ class PypiRemoteRepository : RemoteRepository() {
}
}.start()
val stringBuilder = StringBuilder()
storageService.load(node.sha256!!, Range.full(node.size), context.storageCredentials)?.use {
artifactInputStream ->
storageManager.loadFullArtifactInputStream(node, context.storageCredentials)?.use { artifactInputStream ->
var line: String?
val br = BufferedReader(InputStreamReader(artifactInputStream))
while (br.readLine().also { line = it } != null) {
Original file line number Diff line number Diff line change
@@ -28,9 +28,8 @@
package com.tencent.bkrepo.replication.replica.repository.internal.type

import com.tencent.bkrepo.common.artifact.exception.ArtifactNotFoundException
import com.tencent.bkrepo.common.artifact.manager.StorageManager
import com.tencent.bkrepo.common.artifact.pojo.RepositoryType
import com.tencent.bkrepo.common.artifact.stream.Range
import com.tencent.bkrepo.common.storage.core.StorageService
import com.tencent.bkrepo.replication.constant.BLOB_PATH_REFRESHED_KEY
import com.tencent.bkrepo.replication.constant.DOCKER_LAYER_FULL_PATH
import com.tencent.bkrepo.replication.constant.DOCKER_MANIFEST_JSON_FULL_PATH
@@ -51,7 +50,7 @@ import org.springframework.stereotype.Component
@Component
class DockerPackageNodeMapper(
private val nodeClient: NodeClient,
private val storageService: StorageService,
private val storageManager: StorageManager,
private val repositoryClient: RepositoryClient
) : PackageNodeMapper {

@@ -79,11 +78,7 @@ class DockerPackageNodeMapper(
nodeClient.getNodeDetail(projectId, repoName, manifestFullPath).data!!
}
if (nodeDetail.sha256.isNullOrEmpty()) throw ArtifactNotFoundException(manifestFullPath)
val inputStream = storageService.load(
nodeDetail.sha256!!,
Range.full(nodeDetail.size),
repository.storageCredentials
)!!
val inputStream = storageManager.loadFullArtifactInputStream(nodeDetail, repository.storageCredentials)!!
val manifestInfo = try {
ManifestParser.parseManifest(inputStream)
} catch (e: Exception) {
Original file line number Diff line number Diff line change
@@ -52,7 +52,6 @@ import com.tencent.bkrepo.common.artifact.repository.local.LocalRepository
import com.tencent.bkrepo.common.artifact.resolve.file.ArtifactFileFactory
import com.tencent.bkrepo.common.artifact.resolve.response.ArtifactChannel
import com.tencent.bkrepo.common.artifact.resolve.response.ArtifactResource
import com.tencent.bkrepo.common.artifact.stream.Range
import com.tencent.bkrepo.common.artifact.util.PackageKeys
import com.tencent.bkrepo.common.query.enums.OperationType
import com.tencent.bkrepo.common.query.model.PageLimit
@@ -878,7 +877,9 @@ class RpmLocalRepository(
return
}
logger.info("find primary index: ${indexNode.fullPath}")
val originXmlFile = storageService.load(indexNode.sha256!!, Range.full(indexNode.size), null)!!

val originXmlFile = storageManager
.loadFullArtifactInputStream(NodeDetail(indexNode), repoDetail.storageCredentials)!!
.use { it.unGzipInputStream() }
logger.info("originIndexMd5: ${originXmlFile.md5()}")
try {

0 comments on commit 8c1e5c5

Please sign in to comment.