From 751762681293cfd78b8a02a65d7400a113bf24ad Mon Sep 17 00:00:00 2001 From: zacYL Date: Thu, 16 Jan 2025 10:39:30 +0800 Subject: [PATCH 1/5] =?UTF-8?q?feat:=20blob=E4=B8=8A=E4=BC=A0=E4=BC=98?= =?UTF-8?q?=E5=8C=96=20#2800?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../repository/OciRegistryLocalRepository.kt | 51 +++++++++++++++++-- 1 file changed, 48 insertions(+), 3 deletions(-) diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryLocalRepository.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryLocalRepository.kt index 650be3f3c8..607efc56c0 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryLocalRepository.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryLocalRepository.kt @@ -37,6 +37,7 @@ import com.tencent.bk.audit.annotations.AuditEntry import com.tencent.bk.audit.annotations.AuditInstanceRecord import com.tencent.bkrepo.common.api.constant.HttpStatus import com.tencent.bkrepo.common.api.constant.MediaTypes +import com.tencent.bkrepo.common.api.constant.StringPool import com.tencent.bkrepo.common.artifact.api.ArtifactInfo import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContextHolder import com.tencent.bkrepo.common.artifact.repository.context.ArtifactDownloadContext @@ -88,13 +89,16 @@ import com.tencent.bkrepo.repository.pojo.node.service.NodeDeleteRequest import com.tencent.bkrepo.repository.pojo.packages.PackageVersion import com.tencent.bkrepo.repository.pojo.packages.VersionListOption import org.slf4j.LoggerFactory +import org.springframework.data.redis.core.RedisTemplate import org.springframework.stereotype.Component import java.util.Locale +import java.util.concurrent.TimeUnit @Component class OciRegistryLocalRepository( private val ociOperationService: OciOperationService, - private val artifactConfigurerSupport: OciRegistryArtifactConfigurer + private val artifactConfigurerSupport: OciRegistryArtifactConfigurer, + private val redisTemplate: RedisTemplate? = null, ) : LocalRepository() { /** @@ -176,9 +180,19 @@ class OciRegistryLocalRepository( ) } } + // 临时存储分块文件sha256和md5, 查看https://github.com/moby/moby/blob/master/distribution/push_v2.go + // 发现上传时使用的是分块上传，但是不管blob文件大小多大，都是把blob文件当成1个分块 + // 此处记录对于sha256和md5, put请求时对比下sha256，如果一样则可以避免再次计算对应sha256和md5 + val artifactFile = context.getArtifactFile() + val key = buildRedisStr(DOCKER_REDIS_KEY_PREFIX, uuid!!) + val chunkSha256 = artifactFile.getFileSha256() + val chunkMd5 = artifactFile.getFileMd5() + val chunkSize = artifactFile.getSize().toString() + val value = buildRedisStr(chunkSha256, chunkMd5, chunkSize) + redisTemplate?.opsForValue()?.set(key, value, KEY_EXPIRED_TIME.toLong(), TimeUnit.SECONDS) val patchLen = storageService.append( appendId = uuid!!, - artifactFile = context.getArtifactFile(), + artifactFile = artifactFile, storageCredentials = context.repositoryDetail.storageCredentials ) return ResponseProperty( @@ -275,12 +289,28 @@ class OciRegistryLocalRepository( val artifactInfo = context.artifactInfo as OciBlobArtifactInfo val sha256 = artifactInfo.getDigestHex() val fileInfo = try { + val key = buildRedisStr(DOCKER_REDIS_KEY_PREFIX, artifactInfo.uuid!!) + val fileInfoStr = redisTemplate?.opsForValue()?.get(key)?.toString() + val chunkFileInfo = if (fileInfoStr.isNullOrEmpty()) { + null + } else { + val (chunkSha256, chunkMd5, chunkSize) = splitRedisStrValue(fileInfoStr) + if (chunkSha256 != null && chunkSha256 == sha256) { + FileInfo(chunkSha256, chunkMd5!!, chunkSize!!.toLong()) + } else { + null + } + } storageService.append( appendId = artifactInfo.uuid!!, artifactFile = context.getArtifactFile(), storageCredentials = context.repositoryDetail.storageCredentials ) - val fileInfo = storageService.finishAppend(artifactInfo.uuid!!, context.repositoryDetail.storageCredentials) + val fileInfo = storageService.finishAppend( + artifactInfo.uuid!!, + context.repositoryDetail.storageCredentials, + chunkFileInfo + ) if (fileInfo.sha256 != sha256) throw OciBadRequestException(OciMessageCode.OCI_DIGEST_INVALID, sha256) // 当并发情况下文件被删可能导致文件size为0 @@ -556,7 +586,22 @@ class OciRegistryLocalRepository( } } + private fun buildRedisStr(first: String, second: String, third: String? = null): String { + var result = first + StringPool.COLON + second + third?.let { result + StringPool.COLON + third } + return result + } + + private fun splitRedisStrValue(value: String): Triple { + val values = value.split(StringPool.COLON) + if (values.size < 3) return Triple(null, null, null) + return Triple(values.first(), values[1], values.last()) + } + companion object { private val logger = LoggerFactory.getLogger(OciRegistryLocalRepository::class.java) + private const val DOCKER_REDIS_KEY_PREFIX = "docker:blob_uuid" + // 6小时过期 + private const val KEY_EXPIRED_TIME = 21600 } } From 1af24125883f040b913437cf68f4bb92d1740d2a Mon Sep 17 00:00:00 2001 From: zacYL Date: Thu, 16 Jan 2025 11:35:14 +0800 Subject: [PATCH 2/5] =?UTF-8?q?feat:=20blob=E4=B8=8A=E4=BC=A0=E4=BC=98?= =?UTF-8?q?=E5=8C=96=20#2800?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../bkrepo/oci/artifact/repository/OciRegistryLocalRepository.kt | 1 + 1 file changed, 1 insertion(+) diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryLocalRepository.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryLocalRepository.kt index 607efc56c0..d16009f9f4 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryLocalRepository.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryLocalRepository.kt @@ -296,6 +296,7 @@ class OciRegistryLocalRepository( } else { val (chunkSha256, chunkMd5, chunkSize) = splitRedisStrValue(fileInfoStr) if (chunkSha256 != null && chunkSha256 == sha256) { + logger.info("blob sha256 $chunkSha256, md5 $chunkMd5, size $chunkSize") FileInfo(chunkSha256, chunkMd5!!, chunkSize!!.toLong()) } else { null From 4c79bdb91ea4def9cc25a010fd51926dde9431ff Mon Sep 17 00:00:00 2001 From: zacYL Date: Thu, 16 Jan 2025 14:38:33 +0800 Subject: [PATCH 3/5] =?UTF-8?q?feat:=20blob=E4=B8=8A=E4=BC=A0=E4=BC=98?= =?UTF-8?q?=E5=8C=96=20#2800?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../oci/artifact/repository/OciRegistryLocalRepository.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryLocalRepository.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryLocalRepository.kt index d16009f9f4..2e83bc611a 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryLocalRepository.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryLocalRepository.kt @@ -589,7 +589,7 @@ class OciRegistryLocalRepository( private fun buildRedisStr(first: String, second: String, third: String? = null): String { var result = first + StringPool.COLON + second - third?.let { result + StringPool.COLON + third } + third?.let { result = result + StringPool.COLON + third } return result } From f3f540a51d08d9932646d5577d1c5b4005f509f1 Mon Sep 17 00:00:00 2001 From: zacYL Date: Thu, 16 Jan 2025 15:03:37 +0800 Subject: [PATCH 4/5] =?UTF-8?q?feat:=20blob=E4=B8=8A=E4=BC=A0=E4=BC=98?= =?UTF-8?q?=E5=8C=96=20#2800?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../oci/artifact/repository/OciRegistryLocalRepository.kt | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryLocalRepository.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryLocalRepository.kt index 2e83bc611a..4fb6cb9dd4 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryLocalRepository.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryLocalRepository.kt @@ -184,7 +184,7 @@ class OciRegistryLocalRepository( // 发现上传时使用的是分块上传，但是不管blob文件大小多大，都是把blob文件当成1个分块 // 此处记录对于sha256和md5, put请求时对比下sha256，如果一样则可以避免再次计算对应sha256和md5 val artifactFile = context.getArtifactFile() - val key = buildRedisStr(DOCKER_REDIS_KEY_PREFIX, uuid!!) + val key = buildRedisStr(DOCKER_REDIS_KEY_PREFIX, context.artifactInfo.getRepoIdentify(), uuid!!) val chunkSha256 = artifactFile.getFileSha256() val chunkMd5 = artifactFile.getFileMd5() val chunkSize = artifactFile.getSize().toString() @@ -289,7 +289,11 @@ class OciRegistryLocalRepository( val artifactInfo = context.artifactInfo as OciBlobArtifactInfo val sha256 = artifactInfo.getDigestHex() val fileInfo = try { - val key = buildRedisStr(DOCKER_REDIS_KEY_PREFIX, artifactInfo.uuid!!) + val key = buildRedisStr( + DOCKER_REDIS_KEY_PREFIX, + context.artifactInfo.getRepoIdentify(), + artifactInfo.uuid!!, + ) val fileInfoStr = redisTemplate?.opsForValue()?.get(key)?.toString() val chunkFileInfo = if (fileInfoStr.isNullOrEmpty()) { null From fc1c03fe79f6a53a32714181d512e96d33be5786 Mon Sep 17 00:00:00 2001 From: zacYL Date: Thu, 16 Jan 2025 15:38:26 +0800 Subject: [PATCH 5/5] =?UTF-8?q?feat:=20blob=E4=B8=8A=E4=BC=A0=E4=BC=98?= =?UTF-8?q?=E5=8C=96=20#2800?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../oci/artifact/repository/OciRegistryLocalRepository.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryLocalRepository.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryLocalRepository.kt index 4fb6cb9dd4..845f87191b 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryLocalRepository.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryLocalRepository.kt @@ -189,12 +189,12 @@ class OciRegistryLocalRepository( val chunkMd5 = artifactFile.getFileMd5() val chunkSize = artifactFile.getSize().toString() val value = buildRedisStr(chunkSha256, chunkMd5, chunkSize) - redisTemplate?.opsForValue()?.set(key, value, KEY_EXPIRED_TIME.toLong(), TimeUnit.SECONDS) val patchLen = storageService.append( appendId = uuid!!, artifactFile = artifactFile, storageCredentials = context.repositoryDetail.storageCredentials ) + redisTemplate?.opsForValue()?.set(key, value, KEY_EXPIRED_TIME.toLong(), TimeUnit.SECONDS) return ResponseProperty( location = OciLocationUtils.blobUUIDLocation(uuid!!, this), status = HttpStatus.ACCEPTED,