Skip to content

feat(localRegistry): expose pod resources #2964

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 7 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -38,3 +38,5 @@ profile.out

# Ignore DevSpace cache and log folder
.devspace/

.env
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@

<details className="config-field" data-expandable="false" open>
<summary>

### `annotations` <span className="config-field-required" data-required="false">required</span> <span className="config-field-type">integer</span> <span className="config-field-default"></span> <span className="config-field-enum"></span> {#localRegistry-port}

Additional annotations used in the registry and buildKit containers. Default is
an empty dictionary.

</summary>



</details>
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,9 @@ import PartialImage from "./localRegistry/image.mdx"
import PartialBuildKitImage from "./localRegistry/buildKitImage.mdx"
import PartialPort from "./localRegistry/port.mdx"
import PartialPersistencereference from "./localRegistry/persistence_reference.mdx"
import PartialResourcesreference from "./dev/resources.mdx"
import PartialAnnotations from "./localRegistry/annotations.mdx"


<PartialEnabled />

Expand All @@ -28,8 +31,6 @@ import PartialPersistencereference from "./localRegistry/persistence_reference.m

<PartialPort />



<details className="config-field" data-expandable="true">
<summary>

Expand All @@ -41,5 +42,8 @@ Persistence settings for the local registry

<PartialPersistencereference />


</details>

<PartialResourcesreference />

<PartialAnnotations />
13 changes: 9 additions & 4 deletions docs/yarn.lock
Original file line number Diff line number Diff line change
Expand Up @@ -1190,12 +1190,12 @@
integrity sha512-x/rqGMdzj+fWZvCOYForTghzbtqPDZ5gPwaoNGHdgDfF2QA/XZbCBp4Moo5scrkAMPhB7z26XM/AaHuIJdgauA==

"@babel/runtime-corejs3@^7.18.6":
version "7.22.5"
resolved "https://registry.yarnpkg.com/@babel/runtime-corejs3/-/runtime-corejs3-7.22.5.tgz#bbc769b48edb2bdfd404b65ad1fc3952bf33e3c2"
integrity sha512-TNPDN6aBFaUox2Lu+H/Y1dKKQgr4ucz/FGyCz67RVYLsBpVpUFf1dDngzg+Od8aqbrqwyztkaZjtWCZEUOT8zA==
version "7.26.10"
resolved "https://registry.yarnpkg.com/@babel/runtime-corejs3/-/runtime-corejs3-7.26.10.tgz#5a3185ca2813f8de8ae68622572086edf5cf51f2"
integrity sha512-uITFQYO68pMEYR46AHgQoyBg7KPPJDAbGn4jUTIRgCFJIp88MIBUianVOplhZDEec07bp9zIyr4Kp0FCyQzmWg==
dependencies:
core-js-pure "^3.30.2"
regenerator-runtime "^0.13.11"
regenerator-runtime "^0.14.0"

"@babel/runtime@^7.1.2", "@babel/runtime@^7.10.3", "@babel/runtime@^7.12.13", "@babel/runtime@^7.12.5", "@babel/runtime@^7.17.8", "@babel/runtime@^7.18.6", "@babel/runtime@^7.20.13", "@babel/runtime@^7.8.4":
version "7.22.5"
Expand Down Expand Up @@ -7564,6 +7564,11 @@ regenerator-runtime@^0.13.11:
resolved "https://registry.yarnpkg.com/regenerator-runtime/-/regenerator-runtime-0.13.11.tgz#f6dca3e7ceec20590d07ada785636a90cdca17f9"
integrity sha512-kY1AZVr2Ra+t+piVaJ4gxaFaReZVH40AKNo7UCX6W+dEwBo/2oZJzqfuN1qLq1oL45o56cPaTXELwrTh8Fpggg==

regenerator-runtime@^0.14.0:
version "0.14.1"
resolved "https://registry.yarnpkg.com/regenerator-runtime/-/regenerator-runtime-0.14.1.tgz#356ade10263f685dda125100cd862c1db895327f"
integrity sha512-dYnhHh0nJoMfnkZs6GmmhFknAGRrLznOu5nc9ML+EJxGvrx6H7teuevqVqCuPcPK//3eDrrjQhehXVx9cnkGdw==

regenerator-transform@^0.15.1:
version "0.15.1"
resolved "https://registry.yarnpkg.com/regenerator-transform/-/regenerator-transform-0.15.1.tgz#f6c4e99fc1b4591f780db2586328e4d9a9d8dc56"
Expand Down
113 changes: 86 additions & 27 deletions pkg/devspace/build/localregistry/deployment.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,17 @@ const BuildKitContainer = "buildkitd"

func (r *LocalRegistry) ensureDeployment(ctx devspacecontext.Context) (*appsv1.Deployment, error) {
// Switching from a persistent registry, delete the statefulset.
_, err := ctx.KubeClient().KubeClient().AppsV1().StatefulSets(r.Namespace).Get(ctx.Context(), r.Name, metav1.GetOptions{})
_, err := ctx.KubeClient().
KubeClient().
AppsV1().
StatefulSets(r.Namespace).
Get(ctx.Context(), r.Name, metav1.GetOptions{})
if err == nil {
err := ctx.KubeClient().KubeClient().AppsV1().StatefulSets(r.Namespace).Delete(ctx.Context(), r.Name, metav1.DeleteOptions{})
err := ctx.KubeClient().
KubeClient().
AppsV1().
StatefulSets(r.Namespace).
Delete(ctx.Context(), r.Name, metav1.DeleteOptions{})
if err != nil && kerrors.IsNotFound(err) {
return nil, err
}
Expand All @@ -31,29 +39,41 @@ func (r *LocalRegistry) ensureDeployment(ctx devspacecontext.Context) (*appsv1.D
var existing *appsv1.Deployment
desired := r.getDeployment()
kubeClient := ctx.KubeClient()
err = wait.PollUntilContextTimeout(ctx.Context(), time.Second, 30*time.Second, true, func(ctx context.Context) (bool, error) {
var err error

existing, err = kubeClient.KubeClient().AppsV1().Deployments(r.Namespace).Get(ctx, r.Name, metav1.GetOptions{})
if err == nil {
return true, nil
}
err = wait.PollUntilContextTimeout(
ctx.Context(),
time.Second,
30*time.Second,
true,
func(ctx context.Context) (bool, error) {
var err error

if kerrors.IsNotFound(err) {
existing, err = kubeClient.KubeClient().AppsV1().Deployments(r.Namespace).Create(ctx, desired, metav1.CreateOptions{})
existing, err = kubeClient.KubeClient().
AppsV1().
Deployments(r.Namespace).
Get(ctx, r.Name, metav1.GetOptions{})
if err == nil {
return true, nil
}

if kerrors.IsAlreadyExists(err) {
return false, nil
if kerrors.IsNotFound(err) {
existing, err = kubeClient.KubeClient().
AppsV1().
Deployments(r.Namespace).
Create(ctx, desired, metav1.CreateOptions{})
if err == nil {
return true, nil
}

if kerrors.IsAlreadyExists(err) {
return false, nil
}

return false, err
}

return false, err
}

return false, err
})
},
)
if err != nil {
return nil, err
}
Expand All @@ -72,7 +92,8 @@ func (r *LocalRegistry) ensureDeployment(ctx devspacecontext.Context) (*appsv1.D
},
)
if kerrors.IsUnsupportedMediaType(err) {
ctx.Log().Debugf("Server-side apply not available on the server for localRegistry deployment: (%v)", err)
ctx.Log().
Debugf("Server-side apply not available on the server for localRegistry deployment: (%v)", err)
// Unsupport server-side apply, we use existing or created deployment
return existing, nil
}
Expand All @@ -96,11 +117,18 @@ func (r *LocalRegistry) getDeployment() *appsv1.Deployment {
Labels: map[string]string{
"app": r.Name,
},
Annotations: getAnnotations(r.LocalBuild),
Annotations: getAnnotations(r.LocalBuild, r.Annotations),
},
Spec: corev1.PodSpec{
EnableServiceLinks: new(bool),
Containers: getContainers(r.RegistryImage, r.BuildKitImage, "registry", int32(r.Port), r.LocalBuild),
Containers: getContainers(
r.RegistryImage,
r.BuildKitImage,
"registry",
int32(r.Port),
r.LocalBuild,
r.Resources,
),
Volumes: []corev1.Volume{
{
VolumeSource: corev1.VolumeSource{
Expand All @@ -121,24 +149,42 @@ func (r *LocalRegistry) getDeployment() *appsv1.Deployment {
}
}

func getAnnotations(localbuild bool) map[string]string {
if !localbuild {
return map[string]string{
func getAnnotations(isLocalbuild bool, annotations map[string]string) map[string]string {
if !isLocalbuild {
combined := map[string]string{
"container.apparmor.security.beta.kubernetes.io/buildkitd": "unconfined",
}

for k, v := range annotations {
combined[k] = v
}

return combined
}
return map[string]string{}

return annotations
}

// this returns a different deployment, if we're using a local docker build or not.
func getContainers(registryImage, buildKitImage, volume string, port int32, localbuild bool) []corev1.Container {
buildContainers := getRegistryContainers(registryImage, volume, port)
func getContainers(
registryImage, buildKitImage, volume string,
port int32,
localbuild bool,
registryResources *corev1.ResourceRequirements,
) []corev1.Container {
buildContainers := getRegistryContainers(registryImage, volume, port, registryResources)
if localbuild {
// in case we're using local builds just return the deployment with only the
// registry container inside
return buildContainers
}

resources := corev1.ResourceRequirements{}

if registryResources != nil {
resources = *registryResources
}

buildKitContainer := []corev1.Container{
{
Name: BuildKitContainer,
Expand Down Expand Up @@ -185,6 +231,7 @@ func getContainers(registryImage, buildKitImage, volume string, port int32, loca
MountPath: "/home/user/.local/share/buildkit",
},
},
Resources: resources,
},
}

Expand All @@ -193,7 +240,18 @@ func getContainers(registryImage, buildKitImage, volume string, port int32, loca
return append(buildKitContainer, buildContainers...)
}

func getRegistryContainers(registryImage, volume string, port int32) []corev1.Container {
func getRegistryContainers(
registryImage, volume string,
port int32,
registryResources *corev1.ResourceRequirements,
) []corev1.Container {

resources := corev1.ResourceRequirements{}

if registryResources != nil {
resources = *registryResources
}

return []corev1.Container{
{
Name: "registry",
Expand Down Expand Up @@ -240,6 +298,7 @@ func getRegistryContainers(registryImage, volume string, port int32) []corev1.Co
MountPath: "/var/lib/registry",
},
},
Resources: resources,
},
}
}
59 changes: 58 additions & 1 deletion pkg/devspace/build/localregistry/options.go
Original file line number Diff line number Diff line change
@@ -1,9 +1,13 @@
package localregistry

import (
"fmt"
"path"

"github.com/loft-sh/devspace/pkg/devspace/config/versions/latest"

corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
)

var (
Expand All @@ -24,6 +28,8 @@ type Options struct {
StorageEnabled bool
StorageSize string
StorageClassName string
Resources *corev1.ResourceRequirements
Annotations map[string]string
}

func getID(o Options) string {
Expand All @@ -41,6 +47,8 @@ func NewDefaultOptions() Options {
StorageEnabled: false,
StorageSize: RegistryDefaultStorage,
StorageClassName: "",
Resources: nil,
Annotations: map[string]string{},
}
}

Expand Down Expand Up @@ -113,6 +121,52 @@ func (o Options) WithStorageSize(storageSize string) Options {
return newOptions
}

func (o Options) WithResources(resources *latest.PodResources) Options {
if resources == nil {
return o
}

// helper converts a map[string]string -> corev1.ResourceList
toList := func(src map[string]string) (corev1.ResourceList, error) {
if len(src) == 0 {
return nil, nil
}
dst := corev1.ResourceList{}
for k, v := range src {
q, err := resource.ParseQuantity(v)
if err != nil {
return nil, fmt.Errorf("invalid quantity %q for %s: %w", v, k, err)
}
dst[corev1.ResourceName(k)] = q
}
return dst, nil
}

reqs, err := toList(resources.Requests)
if reqs == nil || err != nil {
return o
}
lims, err := toList(resources.Limits)
if lims == nil || err != nil {
return o
}

o.Resources = &corev1.ResourceRequirements{
Requests: reqs,
Limits: lims,
}

return o
}

func (o Options) WithAnnotations(annotations map[string]string) Options {
newOptions := o
if annotations != nil {
newOptions.Annotations = annotations
}
return newOptions
}

func (o Options) WithLocalRegistryConfig(config *latest.LocalRegistryConfig) Options {
newOptions := o
if config != nil {
Expand All @@ -122,9 +176,12 @@ func (o Options) WithLocalRegistryConfig(config *latest.LocalRegistryConfig) Opt
WithImage(config.Image).
WithBuildKitImage(config.BuildKitImage).
WithPort(config.Port).
WithResources(config.Resources).
WithAnnotations(config.Annotations).
WithLocalBuild(config.LocalBuild)

if config.Persistence != nil && config.Persistence.Enabled != nil && *config.Persistence.Enabled {
if config.Persistence != nil && config.Persistence.Enabled != nil &&
*config.Persistence.Enabled {
newOptions = newOptions.
EnableStorage().
WithStorageClassName(config.Persistence.StorageClassName).
Expand Down
Loading
Loading