[Additional Layer Store] Add authentication helper

Dockerfile

@@ -18,7 +18,7 @@ ARG CNI_PLUGINS_VERSION=v1.4.1
 ARG NERDCTL_VERSION=1.7.6
 
 ARG PODMAN_VERSION=v5.1.1
-ARG CRIO_VERSION=main
+ARG CRIO_VERSION=v1.31.0
 ARG CONMON_VERSION=v2.1.11
 ARG COMMON_VERSION=v0.58.2
 ARG CRIO_TEST_PAUSE_IMAGE_NAME=registry.k8s.io/pause:3.6
@@ -108,7 +108,7 @@ ARG CTR_REMOTE_BUILD_FLAGS
 COPY . $GOPATH/src/github.com/containerd/stargz-snapshotter
 ARG CGO_ENABLED
 RUN cd $GOPATH/src/github.com/containerd/stargz-snapshotter && \
-    PREFIX=/out/ GOARCH=${TARGETARCH:-amd64} GO_BUILD_FLAGS=${SNAPSHOTTER_BUILD_FLAGS} make stargz-store
+    PREFIX=/out/ GOARCH=${TARGETARCH:-amd64} GO_BUILD_FLAGS=${SNAPSHOTTER_BUILD_FLAGS} make stargz-store stargz-store-helper
 
 # Build podman
 FROM golang-base AS podman-dev

Makefile

@@ -45,6 +45,9 @@ ctr-remote: FORCE
 stargz-store: FORCE
 	cd cmd/ ; GO111MODULE=$(GO111MODULE_VALUE) go build -o $(PREFIX)$@ $(GO_BUILD_FLAGS) $(GO_LD_FLAGS) -v ./stargz-store
 
+stargz-store-helper: FORCE
+	cd cmd/ ; GO111MODULE=$(GO111MODULE_VALUE) go build -o $(PREFIX)$@ $(GO_BUILD_FLAGS) $(GO_LD_FLAGS) -v ./stargz-store/helper
+
 check:
 	@echo "$@"
 	@GO111MODULE=$(GO111MODULE_VALUE) $(shell go env GOPATH)/bin/golangci-lint run

cmd/stargz-store/helper/main.go

@@ -0,0 +1,66 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package main
+
+import (
+	"context"
+	"io"
+	"os"
+	"time"
+
+	"github.com/containerd/containerd/v2/defaults"
+	"github.com/containerd/containerd/v2/pkg/dialer"
+	"github.com/containerd/stargz-snapshotter/store/pb"
+	grpc "google.golang.org/grpc"
+	"google.golang.org/grpc/backoff"
+	"google.golang.org/grpc/credentials/insecure"
+)
+
+func main() {
+	var addr = "/var/lib/stargz-store/store.sock" // default
+	if len(os.Args) >= 2 {
+		addr = os.Args[1]
+	}
+	data, err := io.ReadAll(os.Stdin)
+	if err != nil {
+		panic(err)
+	}
+
+	backoffConfig := backoff.DefaultConfig
+	backoffConfig.MaxDelay = 3 * time.Second
+	connParams := grpc.ConnectParams{
+		Backoff: backoffConfig,
+	}
+	gopts := []grpc.DialOption{
+		grpc.WithTransportCredentials(insecure.NewCredentials()),
+		grpc.WithConnectParams(connParams),
+		grpc.WithContextDialer(dialer.ContextDialer),
+		grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(defaults.DefaultMaxRecvMsgSize)),
+		grpc.WithDefaultCallOptions(grpc.MaxCallSendMsgSize(defaults.DefaultMaxSendMsgSize)),
+	}
+	conn, err := grpc.NewClient(dialer.DialAddress(addr), gopts...)
+	if err != nil {
+		panic(err)
+	}
+	c := pb.NewControllerClient(conn)
+	_, err = c.AddCredential(context.Background(), &pb.AddCredentialRequest{
+		Data: data,
+	})
+	if err != nil {
+		panic(err)
+	}
+}

cmd/stargz-store/main.go

@@ -17,30 +17,37 @@
 package main
 
 import (
+	"bytes"
 	"context"
+	"encoding/json"
+	"errors"
 	"flag"
 	"fmt"
 	"io"
 	golog "log"
 	"math/rand"
+	"net"
 	"os"
 	"os/signal"
 	"path/filepath"
+	"sync"
 	"syscall"
 	"time"
 
+	"github.com/containerd/containerd/v2/pkg/reference"
 	"github.com/containerd/log"
 	dbmetadata "github.com/containerd/stargz-snapshotter/cmd/containerd-stargz-grpc/db"
 	"github.com/containerd/stargz-snapshotter/fs/config"
 	"github.com/containerd/stargz-snapshotter/metadata"
 	memorymetadata "github.com/containerd/stargz-snapshotter/metadata/memory"
-	"github.com/containerd/stargz-snapshotter/service/keychain/dockerconfig"
 	"github.com/containerd/stargz-snapshotter/service/keychain/kubeconfig"
 	"github.com/containerd/stargz-snapshotter/service/resolver"
 	"github.com/containerd/stargz-snapshotter/store"
+	"github.com/containerd/stargz-snapshotter/store/pb"
 	sddaemon "github.com/coreos/go-systemd/v22/daemon"
 	"github.com/pelletier/go-toml"
 	bolt "go.etcd.io/bbolt"
+	grpc "google.golang.org/grpc"
 )
 
 const (
@@ -53,6 +60,7 @@ var (
 	configPath = flag.String("config", defaultConfigPath, "path to the configuration file")
 	logLevel   = flag.String("log-level", defaultLogLevel.String(), "set the logging level [trace, debug, info, warn, error, fatal, panic]")
 	rootDir    = flag.String("root", defaultRootDir, "path to the root directory for this snapshotter")
+	listenaddr = flag.String("addr", filepath.Join(defaultRootDir, "store.sock"), "path to the socket listened by this snapshotter")
 )
 
 type Config struct {
@@ -108,8 +116,12 @@ func main() {
 		}
 	}
 
+	sk := new(storeKeychain)
+
+	errCh := serveController(*listenaddr, sk)
+
 	// Prepare kubeconfig-based keychain if required
-	credsFuncs := []resolver.Credential{dockerconfig.NewDockerconfigKeychain(ctx)}
+	credsFuncs := []resolver.Credential{sk.credentials}
 	if config.KubeconfigKeychainConfig.EnableKeychain {
 		var opts []kubeconfig.Option
 		if kcp := config.KubeconfigKeychainConfig.KubeconfigPath; kcp != "" {
@@ -158,14 +170,22 @@ func main() {
 		}
 	}()
 
-	waitForSIGINT()
-	log.G(ctx).Info("Got SIGINT")
+	if err := waitForSignal(ctx, errCh); err != nil {
+		log.G(ctx).Errorf("error: %v", err)
+		os.Exit(1)
+	}
 }
 
-func waitForSIGINT() {
+func waitForSignal(ctx context.Context, errCh <-chan error) error {
 	c := make(chan os.Signal, 1)
 	signal.Notify(c, os.Interrupt)
-	<-c
+	select {
+	case s := <-c:
+		log.G(ctx).Infof("Got %v", s)
+	case err := <-errCh:
+		return err
+	}
+	return nil
 }
 
 const (
@@ -195,3 +215,80 @@ func getMetadataStore(rootDir string, config Config) (metadata.Store, error) {
 			config.MetadataStore, memoryMetadataType, dbMetadataType)
 	}
 }
+
+func newController(addCredentialFunc func(data []byte) error) *controller {
+	return &controller{
+		addCredentialFunc: addCredentialFunc,
+	}
+}
+
+type controller struct {
+	addCredentialFunc func(data []byte) error
+}
+
+func (c *controller) AddCredential(ctx context.Context, req *pb.AddCredentialRequest) (resp *pb.AddCredentialResponse, _ error) {
+	return &pb.AddCredentialResponse{}, c.addCredentialFunc(req.Data)
+}
+
+type authConfig struct {
+	Username      string `json:"username,omitempty"`
+	Password      string `json:"password,omitempty"`
+	IdentityToken string `json:"identityToken,omitempty"`
+}
+
+type storeKeychain struct {
+	config   map[string]authConfig
+	configMu sync.Mutex
+}
+
+func (sk *storeKeychain) add(data []byte) error {
+	conf := make(map[string]authConfig)
+	if err := json.NewDecoder(bytes.NewReader(data)).Decode(&conf); err != nil && !errors.Is(err, io.EOF) {
+		return err
+	}
+	sk.configMu.Lock()
+	if sk.config == nil {
+		sk.config = make(map[string]authConfig)
+	}
+	for k, c := range conf {
+		sk.config[k] = c
+	}
+	sk.configMu.Unlock()
+	return nil
+}
+
+func (sk *storeKeychain) credentials(host string, refspec reference.Spec) (string, string, error) {
+	if host != refspec.Hostname() {
+		return "", "", nil // Do not use creds for mirrors
+	}
+	sk.configMu.Lock()
+	defer sk.configMu.Unlock()
+	if acfg, ok := sk.config[refspec.String()]; ok {
+		if acfg.IdentityToken != "" {
+			return "", acfg.IdentityToken, nil
+		} else if !(acfg.Username == "" && acfg.Password == "") {
+			return acfg.Username, acfg.Password, nil
+		}
+	}
+	return "", "", nil
+}
+
+func serveController(addr string, sk *storeKeychain) <-chan error {
+	// Try to remove the socket file to avoid EADDRINUSE
+	os.Remove(addr)
+	rpc := grpc.NewServer()
+	c := newController(sk.add)
+	pb.RegisterControllerServer(rpc, c)
+	errCh := make(chan error, 1)
+	go func() {
+		l, err := net.Listen("unix", addr)
+		if err != nil {
+			errCh <- fmt.Errorf("error on listen socket %q: %w", addr, err)
+			return
+		}
+		if err := rpc.Serve(l); err != nil {
+			errCh <- fmt.Errorf("error on serving via socket %q: %w", addr, err)
+		}
+	}()
+	return errCh
+}

go.mod

@@ -14,6 +14,7 @@ require (
 	github.com/distribution/reference v0.6.0
 	github.com/docker/cli v27.1.2+incompatible
 	github.com/docker/go-metrics v0.0.1
+	github.com/gogo/protobuf v1.3.2
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da
 	github.com/hanwen/go-fuse/v2 v2.5.1
 	github.com/hashicorp/go-multierror v1.1.1
@@ -60,7 +61,6 @@ require (
 	github.com/go-openapi/jsonpointer v0.19.6 // indirect
 	github.com/go-openapi/jsonreference v0.20.2 // indirect
 	github.com/go-openapi/swag v0.22.4 // indirect
-	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/google/gnostic-models v0.6.8 // indirect
 	github.com/google/go-cmp v0.6.0 // indirect

script/config-cri-o/etc/containers/registries.conf

@@ -1 +1,2 @@
 unqualified-search-registries = ['docker.io']
+additional-layer-store-auth-helper = "stargz-store-helper"

script/cri-o/test-stargz.sh

@@ -20,6 +20,7 @@ CONTEXT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/"
 REPO="${CONTEXT}../../"
 
 REGISTRY_HOST="cri-registry"
+REGISTRY_HOST_AUTH="cri-registry-auth"
 TEST_NODE_NAME="cri-testenv-container"
 CRIO_SOCK=unix:///run/crio/crio.sock
 PREPARE_NODE_NAME="cri-prepare-node"
@@ -36,6 +37,9 @@ STORE_CONFIG=$(mktemp)
 TMPFILE=$(mktemp)
 LOG_FILE=$(mktemp)
 MIRROR_TMP=$(mktemp -d)
+AUTH_DIR=$(mktemp -d)
+DOCKERCONFIG=$(mktemp)
+echo "${DOCKER_COMPOSE_YAML}"
 function cleanup {
     ORG_EXIT_CODE="${1}"
     docker compose -f "${DOCKER_COMPOSE_YAML}" down -v || true
@@ -46,6 +50,8 @@ function cleanup {
     rm "${TMPFILE}" || true
     rm "${LOG_FILE}" || true
     rm -rf "${MIRROR_TMP}" || true
+    rm -rf "${AUTH_DIR}" || true
+    rm "${DOCKERCONFIG}" || true
     exit "${ORG_EXIT_CODE}"
 }
 trap 'cleanup "$?"' EXIT SIGHUP SIGINT SIGQUIT SIGTERM
@@ -70,6 +76,12 @@ function retry {
     fi
 }
 
+DUMMYUSER=dummyuser
+DUMMYPASS=dummypass
+echo "Preparing creds..."
+prepare_creds "${AUTH_DIR}" "${REGISTRY_HOST_AUTH}" "${DUMMYUSER}" "${DUMMYPASS}"
+echo -n '{"auths":{"'"${REGISTRY_HOST_AUTH}"':5000":{"auth":"'$(echo -n "${DUMMYUSER}:${DUMMYPASS}" | base64 -i -w 0)'"}}}' > "${DOCKERCONFIG}"
+
 # Prepare the testing node and registry
 cat <<EOF > "${DOCKER_COMPOSE_YAML}"
 version: "3.3"
@@ -84,6 +96,7 @@ services:
     - /dev/fuse:/dev/fuse
     - "critest-crio-data:/var/lib/containers"
     - "critest-crio-stargz-store-data:/var/lib/stargz-store"
+    - ${AUTH_DIR}:/auth
   image-prepare:
     image: "${PREPARE_NODE_IMAGE}"
     container_name: "${PREPARE_NODE_NAME}"
@@ -100,9 +113,21 @@ services:
     - "critest-prepare-containerd-stargz-grpc-data:/var/lib/containerd-stargz-grpc"
     - "${REPO}:/go/src/github.com/containerd/stargz-snapshotter:ro"
     - "${MIRROR_TMP}:/tools/"
+    - ${AUTH_DIR}:/auth
   registry:
     image: registry:2
     container_name: ${REGISTRY_HOST}
+  registryauth:
+    image: registry:2
+    container_name: ${REGISTRY_HOST_AUTH}
+    environment:
+    - REGISTRY_AUTH=htpasswd
+    - REGISTRY_AUTH_HTPASSWD_REALM="Registry Realm"
+    - REGISTRY_AUTH_HTPASSWD_PATH=/auth/auth/htpasswd
+    - REGISTRY_HTTP_TLS_CERTIFICATE=/auth/certs/domain.crt
+    - REGISTRY_HTTP_TLS_KEY=/auth/certs/domain.key
+    volumes:
+    - ${AUTH_DIR}:/auth
 volumes:
   critest-crio-data:
   critest-crio-stargz-store-data:
@@ -113,6 +138,18 @@ docker compose -f "${DOCKER_COMPOSE_YAML}" up -d --force-recreate
 
 retry docker exec "${PREPARE_NODE_NAME}" curl -k --head "http://${REGISTRY_HOST}:5000/v2/"
 
+# Mirror images used for auth test
+docker exec "${PREPARE_NODE_NAME}" cp /auth/certs/domain.crt /usr/local/share/ca-certificates
+docker exec "${PREPARE_NODE_NAME}" update-ca-certificates
+docker exec "${PREPARE_NODE_NAME}" go install github.com/google/go-containerregistry/cmd/crane@latest
+docker exec "${PREPARE_NODE_NAME}" mkdir /root/.docker/
+docker cp "${DOCKERCONFIG}" "${PREPARE_NODE_NAME}:/root/.docker/config.json"
+docker exec "${PREPARE_NODE_NAME}" crane copy ghcr.io/stargz-containers/ubuntu:22.04-esgz "${REGISTRY_HOST_AUTH}":5000/ubuntu:22.04-esgz
+
+# Configure registry cert
+docker exec "${TEST_NODE_NAME}" cp /auth/certs/domain.crt /usr/local/share/ca-certificates
+docker exec "${TEST_NODE_NAME}" update-ca-certificates
+
 # Mirror and optimize all images used in tests
 echo "${REGISTRY_HOST}:5000" > "${MIRROR_TMP}/host"
 cp "${IMAGE_LIST}" "${MIRROR_TMP}/list"
@@ -183,6 +220,11 @@ echo "===== VERSION INFORMATION ====="
 docker exec "${TEST_NODE_NAME}" runc --version
 docker exec "${TEST_NODE_NAME}" crio --version
 echo "==============================="
+
+# Do auth test
+docker exec "${TEST_NODE_NAME}" /go/bin/crictl --runtime-endpoint=${CRIO_SOCK} pull --creds "${DUMMYUSER}:${DUMMYPASS}" "${REGISTRY_HOST_AUTH}":5000/ubuntu:22.04-esgz
+
+# Do CRI Validation test
 docker exec "${TEST_NODE_NAME}" /go/bin/critest --runtime-endpoint=${CRIO_SOCK}
 
 echo "Check all remote snapshots are created successfully"

script/podman/config/podman-rootless-stargz-store.service

@@ -4,7 +4,7 @@ Description=Stargz Store service for Podman
 [Service]
 # This pipes stargz-store's IO to the shell otherwise they are lost and can't be managed by systemd.
 # TODO: fix this and do not use pipe
-ExecStart=/bin/bash -c "podman unshare stargz-store --log-level=debug --root %h/.local/share/stargz-store/data %h/.local/share/stargz-store/store 2>&1 | cat"
+ExecStart=/bin/bash -c "podman unshare stargz-store --log-level=debug --root %h/.local/share/stargz-store/data --addr %h/.local/share/stargz-store/store.sock %h/.local/share/stargz-store/store 2>&1 | cat"
 ExecStopPost=podman unshare umount %h/.local/share/stargz-store/store
 
 [Install]