diff --git a/test/cri-containerd/jobcontainer_test.go b/test/cri-containerd/jobcontainer_test.go index e495decfef..32e1421071 100644 --- a/test/cri-containerd/jobcontainer_test.go +++ b/test/cri-containerd/jobcontainer_test.go @@ -1,7 +1,7 @@ //go:build windows && functional // +build windows,functional -package cri_containerd +package azcri_containerd import ( "context" @@ -14,14 +14,13 @@ import ( "testing" "time" + runtime "dev.azure.com/msazure/ContainerPlatform/_git/azcri.git/api/azcri/v1" "github.com/Microsoft/go-winio/vhd" - runtime "k8s.io/cri-api/pkg/apis/runtime/v1" "github.com/Microsoft/hcsshim/hcn" "github.com/Microsoft/hcsshim/osversion" "github.com/Microsoft/hcsshim/pkg/annotations" "github.com/Microsoft/hcsshim/test/pkg/definitions/winapi" - "github.com/Microsoft/hcsshim/test/pkg/require" ) func getJobContainerPodRequestWCOW(t *testing.T) *runtime.RunPodSandboxRequest { @@ -262,7 +261,7 @@ func Test_RunContainer_HNS_JobContainer_WCOW(t *testing.T) { // 2. If it did work we need to delete it. network, err := hcn.GetNetworkByName(networkName) if err != nil { - if _, ok := err.(hcn.NetworkNotFoundError); ok { //nolint:errorlint + if _, ok := err.(hcn.NetworkNotFoundError); ok { t.Fatalf("no network/switch with name %q found: %s", networkName, err) } t.Fatalf("failed to get network/switch with name %q: %s", networkName, err) @@ -419,10 +418,9 @@ func Test_RunContainer_HostVolumes_JobContainer_WCOW(t *testing.T) { } func Test_RunContainer_JobContainer_VolumeMount(t *testing.T) { - requireFeatures(t, featureWCOWProcess, featureHostProcess) + client := newTestRuntimeClient(t) require.ExactBuild(t, osversion.RS5) - client := newTestRuntimeClient(t) dir := t.TempDir() tmpfn := filepath.Join(dir, "tmpfile") @@ -452,43 +450,49 @@ func Test_RunContainer_JobContainer_VolumeMount(t *testing.T) { } type config struct { - name string - containerName string - sandboxImage string - containerImage string - exec []string - mounts []*runtime.Mount + name string + containerName string + requiredFeatures []string + sandboxImage string + containerImage string + exec []string + mounts []*runtime.Mount } tests := []config{ { - name: "JobContainer_VolumeMount_DriveLetter", - containerName: t.Name() + "-Container-DriveLetter", - sandboxImage: imageWindowsNanoserver, - containerImage: imageWindowsNanoserver, - mounts: mountDriveLetter, - exec: []string{"cmd", "/c", "dir", "%CONTAINER_SANDBOX_MOUNT_POINT%\\path\\in\\container\\tmpfile"}, + name: "JobContainer_VolumeMount_DriveLetter", + containerName: t.Name() + "-Container-DriveLetter", + requiredFeatures: []string{featureWCOWProcess, featureHostProcess}, + sandboxImage: imageWindowsNanoserver, + containerImage: imageWindowsNanoserver, + mounts: mountDriveLetter, + exec: []string{"cmd", "/c", "dir", "%CONTAINER_SANDBOX_MOUNT_POINT%\\path\\in\\container\\tmpfile"}, }, { - name: "JobContainer_VolumeMount_NoDriveLetter", - containerName: t.Name() + "-Container-NoDriveLetter", - sandboxImage: imageWindowsNanoserver, - containerImage: imageWindowsNanoserver, - mounts: mountNoDriveLetter, - exec: []string{"cmd", "/c", "dir", "%CONTAINER_SANDBOX_MOUNT_POINT%\\path\\in\\container\\tmpfile"}, + name: "JobContainer_VolumeMount_NoDriveLetter", + containerName: t.Name() + "-Container-NoDriveLetter", + requiredFeatures: []string{featureWCOWProcess, featureHostProcess}, + sandboxImage: imageWindowsNanoserver, + containerImage: imageWindowsNanoserver, + mounts: mountNoDriveLetter, + exec: []string{"cmd", "/c", "dir", "%CONTAINER_SANDBOX_MOUNT_POINT%\\path\\in\\container\\tmpfile"}, }, { - name: "JobContainer_VolumeMount_SingleFile", - containerName: t.Name() + "-Container-SingleFile", - sandboxImage: imageWindowsNanoserver, - containerImage: imageWindowsNanoserver, - mounts: mountSingleFile, - exec: []string{"cmd", "/c", "type", "%CONTAINER_SANDBOX_MOUNT_POINT%\\path\\in\\container\\testfile"}, + name: "JobContainer_VolumeMount_SingleFile", + containerName: t.Name() + "-Container-SingleFile", + requiredFeatures: []string{featureWCOWProcess, featureHostProcess}, + sandboxImage: imageWindowsNanoserver, + containerImage: imageWindowsNanoserver, + mounts: mountSingleFile, + exec: []string{"cmd", "/c", "type", "%CONTAINER_SANDBOX_MOUNT_POINT%\\path\\in\\container\\testfile"}, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { + requireFeatures(t, test.requiredFeatures...) + requiredImages := []string{test.sandboxImage, test.containerImage} pullRequiredImages(t, requiredImages) @@ -520,48 +524,54 @@ func Test_RunContainer_JobContainer_VolumeMount(t *testing.T) { } func Test_RunContainer_JobContainer_Environment(t *testing.T) { - requireFeatures(t, featureWCOWProcess, featureHostProcess) - client := newTestRuntimeClient(t) type config struct { - name string - containerName string - sandboxImage string - containerImage string - env []*runtime.KeyValue - exec []string + name string + containerName string + requiredFeatures []string + sandboxImage string + containerImage string + env []*runtime.KeyValue + exec []string + expectedExecResult string } tests := []config{ { - name: "JobContainer_Env_NoMountPoint", - containerName: t.Name() + "-Container-WithNoMountPoint", - sandboxImage: imageWindowsNanoserver, - containerImage: imageWindowsNanoserver, + name: "JobContainer_Env_NoMountPoint", + containerName: t.Name() + "-Container-WithNoMountPoint", + requiredFeatures: []string{featureWCOWProcess, featureHostProcess}, + sandboxImage: imageWindowsNanoserver, + containerImage: imageWindowsNanoserver, env: []*runtime.KeyValue{ { Key: "PATH", Value: "C:\\Windows\\system32;C:\\Windows", }, }, - exec: []string{"cmd", "/c", "IF", "%PATH%", "==", "C:\\Windows\\system32;C:\\Windows", "( exit 0 )", "ELSE", "(exit -1)"}, + exec: []string{"cmd", "/c", "echo %PATH%"}, + expectedExecResult: "C:\\Windows\\system32;C:\\Windows;C:\\hpc\\;C:\\WINDOWS\\System32\\WindowsPowerShell\\v1.0\\;C:\\WINDOWS\\System32\\Wbem", }, { - name: "JobContainer_VolumeMount_WithMountPoint", - containerName: t.Name() + "-Container-WithMountPoint", - sandboxImage: imageWindowsNanoserver, - containerImage: imageWindowsNanoserver, + name: "JobContainer_VolumeMount_WithMountPoint", + containerName: t.Name() + "-Container-WithMountPoint", + requiredFeatures: []string{featureWCOWProcess, featureHostProcess}, + sandboxImage: imageWindowsNanoserver, + containerImage: imageWindowsNanoserver, env: []*runtime.KeyValue{ { Key: "PATH", Value: "%CONTAINER_SANDBOX_MOUNT_POINT%\\apps\\vim\\;C:\\Windows\\system32;C:\\Windows", }, }, - exec: []string{"cmd", "/c", "IF", "%PATH%", "==", "%CONTAINER_SANDBOX_MOUNT_POINT%\\apps\\vim\\;C:\\Windows\\system32;C:\\Windows", "( exit -1 )", "ELSE", "(exit 0)"}, + exec: []string{"cmd", "/c", "echo %PATH%"}, + expectedExecResult: `C:\hpc\apps\vim\;C:\Windows\system32;C:\Windows;C:\hpc\;C:\WINDOWS\System32\WindowsPowerShell\v1.0\;C:\WINDOWS\System32\Wbem`, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { + requireFeatures(t, test.requiredFeatures...) + requiredImages := []string{test.sandboxImage, test.containerImage} pullRequiredImages(t, requiredImages) @@ -589,57 +599,66 @@ func Test_RunContainer_JobContainer_Environment(t *testing.T) { if r.ExitCode != 0 { t.Fatalf("failed with exit code %d checking for job container mount: %s", r.ExitCode, string(r.Stderr)) } + stdout := strings.TrimSpace(string(r.Stdout)) + if stdout != test.expectedExecResult { + t.Fatalf("PATH does not match. Expected: %v Actual: %v", test.expectedExecResult, stdout) + } }) } } func Test_RunContainer_WorkingDirectory_JobContainer_WCOW(t *testing.T) { - requireFeatures(t, featureWCOWProcess, featureHostProcess) - require.ExactBuild(t, osversion.RS5) - client := newTestRuntimeClient(t) + require.ExactBuild(t, osversion.RS5) type config struct { - name string - containerName string //nolint:unused // may be used in future tests - workDir string - sandboxImage string - containerImage string - cmd []string + name string + containerName string //nolint:unused // may be used in future tests + workDir string + requiredFeatures []string + sandboxImage string + containerImage string + cmd []string } tests := []config{ { - name: "JobContainer_WorkDir_DriveLetter", - workDir: "C:\\go\\", - sandboxImage: imageWindowsNanoserver, - containerImage: imageJobContainerWorkDir, - cmd: []string{"src\\workdir\\workdir.exe"}, + name: "JobContainer_WorkDir_DriveLetter", + workDir: "C:\\go\\", + requiredFeatures: []string{featureWCOWProcess, featureHostProcess}, + sandboxImage: imageWindowsNanoserver, + containerImage: imageJobContainerWorkDir, + cmd: []string{"src\\workdir\\workdir.exe"}, }, { - name: "JobContainer_WorkDir_NoDriveLetter", - workDir: "/go", - sandboxImage: imageWindowsNanoserver, - containerImage: imageJobContainerWorkDir, - cmd: []string{"src/workdir/workdir.exe"}, + name: "JobContainer_WorkDir_NoDriveLetter", + workDir: "/go", + requiredFeatures: []string{featureWCOWProcess, featureHostProcess}, + sandboxImage: imageWindowsNanoserver, + containerImage: imageJobContainerWorkDir, + cmd: []string{"src/workdir/workdir.exe"}, }, { - name: "JobContainer_WorkDir_Default", // Just use the workdir from the image, which is C:\\go\\src\\workdir - sandboxImage: imageWindowsNanoserver, - containerImage: imageJobContainerWorkDir, - cmd: []string{"workdir.exe"}, + name: "JobContainer_WorkDir_Default", // Just use the workdir from the image, which is C:\\go\\src\\workdir + requiredFeatures: []string{featureWCOWProcess, featureHostProcess}, + sandboxImage: imageWindowsNanoserver, + containerImage: imageJobContainerWorkDir, + cmd: []string{"workdir.exe"}, }, { - name: "JobContainer_WorkDir_EnvVar", // Test that putting the envvar in the workdir functions. - workDir: "$env:CONTAINER_SANDBOX_MOUNT_POINT\\go\\src\\workdir\\", - sandboxImage: imageWindowsNanoserver, - containerImage: imageJobContainerWorkDir, - cmd: []string{"workdir.exe"}, + name: "JobContainer_WorkDir_EnvVar", // Test that putting the envvar in the workdir functions. + workDir: "$env:CONTAINER_SANDBOX_MOUNT_POINT\\go\\src\\workdir\\", + requiredFeatures: []string{featureWCOWProcess, featureHostProcess}, + sandboxImage: imageWindowsNanoserver, + containerImage: imageJobContainerWorkDir, + cmd: []string{"workdir.exe"}, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { + requireFeatures(t, test.requiredFeatures...) + requiredImages := []string{test.sandboxImage, test.containerImage} pullRequiredImages(t, requiredImages) diff --git a/test/cri-containerd/test-images/hpc_cmdline/Dockerfile b/test/cri-containerd/test-images/hpc_cmdline/Dockerfile new file mode 100644 index 0000000000..51a47c6a1c --- /dev/null +++ b/test/cri-containerd/test-images/hpc_cmdline/Dockerfile @@ -0,0 +1,5 @@ +FROM mcr.microsoft.com/windows/nanoserver:ltsc2022 + +USER containeradministrator + +COPY ./cmdline.exe / \ No newline at end of file diff --git a/test/cri-containerd/test-images/jobcontainer_cmdline/main.go b/test/cri-containerd/test-images/hpc_cmdline/main.go similarity index 100% rename from test/cri-containerd/test-images/jobcontainer_cmdline/main.go rename to test/cri-containerd/test-images/hpc_cmdline/main.go diff --git a/test/cri-containerd/test-images/hpc_etw/Dockerfile b/test/cri-containerd/test-images/hpc_etw/Dockerfile new file mode 100644 index 0000000000..2e56f56aab --- /dev/null +++ b/test/cri-containerd/test-images/hpc_etw/Dockerfile @@ -0,0 +1,6 @@ +FROM mcr.microsoft.com/windows/nanoserver:ltsc2022 + +USER containeradministrator + +COPY ./etw.exe / +COPY test.wprp / \ No newline at end of file diff --git a/test/cri-containerd/test-images/jobcontainer_etw/main.go b/test/cri-containerd/test-images/hpc_etw/main.go similarity index 84% rename from test/cri-containerd/test-images/jobcontainer_etw/main.go rename to test/cri-containerd/test-images/hpc_etw/main.go index 43f34758f5..f131ffbabd 100644 --- a/test/cri-containerd/test-images/jobcontainer_etw/main.go +++ b/test/cri-containerd/test-images/hpc_etw/main.go @@ -17,7 +17,11 @@ func main() { os.Exit(1) } - path := filepath.Join(filepath.Dir(os.Args[0]), "Test.wprp") + path := filepath.Join(filepath.Dir(os.Args[0]), "test.wprp") + if _, err := os.Stat(path); os.IsNotExist(err) { + log.Fatalf("failed to find wprp file %v: %s", path, err) + } + if err := exec.Command("wpr", "-start", path).Run(); err != nil { log.Fatalf("failed to wpr start: %s", err) } diff --git a/test/cri-containerd/test-images/jobcontainer_etw/test.wprp b/test/cri-containerd/test-images/hpc_etw/test.wprp similarity index 100% rename from test/cri-containerd/test-images/jobcontainer_etw/test.wprp rename to test/cri-containerd/test-images/hpc_etw/test.wprp diff --git a/test/cri-containerd/test-images/hpc_hns/Dockerfile b/test/cri-containerd/test-images/hpc_hns/Dockerfile new file mode 100644 index 0000000000..2d28ebe63a --- /dev/null +++ b/test/cri-containerd/test-images/hpc_hns/Dockerfile @@ -0,0 +1,2 @@ +FROM mcr.microsoft.com/windows/nanoserver:ltsc2022 +COPY ./hns.exe / diff --git a/test/cri-containerd/test-images/jobcontainer_hns/main.go b/test/cri-containerd/test-images/hpc_hns/main.go similarity index 100% rename from test/cri-containerd/test-images/jobcontainer_hns/main.go rename to test/cri-containerd/test-images/hpc_hns/main.go diff --git a/test/cri-containerd/test-images/hpc_vhd/Dockerfile b/test/cri-containerd/test-images/hpc_vhd/Dockerfile new file mode 100644 index 0000000000..983da5a5d0 --- /dev/null +++ b/test/cri-containerd/test-images/hpc_vhd/Dockerfile @@ -0,0 +1,2 @@ +FROM mcr.microsoft.com/windows/nanoserver:ltsc2022 +COPY ./vhd.exe / diff --git a/test/cri-containerd/test-images/jobcontainer_createvhd/main.go b/test/cri-containerd/test-images/hpc_vhd/main.go similarity index 100% rename from test/cri-containerd/test-images/jobcontainer_createvhd/main.go rename to test/cri-containerd/test-images/hpc_vhd/main.go diff --git a/test/cri-containerd/test-images/hpc_workdir/Dockerfile b/test/cri-containerd/test-images/hpc_workdir/Dockerfile new file mode 100644 index 0000000000..7b231e3694 --- /dev/null +++ b/test/cri-containerd/test-images/hpc_workdir/Dockerfile @@ -0,0 +1,2 @@ +FROM mcr.microsoft.com/windows/nanoserver:ltsc2022 +COPY ./hpc_workdir.exe / diff --git a/test/cri-containerd/test-images/jobcontainer_workdir/main.go b/test/cri-containerd/test-images/hpc_workdir/main.go similarity index 100% rename from test/cri-containerd/test-images/jobcontainer_workdir/main.go rename to test/cri-containerd/test-images/hpc_workdir/main.go diff --git a/test/cri-containerd/test-images/jobcontainer_cmdline/dockerfile b/test/cri-containerd/test-images/jobcontainer_cmdline/dockerfile deleted file mode 100644 index c3bfda382c..0000000000 --- a/test/cri-containerd/test-images/jobcontainer_cmdline/dockerfile +++ /dev/null @@ -1,11 +0,0 @@ -# Irrelevant what image version we use for job containers as there's no container <-> host OS version restraint. -FROM golang:1.15.10-nanoserver-1809 - -# Get administrator privileges -USER containeradministrator - -WORKDIR C:\\go\\src\\cmdline -COPY main.go . - -RUN go get -d -v ./... -RUN go build -mod=mod \ No newline at end of file diff --git a/test/cri-containerd/test-images/jobcontainer_createvhd/Dockerfile b/test/cri-containerd/test-images/jobcontainer_createvhd/Dockerfile deleted file mode 100644 index 21bc806c7b..0000000000 --- a/test/cri-containerd/test-images/jobcontainer_createvhd/Dockerfile +++ /dev/null @@ -1,11 +0,0 @@ -# Irrelevant what image version we use for job containers as there's no container <-> host OS version restraint. -FROM golang:1.15.10-nanoserver-1809 - -# Get administrator privileges -USER containeradministrator - -WORKDIR C:\\go\\src\\vhd -COPY main.go . - -RUN go get -d -v ./... -RUN go build -mod=mod \ No newline at end of file diff --git a/test/cri-containerd/test-images/jobcontainer_etw/Dockerfile b/test/cri-containerd/test-images/jobcontainer_etw/Dockerfile deleted file mode 100644 index 97976860a7..0000000000 --- a/test/cri-containerd/test-images/jobcontainer_etw/Dockerfile +++ /dev/null @@ -1,11 +0,0 @@ -# Irrelevant what image version we use for job containers as there's no container <-> host OS version restraint. -FROM golang:1.15.10-nanoserver-1809 - -# Get administrator privileges -USER containeradministrator - -WORKDIR C:\\go\\src\\etw -COPY . . - -RUN go get -d -v ./... -RUN go build -mod=mod \ No newline at end of file diff --git a/test/cri-containerd/test-images/jobcontainer_hns/Dockerfile b/test/cri-containerd/test-images/jobcontainer_hns/Dockerfile deleted file mode 100644 index 4dbe8f441c..0000000000 --- a/test/cri-containerd/test-images/jobcontainer_hns/Dockerfile +++ /dev/null @@ -1,15 +0,0 @@ -# This dockerfile builds a super barebones container image that includes a binary to do a single HNS operation to -# validate that we can actually talk to HNS in a job container. As this is a huge usecase for job containers this is paramount -# to test. The binary in the image will NOT function if this image is used for a normal windows container, both process and hypervisor isolated. - -# Irrelevant what image version we use for job containers as there's no container <-> host OS version restraint. -FROM golang:1.15.10-nanoserver-1809 - -# Get administrator privileges -USER containeradministrator - -WORKDIR C:\\go\\src\\hns -COPY main.go . - -RUN go get -d -v ./... -RUN go build -mod=mod \ No newline at end of file diff --git a/test/cri-containerd/test-images/jobcontainer_workdir/Dockerfile b/test/cri-containerd/test-images/jobcontainer_workdir/Dockerfile deleted file mode 100644 index 7a23de790f..0000000000 --- a/test/cri-containerd/test-images/jobcontainer_workdir/Dockerfile +++ /dev/null @@ -1,15 +0,0 @@ -# This dockerfile builds a super barebones container image that includes a binary to do a single HNS operation to -# validate that we can actually talk to HNS in a job container. As this is a huge usecase for job containers this is paramount -# to test. The binary in the image will NOT function if this image is used for a normal windows container, both process and hypervisor isolated. - -# Irrelevant what image version we use for job containers as there's no container <-> host OS version restraint. -FROM golang:1.15.10-nanoserver-1809 - -# Get administrator privileges -USER containeradministrator - -WORKDIR C:\\go\\src\\workdir -COPY main.go . - -RUN go get -d -v ./... -RUN go build -mod=mod \ No newline at end of file