diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml index de07e0f5..00fe719c 100644 --- a/.github/workflows/e2e-tests.yaml +++ b/.github/workflows/e2e-tests.yaml @@ -85,6 +85,17 @@ jobs: arch: "arm" family: "Ubuntu2204" kubernetes-version: "1.31.0" + # Since we only enable enforcing mode for SELinux in AL2023, it's easier to list it in "include" + # field rather than trying to exclude all other variants. + - cluster-type: "kops" + arch: "arm" + family: "AmazonLinux2023" + kubernetes-version: "1.28.13" + selinux-mode: "enforcing" + - cluster-type: "eksctl" + arch: "x86" + family: "AmazonLinux2023" + selinux-mode: "enforcing" exclude: - cluster-type: "kops" family: "Bottlerocket" @@ -100,6 +111,10 @@ jobs: kubernetes-version: "1.30.4" - cluster-type: "kops" kubernetes-version: "1.31.0" + # TODO: This is temporary, we wanted to run our tests on SELinux enforced hosts even before we support it. + # Our tests will be passing on SELinux enforced hosts as well once we support it and we will remove this configuration. + continue-on-error: ${{ matrix.selinux-mode == 'enforcing' }} + runs-on: ubuntu-latest environment: ${{ inputs.environment }} permissions: @@ -115,6 +130,7 @@ jobs: # and in order to get latest supported version by envtest we convert it to 1.28. K8S_VERSION: "${{ matrix.kubernetes-version }}" ENVTEST_K8S_VERSION: "${K8S_VERSION%.*}" + SELINUX_MODE: ${{ matrix.selinux-mode }} steps: - name: Checkout uses: actions/checkout@v4 diff --git a/tests/e2e-kubernetes/scripts/eksctl-patch-selinux-enforcing.json b/tests/e2e-kubernetes/scripts/eksctl-patch-selinux-enforcing.json new file mode 100644 index 00000000..d8bc2436 --- /dev/null +++ b/tests/e2e-kubernetes/scripts/eksctl-patch-selinux-enforcing.json @@ -0,0 +1,12 @@ +[ + { + "op": "add", + "path": "/managedNodeGroups/0/preBootstrapCommands", + "value": [ + "dnf install -y container-selinux", + "setenforce 1", + "sed -i \"s/SELINUX=permissive/SELINUX=enforcing/\" /etc/selinux/config", + "reboot" + ] + } + ] \ No newline at end of file diff --git a/tests/e2e-kubernetes/scripts/eksctl.sh b/tests/e2e-kubernetes/scripts/eksctl.sh index e36f24dd..973db3ce 100644 --- a/tests/e2e-kubernetes/scripts/eksctl.sh +++ b/tests/e2e-kubernetes/scripts/eksctl.sh @@ -25,6 +25,7 @@ function eksctl_create_cluster() { NODE_TYPE=${10} AMI_FAMILY=${11} K8S_VERSION=${12} + EKSCTL_PATCH_SELINUX_ENFORCING_FILE=${13} eksctl_delete_cluster "$BIN" "$CLUSTER_NAME" "$REGION" @@ -42,6 +43,12 @@ function eksctl_create_cluster() { CLUSTER_FILE_TMP="${CLUSTER_FILE}.tmp" ${KUBECTL_BIN} patch -f $CLUSTER_FILE --local --type json --patch "$(cat $EKSCTL_PATCH_FILE)" -o yaml > $CLUSTER_FILE_TMP mv $CLUSTER_FILE_TMP $CLUSTER_FILE + + if [ -n "$EKSCTL_PATCH_SELINUX_ENFORCING_FILE" ]; then + ${KUBECTL_BIN} patch -f $CLUSTER_FILE --local --type json --patch "$(cat $EKSCTL_PATCH_SELINUX_ENFORCING_FILE)" -o yaml > $CLUSTER_FILE_TMP + mv $CLUSTER_FILE_TMP $CLUSTER_FILE + fi + ${BIN} create cluster -f "${CLUSTER_FILE}" --kubeconfig "${KUBECONFIG}" if [ -n "$CI_ROLE_ARN" ]; then diff --git a/tests/e2e-kubernetes/scripts/kops-patch-node-selinux-enforcing.yaml b/tests/e2e-kubernetes/scripts/kops-patch-node-selinux-enforcing.yaml new file mode 100644 index 00000000..0be2ec41 --- /dev/null +++ b/tests/e2e-kubernetes/scripts/kops-patch-node-selinux-enforcing.yaml @@ -0,0 +1,8 @@ +spec: + additionalUserData: + - name: selinux-settings.txt + type: text/cloud-config + content: | + #cloud-config + selinux: + mode: enforcing diff --git a/tests/e2e-kubernetes/scripts/kops.sh b/tests/e2e-kubernetes/scripts/kops.sh index 02c4c8bc..9c750875 100755 --- a/tests/e2e-kubernetes/scripts/kops.sh +++ b/tests/e2e-kubernetes/scripts/kops.sh @@ -33,6 +33,7 @@ function kops_create_cluster() { KOPS_PATCH_NODE_FILE=${11} KOPS_STATE_FILE=${12} SSH_KEY=${13} + KOPS_PATCH_NODE_SELINUX_ENFORCING_FILE=${14} if kops_cluster_exists "${CLUSTER_NAME}" "${BIN}" "${KOPS_STATE_FILE}"; then kops_delete_cluster "$BIN" "$CLUSTER_NAME" "$KOPS_STATE_FILE" @@ -58,6 +59,10 @@ function kops_create_cluster() { kops_patch_cluster_file "$CLUSTER_FILE" "$KOPS_PATCH_FILE" "Cluster" "" kops_patch_cluster_file "$CLUSTER_FILE" "$KOPS_PATCH_NODE_FILE" "InstanceGroup" "Node" + if [ -n "$KOPS_PATCH_NODE_SELINUX_ENFORCING_FILE" ]; then + kops_patch_cluster_file "$CLUSTER_FILE" "$KOPS_PATCH_NODE_SELINUX_ENFORCING_FILE" "InstanceGroup" "Node" + fi + ${BIN} create --state "${KOPS_STATE_FILE}" -f "${CLUSTER_FILE}" ${BIN} update cluster --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}" --yes ${BIN} export kubecfg --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}" --admin --kubeconfig "${KUBECONFIG}" diff --git a/tests/e2e-kubernetes/scripts/run.sh b/tests/e2e-kubernetes/scripts/run.sh index 22fb6878..178eb525 100755 --- a/tests/e2e-kubernetes/scripts/run.sh +++ b/tests/e2e-kubernetes/scripts/run.sh @@ -27,6 +27,7 @@ KUBECTL_BIN=${KUBECTL_INSTALL_PATH}/kubectl CLUSTER_TYPE=${CLUSTER_TYPE:-kops} ARCH=${ARCH:-x86} AMI_FAMILY=${AMI_FAMILY:-AmazonLinux2} +SELINUX_MODE=${SELINUX_MODE:-} # kops: must include patch version (e.g. 1.19.1) # eksctl: mustn't include patch version (e.g. 1.19) @@ -54,6 +55,7 @@ KUBECONFIG=${KUBECONFIG:-"${TEST_DIR}/${CLUSTER_NAME}.kubeconfig"} KOPS_VERSION=1.28.5 ZONES=${AWS_AVAILABILITY_ZONES:-$(aws ec2 describe-availability-zones --region ${REGION} | jq -c '.AvailabilityZones[].ZoneName' | grep -v "us-east-1e" | tr '\n' ',' | sed 's/"//g' | sed 's/.$//')} # excluding us-east-1e, see: https://github.com/eksctl-io/eksctl/issues/817 NODE_COUNT=${NODE_COUNT:-3} + if [[ "${ARCH}" == "x86" ]]; then INSTANCE_TYPE_DEFAULT=c5.large AMI_ID_DEFAULT=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-x86_64 --region ${REGION} --query 'Parameters[0].Value' --output text) @@ -61,17 +63,27 @@ else INSTANCE_TYPE_DEFAULT=m7g.medium AMI_ID_DEFAULT=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-arm64 --region ${REGION} --query 'Parameters[0].Value' --output text) fi + + INSTANCE_TYPE=${INSTANCE_TYPE:-$INSTANCE_TYPE_DEFAULT} AMI_ID=${AMI_ID:-$AMI_ID_DEFAULT} CLUSTER_FILE=${TEST_DIR}/${CLUSTER_NAME}.${CLUSTER_TYPE}.yaml KOPS_PATCH_FILE=${KOPS_PATCH_FILE:-${BASE_DIR}/kops-patch.yaml} KOPS_PATCH_NODE_FILE=${KOPS_PATCH_NODE_FILE:-${BASE_DIR}/kops-patch-node.yaml} +KOPS_PATCH_NODE_SELINUX_ENFORCING_FILE=${KOPS_PATCH_NODE_SELINUX_ENFORCING_FILE:-${BASE_DIR}/kops-patch-node-selinux-enforcing.yaml} +if [[ "${SELINUX_MODE}" != "enforcing" ]]; then + KOPS_PATCH_NODE_SELINUX_ENFORCING_FILE="" +fi KOPS_STATE_FILE=${KOPS_STATE_FILE:-"s3://mountpoint-s3-csi-driver-kops-state-store"} SSH_KEY=${SSH_KEY:-""} HELM_RELEASE_NAME=mountpoint-s3-csi-driver -EKSCTL_VERSION=${EKSCTL_VERSION:-0.191.0} +EKSCTL_VERSION=${EKSCTL_VERSION:-0.201.0} EKSCTL_PATCH_FILE=${EKSCTL_PATCH_FILE:-${BASE_DIR}/eksctl-patch.json} +EKSCTL_PATCH_SELINUX_ENFORCING_FILE=${EKSCTL_PATCH_SELINUX_ENFORCING_FILE:-${BASE_DIR}/eksctl-patch-selinux-enforcing.json} +if [[ "${SELINUX_MODE}" != "enforcing" ]]; then + EKSCTL_PATCH_SELINUX_ENFORCING_FILE="" +fi CI_ROLE_ARN=${CI_ROLE_ARN:-""} mkdir -p ${TEST_DIR} @@ -120,7 +132,8 @@ function create_cluster() { "$KOPS_PATCH_FILE" \ "$KOPS_PATCH_NODE_FILE" \ "$KOPS_STATE_FILE" \ - "$SSH_KEY" + "$SSH_KEY" \ + "$KOPS_PATCH_NODE_SELINUX_ENFORCING_FILE" elif [[ "${CLUSTER_TYPE}" == "eksctl" ]]; then eksctl_create_cluster \ "$CLUSTER_NAME" \ @@ -134,7 +147,8 @@ function create_cluster() { "$CI_ROLE_ARN" \ "$INSTANCE_TYPE" \ "$AMI_FAMILY" \ - "$K8S_VERSION_EKSCTL" + "$K8S_VERSION_EKSCTL"\ + "$EKSCTL_PATCH_SELINUX_ENFORCING_FILE" fi }