From 66bcec4734f3ce321bf0d6d49931fe5e4cfa11d1 Mon Sep 17 00:00:00 2001 From: KMAnju-2021 Date: Thu, 19 Dec 2024 11:21:41 +0530 Subject: [PATCH] adding CNCF runner Signed-off-by: KMAnju-2021 --- .github/workflows/kind.yml | 63 ++++++++++++++++++++++++++++++++++++++ ci/kind/kind-setup.sh | 48 +++++++++++++++++++++++++---- ci/kind/test-e2e-kind.sh | 21 ++++++++++++- 3 files changed, 125 insertions(+), 7 deletions(-) diff --git a/.github/workflows/kind.yml b/.github/workflows/kind.yml index 168bf029b33..6c60bbaf8f5 100644 --- a/.github/workflows/kind.yml +++ b/.github/workflows/kind.yml @@ -277,6 +277,69 @@ jobs: path: log.tar.gz retention-days: 30 + test-e2e-ipam-feature-enabled: + name: E2e tests on a Kind cluster on Linux with FlexibleIPAM feature enabled + needs: [build-antrea-coverage-image] + runs-on: [ubuntu-latest-4-cores] + steps: + - uses: actions/checkout@v4 + with: + show-progress: false + - uses: actions/setup-go@v5 + with: + go-version-file: 'go.mod' + - name: Download Antrea image from previous job + uses: actions/download-artifact@v4 + with: + name: antrea-ubuntu-cov + - name: Load Antrea image + run: | + docker load -i antrea-ubuntu.tar + - name: Install Kind + run: | + KIND_VERSION=$(head -n1 ./ci/kind/version) + curl -Lo ./kind https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-$(uname)-amd64 + chmod +x ./kind + sudo mv kind /usr/local/bin + - name: Run ipam e2e tests + # We enable multicast as some FlexibleIPAM e2e tests require it + run: | + mkdir log + mkdir test-ipam-e2e-coverage + ANTREA_LOG_DIR=$PWD/log ANTREA_COV_DIR=$PWD/test-ipam-e2e-coverage ./ci/kind/test-e2e-kind.sh \ + --encap-mode noEncap \ + --coverage \ + --multicast \ + --flexible-ipam \ + - name: Tar coverage files + run: tar -czf test-ipam-e2e-coverage.tar.gz test-ipam-e2e-coverage + - name: Upload coverage for test-ipam-e2e-coverage + uses: actions/upload-artifact@v4 + with: + name: test-ipam-e2e-coverage + path: test-ipam-e2e-coverage.tar.gz + retention-days: 30 + - name: Codecov + uses: codecov/codecov-action@v5 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: '*.cov.out*' + disable_search: true + flags: kind-e2e-tests + name: test-ipam-e2e-coverage + directory: test-ipam-e2e-coverage + fail_ci_if_error: ${{ github.event_name == 'push' }} + - name: Tar log files + if: ${{ failure() }} + run: tar -czf log.tar.gz log + - name: Upload test log + uses: actions/upload-artifact@v4 + if: ${{ failure() }} + with: + name: e2e-kind-ipam-features-enabled.tar.gz + path: log.tar.gz + retention-days: 30 + test-e2e-noencap: name: E2e tests on a Kind cluster on Linux (noEncap) needs: [build-antrea-coverage-image] diff --git a/ci/kind/kind-setup.sh b/ci/kind/kind-setup.sh index e6f5f676130..6ebdbaeccc8 100755 --- a/ci/kind/kind-setup.sh +++ b/ci/kind/kind-setup.sh @@ -300,6 +300,20 @@ function configure_vlan_subnets { docker_run_with_host_net iptables -t filter -A FORWARD -i ${vlan_interfaces[j]} -o ${vlan_interfaces[i]} -j ACCEPT done done + + # Adding iptables rules to skip default masquerading rule added by docker. + if [[ $FLEXIBLE_IPAM == true ]]; then + docker_run_with_host_net ipset create excluded_subnets hash:net + docker_run_with_host_net ipset add excluded_subnets 192.168.241.0/24 + docker_run_with_host_net ipset add excluded_subnets 192.168.242.0/24 + docker_run_with_host_net ipset add excluded_subnets 192.168.240.0/24 + docker_run_with_host_net ipset list excluded_subnets + + # drop traffic when destination is vlan pods and src is ipam pods. + docker_run_with_host_net iptables -t nat -I POSTROUTING 1 ! -o $bridge_interface -s 192.168.240.0/24 -m set --match-set excluded_subnets dst -j RETURN + # drop traffic when destinations are ipam and vlan pods and src is non-ipam pods. + docker_run_with_host_net iptables -t nat -A POSTROUTING ! -o $bridge_interface -s 10.244.0.0/16 -m set ! --match-set excluded_subnets dst -j MASQUERADE + fi } function delete_vlan_subnets { @@ -318,17 +332,35 @@ function delete_vlan_subnets { docker_run_with_host_net ip link del $interface_name fi done + + # if [[ $FLEXIBLE_IPAM == true ]]; then + docker_run_with_host_net iptables -t nat -D POSTROUTING ! -o $bridge_interface -s 192.168.240.0/24 -m set --match-set excluded_subnets dst -j RETURN || true + docker_run_with_host_net iptables -t nat -D POSTROUTING ! -o $bridge_interface -s 10.244.0.0/16 -m set ! --match-set excluded_subnets dst -j MASQUERADE || true + # fi } -function delete_networks { - networks=$(docker network ls -f name=antrea --format '{{.Name}}') - networks="$(echo $networks)" - if [[ ! -z $networks ]]; then +function delete_network_by_filter { + local filter="$1" # Filter passed as a parameter + local networks + networks=$(docker network ls -f name="$filter" --format '{{.Name}}') + + if [[ -n $networks ]]; then docker network rm $networks > /dev/null 2>&1 - echo "deleted networks $networks" + echo "Deleted networks: $networks" fi } +function delete_networks { + local filters=("antrea" "kind") # Define network filters to process + + for filter in "${filters[@]}"; do + if [[ $filter == "kind" && $FLEXIBLE_IPAM != true ]]; then + continue + fi + delete_network_by_filter "$filter" + done +} + function load_images { echo "load images" set +e @@ -711,7 +743,6 @@ if [[ $ACTION == "destroy" ]]; then exit fi - kind_version=$(kind version | awk '{print $2}') kind_version=${kind_version:1} # strip leading 'v' function version_lt() { test "$(printf '%s\n' "$@" | sort -rV | head -n 1)" != "$1"; } @@ -728,5 +759,10 @@ if [[ $ACTION == "create" ]]; then echoerr "Only one of '--subnets' and '--extra-networks' can be specified" exit 1 fi + + # Reserve IPs after 192.168.240.63 for e2e tests. + if [[ $FLEXIBLE_IPAM == true ]]; then + docker network create -d bridge --subnet 192.168.240.0/24 --gateway 192.168.240.1 --ip-range 192.168.240.0/26 kind + fi create fi diff --git a/ci/kind/test-e2e-kind.sh b/ci/kind/test-e2e-kind.sh index 5adc2a8352c..eabc9952416 100755 --- a/ci/kind/test-e2e-kind.sh +++ b/ci/kind/test-e2e-kind.sh @@ -89,6 +89,7 @@ setup_only=false cleanup_only=false test_only=false run="" +flexible_ipam=false antrea_controller_image="antrea/antrea-controller-ubuntu" antrea_agent_image="antrea/antrea-agent-ubuntu" use_non_default_images=false @@ -110,6 +111,10 @@ case $key in proxy_all=true shift ;; + --flexible-ipam) + flexible_ipam=true + shift + ;; --no-kube-proxy) no_kube_proxy=true shift @@ -248,6 +253,9 @@ fi if $flow_visibility; then manifest_args="$manifest_args --feature-gates FlowExporter=true,L7FlowExporter=true --extra-helm-values-file $FLOW_VISIBILITY_HELM_VALUES" fi +if $flexible_ipam; then + manifest_args="$manifest_args --flexible-ipam" +fi COMMON_IMAGES_LIST=("registry.k8s.io/e2e-test-images/agnhost:2.40" \ "antrea/nginx:1.21.6-alpine" \ @@ -302,6 +310,10 @@ if $extra_vlan; then fi fi +if $flexible_ipam; then + vlan_args="$vlan_args --vlan-subnets 11=192.168.241.1/24 --vlan-subnets 12=192.168.242.1/24" +fi + function setup_cluster { args=$1 @@ -328,7 +340,9 @@ function setup_cluster { if $bgp_policy; then args="$args --deploy-external-frr" fi - + if $flexible_ipam; then + args="$args --flexible-ipam" + fi echo "creating test bed with args $args" eval "timeout 600 $TESTBED_CMD create kind $args" } @@ -401,6 +415,11 @@ function run_test { EXTRA_ARGS="$EXTRA_ARGS --external-frr-cid $external_frr_cid --external-frr-ips $external_frr_ips" fi + if $flexible_ipam; then + EXTRA_ARGS="$EXTRA_ARGS --antrea-ipam" + timeout="100m" + fi + go test -v -timeout=$timeout $RUN_OPT antrea.io/antrea/test/e2e $flow_visibility_args -provider=kind --logs-export-dir=$ANTREA_LOG_DIR $np_evaluation_flag --skip-cases=$skiplist $coverage_args $EXTRA_ARGS if $coverage; then