Skip to content

BPF: Allows users to set DSCP on traffic toward hosts/hostendpoints #10997

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
mazdakn merged 6 commits into from
Sep 15, 2025
Merged

BPF: Allows users to set DSCP on traffic toward hosts/hostendpoints #10997

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
0fee45a
First commit
mazdakn Sep 11, 2025
7ecfecc
fix merge
mazdakn Sep 11, 2025
5eddd50
cleanup
mazdakn Sep 11, 2025
f3752c9
markup
mazdakn Sep 12, 2025
f726794
[BPF] make as few route lookups as possible
tomastigera Sep 12, 2025
037e455
fix bpf ut
mazdakn Sep 15, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 5 additions & 19 deletions felix/bpf-gpl/routes.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,8 @@ static CALI_BPF_INLINE enum cali_rt_flags cali_rt_lookup_flags(ipv46_addr_t *add
#define cali_rt_flags_local_tunneled_host(t) (((t) & (CALI_RT_LOCAL | CALI_RT_HOST | CALI_RT_TUNNELED)) == (CALI_RT_LOCAL | CALI_RT_HOST | CALI_RT_TUNNELED))
#define cali_rt_flags_is_in_pool(t) (((t) & CALI_RT_IN_POOL) == CALI_RT_IN_POOL)
#define cali_rt_flags_skip_ingress_redirect(t) (((t) & CALI_RT_SKIP_INGRESS_REDIRECT))
#define cali_rt_flags_external(t) (!((t) & (CALI_RT_WORKLOAD | CALI_RT_HOST)))

#define cali_rt_flags_should_set_dscp(t) (!cali_rt_flags_is_in_pool(t) && !cali_rt_flags_local_host(t))

static CALI_BPF_INLINE bool rt_addr_is_local_host(ipv46_addr_t *addr)
{
Expand All @@ -118,28 +119,13 @@ static CALI_BPF_INLINE bool rt_addr_is_local_tunneled_host(ipv46_addr_t *addr)
return cali_rt_flags_local_tunneled_host(cali_rt_lookup_flags(addr));
}

static CALI_BPF_INLINE bool rt_addr_is_external(ipv46_addr_t *addr)
{
return cali_rt_flags_external(cali_rt_lookup_flags(addr));
}

static CALI_BPF_INLINE bool rt_addr_is_host_or_in_pool(ipv46_addr_t *addr)
{
__u32 flags = cali_rt_lookup_flags(addr);
return cali_rt_flags_host(flags) || cali_rt_flags_is_in_pool(flags);
}

// Don't perform SNAT if either:
// - packet is destined to an address in an IP pool;
// - packet is destined to local host; or
// - packet is destined to a host and the CALI_GLOBALS_NATOUTGOING_EXCLUDE_HOSTS global flag is set
static CALI_BPF_INLINE bool rt_flags_should_perform_nat_outgoing(enum cali_rt_flags flags, bool exclude_hosts)
static CALI_BPF_INLINE bool cali_rt_flags_should_perform_nat_outgoing(__u32 flags, bool exclude_hosts)
{
if cali_rt_flags_is_in_pool(flags) return false;
if cali_rt_flags_host(flags) {
if cali_rt_flags_local(flags) return false;
if (exclude_hosts) return false;
}
return true;
return !(cali_rt_flags_is_in_pool(flags) ||
(cali_rt_flags_host(flags) && (exclude_hosts || cali_rt_flags_local(flags))));
}
#endif /* __CALI_ROUTES_H__ */
52 changes: 27 additions & 25 deletions felix/bpf-gpl/tc.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -510,9 +510,19 @@ static CALI_BPF_INLINE void calico_tc_process_ct_lookup(struct cali_tc_ctx *ctx)
}
}

struct cali_rt *src_rt = cali_rt_lookup(&ctx->state->ip_src);
struct cali_rt *dst_rt = NULL;

if (CALI_F_TO_HEP || CALI_F_FROM_WEP) {
dst_rt = cali_rt_lookup(&ctx->state->post_nat_ip_dst);
}

__u32 src_flags = src_rt ? src_rt->flags : CALI_RT_UNKNOWN;
__u32 dst_flags = dst_rt ? dst_rt->flags : CALI_RT_UNKNOWN;

if (CALI_F_TO_WEP && (!skb_seen(ctx->skb) ||
skb_mark_equals(ctx->skb, CALI_SKB_MARK_FROM_NAT_IFACE_OUT, CALI_SKB_MARK_FROM_NAT_IFACE_OUT)) &&
cali_rt_flags_local_host(cali_rt_lookup_flags(&ctx->state->ip_src))) {
cali_rt_flags_local_host(src_flags)) {
/* Host to workload traffic always allowed. We discount traffic that was
* seen by another program since it must have come in via another interface.
*/
Expand All @@ -525,53 +535,45 @@ static CALI_BPF_INLINE void calico_tc_process_ct_lookup(struct cali_tc_ctx *ctx)
&& !(ctx->state->ip_proto == IPPROTO_ICMPV6 && ip_link_local(ctx->state->ip_src))
#endif
) {
struct cali_rt *r = cali_rt_lookup(&ctx->state->ip_src);
/* Do RPF check since it's our responsibility to police that. */
if (wep_rpf_check(ctx, r) == RPF_RES_FAIL) {
if (wep_rpf_check(ctx, src_rt) == RPF_RES_FAIL) {
goto deny;
}

if (cali_rt_flags_skip_ingress_redirect(r->flags)) {
if (cali_rt_flags_skip_ingress_redirect(src_flags)) {
ctx->state->flags |= CALI_ST_SKIP_REDIR_PEER;
}

// Check whether the workload needs outgoing NAT to this address.
if (r->flags & CALI_RT_NAT_OUT) {
struct cali_rt *rt = cali_rt_lookup(&ctx->state->post_nat_ip_dst);
enum cali_rt_flags flags = CALI_RT_UNKNOWN;
if (rt) {
flags = rt->flags;
}
if (src_flags & CALI_RT_NAT_OUT) {
bool exclude_hosts = (GLOBAL_FLAGS & CALI_GLOBALS_NATOUTGOING_EXCLUDE_HOSTS);
Copy link
Copy Markdown
Contributor

@tomastigera tomastigera Sep 12, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The original idea here was that you should do nat outgoing if you are going out of a pool and that includes hosts. But we have a config option now that excludes hosts from nat outgoing.

if (rt_flags_should_perform_nat_outgoing(flags, exclude_hosts)) {
if (cali_rt_flags_should_perform_nat_outgoing(dst_flags, exclude_hosts)) {
CALI_DEBUG("Source is in NAT-outgoing pool but dest is not, need to SNAT.");
ctx->state->flags |= CALI_ST_NAT_OUTGOING;
}
}
// Check if traffic is leaving cluster. We might need to set DSCP later.
if (cali_rt_flags_is_in_pool(r->flags) && rt_addr_is_external(&ctx->state->post_nat_ip_dst)) {
CALI_DEBUG("Outside cluster dest " IP_FMT "", debug_ip(ctx->state->post_nat_ip_dst));
ctx->state->flags |= CALI_ST_CLUSTER_EXTERNAL;
}
/* If 3rd party CNI is used and dest is outside cluster. See commit fc711b192f for details. */
if (!(cali_rt_flags_is_in_pool(r->flags))) {
if (!(cali_rt_flags_is_in_pool(src_flags))) {
CALI_DEBUG("Source " IP_FMT " not in IP pool", debug_ip(ctx->state->ip_src));
if (rt_addr_is_external(&ctx->state->post_nat_ip_dst)) {
if (!(dst_flags & (CALI_RT_WORKLOAD | CALI_RT_HOST))) {
CALI_DEBUG("Outside cluster dest " IP_FMT "", debug_ip(ctx->state->post_nat_ip_dst));
ctx->state->flags |= CALI_ST_SKIP_FIB;
}
} else {
if (cali_rt_flags_should_set_dscp(dst_flags)) {
CALI_DEBUG("Remote host or outside cluster dest " IP_FMT "", debug_ip(ctx->state->post_nat_ip_dst));
ctx->state->flags |= CALI_ST_CLUSTER_EXTERNAL;
}
}
}

// If either source or destination is outside cluster, set flag as might need to update DSCP later.
if ((CALI_F_TO_HEP) && (rt_addr_is_local_host(&ctx->state->ip_src)) &&
(rt_addr_is_external(&ctx->state->post_nat_ip_dst))) {
CALI_DEBUG("Outside cluster dest " IP_FMT "", debug_ip(ctx->state->post_nat_ip_dst));
/* If either source or destination is outside cluster, set flag as might need to update DSCP later. */
if ((CALI_F_TO_HEP) && (cali_rt_flags_local_host(src_flags)) && cali_rt_flags_should_set_dscp(dst_flags)) {
CALI_DEBUG("Remote host or outside cluster dest " IP_FMT "", debug_ip(ctx->state->post_nat_ip_dst));
ctx->state->flags |= CALI_ST_CLUSTER_EXTERNAL;
}
if ((CALI_F_FROM_HEP) && (rt_addr_is_host_or_in_pool(&ctx->state->post_nat_ip_dst)) &&
(rt_addr_is_external(&ctx->state->ip_src))) {
CALI_DEBUG("Outside cluster source " IP_FMT "", debug_ip(ctx->state->ip_src));
if ((CALI_F_FROM_HEP) && cali_rt_flags_should_set_dscp(src_flags)) {
CALI_DEBUG("Remote host or outside cluster source " IP_FMT "", debug_ip(ctx->state->ip_src));
ctx->state->flags |= CALI_ST_CLUSTER_EXTERNAL;
}

Expand Down
10 changes: 5 additions & 5 deletions felix/bpf/ut/qos_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -191,8 +191,8 @@ func TestDSCPv4_HEP(t *testing.T) {
{"calico_to_host_ep", tcdefs.MarkSeen, externalAddr, srcIP, resTC_ACT_UNSPEC, 8, -1},
{"calico_from_host_ep", 0, externalAddr, dstIP, resTC_ACT_UNSPEC, 40, -1},

// Src and dest both inside cluster.
{"calico_to_host_ep", tcdefs.MarkSeen, srcIP, dstIP, resTC_ACT_UNSPEC, 8, -1},
// Src and dest are both hosts.
{"calico_to_host_ep", tcdefs.MarkSeen, srcIP, dstIP, resTC_ACT_UNSPEC, 8, 8},
} {
skbMark = tc.expectedSKBMark
runBpfTest(t, tc.progName, rulesAllowUDP, func(bpfrun bpfProgRunFn) {
Expand Down Expand Up @@ -272,7 +272,7 @@ func TestDSCPv6_HEP(t *testing.T) {
Expect(err).NotTo(HaveOccurred())

rtKey = routes.NewKeyV6(dstV6CIDR).AsBytes()
rtVal = routes.NewValueV6WithIfIndex(routes.FlagsRemoteWorkload|routes.FlagInIPAMPool, ifIndex).AsBytes()
rtVal = routes.NewValueV6WithIfIndex(routes.FlagsRemoteHost, ifIndex).AsBytes()
err = rtMapV6.Update(rtKey, rtVal)
Expect(err).NotTo(HaveOccurred())
defer resetRTMap(rtMapV6)
Expand All @@ -288,8 +288,8 @@ func TestDSCPv6_HEP(t *testing.T) {
{"calico_to_host_ep", tcdefs.MarkSeen, externalAddr, dstIPv6, resTC_ACT_UNSPEC, 8, -1},
{"calico_from_host_ep", 0, externalAddr, dstIPv6, resTC_ACT_UNSPEC, 40, -1},

// Src and dest both inside cluster.
{"calico_to_host_ep", tcdefs.MarkSeen, srcIPv6, dstIPv6, resTC_ACT_UNSPEC, 8, -1},
// Src and dest both hosts.
{"calico_to_host_ep", tcdefs.MarkSeen, srcIPv6, dstIPv6, resTC_ACT_UNSPEC, 8, 8},
} {
skbMark = tc.expectedSKBMark
runBpfTest(t, tc.progName, rulesDefaultAllow, func(bpfrun bpfProgRunFn) {
Expand Down