bpf, xdp: clean adjust_{head,meta} memory when offset < 0

include/uapi/linux/bpf.h

@@ -2765,8 +2765,9 @@ union bpf_attr {
  *
  * long bpf_xdp_adjust_head(struct xdp_buff *xdp_md, int delta)
  * 	Description
- * 		Adjust (move) *xdp_md*\ **->data** by *delta* bytes. Note that
- * 		it is possible to use a negative value for *delta*. This helper
+ *		Adjust (move) *xdp_md*\ **->data** by *delta* bytes. Note that
+ *		it is possible to use a negative value for *delta*. If *delta*
+ *		is negative, the new header will be memset to zero. This helper
  * 		can be used to prepare the packet for pushing or popping
  * 		headers.
  *
@@ -2994,7 +2995,8 @@ union bpf_attr {
  * long bpf_xdp_adjust_meta(struct xdp_buff *xdp_md, int delta)
  * 	Description
  * 		Adjust the address pointed by *xdp_md*\ **->data_meta** by
- * 		*delta* (which can be positive or negative). Note that this
+ *		*delta* (which can be positive or negative). If *delta* is
+ *		negative, the new meta will be memset to zero. Note that this
  * 		operation modifies the address stored in *xdp_md*\ **->data**,
  * 		so the latter must be loaded only after the helper has been
  * 		called.

net/core/filter.c

@@ -3947,6 +3947,8 @@ BPF_CALL_2(bpf_xdp_adjust_head, struct xdp_buff *, xdp, int, offset)
 	if (metalen)
 		memmove(xdp->data_meta + offset,
 			xdp->data_meta, metalen);
+	if (offset < 0)
+		memset(data, 0, -offset);
 	xdp->data_meta += offset;
 	xdp->data = data;
 
@@ -4239,7 +4241,8 @@ BPF_CALL_2(bpf_xdp_adjust_meta, struct xdp_buff *, xdp, int, offset)
 		return -EINVAL;
 	if (unlikely(xdp_metalen_invalid(metalen)))
 		return -EACCES;
-
+	if (offset < 0)
+		memset(meta, 0, -offset);
 	xdp->data_meta = meta;
 
 	return 0;

tools/include/uapi/linux/bpf.h

@@ -2766,7 +2766,8 @@ union bpf_attr {
  * long bpf_xdp_adjust_head(struct xdp_buff *xdp_md, int delta)
  * 	Description
  * 		Adjust (move) *xdp_md*\ **->data** by *delta* bytes. Note that
- * 		it is possible to use a negative value for *delta*. This helper
+ *		it is possible to use a negative value for *delta*. If *delta*
+ *		is negative, the new header will be memset to zero. This helper
  * 		can be used to prepare the packet for pushing or popping
  * 		headers.
  *
@@ -2994,7 +2995,8 @@ union bpf_attr {
  * long bpf_xdp_adjust_meta(struct xdp_buff *xdp_md, int delta)
  * 	Description
  * 		Adjust the address pointed by *xdp_md*\ **->data_meta** by
- * 		*delta* (which can be positive or negative). Note that this
+ *		*delta* (which can be positive or negative). If *delta* is
+ *		negative, the new meta will be memset to zero. Note that this
  * 		operation modifies the address stored in *xdp_md*\ **->data**,
  * 		so the latter must be loaded only after the helper has been
  * 		called.

tools/testing/selftests/bpf/prog_tests/xdp_perf.c

@@ -1,10 +1,11 @@
 // SPDX-License-Identifier: GPL-2.0
 #include <test_progs.h>
+#include <network_helpers.h>
+#include "xdp_dummy.skel.h"
 
 void test_xdp_perf(void)
 {
-	const char *file = "./xdp_dummy.bpf.o";
-	struct bpf_object *obj;
+	struct xdp_dummy *skel;
 	char in[128], out[128];
 	int err, prog_fd;
 	LIBBPF_OPTS(bpf_test_run_opts, topts,
@@ -15,14 +16,51 @@ void test_xdp_perf(void)
 		.repeat = 1000000,
 	);
 
-	err = bpf_prog_test_load(file, BPF_PROG_TYPE_XDP, &obj, &prog_fd);
-	if (CHECK_FAIL(err))
-		return;
-
+	skel = xdp_dummy__open_and_load();
+	prog_fd = bpf_program__fd(skel->progs.xdp_dummy_prog);
 	err = bpf_prog_test_run_opts(prog_fd, &topts);
 	ASSERT_OK(err, "test_run");
 	ASSERT_EQ(topts.retval, XDP_PASS, "test_run retval");
 	ASSERT_EQ(topts.data_size_out, 128, "test_run data_size_out");
 
-	bpf_object__close(obj);
+	xdp_dummy__destroy(skel);
+}
+
+void test_xdp_adjust_head_perf(void)
+{
+	struct xdp_dummy *skel;
+	int repeat = 9000000;
+	struct xdp_md ctx_in;
+	char data[100];
+	int err, prog_fd;
+	size_t test_header_size[] = {
+		ETH_ALEN,
+		sizeof(struct iphdr),
+		sizeof(struct ipv6hdr),
+		200,
+	};
+	DECLARE_LIBBPF_OPTS(bpf_test_run_opts, topts,
+			    .data_in = &data,
+			    .data_size_in = sizeof(data),
+			    .repeat = repeat,
+	);
+
+	topts.ctx_in = &ctx_in;
+	topts.ctx_size_in = sizeof(ctx_in);
+	memset(&ctx_in, 0, sizeof(ctx_in));
+	ctx_in.data_meta = 0;
+	ctx_in.data_end = ctx_in.data + sizeof(data);
+
+	skel = xdp_dummy__open_and_load();
+	prog_fd = bpf_program__fd(skel->progs.xdp_dummy_adjust_head);
+
+	for (int i = 0; i < ARRAY_SIZE(test_header_size); i++) {
+		skel->bss->head_size = test_header_size[i];
+		err = bpf_prog_test_run_opts(prog_fd, &topts);
+		ASSERT_OK(err, "test_run");
+		ASSERT_EQ(topts.retval, XDP_PASS, "test_run retval");
+		fprintf(stdout, "run adjust head with size %zd cost %d ns\n",
+			test_header_size[i], topts.duration);
+	}
+	xdp_dummy__destroy(skel);
 }

tools/testing/selftests/bpf/progs/xdp_dummy.c

@@ -4,10 +4,24 @@
 #include <linux/bpf.h>
 #include <bpf/bpf_helpers.h>
 
+int head_size;
+
 SEC("xdp")
 int xdp_dummy_prog(struct xdp_md *ctx)
 {
 	return XDP_PASS;
 }
 
+SEC("xdp")
+int xdp_dummy_adjust_head(struct xdp_md *ctx)
+{
+	if (bpf_xdp_adjust_head(ctx, -head_size))
+		return XDP_DROP;
+
+	if (bpf_xdp_adjust_head(ctx, head_size))
+		return XDP_DROP;
+
+	return XDP_PASS;
+}
+
 char _license[] SEC("license") = "GPL";