Merge pull request #315 from libos-nuse/feature-qdisc

lkl: add LKL_HIJACK_NET_QDISC to configure qdisc policy

Author: liuyuan10

Documentation/lkl.txt

@@ -202,6 +202,12 @@ are the list of those variable for your environment.
      $ LKL_HIJACK_NET_NEIGHBOR="192.168.13.100|12:34:56:78:9a:bc;2001:db8:0:f101::3|12:34:56:78:9a:be"
      lkl-hijack.sh ip neighbor show
 ```
+* LKL_HIJACK_NET_QDISC
+
+  Add a qdisc entry in the form of "root|type;root|type;...".
+```
+     $ LKL_HIJACK_NET_QDISC="root|fq" lkl-hijack.sh tc qdisc
+```
 * LKL_HIJACK_DEBUG
 
   Setting it causes some debug information (both from the kernel and the

arch/lkl/Kconfig

@@ -25,6 +25,8 @@ config LKL
        select IP_PNP_DHCP
        select TCP_CONG_BBR
        select HIGH_RES_TIMERS
+       select NET_SCHED
+       select NET_SCH_FQ
 
 config OUTPUTFORMAT
        string

arch/lkl/include/uapi/asm/syscalls.h

@@ -128,6 +128,7 @@ struct sockaddr {
 #include <linux/virtio_blk.h>
 #include <linux/virtio_net.h>
 #include <linux/virtio_ring.h>
+#include <linux/pkt_sched.h>
 
 struct user_msghdr {
 	void		__user *msg_name;	/* ptr to socket address structure */

tools/lkl/include/lkl.h

@@ -461,6 +461,24 @@ int lkl_if_wait_ipv6_dad(int ifindex, void *addr);
  */
 int lkl_set_fd_limit(unsigned int fd_limit);
 
+/**
+ * lkl_qdisc_add - set qdisc rule onto an interface
+ *
+ * @ifindex - the ifindex of the interface
+ * @root - the name of root class (e.g., "root");
+ * @type - the type of qdisc (e.g., "fq")
+ */
+int lkl_qdisc_add(int ifindex, char *root, char *type);
+
+/**
+ * lkl_qdisc_parse_add - Add a qdisc entry for an interface with strings
+ *
+ * @ifindex - the ifindex of the interface
+ * @entries - strings of qdisc configurations in the form of
+ *            "root|type;root|type;..."
+ */
+void lkl_qdisc_parse_add(int ifindex, char *entries);
+
 #ifdef __cplusplus
 }
 #endif

tools/lkl/lib/hijack/init.c

@@ -202,6 +202,7 @@ hijack_init(void)
 	char *mount = getenv("LKL_HIJACK_MOUNT");
 	struct lkl_netdev_args nd_args;
 	char *neigh_entries = getenv("LKL_HIJACK_NET_NEIGHBOR");
+	char *qdisc_entries = getenv("LKL_HIJACK_NET_QDISC");
 	/* single_cpu mode:
 	 * 0: Don't pin to single CPU (default).
 	 * 1: Pin only LKL kernel threads to single CPU.
@@ -429,8 +430,12 @@ hijack_init(void)
 	if (mount)
 		mount_cmds_exec(mount, lkl_mount_fs);
 
-	if (nd_ifindex >=0 && neigh_entries)
+	if (nd_ifindex >= 0 && neigh_entries)
 		add_neighbor(nd_ifindex, neigh_entries);
+
+	if (nd_ifindex >= 0 && qdisc_entries)
+		lkl_qdisc_parse_add(nd_ifindex, qdisc_entries);
+
 }
 
 void __attribute__((destructor))

tools/lkl/lib/net.c

@@ -529,3 +529,66 @@ int lkl_if_del_ip(int ifindex, int af, void *addr, unsigned int netprefix_len)
 	return ipaddr_modify(LKL_RTM_DELADDR, 0, ifindex, af,
 			     addr, netprefix_len);
 }
+
+static int qdisc_add(int cmd, int flags, int ifindex,
+		     char *root, char *type)
+{
+	struct {
+		struct lkl_nlmsghdr n;
+		struct lkl_tcmsg tc;
+		char buf[64*1024];
+	} req = {
+		.n.nlmsg_len = LKL_NLMSG_LENGTH(sizeof(struct lkl_tcmsg)),
+		.n.nlmsg_flags = LKL_NLM_F_REQUEST|flags,
+		.n.nlmsg_type = cmd,
+		.tc.tcm_family = LKL_AF_UNSPEC,
+	};
+	int err, fd;
+
+	if (!root || !type) {
+		lkl_printf("root and type arguments\n");
+		return -1;
+	}
+
+	if (strcmp(root, "root") == 0)
+		req.tc.tcm_parent = LKL_TC_H_ROOT;
+	req.tc.tcm_ifindex = ifindex;
+
+	fd = netlink_sock(0);
+	if (fd < 0)
+		return fd;
+
+	// create the qdisc attribute
+	addattr_l(&req.n, sizeof(req), LKL_TCA_KIND, type, 2);
+
+	err = rtnl_talk(fd, &req.n);
+	lkl_sys_close(fd);
+	return err;
+}
+
+int lkl_qdisc_add(int ifindex, char *root, char *type)
+{
+	return qdisc_add(LKL_RTM_NEWQDISC, LKL_NLM_F_CREATE | LKL_NLM_F_EXCL,
+			 ifindex, root, type);
+}
+
+/* Add a qdisc entry for an interface in the form of
+ * "root|type;root|type;..."
+ */
+void lkl_qdisc_parse_add(int ifindex, char *entries)
+{
+	char *token = NULL;
+	char *root = NULL, *type = NULL;
+	int ret = 0;
+
+	for (token = strtok(entries, ";"); token; token = strtok(NULL, ";")) {
+		root = strtok(token, "|");
+		type = strtok(NULL, "|");
+		ret = lkl_qdisc_add(ifindex, root, type);
+		if (ret) {
+			fprintf(stderr, "Failed to add qdisc entry: %s\n",
+				lkl_strerror(ret));
+			return;
+		}
+	}
+}

tools/lkl/tests/hijack-test.sh

@@ -129,6 +129,12 @@ LKL_HIJACK_OFFLOAD=0x8803 sh ${script_dir}/run_netperf.sh 192.168.13.1 1 0 TCP_M
 sh ${script_dir}/run_netperf.sh 192.168.13.1 1 0 TCP_RR
 sh ${script_dir}/run_netperf.sh fc03::1 1 0 TCP_STREAM
 
+# QDISC test
+qdisc=$(LKL_HIJACK_NET_QDISC="root|fq" ${hijack_script} tc -s -d qdisc show)
+echo "$qdisc"
+echo "$qdisc" | grep "qdisc fq" > /dev/null
+echo "$qdisc" | grep throttled > /dev/null
+
 if [ -z "`printenv CONFIG_AUTO_LKL_VIRTIO_NET_VDE`" ]; then
     exit 0
 fi