Send additional fields from ebpf-discovery: url-scheme, domain (#94)

* Send additional fields from ebpf-discovery: url-scheme, domain

* fixup! Send additional fields from ebpf-discovery: url-scheme, domain

libebpfdiscovery/src/Discovery.cpp

@@ -119,7 +119,7 @@ void Discovery::handleBufferLookupSuccess(DiscoverySavedBuffer& savedBuffer, Dis
 }
 
 void Discovery::handleExistingSession(SavedSessionsCacheType::iterator it, std::string_view& bufferView, DiscoveryEvent& event) {
-	savedSessions.update(it, [bufferView = std::move(bufferView)](auto& session) { session.parser.parse(std::move(bufferView)); });
+	savedSessions.update(it, [bufferView = std::move(bufferView), flags = event.flags](auto& session) { session.parser.parse(std::move(bufferView), flags); });
 	if (it->second.parser.isInvalidState()) {
 		bpfDiscoveryDeleteSession(event.key);
 		savedSessions.erase(it);
@@ -138,7 +138,7 @@ void Discovery::handleExistingSession(SavedSessionsCacheType::iterator it, std::
 
 void Discovery::handleNewSession(std::string_view& bufferView, DiscoveryEvent& event) {
 	Session session;
-	session.parser.parse(std::move(bufferView));
+	session.parser.parse(std::move(bufferView), event.flags);
 	if (session.parser.isInvalidState()) {
 		return;
 	}

libebpfdiscoveryproto/ebpfdiscoveryproto/service.proto

@@ -23,6 +23,8 @@ message ServicesList {
 message Service {
   uint32 pid = 1;
   string endpoint = 2;
-  uint32 internalClientsNumber = 3;
-  uint32 externalClientsNumber = 4;
+  string domain = 3;
+  string scheme = 4;
+  uint32 internalClientsNumber = 5;
+  uint32 externalClientsNumber = 6;
 }

libebpfdiscoveryproto/src/Translator.cpp

@@ -28,6 +28,8 @@ ServicesList internalToProto(const std::vector<std::reference_wrapper<service::S
 
 		protoService->set_pid(service.pid);
 		protoService->set_endpoint(service.endpoint);
+		protoService->set_domain(service.domain);
+		protoService->set_scheme(service.scheme);
 		protoService->set_internalclientsnumber(service.internalClientsNumber);
 		protoService->set_externalclientsnumber(service.externalClientsNumber);
 	}

libebpfdiscoveryproto/test/TranslatorTest.cpp

@@ -29,15 +29,22 @@ TEST_F(ProtobufTranslatorTest, successfulTranslationToJson) {
 	service::Service service2{.pid = 2, .endpoint = "/endpoint/1", .internalClientsNumber = 1, .externalClientsNumber = 2};
 	service::Service service3{.pid = 3, .endpoint = "/endpoint/2", .internalClientsNumber = 1, .externalClientsNumber = 2};
 
+	service::Service service4{.pid = 4, .endpoint = "google.com/endpoint/3", .domain = "google.com", .scheme = "http", .internalClientsNumber = 1, .externalClientsNumber = 2};
+	service::Service service5{.pid = 5, .endpoint = "dynatrace.com/endpoint/4", .domain = "dynatrace.com", .scheme = "https", .internalClientsNumber = 1, .externalClientsNumber = 2};
+
 	internalServices.push_back(service1);
 	internalServices.push_back(service2);
 	internalServices.push_back(service3);
+	internalServices.push_back(service4);
+	internalServices.push_back(service5);
 
 	const auto proto{internalToProto(internalServices)};
 	const auto result{protoToJson(proto)};
 	const std::string expected{"{\"service\":[{\"pid\":1,\"endpoint\":\"/endpoint/"
 							   "1\",\"internalClientsNumber\":1,\"externalClientsNumber\":2},{\"pid\":2,\"endpoint\":\"/endpoint/"
 							   "1\",\"internalClientsNumber\":1,\"externalClientsNumber\":2},{\"pid\":3,\"endpoint\":\"/endpoint/"
-							   "2\",\"internalClientsNumber\":1,\"externalClientsNumber\":2}]}"};
+							   "2\",\"internalClientsNumber\":1,\"externalClientsNumber\":2},{\"pid\":4,\"endpoint\":\"google.com/"
+							   "endpoint/3\",\"domain\":\"google.com\",\"scheme\":\"http\",\"internalClientsNumber\":1,\"externalClientsNumber\":2},{\"pid\":5,\"endpoint\":\"dynatrace.com/"
+							   "endpoint/4\",\"domain\":\"dynatrace.com\",\"scheme\":\"https\",\"internalClientsNumber\":1,\"externalClientsNumber\":2}]}"};
 	EXPECT_EQ(result, expected);
 }

libhttpparser/headers/httpparser/HttpRequestParser.h

@@ -17,6 +17,7 @@
 #pragma once
 
 #include "ebpfdiscoveryshared/Constants.h"
+#include "ebpfdiscoveryshared/Types.h"
 
 #include <string>
 #include <string_view>
@@ -30,6 +31,7 @@ struct HttpRequest {
 	std::string protocol;
 	std::string host;
 	std::vector<std::string> xForwardedFor;
+	bool isHttps;
 
 	HttpRequest();
 	void clear();
@@ -40,7 +42,7 @@ class HttpRequestParser {
 	HttpRequestParser();
 
 	// Parse passed data while advancing the state machine
-	size_t parse(std::string_view data);
+	size_t parse(std::string_view data, __u8 discoveryFlags);
 
 	bool isInvalidState() const;
 	bool isFinished() const;

libhttpparser/src/HttpRequestParser.cpp

@@ -67,6 +67,7 @@ inline static bool isValidXForwardedForHeaderValueChar(const char ch) {
 HttpRequest::HttpRequest() {
 	method.reserve(4);
 	protocol.reserve(8);
+	isHttps = false;
 }
 
 void HttpRequest::clear() {
@@ -75,12 +76,13 @@ void HttpRequest::clear() {
 	protocol.clear();
 	host.clear();
 	xForwardedFor.clear();
+	isHttps = false;
 }
 
 HttpRequestParser::HttpRequestParser() : state{State::METHOD}, length{0} {
 }
 
-size_t HttpRequestParser::parse(std::string_view data) {
+size_t HttpRequestParser::parse(std::string_view data, __u8 discoveryFlags) {
 	size_t i{0};
 	while (i < data.size()) {
 		if (length > DISCOVERY_MAX_HTTP_REQUEST_LENGTH) {
@@ -94,6 +96,7 @@ size_t HttpRequestParser::parse(std::string_view data) {
 		length++;
 
 		if (state == State::FINISHED || state == State::INVALID) {
+			result.isHttps = discoveryFlags & DISCOVERY_FLAG_SESSION_SSL_HTTP;
 			return i;
 		}
 	}

libhttpparser/test/HttpRequestParserTest.cpp

@@ -40,6 +40,7 @@ struct HttpRequestTestData {
 	std::string protocol;
 	std::string host;
 	std::vector<std::string> xForwardedFor;
+	bool isHttps;
 	bool expectFinished;
 	size_t expectTotalBytesParsed;
 
@@ -50,6 +51,7 @@ struct HttpRequestTestData {
 			std::string protocol = "",
 			std::string host = "",
 			std::vector<std::string> xForwardedFor = {},
+			bool isHttps = false,
 			bool expectFinished = true,
 			std::optional<size_t> expectTotalBytesParsed_ = std::nullopt)
 			: requestChunks(std::move(requestChunks_)),
@@ -58,6 +60,7 @@ struct HttpRequestTestData {
 			  protocol(std::move(protocol)),
 			  host(std::move(host)),
 			  xForwardedFor(std::move(xForwardedFor)),
+			  isHttps(isHttps),
 			  expectFinished(expectFinished),
 			  expectTotalBytesParsed(expectTotalBytesParsed_.value_or(std::accumulate(
 					  requestChunks.begin(), requestChunks.end(), 0, [](int sum, const std::string& str) { return sum + str.length(); }))) {
@@ -71,7 +74,7 @@ TEST_P(HttpRequestParserTest, TestValidRequest) {
 	HttpRequestParser parser;
 	size_t totalBytesParsed{0};
 	for (const auto& chunk : testData.requestChunks) {
-		totalBytesParsed += parser.parse(chunk);
+		totalBytesParsed += parser.parse(chunk, testData.isHttps ? DISCOVERY_FLAG_SESSION_SSL_HTTP : DISCOVERY_FLAG_SESSION_UNENCRYPTED_HTTP);
 	}
 
 	EXPECT_EQ(parser.isFinished(), testData.expectFinished);
@@ -82,6 +85,7 @@ TEST_P(HttpRequestParserTest, TestValidRequest) {
 	EXPECT_EQ(parser.result.protocol, testData.protocol);
 	EXPECT_EQ(parser.result.host, testData.host);
 	EXPECT_EQ(parser.result.xForwardedFor, testData.xForwardedFor);
+	EXPECT_EQ(parser.result.isHttps, testData.isHttps);
 }
 
 struct HttpRequestTestInvalidData {
@@ -96,7 +100,7 @@ TEST_P(HttpRequestParserTestInvalid, testInvalidRequest) {
 	HttpRequestParser parser;
 	size_t totalBytesParsed{0};
 	for (const auto& chunk : testData.requestChunks) {
-		totalBytesParsed += parser.parse(chunk);
+		totalBytesParsed += parser.parse(chunk, DISCOVERY_FLAG_SESSION_UNENCRYPTED_HTTP);
 	}
 
 	EXPECT_TRUE(parser.isFinished());
@@ -121,7 +125,7 @@ INSTANTIATE_TEST_SUITE_P(
 				HttpRequestTestData{
 						{"POST /example HTTP/1.0\r\nHost: example.com\r\n\r\n"}, "POST", "/example", "HTTP/1.0", "example.com", {}},
 				HttpRequestTestData{
-						{"GET /example HTTP/1.1\r\nHost: example.com\r\n\r"}, "GET", "/example", "HTTP/1.1", "example.com", {}, false},
+						{"GET /example HTTP/1.1\r\nHost: example.com\r\n\r"}, "GET", "/example", "HTTP/1.1", "example.com", {}, false, false},
 				HttpRequestTestData{
 						{"GET /Hello%20World/index.html HTTP/1.1\r\nHost:  example.com\r\nx-forwarded-for:  127.0.0.1\r\n\r\n"},
 						"GET",
@@ -152,6 +156,18 @@ INSTANTIATE_TEST_SUITE_P(
 						"HTTP/1.1",
 						"example.com",
 						{"192.168.0.1", "10.0.0.1", "2001:0db8:85a3::8a2e:0370:7334"},
+						false,
+						true,
+						163},
+				HttpRequestTestData{
+						{"POST /example/ HTTP/1.1\r\nHost: example.com\r\nUser-Agent: curl/7.81.0\r\nAccept: */*\r\nX-Forwarded-For: "
+						 "192.168.0.1:8080, 10.0.0.1, [2001:0db8:85a3::8a2e:0370:7334]\r\n\r\n{\"name\":\"example\"}\r\n"},
+						"POST",
+						"/example/",
+						"HTTP/1.1",
+						"example.com",
+						{"192.168.0.1", "10.0.0.1", "2001:0db8:85a3::8a2e:0370:7334"},
+						true,
 						true,
 						163},
 				HttpRequestTestData{
@@ -165,9 +181,9 @@ INSTANTIATE_TEST_SUITE_P(
 						"example.com",
 						{"10.0.0.1", "127.0.0.1", "2001:0db8:85a3::8a2e:0370:7335"},
 				},
-				HttpRequestTestData{chunkString("GET / HTTP/1.1\r\n", 1), "GET", "/", "HTTP/1.1", "", {}, false},
-				HttpRequestTestData{{"GET /"}, "GET", "/", "", "", {}, false},
-				HttpRequestTestData{{"", ""}, "", "", "", "", {}, false}));
+				HttpRequestTestData{chunkString("GET / HTTP/1.1\r\n", 1), "GET", "/", "HTTP/1.1", "", {}, false, false},
+				HttpRequestTestData{{"GET /"}, "GET", "/", "", "", {}, false, false},
+				HttpRequestTestData{{"", ""}, "", "", "", "", {}, false, false}));
 
 INSTANTIATE_TEST_SUITE_P(
 		Default,

libservice/headers/service/Service.h

@@ -24,11 +24,13 @@ namespace service {
 struct Service {
 	uint32_t pid;
 	std::string endpoint;
+	std::string domain;
+	std::string scheme;
 	uint32_t internalClientsNumber{0u};
 	uint32_t externalClientsNumber{0u};
 
 	bool operator==(const Service& other) const {
-		return pid == other.pid && endpoint == other.endpoint && internalClientsNumber == other.internalClientsNumber &&
+		return pid == other.pid && endpoint == other.endpoint && domain == other.domain && scheme == other.scheme && internalClientsNumber == other.internalClientsNumber &&
 			   externalClientsNumber == other.externalClientsNumber;
 	}
 };

libservice/src/Aggregator.cpp

@@ -85,6 +85,8 @@ static Service toService(const IpAddressChecker& ipChecker, const httpparser::Ht
 	Service service;
 	service.pid = meta.pid;
 	service.endpoint = getEndpoint(request.host, request.url);
+	service.domain = request.host.substr(0, request.host.find(':'));
+	service.scheme = request.isHttps ? "https" : "http";
 	incrementServiceClientsNumber(ipChecker, service, request, meta);
 	return service;
 }

libservice/test/AggregatorTest.cpp

@@ -42,6 +42,9 @@ struct ServiceAggregatorTest : public testing::Test {
 		httpparser::HttpRequest request;
 		request.host = host;
 		request.url = url;
+		if (flags.has_value()) {
+			request.isHttps = flags.value() & DISCOVERY_FLAG_SESSION_SSL_HTTP;
+		}
 		DiscoverySessionMeta meta{};
 		if (flags) {
 			meta.flags |= *flags;
@@ -89,21 +92,46 @@ TEST_F(ServiceAggregatorTest, aggregate) {
 		EXPECT_CALL(ipCheckerMock, isV4AddressExternal).WillOnce(testing::Return(true));
 		aggregator.newRequest(request, meta);
 	}
+	// Service 4
+	{
+		const auto [request, meta]{makeRequest(400, "google.com", "/url123", DISCOVERY_FLAG_SESSION_IPV4 | DISCOVERY_FLAG_SESSION_UNENCRYPTED_HTTP)};
+		EXPECT_CALL(ipCheckerMock, isV4AddressExternal).WillOnce(testing::Return(true));
+		aggregator.newRequest(request, meta);
+	}
+	// Service 5
+	{
+		const auto [request, meta]{makeRequest(500, "8.8.8.8", "/url123", DISCOVERY_FLAG_SESSION_IPV4 | DISCOVERY_FLAG_SESSION_UNENCRYPTED_HTTP)};
+		EXPECT_CALL(ipCheckerMock, isV4AddressExternal).WillOnce(testing::Return(true));
+		aggregator.newRequest(request, meta);
+	}
+	// Service 6
+	{
+		const auto [request, meta]{makeRequest(600, "dynatrace.com", "/url123", DISCOVERY_FLAG_SESSION_IPV4 | DISCOVERY_FLAG_SESSION_SSL_HTTP)};
+		EXPECT_CALL(ipCheckerMock, isV4AddressExternal).WillOnce(testing::Return(true));
+		aggregator.newRequest(request, meta);
+	}
+
 
 	{
 		auto services{aggregator.collectServices()};
-		EXPECT_EQ(services.size(), 3);
+		EXPECT_EQ(services.size(), 6);
 
-		Service expectedService1{.pid = 100, .endpoint{"host/url"}, .internalClientsNumber = 0, .externalClientsNumber = 1};
-		Service expectedService2{.pid = 100, .endpoint{"host/url2"}, .internalClientsNumber = 1, .externalClientsNumber = 0};
-		Service expectedService3{.pid = 200, .endpoint{"host/url2"}, .internalClientsNumber = 1, .externalClientsNumber = 2};
+		Service expectedService1{.pid = 100, .endpoint{"host/url"}, .domain = "host", .scheme = "http", .internalClientsNumber = 0, .externalClientsNumber = 1};
+		Service expectedService2{.pid = 100, .endpoint{"host/url2"}, .domain = "host", .scheme = "http", .internalClientsNumber = 1, .externalClientsNumber = 0};
+		Service expectedService3{.pid = 200, .endpoint{"host/url2"}, .domain = "host", .scheme = "http", .internalClientsNumber = 1, .externalClientsNumber = 2};
+		Service expectedService4{.pid = 400, .endpoint{"google.com/url123"}, .domain = "google.com", .scheme = "http", .internalClientsNumber = 0, .externalClientsNumber = 1};
+		Service expectedService5{.pid = 500, .endpoint{"8.8.8.8/url123"}, .domain = "8.8.8.8", .scheme = "http", .internalClientsNumber = 0, .externalClientsNumber = 1};
+		Service expectedService6{.pid = 600, .endpoint{"dynatrace.com/url123"}, .domain = "dynatrace.com", .scheme = "https", .internalClientsNumber = 0, .externalClientsNumber = 1};
 
 		std::vector<Service> servicesCopy;
 		std::transform(services.begin(), services.end(), std::back_inserter(servicesCopy), [](const auto& ref) { return ref.get(); });
 
 		EXPECT_THAT(servicesCopy, testing::Contains(expectedService1));
 		EXPECT_THAT(servicesCopy, testing::Contains(expectedService2));
 		EXPECT_THAT(servicesCopy, testing::Contains(expectedService3));
+		EXPECT_THAT(servicesCopy, testing::Contains(expectedService4));
+		EXPECT_THAT(servicesCopy, testing::Contains(expectedService5));
+		EXPECT_THAT(servicesCopy, testing::Contains(expectedService6));
 	}
 
 	aggregator.clear();