Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Send additional fields from ebpf-discovery: url-scheme, domain #94

Merged
merged 2 commits into from
Jun 26, 2024
Merged

Send additional fields from ebpf-discovery: url-scheme, domain #94

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
04b3fba
Send additional fields from ebpf-discovery: url-scheme, domain
lnarolski Jun 24, 2024
f996b0e
fixup! Send additional fields from ebpf-discovery: url-scheme, domain
lnarolski Jun 26, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions libebpfdiscovery/src/Discovery.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -119,7 +119,7 @@ void Discovery::handleBufferLookupSuccess(DiscoverySavedBuffer& savedBuffer, Dis
}

void Discovery::handleExistingSession(SavedSessionsCacheType::iterator it, std::string_view& bufferView, DiscoveryEvent& event) {
savedSessions.update(it, [bufferView = std::move(bufferView)](auto& session) { session.parser.parse(std::move(bufferView)); });
savedSessions.update(it, [bufferView = std::move(bufferView), flags = event.flags](auto& session) { session.parser.parse(std::move(bufferView), flags); });
if (it->second.parser.isInvalidState()) {
bpfDiscoveryDeleteSession(event.key);
savedSessions.erase(it);
Expand All @@ -138,7 +138,7 @@ void Discovery::handleExistingSession(SavedSessionsCacheType::iterator it, std::

void Discovery::handleNewSession(std::string_view& bufferView, DiscoveryEvent& event) {
Session session;
session.parser.parse(std::move(bufferView));
session.parser.parse(std::move(bufferView), event.flags);
if (session.parser.isInvalidState()) {
return;
}
Expand Down
6 changes: 4 additions & 2 deletions libebpfdiscoveryproto/ebpfdiscoveryproto/service.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@ message ServicesList {
message Service {
uint32 pid = 1;
string endpoint = 2;
uint32 internalClientsNumber = 3;
uint32 externalClientsNumber = 4;
string domain = 3;
string scheme = 4;
uint32 internalClientsNumber = 5;
uint32 externalClientsNumber = 6;
}
2 changes: 2 additions & 0 deletions libebpfdiscoveryproto/src/Translator.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,8 @@ ServicesList internalToProto(const std::vector<std::reference_wrapper<service::S

protoService->set_pid(service.pid);
protoService->set_endpoint(service.endpoint);
protoService->set_domain(service.domain);
protoService->set_scheme(service.scheme);
protoService->set_internalclientsnumber(service.internalClientsNumber);
protoService->set_externalclientsnumber(service.externalClientsNumber);
}
Expand Down
9 changes: 8 additions & 1 deletion libebpfdiscoveryproto/test/TranslatorTest.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,15 +29,22 @@ TEST_F(ProtobufTranslatorTest, successfulTranslationToJson) {
service::Service service2{.pid = 2, .endpoint = "/endpoint/1", .internalClientsNumber = 1, .externalClientsNumber = 2};
service::Service service3{.pid = 3, .endpoint = "/endpoint/2", .internalClientsNumber = 1, .externalClientsNumber = 2};

service::Service service4{.pid = 4, .endpoint = "google.com/endpoint/3", .domain = "google.com", .scheme = "http", .internalClientsNumber = 1, .externalClientsNumber = 2};
service::Service service5{.pid = 5, .endpoint = "dynatrace.com/endpoint/4", .domain = "dynatrace.com", .scheme = "https", .internalClientsNumber = 1, .externalClientsNumber = 2};

internalServices.push_back(service1);
internalServices.push_back(service2);
internalServices.push_back(service3);
internalServices.push_back(service4);
internalServices.push_back(service5);

const auto proto{internalToProto(internalServices)};
const auto result{protoToJson(proto)};
const std::string expected{"{\"service\":[{\"pid\":1,\"endpoint\":\"/endpoint/"
"1\",\"internalClientsNumber\":1,\"externalClientsNumber\":2},{\"pid\":2,\"endpoint\":\"/endpoint/"
"1\",\"internalClientsNumber\":1,\"externalClientsNumber\":2},{\"pid\":3,\"endpoint\":\"/endpoint/"
"2\",\"internalClientsNumber\":1,\"externalClientsNumber\":2}]}"};
"2\",\"internalClientsNumber\":1,\"externalClientsNumber\":2},{\"pid\":4,\"endpoint\":\"google.com/"
"endpoint/3\",\"domain\":\"google.com\",\"scheme\":\"http\",\"internalClientsNumber\":1,\"externalClientsNumber\":2},{\"pid\":5,\"endpoint\":\"dynatrace.com/"
"endpoint/4\",\"domain\":\"dynatrace.com\",\"scheme\":\"https\",\"internalClientsNumber\":1,\"externalClientsNumber\":2}]}"};
EXPECT_EQ(result, expected);
}
4 changes: 3 additions & 1 deletion libhttpparser/headers/httpparser/HttpRequestParser.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
#pragma once

#include "ebpfdiscoveryshared/Constants.h"
#include "ebpfdiscoveryshared/Types.h"

#include <string>
#include <string_view>
Expand All @@ -30,6 +31,7 @@ struct HttpRequest {
std::string protocol;
std::string host;
std::vector<std::string> xForwardedFor;
bool isHttps;

HttpRequest();
void clear();
Expand All @@ -40,7 +42,7 @@ class HttpRequestParser {
HttpRequestParser();

// Parse passed data while advancing the state machine
size_t parse(std::string_view data);
size_t parse(std::string_view data, __u8 discoveryFlags);

bool isInvalidState() const;
bool isFinished() const;
Expand Down
5 changes: 4 additions & 1 deletion libhttpparser/src/HttpRequestParser.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,7 @@ inline static bool isValidXForwardedForHeaderValueChar(const char ch) {
HttpRequest::HttpRequest() {
method.reserve(4);
protocol.reserve(8);
isHttps = false;
}

void HttpRequest::clear() {
Expand All @@ -75,12 +76,13 @@ void HttpRequest::clear() {
protocol.clear();
host.clear();
xForwardedFor.clear();
isHttps = false;
}

HttpRequestParser::HttpRequestParser() : state{State::METHOD}, length{0} {
}

size_t HttpRequestParser::parse(std::string_view data) {
size_t HttpRequestParser::parse(std::string_view data, __u8 discoveryFlags) {
size_t i{0};
while (i < data.size()) {
if (length > DISCOVERY_MAX_HTTP_REQUEST_LENGTH) {
Expand All @@ -94,6 +96,7 @@ size_t HttpRequestParser::parse(std::string_view data) {
length++;

if (state == State::FINISHED || state == State::INVALID) {
result.isHttps = discoveryFlags & DISCOVERY_FLAG_SESSION_SSL_HTTP;
return i;
}
}
Expand Down
28 changes: 22 additions & 6 deletions libhttpparser/test/HttpRequestParserTest.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ struct HttpRequestTestData {
std::string protocol;
std::string host;
std::vector<std::string> xForwardedFor;
bool isHttps;
bool expectFinished;
size_t expectTotalBytesParsed;

Expand All @@ -50,6 +51,7 @@ struct HttpRequestTestData {
std::string protocol = "",
std::string host = "",
std::vector<std::string> xForwardedFor = {},
bool isHttps = false,
bool expectFinished = true,
std::optional<size_t> expectTotalBytesParsed_ = std::nullopt)
: requestChunks(std::move(requestChunks_)),
Expand All @@ -58,6 +60,7 @@ struct HttpRequestTestData {
protocol(std::move(protocol)),
host(std::move(host)),
xForwardedFor(std::move(xForwardedFor)),
isHttps(isHttps),
expectFinished(expectFinished),
expectTotalBytesParsed(expectTotalBytesParsed_.value_or(std::accumulate(
requestChunks.begin(), requestChunks.end(), 0, [](int sum, const std::string& str) { return sum + str.length(); }))) {
Expand All @@ -71,7 +74,7 @@ TEST_P(HttpRequestParserTest, TestValidRequest) {
HttpRequestParser parser;
size_t totalBytesParsed{0};
for (const auto& chunk : testData.requestChunks) {
totalBytesParsed += parser.parse(chunk);
totalBytesParsed += parser.parse(chunk, testData.isHttps ? DISCOVERY_FLAG_SESSION_SSL_HTTP : DISCOVERY_FLAG_SESSION_UNENCRYPTED_HTTP);
}

EXPECT_EQ(parser.isFinished(), testData.expectFinished);
Expand All @@ -82,6 +85,7 @@ TEST_P(HttpRequestParserTest, TestValidRequest) {
EXPECT_EQ(parser.result.protocol, testData.protocol);
EXPECT_EQ(parser.result.host, testData.host);
EXPECT_EQ(parser.result.xForwardedFor, testData.xForwardedFor);
EXPECT_EQ(parser.result.isHttps, testData.isHttps);
}

struct HttpRequestTestInvalidData {
Expand All @@ -96,7 +100,7 @@ TEST_P(HttpRequestParserTestInvalid, testInvalidRequest) {
HttpRequestParser parser;
size_t totalBytesParsed{0};
for (const auto& chunk : testData.requestChunks) {
totalBytesParsed += parser.parse(chunk);
totalBytesParsed += parser.parse(chunk, DISCOVERY_FLAG_SESSION_UNENCRYPTED_HTTP);
}

EXPECT_TRUE(parser.isFinished());
Expand All @@ -121,7 +125,7 @@ INSTANTIATE_TEST_SUITE_P(
HttpRequestTestData{
{"POST /example HTTP/1.0\r\nHost: example.com\r\n\r\n"}, "POST", "/example", "HTTP/1.0", "example.com", {}},
HttpRequestTestData{
{"GET /example HTTP/1.1\r\nHost: example.com\r\n\r"}, "GET", "/example", "HTTP/1.1", "example.com", {}, false},
{"GET /example HTTP/1.1\r\nHost: example.com\r\n\r"}, "GET", "/example", "HTTP/1.1", "example.com", {}, false, false},
HttpRequestTestData{
{"GET /Hello%20World/index.html HTTP/1.1\r\nHost: example.com\r\nx-forwarded-for: 127.0.0.1\r\n\r\n"},
"GET",
Expand Down Expand Up @@ -152,6 +156,18 @@ INSTANTIATE_TEST_SUITE_P(
"HTTP/1.1",
"example.com",
{"192.168.0.1", "10.0.0.1", "2001:0db8:85a3::8a2e:0370:7334"},
false,
true,
163},
HttpRequestTestData{
{"POST /example/ HTTP/1.1\r\nHost: example.com\r\nUser-Agent: curl/7.81.0\r\nAccept: */*\r\nX-Forwarded-For: "
"192.168.0.1:8080, 10.0.0.1, [2001:0db8:85a3::8a2e:0370:7334]\r\n\r\n{\"name\":\"example\"}\r\n"},
"POST",
"/example/",
"HTTP/1.1",
"example.com",
{"192.168.0.1", "10.0.0.1", "2001:0db8:85a3::8a2e:0370:7334"},
true,
true,
163},
HttpRequestTestData{
Expand All @@ -165,9 +181,9 @@ INSTANTIATE_TEST_SUITE_P(
"example.com",
{"10.0.0.1", "127.0.0.1", "2001:0db8:85a3::8a2e:0370:7335"},
},
HttpRequestTestData{chunkString("GET / HTTP/1.1\r\n", 1), "GET", "/", "HTTP/1.1", "", {}, false},
HttpRequestTestData{{"GET /"}, "GET", "/", "", "", {}, false},
HttpRequestTestData{{"", ""}, "", "", "", "", {}, false}));
HttpRequestTestData{chunkString("GET / HTTP/1.1\r\n", 1), "GET", "/", "HTTP/1.1", "", {}, false, false},
HttpRequestTestData{{"GET /"}, "GET", "/", "", "", {}, false, false},
HttpRequestTestData{{"", ""}, "", "", "", "", {}, false, false}));

INSTANTIATE_TEST_SUITE_P(
Default,
Expand Down
4 changes: 3 additions & 1 deletion libservice/headers/service/Service.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,11 +24,13 @@ namespace service {
struct Service {
uint32_t pid;
std::string endpoint;
std::string domain;
lnarolski marked this conversation as resolved.
Show resolved Hide resolved
std::string scheme;
uint32_t internalClientsNumber{0u};
uint32_t externalClientsNumber{0u};

bool operator==(const Service& other) const {
return pid == other.pid && endpoint == other.endpoint && internalClientsNumber == other.internalClientsNumber &&
return pid == other.pid && endpoint == other.endpoint && domain == other.domain && scheme == other.scheme && internalClientsNumber == other.internalClientsNumber &&
externalClientsNumber == other.externalClientsNumber;
}
};
Expand Down
2 changes: 2 additions & 0 deletions libservice/src/Aggregator.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,8 @@ static Service toService(const IpAddressChecker& ipChecker, const httpparser::Ht
Service service;
service.pid = meta.pid;
service.endpoint = getEndpoint(request.host, request.url);
service.domain = request.host.substr(0, request.host.find(':'));
service.scheme = request.isHttps ? "https" : "http";
incrementServiceClientsNumber(ipChecker, service, request, meta);
return service;
}
Expand Down
36 changes: 32 additions & 4 deletions libservice/test/AggregatorTest.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,9 @@ struct ServiceAggregatorTest : public testing::Test {
httpparser::HttpRequest request;
request.host = host;
request.url = url;
if (flags.has_value()) {
request.isHttps = flags.value() & DISCOVERY_FLAG_SESSION_SSL_HTTP;
}
DiscoverySessionMeta meta{};
if (flags) {
meta.flags |= *flags;
Expand Down Expand Up @@ -89,21 +92,46 @@ TEST_F(ServiceAggregatorTest, aggregate) {
EXPECT_CALL(ipCheckerMock, isV4AddressExternal).WillOnce(testing::Return(true));
aggregator.newRequest(request, meta);
}
// Service 4
{
const auto [request, meta]{makeRequest(400, "google.com", "/url123", DISCOVERY_FLAG_SESSION_IPV4 | DISCOVERY_FLAG_SESSION_UNENCRYPTED_HTTP)};
EXPECT_CALL(ipCheckerMock, isV4AddressExternal).WillOnce(testing::Return(true));
aggregator.newRequest(request, meta);
}
// Service 5
{
const auto [request, meta]{makeRequest(500, "8.8.8.8", "/url123", DISCOVERY_FLAG_SESSION_IPV4 | DISCOVERY_FLAG_SESSION_UNENCRYPTED_HTTP)};
EXPECT_CALL(ipCheckerMock, isV4AddressExternal).WillOnce(testing::Return(true));
aggregator.newRequest(request, meta);
}
// Service 6
{
const auto [request, meta]{makeRequest(600, "dynatrace.com", "/url123", DISCOVERY_FLAG_SESSION_IPV4 | DISCOVERY_FLAG_SESSION_SSL_HTTP)};
EXPECT_CALL(ipCheckerMock, isV4AddressExternal).WillOnce(testing::Return(true));
aggregator.newRequest(request, meta);
}


{
auto services{aggregator.collectServices()};
EXPECT_EQ(services.size(), 3);
EXPECT_EQ(services.size(), 6);

Service expectedService1{.pid = 100, .endpoint{"host/url"}, .internalClientsNumber = 0, .externalClientsNumber = 1};
Service expectedService2{.pid = 100, .endpoint{"host/url2"}, .internalClientsNumber = 1, .externalClientsNumber = 0};
Service expectedService3{.pid = 200, .endpoint{"host/url2"}, .internalClientsNumber = 1, .externalClientsNumber = 2};
Service expectedService1{.pid = 100, .endpoint{"host/url"}, .domain = "host", .scheme = "http", .internalClientsNumber = 0, .externalClientsNumber = 1};
Service expectedService2{.pid = 100, .endpoint{"host/url2"}, .domain = "host", .scheme = "http", .internalClientsNumber = 1, .externalClientsNumber = 0};
Service expectedService3{.pid = 200, .endpoint{"host/url2"}, .domain = "host", .scheme = "http", .internalClientsNumber = 1, .externalClientsNumber = 2};
Service expectedService4{.pid = 400, .endpoint{"google.com/url123"}, .domain = "google.com", .scheme = "http", .internalClientsNumber = 0, .externalClientsNumber = 1};
Service expectedService5{.pid = 500, .endpoint{"8.8.8.8/url123"}, .domain = "8.8.8.8", .scheme = "http", .internalClientsNumber = 0, .externalClientsNumber = 1};
Service expectedService6{.pid = 600, .endpoint{"dynatrace.com/url123"}, .domain = "dynatrace.com", .scheme = "https", .internalClientsNumber = 0, .externalClientsNumber = 1};

std::vector<Service> servicesCopy;
std::transform(services.begin(), services.end(), std::back_inserter(servicesCopy), [](const auto& ref) { return ref.get(); });

EXPECT_THAT(servicesCopy, testing::Contains(expectedService1));
EXPECT_THAT(servicesCopy, testing::Contains(expectedService2));
EXPECT_THAT(servicesCopy, testing::Contains(expectedService3));
EXPECT_THAT(servicesCopy, testing::Contains(expectedService4));
EXPECT_THAT(servicesCopy, testing::Contains(expectedService5));
EXPECT_THAT(servicesCopy, testing::Contains(expectedService6));
}

aggregator.clear();
Expand Down