Skip to content

Commit 23ff42f

Browse files
authored
feat(netprobe): HTTP(S) + DoH endpoint probes via libcurl (Refs #697) (#792)
* feat(http): add libcurl HttpClient transport (curl_multi on uvw loop) (Refs #697) * fix(http): guard HttpClient request/timer/socket callbacks against closed state (Refs #697) * feat(netprobe): HttpProbe issuing requests via the libcurl HttpClient (Refs #697) * fix(netprobe): only allocate HttpClient for http streams; clarify timer capture (Refs #697) * feat(netprobe): status-aware HTTP metrics, top_status_codes, http_response_phases group (Refs #697) * test(netprobe): end-to-end HTTP probe test + docs (Refs #697) * fix(http): bound redirects to 10, document timeout_msec:0, assert per-phase timings (Refs #697) * fix(http): handle curl alloc failures, fix watchdog/server-ready test races, README accuracy (Refs #697) * fix(netprobe): lock _targets_metrics insert in process_attempts/process_failure (Refs #697) * build(deps): enable HTTP/2 in libcurl via libnghttp2 (Refs #697) * feat(http): HttpClient request body, headers, and response-body capture (Refs #697) * feat(netprobe): DohProbe issuing DoH queries via the libcurl HttpClient (Refs #697) * feat(netprobe): DNS-aware DoH metrics — dns_response_failures, top_rcodes (Refs #697) * test(netprobe): end-to-end DoH probe test + docs (Refs #697) * fix(http): include <stdexcept> for std::runtime_error (gcc/libstdc++) (Refs #697) * fix(netprobe): address PR review — curl add_handle check, DoH method validation, metric help-text, e2e server-ready/port checks (Refs #697) * feat(netprobe): v1 hardening — URL validation, NOSIGNAL, DoH content-type/question validation, curl-error logging, request() reentrancy guard, POLL_NONE stop, DoH top_status_codes (Refs #697) * refactor(http): encapsulate URL validation in visor_http_client; guard curl_slist_append OOM (Refs #697) * fix(netprobe): init curl before URL validation; normalize DoH qtype to uppercase (Refs #697) * fix(netprobe): restrict redirects to http(s); require fully-parseable DoH response (TC bit + parsed-vs-declared records); accurate timeout comments (Refs #697) * chore(http): add Visor::Lib::Http alias; align http_status_failures README with actual semantics (Refs #697) * fix(netprobe): apply topn settings to per-target status/rcode TopN; validate DoH qname + guard addQuery (Refs #697) * feat(netprobe): support DoH root ('.') queries — accept root in qname validation, build/echo via the empty name pcpp uses (Refs #697)
1 parent 3eb47ea commit 23ff42f

21 files changed

Lines changed: 2692 additions & 30 deletions

‎conanfile.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,12 +28,16 @@ def requirements(self):
2828
self.requires("yaml-cpp/0.9.0")
2929
self.requires("robin-hood-hashing/3.11.5")
3030
self.requires("libcurl/8.20.0")
31+
self.requires("libnghttp2/1.61.0")
3132
if (
3233
"libc" not in self.settings.compiler.fields
3334
or self.settings.compiler.libc != "musl"
3435
):
3536
self.requires("sentry-crashpad/0.6.5")
3637

38+
def configure(self):
39+
self.options["libcurl"].with_nghttp2 = True
40+
3741
def build_requirements(self):
3842
self.tool_requires("protobuf/6.33.5")
3943

‎libs/CMakeLists.txt‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,4 +4,5 @@ add_subdirectory(visor_test)
44
add_subdirectory(visor_transaction)
55
add_subdirectory(visor_tcp)
66
add_subdirectory(visor_dns)
7-
add_subdirectory(visor_utils)
7+
add_subdirectory(visor_utils)
8+
add_subdirectory(visor_http_client)
Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
message(STATUS "Visor HTTP Client Library")
2+
3+
find_package(CURL REQUIRED)
4+
find_package(uvw REQUIRED)
5+
find_package(httplib REQUIRED)
6+
find_package(Catch2 REQUIRED)
7+
8+
add_library(VisorHttpClient STATIC HttpClient.cpp)
9+
# Namespaced alias for consistency with the other libs (Visor::Lib::Dns, Visor::Lib::Tcp, ...)
10+
# and cleaner downstream consumption.
11+
add_library(Visor::Lib::Http ALIAS VisorHttpClient)
12+
target_include_directories(VisorHttpClient PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
13+
target_link_libraries(VisorHttpClient PUBLIC CURL::libcurl uvw::uvw)
14+
15+
add_executable(unit-tests-visor-http-client test_http_client.cpp)
16+
target_link_libraries(unit-tests-visor-http-client PRIVATE VisorHttpClient Catch2::Catch2WithMain httplib::httplib)
17+
add_test(NAME unit-tests-visor-http-client COMMAND unit-tests-visor-http-client)

‎libs/visor_http_client/HttpClient.cpp‎

Lines changed: 384 additions & 0 deletions
Large diffs are not rendered by default.
Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
#pragma once
2+
#include "HttpTypes.h"
3+
#include <curl/curl.h>
4+
#include <functional>
5+
#include <memory>
6+
#include <optional>
7+
#include <unordered_map>
8+
#include <uvw/loop.h>
9+
#include <uvw/poll.h>
10+
#include <uvw/timer.h>
11+
12+
namespace visor::http {
13+
14+
// Threading contract: HttpClient is CONSTRUCTED on the control thread, before the
15+
// netprobe io thread is spawned (no concurrency at that point — same as _async_h/_timer
16+
// in NetProbeInputStream). Thereafter request(), the curl/uvw callbacks, and close()
17+
// run ONLY on the netprobe io (loop) thread. curl_multi is not thread-safe, so one
18+
// HttpClient is touched by exactly one thread for its whole active life.
19+
//
20+
// Reentrancy: the ResultCallback (on_done) MAY call request() — a follow-up request issued
21+
// from within a completion callback is safe. The `_processing` guard makes this work:
22+
// - check_multi_info() collects (callback, result) pairs FIRST, then fires callbacks; and
23+
// it sets _processing=true for the whole drain, so a re-entrant check_multi_info() (e.g.
24+
// triggered by a request() from inside a callback) returns immediately instead of
25+
// recursively mutating _easy/_sockets under the outer drain.
26+
// - request(), when called while _processing, does NOT drive curl_multi_socket_action /
27+
// check_multi_info() inline; it adds the handle and arms the timer for 0 ms so the kick
28+
// happens on the next loop iteration, after the outer drain unwinds.
29+
// close() from inside on_done is still discouraged: the drain loop uses local copies so it
30+
// won't crash, but tearing the client down mid-callback is confusing — prefer deferring it.
31+
class HttpClient
32+
{
33+
public:
34+
using ResultCallback = std::function<void(const HttpResult &)>;
35+
explicit HttpClient(std::shared_ptr<uvw::loop> loop);
36+
~HttpClient();
37+
void request(const HttpRequest &req, ResultCallback on_done);
38+
void close();
39+
40+
private:
41+
// per-easy-handle context (owns the result callback + a write sink)
42+
struct EasyContext {
43+
CURL *easy{nullptr};
44+
ResultCallback on_done;
45+
char errbuf[CURL_ERROR_SIZE]{};
46+
curl_slist *headers{nullptr}; // owned; freed in dtor (after curl_easy_cleanup)
47+
bool capture{false};
48+
std::string response; // captured body (bounded to 64 KB)
49+
~EasyContext() { if (headers) curl_slist_free_all(headers); }
50+
};
51+
// per-socket context: a uvw poll handle curl watches (owned in _sockets below)
52+
struct SocketContext {
53+
std::shared_ptr<uvw::poll_handle> poll;
54+
curl_socket_t sockfd{};
55+
};
56+
57+
static int socket_cb(CURL *easy, curl_socket_t s, int what, void *userp, void *socketp);
58+
static int timer_cb(CURLM *multi, long timeout_ms, void *userp);
59+
static size_t write_discard(char *ptr, size_t size, size_t nmemb, void *userdata);
60+
static size_t write_capture(char *ptr, size_t size, size_t nmemb, void *userdata);
61+
void on_socket_event(curl_socket_t sockfd, int events);
62+
void on_timeout();
63+
void check_multi_info();
64+
65+
std::shared_ptr<uvw::loop> _loop;
66+
CURLM *_multi{nullptr};
67+
bool _closed{false};
68+
bool _processing{false}; // true while check_multi_info() drains; guards re-entrant request()/check_multi_info()
69+
std::shared_ptr<uvw::timer_handle> _timer;
70+
std::unordered_map<CURL *, std::unique_ptr<EasyContext>> _easy;
71+
// we OWN the socket contexts here (curl_multi_assign stores the bare pointer);
72+
// close() sweeps these directly rather than relying on CURL_POLL_REMOVE firing for all.
73+
std::unordered_map<curl_socket_t, std::unique_ptr<SocketContext>> _sockets;
74+
};
75+
76+
// Validate that `url` is a well-formed http/https URL (uses libcurl's URL parser internally, so
77+
// callers don't reach into the curl API). Returns std::nullopt when valid; otherwise a short,
78+
// human-readable reason (e.g. "is not a valid http(s) URL: '<url>'") the caller can surface.
79+
std::optional<std::string> validate_http_url(const std::string &url);
80+
}

‎libs/visor_http_client/HttpTypes.h‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
#pragma once
2+
#include <cstdint>
3+
#include <string>
4+
#include <vector>
5+
6+
namespace visor::http {
7+
8+
struct HttpTimings {
9+
uint64_t total_us{0};
10+
uint64_t dns_us{0};
11+
uint64_t connect_us{0};
12+
uint64_t tls_us{0};
13+
uint64_t ttfb_us{0};
14+
};
15+
struct HttpRequest {
16+
std::string url;
17+
std::string method{"GET"};
18+
uint64_t timeout_ms{0};
19+
bool follow_redirects{true};
20+
bool verify_tls{true};
21+
std::string body; // request body bytes (empty => no body)
22+
std::vector<std::string> headers; // extra request headers, each "Key: Value"
23+
bool capture_response{false}; // when true, capture the response body
24+
};
25+
struct HttpResult {
26+
bool transport_ok{false};
27+
long curl_code{0};
28+
long status_code{0};
29+
HttpTimings timings;
30+
std::string response_body; // populated only when HttpRequest.capture_response
31+
std::string content_type; // raw response Content-Type header when transport_ok (compare case-insensitively)
32+
std::string error_msg; // human-readable curl error detail when !transport_ok
33+
};
34+
}
Lines changed: 234 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,234 @@
1+
#include "HttpClient.h"
2+
#include <catch2/catch_test_macros.hpp>
3+
#include <httplib.h>
4+
#include <uvw/loop.h>
5+
#include <thread>
6+
7+
using namespace visor::http;
8+
9+
// Spin a throwaway httplib server on an ephemeral port; returns port.
10+
static int start_test_server(httplib::Server &svr, std::thread &t)
11+
{
12+
svr.Get("/ok", [](const httplib::Request &, httplib::Response &res) { res.set_content("hi", "text/plain"); });
13+
svr.Get("/notfound", [](const httplib::Request &, httplib::Response &res) { res.status = 404; });
14+
svr.Get("/error", [](const httplib::Request &, httplib::Response &res) { res.status = 500; });
15+
svr.Get("/slow", [](const httplib::Request &, httplib::Response &res) {
16+
std::this_thread::sleep_for(std::chrono::milliseconds(500));
17+
res.set_content("late", "text/plain");
18+
});
19+
svr.Post("/echo", [](const httplib::Request &req, httplib::Response &res) {
20+
res.set_content(req.body, "application/octet-stream");
21+
});
22+
int port = svr.bind_to_any_port("127.0.0.1");
23+
REQUIRE(port > 0);
24+
t = std::thread([&svr] { svr.listen_after_bind(); });
25+
// Wait until the server is actually accepting connections before the test
26+
// connects — avoids a connect-before-listen race on slow/loaded CI machines.
27+
svr.wait_until_ready();
28+
return port;
29+
}
30+
31+
// Arm a watchdog timer that calls loop->stop() after timeout_ms.
32+
// The handle is unreferenced so loop->run() returns as soon as the real work
33+
// handles close — the watchdog fires only if the loop would otherwise stall
34+
// forever (hang-guard). The caller must close the returned handle after
35+
// loop->run() completes.
36+
static std::shared_ptr<uvw::timer_handle> arm_watchdog(std::shared_ptr<uvw::loop> loop, uint64_t timeout_ms)
37+
{
38+
auto wd = loop->resource<uvw::timer_handle>();
39+
wd->on<uvw::timer_event>([loop](const auto &, auto &) { loop->stop(); });
40+
wd->start(uvw::timer_handle::time{timeout_ms}, uvw::timer_handle::time{0});
41+
// Unreference: the watchdog must not prevent the loop from exiting when all
42+
// real work handles (curl poll + timer) have closed. uv_unref makes the
43+
// handle "idle" w.r.t. loop liveness — the loop exits when only unreferenced
44+
// handles remain active, then we close the watchdog in disarm_watchdog.
45+
wd->unreference();
46+
return wd;
47+
}
48+
49+
// Disarm the watchdog and drain its close on the loop.
50+
static void disarm_watchdog(std::shared_ptr<uvw::loop> loop, std::shared_ptr<uvw::timer_handle> wd)
51+
{
52+
if (wd && !wd->closing()) {
53+
wd->stop();
54+
wd->close();
55+
loop->run();
56+
}
57+
}
58+
59+
TEST_CASE("HttpClient basic results", "[http][client]")
60+
{
61+
httplib::Server svr;
62+
std::thread server_thread;
63+
int port = start_test_server(svr, server_thread);
64+
65+
auto loop = uvw::loop::create();
66+
HttpClient client(loop);
67+
std::string base = "http://127.0.0.1:" + std::to_string(port);
68+
69+
std::vector<HttpResult> results;
70+
auto on_done = [&](const HttpResult &r) { results.push_back(r); };
71+
// Named init (not positional aggregate) so adding HttpRequest fields doesn't warn.
72+
auto get_req = [](std::string url, uint64_t timeout_ms) {
73+
HttpRequest r;
74+
r.url = std::move(url);
75+
r.timeout_ms = timeout_ms;
76+
return r;
77+
};
78+
79+
SECTION("200 OK")
80+
{
81+
client.request(get_req(base + "/ok", 2000), on_done);
82+
auto wd = arm_watchdog(loop, 5000);
83+
loop->run();
84+
disarm_watchdog(loop, wd);
85+
REQUIRE(results.size() == 1);
86+
CHECK(results[0].transport_ok);
87+
CHECK(results[0].status_code == 200);
88+
const auto &t = results[0].timings;
89+
CHECK(t.total_us > 0);
90+
// Per-phase delta sanity (plain HTTP, no TLS): the TLS phase must be 0, and every
91+
// phase delta must stay within the total — guards the delta arithmetic against a
92+
// wrong base or an unsigned underflow producing a huge bogus value.
93+
CHECK(t.tls_us == 0);
94+
CHECK(t.dns_us <= t.total_us);
95+
CHECK(t.connect_us <= t.total_us);
96+
CHECK(t.ttfb_us <= t.total_us);
97+
}
98+
SECTION("404")
99+
{
100+
client.request(get_req(base + "/notfound", 2000), on_done);
101+
auto wd = arm_watchdog(loop, 5000);
102+
loop->run();
103+
disarm_watchdog(loop, wd);
104+
REQUIRE(results.size() == 1);
105+
CHECK(results[0].transport_ok);
106+
CHECK(results[0].status_code == 404);
107+
}
108+
SECTION("connection refused")
109+
{
110+
client.request(get_req("http://127.0.0.1:1/x", 2000), on_done);
111+
auto wd = arm_watchdog(loop, 5000);
112+
loop->run();
113+
disarm_watchdog(loop, wd);
114+
REQUIRE(results.size() == 1);
115+
CHECK_FALSE(results[0].transport_ok);
116+
}
117+
SECTION("timeout")
118+
{
119+
client.request(get_req(base + "/slow", 100), on_done);
120+
auto wd = arm_watchdog(loop, 5000);
121+
loop->run();
122+
disarm_watchdog(loop, wd);
123+
REQUIRE(results.size() == 1);
124+
CHECK_FALSE(results[0].transport_ok);
125+
CHECK(results[0].curl_code == CURLE_OPERATION_TIMEDOUT);
126+
}
127+
SECTION("close while in flight")
128+
{
129+
// Issue the slow request but then immediately close() before loop->run() completes it.
130+
// The process must not crash and results must stay empty (interrupted = no metric recorded).
131+
client.request(get_req(base + "/slow", 5000), on_done);
132+
client.close();
133+
auto wd = arm_watchdog(loop, 3000);
134+
loop->run();
135+
disarm_watchdog(loop, wd);
136+
CHECK(results.empty());
137+
}
138+
139+
// close() is idempotent — this is a no-op for the "close while in flight" section.
140+
client.close();
141+
loop->run(); // drain handle closes
142+
svr.stop();
143+
if (server_thread.joinable()) server_thread.join();
144+
}
145+
146+
TEST_CASE("HttpClient POST body + headers + response capture", "[http][client]")
147+
{
148+
httplib::Server svr;
149+
std::thread server_thread;
150+
int port = start_test_server(svr, server_thread);
151+
auto loop = uvw::loop::create();
152+
HttpClient client(loop);
153+
std::string base = "http://127.0.0.1:" + std::to_string(port);
154+
155+
std::vector<HttpResult> results;
156+
auto on_done = [&](const HttpResult &r) { results.push_back(r); };
157+
158+
SECTION("POST echoes body when capture_response")
159+
{
160+
HttpRequest req;
161+
req.url = base + "/echo";
162+
req.method = "POST";
163+
req.body = std::string("\x00\x01hello", 7); // binary-safe
164+
req.headers = {"Content-Type: application/octet-stream"};
165+
req.capture_response = true;
166+
req.timeout_ms = 2000;
167+
client.request(req, on_done);
168+
auto wd = arm_watchdog(loop, 5000);
169+
loop->run();
170+
disarm_watchdog(loop, wd);
171+
REQUIRE(results.size() == 1);
172+
CHECK(results[0].transport_ok);
173+
CHECK(results[0].status_code == 200);
174+
CHECK(results[0].response_body == std::string("\x00\x01hello", 7));
175+
}
176+
SECTION("body NOT captured by default")
177+
{
178+
HttpRequest req;
179+
req.url = base + "/ok";
180+
req.capture_response = false;
181+
req.timeout_ms = 2000;
182+
client.request(req, on_done);
183+
auto wd = arm_watchdog(loop, 5000);
184+
loop->run();
185+
disarm_watchdog(loop, wd);
186+
REQUIRE(results.size() == 1);
187+
CHECK(results[0].response_body.empty());
188+
}
189+
190+
client.close();
191+
loop->run();
192+
svr.stop();
193+
if (server_thread.joinable()) server_thread.join();
194+
}
195+
196+
TEST_CASE("HttpClient request() issued from within a completion callback is safe", "[http][client]")
197+
{
198+
httplib::Server svr;
199+
std::thread server_thread;
200+
int port = start_test_server(svr, server_thread);
201+
202+
auto loop = uvw::loop::create();
203+
HttpClient client(loop);
204+
std::string base = "http://127.0.0.1:" + std::to_string(port);
205+
206+
std::vector<long> statuses;
207+
// In the first request's completion callback, issue a SECOND request (reentrant). The
208+
// _processing guard must keep this safe and let both complete.
209+
auto second = [&](const HttpResult &r) { statuses.push_back(r.status_code); };
210+
auto first = [&](const HttpResult &r) {
211+
statuses.push_back(r.status_code);
212+
HttpRequest r2;
213+
r2.url = base + "/notfound";
214+
r2.timeout_ms = 2000;
215+
client.request(r2, second); // reentrant: called from inside check_multi_info()'s callback fire
216+
};
217+
HttpRequest r1;
218+
r1.url = base + "/ok";
219+
r1.timeout_ms = 2000;
220+
client.request(r1, first);
221+
222+
auto wd = arm_watchdog(loop, 5000);
223+
loop->run();
224+
disarm_watchdog(loop, wd);
225+
226+
REQUIRE(statuses.size() == 2);
227+
CHECK(statuses[0] == 200);
228+
CHECK(statuses[1] == 404);
229+
230+
client.close();
231+
loop->run();
232+
svr.stop();
233+
if (server_thread.joinable()) server_thread.join();
234+
}

0 commit comments

Comments
 (0)