From a5458c50c4808b3070bd8a1a02eb2eb335167986 Mon Sep 17 00:00:00 2001 From: Charles-Henri Bruyand Date: Thu, 13 Feb 2025 11:00:11 +0100 Subject: [PATCH] dnsdist: add support for dnstap new http_protocol field --- pdns/dnsdistdist/dnsdist-actions-factory.cc | 29 +++++---- pdns/dnstap.cc | 10 ++- pdns/dnstap.hh | 9 ++- pdns/dnstap.proto | 23 +++++-- regression-tests.dnsdist/test_Dnstap.py | 70 ++++++++++++++++++--- 5 files changed, 111 insertions(+), 30 deletions(-) diff --git a/pdns/dnsdistdist/dnsdist-actions-factory.cc b/pdns/dnsdistdist/dnsdist-actions-factory.cc index 21e6dd3ab89a..26614c34a3ea 100644 --- a/pdns/dnsdistdist/dnsdist-actions-factory.cc +++ b/pdns/dnsdistdist/dnsdist-actions-factory.cc @@ -1433,28 +1433,31 @@ class SetECSAction : public DNSAction }; #ifndef DISABLE_PROTOBUF -static DnstapMessage::ProtocolType ProtocolToDNSTap(dnsdist::Protocol protocol) +std::tuple> ProtocolToDNSTap(dnsdist::Protocol protocol) { if (protocol == dnsdist::Protocol::DoUDP) { - return DnstapMessage::ProtocolType::DoUDP; + return {DnstapMessage::ProtocolType::DoUDP, boost::none}; } if (protocol == dnsdist::Protocol::DoTCP) { - return DnstapMessage::ProtocolType::DoTCP; + return {DnstapMessage::ProtocolType::DoTCP, boost::none}; } if (protocol == dnsdist::Protocol::DoT) { - return DnstapMessage::ProtocolType::DoT; + return {DnstapMessage::ProtocolType::DoT, boost::none}; } - if (protocol == dnsdist::Protocol::DoH || protocol == dnsdist::Protocol::DoH3) { - return DnstapMessage::ProtocolType::DoH; + if (protocol == dnsdist::Protocol::DoH) { + return {DnstapMessage::ProtocolType::DoH, DnstapMessage::HttpProtocolType::HTTP2}; + } + if (protocol == dnsdist::Protocol::DoH3) { + return {DnstapMessage::ProtocolType::DoH, DnstapMessage::HttpProtocolType::HTTP3}; } if (protocol == dnsdist::Protocol::DNSCryptUDP) { - return DnstapMessage::ProtocolType::DNSCryptUDP; + return {DnstapMessage::ProtocolType::DNSCryptUDP, boost::none}; } if (protocol == dnsdist::Protocol::DNSCryptTCP) { - return DnstapMessage::ProtocolType::DNSCryptTCP; + return {DnstapMessage::ProtocolType::DNSCryptTCP, boost::none}; } if (protocol == dnsdist::Protocol::DoQ) { - return DnstapMessage::ProtocolType::DoQ; + return {DnstapMessage::ProtocolType::DoQ, boost::none}; } throw std::runtime_error("Unhandled protocol for dnstap: " + protocol.toPrettyString()); } @@ -1493,9 +1496,9 @@ class DnstapLogAction : public DNSAction, public boost::noncopyable static thread_local std::string data; data.clear(); - DnstapMessage::ProtocolType protocol = ProtocolToDNSTap(dnsquestion->getProtocol()); + auto [protocol, httpProtocol] = ProtocolToDNSTap(dnsquestion->getProtocol()); // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) - DnstapMessage message(std::move(data), !dnsquestion->getHeader()->qr ? DnstapMessage::MessageType::client_query : DnstapMessage::MessageType::client_response, d_identity, &dnsquestion->ids.origRemote, &dnsquestion->ids.origDest, protocol, reinterpret_cast(dnsquestion->getData().data()), dnsquestion->getData().size(), &dnsquestion->getQueryRealTime(), nullptr); + DnstapMessage message(std::move(data), !dnsquestion->getHeader()->qr ? DnstapMessage::MessageType::client_query : DnstapMessage::MessageType::client_response, d_identity, &dnsquestion->ids.origRemote, &dnsquestion->ids.origDest, protocol, reinterpret_cast(dnsquestion->getData().data()), dnsquestion->getData().size(), &dnsquestion->getQueryRealTime(), nullptr, boost::none, httpProtocol); { if (d_alterFunc) { auto lock = g_lua.lock(); @@ -1699,9 +1702,9 @@ class DnstapLogResponseAction : public DNSResponseAction, public boost::noncopya gettime(&now, true); data.clear(); - DnstapMessage::ProtocolType protocol = ProtocolToDNSTap(response->getProtocol()); + auto [protocol, httpProtocol] = ProtocolToDNSTap(response->getProtocol()); // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) - DnstapMessage message(std::move(data), DnstapMessage::MessageType::client_response, d_identity, &response->ids.origRemote, &response->ids.origDest, protocol, reinterpret_cast(response->getData().data()), response->getData().size(), &response->getQueryRealTime(), &now); + DnstapMessage message(std::move(data), DnstapMessage::MessageType::client_response, d_identity, &response->ids.origRemote, &response->ids.origDest, protocol, reinterpret_cast(response->getData().data()), response->getData().size(), &response->getQueryRealTime(), &now, boost::none, httpProtocol); { if (d_alterFunc) { auto lock = g_lua.lock(); diff --git a/pdns/dnstap.cc b/pdns/dnstap.cc index b032da987663..dffd30d27a8b 100644 --- a/pdns/dnstap.cc +++ b/pdns/dnstap.cc @@ -1,6 +1,7 @@ #include #include "config.h" #include "gettime.hh" +#include "protozero/types.hpp" #include "dnstap.hh" #ifndef DISABLE_PROTOBUF @@ -53,7 +54,9 @@ enum : protozero::pbf_tag_type query_zone = 11, response_time_sec = 12, response_time_nsec = 13, - response_message = 14 + response_message = 14, + policy = 15, + http_protocol = 16, }; } @@ -62,7 +65,7 @@ std::string&& DnstapMessage::getBuffer() return std::move(d_buffer); } -DnstapMessage::DnstapMessage(std::string&& buffer, DnstapMessage::MessageType type, const std::string& identity, const ComboAddress* requestor, const ComboAddress* responder, DnstapMessage::ProtocolType protocol, const char* packet, const size_t len, const struct timespec* queryTime, const struct timespec* responseTime, const boost::optional& auth) : +DnstapMessage::DnstapMessage(std::string&& buffer, DnstapMessage::MessageType type, const std::string& identity, const ComboAddress* requestor, const ComboAddress* responder, DnstapMessage::ProtocolType protocol, const char* packet, const size_t len, const struct timespec* queryTime, const struct timespec* responseTime, const boost::optional& auth, const boost::optional httpProtocol) : d_buffer(std::move(buffer)) { protozero::pbf_writer pbf{d_buffer}; @@ -128,6 +131,9 @@ DnstapMessage::DnstapMessage(std::string&& buffer, DnstapMessage::MessageType ty pbf_message.add_bytes(DnstapMessageFields::response_message, packet, len); } } + if (httpProtocol) { + pbf_message.add_enum(DnstapMessageFields::http_protocol, static_cast(*httpProtocol)); + } if (auth) { pbf_message.add_bytes(DnstapMessageFields::query_zone, auth->toDNSString()); diff --git a/pdns/dnstap.hh b/pdns/dnstap.hh index d82ce9a7d73b..070e2d03052e 100644 --- a/pdns/dnstap.hh +++ b/pdns/dnstap.hh @@ -22,6 +22,7 @@ #pragma once #include +#include #include #include "config.h" @@ -59,8 +60,14 @@ public: DNSCryptTCP = 6, DoQ = 7 }; + enum class HttpProtocolType : uint32_t + { + HTTP1 = 1, + HTTP2 = 2, + HTTP3 = 3, + }; - DnstapMessage(std::string&& buffer, MessageType type, const std::string& identity, const ComboAddress* requestor, const ComboAddress* responder, ProtocolType protocol, const char* packet, size_t len, const struct timespec* queryTime, const struct timespec* responseTime, const boost::optional& auth = boost::none); + DnstapMessage(std::string&& buffer, MessageType type, const std::string& identity, const ComboAddress* requestor, const ComboAddress* responder, ProtocolType protocol, const char* packet, size_t len, const struct timespec* queryTime, const struct timespec* responseTime, const boost::optional& auth = boost::none, const boost::optional httpProtocol = boost::none); void setExtra(const std::string& extra); std::string&& getBuffer(); diff --git a/pdns/dnstap.proto b/pdns/dnstap.proto index 3780c4934f2b..4e98683036fd 100644 --- a/pdns/dnstap.proto +++ b/pdns/dnstap.proto @@ -3,7 +3,7 @@ // This file contains the protobuf schemas for the "dnstap" structured event // replication format for DNS software. -// Written in 2013-2014 by Farsight Security, Inc. +// Written in 2013-2025 by the dnstap contributors. // // To the extent possible under law, the author(s) have dedicated all // copyright and related and neighboring rights to this file to the public @@ -12,7 +12,7 @@ // You should have received a copy of the CC0 Public Domain Dedication along // with this file. If not, see: // -// . +// . syntax = "proto2"; package dnstap; @@ -64,6 +64,15 @@ enum SocketProtocol { DOH = 4; // DNS over HTTPS (RFC 8484) DNSCryptUDP = 5; // DNSCrypt over UDP (https://dnscrypt.info/protocol) DNSCryptTCP = 6; // DNSCrypt over TCP (https://dnscrypt.info/protocol) + DOQ = 7; // DNS over QUIC (RFC 9250) +} + +// HttpProtocol: the HTTP protocol version used to transport a DNS message over +// an HTTP-based protocol such as DNS over HTTPS. +enum HttpProtocol { + HTTP1 = 1; // HTTP/1 + HTTP2 = 2; // HTTP/2 + HTTP3 = 3; // HTTP/3 } // Policy: information about any name server operator policy @@ -215,13 +224,13 @@ message Message { // tool from a DNS server, from the perspective of the tool. TOOL_RESPONSE = 12; - // UPDATE_QUERY is a DNS update query message received from a resolver + // UPDATE_QUERY is a Dynamic DNS Update request (RFC 2136) received // by an authoritative name server, from the perspective of the // authoritative name server. UPDATE_QUERY = 13; - // UPDATE_RESPONSE is a DNS update response message sent from an - // authoritative name server to a resolver, from the perspective of the + // UPDATE_RESPONSE is a Dynamic DNS Update response (RFC 2136) sent + // from an authoritative name server, from the perspective of the // authoritative name server. UPDATE_RESPONSE = 14; } @@ -284,6 +293,10 @@ message Message { // Operator policy applied to the processing of this message, if any. optional Policy policy = 15; + + // One of the HttpProtocol values described above. This field should only be + // set if socket_protocol is set to DOH. + optional HttpProtocol http_protocol = 16; } // All fields except for 'type' in the Message schema are optional. diff --git a/regression-tests.dnsdist/test_Dnstap.py b/regression-tests.dnsdist/test_Dnstap.py index c9457afbe6d8..9800f4241dc9 100644 --- a/regression-tests.dnsdist/test_Dnstap.py +++ b/regression-tests.dnsdist/test_Dnstap.py @@ -17,7 +17,7 @@ FSTRM_CONTROL_FINISH = 0x05 -def checkDnstapBase(testinstance, dnstap, protocol, initiator): +def checkDnstapBase(testinstance, dnstap, protocol, initiator, response_port): testinstance.assertTrue(dnstap) testinstance.assertTrue(dnstap.HasField('identity')) testinstance.assertEqual(dnstap.identity, b'a.server') @@ -35,16 +35,24 @@ def checkDnstapBase(testinstance, dnstap, protocol, initiator): testinstance.assertTrue(dnstap.message.HasField('response_address')) testinstance.assertEqual(socket.inet_ntop(socket.AF_INET, dnstap.message.response_address), initiator) testinstance.assertTrue(dnstap.message.HasField('response_port')) - testinstance.assertEqual(dnstap.message.response_port, testinstance._dnsDistPort) + testinstance.assertEqual(dnstap.message.response_port, response_port) -def checkDnstapQuery(testinstance, dnstap, protocol, query, initiator='127.0.0.1'): +def checkDnstapQuery(testinstance, dnstap, protocol, query, initiator='127.0.0.1', response_port=0, http_protocol=0): + testinstance.assertEqual(dnstap.message.type, dnstap_pb2.Message.CLIENT_QUERY) - checkDnstapBase(testinstance, dnstap, protocol, initiator) + if response_port == 0 : + response_port = testinstance._dnsDistPort + + checkDnstapBase(testinstance, dnstap, protocol, initiator, response_port) testinstance.assertTrue(dnstap.message.HasField('query_time_sec')) testinstance.assertTrue(dnstap.message.HasField('query_time_nsec')) + if http_protocol != 0 : + testinstance.assertTrue(dnstap.message.HasField('http_protocol')) + testinstance.assertEqual(dnstap.message.http_protocol, http_protocol) + testinstance.assertTrue(dnstap.message.HasField('query_message')) wire_message = dns.message.from_wire(dnstap.message.query_message) testinstance.assertEqual(wire_message, query) @@ -54,14 +62,15 @@ def checkDnstapExtra(testinstance, dnstap, expected): testinstance.assertTrue(dnstap.HasField('extra')) testinstance.assertEqual(dnstap.extra, expected) - def checkDnstapNoExtra(testinstance, dnstap): testinstance.assertFalse(dnstap.HasField('extra')) -def checkDnstapResponse(testinstance, dnstap, protocol, response, initiator='127.0.0.1'): +def checkDnstapResponse(testinstance, dnstap, protocol, response, initiator='127.0.0.1', response_port=0): testinstance.assertEqual(dnstap.message.type, dnstap_pb2.Message.CLIENT_RESPONSE) - checkDnstapBase(testinstance, dnstap, protocol, initiator) + if response_port == 0 : + response_port = testinstance._dnsDistPort + checkDnstapBase(testinstance, dnstap, protocol, initiator, response_port) testinstance.assertTrue(dnstap.message.HasField('query_time_sec')) testinstance.assertTrue(dnstap.message.HasField('query_time_nsec')) @@ -574,16 +583,25 @@ def fstrm_handle_bidir_connection(conn, on_data, exit_early=False): if exit_early: break - class TestDnstapOverFrameStreamUnixLogger(DNSDistTest): + _serverKey = 'server.key' + _serverCert = 'server.chain' + _serverName = 'tls.tests.dnsdist.org' + _caCert = 'ca.pem' + _dohServerPort = pickAvailablePort() + _doh3ServerPort = pickAvailablePort() + _dohBaseURL = ("https://%s:%d/" % (_serverName, _dohServerPort)) + _fstrmLoggerAddress = '/tmp/fslutest.sock' _fstrmLoggerQueue = Queue() _fstrmLoggerCounter = 0 - _config_params = ['_testServerPort', '_fstrmLoggerAddress'] + _config_params = ['_testServerPort', '_fstrmLoggerAddress', '_dohServerPort', '_serverCert', '_serverKey', '_doh3ServerPort', '_serverCert', '_serverKey'] _config_template = """ newServer{address="127.0.0.1:%s", useClientSubnet=true} fslu = newFrameStreamUnixLogger('%s') + addDOHLocal("127.0.0.1:%d", "%s", "%s", { "/" }, {}) + addDOH3Local("127.0.0.1:%d", "%s", "%s") addAction(AllRule(), DnstapLogAction("a.server", fslu)) """ @@ -660,6 +678,40 @@ def testDnstapOverFrameStreamUnix(self): checkDnstapQuery(self, dnstap, dnstap_pb2.UDP, query) checkDnstapNoExtra(self, dnstap) + def testDnstapHttpProtocol(self): + """ + DOH and DOH3: Make sure http protocol field is correctly set + """ + name = 'simple.doh.tests.powerdns.com.' + query = dns.message.make_query(name, 'A', 'IN', use_edns=False) + query.id = 0 + expectedQuery = dns.message.make_query(name, 'A', 'IN', use_edns=True, payload=4096) + expectedQuery.id = 0 + response = dns.message.make_response(query) + rrset = dns.rrset.from_text(name, + 3600, + dns.rdataclass.IN, + dns.rdatatype.A, + '127.0.0.1') + response.answer.append(rrset) + + protocols = [ + {"method": "sendDOH3QueryWrapper", "port": self._doh3ServerPort, "expected_protocol": dnstap_pb2.HttpProtocol.HTTP3}, + {"method": "sendDOHQueryWrapper", "port": self._dohServerPort, "expected_protocol": dnstap_pb2.HttpProtocol.HTTP2}, + ] + for protocol in protocols : + sender = getattr(self, protocol["method"]) + (receivedQuery, receivedResponse) = sender(query, response) + receivedQuery.id = query.id + self.assertEqual(query, receivedQuery) + self.assertEqual(response, receivedResponse) + + # check the dnstap message corresponding to the UDP query + dnstap = self.getFirstDnstap() + + checkDnstapQuery(self, dnstap, dnstap_pb2.DOH, query, '127.0.0.1', protocol["port"], protocol["expected_protocol"]) + checkDnstapNoExtra(self, dnstap) + class TestDnstapOverRemotePoolUnixLogger(DNSDistTest): _fstrmLoggerAddress = '/tmp/fslutest.sock' _fstrmLoggerQueue = Queue()