From d80f72638708982f6ebbb655da1104a614de8cff Mon Sep 17 00:00:00 2001 From: Masaori Koshiba Date: Thu, 19 Nov 2020 12:51:46 +0900 Subject: [PATCH] Cleanup incoming PROXY Protocol v1 --- iocore/net/I_NetVConnection.h | 91 ++-------------- iocore/net/NetVConnection.cc | 57 ++++++++++ iocore/net/P_NetVConnection.h | 3 +- iocore/net/ProxyProtocol.cc | 161 +++++++++++++--------------- iocore/net/ProxyProtocol.h | 51 ++++----- iocore/net/SSLNetVConnection.cc | 6 +- proxy/ProtocolProbeSessionAccept.cc | 6 +- proxy/http/HttpTransact.cc | 10 +- proxy/http/HttpTransact.h | 2 +- proxy/logging/LogAccess.cc | 12 +-- 10 files changed, 188 insertions(+), 211 deletions(-) diff --git a/iocore/net/I_NetVConnection.h b/iocore/net/I_NetVConnection.h index c2aa346236a..69e49ee585b 100644 --- a/iocore/net/I_NetVConnection.h +++ b/iocore/net/I_NetVConnection.h @@ -23,6 +23,8 @@ */ #pragma once +#include "ProxyProtocol.h" + #include #include @@ -814,116 +816,45 @@ class NetVConnection : public VConnection, public PluginUserArgsget_proxy_protocol_addr(ProxyProtocolData::SRC)); } sockaddr const * - get_proxy_protocol_dst_addr() + get_proxy_protocol_dst_addr() const { return get_proxy_protocol_addr(ProxyProtocolData::DST); } uint16_t - get_proxy_protocol_dst_port() + get_proxy_protocol_dst_port() const { return ats_ip_port_host_order(this->get_proxy_protocol_addr(ProxyProtocolData::DST)); }; - struct ProxyProtocol { - ProxyProtocolVersion proxy_protocol_version = ProxyProtocolVersion::UNDEFINED; - uint16_t ip_family; - IpEndpoint src_addr; - IpEndpoint dst_addr; - }; - - ProxyProtocol pp_info; + bool has_proxy_protocol(IOBufferReader *); + bool has_proxy_protocol(char *, int64_t *); protected: IpEndpoint local_addr; IpEndpoint remote_addr; + ProxyProtocol pp_info; bool got_local_addr = false; bool got_remote_addr = false; diff --git a/iocore/net/NetVConnection.cc b/iocore/net/NetVConnection.cc index a4f92399cba..512a1ce48ad 100644 --- a/iocore/net/NetVConnection.cc +++ b/iocore/net/NetVConnection.cc @@ -33,6 +33,9 @@ #include "P_Net.h" #include "ts/apidefs.h" +//// +// NetVConnection +// Action * NetVConnection::send_OOB(Continuation *, char *, int) { @@ -45,6 +48,60 @@ NetVConnection::cancel_OOB() return; } +/** + PROXY Protocol check with IOBufferReader + + If the buffer has PROXY Protocol, it will be consumed by this function. + */ +bool +NetVConnection::has_proxy_protocol(IOBufferReader *reader) +{ + char buf[PPv1_CONNECTION_HEADER_LEN_MAX + 1]; + ts::TextView tv; + tv.assign(buf, reader->memcpy(buf, sizeof(buf), 0)); + + size_t len = proxy_protocol_parse(&this->pp_info, tv); + + if (len > 0) { + reader->consume(len); + return true; + } + + return false; +} + +/** + PROXY Protocol check with buffer + + If the buffer has PROXY Protocol, it will be consumed by this function. + */ +bool +NetVConnection::has_proxy_protocol(char *buffer, int64_t *bytes_r) +{ + ts::TextView tv; + tv.assign(buffer, *bytes_r); + + size_t len = proxy_protocol_parse(&this->pp_info, tv); + + if (len <= 0) { + *bytes_r = -EAGAIN; + return false; + } + + *bytes_r -= len; + if (*bytes_r <= 0) { + *bytes_r = -EAGAIN; + } else { + Debug("ssl", "Moving %" PRId64 " characters remaining in the buffer from %p to %p", *bytes_r, buffer + len, buffer); + memmove(buffer, buffer + len, *bytes_r); + } + + return true; +} + +//// +// NetVCOptions +// std::string_view NetVCOptions::get_proto_string() const { diff --git a/iocore/net/P_NetVConnection.h b/iocore/net/P_NetVConnection.h index ff3c91a4c95..3f379af2fb4 100644 --- a/iocore/net/P_NetVConnection.h +++ b/iocore/net/P_NetVConnection.h @@ -22,12 +22,13 @@ */ #include "I_NetVConnection.h" +#include "ProxyProtocol.h" inline sockaddr const * NetVConnection::get_remote_addr() { if (!got_remote_addr) { - if (pp_info.proxy_protocol_version != ProxyProtocolVersion::UNDEFINED) { + if (pp_info.version != ProxyProtocolVersion::UNDEFINED) { set_remote_addr(get_proxy_protocol_src_addr()); } else { set_remote_addr(); diff --git a/iocore/net/ProxyProtocol.cc b/iocore/net/ProxyProtocol.cc index 83c3bcf9dfe..c0cd45a003a 100644 --- a/iocore/net/ProxyProtocol.cc +++ b/iocore/net/ProxyProtocol.cc @@ -21,87 +21,39 @@ * limitations under the License. */ -#include "tscore/ink_assert.h" -#include "tscpp/util/TextView.h" #include "ProxyProtocol.h" + +#include "I_EventSystem.h" #include "I_NetVConnection.h" -bool -ssl_has_proxy_v1(NetVConnection *sslvc, char *buffer, int64_t *bytes_r) +#include "tscore/ink_assert.h" +#include "tscpp/util/TextView.h" + +namespace { - ts::TextView tv; +using namespace std::literals; - tv.assign(buffer, *bytes_r); +constexpr ts::TextView PPv1_CONNECTION_PREFACE = "PROXY"sv; +constexpr ts::TextView PPv2_CONNECTION_PREFACE = "\x0D\x0A\x0D\x0A\x00\x0D\x0A\x51\x55\x49\x54\x0A\x02"sv; - // Client must send at least 15 bytes to get a reasonable match. - if (tv.size() < PROXY_V1_CONNECTION_HEADER_LEN_MIN) { - Debug("proxyprotocol_v1", "ssl_has_proxy_v1: not enough recv'd"); - return false; - } +constexpr size_t PPv1_CONNECTION_HEADER_LEN_MIN = 15; +constexpr size_t PPv2_CONNECTION_HEADER_LEN_MIN = 16; - // if we don't have the PROXY preface, we don't have a ProxyV1 header - if (0 != memcmp(PROXY_V1_CONNECTION_PREFACE, buffer, PROXY_V1_CONNECTION_PREFACE_LEN)) { - Debug("proxyprotocol_v1", "ssl_has_proxy_v1: failed the memcmp(%s, %s, %lu)", PROXY_V1_CONNECTION_PREFACE, buffer, - PROXY_V1_CONNECTION_PREFACE_LEN); - return false; - } +/** + PROXY Protocol v1 Parser + @return read length + */ +size_t +proxy_protocol_v1_parse(ProxyProtocol *pp_info, ts::TextView hdr) +{ // Find the terminating newline - ts::TextView::size_type pos = tv.find('\n'); - if (pos == tv.npos) { + ts::TextView::size_type pos = hdr.find('\n'); + if (pos == hdr.npos) { Debug("proxyprotocol_v1", "ssl_has_proxy_v1: newline not found"); - return false; - } - - // Parse the TextView before moving the bytes in the buffer - if (!proxy_protov1_parse(sslvc, tv)) { - *bytes_r = -EAGAIN; - return false; - } - *bytes_r -= pos + 1; - if (*bytes_r <= 0) { - *bytes_r = -EAGAIN; - } else { - Debug("ssl", "Moving %" PRId64 " characters remaining in the buffer from %p to %p", *bytes_r, buffer + pos + 1, buffer); - memmove(buffer, buffer + pos + 1, *bytes_r); + return 0; } - return true; -} -bool -http_has_proxy_v1(IOBufferReader *reader, NetVConnection *netvc) -{ - char buf[PROXY_V1_CONNECTION_HEADER_LEN_MAX + 1]; - ts::TextView tv; - - tv.assign(buf, reader->memcpy(buf, sizeof(buf), 0)); - - // Client must send at least 15 bytes to get a reasonable match. - if (tv.size() < PROXY_V1_CONNECTION_HEADER_LEN_MIN) { - return false; - } - - if (0 != memcmp(PROXY_V1_CONNECTION_PREFACE, buf, PROXY_V1_CONNECTION_PREFACE_LEN)) { - return false; - } - - // Find the terminating LF, which should already be in the buffer. - ts::TextView::size_type pos = tv.find('\n'); - if (pos == tv.npos) { // not found, it's not a proxy protocol header. - return false; - } - reader->consume(pos + 1); // clear out the header. - - // Now that we know we have a valid PROXY V1 preface, let's parse the - // remainder of the header - - return proxy_protov1_parse(netvc, tv); -} - -bool -proxy_protov1_parse(NetVConnection *netvc, ts::TextView hdr) -{ - static const std::string_view PREFACE{PROXY_V1_CONNECTION_PREFACE, PROXY_V1_CONNECTION_PREFACE_LEN}; ts::TextView token; in_port_t port; @@ -109,17 +61,17 @@ proxy_protov1_parse(NetVConnection *netvc, ts::TextView hdr) // The header should begin with the PROXY preface token = hdr.split_prefix_at(' '); - if (0 == token.size() || token != PREFACE) { + if (0 == token.size() || token != PPv1_CONNECTION_PREFACE) { Debug("proxyprotocol_v1", "proxy_protov1_parse: header [%.*s] does not start with preface [%.*s]", static_cast(hdr.size()), - hdr.data(), static_cast(PREFACE.size()), PREFACE.data()); - return false; + hdr.data(), static_cast(PPv1_CONNECTION_PREFACE.size()), PPv1_CONNECTION_PREFACE.data()); + return 0; } Debug("proxyprotocol_v1", "proxy_protov1_parse: [%.*s] = PREFACE", static_cast(token.size()), token.data()); // The INET protocol family - TCP4, TCP6 or UNKNOWN token = hdr.split_prefix_at(' '); if (0 == token.size()) { - return false; + return 0; } Debug("proxyprotocol_v1", "proxy_protov1_parse: [%.*s] = INET Family", static_cast(token.size()), token.data()); @@ -127,53 +79,88 @@ proxy_protov1_parse(NetVConnection *netvc, ts::TextView hdr) // - 255.255.255.255 or ffff:f...f:ffff ffff:f...f:fff token = hdr.split_prefix_at(' '); if (0 == token.size()) { - return false; + return 0; } Debug("proxyprotocol_v1", "proxy_protov1_parse: [%.*s] = Source Address", static_cast(token.size()), token.data()); - if (0 != netvc->set_proxy_protocol_src_addr(token)) { - return false; + if (0 != ats_ip_pton(token, &pp_info->src_addr)) { + return 0; } // Next is the layer3 destination address // - 255.255.255.255 or ffff:f...f:ffff ffff:f...f:fff token = hdr.split_prefix_at(' '); if (0 == token.size()) { - return false; + return 0; } Debug("proxyprotocol_v1", "proxy_protov1_parse: [%.*s] = Destination Address", static_cast(token.size()), token.data()); - if (0 != netvc->set_proxy_protocol_dst_addr(token)) { - return false; + if (0 != ats_ip_pton(token, &pp_info->dst_addr)) { + return 0; } // Next is the TCP source port represented as a decimal number in the range of [0..65535] inclusive. token = hdr.split_prefix_at(' '); if (0 == token.size()) { - return false; + return 0; } Debug("proxyprotocol_v1", "proxy_protov1_parse: [%.*s] = Source Port", static_cast(token.size()), token.data()); if (0 == (port = ts::svtoi(token))) { Debug("proxyprotocol_v1", "proxy_protov1_parse: src port [%d] token [%.*s] failed to parse", port, static_cast(token.size()), token.data()); - return false; + return 0; } - netvc->set_proxy_protocol_src_port(port); + pp_info->src_addr.port() = htons(port); // Next is the TCP destination port represented as a decimal number in the range of [0..65535] inclusive. // Final trailer is CR LF so split at CR. token = hdr.split_prefix_at('\r'); if (0 == token.size()) { - return false; + return 0; } Debug("proxyprotocol_v1", "proxy_protov1_parse: [%.*s] = Destination Port", static_cast(token.size()), token.data()); if (0 == (port = ts::svtoi(token))) { Debug("proxyprotocol_v1", "proxy_protov1_parse: dst port [%d] token [%.*s] failed to parse", port, static_cast(token.size()), token.data()); - return false; + return 0; } - netvc->set_proxy_protocol_dst_port(port); + pp_info->dst_addr.port() = htons(port); + + pp_info->version = ProxyProtocolVersion::V1; + + return pos + 1; +} + +/** + PROXY Protocol v2 Parser - netvc->set_proxy_protocol_version(NetVConnection::ProxyProtocolVersion::V1); + @return read length + */ +size_t +proxy_protocol_v2_parse(ProxyProtocol * /*pp_info*/, ts::TextView /*hdr*/) +{ + return 0; +} +} // namespace + +/** + PROXY Protocol Parser + */ +size_t +proxy_protocol_parse(ProxyProtocol *pp_info, ts::TextView tv) +{ + size_t len = 0; + + // Parse the TextView before moving the bytes in the buffer + if (tv.size() >= PPv1_CONNECTION_HEADER_LEN_MIN && PPv1_CONNECTION_PREFACE.isPrefixOf(tv)) { + // Client must send at least 15 bytes to get a reasonable match. + len = proxy_protocol_v1_parse(pp_info, tv); + } else if (tv.size() >= PPv2_CONNECTION_HEADER_LEN_MIN && PPv2_CONNECTION_PREFACE.isPrefixOf(tv)) { + len = proxy_protocol_v2_parse(pp_info, tv); + } else { + // if we don't have the PROXY preface, we don't have a ProxyProtocol header + // TODO: print hexdump of buffer safely + Debug("proxyprotocol", "failed to find ProxyProtocol preface"); + } - return true; + return len; } diff --git a/iocore/net/ProxyProtocol.h b/iocore/net/ProxyProtocol.h index f5961af4e6a..84491bc6b14 100644 --- a/iocore/net/ProxyProtocol.h +++ b/iocore/net/ProxyProtocol.h @@ -25,28 +25,29 @@ #pragma once -#include "tscore/ink_defs.h" -#include "tscore/ink_memory.h" -#include -#include -#include "I_VConnection.h" -#include "I_NetVConnection.h" -#include "I_IOBuffer.h" - -// http://www.haproxy.org/download/1.8/doc/proxy-protocol.txt - -extern bool proxy_protov1_parse(NetVConnection *, ts::TextView hdr); -extern bool ssl_has_proxy_v1(NetVConnection *, char *, int64_t *); -extern bool http_has_proxy_v1(IOBufferReader *, NetVConnection *); - -const char *const PROXY_V1_CONNECTION_PREFACE = R"(PROXY)"; -const char *const PROXY_V2_CONNECTION_PREFACE = "\x0D\x0A\x0D\x0A\x00\x0D\x0A\x51\x55\x49\x54\x0A\x02"; - -const size_t PROXY_V1_CONNECTION_PREFACE_LEN = strlen(PROXY_V1_CONNECTION_PREFACE); // 5 -const size_t PROXY_V2_CONNECTION_PREFACE_LEN = 13; - -const size_t PROXY_V1_CONNECTION_HEADER_LEN_MIN = 15; -const size_t PROXY_V2_CONNECTION_HEADER_LEN_MIN = 16; - -const size_t PROXY_V1_CONNECTION_HEADER_LEN_MAX = 108; -const size_t PROXY_V2_CONNECTION_HEADER_LEN_MAX = 16; +#include +#include + +enum class ProxyProtocolVersion { + UNDEFINED, + V1, + V2, +}; + +enum class ProxyProtocolData { + UNDEFINED, + SRC, + DST, +}; + +struct ProxyProtocol { + ProxyProtocolVersion version = ProxyProtocolVersion::UNDEFINED; + uint16_t ip_family = AF_UNSPEC; + IpEndpoint src_addr = {}; + IpEndpoint dst_addr = {}; +}; + +const size_t PPv1_CONNECTION_HEADER_LEN_MAX = 108; +const size_t PPv2_CONNECTION_HEADER_LEN_MAX = 16; + +extern size_t proxy_protocol_parse(ProxyProtocol *pp_info, ts::TextView tv); diff --git a/iocore/net/SSLNetVConnection.cc b/iocore/net/SSLNetVConnection.cc index ff39b45de49..a12b2fa21ef 100644 --- a/iocore/net/SSLNetVConnection.cc +++ b/iocore/net/SSLNetVConnection.cc @@ -405,7 +405,7 @@ SSLNetVConnection::read_raw_data() IpMap *pp_ipmap; pp_ipmap = SSLConfigParams::proxy_protocol_ipmap; - if (this->get_is_proxy_protocol()) { + if (this->get_is_proxy_protocol() && this->get_proxy_protocol_version() == ProxyProtocolVersion::UNDEFINED) { Debug("proxyprotocol", "proxy protocol is enabled on this port"); if (pp_ipmap->count() > 0) { Debug("proxyprotocol", "proxy protocol has a configured allowlist of trusted IPs - checking"); @@ -430,8 +430,8 @@ SSLNetVConnection::read_raw_data() "proxy protocol is enabled on this port - processing all connections"); } - if (ssl_has_proxy_v1(this, buffer, &r)) { - Debug("proxyprotocol", "ssl has proxy_v1 header"); + if (this->has_proxy_protocol(buffer, &r)) { + Debug("proxyprotocol", "ssl has proxy protocol header"); set_remote_addr(get_proxy_protocol_src_addr()); } else { Debug("proxyprotocol", "proxy protocol was enabled, but required header was not present in the " diff --git a/proxy/ProtocolProbeSessionAccept.cc b/proxy/ProtocolProbeSessionAccept.cc index ac210f0091f..cdbffcf24ad 100644 --- a/proxy/ProtocolProbeSessionAccept.cc +++ b/proxy/ProtocolProbeSessionAccept.cc @@ -100,7 +100,7 @@ struct ProtocolProbeTrampoline : public Continuation, public ProtocolProbeSessio IpMap *pp_ipmap; pp_ipmap = probeParent->proxy_protocol_ipmap; - if (netvc->get_is_proxy_protocol()) { + if (netvc->get_is_proxy_protocol() && netvc->get_proxy_protocol_version() == ProxyProtocolVersion::UNDEFINED) { Debug("proxyprotocol", "ioCompletionEvent: proxy protocol is enabled on this port"); if (pp_ipmap->count() > 0) { Debug("proxyprotocol", "ioCompletionEvent: proxy protocol has a configured allowlist of trusted IPs - checking"); @@ -120,8 +120,8 @@ struct ProtocolProbeTrampoline : public Continuation, public ProtocolProbeSessio "ernabled on this port - processing all connections"); } - if (http_has_proxy_v1(reader, netvc)) { - Debug("proxyprotocol", "ioCompletionEvent: http has proxy_v1 header"); + if (netvc->has_proxy_protocol(reader)) { + Debug("proxyprotocol", "ioCompletionEvent: http has proxy protocol header"); netvc->set_remote_addr(netvc->get_proxy_protocol_src_addr()); } else { Debug("proxyprotocol", diff --git a/proxy/http/HttpTransact.cc b/proxy/http/HttpTransact.cc index 28ae2362776..9f2dc65e11d 100644 --- a/proxy/http/HttpTransact.cc +++ b/proxy/http/HttpTransact.cc @@ -5771,11 +5771,11 @@ HttpTransact::initialize_state_variables_from_request(State *s, HTTPHdr *obsolet ats_ip_copy(&s->request_data.src_ip, &s->client_info.src_addr); memset(&s->request_data.dest_ip, 0, sizeof(s->request_data.dest_ip)); if (vc) { - s->request_data.incoming_port = vc->get_local_port(); - s->pp_info.proxy_protocol_version = vc->get_proxy_protocol_version(); - if (s->pp_info.proxy_protocol_version != NetVConnection::ProxyProtocolVersion::UNDEFINED) { - ats_ip_copy(s->pp_info.src_addr, vc->pp_info.src_addr); - ats_ip_copy(s->pp_info.dst_addr, vc->pp_info.dst_addr); + s->request_data.incoming_port = vc->get_local_port(); + s->pp_info.version = vc->get_proxy_protocol_version(); + if (s->pp_info.version != ProxyProtocolVersion::UNDEFINED) { + ats_ip_copy(s->pp_info.src_addr, vc->get_proxy_protocol_src_addr()); + ats_ip_copy(s->pp_info.dst_addr, vc->get_proxy_protocol_dst_addr()); } } s->request_data.xact_start = s->client_request_time; diff --git a/proxy/http/HttpTransact.h b/proxy/http/HttpTransact.h index bfca7f59407..01648bbbdd2 100644 --- a/proxy/http/HttpTransact.h +++ b/proxy/http/HttpTransact.h @@ -910,7 +910,7 @@ class HttpTransact internal_msg_buffer_size = 0; } - NetVConnection::ProxyProtocol pp_info; + ProxyProtocol pp_info; private: // Make this a raw byte array, so it will be accessed through the my_txn_conf() member function. diff --git a/proxy/logging/LogAccess.cc b/proxy/logging/LogAccess.cc index 08957c379e7..37849d2d273 100644 --- a/proxy/logging/LogAccess.cc +++ b/proxy/logging/LogAccess.cc @@ -1359,15 +1359,15 @@ LogAccess::marshal_proxy_protocol_version(char *buf) int len = INK_MIN_ALIGN; if (m_http_sm) { - NetVConnection::ProxyProtocolVersion ver = m_http_sm->t_state.pp_info.proxy_protocol_version; + ProxyProtocolVersion ver = m_http_sm->t_state.pp_info.version; switch (ver) { - case NetVConnection::ProxyProtocolVersion::V1: + case ProxyProtocolVersion::V1: version_str = "V1"; break; - case NetVConnection::ProxyProtocolVersion::V2: + case ProxyProtocolVersion::V2: version_str = "V2"; break; - case NetVConnection::ProxyProtocolVersion::UNDEFINED: + case ProxyProtocolVersion::UNDEFINED: default: version_str = "-"; break; @@ -1387,7 +1387,7 @@ int LogAccess::marshal_proxy_protocol_src_ip(char *buf) { sockaddr const *ip = nullptr; - if (m_http_sm && m_http_sm->t_state.pp_info.proxy_protocol_version != NetVConnection::ProxyProtocolVersion::UNDEFINED) { + if (m_http_sm && m_http_sm->t_state.pp_info.version != ProxyProtocolVersion::UNDEFINED) { ip = &m_http_sm->t_state.pp_info.src_addr.sa; } return marshal_ip(buf, ip); @@ -1399,7 +1399,7 @@ int LogAccess::marshal_proxy_protocol_dst_ip(char *buf) { sockaddr const *ip = nullptr; - if (m_http_sm && m_http_sm->t_state.pp_info.proxy_protocol_version != NetVConnection::ProxyProtocolVersion::UNDEFINED) { + if (m_http_sm && m_http_sm->t_state.pp_info.version != ProxyProtocolVersion::UNDEFINED) { ip = &m_http_sm->t_state.pp_info.dst_addr.sa; } return marshal_ip(buf, ip);