From d546286ecf94c43cef759307a5e8dac4313cfc70 Mon Sep 17 00:00:00 2001 From: Damir Zainullin Date: Sun, 3 Nov 2024 23:05:33 +0100 Subject: [PATCH 1/6] Refactor TLS parser --- process/tls_parser.cpp | 644 ++++++++++++++++++++++++++--------------- process/tls_parser.hpp | 267 ++++++++++++++--- 2 files changed, 639 insertions(+), 272 deletions(-) diff --git a/process/tls_parser.cpp b/process/tls_parser.cpp index 30fd30865..bc38e1323 100644 --- a/process/tls_parser.cpp +++ b/process/tls_parser.cpp @@ -7,19 +7,18 @@ * \brief Class for parsing TLS traffic. * \author Andrej Lukacovic lukacan1@fit.cvut.cz * \author Karel Hynek - * \date 2022 + * \author Zainullin Damir + * \date 2024 */ #include "tls_parser.hpp" + #include +#include +#include namespace ipxp { -TLSParser::TLSParser() -{ - tls_hs = NULL; -} - -uint64_t quic_get_variable_length(uint8_t *start, uint64_t &offset) +uint64_t quic_get_variable_length(const uint8_t *start, uint64_t &offset) { // find out length of parameter field (and load parameter, then move offset) , defined in: // https://www.rfc-editor.org/rfc/rfc9000.html#name-summary-of-integer-encoding @@ -35,17 +34,17 @@ uint64_t quic_get_variable_length(uint8_t *start, uint64_t &offset) return tmp; case 64: - tmp = be16toh(*(uint16_t *) (start + offset)) & 0x3FFF; + tmp = be16toh(* reinterpret_cast(start + offset)) & 0x3FFF; offset += sizeof(uint16_t); return tmp; case 128: - tmp = be32toh(*(uint32_t *) (start + offset)) & 0x3FFFFFFF; + tmp = be32toh(*reinterpret_cast< const uint32_t*>(start + offset)) & 0x3FFFFFFF; offset += sizeof(uint32_t); return tmp; case 192: - tmp = be64toh(*(uint64_t *) (start + offset)) & 0x3FFFFFFFFFFFFFFF; + tmp = be64toh(*reinterpret_cast(start + offset)) & 0x3FFFFFFFFFFFFFFF; offset += sizeof(uint64_t); return tmp; @@ -54,291 +53,472 @@ uint64_t quic_get_variable_length(uint8_t *start, uint64_t &offset) } } // quic_get_variable_length -bool TLSParser::tls_is_grease_value(uint16_t val) +bool TLSParser::is_grease_value(uint16_t val) { - if (val != 0 && !(val & ~(0xFAFA)) && ((0x00FF & val) == (val >> 8))) { - return true; - } - return false; + return val != 0 && !(val & ~(0xFAFA)) && ((0x00FF & val) == (val >> 8)); } -void TLSParser::tls_get_quic_user_agent(TLSData &data, char *buffer, size_t buffer_size) +bool TLSParser::parse_tls(const uint8_t* packet, uint32_t length) { - // compute end of quic_transport_parameters - const uint16_t quic_transport_params_len = ntohs(*(uint16_t *) data.start); - const uint8_t *quic_transport_params_end = data.start + quic_transport_params_len - + sizeof(quic_transport_params_len); + return parse(packet, length, false); +} - if (quic_transport_params_end > data.end) { - return; - } +bool TLSParser::parse_quic_tls(const uint8_t* packet, uint32_t length) +{ + return parse(packet, length, true); +} - uint64_t offset = 0; - uint64_t param = 0; - uint64_t length = 0; - - while (data.start + offset < quic_transport_params_end) { - param = quic_get_variable_length((uint8_t *) data.start, offset); - length = quic_get_variable_length((uint8_t *) data.start, offset); - if (param == TLS_EXT_GOOGLE_USER_AGENT) { - if (length + (size_t) 1 > buffer_size) { - length = buffer_size - 1; - } - memcpy(buffer, data.start + offset, length); - buffer[length] = 0; - data.obejcts_parsed++; - } - offset += length; - } - return; +bool TLSParser::parse(const uint8_t* packet, uint32_t length, bool is_quic) +{ + m_packet_data = packet; + m_packet_length = length; + clear_parsed_data(); + + if (!parse_tls_header(is_quic)) { + return false; + } + if (!parse_tls_handshake()) { + return false; + } + + if (m_handshake->type != TLS_HANDSHAKE_CLIENT_HELLO + && m_handshake->type != TLS_HANDSHAKE_SERVER_HELLO) { + return false; + } + + if (!parse_session_id()) { + return false; + } + if (!parse_cipher_suites()) { + return false; + } + if (!parse_compression_methods()) { + return false; + } + return true; } -void TLSParser::tls_get_server_name(TLSData &data, char *buffer, size_t buffer_size) +bool TLSParser::parse_tls_header(bool is_quic) noexcept { - uint16_t list_len = ntohs(*(uint16_t *) data.start); - uint16_t offset = sizeof(list_len); - const uint8_t *list_end = data.start + list_len + offset; - size_t buff_offset = 0; + if (is_quic) { + m_header_section_size = 0; + return true; + } + const auto* tls_header = reinterpret_cast(m_packet_data); + + if (sizeof(TLSHeader) > m_packet_length) { + return false; + } + if (tls_header == nullptr) { + return false; + } + if (tls_header->type != TLS_HANDSHAKE) { + return false; + } + if (tls_header->version.major != 3 || tls_header->version.minor > 3) { + return false; + } + m_header_section_size = sizeof(TLSHeader); + return true; +} - if (list_end > data.end) { - // data.valid = false; - return; - } +bool handshake_has_supported_version(const TLSHandshake *handshake) +{ + return handshake->version.major == 3 && handshake->version.minor >= 1 && handshake->version.minor <= 3; +} - while (data.start + sizeof(tls_ext_sni) + offset < list_end) { - tls_ext_sni *tmp_sni = (tls_ext_sni *) (data.start + offset); - uint16_t sni_len = ntohs(tmp_sni->length); - - offset += sizeof(tls_ext_sni); - if (data.start + offset + sni_len > list_end) { - break; - } - if (sni_len + (size_t) 1 + buff_offset > buffer_size) { - sni_len = buffer_size - 1 - buff_offset; - } - memcpy(buffer + buff_offset, data.start + offset, sni_len); - - buff_offset += sni_len + 1; - buffer[buff_offset - 1] = 0; - data.obejcts_parsed++; - offset += ntohs(tmp_sni->length); - } - return; +bool handshake_has_supported_type(const TLSHandshake *handshake) +{ + return handshake->type == TLS_HANDSHAKE_CLIENT_HELLO || handshake->type == TLS_HANDSHAKE_SERVER_HELLO; } -void TLSParser::tls_get_supp_ver(TLSData &data, uint16_t &version) +bool TLSParser::parse_tls_handshake() noexcept { - tls_version* ext_ver = (tls_version *) data.start; - version = ((uint16_t) ext_ver->major << 8) | ext_ver->minor; - return; + const auto* handshake = reinterpret_cast(m_packet_data + m_header_section_size); + + if ( m_header_section_size + sizeof(TLSHandshake) > m_packet_length) { + return false; + } + if (!handshake_has_supported_type(handshake)) { + return false; + } + if (!handshake_has_supported_version(handshake)) { + return false; + } + m_handshake = *handshake; + return true; } -void TLSParser::tls_get_alpn(TLSData &data, char *buffer, size_t buffer_size) +bool TLSParser::parse_session_id() noexcept { - uint16_t list_len = ntohs(*(uint16_t *) data.start); - uint16_t offset = sizeof(list_len); - const uint8_t *list_end = data.start + list_len + offset; + const size_t session_id_section_offset + = m_header_section_size + sizeof(TLSHandshake) + TLS_RANDOM_BYTES_LENGTH; + if (session_id_section_offset > m_packet_length) { + return false; + } + + const uint8_t sessionIdLength = *(m_packet_data + session_id_section_offset); + m_session_id_section_length = sizeof(sessionIdLength) + sessionIdLength; + if (session_id_section_offset + m_session_id_section_length > m_packet_length) { + return false; + } + return true; +} - if (list_end > data.end) { - // data.valid = false; - return; - } - if (buffer[0] != 0) { - return; - } +bool TLSParser::parse_cipher_suites() noexcept +{ + const size_t cipher_suite_section_offset = m_header_section_size + + sizeof(TLSHandshake) + TLS_RANDOM_BYTES_LENGTH + m_session_id_section_length; + if (cipher_suite_section_offset + sizeof(uint16_t) > m_packet_length) { + return false; + } + + if (m_handshake->type == TLS_HANDSHAKE_SERVER_HELLO) { + m_cipher_suites_section_length = sizeof(uint16_t); + return true; + } + + // Else parse Client Hello + const uint16_t client_cipher_suites_length + = ntohs(*reinterpret_cast(m_packet_data + cipher_suite_section_offset)); + if (cipher_suite_section_offset + sizeof(client_cipher_suites_length) + client_cipher_suites_length + > m_packet_length) { + return false; + } + + const uint8_t* cipher_suites_begin + = m_packet_data + cipher_suite_section_offset + sizeof(client_cipher_suites_length); + const uint8_t* cipher_suites_end = cipher_suites_begin + client_cipher_suites_length; + for (const uint8_t* cipher_suite = cipher_suites_begin; cipher_suite < cipher_suites_end; + cipher_suite += sizeof(uint16_t)) { + const uint16_t type_id = ntohs(*reinterpret_cast(cipher_suite)); + if (!is_grease_value(type_id)) { + m_cipher_suits.push_back(type_id); + } + } + m_cipher_suites_section_length = sizeof(client_cipher_suites_length) + client_cipher_suites_length; + return true; +} - uint16_t alpn_written = 0; - - while (data.start + sizeof(uint8_t) + offset < list_end) { - uint8_t alpn_len = *(uint8_t *) (data.start + offset); - const uint8_t *alpn_str = data.start + offset + sizeof(uint8_t); - - offset += sizeof(uint8_t) + alpn_len; - if (data.start + offset > list_end) { - break; - } - if (alpn_written + alpn_len + (size_t) 2 >= buffer_size) { - break; - } - - if (alpn_written != 0) { - buffer[alpn_written++] = ';'; - } - memcpy(buffer + alpn_written, alpn_str, alpn_len); - alpn_written += alpn_len; - buffer[alpn_written] = 0; - } - return; -} // TLSParser::tls_get_alpn +bool TLSParser::parse_compression_methods() noexcept +{ + const size_t compression_methods_section_offset = m_header_section_size + + sizeof(TLSHandshake) + TLS_RANDOM_BYTES_LENGTH + m_session_id_section_length + m_cipher_suites_section_length; + if (compression_methods_section_offset > m_packet_length) { + return false; + } + + if (m_handshake->type == TLS_HANDSHAKE_SERVER_HELLO) { + m_compression_methods_section_length = 1; + return true; + } + // Else parse Client Hello + const uint8_t compression_methods_length + = *static_cast(m_packet_data + compression_methods_section_offset); + if (sizeof(compression_methods_length) + compression_methods_length > m_packet_length) { + return false; + } + m_compression_methods_section_length + = sizeof(compression_methods_length) + compression_methods_length; + return true; +} -tls_handshake TLSParser::tls_get_handshake() +void TLSParser::parse_server_names(const uint8_t* extension_data, uint16_t extension_length) { - if (tls_hs != NULL) { - return *tls_hs; - } - return { }; + if (sizeof(uint16_t) > extension_length) { + return; + } + const uint16_t servername_list_length = ntohs(*reinterpret_cast(extension_data)); + if (sizeof(servername_list_length) + servername_list_length > extension_length) { + return; + } + const uint8_t* sni_begin = extension_data + sizeof(servername_list_length); + const uint8_t* sni_end = sni_begin + servername_list_length; + + for (const uint8_t* sni = sni_begin; sni + sizeof(TLSExtensionSNI) <= sni_end;) { + const uint16_t sni_length = ntohs((reinterpret_cast(sni))->length); + + if (sni + sizeof(TLSExtensionSNI) + sni_length > extension_data + extension_length) { + break; + } + m_server_names.emplace_back(reinterpret_cast(sni) + sizeof(TLSExtensionSNI), sni_length); + + sni += sizeof(TLSExtensionSNI) + sni_length; + m_objects_parsed++; + } } -bool TLSParser::tls_check_handshake(TLSData & payload) +void TLSParser::parse_elliptic_curves(const uint8_t* extension_payload, uint16_t extension_length) noexcept { - tls_hs = (tls_handshake *) payload.start; - const uint8_t tmp_hs_type = tls_hs->type; + if (sizeof(uint16_t) > extension_length) { + return; + } + const uint16_t supported_groups_length = ntohs(*reinterpret_cast(extension_payload)); + if (sizeof(supported_groups_length) + supported_groups_length > extension_length) { + return; + } + + const uint8_t* supported_groups_begin = extension_payload + sizeof(supported_groups_length); + const uint8_t* supported_groups_end = supported_groups_begin + supported_groups_length; + + for (const uint8_t* supported_group = supported_groups_begin; supported_group < supported_groups_end; + supported_group += sizeof(uint16_t)) { + const uint16_t supported_group_type = ntohs(*reinterpret_cast(supported_group)); + if (!is_grease_value(supported_group_type)) { + m_elliptic_curves.push_back(supported_group_type); + } + } +} - if (payload.start + sizeof(tls_handshake) > payload.end || - !(tmp_hs_type == TLS_HANDSHAKE_CLIENT_HELLO || tmp_hs_type == TLS_HANDSHAKE_SERVER_HELLO)) { - return false; - } - if (payload.start + 44 > payload.end || - tls_hs->version.major != 3 || - tls_hs->version.minor < 1 || - tls_hs->version.minor > 3) { - return false; - } - payload.start += sizeof(tls_handshake); - return true; +void TLSParser::parse_elliptic_curve_point_formats(const uint8_t* extension_payload, uint16_t extension_length) noexcept +{ + if (sizeof(uint8_t) > extension_length) { + return; + } + const uint8_t supported_formats_length = *extension_payload; + if (sizeof(supported_formats_length) + supported_formats_length > extension_length) { + return; + } + + const uint8_t* supportedFormatsBegin = extension_payload + sizeof(supported_formats_length); + const uint8_t* supportedFormatsEnd = supportedFormatsBegin + supported_formats_length; + std::string supportedFormats; + + for (const uint8_t* supported_format_pointer = supportedFormatsBegin; + supported_format_pointer < supportedFormatsEnd; + supported_format_pointer++) { + const uint8_t supported_format = *supported_format_pointer; + if (!is_grease_value(supported_format)) { + m_elliptic_curve_point_formats.push_back(supported_format); + } + } } -bool TLSParser::tls_check_rec(TLSData & payload) +void TLSParser::parse_alpn(const uint8_t* extension_data, uint16_t extension_length) { - tls_rec *tls = (tls_rec *) payload.start; + if (sizeof(uint16_t) > extension_length) { + return; + } + const uint16_t alpnExtensionLength = ntohs(*reinterpret_cast(extension_data)); + if (sizeof(uint16_t) + alpnExtensionLength > extension_length) { + return; + } + + const uint8_t* alpn_begin = extension_data + sizeof(uint16_t); + const uint8_t* alpn_end = alpn_begin + alpnExtensionLength; + + for (const uint8_t* alpn = alpn_begin; alpn + sizeof(uint8_t) <= alpn_end; ) { + const uint8_t alpn_length = *alpn; + + if (alpn + sizeof(alpn_length) + alpn_length > alpn_begin + extension_length) { + break; + } + m_alpns.emplace_back(reinterpret_cast(alpn) + sizeof(alpn_length), alpn_length); + alpn += sizeof(uint8_t) + alpn_length; + m_objects_parsed++; + } +} - if (payload.start + sizeof(tls_rec) > payload.end || !tls || tls->type != TLS_HANDSHAKE || - tls->version.major != 3 || tls->version.minor > 3) { - return false; - } - payload.start += sizeof(tls_rec); - return true; +void TLSParser::parse_signature_algorithms(const uint8_t* extension_data, uint16_t extension_length) noexcept +{ + const auto* signature_algorithm = reinterpret_cast(extension_data); + std::for_each_n(signature_algorithm, + extension_length /sizeof(uint16_t),[this](uint16_t algorithm){ + m_signature_algorithms.push_back(ntohs(algorithm)); + }); } -bool TLSParser::tls_skip_random(TLSData& payload) +void TLSParser::parse_supported_versions(const uint8_t* extension_data, uint16_t extension_length) noexcept { - if (payload.start + 32 > payload.end) { - return false; - } - payload.start += 32; + if (m_handshake->type == TLS_HANDSHAKE_SERVER_HELLO) { + if (sizeof(uint16_t) > extension_length) { + return; + } + m_supported_versions.push_back(ntohs(*reinterpret_cast(extension_data))); + return; + } + //Else parse client hello + if (sizeof(uint8_t) > extension_length) { + return; + } + const uint8_t versions_length = *extension_data; + if (sizeof(uint8_t) + versions_length > extension_length) { + return; + } + + const auto version = reinterpret_cast(extension_data + sizeof(versions_length)); + std::for_each_n(version, versions_length /2,[this](auto version){ + if (!is_grease_value(version)) { + m_supported_versions.push_back(ntohs(version)); + } + }); +} + +bool TLSParser::parse_extensions(const std::function& callable) noexcept{ + if (!has_valid_extension_length()) { + return false; + } + const size_t extensions_section_offset = m_header_section_size + + sizeof(TLSHandshake) + TLS_RANDOM_BYTES_LENGTH + m_session_id_section_length + m_cipher_suites_section_length + + m_compression_methods_section_length; + const uint16_t extensions_section_length + = ntohs(*reinterpret_cast(m_packet_data + extensions_section_offset)); + + const uint8_t* extensions_begin + = m_packet_data + extensions_section_offset + sizeof(extensions_section_length); + const uint8_t* extensions_end = extensions_begin + extensions_section_length; + + for (const uint8_t* extension_ptr = extensions_begin; extension_ptr < extensions_end;) { + const auto* extension = reinterpret_cast(extension_ptr); + const uint16_t extension_length = ntohs(extension->length); + const uint16_t extension_type = ntohs(extension->type); + + if (extension_ptr + sizeof(TLSExtension) + extension_length > extensions_end) { + break; + } + + const uint8_t* extensionPayload = extension_ptr + sizeof(TLSExtension); + callable(extension_type, extensionPayload, extension_length); + + extension_ptr += sizeof(TLSExtension) + extension_length; + } + return true; +} + +bool TLSParser::has_valid_extension_length() const noexcept +{ + const size_t extensions_section_offset = m_header_section_size + + sizeof(TLSHandshake) + TLS_RANDOM_BYTES_LENGTH + m_session_id_section_length + m_cipher_suites_section_length + + m_compression_methods_section_length; + if (extensions_section_offset > m_packet_length) { + return false; + } + const uint16_t extension_section_length + = ntohs(*reinterpret_cast(m_packet_data + extensions_section_offset)); + if (extensions_section_offset + extension_section_length > m_packet_length) { + return false; + } return true; } -bool TLSParser::tls_skip_sessid(TLSData& payload) +const std::optional& TLSParser::get_handshake() const noexcept { - uint8_t sess_id_len = *(uint8_t *) payload.start; + return m_handshake; +} - if (payload.start + sizeof(sess_id_len) + sess_id_len > payload.end) { - return false; - } - payload.start += sizeof(sess_id_len) + sess_id_len; - return true; +bool TLSParser::is_client_hello() const noexcept +{ + return m_handshake->type == TLS_HANDSHAKE_CLIENT_HELLO; } -bool TLSParser::tls_skip_cipher_suites(TLSData& payload) +bool TLSParser::is_server_hello() const noexcept { - uint16_t cipher_suite_len = ntohs(*(uint16_t *) payload.start); + return m_handshake->type == TLS_HANDSHAKE_SERVER_HELLO; +} - if (payload.start + sizeof(cipher_suite_len) + +cipher_suite_len > payload.end) { - return false; - } - payload.start += sizeof(cipher_suite_len) + cipher_suite_len; - return true; +const std::vector& TLSParser::get_extensions() const noexcept +{ + return m_extensions; } -bool TLSParser::tls_skip_compression_met(TLSData& payload) +const std::vector& TLSParser::get_cipher_suits() const noexcept { - uint8_t compression_met_len = *(uint8_t *) payload.start; + return m_cipher_suits; +} - if (payload.start + sizeof(compression_met_len) + compression_met_len > payload.end) { - return false; - } - payload.start += sizeof(compression_met_len) + compression_met_len; - return true; +const std::vector& TLSParser::get_elliptic_curves() const noexcept +{ + return m_elliptic_curves; } -bool TLSParser::tls_check_ext_len(TLSData& payload) +const std::vector& TLSParser::get_elliptic_curve_point_formats() const noexcept { - const uint8_t *ext_end = payload.start + ntohs(*(uint16_t *) payload.start) + sizeof(uint16_t); + return m_elliptic_curve_point_formats; +} - payload.start += 2; - if (ext_end > payload.end) { - return false; - } - if (ext_end <= payload.end) { - payload.end = ext_end; - } - return true; +const std::vector& TLSParser::get_alpns() const noexcept +{ + return m_alpns; } -bool TLSParser::tls_get_ja3_cipher_suites(std::string &ja3, TLSData &data) +const std::vector& TLSParser::get_server_names() const noexcept { - uint16_t cipher_suites_length = ntohs(*(uint16_t *) data.start); - uint16_t type_id = 0; - const uint8_t *section_end = data.start + cipher_suites_length; + return m_server_names; +} - if (data.start + cipher_suites_length + 1 > data.end) { - // data.valid = false; - return false; - } - data.start += sizeof(cipher_suites_length); - - for (; data.start <= section_end; data.start += sizeof(uint16_t)) { - type_id = ntohs(*(uint16_t *) (data.start)); - if (!tls_is_grease_value(type_id)) { - ja3 += std::to_string(type_id); - if (data.start < section_end) { - ja3 += '-'; - } - } - } - ja3 += ','; - return true; +const std::vector& TLSParser::get_supported_versions() const noexcept +{ + return m_supported_versions; } -std::string TLSParser::tls_get_ja3_ecpliptic_curves(TLSData &data) +const std::vector& TLSParser::get_signature_algorithms() const noexcept { - std::string collected_types; - uint16_t type_id = 0; - uint16_t list_len = ntohs(*(uint16_t *) data.start); - const uint8_t *list_end = data.start + list_len + sizeof(list_len); - uint16_t offset = sizeof(list_len); + return m_signature_algorithms; +} - if (list_end > data.end) { - // data.valid = false; - return ""; - } +static void save_to_buffer(char* destination, const std::vector& source, uint32_t size, char delimiter) noexcept +{ + std::for_each(source.begin(), source.end(), [destination, write_pos = 0UL, size,delimiter](const std::string_view& alpn) mutable { + if (alpn.length() + 2U > size - write_pos) { + destination[write_pos] = 0; + return; + } + const size_t bytes_to_write = std::min(size - write_pos - 2U, alpn.length() + 2UL); + memcpy(destination + write_pos, alpn.data(), bytes_to_write); + write_pos += alpn.length(); + destination[write_pos++] = delimiter; + }); +} - while (data.start + sizeof(uint16_t) + offset <= list_end) { - type_id = ntohs(*(uint16_t *) (data.start + offset)); - offset += sizeof(uint16_t); - if (!tls_is_grease_value(type_id)) { - collected_types += std::to_string(type_id); +void TLSParser::save_server_names(char* destination, uint32_t size) const noexcept +{ + save_to_buffer(destination, m_server_names, size, 0); +} - if (data.start + sizeof(uint16_t) + offset <= list_end) { - collected_types += '-'; - } - } - } - return collected_types; +void TLSParser::save_alpns(char* destination, uint32_t size) const noexcept +{ + save_to_buffer(destination, m_alpns, size, 0); } -std::string TLSParser::tls_get_ja3_ec_point_formats(TLSData &data) +void TLSParser::save_quic_user_agent(char* destination, uint32_t size) const noexcept { - std::string collected_formats; - uint8_t list_len = *data.start; - uint16_t offset = sizeof(list_len); - const uint8_t *list_end = data.start + list_len + offset; - uint8_t format; + save_to_buffer(destination, m_quic_user_agents, size, 0); +} - if (list_end > data.end) { - // data.valid = false; - return ""; - } +void TLSParser::parse_quic_user_agent(const uint8_t* extension_payload, uint16_t extension_length) noexcept +{ + const uint8_t* quic_transport_parameters_begin = extension_payload; + const uint8_t* quic_transport_parameters_end = quic_transport_parameters_begin + extension_length; + for (const uint8_t* parameter = quic_transport_parameters_begin; parameter < quic_transport_parameters_end; ) { + size_t offset = 0UL; + const size_t parameter_id = quic_get_variable_length(parameter, offset); + const size_t parameter_length = quic_get_variable_length(parameter, offset); + if (parameter + offset + parameter_length > quic_transport_parameters_end) { + return; + } + if (parameter_id == TLS_EXT_GOOGLE_USER_AGENT) { + m_objects_parsed++; + m_quic_user_agents.emplace_back(reinterpret_cast(parameter + offset), parameter_length); + } + parameter += offset + parameter_length; + } +} - while (data.start + sizeof(uint8_t) + offset <= list_end) { - format = *(data.start + offset); - collected_formats += std::to_string((int) format); - offset += sizeof(uint8_t); - if (data.start + sizeof(uint8_t) + offset <= list_end) { - collected_formats += '-'; - } - } - return collected_formats; +void TLSParser::clear_parsed_data() noexcept +{ + m_extensions.clear(); + m_cipher_suits.clear(); + m_signature_algorithms.clear(); + m_elliptic_curves.clear(); + m_elliptic_curve_point_formats.clear(); + m_alpns.clear(); + m_supported_versions.clear(); + m_server_names.clear(); } + +void TLSParser::add_extension(uint16_t extension_type, uint16_t extension_length) noexcept +{ + m_extensions.emplace_back(TLSExtension{extension_type, extension_length}); +} + } diff --git a/process/tls_parser.hpp b/process/tls_parser.hpp index 7a5e3928f..ed7b84f64 100644 --- a/process/tls_parser.hpp +++ b/process/tls_parser.hpp @@ -7,13 +7,17 @@ * \brief Class for parsing TLS traffic. * \author Andrej Lukacovic lukacan1@fit.cvut.cz * \author Karel Hynek - * \date 2022 + * \author Zainullin Damir + * \date 2024 */ +#include #include #include -#include +#include +#include +#include #define TLS_HANDSHAKE_CLIENT_HELLO 1 #define TLS_HANDSHAKE_SERVER_HELLO 2 @@ -26,29 +30,30 @@ // draf-02 az draft-12 have this value defined as 0x26 == 38 #define TLS_EXT_QUIC_TRANSPORT_PARAMETERS_V2 0x26 #define TLS_EXT_GOOGLE_USER_AGENT 0x3129 -#define MAX_TLS_EXT_LEN 30 +#define MAX_TLS_EXT_LEN 30UL +#define TLS_EXT_SERVER_NAME 0 +#define TLS_EXT_ECLIPTIC_CURVES 10 // AKA supported_groups +#define TLS_EXT_EC_POINT_FORMATS 11 +#define TLS_EXT_SIGNATURE_ALGORITHMS 13 +#define TLS_EXT_ALPN 16 +#define TLS_EXT_SUPPORTED_VER 43 namespace ipxp { -typedef struct TLSData { - const uint8_t *start; - const uint8_t *end; - int obejcts_parsed; -} TLSData; -struct __attribute__((packed)) tls_ext_sni { +struct __attribute__((packed)) TLSExtensionSNI { uint8_t type; uint16_t length; /* Hostname bytes... */ }; -struct __attribute__((packed)) tls_ext { +struct __attribute__((packed)) TLSExtension { uint16_t type; uint16_t length; - /* Extension pecific data... */ + /* Extension specific data... */ }; -union __attribute__((packed)) tls_version { +union __attribute__((packed)) TLSVersion { uint16_t version; struct { uint8_t major; @@ -56,49 +61,231 @@ union __attribute__((packed)) tls_version { }; }; -struct __attribute__((packed)) tls_handshake { +struct __attribute__((packed)) TLSHandshake { uint8_t type; uint8_t length1; // length field is 3 bytes long... uint16_t length2; - tls_version version; + TLSVersion version; /* Handshake data... */ }; #define TLS_HANDSHAKE 22 -struct __attribute__((packed)) tls_rec { +struct __attribute__((packed)) TLSHeader { uint8_t type; - tls_version version; + TLSVersion version; uint16_t length; /* Record data... */ }; class TLSParser { -private: - tls_handshake *tls_hs; public: - TLSParser(); - bool tls_skip_random(TLSData&); - bool tls_skip_sessid(TLSData&); - bool tls_skip_cipher_suites(TLSData&); - bool tls_skip_compression_met(TLSData&); - bool tls_check_ext_len(TLSData&); - bool tls_check_rec(TLSData&); - void tls_get_server_name(TLSData &, char *, size_t); - void tls_get_alpn(TLSData &, char *, size_t); - void tls_get_supp_ver(TLSData &, uint16_t &); - - void tls_get_quic_user_agent(TLSData &, char *, size_t); - bool tls_check_handshake(TLSData&); - bool tls_get_ja3_cipher_suites(std::string&, TLSData&); - - bool tls_is_grease_value(uint16_t); - - tls_handshake tls_get_handshake(); - uint8_t tls_get_hstype(); - std::string tls_get_version_ja3(); - std::string tls_get_ja3_ecpliptic_curves(TLSData &data); - std::string tls_get_ja3_ec_point_formats(TLSData &data); + /** + * @brief Parses given payload as a normal TLS packet which is not part of the QUIC protocol. + * @param packet Pointer to the payload. + * @param length Length of the payload. + * @return True if parsed succesfully, false otherwise. + */ + bool parse_tls(const uint8_t* packet, uint32_t length); + + /** + * @brief Parses given payload as a TLS part of QUIC protocol which doesn't have TLS header. + * @param packet Pointer to the payload. + * @param length Length of the payload. + * @return True if parsed succesfully, false otherwise. + */ + bool parse_quic_tls(const uint8_t* packet, uint32_t length); + + /** + * @brief Provide custom extensions parser of TLS Client or Server Hello packet. + * @param callable Callable that will be called for each extension, takes type of extension, + * pointer to its begin and its length. + * @return True if extensions section has valid data, false otherwise. + */ + bool parse_extensions(const std::function& callable) noexcept; + + /** + * @brief Parses TLS SNI extension. + * @param extension_data Pointer to the extension's begin. + * @param extension_length Extension's length. + */ + void parse_server_names(const uint8_t* extension_data, uint16_t extension_length); + + /** + * @brief Parses TLS QUIC transport parameters extension. + * @param extension_data Pointer to the extension's begin. + * @param extension_length Extension's length. + */ + void parse_quic_user_agent(const uint8_t* extension_payload, uint16_t extension_length) noexcept; + + /** + * @brief Checks if given TLS packet is Client Hello. + * @return True if it is, false otherwise. + */ + bool is_client_hello() const noexcept; + + /** + * @brief Checks if given TLS packet is Server Hello. + * @return True if it is, false otherwise. + */ + bool is_server_hello() const noexcept; + + /** + * @brief Getter for TLS packet handshake section. + * @return Handshake section if present, nullopt otherwise. + */ + const std::optional& get_handshake() const noexcept; + + /** + * @brief Getter for parsed extension. + * @return Parsed extension. + */ + const std::vector& get_extensions() const noexcept; + + /** + * @brief Adds given extension to parsed extensions. + * @param extension_type Given extension type. + * @param extension_length Given extension length. + */ + void add_extension(uint16_t extension_type, uint16_t extension_length) noexcept; + + /** + * @brief Getter for parsed cipher suits. + * @return Parsed cipher suits types. + */ + const std::vector& get_cipher_suits() const noexcept; + + /** + * @brief Getter for parsed elliptic curves. + * @return Parsed elliptic curves ids. + */ + const std::vector& get_elliptic_curves() const noexcept; + + /** + * @brief Getter for parsed elliptic curves point formats. + * @return Parsed elliptic curves point formats types. + */ + const std::vector& get_elliptic_curve_point_formats() const noexcept; + + /** + * @brief Getter for parsed application layer protocol negotiations. + * @return Pointers to parsed protocol names and its lengths. + */ + const std::vector& get_alpns() const noexcept; + + /** + * @brief Getter for parsed supported versions. + * @return Parsed supported versions. + */ + const std::vector& get_supported_versions() const noexcept; + + /** + * @brief Getter for parsed server names from SNI extension. + * @return Pointers to SNI names and its lengths. + */ + const std::vector& get_server_names() const noexcept; + + /** + * @brief Getter for parsed signature algorithms. + * @return Parsed signature algorithms types. + */ + const std::vector& get_signature_algorithms() const noexcept; + + /** + * @brief Save parsed alpns to given buffer restricted with buffer length. + * @param destination Destination buffer. + * @param size Destination buffer size. + */ + void save_alpns(char* destination, uint32_t size) const noexcept; + + /** + * @brief Save parsed server names from SNI extension restricted with buffer length. + * @param destination Destination buffer. + * @param size Destination buffer size. + */ + void save_server_names(char* destination, uint32_t size) const noexcept; + + /** + * @brief Save parsed QUIC user agent from QUIC transport parameters extension restricted with buffer length. + * @param destination Destination buffer. + * @param size Destination buffer size. + */ + void save_quic_user_agent(char* destination, uint32_t size) const noexcept; + + /** + * @brief Checks if given value is GREASE. + * @param value Value to check. + * @return True if value is GREASE, false otherwise + */ + static bool is_grease_value(uint16_t value); + + /** + * @brief Parse TLS application layer protocol negotiation extension. + * @param extension_data Pointer to the extension's begin. + * @param extension_length Extension's length. + */ + void parse_alpn(const uint8_t* extension_data, uint16_t extension_length); + + /** + * @brief Parse TLS elliptic curves extension. + * @param extension_payload Pointer to the extension's begin. + * @param extension_length Extension's length. + */ + void parse_elliptic_curves(const uint8_t* extension_payload, uint16_t extension_length)noexcept; + + /** + * @brief Parse elliptic curve point formats extension. + * @param extension_payload Pointer to the extension's begin. + * @param extension_length Extension's length. + */ + void + parse_elliptic_curve_point_formats(const uint8_t* extension_payload, uint16_t extension_length) noexcept; + + /** + * @brief Parse TLS supported versions extension. + * @param extension_data Pointer to the extension's begin. + * @param extension_length Extension's length. + */ + void parse_supported_versions(const uint8_t* extension_data, uint16_t extension_length)noexcept; + + /** + * @brief Parse TLS signature algorithms extension. + * @param extension_data Pointer to the extension's begin. + * @param extension_length Extension's length. + * @return True if parsed succesfully, false otherwise. + */ + void parse_signature_algorithms(const uint8_t* extension_data, uint16_t extension_length)noexcept; +private: + bool parse(const uint8_t* packet, uint32_t length, bool is_quic); + bool parse_tls_handshake() noexcept; + bool parse_tls_header(bool is_quic) noexcept; + bool parse_session_id() noexcept; + bool parse_cipher_suites() noexcept; + bool parse_compression_methods()noexcept; + bool has_valid_extension_length() const noexcept; + void clear_parsed_data() noexcept; + + const uint8_t* m_packet_data{nullptr}; + uint32_t m_packet_length{0}; + + uint32_t m_header_section_size {0}; + static constexpr uint32_t TLS_RANDOM_BYTES_LENGTH = 32; + uint32_t m_session_id_section_length {0}; + uint32_t m_cipher_suites_section_length {0}; + uint32_t m_compression_methods_section_length {0}; + + std::vector m_extensions; + std::vector m_cipher_suits; + std::vector m_signature_algorithms; + std::vector m_elliptic_curves; + std::vector m_elliptic_curve_point_formats; + std::vector m_alpns; + std::vector m_supported_versions; + std::vector m_server_names; + std::vector m_quic_user_agents; + + std::optional m_handshake; + uint16_t m_objects_parsed {0}; }; } From e6604c81560ac866e6ed5e98dcff04e5d0c20a0b Mon Sep 17 00:00:00 2001 From: Damir Zainullin Date: Sat, 23 Nov 2024 18:03:51 +0100 Subject: [PATCH 2/6] Update IPFIX element for JA4 --- include/ipfixprobe/ipfix-elements.hpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/include/ipfixprobe/ipfix-elements.hpp b/include/ipfixprobe/ipfix-elements.hpp index 666ccd110..d9f894bcf 100644 --- a/include/ipfixprobe/ipfix-elements.hpp +++ b/include/ipfixprobe/ipfix-elements.hpp @@ -185,10 +185,10 @@ namespace ipxp { #define TLS_VERSION(F) F(39499, 333, 2, nullptr) #define TLS_ALPN(F) F(39499, 337, -1, nullptr) #define TLS_JA3(F) F(39499, 357, -1, nullptr) +#define TLS_JA4(F) F(39499, 358, -1, nullptr) #define TLS_EXT_TYPE(F) F(0, 291, -1, nullptr) #define TLS_EXT_LEN(F) F(0, 291, -1, nullptr) - #define SMTP_COMMANDS(F) F(8057, 810, 4, nullptr) #define SMTP_MAIL_COUNT(F) F(8057, 811, 4, nullptr) #define SMTP_RCPT_COUNT(F) F(8057, 812, 4, nullptr) @@ -389,6 +389,7 @@ namespace ipxp { F(TLS_SNI) \ F(TLS_ALPN) \ F(TLS_JA3) \ + F(TLS_JA4) \ F(TLS_EXT_TYPE) \ F(TLS_EXT_LEN) From 78aee80cf0e2122461b6762be364744e22314f46 Mon Sep 17 00:00:00 2001 From: Damir Zainullin Date: Sun, 3 Nov 2024 23:05:16 +0100 Subject: [PATCH 3/6] Refactor TLS plugin --- Makefile.am | 1 + process/tls.cpp | 378 +++++++++++++++++++++++++++++++++--------------- process/tls.hpp | 156 +++++++++----------- 3 files changed, 329 insertions(+), 206 deletions(-) diff --git a/Makefile.am b/Makefile.am index 2fc539f71..c755d8d64 100644 --- a/Makefile.am +++ b/Makefile.am @@ -99,6 +99,7 @@ ipfixprobe_process_src=\ process/tls.hpp \ process/tls_parser.cpp \ process/tls_parser.hpp \ + process/sha256.hpp \ process/smtp.cpp \ process/smtp.hpp \ process/dns-utils.hpp \ diff --git a/process/tls.cpp b/process/tls.cpp index b47c0dfbc..95fe9ef1c 100644 --- a/process/tls.cpp +++ b/process/tls.cpp @@ -15,12 +15,15 @@ #include -#include - +#include #include +#include +#include +#include -#include "tls.hpp" #include "md5.hpp" +#include "tls.hpp" +#include "sha256.hpp" namespace ipxp { int RecordExtTLS::REGISTERED_ID = -1; @@ -49,8 +52,20 @@ __attribute__((constructor)) static void register_this_plugin() # define DEBUG_CODE(code) #endif -TLSPlugin::TLSPlugin() : ext_ptr(nullptr), parsed_sni(0), flow_flush(false) -{ } +OptionsParser* TLSPlugin::get_parser() const +{ + return new OptionsParser("tls", "Parse SNI from TLS traffic"); +} + +std::string TLSPlugin::get_name() const +{ + return "tls"; +} + +RecordExtTLS *TLSPlugin::get_ext() const +{ + return new RecordExtTLS(); +} TLSPlugin::~TLSPlugin() { @@ -58,7 +73,8 @@ TLSPlugin::~TLSPlugin() } void TLSPlugin::init(const char *params) -{ } +{ +} void TLSPlugin::close() { @@ -81,12 +97,12 @@ int TLSPlugin::post_create(Flow &rec, const Packet &pkt) int TLSPlugin::pre_update(Flow &rec, Packet &pkt) { - RecordExtTLS *ext = static_cast(rec.get_extension(RecordExtTLS::REGISTERED_ID)); + auto *ext = static_cast(rec.get_extension(RecordExtTLS::REGISTERED_ID)); if (ext != nullptr) { - if (ext->server_hello_parsed == false) { + if (!ext->server_hello_parsed) { // Add ALPN from server packet - parse_tls(pkt.payload, pkt.payload_len, ext); + parse_tls(pkt.payload, pkt.payload_len, ext, rec.ip_proto); } return 0; } @@ -95,127 +111,259 @@ int TLSPlugin::pre_update(Flow &rec, Packet &pkt) return 0; } -bool TLSPlugin::obtain_tls_data(TLSData &payload, RecordExtTLS *rec, std::string &ja3, uint8_t hs_type) +static std::string concatenate_vector_to_string(const std::vector& vector){ + if (vector.empty()) { + return ""; + } + return std::accumulate( + std::next(vector.begin()), vector.end(), std::to_string(vector[0]), + [](const std::string& a, uint16_t b) { + return a + "-" + std::to_string(b); + }); +} + +static std::string concatenate_vector_to_hex_string(const std::vector& vector){ + auto res = std::accumulate( + vector.begin(), + vector.end(), + std::string {}, + [](const std::string& acc, uint16_t value) { + std::array buffer = {}; + std::snprintf(buffer.data(), buffer.size(), "%04x,", value); + return acc + buffer.data(); + }); + res.pop_back(); + return res; +} + +static std::string concatenate_extensions_vector_to_string(const std::vector& extensions){ + auto res = std::accumulate( + extensions.begin(), extensions.end(), std::string{}, + [](const std::string& a, const auto& extension) { + if (TLSParser::is_grease_value(extension.type)) { + return a; + } + return a + std::to_string(extension.type) + "-"; + }); + res.pop_back(); + return res; +} + +static const char* convert_version_to_label(uint16_t version) { - std::string ecliptic_curves; - std::string ec_point_formats; + switch(version) { + case 0x0304: + return "13"; + case 0x0303: + return "12"; + case 0x0302: + return "11"; + case 0x0301: + return "10"; + case 0x0300: + return "s3"; + case 0x0002: + return "s2"; + case 0xfeff: + return "d1"; + case 0xfefd: + return "d2"; + case 0xfefc: + return "d3"; + default: + return "00"; + } +} - while (payload.start + sizeof(tls_ext) <= payload.end) { - tls_ext *ext = (tls_ext *) payload.start; - uint16_t length = ntohs(ext->length); - uint16_t type = ntohs(ext->type); +static std::string get_ja3_string(const TLSParser& parser) +{ + std::string ja3_string = std::to_string(parser.get_handshake()->version.version) + ','; + ja3_string += concatenate_vector_to_string(parser.get_cipher_suits()) + ','; + ja3_string += concatenate_extensions_vector_to_string(parser.get_extensions()) + ','; + ja3_string += concatenate_vector_to_string(parser.get_elliptic_curves()) + ','; + ja3_string += concatenate_vector_to_string(parser.get_elliptic_curve_point_formats()); + return ja3_string; +} - payload.start += sizeof(tls_ext); - if (payload.start + length > payload.end) { - break; - } +static char convert_alpn_byte_to_label(char alpn_byte, bool high_nibble) +{ + if (std::isalnum(alpn_byte)) { + return alpn_byte; + } else { + uint8_t nibble = high_nibble ? alpn_byte >> 4 : alpn_byte & 0x0F; + return nibble < 0xA ? (char)('0' + nibble) : (char)('A' + nibble - 0xA); + } +} - if (hs_type == TLS_HANDSHAKE_CLIENT_HELLO) { - if (type == TLS_EXT_SERVER_NAME) { - tls_parser.tls_get_server_name(payload, rec->sni, sizeof(rec->sni)); - } else if (type == TLS_EXT_ECLIPTIC_CURVES) { - ecliptic_curves = tls_parser.tls_get_ja3_ecpliptic_curves(payload); - } else if (type == TLS_EXT_EC_POINT_FORMATS) { - ec_point_formats = tls_parser.tls_get_ja3_ec_point_formats(payload); - } +static const char* get_version_label(const TLSParser& parser) +{ + uint16_t version; + if (parser.get_supported_versions().empty()) { + version = parser.get_handshake()->version.version; + } else { + const auto* versions = (const int16_t*)parser.get_supported_versions().data(); + version = *std::max_element(versions, versions + parser.get_supported_versions().size()); + } + return convert_version_to_label(version); +} - if (!rec->tls_ext_len_set && !rec->tls_ext_type_set) { - // Store extension type - if (rec->tls_ext_type_len < MAX_TLS_EXT_LEN) { - rec->tls_ext_type[rec->tls_ext_type_len] = type; - rec->tls_ext_type_len += 1; - } - - // Store extension type length - if (rec->tls_ext_len_len < MAX_TLS_EXT_LEN) { - rec->tls_ext_len[rec->tls_ext_len_len] = length; - rec->tls_ext_len_len += 1; - } - } - } else if (hs_type == TLS_HANDSHAKE_SERVER_HELLO) { - rec->server_hello_parsed = true; - if (type == TLS_EXT_ALPN) { - tls_parser.tls_get_alpn(payload, rec->alpn, BUFF_SIZE); - // not sure, but probably don`t return yet, as - // this is not only field we want to parse - //return true; - } else if (type == TLS_EXT_SUPPORTED_VER){ - tls_parser.tls_get_supp_ver(payload, rec->version); - } - } - payload.start += length; - if (!tls_parser.tls_is_grease_value(type)) { - ja3 += std::to_string(type); +static std::string get_truncated_hash_hex(const std::string& str) +{ + std::array hash{}; + sha256::hash_it((const uint8_t*)str.c_str(), str.length(), (uint8_t*)hash.data()); + std::ostringstream oss; + for (auto i = 0U; i < 6; ++i) { + oss << std::hex << std::setw(2) << std::setfill('0') + << ((uint16_t)hash[i] & 0xFF); + } + return oss.str(); +} - if (payload.start + sizeof(tls_ext) <= payload.end) { - ja3 += '-'; - } - } - } - if (rec->tls_ext_type_len > 0 ) { - rec->tls_ext_type_set = true; - rec->tls_ext_len_set = true; - } +static std::string get_truncated_cipher_hash(const TLSParser& parser) +{ + std::string cipher_string; + std::vector cipher_suits = parser.get_cipher_suits(); + std::sort(cipher_suits.begin(), cipher_suits.end()); + + if (cipher_suits.empty()) { + cipher_string.assign(12, '0'); + return cipher_string; + } + cipher_string = concatenate_vector_to_hex_string(cipher_suits); + return get_truncated_hash_hex(cipher_string); +} +static std::string get_truncated_extensions_hash(const TLSParser& parser) +{ + std::vector extensions; + std::transform(parser.get_extensions().begin(), parser.get_extensions().end(), std::back_inserter(extensions), + [](const TLSExtension& extension) { return extension.type; }); + extensions.erase(std::remove_if(extensions.begin(), extensions.end(), [](uint16_t extension_type){ + return extension_type == TLS_EXT_ALPN || extension_type == TLS_EXT_SERVER_NAME || TLSParser::is_grease_value(extension_type); + }), extensions.end()); + std::sort(extensions.begin(), extensions.end()); + + auto extensions_string = concatenate_vector_to_hex_string(extensions); + std::vector signature_algorithms = parser.get_signature_algorithms(); + if (!signature_algorithms.empty()) { + signature_algorithms.erase(signature_algorithms.begin()); + } + auto signature_algorithms_string = concatenate_vector_to_hex_string(signature_algorithms); + + auto extensions_and_algorithms_string = extensions_string + '_' + signature_algorithms_string; + return get_truncated_hash_hex(extensions_and_algorithms_string); +} - if (hs_type == TLS_HANDSHAKE_SERVER_HELLO) { - return false; - } - ja3 += ',' + ecliptic_curves + ',' + ec_point_formats; - md5_get_bin(ja3, rec->ja3_hash_bin); - return true; -} // TLSPlugin::obtain_tls_data +static std::string get_alpn_label(const TLSParser& parser) +{ + std::string alpn_label; + if (parser.get_alpns().empty() || parser.get_alpns()[0].empty()) { + alpn_label = "00"; + } else { + const auto& alpn_string = parser.get_alpns()[0]; + alpn_label += convert_alpn_byte_to_label(alpn_string[0], true); + alpn_label += convert_alpn_byte_to_label(alpn_string[alpn_string.length() - 1], false); + } + return alpn_label; +} -bool TLSPlugin::parse_tls(const uint8_t *data, uint16_t payload_len, RecordExtTLS *rec) +static std::string get_ja4_string(const TLSParser& parser, uint8_t ip_proto) { - TLSData payload = { - payload.start = data, - payload.end = data + payload_len, - payload.obejcts_parsed = 0, - }; - std::string ja3; + constexpr const uint8_t UDP_ID = 17; + const char protocol = ip_proto == UDP_ID ? 'q' : 't'; + char version_label[3]; + *(uint16_t*)version_label = *(uint16_t*)get_version_label(parser); + version_label[2] = 0; - if (!tls_parser.tls_check_rec(payload)) { - return false; - } - if (!tls_parser.tls_check_handshake(payload)) { - return false; - } - tls_handshake tls_hs = tls_parser.tls_get_handshake(); + const char sni_label = parser.get_server_names().empty() ? 'i' : 'd'; - rec->version = (rec->version == 0)?(((uint16_t) tls_hs.version.major << 8) | tls_hs.version.minor) : rec->version; - ja3 += std::to_string((uint16_t) tls_hs.version.version) + ','; + const uint8_t ciphers_count = std::min(parser.get_cipher_suits().size(), 99UL); - if (!tls_parser.tls_skip_random(payload)) { - return false; - } - if (!tls_parser.tls_skip_sessid(payload)) { - return false; - } + const uint8_t extension_count = std::min(parser.get_extensions().size(), 99UL); - if (tls_hs.type == TLS_HANDSHAKE_CLIENT_HELLO) { - if (!tls_parser.tls_get_ja3_cipher_suites(ja3, payload)) { - return false; - } - if (!tls_parser.tls_skip_compression_met(payload)) { - return false; - } - } else if (tls_hs.type == TLS_HANDSHAKE_SERVER_HELLO) { - payload.start += 2; // Skip cipher suite - payload.start += 1; // Skip compression method - } else { - return false; - } - if (!tls_parser.tls_check_ext_len(payload)) { - return false; + const auto alpn_label = get_alpn_label(parser); + + const auto truncated_cipher_hash = get_truncated_cipher_hash(parser); + + const auto truncated_extensions_hash = get_truncated_extensions_hash(parser); + + return std::string{} + protocol + version_label + sni_label + std::to_string(ciphers_count) + + std::to_string(extension_count) + alpn_label + '_' + truncated_cipher_hash + '_' + + truncated_extensions_hash; +} + +static bool parse_client_hello_extensions(TLSParser& parser) noexcept +{ + return parser.parse_extensions([&parser](uint16_t extension_type, const uint8_t* extension_payload, uint16_t extension_length){ + if (extension_type == TLS_EXT_SERVER_NAME) { + parser.parse_server_names(extension_payload, extension_length); + } else if (extension_type == TLS_EXT_ECLIPTIC_CURVES) { + parser.parse_elliptic_curves(extension_payload, extension_length); + } else if (extension_type == TLS_EXT_EC_POINT_FORMATS) { + parser.parse_elliptic_curve_point_formats(extension_payload, extension_length); + } else if (extension_type == TLS_EXT_ALPN) { + parser.parse_alpn(extension_payload, extension_length); + } else if (extension_type == TLS_EXT_SIGNATURE_ALGORITHMS) { + parser.parse_signature_algorithms(extension_payload, extension_length); + } else if (extension_type == TLS_EXT_SUPPORTED_VER) { + parser.parse_supported_versions(extension_payload, extension_length); + } + parser.add_extension(extension_type, extension_length); + }); +} + +static bool parse_server_hello_extensions(TLSParser& parser) noexcept +{ + return parser.parse_extensions([&parser]( + uint16_t extension_type, + const uint8_t* extension_payload, + uint16_t extension_length) { + if (extension_type == TLS_EXT_ALPN) { + parser.parse_alpn(extension_payload, extension_length); + } else if (extension_type == TLS_EXT_SUPPORTED_VER) { + parser.parse_supported_versions(extension_payload, extension_length); + } + }); +} + +bool TLSPlugin::parse_tls(const uint8_t *data, uint16_t payload_len, RecordExtTLS *rec, uint8_t ip_proto) +{ + TLSParser parser; + if (!parser.parse_tls(data, payload_len)) { + return false; } - if (!obtain_tls_data(payload, rec, ja3, tls_hs.type)) { - return false; + + if (parser.is_client_hello()) { + if (!parse_client_hello_extensions(parser)) { + return false; + } + if (rec->extensions_buffer_size == 0) { + const auto count_to_copy = std::min(rec->extension_types.size(), parser.get_extensions().size()); + std::transform(parser.get_extensions().begin(), parser.get_extensions().begin() + count_to_copy, rec->extension_types.begin(),[](const auto& typeLength){ + return typeLength.type; + }); + std::transform(parser.get_extensions().begin(), parser.get_extensions().begin() + count_to_copy, rec->extension_lengths.begin(),[](const auto& typeLength){ + return typeLength.length; + }); + rec->extensions_buffer_size = count_to_copy; + } + rec->version = parser.get_handshake()->version.version; + parser.save_server_names(rec->sni, sizeof(rec->sni)); + md5_get_bin(get_ja3_string(parser), rec->ja3); + auto ja4 = get_ja4_string(parser, ip_proto); + std::memcpy(rec->ja4, ja4.c_str(), ja4.length()); + return true; + } else if (parser.is_server_hello()) { + if (!parse_server_hello_extensions(parser)) { + return false; + } + rec->server_hello_parsed = true; + parser.save_alpns(rec->alpn, sizeof(rec->alpn)); + rec->version = parser.get_supported_versions().empty() ? rec->version : parser.get_supported_versions()[0]; } - parsed_sni = payload.obejcts_parsed; - return payload.obejcts_parsed != 0 || !ja3.empty(); -} // TLSPlugin::parse_sni + return false; +} void TLSPlugin::add_tls_record(Flow &rec, const Packet &pkt) { @@ -223,9 +371,9 @@ void TLSPlugin::add_tls_record(Flow &rec, const Packet &pkt) ext_ptr = new RecordExtTLS(); } - if (parse_tls(pkt.payload, pkt.payload_len, ext_ptr)) { + if (parse_tls(pkt.payload, pkt.payload_len, ext_ptr, rec.ip_proto)) { DEBUG_CODE(for (int i = 0; i < 16; i++) { - DEBUG_MSG("%02x", ext_ptr->ja3_hash_bin[i]); + DEBUG_MSG("%02x", ext_ptr->ja3[i]); } ) DEBUG_MSG("\n"); diff --git a/process/tls.hpp b/process/tls.hpp index 3ab6e454b..c880992e9 100644 --- a/process/tls.hpp +++ b/process/tls.hpp @@ -17,15 +17,12 @@ #define IPXP_PROCESS_TLS_HPP #include -#include #include - #include #include - -#ifdef WITH_NEMEA -# include "fields.h" -#endif +#include +#include +#include #include #include @@ -35,17 +32,20 @@ #include #include +#ifdef WITH_NEMEA +# include "fields.h" +#endif -#define BUFF_SIZE 255 +#define BUFF_SIZE 255UL namespace ipxp { -#define TLS_UNIREC_TEMPLATE "TLS_SNI,TLS_JA3,TLS_ALPN,TLS_VERSION,TLS_EXT_TYPE,TLS_EXT_LEN" - +#define TLS_UNIREC_TEMPLATE "TLS_SNI,TLS_JA3,TLS_JA4,TLS_ALPN,TLS_VERSION,TLS_EXT_TYPE,TLS_EXT_LEN" UR_FIELDS( string TLS_SNI, string TLS_ALPN, uint16 TLS_VERSION, bytes TLS_JA3, + string TLS_JA4, uint16* TLS_EXT_TYPE, uint16* TLS_EXT_LEN ) @@ -59,39 +59,21 @@ UR_FIELDS( struct RecordExtTLS : public RecordExt { static int REGISTERED_ID; - uint16_t version; - char alpn[BUFF_SIZE] = { 0 }; - char sni[BUFF_SIZE] = { 0 }; - char ja3_hash[33] = { 0 }; - uint8_t ja3_hash_bin[16] = { 0 }; - std::string ja3; - bool server_hello_parsed; - - uint16_t tls_ext_type[MAX_TLS_EXT_LEN]; - uint16_t tls_ext_type_len; - bool tls_ext_type_set; - - uint16_t tls_ext_len[MAX_TLS_EXT_LEN]; - uint8_t tls_ext_len_len; - bool tls_ext_len_set; + uint16_t version{0}; + char alpn[BUFF_SIZE]{}; + char sni[BUFF_SIZE]{}; + uint8_t ja3[16]{0}; + char ja4[36]{0}; + bool server_hello_parsed{false}; + std::array extension_types{}; + std::array extension_lengths{}; + uint32_t extensions_buffer_size {0}; /** * \brief Constructor. */ - RecordExtTLS() : RecordExt(REGISTERED_ID), version(0) + RecordExtTLS() : RecordExt(REGISTERED_ID) { - alpn[0] = 0; - sni[0] = 0; - ja3_hash[0] = 0; - server_hello_parsed = false; - - memset(tls_ext_type, 0, sizeof(tls_ext_type)); - tls_ext_type_len = 0; - tls_ext_type_set = false; - - memset(tls_ext_len, 0, sizeof(tls_ext_len)); - tls_ext_len_len = 0; - tls_ext_len_set = false; } #ifdef WITH_NEMEA @@ -100,14 +82,15 @@ struct RecordExtTLS : public RecordExt { ur_set(tmplt, record, F_TLS_VERSION, version); ur_set_string(tmplt, record, F_TLS_SNI, sni); ur_set_string(tmplt, record, F_TLS_ALPN, alpn); - ur_set_var(tmplt, record, F_TLS_JA3, ja3_hash_bin, 16); - ur_array_allocate(tmplt, record, F_QUIC_TLS_EXT_TYPE, tls_ext_type_len); - for (int i = 0; i < tls_ext_type_len; i++) { - ur_array_set(tmplt, record, F_TLS_EXT_TYPE, i, tls_ext_type[i]); + ur_set_var(tmplt, record, F_TLS_JA3, ja3, sizeof(ja3)); + ur_set_string(tmplt, record, F_TLS_JA4, ja4); + ur_array_allocate(tmplt, record, F_QUIC_TLS_EXT_TYPE, extensions_buffer_size); + for (auto i = 0U; i < extensions_buffer_size; i++) { + ur_array_set(tmplt, record, F_TLS_EXT_TYPE, i, extension_types[i]); } - ur_array_allocate(tmplt, record, F_TLS_EXT_LEN, tls_ext_len_len); - for (int i = 0; i < tls_ext_len_len; i++) { - ur_array_set(tmplt, record, F_TLS_EXT_LEN, i, tls_ext_len[i]); + ur_array_allocate(tmplt, record, F_TLS_EXT_LEN, extensions_buffer_size); + for (auto i = 0U; i < extensions_buffer_size; i++) { + ur_array_set(tmplt, record, F_TLS_EXT_LEN, i, extension_lengths[i]); } } @@ -118,20 +101,20 @@ struct RecordExtTLS : public RecordExt { #endif // ifdef WITH_NEMEA - virtual int fill_ipfix(uint8_t *buffer, int size) + int fill_ipfix(uint8_t *buffer, int size) override { IpfixBasicList basiclist; basiclist.hdrEnterpriseNum = IpfixBasicList::CesnetPEM; - uint16_t sni_len = strlen(sni); - uint16_t alpn_len = strlen(alpn); + const size_t sni_len = strlen(sni); + const size_t alpn_len = strlen(alpn); - uint32_t pos = 0; + size_t pos = 0UL; - uint16_t len_tls_ext_type = sizeof(tls_ext_type[0]) * (tls_ext_type_len) + basiclist.HeaderSize(); - uint16_t len_tls_len = sizeof(tls_ext_len[0]) * (tls_ext_len_len) + basiclist.HeaderSize(); + const size_t len_tls_ext_type = sizeof(extension_types[0]) * (extensions_buffer_size) + basiclist.HeaderSize(); + const size_t len_tls_len = sizeof(extension_lengths[0]) * (extensions_buffer_size) + basiclist.HeaderSize(); - uint32_t req_buff_len = (sni_len + 3) + (alpn_len + 3) + (2) + (16 + 3) + len_tls_ext_type + len_tls_len; // (SNI) + (ALPN) + (VERSION) + (JA3) + const size_t req_buff_len = (sni_len + 3) + (alpn_len + 3) + (2) + (16 + 3) + (sizeof(ja4) + 3) + len_tls_ext_type + len_tls_len; // (SNI) + (ALPN) + (VERSION) + (JA3) + (JA4) if (req_buff_len > (uint32_t) size) { return -1; @@ -144,19 +127,22 @@ struct RecordExtTLS : public RecordExt { pos += variable2ipfix_buffer(buffer + pos, (uint8_t *) alpn, alpn_len); buffer[pos++] = 16; - memcpy(buffer + pos, ja3_hash_bin, 16); + memcpy(buffer + pos, ja3, 16); pos += 16; + + pos += variable2ipfix_buffer(buffer + pos, (uint8_t *) ja4, sizeof(ja4)); + pos += basiclist.FillBuffer( - buffer + pos, - tls_ext_type, - (uint16_t) tls_ext_type_len, - (uint16_t) TLS_EXT_TYPE_FIELD_ID); + buffer + pos, + extension_types.data(), + extensions_buffer_size, + (uint16_t) TLS_EXT_TYPE_FIELD_ID); pos += basiclist.FillBuffer( buffer + pos, - tls_ext_len, - (uint16_t) tls_ext_len_len, + extension_lengths.data(), + extensions_buffer_size, (uint16_t) TLS_EXT_LEN_FIELD_ID); - return pos; + return static_cast(pos); } const char **get_ipfix_tmplt() const @@ -169,7 +155,7 @@ struct RecordExtTLS : public RecordExt { return ipfix_template; } - std::string get_text() const + std::string get_text() const override { std::ostringstream out; @@ -178,19 +164,19 @@ struct RecordExtTLS : public RecordExt { << ",tlsversion=0x" << std::hex << std::setw(4) << std::setfill('0') << version << ",tlsja3="; for (int i = 0; i < 16; i++) { - out << std::hex << std::setw(2) << std::setfill('0') << (unsigned) ja3_hash_bin[i]; + out << std::hex << std::setw(2) << std::setfill('0') << (unsigned) ja3[i]; } out << ",tlsexttype=("; - for (int i = 0; i < tls_ext_type_len; i++) { - out << std::dec << (uint16_t) tls_ext_type[i]; - if (i != tls_ext_type_len - 1) { + for (auto i = 0U; i < extensions_buffer_size; i++) { + out << std::dec << (uint16_t) extension_types[i]; + if (i != extensions_buffer_size - 1) { out << ","; } } out << "),tlsextlen=("; - for (int i = 0; i < tls_ext_len_len; i++) { - out << std::dec << (uint16_t) tls_ext_len[i]; - if (i != tls_ext_len_len - 1) { + for (auto i = 0U; i < extensions_buffer_size; i++) { + out << std::dec << (uint16_t) extension_lengths[i]; + if (i != extensions_buffer_size - 1U) { out << ","; } } @@ -200,49 +186,37 @@ struct RecordExtTLS : public RecordExt { } }; - #define TLS_HANDSHAKE_CLIENT_HELLO 1 #define TLS_HANDSHAKE_SERVER_HELLO 2 - -#define TLS_EXT_SERVER_NAME 0 -#define TLS_EXT_ECLIPTIC_CURVES 10 // AKA supported_groups -#define TLS_EXT_EC_POINT_FORMATS 11 -#define TLS_EXT_ALPN 16 -#define TLS_EXT_SUPPORTED_VER 43 - - /** * \brief Flow cache plugin for parsing HTTPS packets. */ class TLSPlugin : public ProcessPlugin { public: - TLSPlugin(); - ~TLSPlugin(); - void init(const char *params); - void close(); - OptionsParser *get_parser() const { return new OptionsParser("tls", "Parse SNI from TLS traffic"); } + ~TLSPlugin() override; + void init(const char *params) override; + void close() override; + OptionsParser* get_parser() const override; - std::string get_name() const { return "tls"; } + std::string get_name() const override; - RecordExtTLS *get_ext() const { return new RecordExtTLS(); } + RecordExtTLS* get_ext() const override; ProcessPlugin *copy(); - int post_create(Flow &rec, const Packet &pkt); - int pre_update(Flow &rec, Packet &pkt); + int post_create(Flow &rec, const Packet &pkt) override; + int pre_update(Flow &rec, Packet &pkt) override; void finish(bool print_stats); private: void add_tls_record(Flow&, const Packet&); - bool parse_tls(const uint8_t *, uint16_t, RecordExtTLS *); - bool obtain_tls_data(TLSData&, RecordExtTLS *, std::string&, uint8_t); + bool parse_tls(const uint8_t *data, uint16_t payload_len, RecordExtTLS *rec, uint8_t ip_proto); - RecordExtTLS *ext_ptr; - TLSParser tls_parser; - uint32_t parsed_sni; - bool flow_flush; + RecordExtTLS *ext_ptr{nullptr}; + TLSParser tls_parser{}; + uint32_t parsed_sni{0}; }; } #endif /* IPXP_PROCESS_TLS_HPP */ From a0b756fb127e932d939c9ff8180a9bd70d40c8e7 Mon Sep 17 00:00:00 2001 From: Damir Zainullin Date: Mon, 4 Nov 2024 23:37:42 +0100 Subject: [PATCH 4/6] Add sha-256 hash --- process/sha256.hpp | 148 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 148 insertions(+) create mode 100644 process/sha256.hpp diff --git a/process/sha256.hpp b/process/sha256.hpp new file mode 100644 index 000000000..16096a43f --- /dev/null +++ b/process/sha256.hpp @@ -0,0 +1,148 @@ +// Single file sha256 +// https://github.com/LekKit/sha256 + +#include +#include + +namespace sha256 { + +struct sha256_buff { + unsigned long data_size; + unsigned int h[8]; + unsigned char last_chunk[64]; + unsigned char chunk_size; +}; + +void sha256_init(struct sha256_buff* buff) +{ + buff->h[0] = 0x6a09e667; + buff->h[1] = 0xbb67ae85; + buff->h[2] = 0x3c6ef372; + buff->h[3] = 0xa54ff53a; + buff->h[4] = 0x510e527f; + buff->h[5] = 0x9b05688c; + buff->h[6] = 0x1f83d9ab; + buff->h[7] = 0x5be0cd19; + buff->data_size = 0; + buff->chunk_size = 0; +} + +static const unsigned int k[64] = { + 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, + 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, + 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, + 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, + 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, + 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, + 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, + 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2}; + +#define rotate_r(val, bits) (val >> bits | val << (32 - bits)) + +static void sha256_calc_chunk(struct sha256_buff* buff, const unsigned char* chunk) +{ + unsigned int w[64]; + unsigned int tv[8]; + unsigned int i; + + for (i = 0; i < 16; ++i) { + w[i] = (unsigned int) chunk[0] << 24 | (unsigned int) chunk[1] << 16 + | (unsigned int) chunk[2] << 8 | (unsigned int) chunk[3]; + chunk += 4; + } + + for (i = 16; i < 64; ++i) { + unsigned int s0 = rotate_r(w[i - 15], 7) ^ rotate_r(w[i - 15], 18) ^ (w[i - 15] >> 3); + unsigned int s1 = rotate_r(w[i - 2], 17) ^ rotate_r(w[i - 2], 19) ^ (w[i - 2] >> 10); + w[i] = w[i - 16] + s0 + w[i - 7] + s1; + } + + for (i = 0; i < 8; ++i) + tv[i] = buff->h[i]; + + for (i = 0; i < 64; ++i) { + unsigned int S1 = rotate_r(tv[4], 6) ^ rotate_r(tv[4], 11) ^ rotate_r(tv[4], 25); + unsigned int ch = (tv[4] & tv[5]) ^ (~tv[4] & tv[6]); + unsigned int temp1 = tv[7] + S1 + ch + k[i] + w[i]; + unsigned int S0 = rotate_r(tv[0], 2) ^ rotate_r(tv[0], 13) ^ rotate_r(tv[0], 22); + unsigned int maj = (tv[0] & tv[1]) ^ (tv[0] & tv[2]) ^ (tv[1] & tv[2]); + unsigned int temp2 = S0 + maj; + + tv[7] = tv[6]; + tv[6] = tv[5]; + tv[5] = tv[4]; + tv[4] = tv[3] + temp1; + tv[3] = tv[2]; + tv[2] = tv[1]; + tv[1] = tv[0]; + tv[0] = temp1 + temp2; + } + + for (i = 0; i < 8; ++i) + buff->h[i] += tv[i]; +} + +void sha256_update(struct sha256_buff* buff, const void* data, unsigned long size) +{ + const unsigned char* ptr = (const unsigned char*) data; + buff->data_size += size; + /* If there is data left in buff, concatenate it to process as new chunk */ + if (size + buff->chunk_size >= 64) { + unsigned char tmp_chunk[64]; + memcpy(tmp_chunk, buff->last_chunk, buff->chunk_size); + memcpy(tmp_chunk + buff->chunk_size, ptr, 64 - buff->chunk_size); + ptr += (64 - buff->chunk_size); + size -= (64 - buff->chunk_size); + buff->chunk_size = 0; + sha256_calc_chunk(buff, tmp_chunk); + } + /* Run over data chunks */ + while (size >= 64) { + sha256_calc_chunk(buff, ptr); + ptr += 64; + size -= 64; + } + + /* Save remaining data in buff, will be reused on next call or finalize */ + memcpy(buff->last_chunk + buff->chunk_size, ptr, size); + buff->chunk_size += size; +} + +void sha256_finalize(struct sha256_buff* buff) +{ + buff->last_chunk[buff->chunk_size] = 0x80; + buff->chunk_size++; + memset(buff->last_chunk + buff->chunk_size, 0, 64 - buff->chunk_size); + + /* If there isn't enough space to fit int64, pad chunk with zeroes and prepare next chunk */ + if (buff->chunk_size > 56) { + sha256_calc_chunk(buff, buff->last_chunk); + memset(buff->last_chunk, 0, 64); + } + + /* Add total size as big-endian int64 x8 */ + unsigned long size = buff->data_size * 8; + int i; + for (i = 8; i > 0; --i) { + buff->last_chunk[55 + i] = size & 255; + size >>= 8; + } + + sha256_calc_chunk(buff, buff->last_chunk); +} + +void hash_it(const unsigned char* data, unsigned long data_size, unsigned char* hash) +{ + struct sha256_buff buff; + sha256_init(&buff); + sha256_update(&buff, data, data_size); + sha256_finalize(&buff); + for (int i = 0; i < 8; i++) { + hash[i * 4] = (buff.h[i] >> 24) & 255; + hash[i * 4 + 1] = (buff.h[i] >> 16) & 255; + hash[i * 4 + 2] = (buff.h[i] >> 8) & 255; + hash[i * 4 + 3] = buff.h[i] & 255; + } +} + +} \ No newline at end of file From 04192c14e29467be1b50cdfc72ad1da1a18d5f62 Mon Sep 17 00:00:00 2001 From: Damir Zainullin Date: Sun, 17 Nov 2024 19:08:55 +0100 Subject: [PATCH 5/6] Refactor QUIC Parser to use refactored TLS Parser --- process/quic_parser.cpp | 136 ++++++++++++++++------------------------ process/quic_parser.hpp | 2 +- 2 files changed, 55 insertions(+), 83 deletions(-) diff --git a/process/quic_parser.cpp b/process/quic_parser.cpp index 2bc8554ce..acd94478a 100644 --- a/process/quic_parser.cpp +++ b/process/quic_parser.cpp @@ -248,95 +248,65 @@ uint64_t QUICParser::quic_get_variable_length(const uint8_t* start, uint64_t& of } } // QUICParser::quic_get_variable_length -bool QUICParser::quic_obtain_tls_data(TLSData& payload) +bool QUICParser::quic_parse_tls_extensions() { - quic_tls_extension_lengths_pos = 0; - quic_tls_ext_type_pos = 0; - quic_tls_ext_pos = 0; - while (payload.start + sizeof(tls_ext) <= payload.end) { - tls_ext* ext = (tls_ext*) payload.start; - uint16_t type = ntohs(ext->type); - uint16_t length = ntohs(ext->length); - - // Store extension type - if (quic_tls_ext_type_pos < MAX_QUIC_TLS_EXT_LEN) { - quic_tls_ext_type[quic_tls_ext_type_pos] = type; - quic_tls_ext_type_pos += 1; - } - - // Store extension type length - if (quic_tls_extension_lengths_pos < MAX_QUIC_TLS_EXT_LEN) { - quic_tls_extension_lengths[quic_tls_extension_lengths_pos] = length; - quic_tls_extension_lengths_pos += 1; - } - - // - payload.start += sizeof(tls_ext); - - if (payload.start + length > payload.end) { - break; - } - - // Save value payload except for length - if (quic_tls_ext_pos + length < CURRENT_BUFFER_SIZE) { + const bool extensions_parsed = tls_parser.parse_extensions([this]( + uint16_t extension_type, + const uint8_t* extension_payload, + uint16_t extension_length) { + if (extension_type == TLS_EXT_SERVER_NAME && extension_length != 0) { + tls_parser.parse_server_names(extension_payload, extension_length); + } else if ( + (extension_type == TLS_EXT_QUIC_TRANSPORT_PARAMETERS_V1 + || extension_type == TLS_EXT_QUIC_TRANSPORT_PARAMETERS + || extension_type == TLS_EXT_QUIC_TRANSPORT_PARAMETERS_V2) + && extension_length != 0) { + tls_parser.parse_quic_user_agent(extension_payload, extension_length); + } + if (quic_tls_ext_pos + extension_length < CURRENT_BUFFER_SIZE) { #ifndef QUIC_CH_FULL_TLS_EXT - if (type == TLS_EXT_ALPN || type == TLS_EXT_QUIC_TRANSPORT_PARAMETERS_V1 - || type == TLS_EXT_QUIC_TRANSPORT_PARAMETERS - || type == TLS_EXT_QUIC_TRANSPORT_PARAMETERS_V2) { + if (extension_type == TLS_EXT_ALPN || extension_type == TLS_EXT_QUIC_TRANSPORT_PARAMETERS_V1 + || extension_type == TLS_EXT_QUIC_TRANSPORT_PARAMETERS + || extension_type == TLS_EXT_QUIC_TRANSPORT_PARAMETERS_V2) { #endif - memcpy(quic_tls_ext + quic_tls_ext_pos, payload.start, length); - quic_tls_ext_pos += length; + memcpy(quic_tls_ext + quic_tls_ext_pos, extension_payload, extension_length); + quic_tls_ext_pos += extension_length; #ifndef QUIC_CH_FULL_TLS_EXT - } + } #endif - } - - // Legacy extract specific fields - if (type == TLS_EXT_SERVER_NAME && length != 0) { - tls_parser.tls_get_server_name(payload, sni, BUFF_SIZE); - } else if ( - (type == TLS_EXT_QUIC_TRANSPORT_PARAMETERS_V1 - || type == TLS_EXT_QUIC_TRANSPORT_PARAMETERS - || type == TLS_EXT_QUIC_TRANSPORT_PARAMETERS_V2) - && length != 0) { - tls_parser.tls_get_quic_user_agent(payload, user_agent, BUFF_SIZE); - } - payload.start += length; - } - return payload.obejcts_parsed != 0; + } + tls_parser.add_extension(extension_type, extension_length); + }); + if (!extensions_parsed) { + return false; + } + tls_parser.save_server_names(sni, BUFF_SIZE); + tls_parser.save_quic_user_agent(user_agent, BUFF_SIZE); + + const size_t copy_count = std::min(tls_parser.get_extensions().size(), MAX_QUIC_TLS_EXT_LEN); + std::transform(tls_parser.get_extensions().begin(), + tls_parser.get_extensions().begin() + static_cast(copy_count), + std::begin(quic_tls_ext_type), + [](const TLSExtension& typeLength) { + return typeLength.type; + }); + std::transform(tls_parser.get_extensions().begin(), + tls_parser.get_extensions().begin() + static_cast(copy_count), + std::begin(quic_tls_extension_lengths), + [](const TLSExtension& typeLength) { + return typeLength.length; + }); + quic_tls_ext_type_pos = quic_tls_extension_lengths_pos = copy_count; + return true; } bool QUICParser::quic_parse_tls() { - TLSData payload = { - payload.start = final_payload + quic_crypto_start, - payload.end = final_payload + quic_crypto_start + quic_crypto_len, - payload.obejcts_parsed = 0, - }; - - if (!tls_parser.tls_check_handshake(payload)) { - return false; - } - if (!tls_parser.tls_skip_random(payload)) { - return false; - } - if (!tls_parser.tls_skip_sessid(payload)) { - return false; - } - if (!tls_parser.tls_skip_cipher_suites(payload)) { + if (!tls_parser.parse_quic_tls(final_payload + quic_crypto_start, quic_crypto_len)) { return false; } - if (!tls_parser.tls_skip_compression_met(payload)) { - return false; - } - if (!tls_parser.tls_check_ext_len(payload)) { - return false; - } - // If no parameters were extracted. We also accept the QUIC connection. (no error check here) - quic_obtain_tls_data(payload); - - return true; -} // QUICPlugin::quic_parse_tls + return quic_parse_tls_extensions(); +} uint8_t QUICParser::quic_draft_version(uint32_t version) { @@ -1394,14 +1364,16 @@ bool QUICParser::quic_parse_headers(const Packet& pkt, bool forceInitialParsing) bool QUICParser::quic_set_server_port(const Packet& pkt) { - tls_handshake hs = tls_parser.tls_get_handshake(); + if (!tls_parser.get_handshake().has_value()) { + return false; + } switch (packet_type) { case INITIAL: - tls_hs_type = hs.type; - if (hs.type == 1) { + tls_hs_type = tls_parser.get_handshake()->type; + if (tls_hs_type == 1) { server_port = pkt.dst_port; - } else if (hs.type == 2) { + } else if (tls_hs_type == 2) { // Won't be reached, since we don't supply the OCCID to quic_parser server_port = pkt.src_port; } diff --git a/process/quic_parser.hpp b/process/quic_parser.hpp index 629e9d489..27c522e2f 100644 --- a/process/quic_parser.hpp +++ b/process/quic_parser.hpp @@ -119,7 +119,7 @@ class QUICParser { uint64_t quic_get_variable_length(const uint8_t*, uint64_t&); bool quic_check_version(uint32_t, uint8_t); bool quic_check_pointer_pos(const uint8_t*, const uint8_t*); - bool quic_obtain_tls_data(TLSData&); + bool quic_parse_tls_extensions(); bool quic_set_server_port(const Packet& pkt); bool quic_check_min_initial_size(const Packet& pkt); bool quic_check_supported_version(const uint32_t version); From acbdca03d6bb4684addb5e0b7b9bde7c5da8cff7 Mon Sep 17 00:00:00 2001 From: Damir Zainullin Date: Sat, 23 Nov 2024 18:11:00 +0100 Subject: [PATCH 6/6] Update tests --- tests/functional/reference/tls | 58 +++++++++++++++++----------------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/tests/functional/reference/tls b/tests/functional/reference/tls index 8a58be384..3807095a7 100644 --- a/tests/functional/reference/tls +++ b/tests/functional/reference/tls @@ -1,29 +1,29 @@ -104.26.1.201,192.168.0.228,569,1460,0,2023-03-10T09:04:19.289282,2023-03-10T09:04:19.304938,90:5c:44:2e:bb:e0,08:f8:bc:64:5e:6a,1,1,443,52641,772,0,6,24,16,"",cd08e31494f9531f560d64c695473da9,"chrek.stdout.cz",[0|20|0|1|10|2|0|14|5|18|0|43|2|7|3|5|1|200],[31354|0|23|65281|10|11|35|16|5|13|18|51|45|43|27|17513|10794|21] -104.26.6.183,192.168.0.228,599,264,0,2023-03-10T09:04:17.091196,2023-03-10T09:04:17.110050,90:5c:44:2e:bb:e0,08:f8:bc:64:5e:6a,1,1,443,52635,772,0,6,24,24,"",598872011444709307b861ae817a4b60,"cdn.xsd.cz",[0|15|0|1|10|2|0|14|5|18|0|43|2|7|3|5|1|235],[14906|0|23|65281|10|11|35|16|5|13|18|51|45|43|27|17513|10794|41] -104.26.8.145,192.168.0.228,604,264,0,2023-03-10T09:04:16.211748,2023-03-10T09:04:16.225609,90:5c:44:2e:bb:e0,08:f8:bc:64:5e:6a,1,1,443,52627,772,0,6,24,24,"",598872011444709307b861ae817a4b60,"www.aktualne.cz",[0|20|0|1|10|2|0|14|5|18|0|43|2|7|3|5|1|235],[2570|0|23|65281|10|11|35|16|5|13|18|51|45|43|27|17513|35466|41] -160.85.255.180,192.168.88.244,1344,416,0,2020-09-07T06:52:40.858429,2020-09-07T06:52:40.964957,d8:58:d7:00:c9:27,08:f8:bc:64:5e:6a,4,2,443,59673,771,0,6,24,24,"http/1.1",b32309a26951912be7dba376398abc3b,"ja3er.com",[0|14|0|1|10|2|208|14|5|18|0|43|2|11|3|1|3],[47802|0|23|65281|10|11|35|16|5|13|18|51|45|43|27|2570|21] -160.85.255.180,192.168.88.244,1344,416,0,2020-09-07T06:52:40.858553,2020-09-07T06:52:40.965848,d8:58:d7:00:c9:27,08:f8:bc:64:5e:6a,4,2,443,59674,771,0,6,24,24,"http/1.1",b32309a26951912be7dba376398abc3b,"ja3er.com",[0|14|0|1|10|2|208|14|5|18|0|43|2|11|3|1|3],[23130|0|23|65281|10|11|35|16|5|13|18|51|45|43|27|43690|21] -160.85.255.180,192.168.88.244,1344,416,0,2020-09-07T06:52:40.870218,2020-09-07T06:52:40.972697,d8:58:d7:00:c9:27,08:f8:bc:64:5e:6a,4,2,443,59675,771,0,6,24,24,"http/1.1",b32309a26951912be7dba376398abc3b,"ja3er.com",[0|14|0|1|10|2|208|14|5|18|0|43|2|11|3|1|3],[27242|0|23|65281|10|11|35|16|5|13|18|51|45|43|27|51914|21] -160.85.255.180,192.168.88.244,1344,416,0,2020-09-07T06:52:40.870714,2020-09-07T06:52:40.973197,d8:58:d7:00:c9:27,08:f8:bc:64:5e:6a,4,2,443,59676,771,0,6,24,24,"http/1.1",b32309a26951912be7dba376398abc3b,"ja3er.com",[0|14|0|1|10|2|208|14|5|18|0|43|2|11|3|1|3],[10794|0|23|65281|10|11|35|16|5|13|18|51|45|43|27|19018|21] -160.85.255.180,192.168.88.244,1344,416,0,2020-09-07T06:52:40.870851,2020-09-07T06:52:40.975124,d8:58:d7:00:c9:27,08:f8:bc:64:5e:6a,4,2,443,59677,771,0,6,24,24,"http/1.1",b32309a26951912be7dba376398abc3b,"ja3er.com",[0|14|0|1|10|2|208|14|5|18|0|43|2|11|3|1|3],[23130|0|23|65281|10|11|35|16|5|13|18|51|45|43|27|64250|21] -160.85.255.180,192.168.88.244,1428,6872,0,2020-09-07T06:52:40.477735,2020-09-07T06:52:40.549534,d8:58:d7:00:c9:27,08:f8:bc:64:5e:6a,4,8,443,59672,771,0,6,24,24,"http/1.1",b32309a26951912be7dba376398abc3b,"ja3er.com",[0|14|0|1|10|2|0|14|5|18|0|43|2|11|3|1|211],[10794|0|23|65281|10|11|35|16|5|13|18|51|45|43|27|60138|21] -172.67.69.20,192.168.0.228,610,264,0,2023-03-10T09:04:18.815874,2023-03-10T09:04:18.836732,90:5c:44:2e:bb:e0,08:f8:bc:64:5e:6a,1,1,443,52640,772,0,6,24,24,"",598872011444709307b861ae817a4b60,"recommend.aktualne.cz",[0|26|0|1|10|2|0|14|5|18|0|43|2|7|3|5|1|235],[10794|0|23|65281|10|11|35|16|5|13|18|51|45|43|27|17513|51914|41] -172.67.73.164,192.168.0.228,616,264,0,2023-03-10T09:04:16.603334,2023-03-10T09:04:16.622460,90:5c:44:2e:bb:e0,08:f8:bc:64:5e:6a,1,1,443,52629,772,0,6,24,24,"",598872011444709307b861ae817a4b60,"prod-snowly-sasic.stdout.cz",[0|32|0|1|10|2|0|14|5|18|0|43|2|7|3|5|1|235],[43690|0|23|65281|10|11|35|16|5|13|18|51|45|43|27|17513|56026|41] -18.66.15.50,192.168.0.228,569,286,0,2023-03-10T09:04:17.214373,2023-03-10T09:04:17.235193,90:5c:44:2e:bb:e0,08:f8:bc:64:5e:6a,1,1,443,52637,772,0,6,24,24,"",0d69ff451640d67ee8b5122752834766,"sdk.privacy-center.org",[0|27|0|1|10|2|0|14|5|18|0|43|2|7|3|5|1|41|148],[56026|0|23|65281|10|11|35|16|5|13|18|51|45|43|27|17513|23130|21|41] -185.26.182.106,192.168.0.228,569,0,0,2023-03-10T09:04:00.558609,2023-03-10T09:04:00.558609,90:5c:44:2e:bb:e0,08:f8:bc:64:5e:6a,1,0,443,52614,771,0,6,24,0,"",cd08e31494f9531f560d64c695473da9,"features.opera-api.com",[0|27|0|1|10|2|0|14|5|18|0|43|2|7|3|5|1|193],[43690|0|23|65281|10|11|35|16|5|13|18|51|45|43|27|17513|39578|21] -185.59.208.153,192.168.0.228,682,2025,0,2023-03-10T09:04:16.701535,2023-03-10T09:04:16.737482,90:5c:44:2e:bb:e0,08:f8:bc:64:5e:6a,2,3,443,52630,771,0,6,24,24,"h2",cd08e31494f9531f560d64c695473da9,"delivery.r2b2.cz",[0|21|0|1|10|2|0|14|5|18|0|43|2|7|3|5|1|199],[31354|0|23|65281|10|11|35|16|5|13|18|51|45|43|27|17513|43690|21] -23.218.208.236,192.168.0.228,672,316,0,2023-03-10T09:04:16.722338,2023-03-10T09:04:16.744620,90:5c:44:2e:bb:e0,08:f8:bc:64:5e:6a,1,1,443,52631,772,0,6,24,24,"",598872011444709307b861ae817a4b60,"assets.adobedtm.com",[0|24|0|1|10|2|0|14|5|18|0|43|2|7|3|5|1|299],[64250|0|23|65281|10|11|35|16|5|13|18|51|45|43|27|17513|35466|41] -46.255.231.124,192.168.0.228,594,255,0,2023-03-10T09:04:17.063884,2023-03-10T09:04:17.089634,90:5c:44:2e:bb:e0,08:f8:bc:64:5e:6a,1,1,443,52634,772,0,6,24,24,"",598872011444709307b861ae817a4b60,"i0.cz",[0|10|0|1|10|2|0|14|5|18|0|43|2|7|3|5|1|235],[10794|0|23|65281|10|11|35|16|5|13|18|51|45|43|27|17513|56026|41] -46.255.231.204,192.168.0.228,615,255,0,2023-03-10T09:04:17.338077,2023-03-10T09:04:17.355878,90:5c:44:2e:bb:e0,08:f8:bc:64:5e:6a,1,1,443,52638,772,0,6,24,24,"",598872011444709307b861ae817a4b60,"pocasi-backend.aktualne.cz",[0|31|0|1|10|2|0|14|5|18|0|43|2|7|3|5|1|235],[60138|0|23|65281|10|11|35|16|5|13|18|51|45|43|27|17513|14906|41] -52.84.193.121,192.168.0.228,569,286,0,2023-03-10T09:04:16.968115,2023-03-10T09:04:17.000426,90:5c:44:2e:bb:e0,08:f8:bc:64:5e:6a,1,1,443,52632,772,0,6,24,24,"",0d69ff451640d67ee8b5122752834766,"d27xxe7juh1us6.cloudfront.net",[0|34|0|1|10|2|0|14|5|18|0|43|2|7|3|5|1|34|148],[19018|0|23|65281|10|11|35|16|5|13|18|51|45|43|27|17513|47802|21|41] -54.226.148.116,192.168.0.228,747,1689,0,2023-03-10T09:04:17.182933,2023-03-10T09:04:17.411187,90:5c:44:2e:bb:e0,08:f8:bc:64:5e:6a,2,6,443,52636,771,0,6,24,24,"h2",cd08e31494f9531f560d64c695473da9,"goldengate.grammarly.com",[0|29|0|1|10|2|105|14|5|18|0|43|2|7|3|5|1|86],[39578|0|23|65281|10|11|35|16|5|13|18|51|45|43|27|17513|51914|21] -82.145.216.15,192.168.88.244,1428,8616,0,2020-09-07T06:52:41.372241,2020-09-07T06:52:41.418203,d8:58:d7:00:c9:27,08:f8:bc:64:5e:6a,4,8,443,59678,771,0,6,24,24,"http/1.1",b32309a26951912be7dba376398abc3b,"af.opera.com",[0|17|0|1|10|2|0|14|5|18|0|43|2|11|3|1|208],[2570|0|23|65281|10|11|35|16|5|13|18|51|45|43|27|6682|21] -82.145.216.16,192.168.0.228,569,1340,0,2023-03-10T09:04:16.204659,2023-03-10T09:04:16.235645,90:5c:44:2e:bb:e0,08:f8:bc:64:5e:6a,1,1,443,52626,772,0,6,24,16,"",cd08e31494f9531f560d64c695473da9,"sitecheck.opera.com",[0|24|0|1|10|2|0|14|5|18|0|43|2|7|3|5|1|196],[39578|0|23|65281|10|11|35|16|5|13|18|51|45|43|27|17513|43690|21] -82.145.216.16,192.168.0.228,569,1460,0,2023-03-10T09:04:17.460626,2023-03-10T09:04:17.499628,90:5c:44:2e:bb:e0,08:f8:bc:64:5e:6a,1,1,443,52639,772,0,6,24,16,"",cd08e31494f9531f560d64c695473da9,"af.opera.com",[0|17|0|1|10|2|0|14|5|18|0|43|2|7|3|5|1|203],[27242|0|23|65281|10|11|35|16|5|13|18|51|45|43|27|17513|39578|21] -87.106.189.123,172.16.121.155,2170,8910,0,2015-10-20T07:09:33.614159,2015-10-20T07:09:34.161736,00:50:56:e5:80:5b,00:0c:29:9d:b9:d0,14,18,443,3923,771,0,6,26,26,"",9a7b51089c089491dbc4879218db549c,"asecuritysite.com",[1|22|0|0|22|5|0|0|29|0|2|6],[65281|0|23|35|13|5|13172|18|16|30032|11|10] -87.106.189.123,172.16.121.155,992,650,0,2015-10-20T07:09:33.611190,2015-10-20T07:09:33.712851,00:50:56:e5:80:5b,00:0c:29:9d:b9:d0,8,8,443,3919,771,0,6,26,26,"",9a7b51089c089491dbc4879218db549c,"asecuritysite.com",[1|22|0|0|22|5|0|0|29|0|2|6],[65281|0|23|35|13|5|13172|18|16|30032|11|10] -87.106.189.123,172.16.121.155,992,650,0,2015-10-20T07:09:33.612842,2015-10-20T07:09:33.714751,00:50:56:e5:80:5b,00:0c:29:9d:b9:d0,8,8,443,3920,771,0,6,26,26,"",9a7b51089c089491dbc4879218db549c,"asecuritysite.com",[1|22|0|0|22|5|0|0|29|0|2|6],[65281|0|23|35|13|5|13172|18|16|30032|11|10] -87.106.189.123,172.16.121.155,992,650,0,2015-10-20T07:09:33.613610,2015-10-20T07:09:33.716640,00:50:56:e5:80:5b,00:0c:29:9d:b9:d0,8,8,443,3921,771,0,6,26,26,"",9a7b51089c089491dbc4879218db549c,"asecuritysite.com",[1|22|0|0|22|5|0|0|29|0|2|6],[65281|0|23|35|13|5|13172|18|16|30032|11|10] -87.106.189.123,172.16.121.155,992,650,0,2015-10-20T07:09:33.613897,2015-10-20T07:09:33.713722,00:50:56:e5:80:5b,00:0c:29:9d:b9:d0,8,8,443,3922,771,0,6,26,26,"",9a7b51089c089491dbc4879218db549c,"asecuritysite.com",[1|22|0|0|22|5|0|0|29|0|2|6],[65281|0|23|35|13|5|13172|18|16|30032|11|10] -87.106.189.123,172.16.121.155,992,650,0,2015-10-20T07:09:33.614399,2015-10-20T07:09:33.715479,00:50:56:e5:80:5b,00:0c:29:9d:b9:d0,8,8,443,3924,771,0,6,26,26,"",9a7b51089c089491dbc4879218db549c,"asecuritysite.com",[1|22|0|0|22|5|0|0|29|0|2|6],[65281|0|23|35|13|5|13172|18|16|30032|11|10] -ipaddr DST_IP,ipaddr SRC_IP,uint64 BYTES,uint64 BYTES_REV,uint64 LINK_BIT_FIELD,time TIME_FIRST,time TIME_LAST,macaddr DST_MAC,macaddr SRC_MAC,uint32 PACKETS,uint32 PACKETS_REV,uint16 DST_PORT,uint16 SRC_PORT,uint16 TLS_VERSION,uint8 DIR_BIT_FIELD,uint8 PROTOCOL,uint8 TCP_FLAGS,uint8 TCP_FLAGS_REV,string TLS_ALPN,bytes TLS_JA3,string TLS_SNI,uint16* TLS_EXT_LEN,uint16* TLS_EXT_TYPE +104.26.1.201,192.168.0.228,569,1460,0,2023-03-10T09:04:19.289282,2023-03-10T09:04:19.304938,90:5c:44:2e:bb:e0,08:f8:bc:64:5e:6a,1,1,443,52641,772,0,6,24,16,"",cd08e31494f9531f560d64c695473da9,"t13d1518h2_8daaf6152771_e5627efa2ab1","chrek.stdout.cz",[0|20|0|1|10|2|0|14|5|18|0|43|2|7|3|5|1|200],[31354|0|23|65281|10|11|35|16|5|13|18|51|45|43|27|17513|10794|21] +104.26.6.183,192.168.0.228,599,264,0,2023-03-10T09:04:17.091196,2023-03-10T09:04:17.110050,90:5c:44:2e:bb:e0,08:f8:bc:64:5e:6a,1,1,443,52635,772,0,6,24,24,"",598872011444709307b861ae817a4b60,"t13d1518h2_8daaf6152771_9b887d9acb53","cdn.xsd.cz",[0|15|0|1|10|2|0|14|5|18|0|43|2|7|3|5|1|235],[14906|0|23|65281|10|11|35|16|5|13|18|51|45|43|27|17513|10794|41] +104.26.8.145,192.168.0.228,604,264,0,2023-03-10T09:04:16.211748,2023-03-10T09:04:16.225609,90:5c:44:2e:bb:e0,08:f8:bc:64:5e:6a,1,1,443,52627,772,0,6,24,24,"",598872011444709307b861ae817a4b60,"t13d1518h2_8daaf6152771_9b887d9acb53","www.aktualne.cz",[0|20|0|1|10|2|0|14|5|18|0|43|2|7|3|5|1|235],[2570|0|23|65281|10|11|35|16|5|13|18|51|45|43|27|17513|35466|41] +160.85.255.180,192.168.88.244,1344,416,0,2020-09-07T06:52:40.858429,2020-09-07T06:52:40.964957,d8:58:d7:00:c9:27,08:f8:bc:64:5e:6a,4,2,443,59673,771,0,6,24,24,"http/1.1",b32309a26951912be7dba376398abc3b,"t13d1517h2_8daaf6152771_de4a06bb82e3","ja3er.com",[0|14|0|1|10|2|208|14|5|18|0|43|2|11|3|1|3],[47802|0|23|65281|10|11|35|16|5|13|18|51|45|43|27|2570|21] +160.85.255.180,192.168.88.244,1344,416,0,2020-09-07T06:52:40.858553,2020-09-07T06:52:40.965848,d8:58:d7:00:c9:27,08:f8:bc:64:5e:6a,4,2,443,59674,771,0,6,24,24,"http/1.1",b32309a26951912be7dba376398abc3b,"t13d1517h2_8daaf6152771_de4a06bb82e3","ja3er.com",[0|14|0|1|10|2|208|14|5|18|0|43|2|11|3|1|3],[23130|0|23|65281|10|11|35|16|5|13|18|51|45|43|27|43690|21] +160.85.255.180,192.168.88.244,1344,416,0,2020-09-07T06:52:40.870218,2020-09-07T06:52:40.972697,d8:58:d7:00:c9:27,08:f8:bc:64:5e:6a,4,2,443,59675,771,0,6,24,24,"http/1.1",b32309a26951912be7dba376398abc3b,"t13d1517h2_8daaf6152771_de4a06bb82e3","ja3er.com",[0|14|0|1|10|2|208|14|5|18|0|43|2|11|3|1|3],[27242|0|23|65281|10|11|35|16|5|13|18|51|45|43|27|51914|21] +160.85.255.180,192.168.88.244,1344,416,0,2020-09-07T06:52:40.870714,2020-09-07T06:52:40.973197,d8:58:d7:00:c9:27,08:f8:bc:64:5e:6a,4,2,443,59676,771,0,6,24,24,"http/1.1",b32309a26951912be7dba376398abc3b,"t13d1517h2_8daaf6152771_de4a06bb82e3","ja3er.com",[0|14|0|1|10|2|208|14|5|18|0|43|2|11|3|1|3],[10794|0|23|65281|10|11|35|16|5|13|18|51|45|43|27|19018|21] +160.85.255.180,192.168.88.244,1344,416,0,2020-09-07T06:52:40.870851,2020-09-07T06:52:40.975124,d8:58:d7:00:c9:27,08:f8:bc:64:5e:6a,4,2,443,59677,771,0,6,24,24,"http/1.1",b32309a26951912be7dba376398abc3b,"t13d1517h2_8daaf6152771_de4a06bb82e3","ja3er.com",[0|14|0|1|10|2|208|14|5|18|0|43|2|11|3|1|3],[23130|0|23|65281|10|11|35|16|5|13|18|51|45|43|27|64250|21] +160.85.255.180,192.168.88.244,1428,6872,0,2020-09-07T06:52:40.477735,2020-09-07T06:52:40.549534,d8:58:d7:00:c9:27,08:f8:bc:64:5e:6a,4,8,443,59672,771,0,6,24,24,"http/1.1",b32309a26951912be7dba376398abc3b,"t13d1517h2_8daaf6152771_de4a06bb82e3","ja3er.com",[0|14|0|1|10|2|0|14|5|18|0|43|2|11|3|1|211],[10794|0|23|65281|10|11|35|16|5|13|18|51|45|43|27|60138|21] +172.67.69.20,192.168.0.228,610,264,0,2023-03-10T09:04:18.815874,2023-03-10T09:04:18.836732,90:5c:44:2e:bb:e0,08:f8:bc:64:5e:6a,1,1,443,52640,772,0,6,24,24,"",598872011444709307b861ae817a4b60,"t13d1518h2_8daaf6152771_9b887d9acb53","recommend.aktualne.cz",[0|26|0|1|10|2|0|14|5|18|0|43|2|7|3|5|1|235],[10794|0|23|65281|10|11|35|16|5|13|18|51|45|43|27|17513|51914|41] +172.67.73.164,192.168.0.228,616,264,0,2023-03-10T09:04:16.603334,2023-03-10T09:04:16.622460,90:5c:44:2e:bb:e0,08:f8:bc:64:5e:6a,1,1,443,52629,772,0,6,24,24,"",598872011444709307b861ae817a4b60,"t13d1518h2_8daaf6152771_9b887d9acb53","prod-snowly-sasic.stdout.cz",[0|32|0|1|10|2|0|14|5|18|0|43|2|7|3|5|1|235],[43690|0|23|65281|10|11|35|16|5|13|18|51|45|43|27|17513|56026|41] +18.66.15.50,192.168.0.228,569,286,0,2023-03-10T09:04:17.214373,2023-03-10T09:04:17.235193,90:5c:44:2e:bb:e0,08:f8:bc:64:5e:6a,1,1,443,52637,772,0,6,24,24,"",0d69ff451640d67ee8b5122752834766,"t13d1519h2_8daaf6152771_6cdcb247c39b","sdk.privacy-center.org",[0|27|0|1|10|2|0|14|5|18|0|43|2|7|3|5|1|41|148],[56026|0|23|65281|10|11|35|16|5|13|18|51|45|43|27|17513|23130|21|41] +185.26.182.106,192.168.0.228,569,0,0,2023-03-10T09:04:00.558609,2023-03-10T09:04:00.558609,90:5c:44:2e:bb:e0,08:f8:bc:64:5e:6a,1,0,443,52614,771,0,6,24,0,"",cd08e31494f9531f560d64c695473da9,"t13d1518h2_8daaf6152771_e5627efa2ab1","features.opera-api.com",[0|27|0|1|10|2|0|14|5|18|0|43|2|7|3|5|1|193],[43690|0|23|65281|10|11|35|16|5|13|18|51|45|43|27|17513|39578|21] +185.59.208.153,192.168.0.228,682,2025,0,2023-03-10T09:04:16.701535,2023-03-10T09:04:16.737482,90:5c:44:2e:bb:e0,08:f8:bc:64:5e:6a,2,3,443,52630,771,0,6,24,24,"h2",cd08e31494f9531f560d64c695473da9,"t13d1518h2_8daaf6152771_e5627efa2ab1","delivery.r2b2.cz",[0|21|0|1|10|2|0|14|5|18|0|43|2|7|3|5|1|199],[31354|0|23|65281|10|11|35|16|5|13|18|51|45|43|27|17513|43690|21] +23.218.208.236,192.168.0.228,672,316,0,2023-03-10T09:04:16.722338,2023-03-10T09:04:16.744620,90:5c:44:2e:bb:e0,08:f8:bc:64:5e:6a,1,1,443,52631,772,0,6,24,24,"",598872011444709307b861ae817a4b60,"t13d1518h2_8daaf6152771_9b887d9acb53","assets.adobedtm.com",[0|24|0|1|10|2|0|14|5|18|0|43|2|7|3|5|1|299],[64250|0|23|65281|10|11|35|16|5|13|18|51|45|43|27|17513|35466|41] +46.255.231.124,192.168.0.228,594,255,0,2023-03-10T09:04:17.063884,2023-03-10T09:04:17.089634,90:5c:44:2e:bb:e0,08:f8:bc:64:5e:6a,1,1,443,52634,772,0,6,24,24,"",598872011444709307b861ae817a4b60,"t13d1518h2_8daaf6152771_9b887d9acb53","i0.cz",[0|10|0|1|10|2|0|14|5|18|0|43|2|7|3|5|1|235],[10794|0|23|65281|10|11|35|16|5|13|18|51|45|43|27|17513|56026|41] +46.255.231.204,192.168.0.228,615,255,0,2023-03-10T09:04:17.338077,2023-03-10T09:04:17.355878,90:5c:44:2e:bb:e0,08:f8:bc:64:5e:6a,1,1,443,52638,772,0,6,24,24,"",598872011444709307b861ae817a4b60,"t13d1518h2_8daaf6152771_9b887d9acb53","pocasi-backend.aktualne.cz",[0|31|0|1|10|2|0|14|5|18|0|43|2|7|3|5|1|235],[60138|0|23|65281|10|11|35|16|5|13|18|51|45|43|27|17513|14906|41] +52.84.193.121,192.168.0.228,569,286,0,2023-03-10T09:04:16.968115,2023-03-10T09:04:17.000426,90:5c:44:2e:bb:e0,08:f8:bc:64:5e:6a,1,1,443,52632,772,0,6,24,24,"",0d69ff451640d67ee8b5122752834766,"t13d1519h2_8daaf6152771_6cdcb247c39b","d27xxe7juh1us6.cloudfront.net",[0|34|0|1|10|2|0|14|5|18|0|43|2|7|3|5|1|34|148],[19018|0|23|65281|10|11|35|16|5|13|18|51|45|43|27|17513|47802|21|41] +54.226.148.116,192.168.0.228,747,1689,0,2023-03-10T09:04:17.182933,2023-03-10T09:04:17.411187,90:5c:44:2e:bb:e0,08:f8:bc:64:5e:6a,2,6,443,52636,771,0,6,24,24,"h2",cd08e31494f9531f560d64c695473da9,"t13d1518h2_8daaf6152771_e5627efa2ab1","goldengate.grammarly.com",[0|29|0|1|10|2|105|14|5|18|0|43|2|7|3|5|1|86],[39578|0|23|65281|10|11|35|16|5|13|18|51|45|43|27|17513|51914|21] +82.145.216.15,192.168.88.244,1428,8616,0,2020-09-07T06:52:41.372241,2020-09-07T06:52:41.418203,d8:58:d7:00:c9:27,08:f8:bc:64:5e:6a,4,8,443,59678,771,0,6,24,24,"http/1.1",b32309a26951912be7dba376398abc3b,"t13d1517h2_8daaf6152771_de4a06bb82e3","af.opera.com",[0|17|0|1|10|2|0|14|5|18|0|43|2|11|3|1|208],[2570|0|23|65281|10|11|35|16|5|13|18|51|45|43|27|6682|21] +82.145.216.16,192.168.0.228,569,1340,0,2023-03-10T09:04:16.204659,2023-03-10T09:04:16.235645,90:5c:44:2e:bb:e0,08:f8:bc:64:5e:6a,1,1,443,52626,772,0,6,24,16,"",cd08e31494f9531f560d64c695473da9,"t13d1518h2_8daaf6152771_e5627efa2ab1","sitecheck.opera.com",[0|24|0|1|10|2|0|14|5|18|0|43|2|7|3|5|1|196],[39578|0|23|65281|10|11|35|16|5|13|18|51|45|43|27|17513|43690|21] +82.145.216.16,192.168.0.228,569,1460,0,2023-03-10T09:04:17.460626,2023-03-10T09:04:17.499628,90:5c:44:2e:bb:e0,08:f8:bc:64:5e:6a,1,1,443,52639,772,0,6,24,16,"",cd08e31494f9531f560d64c695473da9,"t13d1518h2_8daaf6152771_e5627efa2ab1","af.opera.com",[0|17|0|1|10|2|0|14|5|18|0|43|2|7|3|5|1|203],[27242|0|23|65281|10|11|35|16|5|13|18|51|45|43|27|17513|39578|21] +87.106.189.123,172.16.121.155,2170,8910,0,2015-10-20T07:09:33.614159,2015-10-20T07:09:34.161736,00:50:56:e5:80:5b,00:0c:29:9d:b9:d0,14,18,443,3923,771,0,6,26,26,"",9a7b51089c089491dbc4879218db549c,"t12d1612h1_94fc43e2fc61_c9eaec7dbab4","asecuritysite.com",[1|22|0|0|22|5|0|0|29|0|2|6],[65281|0|23|35|13|5|13172|18|16|30032|11|10] +87.106.189.123,172.16.121.155,992,650,0,2015-10-20T07:09:33.611190,2015-10-20T07:09:33.712851,00:50:56:e5:80:5b,00:0c:29:9d:b9:d0,8,8,443,3919,771,0,6,26,26,"",9a7b51089c089491dbc4879218db549c,"t12d1612h1_94fc43e2fc61_c9eaec7dbab4","asecuritysite.com",[1|22|0|0|22|5|0|0|29|0|2|6],[65281|0|23|35|13|5|13172|18|16|30032|11|10] +87.106.189.123,172.16.121.155,992,650,0,2015-10-20T07:09:33.612842,2015-10-20T07:09:33.714751,00:50:56:e5:80:5b,00:0c:29:9d:b9:d0,8,8,443,3920,771,0,6,26,26,"",9a7b51089c089491dbc4879218db549c,"t12d1612h1_94fc43e2fc61_c9eaec7dbab4","asecuritysite.com",[1|22|0|0|22|5|0|0|29|0|2|6],[65281|0|23|35|13|5|13172|18|16|30032|11|10] +87.106.189.123,172.16.121.155,992,650,0,2015-10-20T07:09:33.613610,2015-10-20T07:09:33.716640,00:50:56:e5:80:5b,00:0c:29:9d:b9:d0,8,8,443,3921,771,0,6,26,26,"",9a7b51089c089491dbc4879218db549c,"t12d1612h1_94fc43e2fc61_c9eaec7dbab4","asecuritysite.com",[1|22|0|0|22|5|0|0|29|0|2|6],[65281|0|23|35|13|5|13172|18|16|30032|11|10] +87.106.189.123,172.16.121.155,992,650,0,2015-10-20T07:09:33.613897,2015-10-20T07:09:33.713722,00:50:56:e5:80:5b,00:0c:29:9d:b9:d0,8,8,443,3922,771,0,6,26,26,"",9a7b51089c089491dbc4879218db549c,"t12d1612h1_94fc43e2fc61_c9eaec7dbab4","asecuritysite.com",[1|22|0|0|22|5|0|0|29|0|2|6],[65281|0|23|35|13|5|13172|18|16|30032|11|10] +87.106.189.123,172.16.121.155,992,650,0,2015-10-20T07:09:33.614399,2015-10-20T07:09:33.715479,00:50:56:e5:80:5b,00:0c:29:9d:b9:d0,8,8,443,3924,771,0,6,26,26,"",9a7b51089c089491dbc4879218db549c,"t12d1612h1_94fc43e2fc61_c9eaec7dbab4","asecuritysite.com",[1|22|0|0|22|5|0|0|29|0|2|6],[65281|0|23|35|13|5|13172|18|16|30032|11|10] +ipaddr DST_IP,ipaddr SRC_IP,uint64 BYTES,uint64 BYTES_REV,uint64 LINK_BIT_FIELD,time TIME_FIRST,time TIME_LAST,macaddr DST_MAC,macaddr SRC_MAC,uint32 PACKETS,uint32 PACKETS_REV,uint16 DST_PORT,uint16 SRC_PORT,uint16 TLS_VERSION,uint8 DIR_BIT_FIELD,uint8 PROTOCOL,uint8 TCP_FLAGS,uint8 TCP_FLAGS_REV,string TLS_ALPN,bytes TLS_JA3,string TLS_JA4,string TLS_SNI,uint16* TLS_EXT_LEN,uint16* TLS_EXT_TYPE