Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add Yandex services detection #1882

Merged
merged 4 commits into from
Feb 9, 2023
Merged

Add Yandex services detection #1882

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
9844ce1
Add Yandex services detection
0xA50C1A1 Feb 5, 2023
b032e13
Add VK and Yandex to the TLS certificate match list
0xA50C1A1 Feb 6, 2023
49fcc99
Truncate and merge Yandex PCAP samples into a single file
0xA50C1A1 Feb 6, 2023
75b66c6
Update Yandex ASN list
0xA50C1A1 Feb 6, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 8 additions & 8 deletions src/include/ndpi_protocol_ids.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,16 +53,16 @@ typedef enum {
NDPI_PROTOCOL_VK = 22,
NDPI_PROTOCOL_MAIL_POPS = 23,
NDPI_PROTOCOL_TAILSCALE = 24,
NDPI_PROTOCOL_FREE_25 = 25, /* FREE */
NDPI_PROTOCOL_YANDEX = 25,
NDPI_PROTOCOL_NTOP = 26,
NDPI_PROTOCOL_COAP = 27,
NDPI_PROTOCOL_VMWARE = 28,
NDPI_PROTOCOL_MAIL_SMTPS = 29,
NDPI_PROTOCOL_DTLS = 30,
NDPI_PROTOCOL_UBNTAC2 = 31, /* Ubiquity UBNT AirControl = 2 */
NDPI_PROTOCOL_KONTIKI = 32,
NDPI_PROTOCOL_FREE_33 = 33, /* FREE */
NDPI_PROTOCOL_FREE_34 = 34, /* FREE */
NDPI_PROTOCOL_YANDEX_MAIL = 33,
NDPI_PROTOCOL_YANDEX_MUSIC = 34,
NDPI_PROTOCOL_GNUTELLA = 35,
NDPI_PROTOCOL_EDONKEY = 36,
NDPI_PROTOCOL_BITTORRENT = 37,
Expand All @@ -84,13 +84,13 @@ typedef enum {
NDPI_PROTOCOL_CPHA = 53,
NDPI_PROTOCOL_PPSTREAM = 54,
NDPI_PROTOCOL_ZATTOO = 55,
NDPI_PROTOCOL_FREE_56 = 56, /* FREE */
NDPI_PROTOCOL_FREE_57 = 57, /* FREE */
NDPI_PROTOCOL_YANDEX_MARKET = 56,
NDPI_PROTOCOL_YANDEX_DISK = 57,
NDPI_PROTOCOL_DISCORD = 58,
NDPI_PROTOCOL_TVUPLAYER = 59,
NDPI_PROTOCOL_MONGODB = 60,
NDPI_PROTOCOL_PLURALSIGHT = 61,
NDPI_PROTOCOL_FREE_62 = 62, /* FREE */
NDPI_PROTOCOL_YANDEX_CLOUD = 62,
NDPI_PROTOCOL_OCSP = 63,
NDPI_PROTOCOL_VXLAN = 64,
NDPI_PROTOCOL_IRC = 65,
Expand Down Expand Up @@ -126,8 +126,8 @@ typedef enum {
NDPI_PROTOCOL_IAX = 95,
NDPI_PROTOCOL_TFTP = 96,
NDPI_PROTOCOL_AFP = 97,
NDPI_PROTOCOL_FREE_98 = 98, /* FREE */
NDPI_PROTOCOL_FREE_99 = 99, /* FREE */
NDPI_PROTOCOL_YANDEX_METRIKA = 98,
NDPI_PROTOCOL_YANDEX_DIRECT = 99,
NDPI_PROTOCOL_SIP = 100,
NDPI_PROTOCOL_TRUPHONE = 101,
NDPI_PROTOCOL_IP_ICMPV6 = 102,
Expand Down
45 changes: 45 additions & 0 deletions src/lib/inc_generated/ndpi_asn_yandex.c.inc
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
/*
*
* This file is generated automatically and part of nDPI
*
* nDPI is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* nDPI is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with nDPI. If not, see <http://www.gnu.org/licenses/>.
*
*/

/* ****************************************************** */


static ndpi_network ndpi_protocol_yandex_protocol_list[] = {
{ 0x052DC000 /* 5.45.192.0/18 */, 18, NDPI_PROTOCOL_YANDEX },
{ 0x05FFC000 /* 5.255.192.0/18 */, 18, NDPI_PROTOCOL_YANDEX },
{ 0x25094000 /* 37.9.64.0/18 */, 18, NDPI_PROTOCOL_YANDEX },
{ 0x258C8000 /* 37.140.128.0/18 */, 18, NDPI_PROTOCOL_YANDEX },
{ 0x4D580000 /* 77.88.0.0/18 */, 18, NDPI_PROTOCOL_YANDEX },
{ 0x54FCA000 /* 84.252.160.0/19 */, 19, NDPI_PROTOCOL_YANDEX },
{ 0x57FAE000 /* 87.250.224.0/19 */, 19, NDPI_PROTOCOL_YANDEX },
{ 0x5A9CB000 /* 90.156.176.0/22 */, 22, NDPI_PROTOCOL_YANDEX },
{ 0x5A9CB400 /* 90.156.180.0/23 */, 23, NDPI_PROTOCOL_YANDEX },
{ 0x5A9CB600 /* 90.156.182.0/24 */, 24, NDPI_PROTOCOL_YANDEX },
{ 0x5D9E8000 /* 93.158.128.0/18 */, 18, NDPI_PROTOCOL_YANDEX },
{ 0x5F6C8000 /* 95.108.128.0/17 */, 17, NDPI_PROTOCOL_YANDEX },
{ 0x642B4000 /* 100.43.64.0/19 */, 19, NDPI_PROTOCOL_YANDEX },
{ 0x8D088000 /* 141.8.128.0/18 */, 18, NDPI_PROTOCOL_YANDEX },
{ 0xB29A8000 /* 178.154.128.0/18 */, 18, NDPI_PROTOCOL_YANDEX },
{ 0xB920B800 /* 185.32.184.0/22 */, 22, NDPI_PROTOCOL_YANDEX },
{ 0xC7156000 /* 199.21.96.0/22 */, 22, NDPI_PROTOCOL_YANDEX },
{ 0xC724F000 /* 199.36.240.0/22 */, 22, NDPI_PROTOCOL_YANDEX },
{ 0xD5B4C000 /* 213.180.192.0/19 */, 19, NDPI_PROTOCOL_YANDEX },
/* End */
{ 0x0, 0, 0 }
};
37 changes: 37 additions & 0 deletions src/lib/inc_generated/ndpi_asn_yandex_cloud.c.inc
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
/*
*
* This file is generated automatically and part of nDPI
*
* nDPI is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* nDPI is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with nDPI. If not, see <http://www.gnu.org/licenses/>.
*
*/

/* ****************************************************** */


static ndpi_network ndpi_protocol_yandex_cloud_protocol_list[] = {
{ 0x33FA0000 /* 51.250.0.0/17 */, 17, NDPI_PROTOCOL_YANDEX_CLOUD },
{ 0x3E547000 /* 62.84.112.0/20 */, 20, NDPI_PROTOCOL_YANDEX_CLOUD },
{ 0x54C98000 /* 84.201.128.0/18 */, 18, NDPI_PROTOCOL_YANDEX_CLOUD },
{ 0x54FC8000 /* 84.252.128.0/20 */, 20, NDPI_PROTOCOL_YANDEX_CLOUD },
{ 0x59A98000 /* 89.169.128.0/18 */, 18, NDPI_PROTOCOL_YANDEX_CLOUD },
{ 0x82C12000 /* 130.193.32.0/19 */, 19, NDPI_PROTOCOL_YANDEX_CLOUD },
{ 0x9EA00000 /* 158.160.0.0/16 */, 16, NDPI_PROTOCOL_YANDEX_CLOUD },
{ 0xB29AC000 /* 178.154.192.0/18 */, 18, NDPI_PROTOCOL_YANDEX_CLOUD },
{ 0xB9CEA400 /* 185.206.164.0/22 */, 22, NDPI_PROTOCOL_YANDEX_CLOUD },
{ 0xC120D800 /* 193.32.216.0/22 */, 22, NDPI_PROTOCOL_YANDEX_CLOUD },
{ 0xD91CE000 /* 217.28.224.0/20 */, 20, NDPI_PROTOCOL_YANDEX_CLOUD },
/* End */
{ 0x0, 0, 0 }
};
12 changes: 12 additions & 0 deletions src/lib/ndpi_content_match.c.inc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1798,6 +1798,16 @@ static ndpi_protocol_match host_match[] =
{ "vkuseraudio.net", "VK", NDPI_PROTOCOL_VK, NDPI_PROTOCOL_CATEGORY_SOCIAL_NETWORK, NDPI_PROTOCOL_FUN, NDPI_PROTOCOL_DEFAULT_LEVEL },
{ "vkuservideo.net", "VK", NDPI_PROTOCOL_VK, NDPI_PROTOCOL_CATEGORY_SOCIAL_NETWORK, NDPI_PROTOCOL_FUN, NDPI_PROTOCOL_DEFAULT_LEVEL },

{ "yandex.", "Yandex", NDPI_PROTOCOL_YANDEX, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_SAFE, NDPI_PROTOCOL_DEFAULT_LEVEL },
{ "yastatic.net", "Yandex", NDPI_PROTOCOL_YANDEX, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_SAFE, NDPI_PROTOCOL_DEFAULT_LEVEL },
{ "mail.yandex.", "YandexMail", NDPI_PROTOCOL_YANDEX_MAIL, NDPI_PROTOCOL_CATEGORY_MAIL, NDPI_PROTOCOL_SAFE, NDPI_PROTOCOL_DEFAULT_LEVEL },
{ "music.yandex.", "YandexMusic", NDPI_PROTOCOL_YANDEX_MUSIC, NDPI_PROTOCOL_CATEGORY_MUSIC, NDPI_PROTOCOL_FUN, NDPI_PROTOCOL_DEFAULT_LEVEL },
{ "market.yandex.", "YandexMarket", NDPI_PROTOCOL_YANDEX_MARKET, NDPI_PROTOCOL_CATEGORY_SHOPPING, NDPI_PROTOCOL_SAFE, NDPI_PROTOCOL_DEFAULT_LEVEL },
{ "disk.yandex.", "YandexDisk", NDPI_PROTOCOL_YANDEX_DISK, NDPI_PROTOCOL_CATEGORY_CLOUD, NDPI_PROTOCOL_SAFE, NDPI_PROTOCOL_DEFAULT_LEVEL },
{ "cloud.yandex.", "YandexCloud", NDPI_PROTOCOL_YANDEX_CLOUD, NDPI_PROTOCOL_CATEGORY_CLOUD, NDPI_PROTOCOL_SAFE, NDPI_PROTOCOL_DEFAULT_LEVEL },
{ "metrika.yandex.", "YandexMetrika", NDPI_PROTOCOL_YANDEX_METRIKA, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_SAFE, NDPI_PROTOCOL_DEFAULT_LEVEL },
{ "direct.yandex.", "YandexDirect", NDPI_PROTOCOL_YANDEX_DIRECT, CUSTOM_CATEGORY_ADVERTISEMENT, NDPI_PROTOCOL_TRACKER_ADS, NDPI_PROTOCOL_DEFAULT_LEVEL },

#ifdef CUSTOM_NDPI_PROTOCOLS
#include "../../../nDPI-custom/custom_ndpi_content_match_host_match.c.inc"
#endif
Expand All @@ -1820,6 +1830,8 @@ static ndpi_tls_cert_name_match tls_certificate_match [] = {
{ "CN=www.update.microsoft.com", NDPI_PROTOCOL_WINDOWS_UPDATE },
{ "CN=*.tunnelbear.com", NDPI_PROTOCOL_TUNNELBEAR },
{ "CN=cloudflareclient.com", NDPI_PROTOCOL_CLOUDFLARE_WARP },
{ "O=V Kontakte LLC", NDPI_PROTOCOL_VK },
{ "O=Yandex LLC", NDPI_PROTOCOL_YANDEX },

{ NULL, 0 }
};
Expand Down
37 changes: 4 additions & 33 deletions src/lib/ndpi_main.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,8 @@
#include "inc_generated/ndpi_asn_discord.c.inc"
#include "inc_generated/ndpi_asn_line.c.inc"
#include "inc_generated/ndpi_asn_vk.c.inc"

#include "inc_generated/ndpi_asn_yandex.c.inc"
#include "inc_generated/ndpi_asn_yandex_cloud.c.inc"

/* Third party libraries */
#include "third_party/include/ndpi_patricia.h"
Expand Down Expand Up @@ -1146,10 +1147,6 @@ static void ndpi_init_protocol_defaults(struct ndpi_detection_module_struct *ndp
"MySQL", NDPI_PROTOCOL_CATEGORY_DATABASE,
ndpi_build_default_ports(ports_a, 3306, 0, 0, 0, 0) /* TCP */,
ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */);
ndpi_set_proto_defaults(ndpi_str, 1 /* cleartext */, 0 /* nw proto */, NDPI_PROTOCOL_POTENTIALLY_DANGEROUS, NDPI_PROTOCOL_FREE_25,
"Free25", NDPI_PROTOCOL_CATEGORY_DOWNLOAD_FT,
ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */,
ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */);
ndpi_set_proto_defaults(ndpi_str, 0 /* encrypted */, 0 /* nw proto */, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_NATS,
"Nats", NDPI_PROTOCOL_CATEGORY_RPC,
ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */,
Expand All @@ -1170,14 +1167,6 @@ static void ndpi_init_protocol_defaults(struct ndpi_detection_module_struct *ndp
"Kontiki", NDPI_PROTOCOL_CATEGORY_MEDIA,
ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */,
ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */);
ndpi_set_proto_defaults(ndpi_str, 1 /* cleartext */, 0 /* nw proto */, NDPI_PROTOCOL_POTENTIALLY_DANGEROUS, NDPI_PROTOCOL_FREE_33,
"Free33", NDPI_PROTOCOL_CATEGORY_DOWNLOAD_FT,
ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */,
ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */);
ndpi_set_proto_defaults(ndpi_str, 1 /* cleartext */, 0 /* nw proto */, NDPI_PROTOCOL_POTENTIALLY_DANGEROUS, NDPI_PROTOCOL_FREE_34,
"Free34", NDPI_PROTOCOL_CATEGORY_DOWNLOAD_FT,
ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */,
ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */);
ndpi_set_proto_defaults(ndpi_str, 1 /* cleartext */, 0 /* nw proto */, NDPI_PROTOCOL_POTENTIALLY_DANGEROUS, NDPI_PROTOCOL_GNUTELLA,
"Gnutella", NDPI_PROTOCOL_CATEGORY_DOWNLOAD_FT,
ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */,
Expand Down Expand Up @@ -1290,14 +1279,6 @@ static void ndpi_init_protocol_defaults(struct ndpi_detection_module_struct *ndp
"Zattoo", NDPI_PROTOCOL_CATEGORY_VIDEO,
ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */,
ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */);
ndpi_set_proto_defaults(ndpi_str, 1 /* cleartext */, 0 /* nw proto */, NDPI_PROTOCOL_FUN, NDPI_PROTOCOL_FREE_56,
"Free56", NDPI_PROTOCOL_CATEGORY_MUSIC,
ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */,
ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */);
ndpi_set_proto_defaults(ndpi_str, 1 /* cleartext */, 0 /* nw proto */, NDPI_PROTOCOL_FUN, NDPI_PROTOCOL_FREE_57,
"Free57", NDPI_PROTOCOL_CATEGORY_VIDEO,
ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */,
ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */);
ndpi_set_proto_defaults(ndpi_str, 0 /* encrypted */, 1 /* app proto */, NDPI_PROTOCOL_FUN, NDPI_PROTOCOL_DISCORD,
"Discord", NDPI_PROTOCOL_CATEGORY_COLLABORATIVE,
ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */,
Expand All @@ -1310,10 +1291,6 @@ static void ndpi_init_protocol_defaults(struct ndpi_detection_module_struct *ndp
"Pluralsight", NDPI_PROTOCOL_CATEGORY_VIDEO,
ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */,
ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */);
ndpi_set_proto_defaults(ndpi_str, 1 /* cleartext */, 1 /* app proto */, NDPI_PROTOCOL_FUN, NDPI_PROTOCOL_FREE_62,
"Free62", NDPI_PROTOCOL_CATEGORY_DOWNLOAD_FT,
ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */,
ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */);
ndpi_set_proto_defaults(ndpi_str, 1 /* cleartext */, 1 /* app proto */, NDPI_PROTOCOL_SAFE, NDPI_PROTOCOL_OCSP,
"OCSP", NDPI_PROTOCOL_CATEGORY_NETWORK,
ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */,
Expand Down Expand Up @@ -1472,14 +1449,6 @@ static void ndpi_init_protocol_defaults(struct ndpi_detection_module_struct *ndp
"CHECKMK", NDPI_PROTOCOL_CATEGORY_DATA_TRANSFER,
ndpi_build_default_ports(ports_a, 6556, 0, 0, 0, 0) /* TCP */,
ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */);
ndpi_set_proto_defaults(ndpi_str, 1 /* cleartext */, 0 /* nw proto */, NDPI_PROTOCOL_POTENTIALLY_DANGEROUS, NDPI_PROTOCOL_FREE_98,
"Free98", NDPI_PROTOCOL_CATEGORY_DOWNLOAD_FT,
ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */,
ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */);
ndpi_set_proto_defaults(ndpi_str, 1 /* cleartext */, 1 /* app proto */, NDPI_PROTOCOL_FUN, NDPI_PROTOCOL_FREE_99,
"Free99", NDPI_PROTOCOL_CATEGORY_DOWNLOAD_FT,
ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */,
ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */);
ndpi_set_proto_defaults(ndpi_str, 1 /* cleartext */, 0 /* nw proto */, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_SIP,
"SIP", NDPI_PROTOCOL_CATEGORY_VOIP,
ndpi_build_default_ports(ports_a, 5060, 5061, 0, 0, 0) /* TCP */,
Expand Down Expand Up @@ -2778,6 +2747,8 @@ struct ndpi_detection_module_struct *ndpi_init_detection_module(ndpi_init_prefs
ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_discord_protocol_list);
ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_line_protocol_list);
ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_vk_protocol_list);
ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_yandex_protocol_list);
ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_yandex_cloud_protocol_list);
}

if(prefs & ndpi_track_flow_payload)
Expand Down
Binary file added tests/pcap/yandex.pcapng
View file
Open in desktop
Binary file not shown.
Loading