From a90c1223480cab66ef7f9eec54f8d629e7c28269 Mon Sep 17 00:00:00 2001 From: Tom cappleman Date: Fri, 6 Aug 2021 14:03:04 +0100 Subject: [PATCH 01/51] [orchagent] Add separate next hop table and orch **What I did** Added a new next hop group table to APP_DB, and orchagent support to use this as an alternative to including next hop information in the route table **Why I did it** Improves performance and occupancy usage when using the new table **How I verified it** Extended SWSS unit tests to cover new code, and ran existing tests Signed-off-by: Thomas Cappleman --- doc/swss-schema.md | 24 +- orchagent/Makefile.am | 1 + orchagent/mplsrouteorch.cpp | 489 ++++++---- orchagent/neighorch.cpp | 4 + orchagent/nexthopkey.h | 9 +- orchagent/nhgorch.cpp | 1386 +++++++++++++++++++++++++++++ orchagent/nhgorch.h | 242 +++++ orchagent/orchdaemon.cpp | 8 +- orchagent/orchdaemon.h | 1 + orchagent/routeorch.cpp | 764 +++++++++------- orchagent/routeorch.h | 44 +- tests/mock_tests/Makefile.am | 1 + tests/mock_tests/aclorch_ut.cpp | 2 +- tests/test_nhg.py | 1477 +++++++++++++++++++++++++++++-- 14 files changed, 3838 insertions(+), 614 deletions(-) create mode 100644 orchagent/nhgorch.cpp create mode 100644 orchagent/nhgorch.h diff --git a/doc/swss-schema.md b/doc/swss-schema.md index 3ccc7b74af..0dab436fdd 100644 --- a/doc/swss-schema.md +++ b/doc/swss-schema.md @@ -159,8 +159,30 @@ and reflects the LAG ports into the redis under: `LAG_TABLE::port` ;Status: Mandatory key = ROUTE_TABLE:prefix nexthop = *prefix, ;IP addresses separated “,” (empty indicates no gateway) - intf = ifindex? PORT_TABLE.key ; zero or more separated by “,” (zero indicates no interface) + ifname = ifindex? PORT_TABLE.key ; zero or more separated by “,” (zero indicates no interface) blackhole = BIT ; Set to 1 if this route is a blackhole (or null0) + weight = weight_list ; List of weights. + nexthop_group = string ; index within the NEXTHOP_GROUP_TABLE, used instead of nexthop and intf fields + +--------------------------------------------- + +###### LABEL_ROUTE_TABLE +; Defines schema for MPLS label route table attributes +key = LABEL_ROUTE_TABLE:mpls_label ; MPLS label +; field = value +nexthop = STRING ; Comma-separated list of nexthops. +ifname = STRING ; Comma-separated list of interfaces. +weight = STRING ; Comma-separated list of weights. +nexthop_group = string ; index within the NEXTHOP_GROUP_TABLE, used instead of nexthop and intf fields + +--------------------------------------------- +### NEXTHOP_GROUP_TABLE + ;Stores a list of groups of one or more next hops + ;Status: Mandatory + key = NEXTHOP_GROUP_TABLE:string ; arbitrary index for the next hop group + nexthop = *prefix, ;IP addresses separated “,” (empty indicates no gateway) + ifname = ifindex? PORT_TABLE.key ; zero or more separated by “,” (zero indicates no interface) + weight = weight_list ; List of weights. --------------------------------------------- ### NEIGH_TABLE diff --git a/orchagent/Makefile.am b/orchagent/Makefile.am index d4f3627203..357c66718f 100644 --- a/orchagent/Makefile.am +++ b/orchagent/Makefile.am @@ -37,6 +37,7 @@ orchagent_SOURCES = \ orchdaemon.cpp \ orch.cpp \ notifications.cpp \ + nhgorch.cpp \ routeorch.cpp \ mplsrouteorch.cpp \ neighorch.cpp \ diff --git a/orchagent/mplsrouteorch.cpp b/orchagent/mplsrouteorch.cpp index 6db46e01a5..c0fb137242 100644 --- a/orchagent/mplsrouteorch.cpp +++ b/orchagent/mplsrouteorch.cpp @@ -6,12 +6,14 @@ #include "swssnet.h" #include "converter.h" #include "crmorch.h" +#include "nhgorch.h" #include "directory.h" extern sai_object_id_t gVirtualRouterId; extern sai_object_id_t gSwitchId; extern CrmOrch *gCrmOrch; +extern NhgOrch *gNhgOrch; void RouteOrch::doLabelTask(Consumer& consumer) { @@ -121,6 +123,8 @@ void RouteOrch::doLabelTask(Consumer& consumer) if (op == SET_COMMAND) { + SWSS_LOG_DEBUG("Set operation"); + string ips; string aliases; string mpls_nhs; @@ -128,6 +132,7 @@ void RouteOrch::doLabelTask(Consumer& consumer) string weights; bool& excp_intfs_flag = ctx.excp_intfs_flag; bool blackhole = false; + string nhg_index; for (auto i : kfvFieldsValues(t)) { @@ -148,85 +153,131 @@ void RouteOrch::doLabelTask(Consumer& consumer) if (fvField(i) == "weight") weights = fvValue(i); + + if (fvField(i) == "nexthop_group") + nhg_index = fvValue(i); } - vector ipv = tokenize(ips, ','); - vector alsv = tokenize(aliases, ','); - vector mpls_nhv = tokenize(mpls_nhs, ','); - /* Resize the ip vector to match ifname vector - * as tokenize(",", ',') will miss the last empty segment. */ - if (alsv.size() == 0 && !blackhole) + SWSS_LOG_INFO("Label route %u has nexthop_group: %s, ips: %s, " + "MPLS NHs: %s, aliases: %s", + label, + nhg_index.c_str(), + ips.c_str(), + mpls_nhs.c_str(), + aliases.c_str()); + + /* + * A route should not fill both nexthop_group and ips / + * aliases. + */ + if (!nhg_index.empty() && (!ips.empty() || !aliases.empty())) { - SWSS_LOG_WARN("Skip the route %s, for it has an empty ifname field.", key.c_str()); + SWSS_LOG_ERROR("Route %s has both nexthop_group and ips/aliases", + key.c_str()); it = consumer.m_toSync.erase(it); continue; } - else if (alsv.size() != ipv.size()) - { - SWSS_LOG_NOTICE("Route %s: resize ipv to match alsv, %zd -> %zd.", key.c_str(), ipv.size(), alsv.size()); - ipv.resize(alsv.size()); - } - /* Set the empty ip(s) to zero - * as IpAddress("") will construct a incorrect ip. */ - for (auto &ip : ipv) + ctx.nhg_index = nhg_index; + + vector ipv; + vector alsv; + vector mpls_nhv; + + /* + * If the nexthop_group is empty, create the next hop group key + * based on the IPs and aliases. Otherwise, get the key from + * the NhgOrch. + */ + if (nhg_index.empty()) { - if (ip.empty()) + + ipv = tokenize(ips, ','); + alsv = tokenize(aliases, ','); + mpls_nhv = tokenize(mpls_nhs, ','); + + /* Resize the ip vector to match ifname vector + * as tokenize(",", ',') will miss the last empty segment. */ + if (alsv.size() == 0 && !blackhole) + { + SWSS_LOG_WARN("Skip the route %s, for it has an empty ifname field.", key.c_str()); + it = consumer.m_toSync.erase(it); + continue; + } + else if (alsv.size() != ipv.size()) { - SWSS_LOG_NOTICE("Route %s: set the empty nexthop ip to zero.", key.c_str()); - ip = "0.0.0.0"; + SWSS_LOG_NOTICE("Route %s: resize ipv to match alsv, %zd -> %zd.", key.c_str(), ipv.size(), alsv.size()); + ipv.resize(alsv.size()); } - } - for (auto alias : alsv) - { - /* skip route to management, docker, loopback - * TODO: for route to loopback interface, the proper - * way is to create loopback interface and then create - * route pointing to it, so that we can traps packets to - * CPU */ - if (alias == "eth0" || alias == "docker0" || - alias == "lo" || !alias.compare(0, strlen(LOOPBACK_PREFIX), LOOPBACK_PREFIX)) + for (auto alias : alsv) { - excp_intfs_flag = true; - break; + /* skip route to management, docker, loopback + * TODO: for route to loopback interface, the proper + * way is to create loopback interface and then create + * route pointing to it, so that we can traps packets to + * CPU */ + if (alias == "eth0" || alias == "docker0" || + alias == "lo" || !alias.compare(0, strlen(LOOPBACK_PREFIX), LOOPBACK_PREFIX)) + { + excp_intfs_flag = true; + break; + } } - } - // TODO: cannot trust m_portsOrch->getPortIdByAlias because sometimes alias is empty - if (excp_intfs_flag) - { - /* If any existing routes are updated to point to the - * above interfaces, remove them from the ASIC. */ - if (removeLabelRoute(ctx)) - it = consumer.m_toSync.erase(it); - else - it++; - continue; - } + // TODO: cannot trust m_portsOrch->getPortIdByAlias because sometimes alias is empty + if (excp_intfs_flag) + { + /* If any existing routes are updated to point to the + * above interfaces, remove them from the ASIC. */ + if (removeLabelRoute(ctx)) + it = consumer.m_toSync.erase(it); + else + it++; + continue; + } - string nhg_str = ""; - NextHopGroupKey& nhg = ctx.nhg; + string nhg_str = ""; + NextHopGroupKey& nhg = ctx.nhg; - if (blackhole) - { - nhg = NextHopGroupKey(); + if (blackhole) + { + nhg = NextHopGroupKey(); + } + else + { + for (uint32_t i = 0; i < ipv.size(); i++) + { + if (i) nhg_str += NHG_DELIMITER; + if (!mpls_nhv.empty() && mpls_nhv[i] != "na") + { + nhg_str += mpls_nhv[i] + LABELSTACK_DELIMITER; + } + nhg_str += ipv[i] + NH_DELIMITER + alsv[i]; + } + + nhg = NextHopGroupKey(nhg_str, weights); + } } else { - for (uint32_t i = 0; i < ipv.size(); i++) + try { - if (i) nhg_str += NHG_DELIMITER; - if (!mpls_nhv.empty() && mpls_nhv[i] != "na") - { - nhg_str += mpls_nhv[i] + LABELSTACK_DELIMITER; - } - nhg_str += ipv[i] + NH_DELIMITER + alsv[i]; + const NextHopGroup& nh_group = gNhgOrch->getNhg(nhg_index); + ctx.nhg = nh_group.getKey(); + ctx.is_temp = nh_group.isTemp(); + } + catch (const std::out_of_range& e) + { + SWSS_LOG_ERROR("Next hop group %s does not exist", + nhg_index.c_str()); + ++it; + continue; } - - nhg = NextHopGroupKey(nhg_str, weights); } + NextHopGroupKey& nhg = ctx.nhg; + if (nhg.getSize() == 1 && nhg.hasIntfNextHop()) { /* blackhole to be done */ @@ -251,7 +302,8 @@ void RouteOrch::doLabelTask(Consumer& consumer) } else if (m_syncdLabelRoutes.find(vrf_id) == m_syncdLabelRoutes.end() || m_syncdLabelRoutes.at(vrf_id).find(label) == m_syncdLabelRoutes.at(vrf_id).end() || - m_syncdLabelRoutes.at(vrf_id).at(label) != nhg) + m_syncdLabelRoutes.at(vrf_id).at(label) != RouteNhg(nhg, nhg_index) || + ctx.is_temp) { if (addLabelRoute(ctx, nhg)) it = consumer.m_toSync.erase(it); @@ -259,12 +311,15 @@ void RouteOrch::doLabelTask(Consumer& consumer) it++; } else + { /* Duplicate entry */ + SWSS_LOG_INFO("Route %s is duplicate entry", key.c_str()); it = consumer.m_toSync.erase(it); + } // If already exhaust the nexthop groups, and there are pending removing routes in bulker, // flush the bulker and possibly collect some released nexthop groups - if (m_nextHopGroupCount >= m_maxNextHopGroupCount && + if (gNhgOrch->getNhgCount() >= gNhgOrch->getMaxNhgCount() && gLabelRouteBulker.removing_entries_count() > 0) { break; @@ -273,6 +328,7 @@ void RouteOrch::doLabelTask(Consumer& consumer) else if (op == DEL_COMMAND) { /* Cannot locate the route or remove succeed */ + SWSS_LOG_DEBUG("Delete operation"); if (removeLabelRoute(ctx)) it = consumer.m_toSync.erase(it); else @@ -341,7 +397,8 @@ void RouteOrch::doLabelTask(Consumer& consumer) } else if (m_syncdLabelRoutes.find(vrf_id) == m_syncdLabelRoutes.end() || m_syncdLabelRoutes.at(vrf_id).find(label) == m_syncdLabelRoutes.at(vrf_id).end() || - m_syncdLabelRoutes.at(vrf_id).at(label) != nhg) + m_syncdLabelRoutes.at(vrf_id).at(label) != RouteNhg(nhg, ctx.nhg_index) || + ctx.is_temp) { if (addLabelRoutePost(ctx, nhg)) it_prev = consumer.m_toSync.erase(it_prev); @@ -381,7 +438,12 @@ void RouteOrch::addTempLabelRoute(LabelRouteBulkContext& ctx, const NextHopGroup /* Remove next hops that are not in m_syncdNextHops */ for (auto it = next_hop_set.begin(); it != next_hop_set.end();) { - if (!m_neighOrch->hasNextHop(*it)) + /* + * Check if the IP next hop exists in NeighOrch. The next hop may be + * a labeled one, which are created by RouteOrch or NhgOrch if the IP + * next hop exists. + */ + if (!m_neighOrch->hasNextHop(it->ipKey())) { SWSS_LOG_INFO("Failed to get next hop %s for %u", (*it).to_string().c_str(), label); @@ -413,6 +475,10 @@ bool RouteOrch::addLabelRoute(LabelRouteBulkContext& ctx, const NextHopGroupKey sai_object_id_t& vrf_id = ctx.vrf_id; Label& label = ctx.label; + SWSS_LOG_NOTICE("Adding route for label %u with next hop(s) %s", + label, + nextHops.to_string().c_str()); + /* next_hop_id indicates the next hop id or next hop group id of this route */ sai_object_id_t next_hop_id; bool blackhole = false; @@ -425,97 +491,102 @@ bool RouteOrch::addLabelRoute(LabelRouteBulkContext& ctx, const NextHopGroupKey auto it_route = m_syncdLabelRoutes.at(vrf_id).find(label); - if (nextHops.getSize() == 0) - { - /* The route is pointing to a blackhole */ - blackhole = true; - } - /* The route is pointing to a next hop */ - else if (nextHops.getSize() == 1) + if (ctx.nhg_index.empty()) { - const NextHopKey& nexthop = *nextHops.getNextHops().begin(); - if (nexthop.isIntfNextHop()) + if (nextHops.getSize() == 0) { - next_hop_id = m_intfsOrch->getRouterIntfsId(nexthop.alias); - /* rif is not created yet */ - if (next_hop_id == SAI_NULL_OBJECT_ID) - { - SWSS_LOG_INFO("Failed to get next hop %s for %u", - nextHops.to_string().c_str(), label); - return false; - } + /* The route is pointing to a blackhole */ + blackhole = true; } - else + /* The route is pointing to a next hop */ + else if (nextHops.getSize() == 1) { - if (m_neighOrch->hasNextHop(nexthop)) + const NextHopKey& nexthop = *nextHops.getNextHops().begin(); + if (nexthop.isIntfNextHop()) { - next_hop_id = m_neighOrch->getNextHopId(nexthop); - } - /* For non-existent MPLS NH, check if IP neighbor NH exists */ - else if (nexthop.isMplsNextHop() && - m_neighOrch->hasNextHop(NextHopKey(nexthop.ip_address, nexthop.alias))) - { - /* since IP neighbor NH exists, neighbor is resolved, add MPLS NH */ - m_neighOrch->addNextHop(nexthop); - next_hop_id = m_neighOrch->getNextHopId(nexthop); + next_hop_id = m_intfsOrch->getRouterIntfsId(nexthop.alias); + /* rif is not created yet */ + if (next_hop_id == SAI_NULL_OBJECT_ID) + { + SWSS_LOG_INFO("Failed to get next hop %s for %u", + nextHops.to_string().c_str(), label); + return false; + } } - /* IP neighbor is not yet resolved */ else { - SWSS_LOG_INFO("Failed to get next hop %s for %u", - nextHops.to_string().c_str(), label); - return false; + if (m_neighOrch->hasNextHop(nexthop)) + { + next_hop_id = m_neighOrch->getNextHopId(nexthop); + } + /* See if there is an IP neighbor nexthop */ + else if (nexthop.isMplsNextHop() && + m_neighOrch->hasNextHop(NextHopKey(nexthop.ip_address, nexthop.alias))) + { + m_neighOrch->addNextHop(nexthop); + next_hop_id = m_neighOrch->getNextHopId(nexthop); + } + else + { + SWSS_LOG_INFO("Failed to get next hop %s for %u", + nextHops.to_string().c_str(), label); + return false; + } } } - } - /* The route is pointing to a next hop group */ - else - { - /* Check if there is already an existing next hop group */ - if (!hasNextHopGroup(nextHops)) + /* The route is pointing to a next hop group */ + else { - /* Try to create a new next hop group */ - if (!addNextHopGroup(nextHops)) + /* Check if there is already an existing next hop group */ + if (!hasNextHopGroup(nextHops)) { - for (auto it = nextHops.getNextHops().begin(); it != nextHops.getNextHops().end(); ++it) + /* Try to create a new next hop group */ + if (!addNextHopGroup(nextHops)) { - const NextHopKey& nextHop = *it; - if (!m_neighOrch->hasNextHop(nextHop)) - { - SWSS_LOG_INFO("Failed to get next hop %s in %s, resolving neighbor", - nextHop.to_string().c_str(), nextHops.to_string().c_str()); - m_neighOrch->resolveNeighbor(nextHop); - } - } + /* Failed to create the next hop group and check if a temporary route is needed */ - /* Failed to create the next hop group and check if a temporary route is needed */ - - /* If the current next hop is part of the next hop group to sync, - * then return false and no need to add another temporary route. */ - if (it_route != m_syncdLabelRoutes.at(vrf_id).end() && it_route->second.getSize() == 1) - { - const NextHopKey& nexthop = *it_route->second.getNextHops().begin(); - if (nextHops.contains(nexthop)) + /* If the current next hop is part of the next hop group to sync, + * then return false and no need to add another temporary route. */ + if (it_route != m_syncdLabelRoutes.at(vrf_id).end() && it_route->second.nhg_key.getSize() == 1) { - return false; + const NextHopKey& nexthop = *it_route->second.nhg_key.getNextHops().begin(); + if (nextHops.contains(nexthop)) + { + return false; + } } - } - /* Add a temporary route when a next hop group cannot be added, - * and there is no temporary route right now or the current temporary - * route is not pointing to a member of the next hop group to sync. */ - addTempLabelRoute(ctx, nextHops); - /* Return false since the original route is not successfully added */ - return false; + /* Add a temporary route when a next hop group cannot be added, + * and there is no temporary route right now or the current temporary + * route is not pointing to a member of the next hop group to sync. */ + addTempLabelRoute(ctx, nextHops); + /* Return false since the original route is not successfully added */ + return false; + } } - } - next_hop_id = m_syncdNextHopGroups[nextHops].next_hop_group_id; + next_hop_id = m_syncdNextHopGroups[nextHops].next_hop_group_id; + } + } + else + { + SWSS_LOG_DEBUG("Next hop group is owned by NhgOrch with index %s", + ctx.nhg_index.c_str()); + try + { + const NextHopGroup& nhg = gNhgOrch->getNhg(ctx.nhg_index); + next_hop_id = nhg.getId(); + } + catch(const std::out_of_range& e) + { + SWSS_LOG_WARN("Next hop group key %s does not exist", + ctx.nhg_index.c_str()); + return false; + } } /* Sync the inseg entry */ sai_inseg_entry_t inseg_entry; - // route_entry.vr_id = vrf_id; No VRF support for MPLS? inseg_entry.switch_id = gSwitchId; inseg_entry.label = label; @@ -559,7 +630,7 @@ bool RouteOrch::addLabelRoute(LabelRouteBulkContext& ctx, const NextHopGroupKey else { /* Set the packet action to forward when there was no next hop (dropped) */ - if (it_route->second.getSize() == 0 && !blackhole) + if (it_route->second.nhg_key.getSize() == 0 && !blackhole) { inseg_attr.id = SAI_INSEG_ENTRY_ATTR_PACKET_ACTION; inseg_attr.value.s32 = SAI_PACKET_ACTION_FORWARD; @@ -607,44 +678,60 @@ bool RouteOrch::addLabelRoutePost(const LabelRouteBulkContext& ctx, const NextHo /* next_hop_id indicates the next hop id or next hop group id of this route */ sai_object_id_t next_hop_id; - if (nextHops.getSize() == 0) - { - /* The route is pointing to a blackhole */ - blackhole = true; - } - if (nextHops.getSize() == 1) + /* Check that the next hop group is not owned by NhgOrch. */ + if (ctx.nhg_index.empty()) { + if (nextHops.getSize() == 0) + { + /* The route is pointing to a blackhole */ + blackhole = true; + } /* The route is pointing to a next hop */ - const NextHopKey& nexthop = *nextHops.getNextHops().begin(); - if (nexthop.isIntfNextHop()) + else if (nextHops.getSize() == 1) { - next_hop_id = m_intfsOrch->getRouterIntfsId(nexthop.alias); - /* rif is not created yet */ - if (next_hop_id == SAI_NULL_OBJECT_ID) + const NextHopKey& nexthop = *nextHops.getNextHops().begin(); + if (nexthop.isIntfNextHop()) { - SWSS_LOG_INFO("Failed to get next hop %s for label %u", - nextHops.to_string().c_str(), label); - return false; + + next_hop_id = m_intfsOrch->getRouterIntfsId(nexthop.alias); + /* rif is not created yet */ + if (next_hop_id == SAI_NULL_OBJECT_ID) + { + SWSS_LOG_INFO("Failed to get next hop %s for label %u", + nextHops.to_string().c_str(), label); + return false; + } + } + else + { + if (!m_neighOrch->hasNextHop(nexthop)) + { + SWSS_LOG_INFO("Failed to get next hop %s for label %u", + nextHops.to_string().c_str(), label); + return false; + } } } - else + /* The route is pointing to a next hop group */ + else if (nextHops.getSize() > 1) { - if (!m_neighOrch->hasNextHop(nexthop)) + if (!hasNextHopGroup(nextHops)) { - SWSS_LOG_INFO("Failed to get next hop %s for label %u", - nextHops.to_string().c_str(), label); + // Previous added an temporary route + auto& tmp_next_hop = ctx.tmp_next_hop; + addLabelRoutePost(ctx, tmp_next_hop); return false; } } } - /* The route is pointing to a next hop group */ else { - if (!hasNextHopGroup(nextHops)) + SWSS_LOG_DEBUG("NhgOrch owns the next hop group with index %s", + ctx.nhg_index.c_str()); + if (!gNhgOrch->hasNhg(ctx.nhg_index)) { - // Previous added an temporary route - auto& tmp_next_hop = ctx.tmp_next_hop; - addLabelRoutePost(ctx, tmp_next_hop); + SWSS_LOG_WARN("Failed to get next hop group with index %s", + ctx.nhg_index.c_str()); return false; } } @@ -668,16 +755,24 @@ bool RouteOrch::addLabelRoutePost(const LabelRouteBulkContext& ctx, const NextHo gCrmOrch->incCrmResUsedCounter(CrmResourceType::CRM_MPLS_INSEG); /* Increase the ref_count for the next hop (group) entry */ - increaseNextHopRefCount(nextHops); - SWSS_LOG_INFO("Post create label %u with next hop(s) %s", - label, nextHops.to_string().c_str()); + if (ctx.nhg_index.empty()) + { + increaseNextHopRefCount(nextHops); + } + else + { + gNhgOrch->incNhgRefCount(ctx.nhg_index); + } + + SWSS_LOG_INFO("Create label route %u with next hop(s) %s", + label, nextHops.to_string().c_str()); } else { sai_status_t status; /* Set the packet action to forward when there was no next hop (dropped) and not pointing to blackhole */ - if (it_route->second.getSize() == 0 && !blackhole) + if (it_route->second.nhg_key.getSize() == 0 && !blackhole) { status = *it_status++; if (status != SAI_STATUS_SUCCESS) @@ -704,14 +799,30 @@ bool RouteOrch::addLabelRoutePost(const LabelRouteBulkContext& ctx, const NextHo } } - /* Increase the ref_count for the next hop (group) entry */ - increaseNextHopRefCount(nextHops); + /* Decrease the ref count for the previous next hop group. */ + if (it_route->second.nhg_index.empty()) + { + decreaseNextHopRefCount(it_route->second.nhg_key); + if (it_route->second.nhg_key.getSize() > 1 + && m_syncdNextHopGroups[it_route->second.nhg_key].ref_count == 0) + { + m_bulkNhgReducedRefCnt.emplace(it_route->second.nhg_key, 0); + } + } + else + { + /* The next hop group is owned by NeighOrch. */ + gNhgOrch->decNhgRefCount(it_route->second.nhg_index); + } - decreaseNextHopRefCount(it_route->second); - if (it_route->second.getSize() > 1 - && m_syncdNextHopGroups[it_route->second].ref_count == 0) + /* Increase the ref_count for the next hop (group) entry */ + if (ctx.nhg_index.empty()) + { + increaseNextHopRefCount(nextHops); + } + else { - m_bulkNhgReducedRefCnt.emplace(it_route->second, 0); + gNhgOrch->incNhgRefCount(ctx.nhg_index); } if (blackhole) @@ -731,10 +842,10 @@ bool RouteOrch::addLabelRoutePost(const LabelRouteBulkContext& ctx, const NextHo } SWSS_LOG_INFO("Post set label %u with next hop(s) %s", - label, nextHops.to_string().c_str()); + label, nextHops.to_string().c_str()); } - m_syncdLabelRoutes[vrf_id][label] = nextHops; + m_syncdLabelRoutes[vrf_id][label] = RouteNhg(nextHops, ctx.nhg_index); return true; } @@ -754,7 +865,6 @@ bool RouteOrch::removeLabelRoute(LabelRouteBulkContext& ctx) } sai_inseg_entry_t inseg_entry; - //inseg_entry.vr_id = vrf_id; No VRF support for MPLS inseg_entry.switch_id = gSwitchId; inseg_entry.label = label; @@ -807,31 +917,38 @@ bool RouteOrch::removeLabelRoutePost(const LabelRouteBulkContext& ctx) gCrmOrch->decCrmResUsedCounter(CrmResourceType::CRM_MPLS_INSEG); - /* - * Decrease the reference count only when the route is pointing to a next hop. - */ - decreaseNextHopRefCount(it_route->second); - if (it_route->second.getSize() > 1 - && m_syncdNextHopGroups[it_route->second].ref_count == 0) - { - m_bulkNhgReducedRefCnt.emplace(it_route->second, 0); - } - /* - * Additionally check if the NH has label and its ref count == 0, then - * remove the label next hop. - */ - else if (it_route->second.getSize() == 1) + if (it_route->second.nhg_index.empty()) { - const NextHopKey& nexthop = *it_route->second.getNextHops().begin(); - if (nexthop.isMplsNextHop() && - (m_neighOrch->getNextHopRefCount(nexthop) == 0)) + /* + * Decrease the reference count only when the route is pointing to a next hop. + */ + decreaseNextHopRefCount(it_route->second.nhg_key); + if (it_route->second.nhg_key.getSize() > 1 + && m_syncdNextHopGroups[it_route->second.nhg_key].ref_count == 0) { - m_neighOrch->removeMplsNextHop(nexthop); + m_bulkNhgReducedRefCnt.emplace(it_route->second.nhg_key, 0); + } + /* + * Additionally check if the NH has label and its ref count == 0, then + * remove the label next hop. + */ + else if (it_route->second.nhg_key.getSize() == 1) + { + NextHopKey nexthop(it_route->second.nhg_key.to_string()); + if (nexthop.isMplsNextHop() && + (m_neighOrch->getNextHopRefCount(nexthop) == 0)) + { + m_neighOrch->removeMplsNextHop(nexthop); + } } } + else + { + gNhgOrch->decNhgRefCount(it_route->second.nhg_index); + } - SWSS_LOG_INFO("Remove label %u with next hop(s) %s", - label, it_route->second.to_string().c_str()); + SWSS_LOG_INFO("Remove label route %u with next hop(s) %s", + label, it_route->second.nhg_key.to_string().c_str()); it_route_table->second.erase(label); @@ -842,4 +959,4 @@ bool RouteOrch::removeLabelRoutePost(const LabelRouteBulkContext& ctx) } return true; -} +} \ No newline at end of file diff --git a/orchagent/neighorch.cpp b/orchagent/neighorch.cpp index e3f9a4175e..10eff1721b 100644 --- a/orchagent/neighorch.cpp +++ b/orchagent/neighorch.cpp @@ -7,6 +7,7 @@ #include "directory.h" #include "muxorch.h" #include "subscriberstatetable.h" +#include "nhgorch.h" extern sai_neighbor_api_t* sai_neighbor_api; extern sai_next_hop_api_t* sai_next_hop_api; @@ -15,6 +16,7 @@ extern PortsOrch *gPortsOrch; extern sai_object_id_t gSwitchId; extern CrmOrch *gCrmOrch; extern RouteOrch *gRouteOrch; +extern NhgOrch *gNhgOrch; extern FgNhgOrch *gFgNhgOrch; extern Directory gDirectory; extern string gMySwitchType; @@ -322,6 +324,7 @@ bool NeighOrch::setNextHopFlag(const NextHopKey &nexthop, const uint32_t nh_flag { case NHFLAGS_IFDOWN: rc = gRouteOrch->invalidnexthopinNextHopGroup(nexthop, count); + rc &= gNhgOrch->invalidateNextHop(nexthop); break; default: assert(0); @@ -351,6 +354,7 @@ bool NeighOrch::clearNextHopFlag(const NextHopKey &nexthop, const uint32_t nh_fl { case NHFLAGS_IFDOWN: rc = gRouteOrch->validnexthopinNextHopGroup(nexthop, count); + rc &= gNhgOrch->validateNextHop(nexthop); break; default: assert(0); diff --git a/orchagent/nexthopkey.h b/orchagent/nexthopkey.h index 87c294415a..f4d2a8fac3 100644 --- a/orchagent/nexthopkey.h +++ b/orchagent/nexthopkey.h @@ -4,6 +4,7 @@ #include "ipaddress.h" #include "tokenize.h" #include "label.h" +#include "intfsorch.h" #define LABELSTACK_DELIMITER '+' #define NH_DELIMITER '@' @@ -160,10 +161,16 @@ struct NextHopKey std::string str; if (isMplsNextHop()) { - label_stack.to_string() + LABELSTACK_DELIMITER; + str = label_stack.to_string() + LABELSTACK_DELIMITER; } return str; } + + // Method to get the underlying IP/interface pair for the next hop. + NextHopKey ipKey() const + { + return NextHopKey(ip_address, alias); + } }; #endif /* SWSS_NEXTHOPKEY_H */ diff --git a/orchagent/nhgorch.cpp b/orchagent/nhgorch.cpp new file mode 100644 index 0000000000..bf83d1e393 --- /dev/null +++ b/orchagent/nhgorch.cpp @@ -0,0 +1,1386 @@ +#include "nhgorch.h" +#include "switchorch.h" +#include "neighorch.h" +#include "crmorch.h" +#include "routeorch.h" +#include "bulker.h" +#include "logger.h" +#include "swssnet.h" + +extern sai_object_id_t gSwitchId; + +extern PortsOrch *gPortsOrch; +extern CrmOrch *gCrmOrch; +extern NeighOrch *gNeighOrch; +extern SwitchOrch *gSwitchOrch; +extern RouteOrch *gRouteOrch; + +extern size_t gMaxBulkSize; + +extern sai_next_hop_group_api_t* sai_next_hop_group_api; +extern sai_next_hop_api_t* sai_next_hop_api; +extern sai_switch_api_t* sai_switch_api; + +unsigned int NextHopGroup::m_count = 0; + +/* Default maximum number of next hop groups */ +#define DEFAULT_NUMBER_OF_ECMP_GROUPS 128 +#define DEFAULT_MAX_ECMP_GROUP_SIZE 32 + +NhgOrch::NhgOrch(DBConnector *db, string tableName) : + Orch(db, tableName) +{ + SWSS_LOG_ENTER(); + + /* Get the switch's maximum next hop group capacity. */ + sai_attribute_t attr; + attr.id = SAI_SWITCH_ATTR_NUMBER_OF_ECMP_GROUPS; + + sai_status_t status = sai_switch_api->get_switch_attribute(gSwitchId, + 1, + &attr); + + if (status != SAI_STATUS_SUCCESS) + { + SWSS_LOG_WARN("Failed to get switch attribute number of ECMP groups. \ + Use default value. rv:%d", status); + m_maxNhgCount = DEFAULT_NUMBER_OF_ECMP_GROUPS; + } + else + { + m_maxNhgCount = attr.value.s32; + + /* + * ASIC specific workaround to re-calculate maximum ECMP groups + * according to diferent ECMP mode used. + * + * On Mellanox platform, the maximum ECMP groups returned is the value + * under the condition that the ECMP group size is 1. Deviding this + * number by DEFAULT_MAX_ECMP_GROUP_SIZE gets the maximum number of + * ECMP groups when the maximum ECMP group size is 32. + */ + char *platform = getenv("platform"); + if (platform && strstr(platform, MLNX_PLATFORM_SUBSTRING)) + { + SWSS_LOG_NOTICE("Mellanox platform - divide capacity by %d", + DEFAULT_MAX_ECMP_GROUP_SIZE); + m_maxNhgCount /= DEFAULT_MAX_ECMP_GROUP_SIZE; + } + } + + /* Set switch's next hop group capacity. */ + vector fvTuple; + fvTuple.emplace_back("MAX_NEXTHOP_GROUP_COUNT", + to_string(m_maxNhgCount)); + gSwitchOrch->set_switch_capability(fvTuple); + + SWSS_LOG_NOTICE("Maximum number of ECMP groups supported is %d", + m_maxNhgCount); +} + +/* + * Purpose: Perform the operations requested by APPL_DB users. + * + * Description: Iterate over the untreated operations list and resolve them. + * The operations supported are SET and DEL. If an operation + * could not be resolved, it will either remain in the list, or be + * removed, depending on the case. + * + * Params: IN consumer - The cosumer object. + * + * Returns: Nothing. + */ +void NhgOrch::doTask(Consumer& consumer) +{ + SWSS_LOG_ENTER(); + + if (!gPortsOrch->allPortsReady()) + { + return; + } + + auto it = consumer.m_toSync.begin(); + + while (it != consumer.m_toSync.end()) + { + KeyOpFieldsValuesTuple t = it->second; + + string index = kfvKey(t); + string op = kfvOp(t); + + SWSS_LOG_INFO("Next hop group table key %s, op %s", + index.c_str(), op.c_str()); + + bool success; + const auto& nhg_it = m_syncdNextHopGroups.find(index); + + if (op == SET_COMMAND) + { + string ips; + string aliases; + string weights; + string mpls_nhs; + + /* Get group's next hop IPs and aliases */ + for (auto i : kfvFieldsValues(t)) + { + if (fvField(i) == "nexthop") + ips = fvValue(i); + + if (fvField(i) == "ifname") + aliases = fvValue(i); + + if (fvField(i) == "weight") + weights = fvValue(i); + + if (fvField(i) == "mpls_nh") + mpls_nhs = fvValue(i); + } + + /* Split ips and alaises strings into vectors of tokens. */ + vector ipv = tokenize(ips, ','); + vector alsv = tokenize(aliases, ','); + vector mpls_nhv = tokenize(mpls_nhs, ','); + + /* Create the next hop group key. */ + string nhg_str; + + for (uint32_t i = 0; i < ipv.size(); i++) + { + if (i) nhg_str += NHG_DELIMITER; + if (!mpls_nhv.empty() && mpls_nhv[i] != "na") + { + nhg_str += mpls_nhv[i] + LABELSTACK_DELIMITER; + } + nhg_str += ipv[i] + NH_DELIMITER + alsv[i]; + } + + NextHopGroupKey nhg_key = NextHopGroupKey(nhg_str, weights); + + /* If the group does not exist, create one. */ + if (nhg_it == m_syncdNextHopGroups.end()) + { + SWSS_LOG_NOTICE("Adding next hop group %s with %s", + index.c_str(), + nhg_str.c_str()); + /* + * If we'd have to create a SAI object for the group, and we + * already reached the limit, we're going to create a temporary + * group, represented by one of it's NH only until we have + * enough resources to sync the whole group. The item is going + * to be kept in the sync list so we keep trying to create the + * actual group when there are enough resources. + */ + if ((nhg_key.getSize() > 1) && + (NextHopGroup::getCount() >= m_maxNhgCount)) + { + SWSS_LOG_WARN("Next hop group count reached its limit."); + + try + { + auto nhg = std::make_unique( + createTempNhg(nhg_key)); + SWSS_LOG_NOTICE("Adding temp next hop group with %s", + nhg->to_string().c_str()); + if (nhg->sync()) + { + SWSS_LOG_INFO("Temporary NHG successfully synced"); + m_syncdNextHopGroups.emplace(index, + NhgEntry(std::move(nhg))); + } + else + { + SWSS_LOG_WARN( + "Failed to sync temporary NHG %s with %s", + index.c_str(), + nhg_key.to_string().c_str()); + } + } + catch (const std::exception& e) + { + SWSS_LOG_WARN( + "Got exception: %s while adding temp group %s", + e.what(), + nhg_key.to_string().c_str()); + } + + /* + * We set the success to false so we keep trying to update + * this group in order to sync the whole group. + */ + success = false; + } + else + { + auto nhg = std::make_unique(nhg_key); + success = nhg->sync(); + + if (success) + { + SWSS_LOG_INFO("NHG successfully synced"); + m_syncdNextHopGroups.emplace(index, + NhgEntry(std::move(nhg))); + } + } + } + /* If the group exists, update it. */ + else + { + SWSS_LOG_NOTICE("Update next hop group %s with %s", + index.c_str(), + nhg_str.c_str()); + + const auto& nhg_ptr = nhg_it->second.nhg; + + + /* + * A NHG update should never change the SAI ID of the NHG if it + * is still referenced by some other objects, as they would not + * be notified about this change. The only exception to this + * rule is for the temporary NHGs, as the referencing objects + * will keep querying the NhgOrch for any SAI ID updates. + */ + if (!nhg_ptr->isTemp() && + ((nhg_key.getSize() == 1) || (nhg_ptr->getSize() == 1)) && + (nhg_it->second.ref_count > 0)) + { + SWSS_LOG_WARN("Next hop group %s update would change SAI " + "ID while referenced, so not performed", + index.c_str()); + success = false; + } + /* + * If the update would mandate promoting a temporary next hop + * group to a multiple next hops group and we do not have the + * resources yet, we have to skip it until we have enough + * resources. + */ + else if (nhg_ptr->isTemp() && + (nhg_key.getSize() > 1) && + (NextHopGroup::getCount() >= m_maxNhgCount)) + { + /* + * If the group was updated in such way that the previously + * chosen next hop does not represent the new group key, + * update the temporary group to choose a new next hop from + * the new key. + */ + if (!nhg_key.contains(nhg_ptr->getKey())) + { + SWSS_LOG_NOTICE("Updating temporary group %s to %s", + index.c_str(), + nhg_key.to_string().c_str()); + + try + { + /* Create the new temporary next hop group. */ + auto new_nhg = std::make_unique( + createTempNhg(nhg_key)); + + /* + * If we successfully sync the new group, update + * only the next hop group entry's pointer so we + * don't mess up the reference counter, as other + * objects may already reference it. + */ + if (new_nhg->sync()) + { + SWSS_LOG_INFO( + "Temporary NHG successfully synced"); + nhg_it->second.nhg = std::move(new_nhg); + } + else + { + SWSS_LOG_WARN( + "Failed to sync updated temp NHG %s with %s", + index.c_str(), + nhg_key.to_string().c_str()); + } + } + catch (const std::exception& e) + { + SWSS_LOG_WARN( + "Got exception: %s while adding temp group %s", + e.what(), + nhg_key.to_string().c_str()); + } + } + + /* + * Because the resources are exhausted, we have to keep + * trying to update the temporary group until we can + * promote it to a fully functional group. + */ + success = false; + } + /* Common update, when all the requirements are met. */ + else + { + success = nhg_ptr->update(nhg_key); + } + } + } + else if (op == DEL_COMMAND) + { + SWSS_LOG_NOTICE("Deleting next hop group %s", index.c_str()); + + /* If the group does not exist, do nothing. */ + if (nhg_it == m_syncdNextHopGroups.end()) + { + SWSS_LOG_WARN("Unable to find group with key %s to remove", + index.c_str()); + /* Mark the operation as successful to consume it. */ + success = true; + } + /* If the group does exist, but it's still referenced, skip. */ + else if (nhg_it->second.ref_count > 0) + { + SWSS_LOG_WARN( + "Unable to remove group %s which is referenced", + index.c_str()); + success = false; + } + /* Else, if the group is no more referenced, delete it. */ + else + { + const auto& nhg = nhg_it->second.nhg; + + success = nhg->desync(); + + if (success) + { + m_syncdNextHopGroups.erase(nhg_it); + } + } + } + else + { + SWSS_LOG_ERROR("Unknown operation type %s\n", op.c_str()); + /* Mark the operation as successful to consume it. */ + success = true; + } + + /* Depending on the operation success, consume it or skip it. */ + if (success) + { + it = consumer.m_toSync.erase(it); + } + else + { + ++it; + } + } +} + +/* + * Purpose: Validate a next hop for any groups that contains it. + * + * Description: Iterate over all next hop groups and validate the next hop in + * those who contain it. + * + * Params: IN nh_key - The next hop to validate. + * + * Returns: true, if the next hop was successfully validated in all + * containing groups; + * false, otherwise. + */ +bool NhgOrch::validateNextHop(const NextHopKey& nh_key) +{ + SWSS_LOG_ENTER(); + SWSS_LOG_INFO("Validating next hop %s", nh_key.to_string().c_str()); + + /* + * Iterate through all groups and validate the next hop in those who + * contain it. + */ + for (auto& it : m_syncdNextHopGroups) + { + auto& nhg = it.second.nhg; + + SWSS_LOG_INFO("Check if next hop in group %s", + it.first.c_str()); + + if (nhg->hasNextHop(nh_key)) + { + SWSS_LOG_INFO("Group has next hop"); + + /* + * If sync fails, exit right away, as we expect it to be due to a + * raeson for which any other future validations will fail too. + */ + if (!nhg->validateNextHop(nh_key)) + { + SWSS_LOG_ERROR("Failed to validate next hop %s in group %s", + nh_key.to_string().c_str(), + it.first.c_str()); + return false; + } + } + } + + return true; +} + +/* + * Purpose: Invalidate a next hop for any groups containing it. + * + * Description: Iterate through the next hop groups and desync the next hop + * from those that contain it. + * + * Params: IN nh_key - The next hop to invalidate. + * + * Returns: true, if the next hop was successfully invalidatedd from all + * containing groups; + * false, otherwise. + */ +bool NhgOrch::invalidateNextHop(const NextHopKey& nh_key) +{ + SWSS_LOG_ENTER(); + SWSS_LOG_INFO("Invalidating next hop %s", nh_key.to_string().c_str()); + + /* + * Iterate through all groups and invalidate the next hop from those who + * contain it. + */ + for (auto& it : m_syncdNextHopGroups) + { + auto& nhg = it.second.nhg; + + SWSS_LOG_INFO("Check if next hop is in group %s", + it.first.c_str()); + + if (nhg->hasNextHop(nh_key)) + { + SWSS_LOG_INFO("Group has next hop"); + + /* If the desync fails, exit right away. */ + if (!nhg->invalidateNextHop(nh_key)) + { + SWSS_LOG_WARN("Failed to invalidate next hop %s from group %s", + nh_key.to_string().c_str(), + it.first.c_str()); + return false; + } + } + } + + return true; +} + +/* + * Purpose: Increase the ref count for a next hop group. + * + * Description: Increment the ref count for a next hop group by 1. + * + * Params: IN index - The index of the next hop group. + * + * Returns: Nothing. + */ +void NhgOrch::incNhgRefCount(const std::string& index) +{ + SWSS_LOG_ENTER(); + + NhgEntry& nhg_entry = m_syncdNextHopGroups.at(index); + + SWSS_LOG_INFO("Increment group %s ref count from %u to %u", + index.c_str(), + nhg_entry.ref_count, + nhg_entry.ref_count + 1); + + ++nhg_entry.ref_count; +} + +/* + * Purpose: Decrease the ref count for a next hop group. + * + * Description: Decrement the ref count for a next hop group by 1. + * + * Params: IN index - The index of the next hop group. + * + * Returns: Nothing. + */ +void NhgOrch::decNhgRefCount(const std::string& index) +{ + SWSS_LOG_ENTER(); + + NhgEntry& nhg_entry = m_syncdNextHopGroups.at(index); + + /* Sanity check so we don't overflow. */ + assert(nhg_entry.ref_count > 0); + + SWSS_LOG_INFO("Decrement group %s ref count from %u to %u", + index.c_str(), + nhg_entry.ref_count, + nhg_entry.ref_count - 1); + + --nhg_entry.ref_count; +} + +/* + * Purpose: Get the next hop ID of the member. + * + * Description: Get the SAI ID of the next hop from NeighOrch. + * + * Params: None. + * + * Returns: The SAI ID of the next hop, or SAI_NULL_OBJECT_ID if the next + * hop is not valid. + */ +sai_object_id_t NextHopGroupMember::getNhId() const +{ + SWSS_LOG_ENTER(); + + sai_object_id_t nh_id = SAI_NULL_OBJECT_ID; + + if (gNeighOrch->hasNextHop(m_nh_key)) + { + SWSS_LOG_INFO("NeighOrch has next hop %s", + m_nh_key.to_string().c_str()); + nh_id = gNeighOrch->getNextHopId(m_nh_key); + } + /* + * If the next hop is labeled and the IP next hop exists, create the + * labeled one over NeighOrch as it doesn't know about these next hops. + * We don't do this in the constructor as the IP next hop may be added + * after the object is created and would never create the labeled next hop + * afterwards. + */ + else if (isLabeled() && gNeighOrch->hasNextHop(m_nh_key.ipKey())) + { + SWSS_LOG_INFO("Create labeled next hop %s", + m_nh_key.to_string().c_str()); + gNeighOrch->addNextHop(m_nh_key); + nh_id = gNeighOrch->getNextHopId(m_nh_key); + } + + return nh_id; +} + +/* + * Purpose: Move assignment operator. + * + * Description: Perform member-wise swap. + * + * Params: IN nhgm - The next hop group member to swap. + * + * Returns: Reference to this object. + */ +NextHopGroupMember& NextHopGroupMember::operator=(NextHopGroupMember&& nhgm) +{ + SWSS_LOG_ENTER(); + + std::swap(m_nh_key, nhgm.m_nh_key); + std::swap(m_gm_id, nhgm.m_gm_id); + + return *this; +} + +/* + * Purpose: Update the weight of a member. + * + * Description: Set the new member's weight and if the member is synced, update + * the SAI attribute as well. + * + * Params: IN weight - The weight of the next hop group member. + * + * Returns: true, if the operation was successful; + * false, otherwise. + */ +bool NextHopGroupMember::updateWeight(uint32_t weight) +{ + SWSS_LOG_ENTER(); + SWSS_LOG_INFO("Update group member %s weight from %u to %u", + m_nh_key.to_string().c_str(), + m_nh_key.weight, + weight); + + bool success = true; + + m_nh_key.weight = weight; + + if (isSynced()) + { + SWSS_LOG_INFO("Updating SAI weight attribute"); + + sai_attribute_t nhgm_attr; + nhgm_attr.id = SAI_NEXT_HOP_GROUP_MEMBER_ATTR_WEIGHT; + nhgm_attr.value.s32 = m_nh_key.weight; + + sai_status_t status = sai_next_hop_group_api-> + set_next_hop_group_member_attribute(m_gm_id, &nhgm_attr); + success = status == SAI_STATUS_SUCCESS; + } + + SWSS_LOG_INFO("Returning %d", success); + return success; +} + +/* + * Purpose: Sync the group member with the given group member ID. + * + * Description: Set the group member's SAI ID to the the one given and + * increment the appropriate ref counters. + * + * Params: IN gm_id - The group member SAI ID to set. + * + * Returns: Nothing. + */ +void NextHopGroupMember::sync(sai_object_id_t gm_id) +{ + SWSS_LOG_ENTER(); + + /* The SAI ID should be updated from invalid to something valid. */ + assert((m_gm_id == SAI_NULL_OBJECT_ID) && (gm_id != SAI_NULL_OBJECT_ID)); + + m_gm_id = gm_id; + gCrmOrch->incCrmResUsedCounter(CrmResourceType::CRM_NEXTHOP_GROUP_MEMBER); + gNeighOrch->increaseNextHopRefCount(m_nh_key); +} + +/* + * Purpose: Desync the group member, resetting it's SAI ID. + * + * Description: Reset the group member's SAI ID and decrement the appropriate + * ref counters. + * + * Params: None. + * + * Returns: Nothing. + */ +void NextHopGroupMember::desync() +{ + SWSS_LOG_ENTER(); + + /* + * If the member is already desynced, exit so we don't decrement the ref + * counters more than once. + */ + if (!isSynced()) + { + SWSS_LOG_INFO("Next hop group member %s is already desynced", + m_nh_key.to_string().c_str()); + return; + } + + m_gm_id = SAI_NULL_OBJECT_ID; + gCrmOrch->decCrmResUsedCounter(CrmResourceType::CRM_NEXTHOP_GROUP_MEMBER); + gNeighOrch->decreaseNextHopRefCount(m_nh_key); +} + +/* + * Purpose: Destructor. + * + * Description: Assert the group member is desynced and remove the labeled + * next hop from NeighOrch if it is unreferenced. + * + * Params: None. + * + * Returns: Nothing. + */ +NextHopGroupMember::~NextHopGroupMember() +{ + SWSS_LOG_ENTER(); + + /* + * The group member should be desynced from it's group before destroying + * it. + */ + assert(!isSynced()); + + /* + * If the labeled next hop is unreferenced, delete it from NeighOrch as + * NhgOrch and RouteOrch are the ones controlling it's lifetime. They both + * watch over these labeled next hops, so it doesn't matter who created + * them as they're both doing the same checks before deleting a labeled + * next hop. + */ + if (isLabeled() && + gNeighOrch->hasNextHop(m_nh_key) && + (gNeighOrch->getNextHopRefCount(m_nh_key) == 0)) + { + SWSS_LOG_INFO("Delete labeled next hop %s", + m_nh_key.to_string().c_str()); + gNeighOrch->removeMplsNextHop(m_nh_key); + } +} + +/* + * Purpose: Constructor. + * + * Description: Initialize the group's members based on the next hop group key. + * + * Params: IN key - The next hop group's key. + * + * Returns: Nothing. + */ +NextHopGroup::NextHopGroup(const NextHopGroupKey& key) : + m_key(key), + m_id(SAI_NULL_OBJECT_ID), + m_is_temp(false) +{ + SWSS_LOG_ENTER(); + SWSS_LOG_INFO("Creating next hop group %s", m_key.to_string().c_str()); + + /* Parse the key and create the members. */ + for (const auto& it : m_key.getNextHops()) + { + SWSS_LOG_INFO("Adding next hop %s to the group", + it.to_string().c_str()); + m_members.emplace(it, NextHopGroupMember(it)); + } +} + +/* + * Purpose: Move constructor. + * + * Description: Initialize the members by doing member-wise move construct. + * + * Params: IN nhg - The rvalue object to initialize from. + * + * Returns: Nothing. + */ +NextHopGroup::NextHopGroup(NextHopGroup&& nhg) : + m_key(std::move(nhg.m_key)), + m_id(std::move(nhg.m_id)), + m_members(std::move(nhg.m_members)), + m_is_temp(nhg.m_is_temp) +{ + SWSS_LOG_ENTER(); + + /* Invalidate the rvalue SAI ID. */ + nhg.m_id = SAI_NULL_OBJECT_ID; +} + +/* + * Purpose: Move assignment operator. + * + * Description: Perform member-wise swap. + * + * Params: IN nhg - The rvalue object to swap with. + * + * Returns: Referene to this object. + */ +NextHopGroup& NextHopGroup::operator=(NextHopGroup&& nhg) +{ + SWSS_LOG_ENTER(); + + std::swap(m_key, nhg.m_key); + std::swap(m_id, nhg.m_id); + std::swap(m_members, nhg.m_members); + m_is_temp = nhg.m_is_temp; + + return *this; +} + +/* + * Purpose: Sync a next hop group. + * + * Description: Fill in the NHG ID. If the group contains only one NH, this ID + * will be the SAI ID of the next hop that NeighOrch owns. If it + * has more than one NH, create a group over the SAI API and then + * add it's members. + * + * Params: None. + * + * Returns: true, if the operation was successful; + * false, otherwise. + */ +bool NextHopGroup::sync() +{ + SWSS_LOG_ENTER(); + + /* If the group is already synced, exit. */ + if (isSynced()) + { + SWSS_LOG_INFO("Group %s is already synced", to_string().c_str()); + return true; + } + + /* + * If the group has only one member, the group ID will be the member's NH + * ID. + */ + if (m_members.size() == 1) + { + const NextHopGroupMember& nhgm = m_members.begin()->second; + + if (nhgm.getNhId() == SAI_NULL_OBJECT_ID) + { + SWSS_LOG_WARN("Next hop %s is not synced", + nhgm.getNhKey().to_string().c_str()); + return false; + } + else + { + m_id = nhgm.getNhId(); + } + } + /* If the key contains more than one NH, create a group. */ + else + { + /* Assert the group is not empty. */ + assert(!m_members.empty()); + + /* Create the group over SAI. */ + sai_attribute_t nhg_attr; + vector nhg_attrs; + + nhg_attr.id = SAI_NEXT_HOP_GROUP_ATTR_TYPE; + nhg_attr.value.s32 = SAI_NEXT_HOP_GROUP_TYPE_ECMP; + nhg_attrs.push_back(nhg_attr); + + sai_status_t status = sai_next_hop_group_api->create_next_hop_group( + &m_id, + gSwitchId, + (uint32_t)nhg_attrs.size(), + nhg_attrs.data()); + + /* If the operation fails, exit. */ + if (status != SAI_STATUS_SUCCESS) + { + SWSS_LOG_ERROR("Failed to create next hop group %s, rv:%d", + m_key.to_string().c_str(), status); + return false; + } + + /* Increment the amount of programmed next hop groups. */ + gCrmOrch->incCrmResUsedCounter(CrmResourceType::CRM_NEXTHOP_GROUP); + + /* Increment the number of synced NHGs. */ + ++m_count; + + /* + * Try creating the next hop group's members over SAI. + */ + if (!syncMembers(m_key.getNextHops())) + { + SWSS_LOG_WARN("Failed to create next hop members of group %s", + to_string().c_str()); + return false; + } + } + + return true; +} + +/* + * Purpose: Create a temporary next hop group when resources are exhausted. + * + * Description: Choose one member to represent the group and create a group + * with only that next hop as a member. Any object referencing + * the SAI ID of a temporary group should keep querying NhgOrch in + * case the group is updated, as it's SAI ID will change at that + * point. + * + * Params: IN index - The CP index of the next hop group. + * + * Returns: The created temporary next hop group. + */ +NextHopGroup NhgOrch::createTempNhg(const NextHopGroupKey& nhg_key) +{ + SWSS_LOG_ENTER(); + SWSS_LOG_INFO("Syncing temporary group %s", nhg_key.to_string().c_str()); + + /* Get a list of all valid next hops in the group. */ + std::list valid_nhs; + + for (const auto& nh_key : nhg_key.getNextHops()) + { + /* + * Check if the IP next hop exists. We check for the IP next hop as + * the group might contain labeled NHs which we should create if their + * IP next hop does exist. + */ + if (gNeighOrch->hasNextHop(nh_key.ipKey())) + { + SWSS_LOG_INFO("Next hop %s is a candidate for temporary group %s", + nh_key.to_string().c_str(), + nhg_key.to_string().c_str()); + valid_nhs.push_back(nh_key); + } + } + + /* If there is no valid member, exit. */ + if (valid_nhs.empty()) + { + SWSS_LOG_INFO("There is no valid NH to sync temporary group %s", + nhg_key.to_string().c_str()); + throw std::logic_error("No valid NH in the key"); + } + + /* Randomly select the valid NH to represent the group. */ + auto it = valid_nhs.begin(); + advance(it, rand() % valid_nhs.size()); + + /* Create the temporary group. */ + NextHopGroup nhg(NextHopGroupKey(it->to_string())); + nhg.setTemp(true); + + return nhg; +} + +/* + * Purpose: Desync the next hop group. + * + * Description: Reset the group's SAI ID. If the group has more than one + * members, desync the members and the group. + * + * Params: None. + * + * Returns: true, if the operation was successful; + * false, otherwise + */ +bool NextHopGroup::desync() +{ + SWSS_LOG_ENTER(); + SWSS_LOG_INFO("Desyncing group %s", to_string().c_str()); + + /* If the group is already desynced, there is nothing to be done. */ + if (!isSynced()) + { + SWSS_LOG_INFO("Group %s is already desynced", + m_key.to_string().c_str()); + return true; + } + + /* + * If the group has more than one members, desync it's members, then the + * group. + */ + if (m_members.size() > 1) + { + /* Desync group's members. If we failed to desync the members, exit. */ + if (!desyncMembers(m_key.getNextHops())) + { + SWSS_LOG_ERROR("Failed to desync group %s members", + to_string().c_str()); + return false; + } + + /* Desync the group. */ + sai_status_t status = sai_next_hop_group_api-> + remove_next_hop_group(m_id); + + if (status != SAI_STATUS_SUCCESS) + { + SWSS_LOG_ERROR("Failed to remove next hop group %s, rv: %d", + m_key.to_string().c_str(), status); + return false; + } + + /* If the operation is successful, release the resources. */ + gCrmOrch->decCrmResUsedCounter( + CrmResourceType::CRM_NEXTHOP_GROUP); + --m_count; + } + + /* Reset the group ID. */ + m_id = SAI_NULL_OBJECT_ID; + + return true; +} + +/* + * Purpose: Sync the given next hop group's members over the SAI API. + * + * Description: Iterate over the given members and sync them. If the member + * is already synced, we skip it. If any of the next hops isn't + * already synced by the neighOrch, this will fail. Any next hop + * which has the neighbor interface down will be skipped. + * + * Params: IN nh_keys - The next hop keys of the members to sync. + * + * Returns: true, if the members were added succesfully; + * false, otherwise. + */ +bool NextHopGroup::syncMembers(const std::set& nh_keys) +{ + SWSS_LOG_ENTER(); + SWSS_LOG_INFO("Adding next hop group %s members", + to_string().c_str()); + + /* This method should not be called for groups with only one NH. */ + assert(m_members.size() > 1); + + ObjectBulker nextHopGroupMemberBulker( + sai_next_hop_group_api, gSwitchId, gMaxBulkSize); + + /* + * Iterate over the given next hops. + * If the group member is already synced, skip it. + * If any next hop is not synced, thus neighOrch doesn't have it, stop + * immediately. + * If a next hop's interface is down, skip it from being synced. + */ + std::map syncingMembers; + + for (const auto& nh_key : nh_keys) + { + SWSS_LOG_INFO("Checking if next hop %s is valid", + nh_key.to_string().c_str()); + + NextHopGroupMember& nhgm = m_members.at(nh_key); + + /* If the member is already synced, continue. */ + if (nhgm.getGmId() != SAI_NULL_OBJECT_ID) + { + SWSS_LOG_INFO("Update member"); + continue; + } + + /* + * If the next hop doesn't exist, stop from syncing the members. + */ + if (nhgm.getNhId() == SAI_NULL_OBJECT_ID) + { + SWSS_LOG_WARN("Failed to get next hop %s in group %s", + nhgm.to_string().c_str(), to_string().c_str()); + return false; + } + + /* If the neighbor's interface is down, skip from being syncd. */ + if (gNeighOrch->isNextHopFlagSet(nh_key, NHFLAGS_IFDOWN)) + { + SWSS_LOG_WARN("Skip next hop %s in group %s, interface is down", + nh_key.to_string().c_str(), to_string().c_str()); + continue; + } + + /* Create the next hop group member's attributes and fill them. */ + vector nhgm_attrs = createNhgmAttrs(nhgm); + + /* Add a bulker entry for this member. */ + nextHopGroupMemberBulker.create_entry(&syncingMembers[nh_key], + (uint32_t)nhgm_attrs.size(), + nhgm_attrs.data()); + } + + /* Flush the bulker to perform the sync. */ + nextHopGroupMemberBulker.flush(); + + /* + * Go through the synced members and increment the Crm ref count for the + * successful ones. + */ + bool success = true; + for (const auto& mbr : syncingMembers) + { + SWSS_LOG_INFO("Checking next hop member %s has a valid SAI ID", + mbr.first.to_string().c_str()); + + /* Check that the returned member ID is valid. */ + if (mbr.second == SAI_NULL_OBJECT_ID) + { + SWSS_LOG_ERROR("Failed to create next hop group %s's member %s", + m_key.to_string().c_str(), mbr.first.to_string().c_str()); + success = false; + } + else + { + m_members.at(mbr.first).sync(mbr.second); + } + } + + SWSS_LOG_INFO("Returning %d", success); + + return success; +} +/* + * Purpose: Desync the given group's members over the SAI API. + * + * Description: Go through the given members and desync them. + * + * Params: IN nh_keys - The next hop keys of the members to desync. + * + * Returns: true, if the operation was successful; + * false, otherwise + */ +bool NextHopGroup::desyncMembers(const std::set& nh_keys) +{ + SWSS_LOG_ENTER(); + SWSS_LOG_INFO("Removing members of group %s", + to_string().c_str()); + + /* This method should not be called for groups with only one NH. */ + assert(m_members.size() > 1); + + ObjectBulker nextHopGroupMemberBulker( + sai_next_hop_group_api, gSwitchId, gMaxBulkSize); + + /* + * Iterate through the given group members add them to be desynced. + * + * Keep track of the SAI delete statuses in case one of them returns an + * error. We assume that deletion should always succeed. If for some + * reason it doesn't, there's nothing we can do, but we'll log an error + * later. + */ + std::map statuses; + + for (const auto& nh_key : nh_keys) + { + SWSS_LOG_INFO("Desyncing member %s", nh_key.to_string().c_str()); + + const NextHopGroupMember& nhgm = m_members.at(nh_key); + + if (nhgm.isSynced()) + { + SWSS_LOG_INFO("Removing next hop group member %s", + nh_key.to_string().c_str()); + nextHopGroupMemberBulker.remove_entry(&statuses[nh_key], + nhgm.getGmId()); + } + } + + /* Flush the bulker to apply the changes. */ + nextHopGroupMemberBulker.flush(); + + /* + * Iterate over the statuses map and check if the removal was successful. + * If it was, decrement the Crm counter and reset the member's ID. If it + * wasn't, log an error message. + */ + bool success = true; + + for (const auto& status : statuses) + { + SWSS_LOG_INFO("Check member's %s status", + status.first.to_string().c_str()); + + if (status.second == SAI_STATUS_SUCCESS) + { + SWSS_LOG_INFO("Member was successfully desynced"); + m_members.at(status.first).desync(); + } + else + { + SWSS_LOG_ERROR("Could not remove next hop group member %s, rv: %d", + status.first.to_string().c_str(), status.second); + success = false; + } + } + + SWSS_LOG_INFO("Returning %d", success); + + return success; +} + +/* + * Purpose: Update the next hop group based on a new next hop group key. + * + * Description: Update the group's members by removing the members that aren't + * in the new next hop group and adding the new members. We first + * remove the missing members to avoid cases where we reached the + * ASIC group members limit. This will not update the group's SAI + * ID in any way, unless we are promoting a temporary group. + * + * Params: IN nhg_key - The new next hop group key to update to. + * + * Returns: true, if the operation was successful; + * false, otherwise. + */ +bool NextHopGroup::update(const NextHopGroupKey& nhg_key) +{ + SWSS_LOG_ENTER(); + SWSS_LOG_INFO("Update group %s with %s", + to_string().c_str(), nhg_key.to_string().c_str()); + + /* + * There are three cases where the SAI ID of the NHG will change: + * - changing a single next hop group to another single next hop group + * - changing a single next hop group to a multiple next hop group + * - changing a multiple next hop group to a single next hop group + * + * For these kind of updates, we can simply swap the existing group with + * the updated group, as we have no way of preserving the existing SAI ID. + * + * Also, we can perform the same operation if the group is not synced at + * all. + */ + if ((nhg_key.getSize() == 1) || (m_members.size() == 1) || !isSynced()) + { + SWSS_LOG_INFO("Updating group without preserving it's SAI ID"); + + bool was_synced = isSynced(); + *this = NextHopGroup(nhg_key); + + /* Sync the group only if it was synced before. */ + return (was_synced ? sync() : true); + } + /* + * If we are updating a multiple next hop group to another multiple next + * hop group, we can preserve it's SAI ID by only updating it's members. + * This way, any objects referencing the SAI ID of this object will + * continue to work. + */ + else + { + /* Update the key. */ + m_key = nhg_key; + + std::set new_nh_keys = nhg_key.getNextHops(); + std::set removed_nh_keys; + + /* Mark the members that need to be removed. */ + for (auto& mbr_it : m_members) + { + const NextHopKey& nh_key = mbr_it.first; + + /* Look for the existing member inside the new ones. */ + const auto& new_nh_key_it = new_nh_keys.find(nh_key); + + /* If the member is not found, then it needs to be removed. */ + if (new_nh_key_it == new_nh_keys.end()) + { + SWSS_LOG_INFO("Add member %s to be desynced", + nh_key.to_string().c_str()); + removed_nh_keys.insert(nh_key); + } + /* If the member is updated, update it's weight. */ + else + { + if (!mbr_it.second.updateWeight(new_nh_key_it->weight)) + { + SWSS_LOG_WARN("Failed to update member %s weight", + nh_key.to_string().c_str()); + return false; + } + + /* + * Erase the member from the new members list as it already + * exists. + */ + new_nh_keys.erase(new_nh_key_it); + } + } + + /* Desync the removed members. */ + if (!desyncMembers(removed_nh_keys)) + { + SWSS_LOG_WARN("Failed to desync members from group %s", + to_string().c_str()); + return false; + } + + /* Remove the desynced members. */ + for (const auto& nh_key : removed_nh_keys) + { + m_members.erase(nh_key); + } + + /* Add any new members to the group. */ + for (const auto& it : new_nh_keys) + { + m_members.emplace(it, NextHopGroupMember(it)); + } + + /* + * Sync all the members of the group. We sync all of them because + * there may be previous members that were not successfully synced + * before the update, so we must make sure we sync those as well. + */ + if (!syncMembers(m_key.getNextHops())) + { + SWSS_LOG_WARN("Failed to sync new members for group %s", + to_string().c_str()); + return false; + } + + return true; + } +} + +/* + * Purpose: Create the attributes vector for a next hop group member. + * + * Description: Create the group ID and next hop ID attributes. + * + * Params: IN nhgm - The next hop group member. + * + * Returns: The attributes vector for the given next hop. + */ +vector NextHopGroup::createNhgmAttrs( + const NextHopGroupMember& nhgm) const +{ + SWSS_LOG_ENTER(); + + vector nhgm_attrs; + sai_attribute_t nhgm_attr; + + /* Fill in the group ID. */ + nhgm_attr.id = SAI_NEXT_HOP_GROUP_MEMBER_ATTR_NEXT_HOP_GROUP_ID; + nhgm_attr.value.oid = m_id; + nhgm_attrs.push_back(nhgm_attr); + + /* Fill in the next hop ID. */ + nhgm_attr.id = SAI_NEXT_HOP_GROUP_MEMBER_ATTR_NEXT_HOP_ID; + nhgm_attr.value.oid = nhgm.getNhId(); + nhgm_attrs.push_back(nhgm_attr); + + /* Fill in the wright. */ + nhgm_attr.id = SAI_NEXT_HOP_GROUP_MEMBER_ATTR_WEIGHT; + nhgm_attr.value.s32 = nhgm.getWeight(); + nhgm_attrs.push_back(nhgm_attr); + + return nhgm_attrs; +} + +/* + * Purpose: Validate a next hop in the group. + * + * Description: Sync the validated next hop group member. + * + * Params: IN nh_key - The next hop to validate. + * + * Returns: true, if the operation was successful; + * false, otherwise. + */ +bool NextHopGroup::validateNextHop(const NextHopKey& nh_key) +{ + SWSS_LOG_ENTER(); + SWSS_LOG_INFO("Validate NH %s in group %s", + nh_key.to_string().c_str(), + to_string().c_str()); + + /* + * If the group has only one member, there is nothing to be done. The + * member is only a reference to the next hop owned by NeighOrch, so it is + * not for us to take any decisions regarding those. + */ + if (m_members.size() == 1) + { + return true; + } + + return syncMembers({nh_key}); +} + +/* + * Purpose: Invalidate a next hop in the group. + * + * Description: Sync the invalidated next hop group member. + * + * Params: IN nh_key - The next hop to invalidate. + * + * Returns: true, if the operation was successful; + * false, otherwise. + */ +bool NextHopGroup::invalidateNextHop(const NextHopKey& nh_key) +{ + SWSS_LOG_ENTER(); + SWSS_LOG_INFO("Invalidate NH %s in group %s", + nh_key.to_string().c_str(), + to_string().c_str()); + + /* + * If the group has only one member, there is nothing to be done. The + * member is only a reference to the next hop owned by NeighOrch, so it is + * not for us to take any decisions regarding those. + */ + if (m_members.size() == 1) + { + return true; + } + + return desyncMembers({nh_key}); +} diff --git a/orchagent/nhgorch.h b/orchagent/nhgorch.h new file mode 100644 index 0000000000..f7e4bcd4b6 --- /dev/null +++ b/orchagent/nhgorch.h @@ -0,0 +1,242 @@ +#pragma once + +#include "orch.h" +#include "nexthopgroupkey.h" + +class NextHopGroupMember +{ +public: + /* Constructors / Assignment operators. */ + NextHopGroupMember(const NextHopKey& nh_key) : + m_nh_key(nh_key), + m_gm_id(SAI_NULL_OBJECT_ID) {} + + NextHopGroupMember(NextHopGroupMember&& nhgm) : + m_nh_key(std::move(nhgm.m_nh_key)), + m_gm_id(nhgm.m_gm_id) + { nhgm.m_gm_id = SAI_NULL_OBJECT_ID; } + + NextHopGroupMember& operator=(NextHopGroupMember&& nhgm); + + /* + * Prevent object copying so we don't end up having multiple objects + * referencing the same SAI objects. + */ + NextHopGroupMember(const NextHopGroupMember&) = delete; + void operator=(const NextHopGroupMember&) = delete; + + /* Destructor. */ + virtual ~NextHopGroupMember(); + + /* Update member's weight and update the SAI attribute as well. */ + bool updateWeight(uint32_t weight); + + /* Sync / Desync. */ + void sync(sai_object_id_t gm_id); + void desync(); + + /* Getters / Setters. */ + inline const NextHopKey& getNhKey() const { return m_nh_key; } + inline uint32_t getWeight() const { return m_nh_key.weight; } + sai_object_id_t getNhId() const; + inline sai_object_id_t getGmId() const { return m_gm_id; } + inline bool isSynced() const { return m_gm_id != SAI_NULL_OBJECT_ID; } + + /* Check if the next hop is labeled. */ + inline bool isLabeled() const { return !m_nh_key.label_stack.empty(); } + + /* Convert member's details to string. */ + std::string to_string() const + { + return m_nh_key.to_string() + + ", SAI ID: " + std::to_string(m_gm_id); + } + +private: + /* The key of the next hop of this member. */ + NextHopKey m_nh_key; + + /* The group member SAI ID for this member. */ + sai_object_id_t m_gm_id; +}; + +/* Map indexed by NextHopKey, containing the SAI ID of the group member. */ +typedef std::map NhgMembers; + +/* + * NextHopGroup class representing a next hop group object. + */ +class NextHopGroup +{ +public: + /* Constructors. */ + explicit NextHopGroup(const NextHopGroupKey& key); + NextHopGroup(NextHopGroup&& nhg); + NextHopGroup& operator=(NextHopGroup&& nhg); + + /* Destructor. */ + virtual ~NextHopGroup() { desync(); } + + /* Sync the group, creating the group's and members SAI IDs. */ + bool sync(); + + /* Desync the group, reseting the group's and members SAI IDs. */ + bool desync(); + + /* + * Update the group based on a new next hop group key. This will also + * perform any sync / desync necessary. + */ + bool update(const NextHopGroupKey& nhg_key); + + /* Check if the group contains the given next hop. */ + inline bool hasNextHop(const NextHopKey& nh_key) const { + return m_members.find(nh_key) != m_members.end(); } + + /* Validate a next hop in the group, syncing it. */ + bool validateNextHop(const NextHopKey& nh_key); + + /* Invalidate a next hop in the group, desyncing it. */ + bool invalidateNextHop(const NextHopKey& nh_key); + + /* Increment the number of existing groups. */ + static inline void incCount() { ++m_count; } + + /* Decrement the number of existing groups. */ + static inline void decCount() { assert(m_count > 0); --m_count; } + + /* Getters / Setters. */ + inline const NextHopGroupKey& getKey() const { return m_key; } + inline sai_object_id_t getId() const { return m_id; } + static inline unsigned int getCount() { return m_count; } + inline bool isTemp() const { return m_is_temp; } + inline void setTemp(bool is_temp) { m_is_temp = is_temp; } + inline bool isSynced() const { return m_id != SAI_NULL_OBJECT_ID; } + inline size_t getSize() const { return m_members.size(); } + + /* Convert NHG's details to a string. */ + std::string to_string() const + { + return m_key.to_string() + ", SAI ID: " + std::to_string(m_id); + } + +private: + + /* The next hop group key of this group. */ + NextHopGroupKey m_key; + + /* The SAI ID of the group. */ + sai_object_id_t m_id; + + /* Members of this next hop group. */ + NhgMembers m_members; + + /* Whether the group is temporary or not. */ + bool m_is_temp; + + /* + * Number of existing groups. Incremented when an object is created and + * decremented when an object is destroyed. This will also account for the + * groups created by RouteOrch. + */ + static unsigned int m_count; + + /* Add group's members over the SAI API for the given keys. */ + bool syncMembers(const std::set& nh_keys); + + /* Remove group's members the SAI API from the given keys. */ + bool desyncMembers(const std::set& nh_keys); + + /* Create the attributes vector for a next hop group member. */ + vector createNhgmAttrs( + const NextHopGroupMember& nhgm) const; +}; + +/* + * Structure describing a next hop group which NhgOrch owns. Beside having a + * unique pointer to that next hop group, we also want to keep a ref count so + * NhgOrch knows how many other objects reference the next hop group in order + * not to delete them while still being referenced. + */ +struct NhgEntry +{ + /* Pointer to the next hop group. NhgOrch is the sole owner of it. */ + std::unique_ptr nhg; + + /* Number of external objects referencing this next hop group. */ + unsigned int ref_count; + + NhgEntry() = default; + explicit NhgEntry(std::unique_ptr&& _nhg, + unsigned int _ref_count = 0) : + nhg(std::move(_nhg)), ref_count(_ref_count) {} +}; + +/* + * Map indexed by next hop group's CP ID, containing the next hop group for + * that ID and the number of objects referencing it. + */ +typedef std::unordered_map NhgTable; + +/* + * Next Hop Group Orchestrator class that handles NEXTHOP_GROUP_TABLE + * updates. + */ +class NhgOrch : public Orch +{ +public: + /* + * Constructor. + */ + NhgOrch(DBConnector *db, string tableName); + + /* Check if the next hop group given by it's index exists. */ + inline bool hasNhg(const std::string& index) const + { return m_syncdNextHopGroups.find(index) != + m_syncdNextHopGroups.end(); } + + /* + * Get the next hop group given by it's index. If the index does not exist + * in map, a std::out_of_range exception will be thrown. + */ + inline const NextHopGroup& getNhg(const std::string& index) const + { return *m_syncdNextHopGroups.at(index).nhg; } + + /* Add a temporary next hop group when resources are exhausted. */ + NextHopGroup createTempNhg(const NextHopGroupKey& nhg_key); + + /* Getters / Setters. */ + inline unsigned int getMaxNhgCount() const { return m_maxNhgCount; } + static inline unsigned int getNhgCount() + { return NextHopGroup::getCount(); } + + /* Validate / Invalidate a next hop. */ + bool validateNextHop(const NextHopKey& nh_key); + bool invalidateNextHop(const NextHopKey& nh_key); + + /* Increase / Decrease the number of next hop groups. */ + inline void incNhgCount() + { + assert(NextHopGroup::getCount() < m_maxNhgCount); + NextHopGroup::incCount(); + } + inline void decNhgCount() { NextHopGroup::decCount(); } + + /* Increase / Decrease ref count for a NHG given by it's index. */ + void incNhgRefCount(const std::string& index); + void decNhgRefCount(const std::string& index); + +private: + + /* + * Switch's maximum number of next hop groups capacity. + */ + unsigned int m_maxNhgCount; + + /* + * The next hop group table. + */ + NhgTable m_syncdNextHopGroups; + + void doTask(Consumer& consumer); +}; diff --git a/orchagent/orchdaemon.cpp b/orchagent/orchdaemon.cpp index 814729ee0c..c2c995c475 100644 --- a/orchagent/orchdaemon.cpp +++ b/orchagent/orchdaemon.cpp @@ -31,6 +31,7 @@ FdbOrch *gFdbOrch; IntfsOrch *gIntfsOrch; NeighOrch *gNeighOrch; RouteOrch *gRouteOrch; +NhgOrch *gNhgOrch; FgNhgOrch *gFgNhgOrch; AclOrch *gAclOrch; PbhOrch *gPbhOrch; @@ -163,7 +164,8 @@ bool OrchDaemon::init() { APP_ROUTE_TABLE_NAME, routeorch_pri }, { APP_LABEL_ROUTE_TABLE_NAME, routeorch_pri } }; - gRouteOrch = new RouteOrch(m_applDb, route_tables, gSwitchOrch, gNeighOrch, gIntfsOrch, vrf_orch, gFgNhgOrch); + gRouteOrch = new RouteOrch(m_applDb, route_tables, gNeighOrch, gIntfsOrch, vrf_orch, gFgNhgOrch); + gNhgOrch = new NhgOrch(m_applDb, APP_NEXTHOP_GROUP_TABLE_NAME); CoppOrch *copp_orch = new CoppOrch(m_applDb, APP_COPP_TABLE_NAME); TunnelDecapOrch *tunnel_decap_orch = new TunnelDecapOrch(m_applDb, APP_TUNNEL_DECAP_TABLE_NAME); @@ -288,7 +290,7 @@ bool OrchDaemon::init() }; gMacsecOrch = new MACsecOrch(m_applDb, m_stateDb, macsec_app_tables, gPortsOrch); - + /* * The order of the orch list is important for state restore of warm start and * the queued processing in m_toSync map after gPortsOrch->allPortsReady() is set. @@ -297,7 +299,7 @@ bool OrchDaemon::init() * when iterating ConsumerMap. This is ensured implicitly by the order of keys in ordered map. * For cases when Orch has to process tables in specific order, like PortsOrch during warm start, it has to override Orch::doTask() */ - m_orchList = { gSwitchOrch, gCrmOrch, gPortsOrch, gBufferOrch, gIntfsOrch, gNeighOrch, gRouteOrch, copp_orch, tunnel_decap_orch, qos_orch, wm_orch, policer_orch, sflow_orch, debug_counter_orch, gMacsecOrch}; + m_orchList = { gSwitchOrch, gCrmOrch, gPortsOrch, gBufferOrch, gIntfsOrch, gNeighOrch, gNhgOrch, gRouteOrch, copp_orch, tunnel_decap_orch, qos_orch, wm_orch, policer_orch, sflow_orch, debug_counter_orch, gMacsecOrch}; bool initialize_dtel = false; if (platform == BFN_PLATFORM_SUBSTRING || platform == VS_PLATFORM_SUBSTRING) diff --git a/orchagent/orchdaemon.h b/orchagent/orchdaemon.h index 460d9052a5..1d1d9f77cf 100644 --- a/orchagent/orchdaemon.h +++ b/orchagent/orchdaemon.h @@ -11,6 +11,7 @@ #include "intfsorch.h" #include "neighorch.h" #include "routeorch.h" +#include "nhgorch.h" #include "copporch.h" #include "tunneldecaporch.h" #include "qosorch.h" diff --git a/orchagent/routeorch.cpp b/orchagent/routeorch.cpp index 9e9bbe9891..0bb583ede4 100644 --- a/orchagent/routeorch.cpp +++ b/orchagent/routeorch.cpp @@ -2,6 +2,7 @@ #include #include #include "routeorch.h" +#include "nhgorch.h" #include "logger.h" #include "swssnet.h" #include "crmorch.h" @@ -18,6 +19,7 @@ extern sai_switch_api_t* sai_switch_api; extern PortsOrch *gPortsOrch; extern CrmOrch *gCrmOrch; extern Directory gDirectory; +extern NhgOrch *gNhgOrch; extern size_t gMaxBulkSize; @@ -25,56 +27,19 @@ extern size_t gMaxBulkSize; #define DEFAULT_NUMBER_OF_ECMP_GROUPS 128 #define DEFAULT_MAX_ECMP_GROUP_SIZE 32 -RouteOrch::RouteOrch(DBConnector *db, vector &tableNames, SwitchOrch *switchOrch, NeighOrch *neighOrch, IntfsOrch *intfsOrch, VRFOrch *vrfOrch, FgNhgOrch *fgNhgOrch) : +RouteOrch::RouteOrch(DBConnector *db, vector &tableNames, NeighOrch *neighOrch, IntfsOrch *intfsOrch, VRFOrch *vrfOrch, FgNhgOrch *fgNhgOrch) : gRouteBulker(sai_route_api, gMaxBulkSize), gLabelRouteBulker(sai_mpls_api, gMaxBulkSize), gNextHopGroupMemberBulker(sai_next_hop_group_api, gSwitchId, gMaxBulkSize), Orch(db, tableNames), - m_switchOrch(switchOrch), m_neighOrch(neighOrch), m_intfsOrch(intfsOrch), m_vrfOrch(vrfOrch), m_fgNhgOrch(fgNhgOrch), - m_nextHopGroupCount(0), m_resync(false) { SWSS_LOG_ENTER(); - sai_attribute_t attr; - attr.id = SAI_SWITCH_ATTR_NUMBER_OF_ECMP_GROUPS; - - sai_status_t status = sai_switch_api->get_switch_attribute(gSwitchId, 1, &attr); - if (status != SAI_STATUS_SUCCESS) - { - SWSS_LOG_WARN("Failed to get switch attribute number of ECMP groups. \ - Use default value. rv:%d", status); - m_maxNextHopGroupCount = DEFAULT_NUMBER_OF_ECMP_GROUPS; - } - else - { - m_maxNextHopGroupCount = attr.value.s32; - - /* - * ASIC specific workaround to re-calculate maximum ECMP groups - * according to different ECMP mode used. - * - * On Mellanox platform, the maximum ECMP groups returned is the value - * under the condition that the ECMP group size is 1. Dividing this - * number by DEFAULT_MAX_ECMP_GROUP_SIZE gets the maximum number of - * ECMP groups when the maximum ECMP group size is 32. - */ - char *platform = getenv("platform"); - if (platform && strstr(platform, MLNX_PLATFORM_SUBSTRING)) - { - m_maxNextHopGroupCount /= DEFAULT_MAX_ECMP_GROUP_SIZE; - } - } - vector fvTuple; - fvTuple.emplace_back("MAX_NEXTHOP_GROUP_COUNT", to_string(m_maxNextHopGroupCount)); - m_switchOrch->set_switch_capability(fvTuple); - - SWSS_LOG_NOTICE("Maximum number of ECMP groups supported is %d", m_maxNextHopGroupCount); - IpPrefix default_ip_prefix("0.0.0.0/0"); sai_route_entry_t unicast_route_entry; @@ -83,10 +48,12 @@ RouteOrch::RouteOrch(DBConnector *db, vector &tableNames, copy(unicast_route_entry.destination, default_ip_prefix); subnet(unicast_route_entry.destination, unicast_route_entry.destination); + sai_attribute_t attr; attr.id = SAI_ROUTE_ENTRY_ATTR_PACKET_ACTION; attr.value.s32 = SAI_PACKET_ACTION_DROP; - status = sai_route_api->create_route_entry(&unicast_route_entry, 1, &attr); + sai_status_t status = sai_route_api->create_route_entry( + &unicast_route_entry, 1, &attr); if (status != SAI_STATUS_SUCCESS) { SWSS_LOG_ERROR("Failed to create IPv4 default route with packet action drop"); @@ -96,7 +63,7 @@ RouteOrch::RouteOrch(DBConnector *db, vector &tableNames, gCrmOrch->incCrmResUsedCounter(CrmResourceType::CRM_IPV4_ROUTE); /* Add default IPv4 route into the m_syncdRoutes */ - m_syncdRoutes[gVirtualRouterId][default_ip_prefix] = NextHopGroupKey(); + m_syncdRoutes[gVirtualRouterId][default_ip_prefix] = RouteNhg(); SWSS_LOG_NOTICE("Create IPv4 default route with packet action drop"); @@ -115,7 +82,7 @@ RouteOrch::RouteOrch(DBConnector *db, vector &tableNames, gCrmOrch->incCrmResUsedCounter(CrmResourceType::CRM_IPV6_ROUTE); /* Add default IPv6 route into the m_syncdRoutes */ - m_syncdRoutes[gVirtualRouterId][v6_default_ip_prefix] = NextHopGroupKey(); + m_syncdRoutes[gVirtualRouterId][v6_default_ip_prefix] = RouteNhg(); SWSS_LOG_NOTICE("Create IPv6 default route with packet action drop"); @@ -277,7 +244,7 @@ void RouteOrch::attach(Observer *observer, const IpAddress& dstAddr, sai_object_ SWSS_LOG_NOTICE("Attached next hop observer of route %s for destination IP %s", observerEntry->second.routeTable.rbegin()->first.to_string().c_str(), dstAddr.to_string().c_str()); - NextHopUpdate update = { vrf_id, dstAddr, route->first, route->second }; + NextHopUpdate update = { vrf_id, dstAddr, route->first, route->second.nhg_key }; observer->update(SUBJECT_TYPE_NEXTHOP_CHANGE, static_cast(&update)); } } @@ -549,12 +516,15 @@ void RouteOrch::doTask(Consumer& consumer) if (op == SET_COMMAND) { + SWSS_LOG_DEBUG("Set operation"); + string ips; string aliases; string mpls_nhs; string vni_labels; string remote_macs; string weights; + string nhg_index; bool& excp_intfs_flag = ctx.excp_intfs_flag; bool overlay_nh = false; bool blackhole = false; @@ -583,108 +553,165 @@ void RouteOrch::doTask(Consumer& consumer) if (fvField(i) == "weight") weights = fvValue(i); + + if (fvField(i) == "nexthop_group") + nhg_index = fvValue(i); } - vector ipv = tokenize(ips, ','); - vector alsv = tokenize(aliases, ','); - vector mpls_nhv = tokenize(mpls_nhs, ','); - vector vni_labelv = tokenize(vni_labels, ','); - vector rmacv = tokenize(remote_macs, ','); + SWSS_LOG_INFO("Route %s has nexthop_group: %s, ips: %s, " + "MPLS nhs: %s, aliases: %s", + ip_prefix.to_string().c_str(), + nhg_index.c_str(), + ips.c_str(), + mpls_nhs.c_str(), + aliases.c_str()); /* - * For backward compatibility, adjust ip string from old format to - * new format. Meanwhile it can deal with some abnormal cases. + * A route should not fill both nexthop_group and ips / + * aliases. */ - - /* Resize the ip vector to match ifname vector - * as tokenize(",", ',') will miss the last empty segment. */ - if (alsv.size() == 0 && !blackhole) + if (!nhg_index.empty() && (!ips.empty() || !aliases.empty())) { - SWSS_LOG_WARN("Skip the route %s, for it has an empty ifname field.", key.c_str()); + SWSS_LOG_ERROR("Route %s has both nexthop_group and ips/aliases", + key.c_str()); it = consumer.m_toSync.erase(it); continue; } - else if (alsv.size() != ipv.size()) - { - SWSS_LOG_NOTICE("Route %s: resize ipv to match alsv, %zd -> %zd.", key.c_str(), ipv.size(), alsv.size()); - ipv.resize(alsv.size()); - } - /* Set the empty ip(s) to zero - * as IpAddress("") will construct a incorrect ip. */ - for (auto &ip : ipv) + ctx.nhg_index = nhg_index; + + /* + * If the nexthop_group is empty, create the next hop group key + * based on the IPs and aliases. Otherwise, get the key from + * the NhgOrch. + */ + vector ipv; + vector alsv; + vector mpls_nhv; + vector vni_labelv; + vector rmacv; + + /* Check if the next hop group is owned by the NhgOrch. */ + if (nhg_index.empty()) { - if (ip.empty()) + ipv = tokenize(ips, ','); + alsv = tokenize(aliases, ','); + mpls_nhv = tokenize(mpls_nhs, ','); + vni_labelv = tokenize(vni_labels, ','); + rmacv = tokenize(remote_macs, ','); + + /* + * For backward compatibility, adjust ip string from old format to + * new format. Meanwhile it can deal with some abnormal cases. + */ + + /* Resize the ip vector to match ifname vector + * as tokenize(",", ',') will miss the last empty segment. */ + if (alsv.size() == 0 && !blackhole) { - SWSS_LOG_NOTICE("Route %s: set the empty nexthop ip to zero.", key.c_str()); - ip = ip_prefix.isV4() ? "0.0.0.0" : "::"; + SWSS_LOG_WARN("Skip the route %s, for it has an empty ifname field.", key.c_str()); + it = consumer.m_toSync.erase(it); + continue; + } + else if (alsv.size() != ipv.size()) + { + SWSS_LOG_NOTICE("Route %s: resize ipv to match alsv, %zd -> %zd.", key.c_str(), ipv.size(), alsv.size()); + ipv.resize(alsv.size()); } - } - for (auto alias : alsv) - { - /* skip route to management, docker, loopback - * TODO: for route to loopback interface, the proper - * way is to create loopback interface and then create - * route pointing to it, so that we can traps packets to - * CPU */ - if (alias == "eth0" || alias == "docker0" || - alias == "lo" || !alias.compare(0, strlen(LOOPBACK_PREFIX), LOOPBACK_PREFIX)) + /* Set the empty ip(s) to zero + * as IpAddress("") will construct a incorrect ip. */ + for (auto &ip : ipv) { - excp_intfs_flag = true; - break; + if (ip.empty()) + { + SWSS_LOG_NOTICE("Route %s: set the empty nexthop ip to zero.", key.c_str()); + ip = ip_prefix.isV4() ? "0.0.0.0" : "::"; + } } - } - // TODO: cannot trust m_portsOrch->getPortIdByAlias because sometimes alias is empty - if (excp_intfs_flag) - { - /* If any existing routes are updated to point to the - * above interfaces, remove them from the ASIC. */ - if (removeRoute(ctx)) - it = consumer.m_toSync.erase(it); - else - it++; - continue; - } + for (auto alias : alsv) + { + /* skip route to management, docker, loopback + * TODO: for route to loopback interface, the proper + * way is to create loopback interface and then create + * route pointing to it, so that we can traps packets to + * CPU */ + if (alias == "eth0" || alias == "docker0" || + alias == "lo" || !alias.compare(0, strlen(LOOPBACK_PREFIX), LOOPBACK_PREFIX)) + { + excp_intfs_flag = true; + break; + } + } - string nhg_str = ""; - NextHopGroupKey& nhg = ctx.nhg; + // TODO: cannot trust m_portsOrch->getPortIdByAlias because sometimes alias is empty + if (excp_intfs_flag) + { + /* If any existing routes are updated to point to the + * above interfaces, remove them from the ASIC. */ + if (removeRoute(ctx)) + it = consumer.m_toSync.erase(it); + else + it++; + continue; + } - if (blackhole) - { - nhg = NextHopGroupKey(); - } - else if (overlay_nh == false) - { - for (uint32_t i = 0; i < ipv.size(); i++) + string nhg_str = ""; + NextHopGroupKey& nhg = ctx.nhg; + + if (blackhole) { - if (i) nhg_str += NHG_DELIMITER; - if (alsv[i] == "tun0" && !(IpAddress(ipv[i]).isZero())) + nhg = NextHopGroupKey(); + } + else if (overlay_nh == false) + { + for (uint32_t i = 0; i < ipv.size(); i++) { - alsv[i] = gIntfsOrch->getRouterIntfsAlias(ipv[i]); + if (i) nhg_str += NHG_DELIMITER; + if (alsv[i] == "tun0" && !(IpAddress(ipv[i]).isZero())) + { + alsv[i] = gIntfsOrch->getRouterIntfsAlias(ipv[i]); + } + if (!mpls_nhv.empty() && mpls_nhv[i] != "na") + { + nhg_str += mpls_nhv[i] + LABELSTACK_DELIMITER; + } + nhg_str += ipv[i] + NH_DELIMITER + alsv[i]; } - if (!mpls_nhv.empty() && mpls_nhv[i] != "na") + + nhg = NextHopGroupKey(nhg_str, weights); + } + else + { + for (uint32_t i = 0; i < ipv.size(); i++) { - nhg_str += mpls_nhv[i] + LABELSTACK_DELIMITER; + if (i) nhg_str += NHG_DELIMITER; + nhg_str += ipv[i] + NH_DELIMITER + "vni" + alsv[i] + NH_DELIMITER + vni_labelv[i] + NH_DELIMITER + rmacv[i]; } - nhg_str += ipv[i] + NH_DELIMITER + alsv[i]; - } - - nhg = NextHopGroupKey(nhg_str, weights); + nhg = NextHopGroupKey(nhg_str, overlay_nh); + } } else { - for (uint32_t i = 0; i < ipv.size(); i++) + try { - if (i) nhg_str += NHG_DELIMITER; - nhg_str += ipv[i] + NH_DELIMITER + "vni" + alsv[i] + NH_DELIMITER + vni_labelv[i] + NH_DELIMITER + rmacv[i]; + const NextHopGroup& nh_group = gNhgOrch->getNhg(nhg_index); + ctx.nhg = nh_group.getKey(); + ctx.is_temp = nh_group.isTemp(); + } + catch (const std::out_of_range& e) + { + SWSS_LOG_ERROR("Next hop group %s does not exist", + nhg_index.c_str()); + ++it; + continue; } - - nhg = NextHopGroupKey(nhg_str, overlay_nh); } + NextHopGroupKey& nhg = ctx.nhg; + if (nhg.getSize() == 1 && nhg.hasIntfNextHop()) { if (alsv[0] == "unknown") @@ -720,9 +747,15 @@ void RouteOrch::doTask(Consumer& consumer) it++; } } + /* + * Check if the route does not exist or needs to be updated or + * if the route is using a temporary next hop group owned by + * NhgOrch. + */ else if (m_syncdRoutes.find(vrf_id) == m_syncdRoutes.end() || m_syncdRoutes.at(vrf_id).find(ip_prefix) == m_syncdRoutes.at(vrf_id).end() || - m_syncdRoutes.at(vrf_id).at(ip_prefix) != nhg) + m_syncdRoutes.at(vrf_id).at(ip_prefix) != RouteNhg(nhg, ctx.nhg_index) || + ctx.is_temp) { if (addRoute(ctx, nhg)) it = consumer.m_toSync.erase(it); @@ -730,18 +763,24 @@ void RouteOrch::doTask(Consumer& consumer) it++; } else + { + SWSS_LOG_INFO("Route %s is duplicate entry", key.c_str()); /* Duplicate entry */ it = consumer.m_toSync.erase(it); + } // If already exhaust the nexthop groups, and there are pending removing routes in bulker, // flush the bulker and possibly collect some released nexthop groups - if (m_nextHopGroupCount >= m_maxNextHopGroupCount && gRouteBulker.removing_entries_count() > 0) + if (gNhgOrch->getNhgCount() >= gNhgOrch->getMaxNhgCount() && + gRouteBulker.removing_entries_count() > 0) { break; } } else if (op == DEL_COMMAND) { + SWSS_LOG_DEBUG("Delete operation"); + if (removeRoute(ctx)) it = consumer.m_toSync.erase(it); else @@ -809,8 +848,9 @@ void RouteOrch::doTask(Consumer& consumer) it_prev++; } else if (m_syncdRoutes.find(vrf_id) == m_syncdRoutes.end() || - m_syncdRoutes.at(vrf_id).find(ip_prefix) == m_syncdRoutes.at(vrf_id).end() || - m_syncdRoutes.at(vrf_id).at(ip_prefix) != nhg) + m_syncdRoutes.at(vrf_id).find(ip_prefix) == m_syncdRoutes.at(vrf_id).end() || + m_syncdRoutes.at(vrf_id).at(ip_prefix) != RouteNhg(nhg, ctx.nhg_index) || + ctx.is_temp) { if (addRoutePost(ctx, nhg)) it_prev = consumer.m_toSync.erase(it_prev); @@ -871,13 +911,13 @@ void RouteOrch::notifyNextHopChangeObservers(sai_object_id_t vrf_id, const IpPre update_required = true; } - entry.second.routeTable.emplace(prefix, nexthops); + entry.second.routeTable.emplace(prefix, RouteNhg(nexthops, "")); } else { - if (route->second != nexthops) + if (route->second.nhg_key != nexthops) { - route->second = nexthops; + route->second.nhg_key = nexthops; /* If changed route is best match update observers */ if (entry.second.routeTable.rbegin()->first == route->first) { @@ -908,7 +948,7 @@ void RouteOrch::notifyNextHopChangeObservers(sai_object_id_t vrf_id, const IpPre assert(!entry.second.routeTable.empty()); auto route = entry.second.routeTable.rbegin(); - NextHopUpdate update = { vrf_id, entry.first.second, route->first, route->second }; + NextHopUpdate update = { vrf_id, entry.first.second, route->first, route->second.nhg_key }; for (auto observer : entry.second.observers) { @@ -985,7 +1025,7 @@ const NextHopGroupKey RouteOrch::getSyncdRouteNhgKey(sai_object_id_t vrf_id, con auto route_entry = route_table->second.find(ipPrefix); if (route_entry != route_table->second.end()) { - nhg = route_entry->second; + nhg = route_entry->second.nhg_key; } } return nhg; @@ -995,7 +1035,7 @@ bool RouteOrch::createFineGrainedNextHopGroup(sai_object_id_t &next_hop_group_id { SWSS_LOG_ENTER(); - if (m_nextHopGroupCount >= m_maxNextHopGroupCount) + if (gNhgOrch->getNhgCount() >= gNhgOrch->getMaxNhgCount()) { SWSS_LOG_DEBUG("Failed to create new next hop group. \ Reaching maximum number of next hop groups."); @@ -1017,7 +1057,7 @@ bool RouteOrch::createFineGrainedNextHopGroup(sai_object_id_t &next_hop_group_id } gCrmOrch->incCrmResUsedCounter(CrmResourceType::CRM_NEXTHOP_GROUP); - m_nextHopGroupCount++; + gNhgOrch->incNhgCount(); return true; } @@ -1039,7 +1079,7 @@ bool RouteOrch::removeFineGrainedNextHopGroup(sai_object_id_t &next_hop_group_id } gCrmOrch->decCrmResUsedCounter(CrmResourceType::CRM_NEXTHOP_GROUP); - m_nextHopGroupCount--; + gNhgOrch->decNhgCount(); return true; } @@ -1050,10 +1090,10 @@ bool RouteOrch::addNextHopGroup(const NextHopGroupKey &nexthops) assert(!hasNextHopGroup(nexthops)); - if (m_nextHopGroupCount >= m_maxNextHopGroupCount) + if (gNhgOrch->getNhgCount() >= gNhgOrch->getMaxNhgCount()) { - SWSS_LOG_DEBUG("Failed to create new next hop group. \ - Reaching maximum number of next hop groups."); + SWSS_LOG_WARN("Reached maximum next hop groups of %u", + gNhgOrch->getMaxNhgCount()); return false; } @@ -1080,11 +1120,10 @@ bool RouteOrch::addNextHopGroup(const NextHopGroupKey &nexthops) } else { - SWSS_LOG_INFO("Failed to get next hop %s in %s", + SWSS_LOG_WARN("Failed to get next hop %s in %s", it.to_string().c_str(), nexthops.to_string().c_str()); return false; } - // skip next hop group member create for neighbor from down port if (m_neighOrch->isNextHopFlagSet(it, NHFLAGS_IFDOWN)) { @@ -1126,7 +1165,7 @@ bool RouteOrch::addNextHopGroup(const NextHopGroupKey &nexthops) } } - m_nextHopGroupCount ++; + gNhgOrch->incNhgCount(); SWSS_LOG_NOTICE("Create next hop group %s", nexthops.to_string().c_str()); gCrmOrch->incCrmResUsedCounter(CrmResourceType::CRM_NEXTHOP_GROUP); @@ -1280,13 +1319,14 @@ bool RouteOrch::removeNextHopGroup(const NextHopGroupKey &nexthops) } } - m_nextHopGroupCount --; + gNhgOrch->decNhgCount(); gCrmOrch->decCrmResUsedCounter(CrmResourceType::CRM_NEXTHOP_GROUP); set next_hop_set = nexthops.getNextHops(); for (auto it : next_hop_set) { m_neighOrch->decreaseNextHopRefCount(it); + if (overlay_nh && !m_neighOrch->getNextHopRefCount(it)) { if(!m_neighOrch->removeTunnelNextHop(it)) @@ -1421,7 +1461,12 @@ void RouteOrch::addTempRoute(RouteBulkContext& ctx, const NextHopGroupKey &nextH /* Remove next hops that are not in m_syncdNextHops */ for (auto it = next_hop_set.begin(); it != next_hop_set.end();) { - if (!m_neighOrch->hasNextHop(*it)) + /* + * Check if the IP next hop exists in NeighOrch. The next hop may be + * a labeled one, which are created by RouteOrch or NhgOrch if the IP + * next hop exists. + */ + if (!m_neighOrch->hasNextHop(it->ipKey())) { SWSS_LOG_INFO("Failed to get next hop %s for %s", (*it).to_string().c_str(), ipPrefix.to_string().c_str()); @@ -1488,140 +1533,158 @@ bool RouteOrch::addRoute(RouteBulkContext& ctx, const NextHopGroupKey &nextHops) return false; } } - else if (nextHops.getSize() == 0) + else if (ctx.nhg_index.empty()) { - /* The route is pointing to a blackhole */ - blackhole = true; - } - else if (nextHops.getSize() == 1) - { - /* The route is pointing to a next hop */ - const NextHopKey& nexthop = *nextHops.getNextHops().begin(); - if (nexthop.isIntfNextHop()) + if (nextHops.getSize() == 0) { - if(gPortsOrch->isInbandPort(nexthop.alias)) - { - //This routes is the static route added for the remote system neighbors - //We do not need this route in the ASIC since the static neighbor creation - //in ASIC adds the same full mask route (host route) in ASIC automatically - //So skip. - return true; - } - - next_hop_id = m_intfsOrch->getRouterIntfsId(nexthop.alias); - /* rif is not created yet */ - if (next_hop_id == SAI_NULL_OBJECT_ID) - { - SWSS_LOG_INFO("Failed to get next hop %s for %s", - nextHops.to_string().c_str(), ipPrefix.to_string().c_str()); - return false; - } + /* The route is pointing to a blackhole */ + blackhole = true; } - else + else if (nextHops.getSize() == 1) { - if (m_neighOrch->hasNextHop(nexthop)) + /* The route is pointing to a next hop */ + const NextHopKey& nexthop = *nextHops.getNextHops().begin(); + if (nexthop.isIntfNextHop()) { - next_hop_id = m_neighOrch->getNextHopId(nexthop); - } - /* For non-existent MPLS NH, check if IP neighbor NH exists */ - else if (nexthop.isMplsNextHop() && - m_neighOrch->hasNextHop(NextHopKey(nexthop.ip_address, nexthop.alias))) - { - /* since IP neighbor NH exists, neighbor is resolved, add MPLS NH */ - m_neighOrch->addNextHop(nexthop); - next_hop_id = m_neighOrch->getNextHopId(nexthop); + if(gPortsOrch->isInbandPort(nexthop.alias)) + { + //This routes is the static route added for the remote system neighbors + //We do not need this route in the ASIC since the static neighbor creation + //in ASIC adds the same full mask route (host route) in ASIC automatically + //So skip. + return true; + } + + next_hop_id = m_intfsOrch->getRouterIntfsId(nexthop.alias); + /* rif is not created yet */ + if (next_hop_id == SAI_NULL_OBJECT_ID) + { + SWSS_LOG_INFO("Failed to get next hop %s for %s", + nextHops.to_string().c_str(), ipPrefix.to_string().c_str()); + return false; + } } /* IP neighbor is not yet resolved */ else { - if(overlay_nh) + if (m_neighOrch->hasNextHop(nexthop)) + { + next_hop_id = m_neighOrch->getNextHopId(nexthop); + } + /* See if there is an IP neighbor nexthop */ + else if (nexthop.isMplsNextHop() && + m_neighOrch->hasNextHop(nexthop.ipKey())) { - SWSS_LOG_INFO("create remote vtep %s", nexthop.to_string(overlay_nh).c_str()); - status = createRemoteVtep(vrf_id, nexthop); - if (status == false) + m_neighOrch->addNextHop(nexthop); + next_hop_id = m_neighOrch->getNextHopId(nexthop); + } + else + { + if(overlay_nh) { - SWSS_LOG_ERROR("Failed to create remote vtep %s", nexthop.to_string(overlay_nh).c_str()); - return false; + SWSS_LOG_INFO("create remote vtep %s", nexthop.to_string(overlay_nh).c_str()); + status = createRemoteVtep(vrf_id, nexthop); + if (status == false) + { + SWSS_LOG_ERROR("Failed to create remote vtep %s", nexthop.to_string(overlay_nh).c_str()); + return false; + } + next_hop_id = m_neighOrch->addTunnelNextHop(nexthop); + if (next_hop_id == SAI_NULL_OBJECT_ID) + { + SWSS_LOG_ERROR("Failed to create Tunnel Nexthop %s", nexthop.to_string(overlay_nh).c_str()); + return false; + } } - next_hop_id = m_neighOrch->addTunnelNextHop(nexthop); - if (next_hop_id == SAI_NULL_OBJECT_ID) + else { - SWSS_LOG_ERROR("Failed to create Tunnel Nexthop %s", nexthop.to_string(overlay_nh).c_str()); + SWSS_LOG_INFO("Failed to get next hop %s for %s, resolving neighbor", + nextHops.to_string().c_str(), ipPrefix.to_string().c_str()); + m_neighOrch->resolveNeighbor(nexthop); return false; } } - else - { - SWSS_LOG_INFO("Failed to get next hop %s for %s, resolving neighbor", - nextHops.to_string().c_str(), ipPrefix.to_string().c_str()); - m_neighOrch->resolveNeighbor(nexthop); - return false; - } } } - } - /* The route is pointing to a next hop group */ - else - { - /* Check if there is already an existing next hop group */ - if (!hasNextHopGroup(nextHops)) + /* The route is pointing to a next hop group */ + else { - /* Try to create a new next hop group */ - if (!addNextHopGroup(nextHops)) + /* Check if there is already an existing next hop group */ + if (!hasNextHopGroup(nextHops)) { - for(auto it = nextHops.getNextHops().begin(); it != nextHops.getNextHops().end(); ++it) + /* Try to create a new next hop group */ + if (!addNextHopGroup(nextHops)) { - const NextHopKey& nextHop = *it; - if(!m_neighOrch->hasNextHop(nextHop)) + for(auto it = nextHops.getNextHops().begin(); it != nextHops.getNextHops().end(); ++it) { - if(overlay_nh) + const NextHopKey& nextHop = *it; + if(!m_neighOrch->hasNextHop(nextHop)) { - SWSS_LOG_INFO("create remote vtep %s ecmp", nextHop.to_string(overlay_nh).c_str()); - status = createRemoteVtep(vrf_id, nextHop); - if (status == false) + if(overlay_nh) { - SWSS_LOG_ERROR("Failed to create remote vtep %s ecmp", nextHop.to_string(overlay_nh).c_str()); - return false; + SWSS_LOG_INFO("create remote vtep %s ecmp", nextHop.to_string(overlay_nh).c_str()); + status = createRemoteVtep(vrf_id, nextHop); + if (status == false) + { + SWSS_LOG_ERROR("Failed to create remote vtep %s ecmp", nextHop.to_string(overlay_nh).c_str()); + return false; + } + next_hop_id = m_neighOrch->addTunnelNextHop(nextHop); + if (next_hop_id == SAI_NULL_OBJECT_ID) + { + SWSS_LOG_ERROR("Failed to create Tunnel Nexthop %s", nextHop.to_string(overlay_nh).c_str()); + return false; + } } - next_hop_id = m_neighOrch->addTunnelNextHop(nextHop); - if (next_hop_id == SAI_NULL_OBJECT_ID) + else { - SWSS_LOG_ERROR("Failed to create Tunnel Nexthop %s", nextHop.to_string(overlay_nh).c_str()); - return false; + SWSS_LOG_INFO("Failed to get next hop %s in %s, resolving neighbor", + nextHop.to_string().c_str(), nextHops.to_string().c_str()); + m_neighOrch->resolveNeighbor(nextHop); } } - else - { - SWSS_LOG_INFO("Failed to get next hop %s in %s, resolving neighbor", - nextHop.to_string().c_str(), nextHops.to_string().c_str()); - m_neighOrch->resolveNeighbor(nextHop); - } } - } - /* Failed to create the next hop group and check if a temporary route is needed */ + /* Failed to create the next hop group and check if a temporary route is needed */ - /* If the current next hop is part of the next hop group to sync, - * then return false and no need to add another temporary route. */ - if (it_route != m_syncdRoutes.at(vrf_id).end() && it_route->second.getSize() == 1) - { - const NextHopKey& nexthop = *it_route->second.getNextHops().begin(); - if (nextHops.contains(nexthop)) + /* If the current next hop is part of the next hop group to sync, + * then return false and no need to add another temporary route. */ + if (it_route != m_syncdRoutes.at(vrf_id).end() && it_route->second.nhg_key.getSize() == 1) { - return false; + const NextHopKey& nexthop = *it_route->second.nhg_key.getNextHops().begin(); + if (nextHops.contains(nexthop)) + { + return false; + } } - } - /* Add a temporary route when a next hop group cannot be added, - * and there is no temporary route right now or the current temporary - * route is not pointing to a member of the next hop group to sync. */ - addTempRoute(ctx, nextHops); - /* Return false since the original route is not successfully added */ - return false; + /* Add a temporary route when a next hop group cannot be added, + * and there is no temporary route right now or the current temporary + * route is not pointing to a member of the next hop group to sync. */ + addTempRoute(ctx, nextHops); + /* Return false since the original route is not successfully added */ + return false; + } } - } - next_hop_id = m_syncdNextHopGroups[nextHops].next_hop_group_id; + next_hop_id = m_syncdNextHopGroups[nextHops].next_hop_group_id; + } + } + else + { + SWSS_LOG_DEBUG("Next hop group is owned by NhgOrch with index %s", + ctx.nhg_index.c_str()); + try + { + const NextHopGroup& nhg = gNhgOrch->getNhg(ctx.nhg_index); + next_hop_id = nhg.getId(); + } + catch(const std::out_of_range& e) + { + SWSS_LOG_WARN("Next hop group key %s does not exist", + ctx.nhg_index.c_str()); + return false; + } } /* Sync the route entry */ @@ -1665,7 +1728,7 @@ bool RouteOrch::addRoute(RouteBulkContext& ctx, const NextHopGroupKey &nextHops) else { /* Set the packet action to forward when there was no next hop (dropped) and not pointing to blackhole*/ - if (it_route->second.getSize() == 0 && !blackhole) + if (it_route->second.nhg_key.getSize() == 0 && !blackhole) { route_attr.id = SAI_ROUTE_ENTRY_ATTR_PACKET_ACTION; route_attr.value.s32 = SAI_PACKET_ACTION_FORWARD; @@ -1676,11 +1739,11 @@ bool RouteOrch::addRoute(RouteBulkContext& ctx, const NextHopGroupKey &nextHops) if (curNhgIsFineGrained && !isFineGrainedNextHopIdChanged) { - /* Don't change route entry if the route is previously fine grained and new nhg is also fine grained. + /* Don't change route entry if the route is previously fine grained and new nhg is also fine grained. * We already modifed sai nhg objs as part of setFgNhg to account for nhg change. */ object_statuses.emplace_back(SAI_STATUS_SUCCESS); } - else + else { route_attr.id = SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID; route_attr.value.oid = next_hop_id; @@ -1699,6 +1762,11 @@ bool RouteOrch::addRoute(RouteBulkContext& ctx, const NextHopGroupKey &nextHops) gRouteBulker.set_entry_attribute(&object_statuses.back(), &route_entry, &route_attr); } } + + SWSS_LOG_NOTICE("Added route %s with next hop(s) %s", + ipPrefix.to_string().c_str(), + nextHops.to_string().c_str()); + return false; } @@ -1719,52 +1787,69 @@ bool RouteOrch::addRoutePost(const RouteBulkContext& ctx, const NextHopGroupKey return false; } - /* next_hop_id indicates the next hop id or next hop group id of this route */ - sai_object_id_t next_hop_id; + SWSS_LOG_DEBUG("Checking next hop group %s", nextHops.to_string().c_str()); if (m_fgNhgOrch->isRouteFineGrained(vrf_id, ipPrefix, nextHops)) { /* Route is pointing to Fine Grained ECMP nexthop group */ isFineGrained = true; } - else if (nextHops.getSize() == 0) + /* Check that the next hop group is not owned by NhgOrch. */ + else if (ctx.nhg_index.empty()) { - /* The route is pointing to a blackhole */ - blackhole = true; - } - else if (nextHops.getSize() == 1) - { - /* The route is pointing to a next hop */ - const NextHopKey& nexthop = *nextHops.getNextHops().begin(); - if (nexthop.isIntfNextHop()) + SWSS_LOG_DEBUG("Next hop group is not owned by NhgOrch"); + if (nextHops.getSize() == 0) + { + /* The route is pointing to a blackhole */ + blackhole = true; + } + else if (nextHops.getSize() == 1) { - next_hop_id = m_intfsOrch->getRouterIntfsId(nexthop.alias); - /* rif is not created yet */ - if (next_hop_id == SAI_NULL_OBJECT_ID) + /* The route is pointing to a next hop */ + const NextHopKey& nexthop = *nextHops.getNextHops().begin(); + if (nexthop.isIntfNextHop()) { - SWSS_LOG_INFO("Failed to get next hop %s for %s", - nextHops.to_string().c_str(), ipPrefix.to_string().c_str()); - return false; + auto next_hop_id = m_intfsOrch->getRouterIntfsId(nexthop.alias); + /* rif is not created yet */ + if (next_hop_id == SAI_NULL_OBJECT_ID) + { + SWSS_LOG_WARN("Failed to get next hop %s for %s", + nextHops.to_string().c_str(), ipPrefix.to_string().c_str()); + return false; + } + } + else + { + if (!m_neighOrch->hasNextHop(nexthop)) + { + SWSS_LOG_WARN("Failed to get next hop %s for %s", + nextHops.to_string().c_str(), ipPrefix.to_string().c_str()); + return false; + } } } + /* The route is pointing to a next hop group */ else { - if (!m_neighOrch->hasNextHop(nexthop)) + if (!hasNextHopGroup(nextHops)) { - SWSS_LOG_INFO("Failed to get next hop %s for %s", - nextHops.to_string().c_str(), ipPrefix.to_string().c_str()); + SWSS_LOG_WARN("Next hop group is temporary, represented by %s", + ctx.tmp_next_hop.to_string().c_str()); + // Previous added an temporary route + auto& tmp_next_hop = ctx.tmp_next_hop; + addRoutePost(ctx, tmp_next_hop); return false; } } } - /* The route is pointing to a next hop group */ else { - if (!hasNextHopGroup(nextHops)) + SWSS_LOG_DEBUG("NhgOrch owns the next hop group with index %s", + ctx.nhg_index.c_str()); + if (!gNhgOrch->hasNhg(ctx.nhg_index)) { - // Previous added an temporary route - auto& tmp_next_hop = ctx.tmp_next_hop; - addRoutePost(ctx, tmp_next_hop); + SWSS_LOG_WARN("Failed to get next hop group with index %s", + ctx.nhg_index.c_str()); return false; } } @@ -1799,16 +1884,16 @@ bool RouteOrch::addRoutePost(const RouteBulkContext& ctx, const NextHopGroupKey else { /* Route already exists */ - auto nh_entry = m_syncdNextHopGroups.find(it_route->second); + auto nh_entry = m_syncdNextHopGroups.find(it_route->second.nhg_key); if (nh_entry != m_syncdNextHopGroups.end()) { /* Case where route was pointing to non-fine grained nhs in the past, * and transitioned to Fine Grained ECMP */ - decreaseNextHopRefCount(it_route->second); - if (it_route->second.getSize() > 1 - && m_syncdNextHopGroups[it_route->second].ref_count == 0) + decreaseNextHopRefCount(it_route->second.nhg_key); + if (it_route->second.nhg_key.getSize() > 1 + && m_syncdNextHopGroups[it_route->second.nhg_key].ref_count == 0) { - m_bulkNhgReducedRefCnt.emplace(it_route->second, 0); + m_bulkNhgReducedRefCnt.emplace(it_route->second.nhg_key, 0); } } SWSS_LOG_INFO("FG Post set route %s with next hop(s) %s", @@ -1822,9 +1907,11 @@ bool RouteOrch::addRoutePost(const RouteBulkContext& ctx, const NextHopGroupKey { SWSS_LOG_ERROR("Failed to create route %s with next hop(s) %s", ipPrefix.to_string().c_str(), nextHops.to_string().c_str()); - /* Clean up the newly created next hop group entry */ - if (nextHops.getSize() > 1) + + /* Check that the next hop group is not owned by NhgOrch. */ + if (ctx.nhg_index.empty() && nextHops.getSize() > 1) { + /* Clean up the newly created next hop group entry */ removeNextHopGroup(nextHops); } task_process_status handle_status = handleSaiCreateStatus(SAI_API_ROUTE, status); @@ -1843,10 +1930,17 @@ bool RouteOrch::addRoutePost(const RouteBulkContext& ctx, const NextHopGroupKey gCrmOrch->incCrmResUsedCounter(CrmResourceType::CRM_IPV6_ROUTE); } - /* Increase the ref_count for the next hop (group) entry */ - increaseNextHopRefCount(nextHops); + /* Increase the ref_count for the next hop group. */ + if (ctx.nhg_index.empty()) + { + increaseNextHopRefCount(nextHops); + } + else + { + gNhgOrch->incNhgRefCount(ctx.nhg_index); + } - SWSS_LOG_INFO("Post create route %s with next hop(s) %s", + SWSS_LOG_NOTICE("Post create route %s with next hop(s) %s", ipPrefix.to_string().c_str(), nextHops.to_string().c_str()); } else @@ -1854,7 +1948,7 @@ bool RouteOrch::addRoutePost(const RouteBulkContext& ctx, const NextHopGroupKey sai_status_t status; /* Set the packet action to forward when there was no next hop (dropped) and not pointing to blackhole */ - if (it_route->second.getSize() == 0 && !blackhole) + if (it_route->second.nhg_key.getSize() == 0 && !blackhole) { status = *it_status++; if (status != SAI_STATUS_SUCCESS) @@ -1881,9 +1975,6 @@ bool RouteOrch::addRoutePost(const RouteBulkContext& ctx, const NextHopGroupKey } } - /* Increase the ref_count for the next hop (group) entry */ - increaseNextHopRefCount(nextHops); - if (m_fgNhgOrch->syncdContainsFgNhg(vrf_id, ipPrefix)) { /* Remove FG nhg since prefix now points to standard nhg/nhs */ @@ -1891,23 +1982,32 @@ bool RouteOrch::addRoutePost(const RouteBulkContext& ctx, const NextHopGroupKey } else { - decreaseNextHopRefCount(it_route->second); - auto ol_nextHops = it_route->second; - if (it_route->second.getSize() > 1 - && m_syncdNextHopGroups[it_route->second].ref_count == 0) + /* Decrease the ref count for the previous next hop group. */ + if (it_route->second.nhg_index.empty()) { - m_bulkNhgReducedRefCnt.emplace(it_route->second, 0); - } - else if (ol_nextHops.is_overlay_nexthop()) - { - SWSS_LOG_NOTICE("Update overlay Nexthop %s", ol_nextHops.to_string().c_str()); - m_bulkNhgReducedRefCnt.emplace(ol_nextHops, vrf_id); + decreaseNextHopRefCount(it_route->second.nhg_key); + auto ol_nextHops = it_route->second.nhg_key; + if (ol_nextHops.getSize() > 1 + && m_syncdNextHopGroups[ol_nextHops].ref_count == 0) + { + m_bulkNhgReducedRefCnt.emplace(ol_nextHops, 0); + } + else if (ol_nextHops.is_overlay_nexthop()) + { + SWSS_LOG_NOTICE("Update overlay Nexthop %s", ol_nextHops.to_string().c_str()); + m_bulkNhgReducedRefCnt.emplace(ol_nextHops, vrf_id); + } + else if (ol_nextHops.getSize() == 1) + { + RouteKey r_key = { vrf_id, ipPrefix }; + auto nexthop = NextHopKey(ol_nextHops.to_string()); + removeNextHopRoute(nexthop, r_key); + } } - else if (ol_nextHops.getSize() == 1) + else { - RouteKey r_key = { vrf_id, ipPrefix }; - auto nexthop = NextHopKey(ol_nextHops.to_string()); - removeNextHopRoute(nexthop, r_key); + /* The next hop group is owned by NeighOrch. */ + gNhgOrch->decNhgRefCount(it_route->second.nhg_index); } } @@ -1927,11 +2027,21 @@ bool RouteOrch::addRoutePost(const RouteBulkContext& ctx, const NextHopGroupKey } } - SWSS_LOG_INFO("Post set route %s with next hop(s) %s", + if (ctx.nhg_index.empty()) + { + /* Increase the ref_count for the next hop (group) entry */ + increaseNextHopRefCount(nextHops); + } + else + { + gNhgOrch->incNhgRefCount(ctx.nhg_index); + } + + SWSS_LOG_NOTICE("Post set route %s with next hop(s) %s", ipPrefix.to_string().c_str(), nextHops.to_string().c_str()); } - if (nextHops.getSize() == 1 && !nextHops.is_overlay_nexthop()) + if (ctx.nhg_index.empty() && nextHops.getSize() == 1 && !nextHops.is_overlay_nexthop()) { RouteKey r_key = { vrf_id, ipPrefix }; auto nexthop = NextHopKey(nextHops.to_string()); @@ -1941,10 +2051,16 @@ bool RouteOrch::addRoutePost(const RouteBulkContext& ctx, const NextHopGroupKey } } - m_syncdRoutes[vrf_id][ipPrefix] = nextHops; + m_syncdRoutes[vrf_id][ipPrefix] = RouteNhg(nextHops, ctx.nhg_index); notifyNextHopChangeObservers(vrf_id, ipPrefix, nextHops, true); - return true; + + /* + * If the route uses a temporary synced NHG owned by NhgOrch, return false + * in order to keep trying to update the route in case the NHG is updated, + * which will update the SAI ID of the group as well. + */ + return !ctx.is_temp; } bool RouteOrch::removeRoute(RouteBulkContext& ctx) @@ -2082,50 +2198,58 @@ bool RouteOrch::removeRoutePost(const RouteBulkContext& ctx) } else { - /* - * Decrease the reference count only when the route is pointing to a next hop. - */ - decreaseNextHopRefCount(it_route->second); + /* Check that the next hop group is not owned by NhgOrch. */ + if (it_route->second.nhg_index.empty()) + { + auto ol_nextHops = it_route->second.nhg_key; - auto ol_nextHops = it_route->second; + /* + * Decrease the reference count only when the route is pointing to a next hop. + */ + decreaseNextHopRefCount(ol_nextHops); - if (it_route->second.getSize() > 1 - && m_syncdNextHopGroups[it_route->second].ref_count == 0) - { - m_bulkNhgReducedRefCnt.emplace(it_route->second, 0); - } - else if (ol_nextHops.is_overlay_nexthop()) - { - SWSS_LOG_NOTICE("Remove overlay Nexthop %s", ol_nextHops.to_string().c_str()); - m_bulkNhgReducedRefCnt.emplace(ol_nextHops, vrf_id); - } - /* - * Additionally check if the NH has label and its ref count == 0, then - * remove the label next hop. - */ - else if (it_route->second.getSize() == 1) - { - const NextHopKey& nexthop = *it_route->second.getNextHops().begin(); - if (nexthop.isMplsNextHop() && - (m_neighOrch->getNextHopRefCount(nexthop) == 0)) + if (ol_nextHops.getSize() > 1 + && m_syncdNextHopGroups[ol_nextHops].ref_count == 0) + { + m_bulkNhgReducedRefCnt.emplace(ol_nextHops, 0); + } + else if (ol_nextHops.is_overlay_nexthop()) { - m_neighOrch->removeMplsNextHop(nexthop); + SWSS_LOG_NOTICE("Remove overlay Nexthop %s", ol_nextHops.to_string().c_str()); + m_bulkNhgReducedRefCnt.emplace(ol_nextHops, vrf_id); } + /* + * Additionally check if the NH has label and its ref count == 0, then + * remove the label next hop. + */ + else if (ol_nextHops.getSize() == 1) + { + const NextHopKey& nexthop = *it_route->second.nhg_key.getNextHops().begin(); + if (nexthop.isMplsNextHop() && + (m_neighOrch->getNextHopRefCount(nexthop) == 0)) + { + m_neighOrch->removeMplsNextHop(nexthop); + } - RouteKey r_key = { vrf_id, ipPrefix }; - removeNextHopRoute(nexthop, r_key); + RouteKey r_key = { vrf_id, ipPrefix }; + removeNextHopRoute(nexthop, r_key); + } + } + else + { + gNhgOrch->decNhgRefCount(it_route->second.nhg_index); } } SWSS_LOG_INFO("Remove route %s with next hop(s) %s", - ipPrefix.to_string().c_str(), it_route->second.to_string().c_str()); + ipPrefix.to_string().c_str(), it_route->second.nhg_key.to_string().c_str()); if (ipPrefix.isDefaultRoute() && vrf_id == gVirtualRouterId) { - it_route_table->second[ipPrefix] = NextHopGroupKey(); + it_route_table->second[ipPrefix] = RouteNhg(); /* Notify about default route next hop change */ - notifyNextHopChangeObservers(vrf_id, ipPrefix, it_route_table->second[ipPrefix], true); + notifyNextHopChangeObservers(vrf_id, ipPrefix, it_route_table->second[ipPrefix].nhg_key, true); } else { diff --git a/orchagent/routeorch.h b/orchagent/routeorch.h index 4f74db62dc..d9066605c6 100644 --- a/orchagent/routeorch.h +++ b/orchagent/routeorch.h @@ -40,6 +40,30 @@ struct NextHopUpdate NextHopGroupKey nexthopGroup; }; +/* + * Structure describing the next hop group used by a route. As the next hop + * groups can either be owned by RouteOrch or by NhgOrch, we have to keep track + * of the next hop group index, as it is the one telling us which one owns it. + */ +struct RouteNhg +{ + NextHopGroupKey nhg_key; + + /* + * Index of the next hop group used. Filled only if referencing a + * NhgOrch's owned next hop group. + */ + std::string nhg_index; + + RouteNhg() = default; + RouteNhg(const NextHopGroupKey& key, const std::string& index) : + nhg_key(key), nhg_index(index) {} + + bool operator==(const RouteNhg& rnhg) + { return ((nhg_key == rnhg.nhg_key) && (nhg_index == rnhg.nhg_index)); } + bool operator!=(const RouteNhg& rnhg) { return !(*this == rnhg); } +}; + struct NextHopObserverEntry; /* Route destination key for a nexthop */ @@ -57,11 +81,11 @@ struct RouteKey /* NextHopGroupTable: NextHopGroupKey, NextHopGroupEntry */ typedef std::map NextHopGroupTable; /* RouteTable: destination network, NextHopGroupKey */ -typedef std::map RouteTable; +typedef std::map RouteTable; /* RouteTables: vrf_id, RouteTable */ typedef std::map RouteTables; /* LabelRouteTable: destination label, next hop address(es) */ -typedef std::map LabelRouteTable; +typedef std::map LabelRouteTable; /* LabelRouteTables: vrf_id, LabelRouteTable */ typedef std::map LabelRouteTables; /* Host: vrf_id, IpAddress */ @@ -82,12 +106,15 @@ struct RouteBulkContext std::deque object_statuses; // Bulk statuses NextHopGroupKey tmp_next_hop; // Temporary next hop NextHopGroupKey nhg; + std::string nhg_index; sai_object_id_t vrf_id; IpPrefix ip_prefix; bool excp_intfs_flag; + // is_temp will track if the NhgOrch's owned NHG is temporary or not + bool is_temp; RouteBulkContext() - : excp_intfs_flag(false) + : excp_intfs_flag(false), is_temp(false) { } @@ -102,6 +129,7 @@ struct RouteBulkContext nhg.clear(); excp_intfs_flag = false; vrf_id = SAI_NULL_OBJECT_ID; + is_temp = false; } }; @@ -110,13 +138,16 @@ struct LabelRouteBulkContext std::deque object_statuses; // Bulk statuses NextHopGroupKey tmp_next_hop; // Temporary next hop NextHopGroupKey nhg; + std::string nhg_index; sai_object_id_t vrf_id; Label label; bool excp_intfs_flag; uint8_t pop_count; + // is_temp will track if the NhgOrch's owned NHG is temporary or not + bool is_temp; LabelRouteBulkContext() - : excp_intfs_flag(false) + : excp_intfs_flag(false), is_temp(false) { } @@ -137,7 +168,7 @@ struct LabelRouteBulkContext class RouteOrch : public Orch, public Subject { public: - RouteOrch(DBConnector *db, vector &tableNames, SwitchOrch *switchOrch, NeighOrch *neighOrch, IntfsOrch *intfsOrch, VRFOrch *vrfOrch, FgNhgOrch *fgNhgOrch); + RouteOrch(DBConnector *db, vector &tableNames, NeighOrch *neighOrch, IntfsOrch *intfsOrch, VRFOrch *vrfOrch, FgNhgOrch *fgNhgOrch); bool hasNextHopGroup(const NextHopGroupKey&) const; sai_object_id_t getNextHopGroupId(const NextHopGroupKey&); @@ -173,14 +204,11 @@ class RouteOrch : public Orch, public Subject std::string getLinkLocalEui64Addr(void); private: - SwitchOrch *m_switchOrch; NeighOrch *m_neighOrch; IntfsOrch *m_intfsOrch; VRFOrch *m_vrfOrch; FgNhgOrch *m_fgNhgOrch; - int m_nextHopGroupCount; - int m_maxNextHopGroupCount; bool m_resync; RouteTables m_syncdRoutes; diff --git a/tests/mock_tests/Makefile.am b/tests/mock_tests/Makefile.am index 5fed8001a4..96f936a622 100644 --- a/tests/mock_tests/Makefile.am +++ b/tests/mock_tests/Makefile.am @@ -39,6 +39,7 @@ tests_SOURCES = aclorch_ut.cpp \ $(top_srcdir)/orchagent/routeorch.cpp \ $(top_srcdir)/orchagent/mplsrouteorch.cpp \ $(top_srcdir)/orchagent/fgnhgorch.cpp \ + $(top_srcdir)/orchagent/nhgorch.cpp \ $(top_srcdir)/orchagent/neighorch.cpp \ $(top_srcdir)/orchagent/intfsorch.cpp \ $(top_srcdir)/orchagent/portsorch.cpp \ diff --git a/tests/mock_tests/aclorch_ut.cpp b/tests/mock_tests/aclorch_ut.cpp index 4cd7688883..645c9b63f0 100644 --- a/tests/mock_tests/aclorch_ut.cpp +++ b/tests/mock_tests/aclorch_ut.cpp @@ -362,7 +362,7 @@ namespace aclorch_test { APP_ROUTE_TABLE_NAME, routeorch_pri }, { APP_LABEL_ROUTE_TABLE_NAME, routeorch_pri } }; - gRouteOrch = new RouteOrch(m_app_db.get(), route_tables, gSwitchOrch, gNeighOrch, gIntfsOrch, gVrfOrch, gFgNhgOrch); + gRouteOrch = new RouteOrch(m_app_db.get(), route_tables, gNeighOrch, gIntfsOrch, gVrfOrch, gFgNhgOrch); PolicerOrch *policer_orch = new PolicerOrch(m_config_db.get(), "POLICER"); diff --git a/tests/test_nhg.py b/tests/test_nhg.py index 7a40b2a942..ef09eb5a82 100644 --- a/tests/test_nhg.py +++ b/tests/test_nhg.py @@ -10,30 +10,115 @@ class TestNextHopGroup(object): - def test_route_nhg(self, dvs, dvs_route, testlog): + ASIC_NHS_STR = "ASIC_STATE:SAI_OBJECT_TYPE_NEXT_HOP" + ASIC_NHG_STR = "ASIC_STATE:SAI_OBJECT_TYPE_NEXT_HOP_GROUP" + ASIC_NHGM_STR = ASIC_NHG_STR + "_MEMBER" + ASIC_RT_STR = "ASIC_STATE:SAI_OBJECT_TYPE_ROUTE_ENTRY" + ASIC_INSEG_STR = "ASIC_STATE:SAI_OBJECT_TYPE_INSEG_ENTRY" + + def get_route_id(self, prefix, dvs): + for k in dvs.get_asic_db().get_keys(self.ASIC_RT_STR): + if json.loads(k)['dest'] == prefix: + return k + + return None + + def get_inseg_id(self, label, dvs): + for k in dvs.get_asic_db().get_keys(self.ASIC_INSEG_STR): + print(json.loads(k)) + if json.loads(k)['label'] == label: + return k + + return None + + def get_nhg_id(self, nhg_index, dvs): + # Add a route with the given index, then retrieve the next hop group ID + # from that route + asic_db = dvs.get_asic_db() + asic_rts_count = len(asic_db.get_keys(self.ASIC_RT_STR)) + + fvs = swsscommon.FieldValuePairs([('nexthop_group', nhg_index)]) + prefix = '255.255.255.255/24' + ps = swsscommon.ProducerStateTable(dvs.get_app_db().db_connection, + "ROUTE_TABLE") + ps.set(prefix, fvs) + + # Assert the route is created + try: + asic_db.wait_for_n_keys(self.ASIC_RT_STR, asic_rts_count + 1) + except Exception as e: + return None + + # Get the route ID for the created route + rt_id = self.get_route_id(prefix, dvs) + assert rt_id != None + + # Get the NHGID + nhgid = asic_db.get_entry(self.ASIC_RT_STR, rt_id)["SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID"] + + # Remove the added route + ps._del(prefix) + asic_db.wait_for_deleted_entry(self.ASIC_RT_STR, rt_id) + + # Return the NHGID + return nhgid + + def get_nhgm_ids(self, nhg_index, dvs): + nhgid = self.get_nhg_id(nhg_index, dvs) + nhgms = [] + asic_db = dvs.get_asic_db() + + for k in asic_db.get_keys(self.ASIC_NHGM_STR): + fvs = asic_db.get_entry(self.ASIC_NHGM_STR, k) + + # Sometimes some of the NHGMs have no fvs for some reason, so + # we skip those + try: + if fvs['SAI_NEXT_HOP_GROUP_MEMBER_ATTR_NEXT_HOP_GROUP_ID'] == nhgid: + nhgms.append(k) + except KeyError as e: + pass + + return nhgms + + def port_name(self, i): + return "Ethernet" + str(i * 4) + + def port_ip(self, i): + return "10.0.0." + str(i * 2) + + def port_ipprefix(self, i): + return self.port_ip(i) + "/31" + + def peer_ip(self, i): + return "10.0.0." + str(i * 2 + 1) + + def port_mac(self, i): + return "00:00:00:00:00:0" + str(i) + + def config_intf(self, i, dvs): config_db = dvs.get_config_db() - fvs = {"NULL": "NULL"} - config_db.create_entry("INTERFACE", "Ethernet0", fvs) - config_db.create_entry("INTERFACE", "Ethernet4", fvs) - config_db.create_entry("INTERFACE", "Ethernet8", fvs) - config_db.create_entry("INTERFACE", "Ethernet0|10.0.0.0/31", fvs) - config_db.create_entry("INTERFACE", "Ethernet4|10.0.0.2/31", fvs) - config_db.create_entry("INTERFACE", "Ethernet8|10.0.0.4/31", fvs) - dvs.runcmd("config interface startup Ethernet0") - dvs.runcmd("config interface startup Ethernet4") - dvs.runcmd("config interface startup Ethernet8") - - dvs.runcmd("arp -s 10.0.0.1 00:00:00:00:00:01") - dvs.runcmd("arp -s 10.0.0.3 00:00:00:00:00:02") - dvs.runcmd("arp -s 10.0.0.5 00:00:00:00:00:03") - - assert dvs.servers[0].runcmd("ip link set down dev eth0") == 0 - assert dvs.servers[1].runcmd("ip link set down dev eth0") == 0 - assert dvs.servers[2].runcmd("ip link set down dev eth0") == 0 - - assert dvs.servers[0].runcmd("ip link set up dev eth0") == 0 - assert dvs.servers[1].runcmd("ip link set up dev eth0") == 0 - assert dvs.servers[2].runcmd("ip link set up dev eth0") == 0 + fvs = {'NULL': 'NULL'} + + config_db.create_entry("INTERFACE", self.port_name(i), fvs) + config_db.create_entry("INTERFACE", "{}|{}".format(self.port_name(i), self.port_ipprefix(i)), fvs) + dvs.runcmd("config interface startup " + self.port_name(i)) + dvs.runcmd("arp -s {} {}".format(self.peer_ip(i), self.port_mac(i))) + assert dvs.servers[i].runcmd("ip link set down dev eth0") == 0 + assert dvs.servers[i].runcmd("ip link set up dev eth0") == 0 + + def flap_intf(self, i, status, dvs): + assert status in ['up', 'down'] + + dvs.servers[i].runcmd("ip link set {} dev eth0".format(status)) == 0 + time.sleep(2) + fvs = dvs.get_app_db().get_entry("PORT_TABLE", "Ethernet%d" % (i * 4)) + assert bool(fvs) + assert fvs["oper_status"] == status + + def test_route_nhg(self, dvs, dvs_route, testlog): + for i in range(3): + self.config_intf(i, dvs) rtprefix = "2.2.2.0/24" @@ -53,20 +138,20 @@ def test_route_nhg(self, dvs, dvs_route, testlog): rtkeys = dvs_route.check_asicdb_route_entries([rtprefix]) # assert the route points to next hop group - fvs = asic_db.get_entry("ASIC_STATE:SAI_OBJECT_TYPE_ROUTE_ENTRY", rtkeys[0]) + fvs = asic_db.get_entry(self.ASIC_RT_STR, rtkeys[0]) nhgid = fvs["SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID"] - fvs = asic_db.get_entry("ASIC_STATE:SAI_OBJECT_TYPE_NEXT_HOP_GROUP", nhgid) + fvs = asic_db.get_entry(self.ASIC_NHG_STR, nhgid) assert bool(fvs) - keys = asic_db.get_keys("ASIC_STATE:SAI_OBJECT_TYPE_NEXT_HOP_GROUP_MEMBER") + keys = asic_db.get_keys(self.ASIC_NHGM_STR) assert len(keys) == 3 for k in keys: - fvs = asic_db.get_entry("ASIC_STATE:SAI_OBJECT_TYPE_NEXT_HOP_GROUP_MEMBER", k) + fvs = asic_db.get_entry(self.ASIC_NHGM_STR, k) assert fvs["SAI_NEXT_HOP_GROUP_MEMBER_ATTR_NEXT_HOP_GROUP_ID"] == nhgid @@ -178,36 +263,22 @@ def test_route_nhg(self, dvs, dvs_route, testlog): # bring links down one-by-one for i in [0, 1, 2]: - dvs.servers[i].runcmd("ip link set down dev eth0") == 0 + self.flap_intf(i, 'down', dvs) - time.sleep(1) - - fvs = app_db.get_entry("PORT_TABLE", "Ethernet%d" % (i * 4)) - - assert bool(fvs) - assert fvs["oper_status"] == "down" - - keys = asic_db.get_keys("ASIC_STATE:SAI_OBJECT_TYPE_NEXT_HOP_GROUP_MEMBER") + keys = asic_db.get_keys(self.ASIC_NHGM_STR) assert len(keys) == 2 - i # bring links up one-by-one for i in [0, 1, 2]: - dvs.servers[i].runcmd("ip link set up dev eth0") == 0 - - time.sleep(1) + self.flap_intf(i, 'up', dvs) - fvs = app_db.get_entry("PORT_TABLE", "Ethernet%d" % (i * 4)) - - assert bool(fvs) - assert fvs["oper_status"] == "up" - - keys = asic_db.get_keys("ASIC_STATE:SAI_OBJECT_TYPE_NEXT_HOP_GROUP_MEMBER") + keys = asic_db.get_keys(self.ASIC_NHGM_STR) assert len(keys) == i + 1 for k in keys: - fvs = asic_db.get_entry("ASIC_STATE:SAI_OBJECT_TYPE_NEXT_HOP_GROUP_MEMBER", k) + fvs = asic_db.get_entry(self.ASIC_NHGM_STR, k) assert fvs["SAI_NEXT_HOP_GROUP_MEMBER_ATTR_NEXT_HOP_GROUP_ID"] == nhgid # Remove route 2.2.2.0/24 @@ -216,6 +287,358 @@ def test_route_nhg(self, dvs, dvs_route, testlog): # Wait for route 2.2.2.0/24 to be removed dvs_route.check_asicdb_deleted_route_entries([rtprefix]) + def test_label_route_nhg(self, dvs, testlog): + for i in range(3): + self.config_intf(i, dvs) + + app_db = dvs.get_app_db() + lr_ps = swsscommon.ProducerStateTable(app_db.db_connection, "LABEL_ROUTE_TABLE") + + asic_db = dvs.get_asic_db() + asic_insegs_count = len(asic_db.get_keys(self.ASIC_INSEG_STR)) + + # add label route + fvs = swsscommon.FieldValuePairs([("nexthop","10.0.0.1,10.0.0.3,10.0.0.5"), + ("ifname", "Ethernet0,Ethernet4,Ethernet8")]) + lr_ps.set("10", fvs) + + # check if route was propagated to ASIC DB + + asic_db.wait_for_n_keys(self.ASIC_INSEG_STR, asic_insegs_count + 1) + + k = self.get_inseg_id('10', dvs) + assert k is not None + + # assert the route points to next hop group + fvs = asic_db.get_entry(self.ASIC_INSEG_STR, k) + + nhgid = fvs["SAI_INSEG_ENTRY_ATTR_NEXT_HOP_ID"] + + fvs = asic_db.get_entry(self.ASIC_NHG_STR, nhgid) + + assert bool(fvs) + + keys = asic_db.get_keys(self.ASIC_NHGM_STR) + + assert len(keys) == 3 + + for k in keys: + fvs = asic_db.get_entry(self.ASIC_NHGM_STR, k) + + assert fvs["SAI_NEXT_HOP_GROUP_MEMBER_ATTR_NEXT_HOP_GROUP_ID"] == nhgid + + # bring links down one-by-one + for i in [0, 1, 2]: + self.flap_intf(i, 'down', dvs) + + keys = asic_db.get_keys(self.ASIC_NHGM_STR) + + assert len(keys) == 2 - i + + # bring links up one-by-one + for i in [0, 1, 2]: + self.flap_intf(i, 'up', dvs) + + keys = asic_db.get_keys(self.ASIC_NHGM_STR) + + assert len(keys) == i + 1 + + for k in keys: + fvs = asic_db.get_entry(self.ASIC_NHGM_STR, k) + assert fvs["SAI_NEXT_HOP_GROUP_MEMBER_ATTR_NEXT_HOP_GROUP_ID"] == nhgid + + # Remove label route 10 + lr_ps._del("10") + + # Wait for label route 10 to be removed + asic_db.wait_for_n_keys(self.ASIC_INSEG_STR, asic_insegs_count) + + def test_nhgorch_labeled_nhs(self, dvs, testlog): + for i in range(2): + self.config_intf(i, dvs) + + app_db = dvs.get_app_db() + asic_db = dvs.get_asic_db() + nhg_ps = swsscommon.ProducerStateTable(app_db.db_connection, "NEXTHOP_GROUP_TABLE") + asic_nhgs_count = len(asic_db.get_keys(self.ASIC_NHG_STR)) + asic_nhgms_count = len(asic_db.get_keys(self.ASIC_NHGM_STR)) + asic_nhs_count = len(asic_db.get_keys(self.ASIC_NHS_STR)) + + # Add a group containing labeled weighted NHs + fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1,10.0.0.3'), + ('mpls_nh', 'push1,push3'), + ('ifname', 'Ethernet0,Ethernet4'), + ('weight', '2,4')]) + nhg_ps.set('group1', fvs) + asic_db.wait_for_n_keys(self.ASIC_NHG_STR, asic_nhgs_count + 1) + asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, asic_nhgms_count + 2) + + # NhgOrch should create two next hops for the labeled ones + asic_db.wait_for_n_keys(self.ASIC_NHS_STR, asic_nhs_count + 2) + + # Assert the weights are properly set + nhgm_ids = self.get_nhgm_ids('group1', dvs) + weights = [] + for k in nhgm_ids: + fvs = asic_db.get_entry(self.ASIC_NHGM_STR, k) + weights.append(fvs['SAI_NEXT_HOP_GROUP_MEMBER_ATTR_WEIGHT']) + assert set(weights) == set(['2', '4']) + + # Create a new single next hop with the same label + fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1'), + ('mpls_nh', 'push1'), + ('ifname', 'Ethernet0')]) + nhg_ps.set('group2', fvs) + + # No new next hop should be added + time.sleep(1) + assert len(asic_db.get_keys(self.ASIC_NHS_STR)) == asic_nhs_count + 2 + + # Create a new single next hop with a different label + fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1'), + ('mpls_nh', 'push2'), + ('ifname', 'Ethernet0')]) + nhg_ps.set('group3', fvs) + + # A new next hop should be added + asic_db.wait_for_n_keys(self.ASIC_NHS_STR, asic_nhs_count + 3) + + # Delete group3 + nhg_ps._del('group3') + + # Group3's NH should be deleted + asic_db.wait_for_n_keys(self.ASIC_NHS_STR, asic_nhs_count + 2) + + # Delete group2 + nhg_ps._del('group2') + + # The number of NHs should be the same as they are still referenced by + # group1 + time.sleep(1) + asic_db.wait_for_n_keys(self.ASIC_NHS_STR, asic_nhs_count + 2) + + # Update group1 with no weights and both labeled and unlabeled NHs + fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1,10.0.0.3'), + ('mpls_nh', 'push2,na'), + ('ifname', 'Ethernet0,Ethernet4')]) + nhg_ps.set('group1', fvs) + + # Group members should be replaced and one NH should get deleted + asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, asic_nhgms_count + 2) + asic_db.wait_for_n_keys(self.ASIC_NHS_STR, asic_nhs_count + 1) + + # Assert the weights of the NHGMs are the expected ones + nhgm_ids = self.get_nhgm_ids('group1', dvs) + weights = [] + for nhgm_id in nhgm_ids: + fvs = asic_db.get_entry(self.ASIC_NHGM_STR, nhgm_id) + weights.append(fvs['SAI_NEXT_HOP_GROUP_MEMBER_ATTR_WEIGHT']) + assert weights == ['0', '0'] + + # Delete group1 + nhg_ps._del('group1') + + # Wait for the group and it's members to be deleted + asic_db.wait_for_n_keys(self.ASIC_NHG_STR, asic_nhgs_count) + asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, asic_nhgms_count) + + # The two next hops should also get deleted + asic_db.wait_for_n_keys(self.ASIC_NHS_STR, asic_nhs_count) + + def test_nhgorch_excp_group_cases(self, dvs, testlog): + for i in range(3): + self.config_intf(i, dvs) + + app_db = dvs.get_app_db() + asic_db = dvs.get_asic_db() + nhg_ps = swsscommon.ProducerStateTable(app_db.db_connection, "NEXTHOP_GROUP_TABLE") + rt_ps = swsscommon.ProducerStateTable(app_db.db_connection, "ROUTE_TABLE") + + def get_nhg_keys(): + return asic_db.get_keys(self.ASIC_NHG_STR) + + def get_nhgm_keys(): + return asic_db.get_keys(self.ASIC_NHGM_STR) + + def get_rt_keys(): + return asic_db.get_keys(self.ASIC_RT_STR) + + # Count existing objects + prev_nhg_keys = get_nhg_keys() + asic_nhgs_count = len(get_nhg_keys()) + asic_nhgms_count = len(get_nhgm_keys()) + asic_nhs_count = len(asic_db.get_keys(self.ASIC_NHS_STR)) + + # Remove a group that does not exist + nhg_ps._del("group1") + time.sleep(1) + assert len(get_nhg_keys()) == asic_nhgs_count + + # Create a next hop group with a member that does not exist - should fail + fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1,10.0.0.3,10.0.0.63'), + ("ifname", "Ethernet0,Ethernet4,Ethernet124")]) + nhg_ps.set("group1", fvs) + time.sleep(1) + assert len(asic_db.get_keys(self.ASIC_NHG_STR)) == asic_nhgs_count + + # Issue an update for this next hop group that doesn't yet exist, + # which contains only valid NHs. This will overwrite the previous + # operation and create the group. + fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1,10.0.0.5'), + ("ifname", "Ethernet0,Ethernet8")]) + nhg_ps.set("group1", fvs) + asic_db.wait_for_n_keys(self.ASIC_NHG_STR, asic_nhgs_count + 1) + asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, asic_nhgms_count + 2) + + # Check the group has it's two members + for nhgid in asic_db.get_keys(self.ASIC_NHG_STR): + if nhgid not in prev_nhg_keys: + break + + count = 0 + for k in asic_db.get_keys(self.ASIC_NHGM_STR): + fvs = asic_db.get_entry(self.ASIC_NHGM_STR, k) + if fvs['SAI_NEXT_HOP_GROUP_MEMBER_ATTR_NEXT_HOP_GROUP_ID'] == nhgid: + count += 1 + assert count == 2 + + # Add a route referencing the new group + asic_rts_count = len(get_rt_keys()) + fvs = swsscommon.FieldValuePairs([('nexthop_group', 'group1')]) + rt_ps.set('2.2.2.0/24', fvs) + asic_db.wait_for_n_keys(self.ASIC_RT_STR, asic_rts_count + 1) + + # Get the route key + for rt_key in get_rt_keys(): + k = json.loads(rt_key) + + if k['dest'] == "2.2.2.0/24": + break + + # Try removing the group while it still has references - should fail + nhg_ps._del('group1') + time.sleep(1) + assert len(get_nhg_keys()) == asic_nhgs_count + 1 + + # Create a new group + fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1,10.0.0.3'), + ('ifname', 'Ethernet0,Ethernet4')]) + nhg_ps.set("group2", fvs) + asic_db.wait_for_n_keys(self.ASIC_NHG_STR, asic_nhgs_count + 2) + asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, asic_nhgms_count + 4) + + # Update the route to point to the new group + fvs = swsscommon.FieldValuePairs([('nexthop_group', 'group2')]) + rt_ps.set('2.2.2.0/24', fvs) + + # The first group should have got deleted + asic_db.wait_for_n_keys(self.ASIC_NHG_STR, asic_nhgs_count + 1) + asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, asic_nhgms_count + 2) + + assert asic_db.get_entry(self.ASIC_RT_STR, rt_key)['SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID'] != nhgid + + # Update the route with routeOrch's owned next hop group + nhgid = asic_db.get_entry(self.ASIC_RT_STR, rt_key)['SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID'] + fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1,10.0.0.3'), + ('ifname', 'Ethernet0,Ethernet4')]) + rt_ps.set('2.2.2.0/24', fvs) + + asic_db.wait_for_n_keys(self.ASIC_NHG_STR, asic_nhgs_count + 2) + asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, asic_nhgms_count + 4) + + # Assert the next hop group ID changed + time.sleep(1) + assert asic_db.get_entry(self.ASIC_RT_STR, rt_key)['SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID'] != nhgid + nhgid = asic_db.get_entry(self.ASIC_RT_STR, rt_key)['SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID'] + + # Update the route to point back to group2 + fvs = swsscommon.FieldValuePairs([('nexthop_group', 'group2')]) + rt_ps.set('2.2.2.0/24', fvs) + + # The routeOrch's owned next hop group should get deleted + asic_db.wait_for_n_keys(self.ASIC_NHG_STR, asic_nhgs_count + 1) + asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, asic_nhgms_count + 2) + + # Assert the route points back to group2 + assert asic_db.get_entry(self.ASIC_RT_STR, rt_key)['SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID'] != nhgid + + # Create a new group with the same members as group2 + nhgid = asic_db.get_entry(self.ASIC_RT_STR, rt_key)['SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID'] + fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1,10.0.0.3'), + ('ifname', 'Ethernet0,Ethernet4')]) + nhg_ps.set("group1", fvs) + asic_db.wait_for_n_keys(self.ASIC_NHG_STR, asic_nhgs_count + 2) + asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, asic_nhgms_count + 4) + + # Update the route to point to the new group + fvs = swsscommon.FieldValuePairs([('nexthop_group', 'group1')]) + rt_ps.set('2.2.2.0/24', fvs) + time.sleep(1) + + # Assert the next hop group ID changed + assert asic_db.get_entry(self.ASIC_RT_STR, rt_key)['SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID'] != nhgid + + # Remove the route + rt_ps._del('2.2.2.0/24') + asic_db.wait_for_n_keys(self.ASIC_RT_STR, asic_rts_count) + + # Remove the groups + nhg_ps._del('group1') + nhg_ps._del('group2') + + asic_db.wait_for_n_keys(self.ASIC_NHG_STR, asic_nhgs_count) + asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, asic_nhgms_count) + + # Create a route with labeled NHs + fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1,10.0.0.3'), + ('mpls_nh', 'push1,push3'), + ('ifname', 'Ethernet0,Ethernet4'), + ('weight', '2,4')]) + rt_ps.set('2.2.2.0/24', fvs) + asic_db.wait_for_n_keys(self.ASIC_RT_STR, asic_rts_count + 1) + + # Two new next hops should be created + asic_db.wait_for_n_keys(self.ASIC_NHS_STR, asic_nhs_count + 2) + + # Create a NHG with the same details + nhg_ps.set('group1', fvs) + + # No new next hops should be created + time.sleep(1) + assert len(asic_db.get_keys(self.ASIC_NHS_STR)) == asic_nhs_count + 2 + + # Update the group with a different NH + fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1,10.0.0.3'), + ('mpls_nh', 'push2,push3'), + ('ifname', 'Ethernet0,Ethernet4'), + ('weight', '2,4')]) + nhg_ps.set('group1', fvs) + + # A new next hop should be created + asic_db.wait_for_n_keys(self.ASIC_NHS_STR, asic_nhs_count + 3) + + # Remove the route + rt_ps._del('2.2.2.0/24') + + # One NH should become unreferenced and should be deleted. The other + # one is still referenced by NhgOrch's owned NHG. + asic_db.wait_for_n_keys(self.ASIC_NHS_STR, asic_nhs_count + 2) + + # Remove the group + nhg_ps._del('group1') + + # Both new next hops should be deleted + asic_db.wait_for_n_keys(self.ASIC_NHS_STR, asic_nhs_count) + + # Add a route with a NHG that does not exist + fvs = swsscommon.FieldValuePairs([('nexthop_group', 'group1')]) + rt_ps.set('2.2.2.0/24', fvs) + time.sleep(1) + assert asic_rts_count == len(asic_db.get_keys(self.ASIC_RT_STR)) + + # Remove the pending route + rt_ps._del('2.2.2.0/24') + def test_route_nhg_exhaust(self, dvs, testlog): """ Test the situation of exhausting ECMP group, assume SAI_SWITCH_ATTR_NUMBER_OF_ECMP_GROUPS is 512 @@ -233,21 +656,6 @@ def test_route_nhg_exhaust(self, dvs, testlog): # TODO: check ECMP 512 - def port_name(i): - return "Ethernet" + str(i * 4) - - def port_ip(i): - return "10.0.0." + str(i * 2) - - def peer_ip(i): - return "10.0.0." + str(i * 2 + 1) - - def port_ipprefix(i): - return port_ip(i) + "/31" - - def port_mac(i): - return "00:00:00:00:00:0" + str(i) - def gen_ipprefix(r): """ Construct route like 2.X.X.0/24 """ ip = ipaddress.IPv4Address(IP_INTEGER_BASE + r * 256) @@ -260,25 +668,16 @@ def gen_nhg_fvs(binary): ifname = [] for i in range(MAX_PORT_COUNT): if binary[i] == '1': - nexthop.append(peer_ip(i)) - ifname.append(port_name(i)) + nexthop.append(self.peer_ip(i)) + ifname.append(self.port_name(i)) nexthop = ','.join(nexthop) ifname = ','.join(ifname) fvs = swsscommon.FieldValuePairs([("nexthop", nexthop), ("ifname", ifname)]) return fvs - def asic_route_exists(keys, ipprefix): - for k in keys: - rt_key = json.loads(k) - - if rt_key['dest'] == ipprefix: - return k - else: - return None - def asic_route_nhg_fvs(k): - fvs = asic_db.get_entry("ASIC_STATE:SAI_OBJECT_TYPE_ROUTE_ENTRY", k) + fvs = asic_db.get_entry(self.ASIC_RT_STR, k) if not fvs: return None @@ -287,7 +686,7 @@ def asic_route_nhg_fvs(k): if nhgid is None: return None - fvs = asic_db.get_entry("ASIC_STATE:SAI_OBJECT_TYPE_NEXT_HOP_GROUP", nhgid) + fvs = asic_db.get_entry(self.ASIC_NHG_STR, nhgid) return fvs MAX_ECMP_COUNT = 512 @@ -297,25 +696,19 @@ def asic_route_nhg_fvs(k): else: IP_INTEGER_BASE = int(ipaddress.IPv4Address(str("2.2.2.0"))) - config_db = dvs.get_config_db() - fvs = {"NULL": "NULL"} - for i in range(MAX_PORT_COUNT): - config_db.create_entry("INTERFACE", port_name(i), fvs) - config_db.create_entry("INTERFACE", "{}|{}".format(port_name(i), port_ipprefix(i)), fvs) - dvs.runcmd("config interface startup " + port_name(i)) - dvs.runcmd("arp -s {} {}".format(peer_ip(i), port_mac(i))) - assert dvs.servers[i].runcmd("ip link set down dev eth0") == 0 - assert dvs.servers[i].runcmd("ip link set up dev eth0") == 0 + self.config_intf(i, dvs) app_db = dvs.get_app_db() asic_db = dvs.get_asic_db() ps = swsscommon.ProducerStateTable(app_db.db_connection, "ROUTE_TABLE") + dvs.runcmd('swssloglevel -c orchagent -l INFO') + # Add first batch of routes with unique nexthop groups in AppDB route_count = 0 r = 0 - asic_routes_count = len(asic_db.get_keys("ASIC_STATE:SAI_OBJECT_TYPE_ROUTE_ENTRY")) + asic_routes_count = len(asic_db.get_keys(self.ASIC_RT_STR)) while route_count < MAX_ECMP_COUNT: r += 1 fmt = '{{0:0{}b}}'.format(MAX_PORT_COUNT) @@ -329,12 +722,35 @@ def asic_route_nhg_fvs(k): route_count += 1 # Wait and check ASIC DB the count of nexthop groups used - asic_db.wait_for_n_keys("ASIC_STATE:SAI_OBJECT_TYPE_NEXT_HOP_GROUP", MAX_ECMP_COUNT) + asic_db.wait_for_n_keys(self.ASIC_NHG_STR, MAX_ECMP_COUNT) # Wait and check ASIC DB the count of routes - asic_db.wait_for_n_keys("ASIC_STATE:SAI_OBJECT_TYPE_ROUTE_ENTRY", asic_routes_count + MAX_ECMP_COUNT) + asic_db.wait_for_n_keys(self.ASIC_RT_STR, asic_routes_count + MAX_ECMP_COUNT) asic_routes_count += MAX_ECMP_COUNT + # Add a route with labeled NHs + asic_nhs_count = len(asic_db.get_keys(self.ASIC_NHS_STR)) + route_ipprefix = gen_ipprefix(route_count) + fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1,10.0.0.3'), + ('mpls_nh', 'push1,push3'), + ('ifname', 'Ethernet0,Ethernet4')]) + ps.set(route_ipprefix, fvs) + route_count += 1 + + # A temporary route should be created + asic_db.wait_for_n_keys(self.ASIC_RT_STR, asic_routes_count + 1) + + # A NH should be elected as the temporary NHG and it should be created + # as it doesn't exist. + asic_db.wait_for_n_keys(self.ASIC_NHS_STR, asic_nhs_count + 1) + + # Delete the route. The route and the added labeled NH should be + # removed. + ps._del(route_ipprefix) + route_count -= 1 + asic_db.wait_for_n_keys(self.ASIC_RT_STR, asic_routes_count) + asic_db.wait_for_n_keys(self.ASIC_NHS_STR, asic_nhs_count) + # Add second batch of routes with unique nexthop groups in AppDB # Add more routes with new nexthop group in AppDBdd route_ipprefix = gen_ipprefix(route_count) @@ -355,18 +771,18 @@ def asic_route_nhg_fvs(k): last_ipprefix = route_ipprefix # Wait until we get expected routes and check ASIC DB on the count of nexthop groups used, and it should not increase - keys = asic_db.wait_for_n_keys("ASIC_STATE:SAI_OBJECT_TYPE_ROUTE_ENTRY", asic_routes_count + 10) - asic_db.wait_for_n_keys("ASIC_STATE:SAI_OBJECT_TYPE_NEXT_HOP_GROUP", MAX_ECMP_COUNT) + asic_db.wait_for_n_keys(self.ASIC_RT_STR, asic_routes_count + 10) + asic_db.wait_for_n_keys(self.ASIC_NHG_STR, MAX_ECMP_COUNT) # Check the route points to next hop group # Note: no need to wait here - k = asic_route_exists(keys, "2.2.2.0/24") + k = self.get_route_id("2.2.2.0/24", dvs) assert k is not None fvs = asic_route_nhg_fvs(k) assert fvs is not None # Check the second batch does not point to next hop group - k = asic_route_exists(keys, base_ipprefix) + k = self.get_route_id(base_ipprefix, dvs) assert k is not None fvs = asic_route_nhg_fvs(k) assert not(fvs) @@ -384,20 +800,893 @@ def asic_route_nhg_fvs(k): route_ipprefix = gen_ipprefix(route_count) ps._del(route_ipprefix) route_count += 1 + asic_routes_count -= MAX_ECMP_COUNT # Wait and check the second batch points to next hop group # Check ASIC DB on the count of nexthop groups used, and it should not increase or decrease - asic_db.wait_for_n_keys("ASIC_STATE:SAI_OBJECT_TYPE_NEXT_HOP_GROUP", 10) - keys = asic_db.get_keys("ASIC_STATE:SAI_OBJECT_TYPE_ROUTE_ENTRY") - k = asic_route_exists(keys, base_ipprefix) + asic_db.wait_for_n_keys(self.ASIC_NHG_STR, 10) + k = self.get_route_id(base_ipprefix, dvs) assert k is not None fvs = asic_route_nhg_fvs(k) assert fvs is not None - k = asic_route_exists(keys, last_ipprefix) + k = self.get_route_id(last_ipprefix, dvs) assert k is not None fvs = asic_route_nhg_fvs(k) assert fvs is not None + # Cleanup + + # Remove second batch of routes + for i in range(10): + route_ipprefix = gen_ipprefix(MAX_ECMP_COUNT + i) + ps._del(route_ipprefix) + + asic_db.wait_for_n_keys(self.ASIC_NHG_STR, 0) + asic_db.wait_for_n_keys(self.ASIC_RT_STR, asic_routes_count) + + def test_nhgorch_nhg_exhaust(self, dvs, testlog): + app_db = dvs.get_app_db() + asic_db = dvs.get_asic_db() + nhg_ps = swsscommon.ProducerStateTable(app_db.db_connection, "NEXTHOP_GROUP_TABLE") + rt_ps = swsscommon.ProducerStateTable(app_db.db_connection, "ROUTE_TABLE") + + MAX_ECMP_COUNT = 512 + MAX_PORT_COUNT = 10 + + r = 0 + fmt = '{{0:0{}b}}'.format(MAX_PORT_COUNT) + asic_nhgs_count = len(asic_db.get_keys(self.ASIC_NHG_STR)) + nhg_count = asic_nhgs_count + first_valid_nhg = nhg_count + + def gen_nhg_fvs(binary): + nexthop = [] + ifname = [] + + for i in range(MAX_PORT_COUNT): + if binary[i] == '1': + nexthop.append(self.peer_ip(i)) + ifname.append(self.port_name(i)) + + nexthop = ','.join(nexthop) + ifname = ','.join(ifname) + fvs = swsscommon.FieldValuePairs([("nexthop", nexthop), ("ifname", ifname)]) + + return fvs + + def gen_nhg_index(nhg_number): + return "group{}".format(nhg_number) + + def gen_valid_binary(): + nonlocal r + + while True: + r += 1 + binary = fmt.format(r) + # We need at least 2 ports for a nexthop group + if binary.count('1') <= 1: + continue + return binary + + def create_temp_nhg(): + nonlocal nhg_count + + binary = gen_valid_binary() + nhg_fvs = gen_nhg_fvs(binary) + nhg_index = gen_nhg_index(nhg_count) + nhg_ps.set(nhg_index, nhg_fvs) + nhg_count += 1 + + return nhg_index, binary + + def delete_nhg(): + nonlocal first_valid_nhg + + del_nhg_index = gen_nhg_index(first_valid_nhg) + del_nhg_id = asic_nhgs[del_nhg_index] + + nhg_ps._del(del_nhg_index) + asic_nhgs.pop(del_nhg_index) + first_valid_nhg += 1 + + return del_nhg_id + + def nhg_exists(nhg_index): + return self.get_nhg_id(nhg_index, dvs) is not None + + # Create interfaces + for i in range(MAX_PORT_COUNT): + self.config_intf(i, dvs) + + asic_nhgs = {} + + # Add first batch of next hop groups to reach the NHG limit + while nhg_count < MAX_ECMP_COUNT: + r += 1 + binary = fmt.format(r) + # We need at least 2 ports for a nexthop group + if binary.count('1') <= 1: + continue + nhg_fvs = gen_nhg_fvs(binary) + nhg_index = gen_nhg_index(nhg_count) + nhg_ps.set(nhg_index, nhg_fvs) + + # Wait for the group to be added + asic_db.wait_for_n_keys(self.ASIC_NHG_STR, asic_nhgs_count + nhg_count + 1) + + # Save the NHG index/ID pair + asic_nhgs[nhg_index] = self.get_nhg_id(nhg_index, dvs) + + nhg_count += 1 + + # Add a new next hop group - it should create a temporary one instead + prev_nhgs = asic_db.get_keys(self.ASIC_NHG_STR) + nhg_index, _ = create_temp_nhg() + + # Assert no new group has been added + asic_db.wait_for_n_keys(self.ASIC_NHG_STR, MAX_ECMP_COUNT) + + # Assert the same NHGs are in ASIC DB + assert prev_nhgs == asic_db.get_keys(self.ASIC_NHG_STR) + + # Delete an existing next hop group + del_nhg_id = delete_nhg() + + # Wait for the key to be deleted + asic_db.wait_for_deleted_entry(self.ASIC_NHG_STR, del_nhg_id) + + # Wait for the temporary group to be promoted and replace the deleted + # NHG + asic_db.wait_for_n_keys(self.ASIC_NHG_STR, MAX_ECMP_COUNT) + + # Save the promoted NHG index/ID + asic_nhgs[nhg_index] = self.get_nhg_id(nhg_index, dvs) + + # Update a group + binary = gen_valid_binary() + nhg_fvs = gen_nhg_fvs(binary) + nhg_index = gen_nhg_index(first_valid_nhg) + + # Save the previous members + prev_nhg_members = self.get_nhgm_ids(nhg_index, dvs) + nhg_ps.set(nhg_index, nhg_fvs) + + # Wait a second so the NHG members get updated + time.sleep(1) + + # Assert the group was updated by checking it's members + assert self.get_nhgm_ids(nhg_index, dvs) != prev_nhg_members + + # Create a new temporary group + nhg_index, _ = create_temp_nhg() + time.sleep(1) + + # Delete the temporary group + nhg_ps._del(nhg_index) + + # Assert the NHG does not exist anymore + assert not nhg_exists(nhg_index) + + # Assert the number of groups is the same + asic_db.wait_for_n_keys(self.ASIC_NHG_STR, MAX_ECMP_COUNT) + + # Create a new temporary group + nhg_index, binary = create_temp_nhg() + + # Save the number of group members + binary_count = binary.count('1') + + # Update the temporary group with a different number of members + while True: + r += 1 + binary = fmt.format(r) + # We need at least 2 ports for a nexthop group + if binary.count('1') <= 1 or binary.count('1') == binary_count: + continue + binary_count = binary.count('1') + break + nhg_fvs = gen_nhg_fvs(binary) + nhg_ps.set(nhg_index, nhg_fvs) + + # Delete a group + del_nhg_id = delete_nhg() + + # Wait for the group to be deleted + asic_db.wait_for_deleted_entry(self.ASIC_NHG_STR, del_nhg_id) + + # The temporary group should be promoted + asic_db.wait_for_n_keys(self.ASIC_NHG_STR, MAX_ECMP_COUNT) + + # Save the promoted NHG index/ID + asic_nhgs[nhg_index] = self.get_nhg_id(nhg_index, dvs) + + # Assert it has the updated details by checking the number of members + assert len(self.get_nhgm_ids(nhg_index, dvs)) == binary_count + + # Add a route + asic_rts_count = len(asic_db.get_keys(self.ASIC_RT_STR)) + + rt_fvs = swsscommon.FieldValuePairs([('nexthop_group', nhg_index)]) + rt_ps.set('2.2.2.0/24', rt_fvs) + + # Assert the route is created + asic_db.wait_for_n_keys(self.ASIC_RT_STR, asic_rts_count + 1) + + # Save the previous NHG ID + prev_nhg_id = asic_nhgs[nhg_index] + + # Create a new temporary group + nhg_index, binary = create_temp_nhg() + + # Get the route ID + rt_id = self.get_route_id('2.2.2.0/24', dvs) + assert rt_id != None + + # Update the route to point to the temporary NHG + rt_fvs = swsscommon.FieldValuePairs([('nexthop_group', nhg_index)]) + rt_ps.set('2.2.2.0/24', rt_fvs) + + # Wait for the route to change it's NHG ID + asic_db.wait_for_field_negative_match(self.ASIC_RT_STR, + rt_id, + {'SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID': prev_nhg_id}) + + # Save the new route NHG ID + prev_nhg_id = asic_db.get_entry(self.ASIC_RT_STR, rt_id)['SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID'] + + # Update the temporary NHG with one that has different NHs + + # Create a new binary that uses the other interfaces than the previous + # binary was using + new_binary = [] + + for i in range(len(binary)): + if binary[i] == '1': + new_binary.append('0') + else: + new_binary.append('1') + + binary = ''.join(new_binary) + assert binary.count('1') > 1 + + nhg_fvs = gen_nhg_fvs(binary) + nhg_ps.set(nhg_index, nhg_fvs) + + # The NHG ID of the route should change + asic_db.wait_for_field_negative_match(self.ASIC_RT_STR, + rt_id, + {'SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID': prev_nhg_id}) + + # Delete a NHG. + del_nhg_id = delete_nhg() + + # Wait for the NHG to be deleted + asic_db.wait_for_deleted_entry(self.ASIC_NHG_STR, del_nhg_id) + + # The temporary group should get promoted. + asic_db.wait_for_n_keys(self.ASIC_NHG_STR, MAX_ECMP_COUNT) + + # Save the promoted NHG index/ID + asic_nhgs[nhg_index] = self.get_nhg_id(nhg_index, dvs) + + # Assert the NHGID of the route changed due to temporary group being + # promoted. + asic_db.wait_for_field_match(self.ASIC_RT_STR, + rt_id, + {'SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID': asic_nhgs[nhg_index]}) + + # Try updating the promoted NHG to a single NHG. Should fail as it no + # longer is a temporary NHG + nhg_id = self.get_nhg_id(nhg_index, dvs) + fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1'), + ('ifname', 'Ethernet0')]) + nhg_ps.set(nhg_index, fvs) + time.sleep(1) + assert bool(asic_db.get_entry(self.ASIC_NHG_STR, nhg_id)) == True + + # Save the NHG index + prev_nhg_id = nhg_id + + # Create a temporary next hop groups + nhg_index, binary = create_temp_nhg() + nhg_id = self.get_nhg_id(nhg_index, dvs) + + # Update the route to point to the temporary group + fvs = swsscommon.FieldValuePairs([('nexthop_group', nhg_index)]) + rt_ps.set('2.2.2.0/24', fvs) + + # The previous group should be updated to a single NHG now, freeing a + # NHG slot in ASIC and promoting the temporary one + asic_db.wait_for_n_keys(self.ASIC_NHG_STR, MAX_ECMP_COUNT) + asic_nhgs[nhg_index] = self.get_nhg_id(nhg_index, dvs) + assert self.get_nhg_id(nhg_index, dvs) != prev_nhg_id + + # Create a temporary next hop groups + nhg_index, binary = create_temp_nhg() + nhg_id = self.get_nhg_id(nhg_index, dvs) + + # Update the route to point to the temporary group + fvs = swsscommon.FieldValuePairs([('nexthop_group', nhg_index)]) + rt_ps.set('2.2.2.0/24', fvs) + + # Update the group to a single NHG + fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1'), + ('ifname', 'Ethernet0')]) + nhg_ps.set(nhg_index, fvs) + nhg_id = self.get_nhg_id(nhg_index, dvs) + + # Wait for the route to update it's NHG ID + asic_db.wait_for_field_match(self.ASIC_RT_STR, + rt_id, + {'SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID': nhg_id}) + + # Try to update the nexthop group to another one + fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.3'), + ('ifname', 'Ethernet4')]) + nhg_ps.set(nhg_index, fvs) + + # Wait a second so the database operations finish + time.sleep(1) + + # The update should fail as the group is not temporary anymore and it + # should keep it's previous NHG ID + asic_db.wait_for_field_match(self.ASIC_RT_STR, + rt_id, + {'SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID': nhg_id}) + + # Create a next hop group that contains labeled NHs that do not exist + # in NeighOrch + asic_nhs_count = len(asic_db.get_keys(self.ASIC_NHS_STR)) + fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1,10.0.0.3'), + ('mpls_nh', 'push1,push3'), + ('ifname', 'Ethernet0,Ethernet4')]) + nhg_index = gen_nhg_index(nhg_count) + nhg_ps.set(nhg_index, fvs) + nhg_count += 1 + + # A temporary next hop should be elected to represent the group and + # thus a new labeled next hop should be created + asic_db.wait_for_n_keys(self.ASIC_NHS_STR, asic_nhs_count + 1) + + # Delete a next hop group + delete_nhg() + + # The group should be promoted and the other labeled NH should also get + # created + asic_db.wait_for_n_keys(self.ASIC_NHG_STR, MAX_ECMP_COUNT) + asic_db.wait_for_n_keys(self.ASIC_NHS_STR, asic_nhs_count + 2) + + # Save the promoted NHG index/ID + asic_nhgs[nhg_index] = self.get_nhg_id(nhg_index, dvs) + + # Update the route with a RouteOrch's owned NHG + binary = gen_valid_binary() + nhg_fvs = gen_nhg_fvs(binary) + rt_ps.set('2.2.2.0/24', nhg_fvs) + + # Assert no new group has been added + asic_db.wait_for_n_keys(self.ASIC_NHG_STR, MAX_ECMP_COUNT) + + # Delete a next hop group + delete_nhg() + + # The temporary group should be promoted + asic_db.wait_for_n_keys(self.ASIC_NHG_STR, MAX_ECMP_COUNT) + + # Create a temporary NHG owned by NhgOrch + nhg_index, _ = create_temp_nhg() + + # Assert no new group has been added + asic_db.wait_for_n_keys(self.ASIC_NHG_STR, MAX_ECMP_COUNT) + + # Delete a next hop group + delete_nhg() + + # Check that the temporary group is promoted + asic_db.wait_for_n_keys(self.ASIC_NHG_STR, MAX_ECMP_COUNT) + + # Save the promoted NHG index/ID + asic_nhgs[nhg_index] = self.get_nhg_id(nhg_index, dvs) + + # Create a temporary NHG that contains only NHs that do not exist + nhg_fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.21,10.0.0.23'), + ('ifname', 'Ethernet40,Ethernet44')]) + nhg_index = gen_nhg_index(nhg_count) + nhg_count += 1 + nhg_ps.set(nhg_index, nhg_fvs) + + # Assert the group is not created + assert not nhg_exists(nhg_index) + + # Update the temporary NHG to a valid one + binary = gen_valid_binary() + nhg_fvs = gen_nhg_fvs(binary) + nhg_ps.set(nhg_index, nhg_fvs) + + # Assert the temporary group was updated and the group got created + nhg_id = self.get_nhg_id(nhg_index, dvs) + assert nhg_id is not None + + # Update the temporary NHG to an invalid one again + nhg_fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.21,10.0.0.23'), + ('ifname', 'Ethernet40,Ethernet44')]) + nhg_ps.set(nhg_index, nhg_fvs) + + # The update should fail and the temporary NHG should still be pointing + # to the old valid NH + assert self.get_nhg_id(nhg_index, dvs) == nhg_id + + # Delete the temporary group + nhg_ps._del(nhg_index) + + # Cleanup + + # Delete the route + rt_ps._del('2.2.2.0/24') + asic_db.wait_for_deleted_entry(self.ASIC_RT_STR, rt_id) + + # Delete the next hop groups + for k in asic_nhgs: + nhg_ps._del(k) + asic_db.wait_for_n_keys(self.ASIC_NHG_STR, asic_nhgs_count) + + def test_nhgorch_multi_nh_group(self, dvs, testlog): + for i in range(4): + self.config_intf(i, dvs) + + app_db = dvs.get_app_db() + asic_db = dvs.get_asic_db() + + # create next hop group in APPL DB + + nhg_ps = swsscommon.ProducerStateTable(app_db.db_connection, "NEXTHOP_GROUP_TABLE") + + prev_nhg_keys = asic_db.get_keys(self.ASIC_NHG_STR) + asic_nhgs_count = len(prev_nhg_keys) + asic_nhgms_count = len(asic_db.get_keys(self.ASIC_NHGM_STR)) + + fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1,10.0.0.3,10.0.0.5'), + ("ifname", "Ethernet0,Ethernet4,Ethernet8")]) + nhg_ps.set("group1", fvs) + + # check if group was propagated to ASIC DB + + asic_db.wait_for_n_keys(self.ASIC_NHG_STR, asic_nhgs_count + 1) + keys = asic_db.get_keys(self.ASIC_NHG_STR) + + found_nhg = False + + for nhgid in keys: + if nhgid not in prev_nhg_keys: + found_nhg = True + break + + assert found_nhg == True + + # check if members were propagated to ASIC DB + + asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, asic_nhgms_count + 3) + keys = asic_db.get_keys(self.ASIC_NHGM_STR) + count = 0 + + for k in keys: + fvs = asic_db.get_entry(self.ASIC_NHGM_STR, k) + + if fvs["SAI_NEXT_HOP_GROUP_MEMBER_ATTR_NEXT_HOP_GROUP_ID"] == nhgid: + count += 1 + + assert count == 3 + + # create route in APPL DB + + rt_ps = swsscommon.ProducerStateTable(app_db.db_connection, "ROUTE_TABLE") + + asic_routes_count = len(asic_db.get_keys(self.ASIC_RT_STR)) + + fvs = swsscommon.FieldValuePairs([("nexthop_group", "group1")]) + rt_ps.set("2.2.2.0/24", fvs) + + # check if route was propagated to ASIC DB + + asic_db.wait_for_n_keys(self.ASIC_RT_STR, asic_routes_count + 1) + + k = self.get_route_id('2.2.2.0/24', dvs) + assert k is not None + + # assert the route points to next hop group + fvs = asic_db.get_entry(self.ASIC_RT_STR, k) + assert fvs["SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID"] == nhgid + + # bring links down one-by-one + for i in [0, 1, 2]: + self.flap_intf(i, 'down', dvs) + + keys = asic_db.get_keys(self.ASIC_NHGM_STR) + + assert len(keys) == (asic_nhgms_count + 2 - i) + assert bool(asic_db.get_entry(self.ASIC_NHG_STR, nhgid)) + + # bring links up one-by-one + for i in [0, 1, 2]: + self.flap_intf(i, 'up', dvs) + + keys = asic_db.get_keys(self.ASIC_NHGM_STR) + + assert len(keys) == (asic_nhgms_count + i + 1) + + count = 0 + for k in keys: + fvs = asic_db.get_entry(self.ASIC_NHGM_STR, k) + if fvs["SAI_NEXT_HOP_GROUP_MEMBER_ATTR_NEXT_HOP_GROUP_ID"] == nhgid: + count += 1 + assert count == i + 1 + + # Bring an interface down + self.flap_intf(1, 'down', dvs) + + # One group member will get deleted + asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, 2) + + # Create a group that contains a NH that uses the down link + fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1,10.0.0.3'), + ("ifname", "Ethernet0,Ethernet4")]) + nhg_ps.set('group2', fvs) + + # The group should get created, but it will not contained the NH that + # has the link down + asic_db.wait_for_n_keys(self.ASIC_NHG_STR, asic_nhgs_count + 2) + asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, asic_nhgms_count + 3) + + # Update the NHG with one interface down + fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.3,10.0.0.1'), + ("ifname", "Ethernet4,Ethernet0")]) + nhg_ps.set("group1", fvs) + + # Wait for group members to update - the group will contain only the + # members that have their links up + asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, asic_nhgms_count + 2) + + # Bring the interface up + self.flap_intf(1, 'up', dvs) + + # Check that the missing member of group1 and group2 is being added + asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, asic_nhgms_count + 4) + + # Remove group2 + nhg_ps._del('group2') + asic_db.wait_for_n_keys(self.ASIC_NHG_STR, asic_nhgs_count + 1) + asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, asic_nhgms_count + 2) + + # Create group2 with a NH that does not exist + fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.3,10.0.0.63'), + ("ifname", "Ethernet4,Ethernet124")]) + nhg_ps.set("group2", fvs) + + # The groups should not be created + time.sleep(1) + asic_db.wait_for_n_keys(self.ASIC_NHG_STR, asic_nhgs_count + 1) + + # Update group1 with a NH that does not exist + fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.3,10.0.0.63'), + ("ifname", "Ethernet4,Ethernet124")]) + nhg_ps.set("group1", fvs) + + # The update should fail, leaving group1 with only the unremoved + # members + asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, asic_nhgms_count + 1) + + # Configure the missing NH's interface + self.config_intf(31, dvs) + + # A couple more routes will be added to ASIC_DB + asic_routes_count += 2 + + # Group2 should get created and group1 should be updated + asic_db.wait_for_n_keys(self.ASIC_NHG_STR, asic_nhgs_count + 2) + asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, asic_nhgms_count + 4) + + # Delete group2 + nhg_ps._del('group2') + asic_db.wait_for_n_keys(self.ASIC_NHG_STR, asic_nhgs_count + 1) + + # Update the NHG, adding two new members + fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1,10.0.0.3,10.0.0.5,10.0.0.7'), + ("ifname", "Ethernet0,Ethernet4,Ethernet8,Ethernet12")]) + nhg_ps.set("group1", fvs) + + # Wait for members to be added + asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, asic_nhgms_count + 4) + + # Update the group to one NH only - orchagent should fail as it is + # referenced + fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1'), ("ifname", "Ethernet0")]) + nhg_ps.set("group1", fvs) + time.sleep(1) + + assert bool(asic_db.get_entry(self.ASIC_NHG_STR, nhgid)) + assert len(asic_db.get_keys(self.ASIC_NHGM_STR)) == asic_nhgms_count + 4 + + # Remove route 2.2.2.0/24 + rt_ps._del("2.2.2.0/24") + + # Wait for route 2.2.2.0/24 to be removed + asic_db.wait_for_n_keys(self.ASIC_RT_STR, asic_routes_count) + + # The group is not referenced anymore, so it should get updated to a + # single next hop group, removing the ASIC group + asic_db.wait_for_n_keys(self.ASIC_NHG_STR, asic_nhgs_count) + asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, asic_nhgms_count) + + # Remove group1 + nhg_ps._del("group1") + + def test_nhgorch_single_nh_group(self, dvs, testlog): + for i in range(2): + self.config_intf(i, dvs) + + app_db = dvs.get_app_db() + asic_db = dvs.get_asic_db() + + # Count existing objects + asic_nhgs_count = len(asic_db.get_keys(self.ASIC_NHG_STR)) + + # Create single next hop group in APPL DB + nhg_ps = swsscommon.ProducerStateTable(app_db.db_connection, "NEXTHOP_GROUP_TABLE") + fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1'), ("ifname", "Ethernet0")]) + nhg_ps.set("group1", fvs) + time.sleep(1) + + # Check that the group was not created in ASIC DB + assert len(asic_db.get_keys(self.ASIC_NHG_STR)) == asic_nhgs_count + + # Get the NHG ID. The UT assumes there is only one next hop with IP 10.0.0.1 + count = 0 + nhgid = 0 + for k in asic_db.get_keys(self.ASIC_NHS_STR): + fvs = asic_db.get_entry(self.ASIC_NHS_STR, k) + if fvs['SAI_NEXT_HOP_ATTR_IP'] == '10.0.0.1': + nhgid = k + count += 1 + assert count == 1 + + # create route in APPL DB + + rt_ps = swsscommon.ProducerStateTable(app_db.db_connection, "ROUTE_TABLE") + + asic_routes_count = len(asic_db.get_keys(self.ASIC_RT_STR)) + + fvs = swsscommon.FieldValuePairs([("nexthop_group", "group1")]) + rt_ps.set("2.2.2.0/24", fvs) + + # check if route was propagated to ASIC DB + + asic_db.wait_for_n_keys(self.ASIC_RT_STR, asic_routes_count + 1) + + k = self.get_route_id('2.2.2.0/24', dvs) + + # assert the route points to next hop group + fvs = asic_db.get_entry(self.ASIC_RT_STR, k) + assert fvs["SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID"] == nhgid + + # Bring group's link down + self.flap_intf(0, 'down', dvs) + + # Check that the group still has the same ID and everything works + assert nhgid == self.get_nhg_id('group1', dvs) + + # Bring group's link back up + self.flap_intf(0, 'up', dvs) + + # Check that the group still has the same ID and everything works + assert nhgid == self.get_nhg_id('group1', dvs) + + # Bring an interface down + self.flap_intf(1, 'down', dvs) + + # Create group2 pointing to the NH which's link is down + fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.3'), ("ifname", "Ethernet4")]) + nhg_ps.set("group2", fvs) + + # The group should be created. To test this, add a route pointing to + # it. If the group exists, the route will be created as well. + fvs = swsscommon.FieldValuePairs([("nexthop_group", "group2")]) + rt_ps.set('2.2.4.0/24', fvs) + asic_db.wait_for_n_keys(self.ASIC_RT_STR, asic_routes_count + 2) + + # Delete the route and the group + rt_ps._del('2.2.4.0/24') + nhg_ps._del('group2') + + # Bring the interface back up + self.flap_intf(1, 'up', dvs) + + # Create group2 pointing to a NH that does not exist + fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.61'), ("ifname", "Ethernet120")]) + nhg_ps.set("group2", fvs) + + # The group should fail to be created. To test this, we add a route + # pointing to it. The route should not be created. + fvs = swsscommon.FieldValuePairs([("nexthop_group", "group2")]) + rt_ps.set('2.2.4.0/24', fvs) + asic_db.wait_for_n_keys(self.ASIC_RT_STR, asic_routes_count + 1) + + # Configure the NH's interface + self.config_intf(30, dvs) + + # A couple of more routes will be added to ASIC_DB + asic_routes_count += 2 + + # The group should be created, so the route should be added + asic_db.wait_for_n_keys(self.ASIC_RT_STR, asic_routes_count + 2) + + # Delete the route and the group + rt_ps._del('2.2.4.0/24') + nhg_ps._del('group2') + + # Update the group to a multiple NH group - should fail as the group + # is referenced + fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1,10.0.0.3'), + ("ifname", "Ethernet0,Ethernet4")]) + nhg_ps.set("group1", fvs) + time.sleep(1) + assert len(asic_db.get_keys(self.ASIC_NHG_STR)) == asic_nhgs_count + + # Update group1 to point to another NH - should fail as the group is + # referenced + nhgid = self.get_nhg_id('group1', dvs) + fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.3'), ("ifname", "Ethernet4")]) + nhg_ps.set("group1", fvs) + assert nhgid == self.get_nhg_id('group1', dvs) + + # Remove route 2.2.2.0/24 + rt_ps._del("2.2.2.0/24") + + # Wait for route 2.2.2.0/24 to be removed + asic_db.wait_for_n_keys(self.ASIC_RT_STR, asic_routes_count) + + # The group is not referenced anymore, so it should be updated + assert nhgid != self.get_nhg_id('group1', dvs) + + # Update the group to a multiple NH group - should work as the group is + # not referenced + fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1,10.0.0.3'), + ("ifname", "Ethernet0,Ethernet4")]) + nhg_ps.set("group1", fvs) + asic_db.wait_for_n_keys(self.ASIC_NHG_STR, asic_nhgs_count + 1) + + # Remove group1 + nhg_ps._del("group1") + + def test_nhgorch_label_route(self, dvs, testlog): + for i in range(4): + self.config_intf(i, dvs) + + app_db = dvs.get_app_db() + asic_db = dvs.get_asic_db() + + # create next hop group in APPL DB + + nhg_ps = swsscommon.ProducerStateTable(app_db.db_connection, "NEXTHOP_GROUP_TABLE") + + prev_nhg_keys = asic_db.get_keys(self.ASIC_NHG_STR) + asic_nhgs_count = len(prev_nhg_keys) + asic_nhgms_count = len(asic_db.get_keys(self.ASIC_NHGM_STR)) + + fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1,10.0.0.3,10.0.0.5'), + ("ifname", "Ethernet0,Ethernet4,Ethernet8")]) + nhg_ps.set("group1", fvs) + + # check if group was propagated to ASIC DB + + asic_db.wait_for_n_keys(self.ASIC_NHG_STR, asic_nhgs_count + 1) + keys = asic_db.get_keys(self.ASIC_NHG_STR) + + found_nhg = False + + for nhgid in keys: + if nhgid not in prev_nhg_keys: + found_nhg = True + break + + assert found_nhg == True + + # check if members were propagated to ASIC DB + + asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, asic_nhgms_count + 3) + keys = asic_db.get_keys(self.ASIC_NHGM_STR) + count = 0 + + for k in keys: + fvs = asic_db.get_entry(self.ASIC_NHGM_STR, k) + + if fvs["SAI_NEXT_HOP_GROUP_MEMBER_ATTR_NEXT_HOP_GROUP_ID"] == nhgid: + count += 1 + + assert count == 3 + + # create prefix route in APPL DB + + rt_ps = swsscommon.ProducerStateTable(app_db.db_connection, "ROUTE_TABLE") + + asic_routes_count = len(asic_db.get_keys(self.ASIC_RT_STR)) + + fvs = swsscommon.FieldValuePairs([("nexthop_group", "group1")]) + rt_ps.set("2.2.2.0/24", fvs) + + # check if route was propagated to ASIC DB + + asic_db.wait_for_n_keys(self.ASIC_RT_STR, asic_routes_count + 1) + + k = self.get_route_id('2.2.2.0/24', dvs) + + # assert the route points to next hop group + fvs = asic_db.get_entry(self.ASIC_RT_STR, k) + assert fvs["SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID"] == nhgid + + # create label route in APPL DB pointing to the same next hop group + + lrt_ps = swsscommon.ProducerStateTable(app_db.db_connection, "LABEL_ROUTE_TABLE") + + asic_insegs_count = len(asic_db.get_keys(self.ASIC_INSEG_STR)) + + fvs = swsscommon.FieldValuePairs([("nexthop_group", "group1")]) + lrt_ps.set("20", fvs) + + # check if label route was propagated to ASIC DB + + asic_db.wait_for_n_keys(self.ASIC_INSEG_STR, asic_insegs_count + 1) + + k = self.get_inseg_id('20', dvs) + assert k is not None + + # assert the route points to next hop group + fvs = asic_db.get_entry(self.ASIC_INSEG_STR, k) + assert fvs["SAI_INSEG_ENTRY_ATTR_NEXT_HOP_ID"] == nhgid + + # bring links down one-by-one + for i in [0, 1, 2]: + self.flap_intf(i, 'down', dvs) + + keys = asic_db.get_keys(self.ASIC_NHGM_STR) + + assert len(keys) == (asic_nhgms_count + 2 - i) + assert bool(asic_db.get_entry(self.ASIC_NHG_STR, nhgid)) + + # bring links up one-by-one + for i in [0, 1, 2]: + self.flap_intf(i, 'up', dvs) + + keys = asic_db.get_keys(self.ASIC_NHGM_STR) + + assert len(keys) == (asic_nhgms_count + i + 1) + + count = 0 + for k in keys: + fvs = asic_db.get_entry(self.ASIC_NHGM_STR, k) + if fvs["SAI_NEXT_HOP_GROUP_MEMBER_ATTR_NEXT_HOP_GROUP_ID"] == nhgid: + count += 1 + assert count == i + 1 + + # Bring an interface down + self.flap_intf(1, 'down', dvs) + + # One group member will get deleted + asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, 2) + + # Remove route 2.2.2.0/24 + rt_ps._del("2.2.2.0/24") + + # Wait for route 2.2.2.0/24 to be removed + asic_db.wait_for_n_keys(self.ASIC_RT_STR, asic_routes_count) + + # Remove label route 20 + lrt_ps._del("20") + + # Wait for route 20 to be removed + asic_db.wait_for_n_keys(self.ASIC_INSEG_STR, asic_insegs_count) + + # Remove group1 + nhg_ps._del("group1") # Add Dummy always-pass test at end as workaroud # for issue when Flaky fail on final test it invokes module tear-down before retrying From 0b3e0405928cf5866128e6986cd3e044c9482caf Mon Sep 17 00:00:00 2001 From: Tom cappleman Date: Fri, 10 Sep 2021 11:20:02 +0100 Subject: [PATCH 02/51] Fix up following review --- doc/swss-schema.md | 19 ++- orchagent/mplsrouteorch.cpp | 46 ++---- orchagent/nhgorch.cpp | 286 ++++++------------------------------ orchagent/nhgorch.h | 42 +++--- orchagent/routeorch.cpp | 53 ++----- orchagent/routeorch.h | 14 +- 6 files changed, 110 insertions(+), 350 deletions(-) diff --git a/doc/swss-schema.md b/doc/swss-schema.md index 0dab436fdd..4da3cb3b2a 100644 --- a/doc/swss-schema.md +++ b/doc/swss-schema.md @@ -160,6 +160,7 @@ and reflects the LAG ports into the redis under: `LAG_TABLE::port` key = ROUTE_TABLE:prefix nexthop = *prefix, ;IP addresses separated “,” (empty indicates no gateway) ifname = ifindex? PORT_TABLE.key ; zero or more separated by “,” (zero indicates no interface) + mpls_nh = STRING ; Comma-separated list of MPLS NH info. blackhole = BIT ; Set to 1 if this route is a blackhole (or null0) weight = weight_list ; List of weights. nexthop_group = string ; index within the NEXTHOP_GROUP_TABLE, used instead of nexthop and intf fields @@ -167,13 +168,16 @@ and reflects the LAG ports into the redis under: `LAG_TABLE::port` --------------------------------------------- ###### LABEL_ROUTE_TABLE -; Defines schema for MPLS label route table attributes -key = LABEL_ROUTE_TABLE:mpls_label ; MPLS label -; field = value -nexthop = STRING ; Comma-separated list of nexthops. -ifname = STRING ; Comma-separated list of interfaces. -weight = STRING ; Comma-separated list of weights. -nexthop_group = string ; index within the NEXTHOP_GROUP_TABLE, used instead of nexthop and intf fields + ; Defines schema for MPLS label route table attributes + key = LABEL_ROUTE_TABLE:mpls_label ; MPLS label + ; field = value + nexthop = STRING ; Comma-separated list of nexthops. + ifname = STRING ; Comma-separated list of interfaces. + mpls_nh = STRING ; Comma-separated list of MPLS NH info. + mpls_pop = STRING ; Number of ingress MPLS labels to POP + weight = STRING ; Comma-separated list of weights. + blackhole = BIT ; Set to 1 if this route is a blackhole (or null0) + nexthop_group = string ; index within the NEXTHOP_GROUP_TABLE, used instead of nexthop and intf fields --------------------------------------------- ### NEXTHOP_GROUP_TABLE @@ -182,6 +186,7 @@ nexthop_group = string ; index within the NEXTHOP_GROUP_TABLE, used instead of n key = NEXTHOP_GROUP_TABLE:string ; arbitrary index for the next hop group nexthop = *prefix, ;IP addresses separated “,” (empty indicates no gateway) ifname = ifindex? PORT_TABLE.key ; zero or more separated by “,” (zero indicates no interface) + mpls_nh = STRING ; Comma-separated list of MPLS NH info. weight = weight_list ; List of weights. --------------------------------------------- diff --git a/orchagent/mplsrouteorch.cpp b/orchagent/mplsrouteorch.cpp index c0fb137242..ffdd0c619e 100644 --- a/orchagent/mplsrouteorch.cpp +++ b/orchagent/mplsrouteorch.cpp @@ -123,8 +123,6 @@ void RouteOrch::doLabelTask(Consumer& consumer) if (op == SET_COMMAND) { - SWSS_LOG_DEBUG("Set operation"); - string ips; string aliases; string mpls_nhs; @@ -158,14 +156,6 @@ void RouteOrch::doLabelTask(Consumer& consumer) nhg_index = fvValue(i); } - SWSS_LOG_INFO("Label route %u has nexthop_group: %s, ips: %s, " - "MPLS NHs: %s, aliases: %s", - label, - nhg_index.c_str(), - ips.c_str(), - mpls_nhs.c_str(), - aliases.c_str()); - /* * A route should not fill both nexthop_group and ips / * aliases. @@ -265,12 +255,11 @@ void RouteOrch::doLabelTask(Consumer& consumer) { const NextHopGroup& nh_group = gNhgOrch->getNhg(nhg_index); ctx.nhg = nh_group.getKey(); - ctx.is_temp = nh_group.isTemp(); + ctx.using_temp_nhg = nh_group.isTemp(); } catch (const std::out_of_range& e) { - SWSS_LOG_ERROR("Next hop group %s does not exist", - nhg_index.c_str()); + SWSS_LOG_ERROR("Next hop group %s does not exist", nhg_index.c_str()); ++it; continue; } @@ -303,7 +292,7 @@ void RouteOrch::doLabelTask(Consumer& consumer) else if (m_syncdLabelRoutes.find(vrf_id) == m_syncdLabelRoutes.end() || m_syncdLabelRoutes.at(vrf_id).find(label) == m_syncdLabelRoutes.at(vrf_id).end() || m_syncdLabelRoutes.at(vrf_id).at(label) != RouteNhg(nhg, nhg_index) || - ctx.is_temp) + ctx.using_temp_nhg) { if (addLabelRoute(ctx, nhg)) it = consumer.m_toSync.erase(it); @@ -328,7 +317,6 @@ void RouteOrch::doLabelTask(Consumer& consumer) else if (op == DEL_COMMAND) { /* Cannot locate the route or remove succeed */ - SWSS_LOG_DEBUG("Delete operation"); if (removeLabelRoute(ctx)) it = consumer.m_toSync.erase(it); else @@ -398,7 +386,7 @@ void RouteOrch::doLabelTask(Consumer& consumer) else if (m_syncdLabelRoutes.find(vrf_id) == m_syncdLabelRoutes.end() || m_syncdLabelRoutes.at(vrf_id).find(label) == m_syncdLabelRoutes.at(vrf_id).end() || m_syncdLabelRoutes.at(vrf_id).at(label) != RouteNhg(nhg, ctx.nhg_index) || - ctx.is_temp) + ctx.using_temp_nhg) { if (addLabelRoutePost(ctx, nhg)) it_prev = consumer.m_toSync.erase(it_prev); @@ -445,8 +433,7 @@ void RouteOrch::addTempLabelRoute(LabelRouteBulkContext& ctx, const NextHopGroup */ if (!m_neighOrch->hasNextHop(it->ipKey())) { - SWSS_LOG_INFO("Failed to get next hop %s for %u", - (*it).to_string().c_str(), label); + SWSS_LOG_INFO("Failed to get next hop %s for %u", (*it).to_string().c_str(), label); it = next_hop_set.erase(it); } else @@ -475,10 +462,6 @@ bool RouteOrch::addLabelRoute(LabelRouteBulkContext& ctx, const NextHopGroupKey sai_object_id_t& vrf_id = ctx.vrf_id; Label& label = ctx.label; - SWSS_LOG_NOTICE("Adding route for label %u with next hop(s) %s", - label, - nextHops.to_string().c_str()); - /* next_hop_id indicates the next hop id or next hop group id of this route */ sai_object_id_t next_hop_id; bool blackhole = false; @@ -521,7 +504,7 @@ bool RouteOrch::addLabelRoute(LabelRouteBulkContext& ctx, const NextHopGroupKey } /* See if there is an IP neighbor nexthop */ else if (nexthop.isMplsNextHop() && - m_neighOrch->hasNextHop(NextHopKey(nexthop.ip_address, nexthop.alias))) + m_neighOrch->hasNextHop(nexthop.ipKey())) { m_neighOrch->addNextHop(nexthop); next_hop_id = m_neighOrch->getNextHopId(nexthop); @@ -547,7 +530,8 @@ bool RouteOrch::addLabelRoute(LabelRouteBulkContext& ctx, const NextHopGroupKey /* If the current next hop is part of the next hop group to sync, * then return false and no need to add another temporary route. */ - if (it_route != m_syncdLabelRoutes.at(vrf_id).end() && it_route->second.nhg_key.getSize() == 1) + if (it_route != m_syncdLabelRoutes.at(vrf_id).end() && + it_route->second.nhg_key.getSize() == 1) { const NextHopKey& nexthop = *it_route->second.nhg_key.getNextHops().begin(); if (nextHops.contains(nexthop)) @@ -570,8 +554,7 @@ bool RouteOrch::addLabelRoute(LabelRouteBulkContext& ctx, const NextHopGroupKey } else { - SWSS_LOG_DEBUG("Next hop group is owned by NhgOrch with index %s", - ctx.nhg_index.c_str()); + SWSS_LOG_DEBUG("Next hop group is owned by NhgOrch with index %s", ctx.nhg_index.c_str()); try { const NextHopGroup& nhg = gNhgOrch->getNhg(ctx.nhg_index); @@ -579,8 +562,7 @@ bool RouteOrch::addLabelRoute(LabelRouteBulkContext& ctx, const NextHopGroupKey } catch(const std::out_of_range& e) { - SWSS_LOG_WARN("Next hop group key %s does not exist", - ctx.nhg_index.c_str()); + SWSS_LOG_WARN("Next hop group key %s does not exist", ctx.nhg_index.c_str()); return false; } } @@ -726,12 +708,10 @@ bool RouteOrch::addLabelRoutePost(const LabelRouteBulkContext& ctx, const NextHo } else { - SWSS_LOG_DEBUG("NhgOrch owns the next hop group with index %s", - ctx.nhg_index.c_str()); + SWSS_LOG_DEBUG("NhgOrch owns the next hop group with index %s", ctx.nhg_index.c_str()); if (!gNhgOrch->hasNhg(ctx.nhg_index)) { - SWSS_LOG_WARN("Failed to get next hop group with index %s", - ctx.nhg_index.c_str()); + SWSS_LOG_WARN("Failed to get next hop group with index %s", ctx.nhg_index.c_str()); return false; } } @@ -934,7 +914,7 @@ bool RouteOrch::removeLabelRoutePost(const LabelRouteBulkContext& ctx) */ else if (it_route->second.nhg_key.getSize() == 1) { - NextHopKey nexthop(it_route->second.nhg_key.to_string()); + const NextHopKey& nexthop = *it_route->second.nhg_key.getNextHops().begin(); if (nexthop.isMplsNextHop() && (m_neighOrch->getNextHopRefCount(nexthop) == 0)) { diff --git a/orchagent/nhgorch.cpp b/orchagent/nhgorch.cpp index bf83d1e393..1df46f6335 100644 --- a/orchagent/nhgorch.cpp +++ b/orchagent/nhgorch.cpp @@ -23,10 +23,6 @@ extern sai_switch_api_t* sai_switch_api; unsigned int NextHopGroup::m_count = 0; -/* Default maximum number of next hop groups */ -#define DEFAULT_NUMBER_OF_ECMP_GROUPS 128 -#define DEFAULT_MAX_ECMP_GROUP_SIZE 32 - NhgOrch::NhgOrch(DBConnector *db, string tableName) : Orch(db, tableName) { @@ -62,8 +58,6 @@ NhgOrch::NhgOrch(DBConnector *db, string tableName) : char *platform = getenv("platform"); if (platform && strstr(platform, MLNX_PLATFORM_SUBSTRING)) { - SWSS_LOG_NOTICE("Mellanox platform - divide capacity by %d", - DEFAULT_MAX_ECMP_GROUP_SIZE); m_maxNhgCount /= DEFAULT_MAX_ECMP_GROUP_SIZE; } } @@ -74,20 +68,16 @@ NhgOrch::NhgOrch(DBConnector *db, string tableName) : to_string(m_maxNhgCount)); gSwitchOrch->set_switch_capability(fvTuple); - SWSS_LOG_NOTICE("Maximum number of ECMP groups supported is %d", - m_maxNhgCount); + SWSS_LOG_NOTICE("Maximum number of ECMP groups supported is %d", m_maxNhgCount); } /* * Purpose: Perform the operations requested by APPL_DB users. - * * Description: Iterate over the untreated operations list and resolve them. * The operations supported are SET and DEL. If an operation * could not be resolved, it will either remain in the list, or be * removed, depending on the case. - * * Params: IN consumer - The cosumer object. - * * Returns: Nothing. */ void NhgOrch::doTask(Consumer& consumer) @@ -108,10 +98,7 @@ void NhgOrch::doTask(Consumer& consumer) string index = kfvKey(t); string op = kfvOp(t); - SWSS_LOG_INFO("Next hop group table key %s, op %s", - index.c_str(), op.c_str()); - - bool success; + bool success = true; const auto& nhg_it = m_syncdNextHopGroups.find(index); if (op == SET_COMMAND) @@ -160,9 +147,6 @@ void NhgOrch::doTask(Consumer& consumer) /* If the group does not exist, create one. */ if (nhg_it == m_syncdNextHopGroups.end()) { - SWSS_LOG_NOTICE("Adding next hop group %s with %s", - index.c_str(), - nhg_str.c_str()); /* * If we'd have to create a SAI object for the group, and we * already reached the limit, we're going to create a temporary @@ -171,8 +155,7 @@ void NhgOrch::doTask(Consumer& consumer) * to be kept in the sync list so we keep trying to create the * actual group when there are enough resources. */ - if ((nhg_key.getSize() > 1) && - (NextHopGroup::getCount() >= m_maxNhgCount)) + if ((nhg_key.getSize() > 1) && (NextHopGroup::getCount() >= m_maxNhgCount)) { SWSS_LOG_WARN("Next hop group count reached its limit."); @@ -180,26 +163,21 @@ void NhgOrch::doTask(Consumer& consumer) { auto nhg = std::make_unique( createTempNhg(nhg_key)); - SWSS_LOG_NOTICE("Adding temp next hop group with %s", - nhg->to_string().c_str()); if (nhg->sync()) { - SWSS_LOG_INFO("Temporary NHG successfully synced"); m_syncdNextHopGroups.emplace(index, NhgEntry(std::move(nhg))); } else { - SWSS_LOG_WARN( - "Failed to sync temporary NHG %s with %s", + SWSS_LOG_WARN("Failed to sync temporary NHG %s with %s", index.c_str(), nhg_key.to_string().c_str()); } } catch (const std::exception& e) { - SWSS_LOG_WARN( - "Got exception: %s while adding temp group %s", + SWSS_LOG_WARN("Got exception: %s while adding temp group %s", e.what(), nhg_key.to_string().c_str()); } @@ -217,7 +195,6 @@ void NhgOrch::doTask(Consumer& consumer) if (success) { - SWSS_LOG_INFO("NHG successfully synced"); m_syncdNextHopGroups.emplace(index, NhgEntry(std::move(nhg))); } @@ -226,13 +203,8 @@ void NhgOrch::doTask(Consumer& consumer) /* If the group exists, update it. */ else { - SWSS_LOG_NOTICE("Update next hop group %s with %s", - index.c_str(), - nhg_str.c_str()); - const auto& nhg_ptr = nhg_it->second.nhg; - /* * A NHG update should never change the SAI ID of the NHG if it * is still referenced by some other objects, as they would not @@ -267,15 +239,10 @@ void NhgOrch::doTask(Consumer& consumer) */ if (!nhg_key.contains(nhg_ptr->getKey())) { - SWSS_LOG_NOTICE("Updating temporary group %s to %s", - index.c_str(), - nhg_key.to_string().c_str()); - try { /* Create the new temporary next hop group. */ - auto new_nhg = std::make_unique( - createTempNhg(nhg_key)); + auto new_nhg = std::make_unique(createTempNhg(nhg_key)); /* * If we successfully sync the new group, update @@ -285,22 +252,18 @@ void NhgOrch::doTask(Consumer& consumer) */ if (new_nhg->sync()) { - SWSS_LOG_INFO( - "Temporary NHG successfully synced"); nhg_it->second.nhg = std::move(new_nhg); } else { - SWSS_LOG_WARN( - "Failed to sync updated temp NHG %s with %s", + SWSS_LOG_WARN("Failed to sync updated temp NHG %s with %s", index.c_str(), nhg_key.to_string().c_str()); } } catch (const std::exception& e) { - SWSS_LOG_WARN( - "Got exception: %s while adding temp group %s", + SWSS_LOG_WARN("Got exception: %s while adding temp group %s", e.what(), nhg_key.to_string().c_str()); } @@ -322,22 +285,17 @@ void NhgOrch::doTask(Consumer& consumer) } else if (op == DEL_COMMAND) { - SWSS_LOG_NOTICE("Deleting next hop group %s", index.c_str()); - /* If the group does not exist, do nothing. */ if (nhg_it == m_syncdNextHopGroups.end()) { - SWSS_LOG_WARN("Unable to find group with key %s to remove", - index.c_str()); + SWSS_LOG_WARN("Unable to find group with key %s to remove", index.c_str()); /* Mark the operation as successful to consume it. */ success = true; } /* If the group does exist, but it's still referenced, skip. */ else if (nhg_it->second.ref_count > 0) { - SWSS_LOG_WARN( - "Unable to remove group %s which is referenced", - index.c_str()); + SWSS_LOG_WARN("Unable to remove group %s which is referenced", index.c_str()); success = false; } /* Else, if the group is no more referenced, delete it. */ @@ -345,7 +303,7 @@ void NhgOrch::doTask(Consumer& consumer) { const auto& nhg = nhg_it->second.nhg; - success = nhg->desync(); + success = nhg->delete(); if (success) { @@ -374,12 +332,9 @@ void NhgOrch::doTask(Consumer& consumer) /* * Purpose: Validate a next hop for any groups that contains it. - * * Description: Iterate over all next hop groups and validate the next hop in * those who contain it. - * * Params: IN nh_key - The next hop to validate. - * * Returns: true, if the next hop was successfully validated in all * containing groups; * false, otherwise. @@ -387,7 +342,6 @@ void NhgOrch::doTask(Consumer& consumer) bool NhgOrch::validateNextHop(const NextHopKey& nh_key) { SWSS_LOG_ENTER(); - SWSS_LOG_INFO("Validating next hop %s", nh_key.to_string().c_str()); /* * Iterate through all groups and validate the next hop in those who @@ -397,13 +351,8 @@ bool NhgOrch::validateNextHop(const NextHopKey& nh_key) { auto& nhg = it.second.nhg; - SWSS_LOG_INFO("Check if next hop in group %s", - it.first.c_str()); - if (nhg->hasNextHop(nh_key)) { - SWSS_LOG_INFO("Group has next hop"); - /* * If sync fails, exit right away, as we expect it to be due to a * raeson for which any other future validations will fail too. @@ -423,12 +372,9 @@ bool NhgOrch::validateNextHop(const NextHopKey& nh_key) /* * Purpose: Invalidate a next hop for any groups containing it. - * - * Description: Iterate through the next hop groups and desync the next hop + * Description: Iterate through the next hop groups and delete the next hop * from those that contain it. - * * Params: IN nh_key - The next hop to invalidate. - * * Returns: true, if the next hop was successfully invalidatedd from all * containing groups; * false, otherwise. @@ -436,7 +382,6 @@ bool NhgOrch::validateNextHop(const NextHopKey& nh_key) bool NhgOrch::invalidateNextHop(const NextHopKey& nh_key) { SWSS_LOG_ENTER(); - SWSS_LOG_INFO("Invalidating next hop %s", nh_key.to_string().c_str()); /* * Iterate through all groups and invalidate the next hop from those who @@ -446,14 +391,9 @@ bool NhgOrch::invalidateNextHop(const NextHopKey& nh_key) { auto& nhg = it.second.nhg; - SWSS_LOG_INFO("Check if next hop is in group %s", - it.first.c_str()); - if (nhg->hasNextHop(nh_key)) { - SWSS_LOG_INFO("Group has next hop"); - - /* If the desync fails, exit right away. */ + /* If the delete fails, exit right away. */ if (!nhg->invalidateNextHop(nh_key)) { SWSS_LOG_WARN("Failed to invalidate next hop %s from group %s", @@ -469,11 +409,8 @@ bool NhgOrch::invalidateNextHop(const NextHopKey& nh_key) /* * Purpose: Increase the ref count for a next hop group. - * * Description: Increment the ref count for a next hop group by 1. - * * Params: IN index - The index of the next hop group. - * * Returns: Nothing. */ void NhgOrch::incNhgRefCount(const std::string& index) @@ -481,22 +418,13 @@ void NhgOrch::incNhgRefCount(const std::string& index) SWSS_LOG_ENTER(); NhgEntry& nhg_entry = m_syncdNextHopGroups.at(index); - - SWSS_LOG_INFO("Increment group %s ref count from %u to %u", - index.c_str(), - nhg_entry.ref_count, - nhg_entry.ref_count + 1); - ++nhg_entry.ref_count; } /* * Purpose: Decrease the ref count for a next hop group. - * * Description: Decrement the ref count for a next hop group by 1. - * * Params: IN index - The index of the next hop group. - * * Returns: Nothing. */ void NhgOrch::decNhgRefCount(const std::string& index) @@ -507,22 +435,13 @@ void NhgOrch::decNhgRefCount(const std::string& index) /* Sanity check so we don't overflow. */ assert(nhg_entry.ref_count > 0); - - SWSS_LOG_INFO("Decrement group %s ref count from %u to %u", - index.c_str(), - nhg_entry.ref_count, - nhg_entry.ref_count - 1); - --nhg_entry.ref_count; } /* * Purpose: Get the next hop ID of the member. - * * Description: Get the SAI ID of the next hop from NeighOrch. - * * Params: None. - * * Returns: The SAI ID of the next hop, or SAI_NULL_OBJECT_ID if the next * hop is not valid. */ @@ -534,8 +453,6 @@ sai_object_id_t NextHopGroupMember::getNhId() const if (gNeighOrch->hasNextHop(m_nh_key)) { - SWSS_LOG_INFO("NeighOrch has next hop %s", - m_nh_key.to_string().c_str()); nh_id = gNeighOrch->getNextHopId(m_nh_key); } /* @@ -547,8 +464,6 @@ sai_object_id_t NextHopGroupMember::getNhId() const */ else if (isLabeled() && gNeighOrch->hasNextHop(m_nh_key.ipKey())) { - SWSS_LOG_INFO("Create labeled next hop %s", - m_nh_key.to_string().c_str()); gNeighOrch->addNextHop(m_nh_key); nh_id = gNeighOrch->getNextHopId(m_nh_key); } @@ -558,11 +473,8 @@ sai_object_id_t NextHopGroupMember::getNhId() const /* * Purpose: Move assignment operator. - * * Description: Perform member-wise swap. - * * Params: IN nhgm - The next hop group member to swap. - * * Returns: Reference to this object. */ NextHopGroupMember& NextHopGroupMember::operator=(NextHopGroupMember&& nhgm) @@ -577,22 +489,15 @@ NextHopGroupMember& NextHopGroupMember::operator=(NextHopGroupMember&& nhgm) /* * Purpose: Update the weight of a member. - * * Description: Set the new member's weight and if the member is synced, update * the SAI attribute as well. - * * Params: IN weight - The weight of the next hop group member. - * * Returns: true, if the operation was successful; * false, otherwise. */ bool NextHopGroupMember::updateWeight(uint32_t weight) { SWSS_LOG_ENTER(); - SWSS_LOG_INFO("Update group member %s weight from %u to %u", - m_nh_key.to_string().c_str(), - m_nh_key.weight, - weight); bool success = true; @@ -600,8 +505,6 @@ bool NextHopGroupMember::updateWeight(uint32_t weight) if (isSynced()) { - SWSS_LOG_INFO("Updating SAI weight attribute"); - sai_attribute_t nhgm_attr; nhgm_attr.id = SAI_NEXT_HOP_GROUP_MEMBER_ATTR_WEIGHT; nhgm_attr.value.s32 = m_nh_key.weight; @@ -611,18 +514,14 @@ bool NextHopGroupMember::updateWeight(uint32_t weight) success = status == SAI_STATUS_SUCCESS; } - SWSS_LOG_INFO("Returning %d", success); return success; } /* * Purpose: Sync the group member with the given group member ID. - * * Description: Set the group member's SAI ID to the the one given and * increment the appropriate ref counters. - * * Params: IN gm_id - The group member SAI ID to set. - * * Returns: Nothing. */ void NextHopGroupMember::sync(sai_object_id_t gm_id) @@ -638,27 +537,22 @@ void NextHopGroupMember::sync(sai_object_id_t gm_id) } /* - * Purpose: Desync the group member, resetting it's SAI ID. - * + * Purpose: Delete the group member, resetting it's SAI ID. * Description: Reset the group member's SAI ID and decrement the appropriate * ref counters. - * * Params: None. - * * Returns: Nothing. */ -void NextHopGroupMember::desync() +void NextHopGroupMember::delete() { SWSS_LOG_ENTER(); /* - * If the member is already desynced, exit so we don't decrement the ref + * If the member is already deleted, exit so we don't decrement the ref * counters more than once. */ if (!isSynced()) { - SWSS_LOG_INFO("Next hop group member %s is already desynced", - m_nh_key.to_string().c_str()); return; } @@ -669,12 +563,9 @@ void NextHopGroupMember::desync() /* * Purpose: Destructor. - * - * Description: Assert the group member is desynced and remove the labeled + * Description: Assert the group member is deleted and remove the labeled * next hop from NeighOrch if it is unreferenced. - * * Params: None. - * * Returns: Nothing. */ NextHopGroupMember::~NextHopGroupMember() @@ -682,8 +573,7 @@ NextHopGroupMember::~NextHopGroupMember() SWSS_LOG_ENTER(); /* - * The group member should be desynced from it's group before destroying - * it. + * The group member should be deleted from its group before destroying it. */ assert(!isSynced()); @@ -698,19 +588,14 @@ NextHopGroupMember::~NextHopGroupMember() gNeighOrch->hasNextHop(m_nh_key) && (gNeighOrch->getNextHopRefCount(m_nh_key) == 0)) { - SWSS_LOG_INFO("Delete labeled next hop %s", - m_nh_key.to_string().c_str()); gNeighOrch->removeMplsNextHop(m_nh_key); } } /* * Purpose: Constructor. - * * Description: Initialize the group's members based on the next hop group key. - * * Params: IN key - The next hop group's key. - * * Returns: Nothing. */ NextHopGroup::NextHopGroup(const NextHopGroupKey& key) : @@ -719,24 +604,18 @@ NextHopGroup::NextHopGroup(const NextHopGroupKey& key) : m_is_temp(false) { SWSS_LOG_ENTER(); - SWSS_LOG_INFO("Creating next hop group %s", m_key.to_string().c_str()); /* Parse the key and create the members. */ for (const auto& it : m_key.getNextHops()) { - SWSS_LOG_INFO("Adding next hop %s to the group", - it.to_string().c_str()); m_members.emplace(it, NextHopGroupMember(it)); } } /* * Purpose: Move constructor. - * * Description: Initialize the members by doing member-wise move construct. - * * Params: IN nhg - The rvalue object to initialize from. - * * Returns: Nothing. */ NextHopGroup::NextHopGroup(NextHopGroup&& nhg) : @@ -753,11 +632,8 @@ NextHopGroup::NextHopGroup(NextHopGroup&& nhg) : /* * Purpose: Move assignment operator. - * * Description: Perform member-wise swap. - * * Params: IN nhg - The rvalue object to swap with. - * * Returns: Referene to this object. */ NextHopGroup& NextHopGroup::operator=(NextHopGroup&& nhg) @@ -774,14 +650,11 @@ NextHopGroup& NextHopGroup::operator=(NextHopGroup&& nhg) /* * Purpose: Sync a next hop group. - * * Description: Fill in the NHG ID. If the group contains only one NH, this ID * will be the SAI ID of the next hop that NeighOrch owns. If it * has more than one NH, create a group over the SAI API and then * add it's members. - * * Params: None. - * * Returns: true, if the operation was successful; * false, otherwise. */ @@ -792,7 +665,6 @@ bool NextHopGroup::sync() /* If the group is already synced, exit. */ if (isSynced()) { - SWSS_LOG_INFO("Group %s is already synced", to_string().c_str()); return true; } @@ -803,16 +675,16 @@ bool NextHopGroup::sync() if (m_members.size() == 1) { const NextHopGroupMember& nhgm = m_members.begin()->second; + sai_object_id_t nhid = nhgm.getNhId(); - if (nhgm.getNhId() == SAI_NULL_OBJECT_ID) + if (nhid == SAI_NULL_OBJECT_ID) { - SWSS_LOG_WARN("Next hop %s is not synced", - nhgm.getNhKey().to_string().c_str()); + SWSS_LOG_WARN("Next hop %s is not synced", nhgm.getNhKey().to_string().c_str()); return false; } else { - m_id = nhgm.getNhId(); + m_id = nhid; } } /* If the key contains more than one NH, create a group. */ @@ -865,21 +737,17 @@ bool NextHopGroup::sync() /* * Purpose: Create a temporary next hop group when resources are exhausted. - * * Description: Choose one member to represent the group and create a group * with only that next hop as a member. Any object referencing * the SAI ID of a temporary group should keep querying NhgOrch in * case the group is updated, as it's SAI ID will change at that * point. - * * Params: IN index - The CP index of the next hop group. - * * Returns: The created temporary next hop group. */ NextHopGroup NhgOrch::createTempNhg(const NextHopGroupKey& nhg_key) { SWSS_LOG_ENTER(); - SWSS_LOG_INFO("Syncing temporary group %s", nhg_key.to_string().c_str()); /* Get a list of all valid next hops in the group. */ std::list valid_nhs; @@ -893,9 +761,6 @@ NextHopGroup NhgOrch::createTempNhg(const NextHopGroupKey& nhg_key) */ if (gNeighOrch->hasNextHop(nh_key.ipKey())) { - SWSS_LOG_INFO("Next hop %s is a candidate for temporary group %s", - nh_key.to_string().c_str(), - nhg_key.to_string().c_str()); valid_nhs.push_back(nh_key); } } @@ -920,44 +785,37 @@ NextHopGroup NhgOrch::createTempNhg(const NextHopGroupKey& nhg_key) } /* - * Purpose: Desync the next hop group. - * + * Purpose: Delete the next hop group. * Description: Reset the group's SAI ID. If the group has more than one - * members, desync the members and the group. - * + * members, delete the members and the group. * Params: None. - * * Returns: true, if the operation was successful; * false, otherwise */ -bool NextHopGroup::desync() +bool NextHopGroup::delete() { SWSS_LOG_ENTER(); - SWSS_LOG_INFO("Desyncing group %s", to_string().c_str()); - /* If the group is already desynced, there is nothing to be done. */ + /* If the group is already deleted, there is nothing to be done. */ if (!isSynced()) { - SWSS_LOG_INFO("Group %s is already desynced", - m_key.to_string().c_str()); return true; } /* - * If the group has more than one members, desync it's members, then the + * If the group has more than one members, delete it's members, then the * group. */ if (m_members.size() > 1) { - /* Desync group's members. If we failed to desync the members, exit. */ - if (!desyncMembers(m_key.getNextHops())) + /* Delete group's members. If we failed to delete the members, exit. */ + if (!deleteMembers(m_key.getNextHops())) { - SWSS_LOG_ERROR("Failed to desync group %s members", - to_string().c_str()); + SWSS_LOG_ERROR("Failed to delete group %s members", to_string().c_str()); return false; } - /* Desync the group. */ + /* Delete the group. */ sai_status_t status = sai_next_hop_group_api-> remove_next_hop_group(m_id); @@ -982,22 +840,17 @@ bool NextHopGroup::desync() /* * Purpose: Sync the given next hop group's members over the SAI API. - * * Description: Iterate over the given members and sync them. If the member * is already synced, we skip it. If any of the next hops isn't * already synced by the neighOrch, this will fail. Any next hop * which has the neighbor interface down will be skipped. - * * Params: IN nh_keys - The next hop keys of the members to sync. - * * Returns: true, if the members were added succesfully; * false, otherwise. */ bool NextHopGroup::syncMembers(const std::set& nh_keys) { SWSS_LOG_ENTER(); - SWSS_LOG_INFO("Adding next hop group %s members", - to_string().c_str()); /* This method should not be called for groups with only one NH. */ assert(m_members.size() > 1); @@ -1016,15 +869,11 @@ bool NextHopGroup::syncMembers(const std::set& nh_keys) for (const auto& nh_key : nh_keys) { - SWSS_LOG_INFO("Checking if next hop %s is valid", - nh_key.to_string().c_str()); - NextHopGroupMember& nhgm = m_members.at(nh_key); /* If the member is already synced, continue. */ if (nhgm.getGmId() != SAI_NULL_OBJECT_ID) { - SWSS_LOG_INFO("Update member"); continue; } @@ -1065,9 +914,6 @@ bool NextHopGroup::syncMembers(const std::set& nh_keys) bool success = true; for (const auto& mbr : syncingMembers) { - SWSS_LOG_INFO("Checking next hop member %s has a valid SAI ID", - mbr.first.to_string().c_str()); - /* Check that the returned member ID is valid. */ if (mbr.second == SAI_NULL_OBJECT_ID) { @@ -1081,25 +927,18 @@ bool NextHopGroup::syncMembers(const std::set& nh_keys) } } - SWSS_LOG_INFO("Returning %d", success); - return success; } /* - * Purpose: Desync the given group's members over the SAI API. - * - * Description: Go through the given members and desync them. - * - * Params: IN nh_keys - The next hop keys of the members to desync. - * + * Purpose: Delete the given group's members over the SAI API. + * Description: Go through the given members and delete them. + * Params: IN nh_keys - The next hop keys of the members to delete. * Returns: true, if the operation was successful; * false, otherwise */ -bool NextHopGroup::desyncMembers(const std::set& nh_keys) +bool NextHopGroup::deleteMembers(const std::set& nh_keys) { SWSS_LOG_ENTER(); - SWSS_LOG_INFO("Removing members of group %s", - to_string().c_str()); /* This method should not be called for groups with only one NH. */ assert(m_members.size() > 1); @@ -1108,7 +947,7 @@ bool NextHopGroup::desyncMembers(const std::set& nh_keys) sai_next_hop_group_api, gSwitchId, gMaxBulkSize); /* - * Iterate through the given group members add them to be desynced. + * Iterate through the given group members add them to be deleted. * * Keep track of the SAI delete statuses in case one of them returns an * error. We assume that deletion should always succeed. If for some @@ -1119,14 +958,10 @@ bool NextHopGroup::desyncMembers(const std::set& nh_keys) for (const auto& nh_key : nh_keys) { - SWSS_LOG_INFO("Desyncing member %s", nh_key.to_string().c_str()); - const NextHopGroupMember& nhgm = m_members.at(nh_key); if (nhgm.isSynced()) { - SWSS_LOG_INFO("Removing next hop group member %s", - nh_key.to_string().c_str()); nextHopGroupMemberBulker.remove_entry(&statuses[nh_key], nhgm.getGmId()); } @@ -1144,13 +979,9 @@ bool NextHopGroup::desyncMembers(const std::set& nh_keys) for (const auto& status : statuses) { - SWSS_LOG_INFO("Check member's %s status", - status.first.to_string().c_str()); - if (status.second == SAI_STATUS_SUCCESS) { - SWSS_LOG_INFO("Member was successfully desynced"); - m_members.at(status.first).desync(); + m_members.at(status.first).delete(); } else { @@ -1160,30 +991,23 @@ bool NextHopGroup::desyncMembers(const std::set& nh_keys) } } - SWSS_LOG_INFO("Returning %d", success); - return success; } /* * Purpose: Update the next hop group based on a new next hop group key. - * * Description: Update the group's members by removing the members that aren't * in the new next hop group and adding the new members. We first * remove the missing members to avoid cases where we reached the * ASIC group members limit. This will not update the group's SAI * ID in any way, unless we are promoting a temporary group. - * * Params: IN nhg_key - The new next hop group key to update to. - * * Returns: true, if the operation was successful; * false, otherwise. */ bool NextHopGroup::update(const NextHopGroupKey& nhg_key) { SWSS_LOG_ENTER(); - SWSS_LOG_INFO("Update group %s with %s", - to_string().c_str(), nhg_key.to_string().c_str()); /* * There are three cases where the SAI ID of the NHG will change: @@ -1199,8 +1023,6 @@ bool NextHopGroup::update(const NextHopGroupKey& nhg_key) */ if ((nhg_key.getSize() == 1) || (m_members.size() == 1) || !isSynced()) { - SWSS_LOG_INFO("Updating group without preserving it's SAI ID"); - bool was_synced = isSynced(); *this = NextHopGroup(nhg_key); @@ -1232,8 +1054,6 @@ bool NextHopGroup::update(const NextHopGroupKey& nhg_key) /* If the member is not found, then it needs to be removed. */ if (new_nh_key_it == new_nh_keys.end()) { - SWSS_LOG_INFO("Add member %s to be desynced", - nh_key.to_string().c_str()); removed_nh_keys.insert(nh_key); } /* If the member is updated, update it's weight. */ @@ -1241,8 +1061,7 @@ bool NextHopGroup::update(const NextHopGroupKey& nhg_key) { if (!mbr_it.second.updateWeight(new_nh_key_it->weight)) { - SWSS_LOG_WARN("Failed to update member %s weight", - nh_key.to_string().c_str()); + SWSS_LOG_WARN("Failed to update member %s weight", nh_key.to_string().c_str()); return false; } @@ -1254,15 +1073,14 @@ bool NextHopGroup::update(const NextHopGroupKey& nhg_key) } } - /* Desync the removed members. */ - if (!desyncMembers(removed_nh_keys)) + /* Delete the removed members. */ + if (!deleteMembers(removed_nh_keys)) { - SWSS_LOG_WARN("Failed to desync members from group %s", - to_string().c_str()); + SWSS_LOG_WARN("Failed to delete members from group %s", to_string().c_str()); return false; } - /* Remove the desynced members. */ + /* Remove the deleted members. */ for (const auto& nh_key : removed_nh_keys) { m_members.erase(nh_key); @@ -1281,8 +1099,7 @@ bool NextHopGroup::update(const NextHopGroupKey& nhg_key) */ if (!syncMembers(m_key.getNextHops())) { - SWSS_LOG_WARN("Failed to sync new members for group %s", - to_string().c_str()); + SWSS_LOG_WARN("Failed to sync new members for group %s", to_string().c_str()); return false; } @@ -1292,11 +1109,8 @@ bool NextHopGroup::update(const NextHopGroupKey& nhg_key) /* * Purpose: Create the attributes vector for a next hop group member. - * * Description: Create the group ID and next hop ID attributes. - * * Params: IN nhgm - The next hop group member. - * * Returns: The attributes vector for the given next hop. */ vector NextHopGroup::createNhgmAttrs( @@ -1327,20 +1141,14 @@ vector NextHopGroup::createNhgmAttrs( /* * Purpose: Validate a next hop in the group. - * * Description: Sync the validated next hop group member. - * * Params: IN nh_key - The next hop to validate. - * * Returns: true, if the operation was successful; * false, otherwise. */ bool NextHopGroup::validateNextHop(const NextHopKey& nh_key) { SWSS_LOG_ENTER(); - SWSS_LOG_INFO("Validate NH %s in group %s", - nh_key.to_string().c_str(), - to_string().c_str()); /* * If the group has only one member, there is nothing to be done. The @@ -1357,20 +1165,14 @@ bool NextHopGroup::validateNextHop(const NextHopKey& nh_key) /* * Purpose: Invalidate a next hop in the group. - * * Description: Sync the invalidated next hop group member. - * * Params: IN nh_key - The next hop to invalidate. - * * Returns: true, if the operation was successful; * false, otherwise. */ bool NextHopGroup::invalidateNextHop(const NextHopKey& nh_key) { SWSS_LOG_ENTER(); - SWSS_LOG_INFO("Invalidate NH %s in group %s", - nh_key.to_string().c_str(), - to_string().c_str()); /* * If the group has only one member, there is nothing to be done. The @@ -1382,5 +1184,5 @@ bool NextHopGroup::invalidateNextHop(const NextHopKey& nh_key) return true; } - return desyncMembers({nh_key}); + return deleteMembers({nh_key}); } diff --git a/orchagent/nhgorch.h b/orchagent/nhgorch.h index f7e4bcd4b6..73046ca83e 100644 --- a/orchagent/nhgorch.h +++ b/orchagent/nhgorch.h @@ -3,6 +3,10 @@ #include "orch.h" #include "nexthopgroupkey.h" +/* Default maximum number of next hop groups */ +#define DEFAULT_NUMBER_OF_ECMP_GROUPS 128 +#define DEFAULT_MAX_ECMP_GROUP_SIZE 32 + class NextHopGroupMember { public: @@ -31,9 +35,9 @@ class NextHopGroupMember /* Update member's weight and update the SAI attribute as well. */ bool updateWeight(uint32_t weight); - /* Sync / Desync. */ + /* Sync / Delete. */ void sync(sai_object_id_t gm_id); - void desync(); + void delete(); /* Getters / Setters. */ inline const NextHopKey& getNhKey() const { return m_nh_key; } @@ -48,8 +52,7 @@ class NextHopGroupMember /* Convert member's details to string. */ std::string to_string() const { - return m_nh_key.to_string() + - ", SAI ID: " + std::to_string(m_gm_id); + return m_nh_key.to_string() + ", SAI ID: " + std::to_string(m_gm_id); } private: @@ -75,28 +78,30 @@ class NextHopGroup NextHopGroup& operator=(NextHopGroup&& nhg); /* Destructor. */ - virtual ~NextHopGroup() { desync(); } + virtual ~NextHopGroup() { delete(); } /* Sync the group, creating the group's and members SAI IDs. */ bool sync(); - /* Desync the group, reseting the group's and members SAI IDs. */ - bool desync(); + /* Delete the group, reseting the group's and members SAI IDs. */ + bool delete(); /* * Update the group based on a new next hop group key. This will also - * perform any sync / desync necessary. + * perform any sync / delete necessary. */ bool update(const NextHopGroupKey& nhg_key); /* Check if the group contains the given next hop. */ - inline bool hasNextHop(const NextHopKey& nh_key) const { - return m_members.find(nh_key) != m_members.end(); } + inline bool hasNextHop(const NextHopKey& nh_key) const + { + return m_members.find(nh_key) != m_members.end(); + } /* Validate a next hop in the group, syncing it. */ bool validateNextHop(const NextHopKey& nh_key); - /* Invalidate a next hop in the group, desyncing it. */ + /* Invalidate a next hop in the group, deleteing it. */ bool invalidateNextHop(const NextHopKey& nh_key); /* Increment the number of existing groups. */ @@ -145,11 +150,10 @@ class NextHopGroup bool syncMembers(const std::set& nh_keys); /* Remove group's members the SAI API from the given keys. */ - bool desyncMembers(const std::set& nh_keys); + bool deleteMembers(const std::set& nh_keys); /* Create the attributes vector for a next hop group member. */ - vector createNhgmAttrs( - const NextHopGroupMember& nhgm) const; + vector createNhgmAttrs(const NextHopGroupMember& nhgm) const; }; /* @@ -192,23 +196,23 @@ class NhgOrch : public Orch /* Check if the next hop group given by it's index exists. */ inline bool hasNhg(const std::string& index) const - { return m_syncdNextHopGroups.find(index) != - m_syncdNextHopGroups.end(); } + { + return m_syncdNextHopGroups.find(index) != m_syncdNextHopGroups.end(); + } /* * Get the next hop group given by it's index. If the index does not exist * in map, a std::out_of_range exception will be thrown. */ inline const NextHopGroup& getNhg(const std::string& index) const - { return *m_syncdNextHopGroups.at(index).nhg; } + { return *m_syncdNextHopGroups.at(index).nhg; } /* Add a temporary next hop group when resources are exhausted. */ NextHopGroup createTempNhg(const NextHopGroupKey& nhg_key); /* Getters / Setters. */ inline unsigned int getMaxNhgCount() const { return m_maxNhgCount; } - static inline unsigned int getNhgCount() - { return NextHopGroup::getCount(); } + static inline unsigned int getNhgCount() { return NextHopGroup::getCount(); } /* Validate / Invalidate a next hop. */ bool validateNextHop(const NextHopKey& nh_key); diff --git a/orchagent/routeorch.cpp b/orchagent/routeorch.cpp index 0bb583ede4..98c2cbd9a8 100644 --- a/orchagent/routeorch.cpp +++ b/orchagent/routeorch.cpp @@ -23,10 +23,6 @@ extern NhgOrch *gNhgOrch; extern size_t gMaxBulkSize; -/* Default maximum number of next hop groups */ -#define DEFAULT_NUMBER_OF_ECMP_GROUPS 128 -#define DEFAULT_MAX_ECMP_GROUP_SIZE 32 - RouteOrch::RouteOrch(DBConnector *db, vector &tableNames, NeighOrch *neighOrch, IntfsOrch *intfsOrch, VRFOrch *vrfOrch, FgNhgOrch *fgNhgOrch) : gRouteBulker(sai_route_api, gMaxBulkSize), gLabelRouteBulker(sai_mpls_api, gMaxBulkSize), @@ -516,8 +512,6 @@ void RouteOrch::doTask(Consumer& consumer) if (op == SET_COMMAND) { - SWSS_LOG_DEBUG("Set operation"); - string ips; string aliases; string mpls_nhs; @@ -558,22 +552,13 @@ void RouteOrch::doTask(Consumer& consumer) nhg_index = fvValue(i); } - SWSS_LOG_INFO("Route %s has nexthop_group: %s, ips: %s, " - "MPLS nhs: %s, aliases: %s", - ip_prefix.to_string().c_str(), - nhg_index.c_str(), - ips.c_str(), - mpls_nhs.c_str(), - aliases.c_str()); - /* * A route should not fill both nexthop_group and ips / * aliases. */ if (!nhg_index.empty() && (!ips.empty() || !aliases.empty())) { - SWSS_LOG_ERROR("Route %s has both nexthop_group and ips/aliases", - key.c_str()); + SWSS_LOG_ERROR("Route %s has both nexthop_group and ips/aliases", key.c_str()); it = consumer.m_toSync.erase(it); continue; } @@ -699,12 +684,11 @@ void RouteOrch::doTask(Consumer& consumer) { const NextHopGroup& nh_group = gNhgOrch->getNhg(nhg_index); ctx.nhg = nh_group.getKey(); - ctx.is_temp = nh_group.isTemp(); + ctx.using_temp_nhg = nh_group.isTemp(); } catch (const std::out_of_range& e) { - SWSS_LOG_ERROR("Next hop group %s does not exist", - nhg_index.c_str()); + SWSS_LOG_ERROR("Next hop group %s does not exist", nhg_index.c_str()); ++it; continue; } @@ -755,7 +739,7 @@ void RouteOrch::doTask(Consumer& consumer) else if (m_syncdRoutes.find(vrf_id) == m_syncdRoutes.end() || m_syncdRoutes.at(vrf_id).find(ip_prefix) == m_syncdRoutes.at(vrf_id).end() || m_syncdRoutes.at(vrf_id).at(ip_prefix) != RouteNhg(nhg, ctx.nhg_index) || - ctx.is_temp) + ctx.using_temp_nhg) { if (addRoute(ctx, nhg)) it = consumer.m_toSync.erase(it); @@ -779,8 +763,6 @@ void RouteOrch::doTask(Consumer& consumer) } else if (op == DEL_COMMAND) { - SWSS_LOG_DEBUG("Delete operation"); - if (removeRoute(ctx)) it = consumer.m_toSync.erase(it); else @@ -850,7 +832,7 @@ void RouteOrch::doTask(Consumer& consumer) else if (m_syncdRoutes.find(vrf_id) == m_syncdRoutes.end() || m_syncdRoutes.at(vrf_id).find(ip_prefix) == m_syncdRoutes.at(vrf_id).end() || m_syncdRoutes.at(vrf_id).at(ip_prefix) != RouteNhg(nhg, ctx.nhg_index) || - ctx.is_temp) + ctx.using_temp_nhg) { if (addRoutePost(ctx, nhg)) it_prev = consumer.m_toSync.erase(it_prev); @@ -1092,8 +1074,8 @@ bool RouteOrch::addNextHopGroup(const NextHopGroupKey &nexthops) if (gNhgOrch->getNhgCount() >= gNhgOrch->getMaxNhgCount()) { - SWSS_LOG_WARN("Reached maximum next hop groups of %u", - gNhgOrch->getMaxNhgCount()); + SWSS_LOG_DEBUG("Failed to create new next hop group. \ + Reaching maximum number of next hop groups."); return false; } @@ -1120,7 +1102,7 @@ bool RouteOrch::addNextHopGroup(const NextHopGroupKey &nexthops) } else { - SWSS_LOG_WARN("Failed to get next hop %s in %s", + SWSS_LOG_INFO("Failed to get next hop %s in %s", it.to_string().c_str(), nexthops.to_string().c_str()); return false; } @@ -1672,8 +1654,6 @@ bool RouteOrch::addRoute(RouteBulkContext& ctx, const NextHopGroupKey &nextHops) } else { - SWSS_LOG_DEBUG("Next hop group is owned by NhgOrch with index %s", - ctx.nhg_index.c_str()); try { const NextHopGroup& nhg = gNhgOrch->getNhg(ctx.nhg_index); @@ -1681,8 +1661,7 @@ bool RouteOrch::addRoute(RouteBulkContext& ctx, const NextHopGroupKey &nextHops) } catch(const std::out_of_range& e) { - SWSS_LOG_WARN("Next hop group key %s does not exist", - ctx.nhg_index.c_str()); + SWSS_LOG_WARN("Next hop group key %s does not exist", ctx.nhg_index.c_str()); return false; } } @@ -1763,10 +1742,6 @@ bool RouteOrch::addRoute(RouteBulkContext& ctx, const NextHopGroupKey &nextHops) } } - SWSS_LOG_NOTICE("Added route %s with next hop(s) %s", - ipPrefix.to_string().c_str(), - nextHops.to_string().c_str()); - return false; } @@ -1787,8 +1762,6 @@ bool RouteOrch::addRoutePost(const RouteBulkContext& ctx, const NextHopGroupKey return false; } - SWSS_LOG_DEBUG("Checking next hop group %s", nextHops.to_string().c_str()); - if (m_fgNhgOrch->isRouteFineGrained(vrf_id, ipPrefix, nextHops)) { /* Route is pointing to Fine Grained ECMP nexthop group */ @@ -1797,7 +1770,6 @@ bool RouteOrch::addRoutePost(const RouteBulkContext& ctx, const NextHopGroupKey /* Check that the next hop group is not owned by NhgOrch. */ else if (ctx.nhg_index.empty()) { - SWSS_LOG_DEBUG("Next hop group is not owned by NhgOrch"); if (nextHops.getSize() == 0) { /* The route is pointing to a blackhole */ @@ -1844,12 +1816,9 @@ bool RouteOrch::addRoutePost(const RouteBulkContext& ctx, const NextHopGroupKey } else { - SWSS_LOG_DEBUG("NhgOrch owns the next hop group with index %s", - ctx.nhg_index.c_str()); if (!gNhgOrch->hasNhg(ctx.nhg_index)) { - SWSS_LOG_WARN("Failed to get next hop group with index %s", - ctx.nhg_index.c_str()); + SWSS_LOG_WARN("Failed to get next hop group with index %s", ctx.nhg_index.c_str()); return false; } } @@ -2060,7 +2029,7 @@ bool RouteOrch::addRoutePost(const RouteBulkContext& ctx, const NextHopGroupKey * in order to keep trying to update the route in case the NHG is updated, * which will update the SAI ID of the group as well. */ - return !ctx.is_temp; + return !ctx.using_temp_nhg; } bool RouteOrch::removeRoute(RouteBulkContext& ctx) diff --git a/orchagent/routeorch.h b/orchagent/routeorch.h index d9066605c6..453138b98a 100644 --- a/orchagent/routeorch.h +++ b/orchagent/routeorch.h @@ -110,11 +110,11 @@ struct RouteBulkContext sai_object_id_t vrf_id; IpPrefix ip_prefix; bool excp_intfs_flag; - // is_temp will track if the NhgOrch's owned NHG is temporary or not - bool is_temp; + // using_temp_nhg will track if the NhgOrch's owned NHG is temporary or not + bool using_temp_nhg; RouteBulkContext() - : excp_intfs_flag(false), is_temp(false) + : excp_intfs_flag(false), using_temp_nhg(false) { } @@ -129,7 +129,7 @@ struct RouteBulkContext nhg.clear(); excp_intfs_flag = false; vrf_id = SAI_NULL_OBJECT_ID; - is_temp = false; + using_temp_nhg = false; } }; @@ -143,11 +143,11 @@ struct LabelRouteBulkContext Label label; bool excp_intfs_flag; uint8_t pop_count; - // is_temp will track if the NhgOrch's owned NHG is temporary or not - bool is_temp; + // using_temp_nhg will track if the NhgOrch's owned NHG is temporary or not + bool using_temp_nhg; LabelRouteBulkContext() - : excp_intfs_flag(false), is_temp(false) + : excp_intfs_flag(false), using_temp_nhg(false) { } From c631af9ab69b0e9fe8e4d21a25591d97c565b879 Mon Sep 17 00:00:00 2001 From: Tom cappleman Date: Fri, 10 Sep 2021 11:56:14 +0100 Subject: [PATCH 03/51] More fixes --- orchagent/mplsrouteorch.cpp | 2 +- orchagent/neighorch.cpp | 16 +++++++-- orchagent/neighorch.h | 1 + orchagent/nhgorch.cpp | 70 ++++++++++++++++++------------------- orchagent/nhgorch.h | 18 +++++----- orchagent/routeorch.cpp | 4 +-- 6 files changed, 61 insertions(+), 50 deletions(-) diff --git a/orchagent/mplsrouteorch.cpp b/orchagent/mplsrouteorch.cpp index ffdd0c619e..a1209c0e28 100644 --- a/orchagent/mplsrouteorch.cpp +++ b/orchagent/mplsrouteorch.cpp @@ -431,7 +431,7 @@ void RouteOrch::addTempLabelRoute(LabelRouteBulkContext& ctx, const NextHopGroup * a labeled one, which are created by RouteOrch or NhgOrch if the IP * next hop exists. */ - if (!m_neighOrch->hasNextHop(it->ipKey())) + if (!m_neighOrch->isNeighborResolved(*it)) { SWSS_LOG_INFO("Failed to get next hop %s for %u", (*it).to_string().c_str(), label); it = next_hop_set.erase(it); diff --git a/orchagent/neighorch.cpp b/orchagent/neighorch.cpp index 10eff1721b..23d7a38b3b 100644 --- a/orchagent/neighorch.cpp +++ b/orchagent/neighorch.cpp @@ -34,7 +34,7 @@ NeighOrch::NeighOrch(DBConnector *appDb, string tableName, IntfsOrch *intfsOrch, SWSS_LOG_ENTER(); m_fdbOrch->attach(this); - + if(gMySwitchType == "voq") { //Add subscriber to process VOQ system neigh @@ -171,6 +171,16 @@ bool NeighOrch::hasNextHop(const NextHopKey &nexthop) return m_syncdNextHops.find(nexthop) != m_syncdNextHops.end(); } +// Check if the underlying neighbor is resolved for a given next hop key. +bool NeighOrch::isNeighborResolved(const NextHopKey &nexthop) +{ + // Extract the IP address and interface from the next hop key, and check if the next hop + // for just that pair exists. + NextHopKey base_nexthop = NextHopKey(nexthop.ip_address, nexthop.alias); + + return hasNextHop(base_nexthop); +} + bool NeighOrch::addNextHop(const NextHopKey &nh) { SWSS_LOG_ENTER(); @@ -1028,7 +1038,7 @@ bool NeighOrch::removeNeighbor(const NeighborEntry &neighborEntry, bool disable) NeighborUpdate update = { neighborEntry, MacAddress(), false }; notify(SUBJECT_TYPE_NEIGH_CHANGE, static_cast(&update)); - + if(gMySwitchType == "voq") { //Sync the neighbor to delete from the CHASSIS_APP_DB @@ -1302,7 +1312,7 @@ void NeighOrch::doVoqSystemNeighTask(Consumer &consumer) // the owner asic's mac is not readily avaiable here, the owner asic mac is derived from // the switch id and lower 5 bytes of asic mac which is assumed to be same for all asics // in the VS system. - // Therefore to make VOQ chassis systems work in VS platform based setups like the setups + // Therefore to make VOQ chassis systems work in VS platform based setups like the setups // using KVMs, it is required that all asics have same base mac in the format given below // :<6th byte = switch_id> diff --git a/orchagent/neighorch.h b/orchagent/neighorch.h index 3b5867a949..7a9a1fa330 100644 --- a/orchagent/neighorch.h +++ b/orchagent/neighorch.h @@ -49,6 +49,7 @@ class NeighOrch : public Orch, public Subject, public Observer ~NeighOrch(); bool hasNextHop(const NextHopKey&); + bool isNeighborResolved(const NextHopKey&); bool addNextHop(const NextHopKey&); bool removeMplsNextHop(const NextHopKey&); diff --git a/orchagent/nhgorch.cpp b/orchagent/nhgorch.cpp index 1df46f6335..4bbdc84889 100644 --- a/orchagent/nhgorch.cpp +++ b/orchagent/nhgorch.cpp @@ -298,12 +298,12 @@ void NhgOrch::doTask(Consumer& consumer) SWSS_LOG_WARN("Unable to remove group %s which is referenced", index.c_str()); success = false; } - /* Else, if the group is no more referenced, delete it. */ + /* Else, if the group is no more referenced, remove it. */ else { const auto& nhg = nhg_it->second.nhg; - success = nhg->delete(); + success = nhg->remove(); if (success) { @@ -372,7 +372,7 @@ bool NhgOrch::validateNextHop(const NextHopKey& nh_key) /* * Purpose: Invalidate a next hop for any groups containing it. - * Description: Iterate through the next hop groups and delete the next hop + * Description: Iterate through the next hop groups and remove the next hop * from those that contain it. * Params: IN nh_key - The next hop to invalidate. * Returns: true, if the next hop was successfully invalidatedd from all @@ -393,7 +393,7 @@ bool NhgOrch::invalidateNextHop(const NextHopKey& nh_key) if (nhg->hasNextHop(nh_key)) { - /* If the delete fails, exit right away. */ + /* If the remove fails, exit right away. */ if (!nhg->invalidateNextHop(nh_key)) { SWSS_LOG_WARN("Failed to invalidate next hop %s from group %s", @@ -462,7 +462,7 @@ sai_object_id_t NextHopGroupMember::getNhId() const * after the object is created and would never create the labeled next hop * afterwards. */ - else if (isLabeled() && gNeighOrch->hasNextHop(m_nh_key.ipKey())) + else if (isLabeled() && gNeighOrch->isNeighborResolved(m_nh_key)) { gNeighOrch->addNextHop(m_nh_key); nh_id = gNeighOrch->getNextHopId(m_nh_key); @@ -537,18 +537,18 @@ void NextHopGroupMember::sync(sai_object_id_t gm_id) } /* - * Purpose: Delete the group member, resetting it's SAI ID. + * Purpose: Remove the group member, resetting it's SAI ID. * Description: Reset the group member's SAI ID and decrement the appropriate * ref counters. * Params: None. * Returns: Nothing. */ -void NextHopGroupMember::delete() +void NextHopGroupMember::remove() { SWSS_LOG_ENTER(); /* - * If the member is already deleted, exit so we don't decrement the ref + * If the member is already removed, exit so we don't decrement the ref * counters more than once. */ if (!isSynced()) @@ -563,7 +563,7 @@ void NextHopGroupMember::delete() /* * Purpose: Destructor. - * Description: Assert the group member is deleted and remove the labeled + * Description: Assert the group member is removed and remove the labeled * next hop from NeighOrch if it is unreferenced. * Params: None. * Returns: Nothing. @@ -573,15 +573,15 @@ NextHopGroupMember::~NextHopGroupMember() SWSS_LOG_ENTER(); /* - * The group member should be deleted from its group before destroying it. + * The group member should be removed from its group before destroying it. */ assert(!isSynced()); /* - * If the labeled next hop is unreferenced, delete it from NeighOrch as + * If the labeled next hop is unreferenced, remove it from NeighOrch as * NhgOrch and RouteOrch are the ones controlling it's lifetime. They both * watch over these labeled next hops, so it doesn't matter who created - * them as they're both doing the same checks before deleting a labeled + * them as they're both doing the same checks before removing a labeled * next hop. */ if (isLabeled() && @@ -759,7 +759,7 @@ NextHopGroup NhgOrch::createTempNhg(const NextHopGroupKey& nhg_key) * the group might contain labeled NHs which we should create if their * IP next hop does exist. */ - if (gNeighOrch->hasNextHop(nh_key.ipKey())) + if (gNeighOrch->isNeighborResolved(nh_key)) { valid_nhs.push_back(nh_key); } @@ -785,37 +785,37 @@ NextHopGroup NhgOrch::createTempNhg(const NextHopGroupKey& nhg_key) } /* - * Purpose: Delete the next hop group. + * Purpose: Remove the next hop group. * Description: Reset the group's SAI ID. If the group has more than one - * members, delete the members and the group. + * members, remove the members and the group. * Params: None. * Returns: true, if the operation was successful; * false, otherwise */ -bool NextHopGroup::delete() +bool NextHopGroup::remove() { SWSS_LOG_ENTER(); - /* If the group is already deleted, there is nothing to be done. */ + /* If the group is already removed, there is nothing to be done. */ if (!isSynced()) { return true; } /* - * If the group has more than one members, delete it's members, then the + * If the group has more than one members, remove it's members, then the * group. */ if (m_members.size() > 1) { - /* Delete group's members. If we failed to delete the members, exit. */ - if (!deleteMembers(m_key.getNextHops())) + /* Remove group's members. If we failed to remove the members, exit. */ + if (!removeMembers(m_key.getNextHops())) { - SWSS_LOG_ERROR("Failed to delete group %s members", to_string().c_str()); + SWSS_LOG_ERROR("Failed to remove group %s members", to_string().c_str()); return false; } - /* Delete the group. */ + /* Remove the group. */ sai_status_t status = sai_next_hop_group_api-> remove_next_hop_group(m_id); @@ -930,13 +930,13 @@ bool NextHopGroup::syncMembers(const std::set& nh_keys) return success; } /* - * Purpose: Delete the given group's members over the SAI API. - * Description: Go through the given members and delete them. - * Params: IN nh_keys - The next hop keys of the members to delete. + * Purpose: Remove the given group's members over the SAI API. + * Description: Go through the given members and remove them. + * Params: IN nh_keys - The next hop keys of the members to remove. * Returns: true, if the operation was successful; * false, otherwise */ -bool NextHopGroup::deleteMembers(const std::set& nh_keys) +bool NextHopGroup::removeMembers(const std::set& nh_keys) { SWSS_LOG_ENTER(); @@ -947,10 +947,10 @@ bool NextHopGroup::deleteMembers(const std::set& nh_keys) sai_next_hop_group_api, gSwitchId, gMaxBulkSize); /* - * Iterate through the given group members add them to be deleted. + * Iterate through the given group members add them to be removed. * - * Keep track of the SAI delete statuses in case one of them returns an - * error. We assume that deletion should always succeed. If for some + * Keep track of the SAI remove statuses in case one of them returns an + * error. We assume that removal should always succeed. If for some * reason it doesn't, there's nothing we can do, but we'll log an error * later. */ @@ -981,7 +981,7 @@ bool NextHopGroup::deleteMembers(const std::set& nh_keys) { if (status.second == SAI_STATUS_SUCCESS) { - m_members.at(status.first).delete(); + m_members.at(status.first).remove(); } else { @@ -1073,14 +1073,14 @@ bool NextHopGroup::update(const NextHopGroupKey& nhg_key) } } - /* Delete the removed members. */ - if (!deleteMembers(removed_nh_keys)) + /* Remove the removed members. */ + if (!removeMembers(removed_nh_keys)) { - SWSS_LOG_WARN("Failed to delete members from group %s", to_string().c_str()); + SWSS_LOG_WARN("Failed to remove members from group %s", to_string().c_str()); return false; } - /* Remove the deleted members. */ + /* Remove the removed members. */ for (const auto& nh_key : removed_nh_keys) { m_members.erase(nh_key); @@ -1184,5 +1184,5 @@ bool NextHopGroup::invalidateNextHop(const NextHopKey& nh_key) return true; } - return deleteMembers({nh_key}); + return removeMembers({nh_key}); } diff --git a/orchagent/nhgorch.h b/orchagent/nhgorch.h index 73046ca83e..f0b48c20bc 100644 --- a/orchagent/nhgorch.h +++ b/orchagent/nhgorch.h @@ -35,9 +35,9 @@ class NextHopGroupMember /* Update member's weight and update the SAI attribute as well. */ bool updateWeight(uint32_t weight); - /* Sync / Delete. */ + /* Sync / Remove. */ void sync(sai_object_id_t gm_id); - void delete(); + void remove(); /* Getters / Setters. */ inline const NextHopKey& getNhKey() const { return m_nh_key; } @@ -78,17 +78,17 @@ class NextHopGroup NextHopGroup& operator=(NextHopGroup&& nhg); /* Destructor. */ - virtual ~NextHopGroup() { delete(); } + virtual ~NextHopGroup() { remove(); } /* Sync the group, creating the group's and members SAI IDs. */ bool sync(); - /* Delete the group, reseting the group's and members SAI IDs. */ - bool delete(); + /* Remove the group, reseting the group's and members SAI IDs. */ + bool remove(); /* * Update the group based on a new next hop group key. This will also - * perform any sync / delete necessary. + * perform any sync / remove necessary. */ bool update(const NextHopGroupKey& nhg_key); @@ -101,7 +101,7 @@ class NextHopGroup /* Validate a next hop in the group, syncing it. */ bool validateNextHop(const NextHopKey& nh_key); - /* Invalidate a next hop in the group, deleteing it. */ + /* Invalidate a next hop in the group, removing it. */ bool invalidateNextHop(const NextHopKey& nh_key); /* Increment the number of existing groups. */ @@ -150,7 +150,7 @@ class NextHopGroup bool syncMembers(const std::set& nh_keys); /* Remove group's members the SAI API from the given keys. */ - bool deleteMembers(const std::set& nh_keys); + bool removeMembers(const std::set& nh_keys); /* Create the attributes vector for a next hop group member. */ vector createNhgmAttrs(const NextHopGroupMember& nhgm) const; @@ -160,7 +160,7 @@ class NextHopGroup * Structure describing a next hop group which NhgOrch owns. Beside having a * unique pointer to that next hop group, we also want to keep a ref count so * NhgOrch knows how many other objects reference the next hop group in order - * not to delete them while still being referenced. + * not to remove them while still being referenced. */ struct NhgEntry { diff --git a/orchagent/routeorch.cpp b/orchagent/routeorch.cpp index 98c2cbd9a8..e5fd55320c 100644 --- a/orchagent/routeorch.cpp +++ b/orchagent/routeorch.cpp @@ -1448,7 +1448,7 @@ void RouteOrch::addTempRoute(RouteBulkContext& ctx, const NextHopGroupKey &nextH * a labeled one, which are created by RouteOrch or NhgOrch if the IP * next hop exists. */ - if (!m_neighOrch->hasNextHop(it->ipKey())) + if (!m_neighOrch->isNeighborResolved(*it)) { SWSS_LOG_INFO("Failed to get next hop %s for %s", (*it).to_string().c_str(), ipPrefix.to_string().c_str()); @@ -1555,7 +1555,7 @@ bool RouteOrch::addRoute(RouteBulkContext& ctx, const NextHopGroupKey &nextHops) } /* See if there is an IP neighbor nexthop */ else if (nexthop.isMplsNextHop() && - m_neighOrch->hasNextHop(nexthop.ipKey())) + m_neighOrch->isNeighborResolved(nexthop)) { m_neighOrch->addNextHop(nexthop); next_hop_id = m_neighOrch->getNextHopId(nexthop); From dfb10fcc2911c41336ae635aa6211d300e1857d0 Mon Sep 17 00:00:00 2001 From: Tom cappleman Date: Fri, 10 Sep 2021 11:56:54 +0100 Subject: [PATCH 04/51] Remove ipKey method --- orchagent/nexthopkey.h | 6 ------ 1 file changed, 6 deletions(-) diff --git a/orchagent/nexthopkey.h b/orchagent/nexthopkey.h index f4d2a8fac3..1e76916dd4 100644 --- a/orchagent/nexthopkey.h +++ b/orchagent/nexthopkey.h @@ -165,12 +165,6 @@ struct NextHopKey } return str; } - - // Method to get the underlying IP/interface pair for the next hop. - NextHopKey ipKey() const - { - return NextHopKey(ip_address, alias); - } }; #endif /* SWSS_NEXTHOPKEY_H */ From c9320bf95bf5a5e1868774127927879371c1d506 Mon Sep 17 00:00:00 2001 From: Tom cappleman Date: Fri, 10 Sep 2021 14:04:15 +0100 Subject: [PATCH 05/51] Remove ipKey method --- orchagent/mplsrouteorch.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/orchagent/mplsrouteorch.cpp b/orchagent/mplsrouteorch.cpp index a1209c0e28..8afdc1af02 100644 --- a/orchagent/mplsrouteorch.cpp +++ b/orchagent/mplsrouteorch.cpp @@ -504,7 +504,7 @@ bool RouteOrch::addLabelRoute(LabelRouteBulkContext& ctx, const NextHopGroupKey } /* See if there is an IP neighbor nexthop */ else if (nexthop.isMplsNextHop() && - m_neighOrch->hasNextHop(nexthop.ipKey())) + m_neighOrch->isNeighborResolved(nexthop)) { m_neighOrch->addNextHop(nexthop); next_hop_id = m_neighOrch->getNextHopId(nexthop); From 06b496f1234a32c4bf97c62fb8d47818d9aa51f6 Mon Sep 17 00:00:00 2001 From: Tom cappleman Date: Tue, 14 Sep 2021 09:31:41 +0100 Subject: [PATCH 06/51] Add neighbor resolution to nhgorch --- orchagent/nhgorch.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/orchagent/nhgorch.cpp b/orchagent/nhgorch.cpp index 4bbdc84889..f256fe7635 100644 --- a/orchagent/nhgorch.cpp +++ b/orchagent/nhgorch.cpp @@ -467,6 +467,10 @@ sai_object_id_t NextHopGroupMember::getNhId() const gNeighOrch->addNextHop(m_nh_key); nh_id = gNeighOrch->getNextHopId(m_nh_key); } + else + { + gNeighOrch->resolveNeighbor(m_nh_key); + } return nh_id; } From 4031ad73148358806873a035f0f52765b1c306a7 Mon Sep 17 00:00:00 2001 From: Tom cappleman Date: Thu, 16 Sep 2021 17:03:59 +0100 Subject: [PATCH 07/51] Fix LGTM alerts --- tests/test_nhg.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/test_nhg.py b/tests/test_nhg.py index ef09eb5a82..d13763b507 100644 --- a/tests/test_nhg.py +++ b/tests/test_nhg.py @@ -1089,7 +1089,6 @@ def nhg_exists(nhg_index): # Create a temporary next hop groups nhg_index, binary = create_temp_nhg() - nhg_id = self.get_nhg_id(nhg_index, dvs) # Update the route to point to the temporary group fvs = swsscommon.FieldValuePairs([('nexthop_group', nhg_index)]) @@ -1103,7 +1102,6 @@ def nhg_exists(nhg_index): # Create a temporary next hop groups nhg_index, binary = create_temp_nhg() - nhg_id = self.get_nhg_id(nhg_index, dvs) # Update the route to point to the temporary group fvs = swsscommon.FieldValuePairs([('nexthop_group', nhg_index)]) From 5a1725774abe0decc53e13dece287da72927da0a Mon Sep 17 00:00:00 2001 From: Alexandru Banu Date: Tue, 21 Sep 2021 12:56:04 +0300 Subject: [PATCH 08/51] Remove unnecessary log and add clarifying comment Signed-off-by: Alexandru Banu --- orchagent/nhgorch.cpp | 7 +++++++ orchagent/routeorch.cpp | 1 - 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/orchagent/nhgorch.cpp b/orchagent/nhgorch.cpp index f256fe7635..803a87f2f6 100644 --- a/orchagent/nhgorch.cpp +++ b/orchagent/nhgorch.cpp @@ -1028,6 +1028,13 @@ bool NextHopGroup::update(const NextHopGroupKey& nhg_key) if ((nhg_key.getSize() == 1) || (m_members.size() == 1) || !isSynced()) { bool was_synced = isSynced(); + + /* + * The previous NHG is going to be destroyed as a consequence of the + * move assignment oprator being invoked on a temporary object. The + * destructor will be invoked right after the evaluation of the line + * below. + */ *this = NextHopGroup(nhg_key); /* Sync the group only if it was synced before. */ diff --git a/orchagent/routeorch.cpp b/orchagent/routeorch.cpp index e5fd55320c..e81d456b25 100644 --- a/orchagent/routeorch.cpp +++ b/orchagent/routeorch.cpp @@ -748,7 +748,6 @@ void RouteOrch::doTask(Consumer& consumer) } else { - SWSS_LOG_INFO("Route %s is duplicate entry", key.c_str()); /* Duplicate entry */ it = consumer.m_toSync.erase(it); } From a05c58a05a32c74ad1b017619805f1c0bf5f04c8 Mon Sep 17 00:00:00 2001 From: Alexandru Banu Date: Fri, 24 Sep 2021 20:50:16 +0300 Subject: [PATCH 09/51] Update code after review --- orchagent/nhgorch.cpp | 64 ++---- orchagent/routeorch.cpp | 418 +++++++++++++++++++--------------------- tests/test_nhg.py | 1 - 3 files changed, 222 insertions(+), 261 deletions(-) diff --git a/orchagent/nhgorch.cpp b/orchagent/nhgorch.cpp index 803a87f2f6..0989df6d63 100644 --- a/orchagent/nhgorch.cpp +++ b/orchagent/nhgorch.cpp @@ -32,9 +32,7 @@ NhgOrch::NhgOrch(DBConnector *db, string tableName) : sai_attribute_t attr; attr.id = SAI_SWITCH_ATTR_NUMBER_OF_ECMP_GROUPS; - sai_status_t status = sai_switch_api->get_switch_attribute(gSwitchId, - 1, - &attr); + sai_status_t status = sai_switch_api->get_switch_attribute(gSwitchId, 1, &attr); if (status != SAI_STATUS_SUCCESS) { @@ -64,8 +62,7 @@ NhgOrch::NhgOrch(DBConnector *db, string tableName) : /* Set switch's next hop group capacity. */ vector fvTuple; - fvTuple.emplace_back("MAX_NEXTHOP_GROUP_COUNT", - to_string(m_maxNhgCount)); + fvTuple.emplace_back("MAX_NEXTHOP_GROUP_COUNT", to_string(m_maxNhgCount)); gSwitchOrch->set_switch_capability(fvTuple); SWSS_LOG_NOTICE("Maximum number of ECMP groups supported is %d", m_maxNhgCount); @@ -98,7 +95,7 @@ void NhgOrch::doTask(Consumer& consumer) string index = kfvKey(t); string op = kfvOp(t); - bool success = true; + bool success = false; const auto& nhg_it = m_syncdNextHopGroups.find(index); if (op == SET_COMMAND) @@ -157,36 +154,28 @@ void NhgOrch::doTask(Consumer& consumer) */ if ((nhg_key.getSize() > 1) && (NextHopGroup::getCount() >= m_maxNhgCount)) { - SWSS_LOG_WARN("Next hop group count reached its limit."); + SWSS_LOG_DEBUG("Next hop group count reached its limit."); try { - auto nhg = std::make_unique( - createTempNhg(nhg_key)); + auto nhg = std::make_unique(createTempNhg(nhg_key)); if (nhg->sync()) { - m_syncdNextHopGroups.emplace(index, - NhgEntry(std::move(nhg))); + m_syncdNextHopGroups.emplace(index, NhgEntry(std::move(nhg))); } else { - SWSS_LOG_WARN("Failed to sync temporary NHG %s with %s", + SWSS_LOG_INFO("Failed to sync temporary NHG %s with %s", index.c_str(), nhg_key.to_string().c_str()); } } catch (const std::exception& e) { - SWSS_LOG_WARN("Got exception: %s while adding temp group %s", + SWSS_LOG_INFO("Got exception: %s while adding temp group %s", e.what(), nhg_key.to_string().c_str()); } - - /* - * We set the success to false so we keep trying to update - * this group in order to sync the whole group. - */ - success = false; } else { @@ -195,8 +184,7 @@ void NhgOrch::doTask(Consumer& consumer) if (success) { - m_syncdNextHopGroups.emplace(index, - NhgEntry(std::move(nhg))); + m_syncdNextHopGroups.emplace(index, NhgEntry(std::move(nhg))); } } } @@ -216,10 +204,9 @@ void NhgOrch::doTask(Consumer& consumer) ((nhg_key.getSize() == 1) || (nhg_ptr->getSize() == 1)) && (nhg_it->second.ref_count > 0)) { - SWSS_LOG_WARN("Next hop group %s update would change SAI " + SWSS_LOG_INFO("Next hop group %s update would change SAI " "ID while referenced, so not performed", index.c_str()); - success = false; } /* * If the update would mandate promoting a temporary next hop @@ -235,7 +222,8 @@ void NhgOrch::doTask(Consumer& consumer) * If the group was updated in such way that the previously * chosen next hop does not represent the new group key, * update the temporary group to choose a new next hop from - * the new key. + * the new key. Otherwise, this will be a no-op as we have + * to wait for resources in order to promote the group. */ if (!nhg_key.contains(nhg_ptr->getKey())) { @@ -256,25 +244,18 @@ void NhgOrch::doTask(Consumer& consumer) } else { - SWSS_LOG_WARN("Failed to sync updated temp NHG %s with %s", + SWSS_LOG_INFO("Failed to sync updated temp NHG %s with %s", index.c_str(), nhg_key.to_string().c_str()); } } catch (const std::exception& e) { - SWSS_LOG_WARN("Got exception: %s while adding temp group %s", + SWSS_LOG_INFO("Got exception: %s while adding temp group %s", e.what(), nhg_key.to_string().c_str()); } } - - /* - * Because the resources are exhausted, we have to keep - * trying to update the temporary group until we can - * promote it to a fully functional group. - */ - success = false; } /* Common update, when all the requirements are met. */ else @@ -288,15 +269,14 @@ void NhgOrch::doTask(Consumer& consumer) /* If the group does not exist, do nothing. */ if (nhg_it == m_syncdNextHopGroups.end()) { - SWSS_LOG_WARN("Unable to find group with key %s to remove", index.c_str()); + SWSS_LOG_INFO("Unable to find group with key %s to remove", index.c_str()); /* Mark the operation as successful to consume it. */ success = true; } /* If the group does exist, but it's still referenced, skip. */ else if (nhg_it->second.ref_count > 0) { - SWSS_LOG_WARN("Unable to remove group %s which is referenced", index.c_str()); - success = false; + SWSS_LOG_INFO("Unable to remove group %s which is referenced", index.c_str()); } /* Else, if the group is no more referenced, remove it. */ else @@ -513,8 +493,7 @@ bool NextHopGroupMember::updateWeight(uint32_t weight) nhgm_attr.id = SAI_NEXT_HOP_GROUP_MEMBER_ATTR_WEIGHT; nhgm_attr.value.s32 = m_nh_key.weight; - sai_status_t status = sai_next_hop_group_api-> - set_next_hop_group_member_attribute(m_gm_id, &nhgm_attr); + sai_status_t status = sai_next_hop_group_api->set_next_hop_group_member_attribute(m_gm_id, &nhgm_attr); success = status == SAI_STATUS_SUCCESS; } @@ -859,8 +838,7 @@ bool NextHopGroup::syncMembers(const std::set& nh_keys) /* This method should not be called for groups with only one NH. */ assert(m_members.size() > 1); - ObjectBulker nextHopGroupMemberBulker( - sai_next_hop_group_api, gSwitchId, gMaxBulkSize); + ObjectBulker nextHopGroupMemberBulker(sai_next_hop_group_api, gSwitchId, gMaxBulkSize); /* * Iterate over the given next hops. @@ -966,8 +944,7 @@ bool NextHopGroup::removeMembers(const std::set& nh_keys) if (nhgm.isSynced()) { - nextHopGroupMemberBulker.remove_entry(&statuses[nh_key], - nhgm.getGmId()); + nextHopGroupMemberBulker.remove_entry(&statuses[nh_key], nhgm.getGmId()); } } @@ -1124,8 +1101,7 @@ bool NextHopGroup::update(const NextHopGroupKey& nhg_key) * Params: IN nhgm - The next hop group member. * Returns: The attributes vector for the given next hop. */ -vector NextHopGroup::createNhgmAttrs( - const NextHopGroupMember& nhgm) const +vector NextHopGroup::createNhgmAttrs(const NextHopGroupMember& nhgm) const { SWSS_LOG_ENTER(); diff --git a/orchagent/routeorch.cpp b/orchagent/routeorch.cpp index e81d456b25..ed2bc47625 100644 --- a/orchagent/routeorch.cpp +++ b/orchagent/routeorch.cpp @@ -575,6 +575,7 @@ void RouteOrch::doTask(Consumer& consumer) vector mpls_nhv; vector vni_labelv; vector rmacv; + NextHopGroupKey& nhg = ctx.nhg; /* Check if the next hop group is owned by the NhgOrch. */ if (nhg_index.empty()) @@ -643,7 +644,6 @@ void RouteOrch::doTask(Consumer& consumer) } string nhg_str = ""; - NextHopGroupKey& nhg = ctx.nhg; if (blackhole) { @@ -683,7 +683,7 @@ void RouteOrch::doTask(Consumer& consumer) try { const NextHopGroup& nh_group = gNhgOrch->getNhg(nhg_index); - ctx.nhg = nh_group.getKey(); + nhg = nh_group.getKey(); ctx.using_temp_nhg = nh_group.isTemp(); } catch (const std::out_of_range& e) @@ -694,8 +694,6 @@ void RouteOrch::doTask(Consumer& consumer) } } - NextHopGroupKey& nhg = ctx.nhg; - if (nhg.getSize() == 1 && nhg.hasIntfNextHop()) { if (alsv[0] == "unknown") @@ -1514,155 +1512,154 @@ bool RouteOrch::addRoute(RouteBulkContext& ctx, const NextHopGroupKey &nextHops) return false; } } - else if (ctx.nhg_index.empty()) + /* NhgOrch owns the NHG */ + else if (!ctx.nhg_index.empty()) { - if (nextHops.getSize() == 0) + try + { + const NextHopGroup& nhg = gNhgOrch->getNhg(ctx.nhg_index); + next_hop_id = nhg.getId(); + } + catch(const std::out_of_range& e) { - /* The route is pointing to a blackhole */ - blackhole = true; + SWSS_LOG_INFO("Next hop group key %s does not exist", ctx.nhg_index.c_str()); + return false; } - else if (nextHops.getSize() == 1) + } + /* RouteOrch owns the NHG */ + else if (nextHops.getSize() == 0) + { + /* The route is pointing to a blackhole */ + blackhole = true; + } + else if (nextHops.getSize() == 1) + { + /* The route is pointing to a next hop */ + const NextHopKey& nexthop = *nextHops.getNextHops().begin(); + if (nexthop.isIntfNextHop()) { - /* The route is pointing to a next hop */ - const NextHopKey& nexthop = *nextHops.getNextHops().begin(); - if (nexthop.isIntfNextHop()) + if(gPortsOrch->isInbandPort(nexthop.alias)) { - if(gPortsOrch->isInbandPort(nexthop.alias)) - { - //This routes is the static route added for the remote system neighbors - //We do not need this route in the ASIC since the static neighbor creation - //in ASIC adds the same full mask route (host route) in ASIC automatically - //So skip. - return true; - } + //This routes is the static route added for the remote system neighbors + //We do not need this route in the ASIC since the static neighbor creation + //in ASIC adds the same full mask route (host route) in ASIC automatically + //So skip. + return true; + } - next_hop_id = m_intfsOrch->getRouterIntfsId(nexthop.alias); - /* rif is not created yet */ - if (next_hop_id == SAI_NULL_OBJECT_ID) - { - SWSS_LOG_INFO("Failed to get next hop %s for %s", - nextHops.to_string().c_str(), ipPrefix.to_string().c_str()); - return false; - } + next_hop_id = m_intfsOrch->getRouterIntfsId(nexthop.alias); + /* rif is not created yet */ + if (next_hop_id == SAI_NULL_OBJECT_ID) + { + SWSS_LOG_INFO("Failed to get next hop %s for %s", + nextHops.to_string().c_str(), ipPrefix.to_string().c_str()); + return false; + } + } + /* IP neighbor is not yet resolved */ + else + { + if (m_neighOrch->hasNextHop(nexthop)) + { + next_hop_id = m_neighOrch->getNextHopId(nexthop); + } + /* See if there is an IP neighbor nexthop */ + else if (nexthop.isMplsNextHop() && + m_neighOrch->isNeighborResolved(nexthop)) + { + m_neighOrch->addNextHop(nexthop); + next_hop_id = m_neighOrch->getNextHopId(nexthop); } - /* IP neighbor is not yet resolved */ else { - if (m_neighOrch->hasNextHop(nexthop)) - { - next_hop_id = m_neighOrch->getNextHopId(nexthop); - } - /* See if there is an IP neighbor nexthop */ - else if (nexthop.isMplsNextHop() && - m_neighOrch->isNeighborResolved(nexthop)) - { - m_neighOrch->addNextHop(nexthop); - next_hop_id = m_neighOrch->getNextHopId(nexthop); - } - else + if(overlay_nh) { - if(overlay_nh) + SWSS_LOG_INFO("create remote vtep %s", nexthop.to_string(overlay_nh).c_str()); + status = createRemoteVtep(vrf_id, nexthop); + if (status == false) { - SWSS_LOG_INFO("create remote vtep %s", nexthop.to_string(overlay_nh).c_str()); - status = createRemoteVtep(vrf_id, nexthop); - if (status == false) - { - SWSS_LOG_ERROR("Failed to create remote vtep %s", nexthop.to_string(overlay_nh).c_str()); - return false; - } - next_hop_id = m_neighOrch->addTunnelNextHop(nexthop); - if (next_hop_id == SAI_NULL_OBJECT_ID) - { - SWSS_LOG_ERROR("Failed to create Tunnel Nexthop %s", nexthop.to_string(overlay_nh).c_str()); - return false; - } + SWSS_LOG_ERROR("Failed to create remote vtep %s", nexthop.to_string(overlay_nh).c_str()); + return false; } - else + next_hop_id = m_neighOrch->addTunnelNextHop(nexthop); + if (next_hop_id == SAI_NULL_OBJECT_ID) { - SWSS_LOG_INFO("Failed to get next hop %s for %s, resolving neighbor", - nextHops.to_string().c_str(), ipPrefix.to_string().c_str()); - m_neighOrch->resolveNeighbor(nexthop); + SWSS_LOG_ERROR("Failed to create Tunnel Nexthop %s", nexthop.to_string(overlay_nh).c_str()); return false; } } + else + { + SWSS_LOG_INFO("Failed to get next hop %s for %s, resolving neighbor", + nextHops.to_string().c_str(), ipPrefix.to_string().c_str()); + m_neighOrch->resolveNeighbor(nexthop); + return false; + } } } - /* The route is pointing to a next hop group */ - else + } + /* The route is pointing to a next hop group */ + else + { + /* Check if there is already an existing next hop group */ + if (!hasNextHopGroup(nextHops)) { - /* Check if there is already an existing next hop group */ - if (!hasNextHopGroup(nextHops)) + /* Try to create a new next hop group */ + if (!addNextHopGroup(nextHops)) { - /* Try to create a new next hop group */ - if (!addNextHopGroup(nextHops)) + for(auto it = nextHops.getNextHops().begin(); it != nextHops.getNextHops().end(); ++it) { - for(auto it = nextHops.getNextHops().begin(); it != nextHops.getNextHops().end(); ++it) + const NextHopKey& nextHop = *it; + if(!m_neighOrch->hasNextHop(nextHop)) { - const NextHopKey& nextHop = *it; - if(!m_neighOrch->hasNextHop(nextHop)) + if(overlay_nh) { - if(overlay_nh) + SWSS_LOG_INFO("create remote vtep %s ecmp", nextHop.to_string(overlay_nh).c_str()); + status = createRemoteVtep(vrf_id, nextHop); + if (status == false) { - SWSS_LOG_INFO("create remote vtep %s ecmp", nextHop.to_string(overlay_nh).c_str()); - status = createRemoteVtep(vrf_id, nextHop); - if (status == false) - { - SWSS_LOG_ERROR("Failed to create remote vtep %s ecmp", nextHop.to_string(overlay_nh).c_str()); - return false; - } - next_hop_id = m_neighOrch->addTunnelNextHop(nextHop); - if (next_hop_id == SAI_NULL_OBJECT_ID) - { - SWSS_LOG_ERROR("Failed to create Tunnel Nexthop %s", nextHop.to_string(overlay_nh).c_str()); - return false; - } + SWSS_LOG_ERROR("Failed to create remote vtep %s ecmp", nextHop.to_string(overlay_nh).c_str()); + return false; } - else + next_hop_id = m_neighOrch->addTunnelNextHop(nextHop); + if (next_hop_id == SAI_NULL_OBJECT_ID) { - SWSS_LOG_INFO("Failed to get next hop %s in %s, resolving neighbor", - nextHop.to_string().c_str(), nextHops.to_string().c_str()); - m_neighOrch->resolveNeighbor(nextHop); + SWSS_LOG_ERROR("Failed to create Tunnel Nexthop %s", nextHop.to_string(overlay_nh).c_str()); + return false; } } + else + { + SWSS_LOG_INFO("Failed to get next hop %s in %s, resolving neighbor", + nextHop.to_string().c_str(), nextHops.to_string().c_str()); + m_neighOrch->resolveNeighbor(nextHop); + } } + } - /* Failed to create the next hop group and check if a temporary route is needed */ + /* Failed to create the next hop group and check if a temporary route is needed */ - /* If the current next hop is part of the next hop group to sync, - * then return false and no need to add another temporary route. */ - if (it_route != m_syncdRoutes.at(vrf_id).end() && it_route->second.nhg_key.getSize() == 1) + /* If the current next hop is part of the next hop group to sync, + * then return false and no need to add another temporary route. */ + if (it_route != m_syncdRoutes.at(vrf_id).end() && it_route->second.nhg_key.getSize() == 1) + { + const NextHopKey& nexthop = *it_route->second.nhg_key.getNextHops().begin(); + if (nextHops.contains(nexthop)) { - const NextHopKey& nexthop = *it_route->second.nhg_key.getNextHops().begin(); - if (nextHops.contains(nexthop)) - { - return false; - } + return false; } - - /* Add a temporary route when a next hop group cannot be added, - * and there is no temporary route right now or the current temporary - * route is not pointing to a member of the next hop group to sync. */ - addTempRoute(ctx, nextHops); - /* Return false since the original route is not successfully added */ - return false; } - } - next_hop_id = m_syncdNextHopGroups[nextHops].next_hop_group_id; - } - } - else - { - try - { - const NextHopGroup& nhg = gNhgOrch->getNhg(ctx.nhg_index); - next_hop_id = nhg.getId(); - } - catch(const std::out_of_range& e) - { - SWSS_LOG_WARN("Next hop group key %s does not exist", ctx.nhg_index.c_str()); - return false; + /* Add a temporary route when a next hop group cannot be added, + * and there is no temporary route right now or the current temporary + * route is not pointing to a member of the next hop group to sync. */ + addTempRoute(ctx, nextHops); + /* Return false since the original route is not successfully added */ + return false; + } } + + next_hop_id = m_syncdNextHopGroups[nextHops].next_hop_group_id; } /* Sync the route entry */ @@ -1740,7 +1737,6 @@ bool RouteOrch::addRoute(RouteBulkContext& ctx, const NextHopGroupKey &nextHops) gRouteBulker.set_entry_attribute(&object_statuses.back(), &route_entry, &route_attr); } } - return false; } @@ -1766,58 +1762,56 @@ bool RouteOrch::addRoutePost(const RouteBulkContext& ctx, const NextHopGroupKey /* Route is pointing to Fine Grained ECMP nexthop group */ isFineGrained = true; } - /* Check that the next hop group is not owned by NhgOrch. */ - else if (ctx.nhg_index.empty()) + /* NhgOrch owns the NHG. */ + else if (!ctx.nhg_index.empty()) { - if (nextHops.getSize() == 0) + if (!gNhgOrch->hasNhg(ctx.nhg_index)) { - /* The route is pointing to a blackhole */ - blackhole = true; + SWSS_LOG_INFO("Failed to get next hop group with index %s", ctx.nhg_index.c_str()); + return false; } - else if (nextHops.getSize() == 1) + } + /* RouteOrch owns the NHG */ + else if (nextHops.getSize() == 0) + { + /* The route is pointing to a blackhole */ + blackhole = true; + } + else if (nextHops.getSize() == 1) + { + /* The route is pointing to a next hop */ + const NextHopKey& nexthop = *nextHops.getNextHops().begin(); + if (nexthop.isIntfNextHop()) { - /* The route is pointing to a next hop */ - const NextHopKey& nexthop = *nextHops.getNextHops().begin(); - if (nexthop.isIntfNextHop()) - { - auto next_hop_id = m_intfsOrch->getRouterIntfsId(nexthop.alias); - /* rif is not created yet */ - if (next_hop_id == SAI_NULL_OBJECT_ID) - { - SWSS_LOG_WARN("Failed to get next hop %s for %s", - nextHops.to_string().c_str(), ipPrefix.to_string().c_str()); - return false; - } - } - else + auto next_hop_id = m_intfsOrch->getRouterIntfsId(nexthop.alias); + /* rif is not created yet */ + if (next_hop_id == SAI_NULL_OBJECT_ID) { - if (!m_neighOrch->hasNextHop(nexthop)) - { - SWSS_LOG_WARN("Failed to get next hop %s for %s", - nextHops.to_string().c_str(), ipPrefix.to_string().c_str()); - return false; - } + SWSS_LOG_INFO("Failed to get next hop %s for %s", + nextHops.to_string().c_str(), ipPrefix.to_string().c_str()); + return false; } } - /* The route is pointing to a next hop group */ else { - if (!hasNextHopGroup(nextHops)) + if (!m_neighOrch->hasNextHop(nexthop)) { - SWSS_LOG_WARN("Next hop group is temporary, represented by %s", - ctx.tmp_next_hop.to_string().c_str()); - // Previous added an temporary route - auto& tmp_next_hop = ctx.tmp_next_hop; - addRoutePost(ctx, tmp_next_hop); + SWSS_LOG_INFO("Failed to get next hop %s for %s", + nextHops.to_string().c_str(), ipPrefix.to_string().c_str()); return false; } } } + /* The route is pointing to a next hop group */ else { - if (!gNhgOrch->hasNhg(ctx.nhg_index)) + if (!hasNextHopGroup(nextHops)) { - SWSS_LOG_WARN("Failed to get next hop group with index %s", ctx.nhg_index.c_str()); + SWSS_LOG_DEBUG("Next hop group is temporary, represented by %s", + ctx.tmp_next_hop.to_string().c_str()); + // Previous added an temporary route + auto& tmp_next_hop = ctx.tmp_next_hop; + addRoutePost(ctx, tmp_next_hop); return false; } } @@ -1908,7 +1902,7 @@ bool RouteOrch::addRoutePost(const RouteBulkContext& ctx, const NextHopGroupKey gNhgOrch->incNhgRefCount(ctx.nhg_index); } - SWSS_LOG_NOTICE("Post create route %s with next hop(s) %s", + SWSS_LOG_INFO("Post create route %s with next hop(s) %s", ipPrefix.to_string().c_str(), nextHops.to_string().c_str()); } else @@ -1948,36 +1942,33 @@ bool RouteOrch::addRoutePost(const RouteBulkContext& ctx, const NextHopGroupKey /* Remove FG nhg since prefix now points to standard nhg/nhs */ m_fgNhgOrch->removeFgNhg(vrf_id, ipPrefix); } - else + /* Decrease the ref count for the previous next hop group. */ + else if (it_route->second.nhg_index.empty()) { - /* Decrease the ref count for the previous next hop group. */ - if (it_route->second.nhg_index.empty()) + decreaseNextHopRefCount(it_route->second.nhg_key); + auto ol_nextHops = it_route->second.nhg_key; + if (ol_nextHops.getSize() > 1 + && m_syncdNextHopGroups[ol_nextHops].ref_count == 0) { - decreaseNextHopRefCount(it_route->second.nhg_key); - auto ol_nextHops = it_route->second.nhg_key; - if (ol_nextHops.getSize() > 1 - && m_syncdNextHopGroups[ol_nextHops].ref_count == 0) - { - m_bulkNhgReducedRefCnt.emplace(ol_nextHops, 0); - } - else if (ol_nextHops.is_overlay_nexthop()) - { - SWSS_LOG_NOTICE("Update overlay Nexthop %s", ol_nextHops.to_string().c_str()); - m_bulkNhgReducedRefCnt.emplace(ol_nextHops, vrf_id); - } - else if (ol_nextHops.getSize() == 1) - { - RouteKey r_key = { vrf_id, ipPrefix }; - auto nexthop = NextHopKey(ol_nextHops.to_string()); - removeNextHopRoute(nexthop, r_key); - } + m_bulkNhgReducedRefCnt.emplace(ol_nextHops, 0); } - else + else if (ol_nextHops.is_overlay_nexthop()) + { + SWSS_LOG_NOTICE("Update overlay Nexthop %s", ol_nextHops.to_string().c_str()); + m_bulkNhgReducedRefCnt.emplace(ol_nextHops, vrf_id); + } + else if (ol_nextHops.getSize() == 1) { - /* The next hop group is owned by NeighOrch. */ - gNhgOrch->decNhgRefCount(it_route->second.nhg_index); + RouteKey r_key = { vrf_id, ipPrefix }; + auto nexthop = NextHopKey(ol_nextHops.to_string()); + removeNextHopRoute(nexthop, r_key); } } + else + { + /* The next hop group is owned by NhgOrch. */ + gNhgOrch->decNhgRefCount(it_route->second.nhg_index); + } if (blackhole) { @@ -2005,7 +1996,7 @@ bool RouteOrch::addRoutePost(const RouteBulkContext& ctx, const NextHopGroupKey gNhgOrch->incNhgRefCount(ctx.nhg_index); } - SWSS_LOG_NOTICE("Post set route %s with next hop(s) %s", + SWSS_LOG_INFO("Post set route %s with next hop(s) %s", ipPrefix.to_string().c_str(), nextHops.to_string().c_str()); } @@ -2164,49 +2155,44 @@ bool RouteOrch::removeRoutePost(const RouteBulkContext& ctx) /* Delete Fine Grained nhg if the revmoved route pointed to it */ m_fgNhgOrch->removeFgNhg(vrf_id, ipPrefix); } - else + /* Check that the next hop group is not owned by NhgOrch. */ + else if (it_route->second.nhg_index.empty()) { - /* Check that the next hop group is not owned by NhgOrch. */ - if (it_route->second.nhg_index.empty()) - { - auto ol_nextHops = it_route->second.nhg_key; + auto ol_nextHops = it_route->second.nhg_key; - /* - * Decrease the reference count only when the route is pointing to a next hop. - */ - decreaseNextHopRefCount(ol_nextHops); - - if (ol_nextHops.getSize() > 1 - && m_syncdNextHopGroups[ol_nextHops].ref_count == 0) - { - m_bulkNhgReducedRefCnt.emplace(ol_nextHops, 0); - } - else if (ol_nextHops.is_overlay_nexthop()) - { - SWSS_LOG_NOTICE("Remove overlay Nexthop %s", ol_nextHops.to_string().c_str()); - m_bulkNhgReducedRefCnt.emplace(ol_nextHops, vrf_id); - } - /* - * Additionally check if the NH has label and its ref count == 0, then - * remove the label next hop. - */ - else if (ol_nextHops.getSize() == 1) - { - const NextHopKey& nexthop = *it_route->second.nhg_key.getNextHops().begin(); - if (nexthop.isMplsNextHop() && - (m_neighOrch->getNextHopRefCount(nexthop) == 0)) - { - m_neighOrch->removeMplsNextHop(nexthop); - } + /* + * Decrease the reference count only when the route is pointing to a next hop. + */ + decreaseNextHopRefCount(ol_nextHops); - RouteKey r_key = { vrf_id, ipPrefix }; - removeNextHopRoute(nexthop, r_key); - } + if (ol_nextHops.getSize() > 1 + && m_syncdNextHopGroups[ol_nextHops].ref_count == 0) + { + m_bulkNhgReducedRefCnt.emplace(ol_nextHops, 0); } - else + else if (ol_nextHops.is_overlay_nexthop()) { - gNhgOrch->decNhgRefCount(it_route->second.nhg_index); + SWSS_LOG_NOTICE("Remove overlay Nexthop %s", ol_nextHops.to_string().c_str()); + m_bulkNhgReducedRefCnt.emplace(ol_nextHops, vrf_id); } + /* + * Additionally check if the NH has label and its ref count == 0, then + * remove the label next hop. + */ + else if (ol_nextHops.getSize() == 1) + { + const NextHopKey& nexthop = *it_route->second.nhg_key.getNextHops().begin(); + if (nexthop.isMplsNextHop() && + (m_neighOrch->getNextHopRefCount(nexthop) == 0)) + { + m_neighOrch->removeMplsNextHop(nexthop); + } + } + } + /* The NHG is owned by NhgOrch */ + else + { + gNhgOrch->decNhgRefCount(it_route->second.nhg_index); } SWSS_LOG_INFO("Remove route %s with next hop(s) %s", diff --git a/tests/test_nhg.py b/tests/test_nhg.py index d13763b507..13f0009483 100644 --- a/tests/test_nhg.py +++ b/tests/test_nhg.py @@ -25,7 +25,6 @@ def get_route_id(self, prefix, dvs): def get_inseg_id(self, label, dvs): for k in dvs.get_asic_db().get_keys(self.ASIC_INSEG_STR): - print(json.loads(k)) if json.loads(k)['label'] == label: return k From ccb7c6ca91751e3fc90b29cf20db3ac5f0212900 Mon Sep 17 00:00:00 2001 From: Alexandru Banu Date: Thu, 30 Sep 2021 08:15:38 +0300 Subject: [PATCH 10/51] Minor UT updates --- tests/test_nhg.py | 72 ++++++++++++++++++++++++++++++++++++----------- 1 file changed, 55 insertions(+), 17 deletions(-) diff --git a/tests/test_nhg.py b/tests/test_nhg.py index 13f0009483..771eb637dc 100644 --- a/tests/test_nhg.py +++ b/tests/test_nhg.py @@ -46,21 +46,19 @@ def get_nhg_id(self, nhg_index, dvs): try: asic_db.wait_for_n_keys(self.ASIC_RT_STR, asic_rts_count + 1) except Exception as e: + ps._del(prefix) return None + else: + # Get the route ID for the created route + rt_id = self.get_route_id(prefix, dvs) + assert rt_id != None - # Get the route ID for the created route - rt_id = self.get_route_id(prefix, dvs) - assert rt_id != None - - # Get the NHGID - nhgid = asic_db.get_entry(self.ASIC_RT_STR, rt_id)["SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID"] - - # Remove the added route - ps._del(prefix) - asic_db.wait_for_deleted_entry(self.ASIC_RT_STR, rt_id) + # Get the NHGID + nhgid = asic_db.get_entry(self.ASIC_RT_STR, rt_id)["SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID"] + ps._del(prefix) + asic_db.wait_for_deleted_entry(self.ASIC_RT_STR, rt_id) - # Return the NHGID - return nhgid + return nhgid def get_nhgm_ids(self, nhg_index, dvs): nhgid = self.get_nhg_id(nhg_index, dvs) @@ -464,7 +462,7 @@ def get_rt_keys(): # Count existing objects prev_nhg_keys = get_nhg_keys() - asic_nhgs_count = len(get_nhg_keys()) + asic_nhgs_count = len(prev_nhg_keys) asic_nhgms_count = len(get_nhgm_keys()) asic_nhs_count = len(asic_db.get_keys(self.ASIC_NHS_STR)) @@ -534,6 +532,7 @@ def get_rt_keys(): asic_db.wait_for_n_keys(self.ASIC_NHG_STR, asic_nhgs_count + 1) asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, asic_nhgms_count + 2) + # The route's group should have changed to the new one assert asic_db.get_entry(self.ASIC_RT_STR, rt_key)['SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID'] != nhgid # Update the route with routeOrch's owned next hop group @@ -596,14 +595,17 @@ def get_rt_keys(): rt_ps.set('2.2.2.0/24', fvs) asic_db.wait_for_n_keys(self.ASIC_RT_STR, asic_rts_count + 1) + # A NHG should be created + asic_db.wait_for_n_keys(self.ASIC_NHG_STR, asic_nhgs_count + 1) + # Two new next hops should be created asic_db.wait_for_n_keys(self.ASIC_NHS_STR, asic_nhs_count + 2) # Create a NHG with the same details nhg_ps.set('group1', fvs) + asic_db.wait_for_n_keys(self.ASIC_NHG_STR, asic_nhgs_count + 2) # No new next hops should be created - time.sleep(1) assert len(asic_db.get_keys(self.ASIC_NHS_STR)) == asic_nhs_count + 2 # Update the group with a different NH @@ -616,8 +618,13 @@ def get_rt_keys(): # A new next hop should be created asic_db.wait_for_n_keys(self.ASIC_NHS_STR, asic_nhs_count + 3) + # group1 should be updated and a new NHG shouldn't be created + time.sleep(1) + assert len(get_nhg_keys()) == asic_nhgs_count + 2 + # Remove the route rt_ps._del('2.2.2.0/24') + asic_db.wait_for_n_keys(self.ASIC_NHG_STR, asic_nhgs_count + 1) # One NH should become unreferenced and should be deleted. The other # one is still referenced by NhgOrch's owned NHG. @@ -625,6 +632,7 @@ def get_rt_keys(): # Remove the group nhg_ps._del('group1') + asic_db.wait_for_n_keys(self.ASIC_NHG_STR, asic_nhgs_count) # Both new next hops should be deleted asic_db.wait_for_n_keys(self.ASIC_NHS_STR, asic_nhs_count) @@ -702,8 +710,6 @@ def asic_route_nhg_fvs(k): asic_db = dvs.get_asic_db() ps = swsscommon.ProducerStateTable(app_db.db_connection, "ROUTE_TABLE") - dvs.runcmd('swssloglevel -c orchagent -l INFO') - # Add first batch of routes with unique nexthop groups in AppDB route_count = 0 r = 0 @@ -922,6 +928,10 @@ def nhg_exists(nhg_index): prev_nhgs = asic_db.get_keys(self.ASIC_NHG_STR) nhg_index, _ = create_temp_nhg() + # Save the temporary NHG's SAI ID + time.sleep(1) + nhg_id = self.get_nhg_id(nhg_index, dvs) + # Assert no new group has been added asic_db.wait_for_n_keys(self.ASIC_NHG_STR, MAX_ECMP_COUNT) @@ -938,6 +948,9 @@ def nhg_exists(nhg_index): # NHG asic_db.wait_for_n_keys(self.ASIC_NHG_STR, MAX_ECMP_COUNT) + # Assert the SAI ID of the previously temporary NHG has been updated + assert nhg_id != self.get_nhg_id(nhg_index, dvs) + # Save the promoted NHG index/ID asic_nhgs[nhg_index] = self.get_nhg_id(nhg_index, dvs) @@ -1505,6 +1518,7 @@ def test_nhgorch_single_nh_group(self, dvs, testlog): # pointing to it. The route should not be created. fvs = swsscommon.FieldValuePairs([("nexthop_group", "group2")]) rt_ps.set('2.2.4.0/24', fvs) + time.sleep(1) asic_db.wait_for_n_keys(self.ASIC_RT_STR, asic_routes_count + 1) # Configure the NH's interface @@ -1519,18 +1533,22 @@ def test_nhgorch_single_nh_group(self, dvs, testlog): # Delete the route and the group rt_ps._del('2.2.4.0/24') nhg_ps._del('group2') + asic_db.wait_for_n_keys(self.ASIC_RT_STR, asic_routes_count + 1) # Update the group to a multiple NH group - should fail as the group # is referenced + nhgid = self.get_nhg_id('group1', dvs) fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1,10.0.0.3'), ("ifname", "Ethernet0,Ethernet4")]) nhg_ps.set("group1", fvs) time.sleep(1) assert len(asic_db.get_keys(self.ASIC_NHG_STR)) == asic_nhgs_count + # Assert the NHG ID hasn't changed + assert nhgid == self.get_nhg_id('group1', dvs) + # Update group1 to point to another NH - should fail as the group is # referenced - nhgid = self.get_nhg_id('group1', dvs) fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.3'), ("ifname", "Ethernet4")]) nhg_ps.set("group1", fvs) assert nhgid == self.get_nhg_id('group1', dvs) @@ -1551,6 +1569,26 @@ def test_nhgorch_single_nh_group(self, dvs, testlog): nhg_ps.set("group1", fvs) asic_db.wait_for_n_keys(self.ASIC_NHG_STR, asic_nhgs_count + 1) + # Create a route referencing group1 + fvs = swsscommon.FieldValuePairs([("nexthop_group", "group1")]) + rt_ps.set('2.2.2.0/24', fvs) + asic_db.wait_for_n_keys(self.ASIC_RT_STR, asic_routes_count + 1) + + # Update the NHG to a single next hop - should fail as it is being + # referenced + nhgid = self.get_nhg_id('group1', dvs) + fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1'), ("ifname", "Ethernet0")]) + nhg_ps.set("group1", fvs) + time.sleep(1) + assert nhgid == self.get_nhg_id('group1', dvs) + + # Remove route 2.2.2.0/24 + rt_ps._del("2.2.2.0/24") + asic_db.wait_for_n_keys(self.ASIC_RT_STR, asic_routes_count) + + # The group is not referenced anymore, so it should be updated + assert nhgid != self.get_nhg_id('group1', dvs) + # Remove group1 nhg_ps._del("group1") From af2ec2765dc0e01e1960eb4a5784a8c769f5344c Mon Sep 17 00:00:00 2001 From: Tom cappleman Date: Thu, 30 Sep 2021 12:10:21 +0100 Subject: [PATCH 11/51] Move NHG count back to routeorch --- orchagent/nhgorch.cpp | 46 ++------------------------ orchagent/orchdaemon.cpp | 2 +- orchagent/routeorch.cpp | 57 ++++++++++++++++++++++++++------- orchagent/routeorch.h | 8 ++++- tests/mock_tests/aclorch_ut.cpp | 4 +-- 5 files changed, 59 insertions(+), 58 deletions(-) diff --git a/orchagent/nhgorch.cpp b/orchagent/nhgorch.cpp index 0989df6d63..9823857cc9 100644 --- a/orchagent/nhgorch.cpp +++ b/orchagent/nhgorch.cpp @@ -1,5 +1,4 @@ #include "nhgorch.h" -#include "switchorch.h" #include "neighorch.h" #include "crmorch.h" #include "routeorch.h" @@ -12,7 +11,6 @@ extern sai_object_id_t gSwitchId; extern PortsOrch *gPortsOrch; extern CrmOrch *gCrmOrch; extern NeighOrch *gNeighOrch; -extern SwitchOrch *gSwitchOrch; extern RouteOrch *gRouteOrch; extern size_t gMaxBulkSize; @@ -27,45 +25,6 @@ NhgOrch::NhgOrch(DBConnector *db, string tableName) : Orch(db, tableName) { SWSS_LOG_ENTER(); - - /* Get the switch's maximum next hop group capacity. */ - sai_attribute_t attr; - attr.id = SAI_SWITCH_ATTR_NUMBER_OF_ECMP_GROUPS; - - sai_status_t status = sai_switch_api->get_switch_attribute(gSwitchId, 1, &attr); - - if (status != SAI_STATUS_SUCCESS) - { - SWSS_LOG_WARN("Failed to get switch attribute number of ECMP groups. \ - Use default value. rv:%d", status); - m_maxNhgCount = DEFAULT_NUMBER_OF_ECMP_GROUPS; - } - else - { - m_maxNhgCount = attr.value.s32; - - /* - * ASIC specific workaround to re-calculate maximum ECMP groups - * according to diferent ECMP mode used. - * - * On Mellanox platform, the maximum ECMP groups returned is the value - * under the condition that the ECMP group size is 1. Deviding this - * number by DEFAULT_MAX_ECMP_GROUP_SIZE gets the maximum number of - * ECMP groups when the maximum ECMP group size is 32. - */ - char *platform = getenv("platform"); - if (platform && strstr(platform, MLNX_PLATFORM_SUBSTRING)) - { - m_maxNhgCount /= DEFAULT_MAX_ECMP_GROUP_SIZE; - } - } - - /* Set switch's next hop group capacity. */ - vector fvTuple; - fvTuple.emplace_back("MAX_NEXTHOP_GROUP_COUNT", to_string(m_maxNhgCount)); - gSwitchOrch->set_switch_capability(fvTuple); - - SWSS_LOG_NOTICE("Maximum number of ECMP groups supported is %d", m_maxNhgCount); } /* @@ -152,7 +111,8 @@ void NhgOrch::doTask(Consumer& consumer) * to be kept in the sync list so we keep trying to create the * actual group when there are enough resources. */ - if ((nhg_key.getSize() > 1) && (NextHopGroup::getCount() >= m_maxNhgCount)) + if ((nhg_key.getSize() > 1) && + (gRouteOrch->getNhgCount() + NextHopGroup::getCount() >= gRouteOrch->getMaxNhgCount())) { SWSS_LOG_DEBUG("Next hop group count reached its limit."); @@ -216,7 +176,7 @@ void NhgOrch::doTask(Consumer& consumer) */ else if (nhg_ptr->isTemp() && (nhg_key.getSize() > 1) && - (NextHopGroup::getCount() >= m_maxNhgCount)) + (gRouteOrch->getNhgCount() + NextHopGroup::getCount() >= gRouteOrch->getMaxNhgCount())) { /* * If the group was updated in such way that the previously diff --git a/orchagent/orchdaemon.cpp b/orchagent/orchdaemon.cpp index c2c995c475..9048613c99 100644 --- a/orchagent/orchdaemon.cpp +++ b/orchagent/orchdaemon.cpp @@ -164,7 +164,7 @@ bool OrchDaemon::init() { APP_ROUTE_TABLE_NAME, routeorch_pri }, { APP_LABEL_ROUTE_TABLE_NAME, routeorch_pri } }; - gRouteOrch = new RouteOrch(m_applDb, route_tables, gNeighOrch, gIntfsOrch, vrf_orch, gFgNhgOrch); + gRouteOrch = new RouteOrch(m_applDb, route_tables, gSwitchOrch, gNeighOrch, gIntfsOrch, vrf_orch, gFgNhgOrch); gNhgOrch = new NhgOrch(m_applDb, APP_NEXTHOP_GROUP_TABLE_NAME); CoppOrch *copp_orch = new CoppOrch(m_applDb, APP_COPP_TABLE_NAME); diff --git a/orchagent/routeorch.cpp b/orchagent/routeorch.cpp index ed2bc47625..00c66872f6 100644 --- a/orchagent/routeorch.cpp +++ b/orchagent/routeorch.cpp @@ -23,19 +23,56 @@ extern NhgOrch *gNhgOrch; extern size_t gMaxBulkSize; -RouteOrch::RouteOrch(DBConnector *db, vector &tableNames, NeighOrch *neighOrch, IntfsOrch *intfsOrch, VRFOrch *vrfOrch, FgNhgOrch *fgNhgOrch) : +RouteOrch::RouteOrch(DBConnector *db, vector &tableNames, SwitchOrch *switchOrch, NeighOrch *neighOrch, IntfsOrch *intfsOrch, VRFOrch *vrfOrch, FgNhgOrch *fgNhgOrch) : gRouteBulker(sai_route_api, gMaxBulkSize), gLabelRouteBulker(sai_mpls_api, gMaxBulkSize), gNextHopGroupMemberBulker(sai_next_hop_group_api, gSwitchId, gMaxBulkSize), Orch(db, tableNames), + m_switchOrch(switchOrch), m_neighOrch(neighOrch), m_intfsOrch(intfsOrch), m_vrfOrch(vrfOrch), m_fgNhgOrch(fgNhgOrch), + m_nextHopGroupCount(0), m_resync(false) { SWSS_LOG_ENTER(); + sai_attribute_t attr; + attr.id = SAI_SWITCH_ATTR_NUMBER_OF_ECMP_GROUPS; + + sai_status_t status = sai_switch_api->get_switch_attribute(gSwitchId, 1, &attr); + if (status != SAI_STATUS_SUCCESS) + { + SWSS_LOG_WARN("Failed to get switch attribute number of ECMP groups. \ + Use default value. rv:%d", status); + m_maxNextHopGroupCount = DEFAULT_NUMBER_OF_ECMP_GROUPS; + } + else + { + m_maxNextHopGroupCount = attr.value.s32; + + /* + * ASIC specific workaround to re-calculate maximum ECMP groups + * according to different ECMP mode used. + * + * On Mellanox platform, the maximum ECMP groups returned is the value + * under the condition that the ECMP group size is 1. Dividing this + * number by DEFAULT_MAX_ECMP_GROUP_SIZE gets the maximum number of + * ECMP groups when the maximum ECMP group size is 32. + */ + char *platform = getenv("platform"); + if (platform && strstr(platform, MLNX_PLATFORM_SUBSTRING)) + { + m_maxNextHopGroupCount /= DEFAULT_MAX_ECMP_GROUP_SIZE; + } + } + vector fvTuple; + fvTuple.emplace_back("MAX_NEXTHOP_GROUP_COUNT", to_string(m_maxNextHopGroupCount)); + m_switchOrch->set_switch_capability(fvTuple); + + SWSS_LOG_NOTICE("Maximum number of ECMP groups supported is %d", m_maxNextHopGroupCount); + IpPrefix default_ip_prefix("0.0.0.0/0"); sai_route_entry_t unicast_route_entry; @@ -44,12 +81,10 @@ RouteOrch::RouteOrch(DBConnector *db, vector &tableNames, copy(unicast_route_entry.destination, default_ip_prefix); subnet(unicast_route_entry.destination, unicast_route_entry.destination); - sai_attribute_t attr; attr.id = SAI_ROUTE_ENTRY_ATTR_PACKET_ACTION; attr.value.s32 = SAI_PACKET_ACTION_DROP; - sai_status_t status = sai_route_api->create_route_entry( - &unicast_route_entry, 1, &attr); + status = sai_route_api->create_route_entry(&unicast_route_entry, 1, &attr); if (status != SAI_STATUS_SUCCESS) { SWSS_LOG_ERROR("Failed to create IPv4 default route with packet action drop"); @@ -752,7 +787,7 @@ void RouteOrch::doTask(Consumer& consumer) // If already exhaust the nexthop groups, and there are pending removing routes in bulker, // flush the bulker and possibly collect some released nexthop groups - if (gNhgOrch->getNhgCount() >= gNhgOrch->getMaxNhgCount() && + if (m_nextHopGroupCount + gNhgOrch->getNhgCount() >= m_maxNextHopGroupCount && gRouteBulker.removing_entries_count() > 0) { break; @@ -1014,7 +1049,7 @@ bool RouteOrch::createFineGrainedNextHopGroup(sai_object_id_t &next_hop_group_id { SWSS_LOG_ENTER(); - if (gNhgOrch->getNhgCount() >= gNhgOrch->getMaxNhgCount()) + if (m_nextHopGroupCount + gNhgOrch->getNhgCount() >= m_maxNextHopGroupCount) { SWSS_LOG_DEBUG("Failed to create new next hop group. \ Reaching maximum number of next hop groups."); @@ -1036,7 +1071,7 @@ bool RouteOrch::createFineGrainedNextHopGroup(sai_object_id_t &next_hop_group_id } gCrmOrch->incCrmResUsedCounter(CrmResourceType::CRM_NEXTHOP_GROUP); - gNhgOrch->incNhgCount(); + m_nextHopGroupCount++; return true; } @@ -1058,7 +1093,7 @@ bool RouteOrch::removeFineGrainedNextHopGroup(sai_object_id_t &next_hop_group_id } gCrmOrch->decCrmResUsedCounter(CrmResourceType::CRM_NEXTHOP_GROUP); - gNhgOrch->decNhgCount(); + m_nextHopGroupCount--; return true; } @@ -1069,7 +1104,7 @@ bool RouteOrch::addNextHopGroup(const NextHopGroupKey &nexthops) assert(!hasNextHopGroup(nexthops)); - if (gNhgOrch->getNhgCount() >= gNhgOrch->getMaxNhgCount()) + if (m_nextHopGroupCount + gNhgOrch->getNhgCount() >= m_maxNextHopGroupCount) { SWSS_LOG_DEBUG("Failed to create new next hop group. \ Reaching maximum number of next hop groups."); @@ -1144,7 +1179,7 @@ bool RouteOrch::addNextHopGroup(const NextHopGroupKey &nexthops) } } - gNhgOrch->incNhgCount(); + m_nextHopGroupCount++; SWSS_LOG_NOTICE("Create next hop group %s", nexthops.to_string().c_str()); gCrmOrch->incCrmResUsedCounter(CrmResourceType::CRM_NEXTHOP_GROUP); @@ -1298,7 +1333,7 @@ bool RouteOrch::removeNextHopGroup(const NextHopGroupKey &nexthops) } } - gNhgOrch->decNhgCount(); + m_nextHopGroupCount--; gCrmOrch->decCrmResUsedCounter(CrmResourceType::CRM_NEXTHOP_GROUP); set next_hop_set = nexthops.getNextHops(); diff --git a/orchagent/routeorch.h b/orchagent/routeorch.h index 453138b98a..a619eab428 100644 --- a/orchagent/routeorch.h +++ b/orchagent/routeorch.h @@ -168,7 +168,7 @@ struct LabelRouteBulkContext class RouteOrch : public Orch, public Subject { public: - RouteOrch(DBConnector *db, vector &tableNames, NeighOrch *neighOrch, IntfsOrch *intfsOrch, VRFOrch *vrfOrch, FgNhgOrch *fgNhgOrch); + RouteOrch(DBConnector *db, vector &tableNames, SwitchOrch *switchOrc, NeighOrch *neighOrch, IntfsOrch *intfsOrch, VRFOrch *vrfOrch, FgNhgOrch *fgNhgOrch); bool hasNextHopGroup(const NextHopGroupKey&) const; sai_object_id_t getNextHopGroupId(const NextHopGroupKey&); @@ -203,12 +203,18 @@ class RouteOrch : public Orch, public Subject void delLinkLocalRouteToMe(sai_object_id_t vrf_id, IpPrefix linklocal_prefix); std::string getLinkLocalEui64Addr(void); + unsigned int getNhgCount() { return m_nextHopGroupCount; } + unsigned int getMaxNhgCount() { return m_maxNextHopGroupCount; } + private: + SwitchOrch *m_switchOrch; NeighOrch *m_neighOrch; IntfsOrch *m_intfsOrch; VRFOrch *m_vrfOrch; FgNhgOrch *m_fgNhgOrch; + unsigned int m_nextHopGroupCount; + unsigned int m_maxNextHopGroupCount; bool m_resync; RouteTables m_syncdRoutes; diff --git a/tests/mock_tests/aclorch_ut.cpp b/tests/mock_tests/aclorch_ut.cpp index 645c9b63f0..8d7b3b6623 100644 --- a/tests/mock_tests/aclorch_ut.cpp +++ b/tests/mock_tests/aclorch_ut.cpp @@ -337,7 +337,7 @@ namespace aclorch_test { APP_VXLAN_FDB_TABLE_NAME, FdbOrch::fdborch_pri}, { APP_MCLAG_FDB_TABLE_NAME, fdborch_pri} }; - + TableConnector stateDbFdb(m_state_db.get(), STATE_FDB_TABLE_NAME); TableConnector stateMclagDbFdb(m_state_db.get(), STATE_MCLAG_REMOTE_FDB_TABLE_NAME); ASSERT_EQ(gFdbOrch, nullptr); @@ -362,7 +362,7 @@ namespace aclorch_test { APP_ROUTE_TABLE_NAME, routeorch_pri }, { APP_LABEL_ROUTE_TABLE_NAME, routeorch_pri } }; - gRouteOrch = new RouteOrch(m_app_db.get(), route_tables, gNeighOrch, gIntfsOrch, gVrfOrch, gFgNhgOrch); + gRouteOrch = new RouteOrch(m_app_db.get(), route_tables, gSwitchOrch, gNeighOrch, gIntfsOrch, gVrfOrch, gFgNhgOrch); PolicerOrch *policer_orch = new PolicerOrch(m_config_db.get(), "POLICER"); From 0d9ea95b1ef1cccacbf75b19d7319594e0dde8b5 Mon Sep 17 00:00:00 2001 From: Tom cappleman Date: Thu, 30 Sep 2021 13:09:30 +0100 Subject: [PATCH 12/51] Use NHG for all sizes of group --- orchagent/nhgorch.cpp | 143 +++++++++++------------------------------- orchagent/nhgorch.h | 3 +- tests/test_nhg.py | 105 ++++--------------------------- 3 files changed, 50 insertions(+), 201 deletions(-) diff --git a/orchagent/nhgorch.cpp b/orchagent/nhgorch.cpp index 9823857cc9..366dc2f8f9 100644 --- a/orchagent/nhgorch.cpp +++ b/orchagent/nhgorch.cpp @@ -104,15 +104,13 @@ void NhgOrch::doTask(Consumer& consumer) if (nhg_it == m_syncdNextHopGroups.end()) { /* - * If we'd have to create a SAI object for the group, and we - * already reached the limit, we're going to create a temporary + * If we've reached the NHG limit, we're going to create a temporary * group, represented by one of it's NH only until we have * enough resources to sync the whole group. The item is going * to be kept in the sync list so we keep trying to create the * actual group when there are enough resources. */ - if ((nhg_key.getSize() > 1) && - (gRouteOrch->getNhgCount() + NextHopGroup::getCount() >= gRouteOrch->getMaxNhgCount())) + if (gRouteOrch->getNhgCount() + NextHopGroup::getCount() >= gRouteOrch->getMaxNhgCount()) { SWSS_LOG_DEBUG("Next hop group count reached its limit."); @@ -139,7 +137,7 @@ void NhgOrch::doTask(Consumer& consumer) } else { - auto nhg = std::make_unique(nhg_key); + auto nhg = std::make_unique(nhg_key, false); success = nhg->sync(); if (success) @@ -153,30 +151,14 @@ void NhgOrch::doTask(Consumer& consumer) { const auto& nhg_ptr = nhg_it->second.nhg; - /* - * A NHG update should never change the SAI ID of the NHG if it - * is still referenced by some other objects, as they would not - * be notified about this change. The only exception to this - * rule is for the temporary NHGs, as the referencing objects - * will keep querying the NhgOrch for any SAI ID updates. - */ - if (!nhg_ptr->isTemp() && - ((nhg_key.getSize() == 1) || (nhg_ptr->getSize() == 1)) && - (nhg_it->second.ref_count > 0)) - { - SWSS_LOG_INFO("Next hop group %s update would change SAI " - "ID while referenced, so not performed", - index.c_str()); - } /* * If the update would mandate promoting a temporary next hop * group to a multiple next hops group and we do not have the * resources yet, we have to skip it until we have enough * resources. */ - else if (nhg_ptr->isTemp() && - (nhg_key.getSize() > 1) && - (gRouteOrch->getNhgCount() + NextHopGroup::getCount() >= gRouteOrch->getMaxNhgCount())) + if (nhg_ptr->isTemp() && + (gRouteOrch->getNhgCount() + NextHopGroup::getCount() >= gRouteOrch->getMaxNhgCount())) { /* * If the group was updated in such way that the previously @@ -541,10 +523,10 @@ NextHopGroupMember::~NextHopGroupMember() * Params: IN key - The next hop group's key. * Returns: Nothing. */ -NextHopGroup::NextHopGroup(const NextHopGroupKey& key) : +NextHopGroup::NextHopGroup(const NextHopGroupKey& key, bool is_temp) : m_key(key), m_id(SAI_NULL_OBJECT_ID), - m_is_temp(false) + m_is_temp(is_temp) { SWSS_LOG_ENTER(); @@ -612,10 +594,10 @@ bool NextHopGroup::sync() } /* - * If the group has only one member, the group ID will be the member's NH + * If the group is temporary, the group ID will be the only member's NH * ID. */ - if (m_members.size() == 1) + if (m_is_temp) { const NextHopGroupMember& nhgm = m_members.begin()->second; sai_object_id_t nhid = nhgm.getNhId(); @@ -630,7 +612,6 @@ bool NextHopGroup::sync() m_id = nhid; } } - /* If the key contains more than one NH, create a group. */ else { /* Assert the group is not empty. */ @@ -721,8 +702,7 @@ NextHopGroup NhgOrch::createTempNhg(const NextHopGroupKey& nhg_key) advance(it, rand() % valid_nhs.size()); /* Create the temporary group. */ - NextHopGroup nhg(NextHopGroupKey(it->to_string())); - nhg.setTemp(true); + NextHopGroup nhg(NextHopGroupKey(it->to_string()), true); return nhg; } @@ -739,42 +719,38 @@ bool NextHopGroup::remove() { SWSS_LOG_ENTER(); - /* If the group is already removed, there is nothing to be done. */ - if (!isSynced()) + /* If the group is already removed, or is temporary, there is nothing to be done - + * just reset the ID. + */ + if (!isSynced() || m_is_temp) { + m_id = SAI_NULL_OBJECT_ID; return true; } - /* - * If the group has more than one members, remove it's members, then the - * group. - */ - if (m_members.size() > 1) + /* Remove group's members. If we failed to remove the members, exit. */ + if (!removeMembers(m_key.getNextHops())) { - /* Remove group's members. If we failed to remove the members, exit. */ - if (!removeMembers(m_key.getNextHops())) - { - SWSS_LOG_ERROR("Failed to remove group %s members", to_string().c_str()); - return false; - } + SWSS_LOG_ERROR("Failed to remove group %s members", to_string().c_str()); + return false; + } - /* Remove the group. */ - sai_status_t status = sai_next_hop_group_api-> - remove_next_hop_group(m_id); + /* Remove the group. */ + sai_status_t status = sai_next_hop_group_api-> + remove_next_hop_group(m_id); - if (status != SAI_STATUS_SUCCESS) - { - SWSS_LOG_ERROR("Failed to remove next hop group %s, rv: %d", - m_key.to_string().c_str(), status); - return false; - } - - /* If the operation is successful, release the resources. */ - gCrmOrch->decCrmResUsedCounter( - CrmResourceType::CRM_NEXTHOP_GROUP); - --m_count; + if (status != SAI_STATUS_SUCCESS) + { + SWSS_LOG_ERROR("Failed to remove next hop group %s, rv: %d", + m_key.to_string().c_str(), status); + return false; } + /* If the operation is successful, release the resources. */ + gCrmOrch->decCrmResUsedCounter( + CrmResourceType::CRM_NEXTHOP_GROUP); + --m_count; + /* Reset the group ID. */ m_id = SAI_NULL_OBJECT_ID; @@ -795,9 +771,6 @@ bool NextHopGroup::syncMembers(const std::set& nh_keys) { SWSS_LOG_ENTER(); - /* This method should not be called for groups with only one NH. */ - assert(m_members.size() > 1); - ObjectBulker nextHopGroupMemberBulker(sai_next_hop_group_api, gSwitchId, gMaxBulkSize); /* @@ -882,9 +855,6 @@ bool NextHopGroup::removeMembers(const std::set& nh_keys) { SWSS_LOG_ENTER(); - /* This method should not be called for groups with only one NH. */ - assert(m_members.size() > 1); - ObjectBulker nextHopGroupMemberBulker( sai_next_hop_group_api, gSwitchId, gMaxBulkSize); @@ -950,39 +920,20 @@ bool NextHopGroup::update(const NextHopGroupKey& nhg_key) { SWSS_LOG_ENTER(); - /* - * There are three cases where the SAI ID of the NHG will change: - * - changing a single next hop group to another single next hop group - * - changing a single next hop group to a multiple next hop group - * - changing a multiple next hop group to a single next hop group - * - * For these kind of updates, we can simply swap the existing group with - * the updated group, as we have no way of preserving the existing SAI ID. - * - * Also, we can perform the same operation if the group is not synced at - * all. - */ - if ((nhg_key.getSize() == 1) || (m_members.size() == 1) || !isSynced()) + /* If we are converting a temporary NHG into a permanent one, just create a new NHG. */ + if (m_is_temp) { - bool was_synced = isSynced(); - /* * The previous NHG is going to be destroyed as a consequence of the * move assignment oprator being invoked on a temporary object. The * destructor will be invoked right after the evaluation of the line * below. */ - *this = NextHopGroup(nhg_key); + *this = NextHopGroup(nhg_key, false); /* Sync the group only if it was synced before. */ - return (was_synced ? sync() : true); + return sync(); } - /* - * If we are updating a multiple next hop group to another multiple next - * hop group, we can preserve it's SAI ID by only updating it's members. - * This way, any objects referencing the SAI ID of this object will - * continue to work. - */ else { /* Update the key. */ @@ -1097,16 +1048,6 @@ bool NextHopGroup::validateNextHop(const NextHopKey& nh_key) { SWSS_LOG_ENTER(); - /* - * If the group has only one member, there is nothing to be done. The - * member is only a reference to the next hop owned by NeighOrch, so it is - * not for us to take any decisions regarding those. - */ - if (m_members.size() == 1) - { - return true; - } - return syncMembers({nh_key}); } @@ -1121,15 +1062,5 @@ bool NextHopGroup::invalidateNextHop(const NextHopKey& nh_key) { SWSS_LOG_ENTER(); - /* - * If the group has only one member, there is nothing to be done. The - * member is only a reference to the next hop owned by NeighOrch, so it is - * not for us to take any decisions regarding those. - */ - if (m_members.size() == 1) - { - return true; - } - return removeMembers({nh_key}); } diff --git a/orchagent/nhgorch.h b/orchagent/nhgorch.h index f0b48c20bc..86825a2d85 100644 --- a/orchagent/nhgorch.h +++ b/orchagent/nhgorch.h @@ -73,7 +73,7 @@ class NextHopGroup { public: /* Constructors. */ - explicit NextHopGroup(const NextHopGroupKey& key); + explicit NextHopGroup(const NextHopGroupKey& key, bool is_temp); NextHopGroup(NextHopGroup&& nhg); NextHopGroup& operator=(NextHopGroup&& nhg); @@ -115,7 +115,6 @@ class NextHopGroup inline sai_object_id_t getId() const { return m_id; } static inline unsigned int getCount() { return m_count; } inline bool isTemp() const { return m_is_temp; } - inline void setTemp(bool is_temp) { m_is_temp = is_temp; } inline bool isSynced() const { return m_id != SAI_NULL_OBJECT_ID; } inline size_t getSize() const { return m_members.size(); } diff --git a/tests/test_nhg.py b/tests/test_nhg.py index 771eb637dc..84a7025704 100644 --- a/tests/test_nhg.py +++ b/tests/test_nhg.py @@ -988,6 +988,8 @@ def nhg_exists(nhg_index): # Save the number of group members binary_count = binary.count('1') + dvs.runcmd('swssloglevel -c orchagent -l DEBUG') + # Update the temporary group with a different number of members while True: r += 1 @@ -1087,63 +1089,6 @@ def nhg_exists(nhg_index): rt_id, {'SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID': asic_nhgs[nhg_index]}) - # Try updating the promoted NHG to a single NHG. Should fail as it no - # longer is a temporary NHG - nhg_id = self.get_nhg_id(nhg_index, dvs) - fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1'), - ('ifname', 'Ethernet0')]) - nhg_ps.set(nhg_index, fvs) - time.sleep(1) - assert bool(asic_db.get_entry(self.ASIC_NHG_STR, nhg_id)) == True - - # Save the NHG index - prev_nhg_id = nhg_id - - # Create a temporary next hop groups - nhg_index, binary = create_temp_nhg() - - # Update the route to point to the temporary group - fvs = swsscommon.FieldValuePairs([('nexthop_group', nhg_index)]) - rt_ps.set('2.2.2.0/24', fvs) - - # The previous group should be updated to a single NHG now, freeing a - # NHG slot in ASIC and promoting the temporary one - asic_db.wait_for_n_keys(self.ASIC_NHG_STR, MAX_ECMP_COUNT) - asic_nhgs[nhg_index] = self.get_nhg_id(nhg_index, dvs) - assert self.get_nhg_id(nhg_index, dvs) != prev_nhg_id - - # Create a temporary next hop groups - nhg_index, binary = create_temp_nhg() - - # Update the route to point to the temporary group - fvs = swsscommon.FieldValuePairs([('nexthop_group', nhg_index)]) - rt_ps.set('2.2.2.0/24', fvs) - - # Update the group to a single NHG - fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1'), - ('ifname', 'Ethernet0')]) - nhg_ps.set(nhg_index, fvs) - nhg_id = self.get_nhg_id(nhg_index, dvs) - - # Wait for the route to update it's NHG ID - asic_db.wait_for_field_match(self.ASIC_RT_STR, - rt_id, - {'SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID': nhg_id}) - - # Try to update the nexthop group to another one - fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.3'), - ('ifname', 'Ethernet4')]) - nhg_ps.set(nhg_index, fvs) - - # Wait a second so the database operations finish - time.sleep(1) - - # The update should fail as the group is not temporary anymore and it - # should keep it's previous NHG ID - asic_db.wait_for_field_match(self.ASIC_RT_STR, - rt_id, - {'SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID': nhg_id}) - # Create a next hop group that contains labeled NHs that do not exist # in NeighOrch asic_nhs_count = len(asic_db.get_keys(self.ASIC_NHS_STR)) @@ -1407,26 +1352,17 @@ def test_nhgorch_multi_nh_group(self, dvs, testlog): # Wait for members to be added asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, asic_nhgms_count + 4) - # Update the group to one NH only - orchagent should fail as it is - # referenced + # Update the group to one NH only fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1'), ("ifname", "Ethernet0")]) nhg_ps.set("group1", fvs) time.sleep(1) assert bool(asic_db.get_entry(self.ASIC_NHG_STR, nhgid)) - assert len(asic_db.get_keys(self.ASIC_NHGM_STR)) == asic_nhgms_count + 4 + asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, asic_nhgms_count + 1) # Remove route 2.2.2.0/24 rt_ps._del("2.2.2.0/24") - # Wait for route 2.2.2.0/24 to be removed - asic_db.wait_for_n_keys(self.ASIC_RT_STR, asic_routes_count) - - # The group is not referenced anymore, so it should get updated to a - # single next hop group, removing the ASIC group - asic_db.wait_for_n_keys(self.ASIC_NHG_STR, asic_nhgs_count) - asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, asic_nhgms_count) - # Remove group1 nhg_ps._del("group1") @@ -1446,18 +1382,11 @@ def test_nhgorch_single_nh_group(self, dvs, testlog): nhg_ps.set("group1", fvs) time.sleep(1) - # Check that the group was not created in ASIC DB - assert len(asic_db.get_keys(self.ASIC_NHG_STR)) == asic_nhgs_count + # Check that the group was created in ASIC DB + asic_db.wait_for_n_keys(self.ASIC_NHG_STR, asic_nhgs_count + 1) - # Get the NHG ID. The UT assumes there is only one next hop with IP 10.0.0.1 - count = 0 - nhgid = 0 - for k in asic_db.get_keys(self.ASIC_NHS_STR): - fvs = asic_db.get_entry(self.ASIC_NHS_STR, k) - if fvs['SAI_NEXT_HOP_ATTR_IP'] == '10.0.0.1': - nhgid = k - count += 1 - assert count == 1 + # Get the NHG ID. + nhgid = self.get_nhg_id("group1", dvs) # create route in APPL DB @@ -1535,20 +1464,14 @@ def test_nhgorch_single_nh_group(self, dvs, testlog): nhg_ps._del('group2') asic_db.wait_for_n_keys(self.ASIC_RT_STR, asic_routes_count + 1) - # Update the group to a multiple NH group - should fail as the group - # is referenced - nhgid = self.get_nhg_id('group1', dvs) + # Update the group to a multiple NH group fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1,10.0.0.3'), ("ifname", "Ethernet0,Ethernet4")]) nhg_ps.set("group1", fvs) time.sleep(1) - assert len(asic_db.get_keys(self.ASIC_NHG_STR)) == asic_nhgs_count - # Assert the NHG ID hasn't changed - assert nhgid == self.get_nhg_id('group1', dvs) - - # Update group1 to point to another NH - should fail as the group is - # referenced + # Update group1 to point to another NH + nhgid = self.get_nhg_id('group1', dvs) fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.3'), ("ifname", "Ethernet4")]) nhg_ps.set("group1", fvs) assert nhgid == self.get_nhg_id('group1', dvs) @@ -1559,11 +1482,7 @@ def test_nhgorch_single_nh_group(self, dvs, testlog): # Wait for route 2.2.2.0/24 to be removed asic_db.wait_for_n_keys(self.ASIC_RT_STR, asic_routes_count) - # The group is not referenced anymore, so it should be updated - assert nhgid != self.get_nhg_id('group1', dvs) - - # Update the group to a multiple NH group - should work as the group is - # not referenced + # Update the group to a multiple NH group fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1,10.0.0.3'), ("ifname", "Ethernet0,Ethernet4")]) nhg_ps.set("group1", fvs) From 00fce94b1351c84350e10aaf35aedc5a0d336c85 Mon Sep 17 00:00:00 2001 From: Tom cappleman Date: Fri, 1 Oct 2021 10:40:18 +0100 Subject: [PATCH 13/51] Fix temp NH handling --- orchagent/nhgorch.cpp | 156 +++++++++++++++++++++++------------------- orchagent/nhgorch.h | 8 +++ tests/test_nhg.py | 8 +-- 3 files changed, 93 insertions(+), 79 deletions(-) diff --git a/orchagent/nhgorch.cpp b/orchagent/nhgorch.cpp index 366dc2f8f9..4b2084e82a 100644 --- a/orchagent/nhgorch.cpp +++ b/orchagent/nhgorch.cpp @@ -12,6 +12,7 @@ extern PortsOrch *gPortsOrch; extern CrmOrch *gCrmOrch; extern NeighOrch *gNeighOrch; extern RouteOrch *gRouteOrch; +extern NhgOrch *gNhgOrch; extern size_t gMaxBulkSize; @@ -199,6 +200,24 @@ void NhgOrch::doTask(Consumer& consumer) } } } + /* + * If the group is temporary but can now be promoted, create and sync a new group for + * the desired next hops. + */ + else if (nhg_ptr->isTemp()) + { + auto nhg = std::make_unique(nhg_key, false); + success = nhg->sync(); + + if (success) + { + /* + * Placing the new group in the map will replace the temporary group, causing + * it to be removed and freed. + */ + nhg_it->second.nhg = std::move(nhg); + } + } /* Common update, when all the requirements are met. */ else { @@ -636,7 +655,12 @@ bool NextHopGroup::sync() { SWSS_LOG_ERROR("Failed to create next hop group %s, rv:%d", m_key.to_string().c_str(), status); - return false; + + task_process_status handle_status = gNhgOrch->handleSaiCreateStatus(SAI_API_NEXT_HOP_GROUP, status); + if (handle_status != task_success) + { + return gNhgOrch->parseHandleSaiStatusFailure(handle_status); + } } /* Increment the amount of programmed next hop groups. */ @@ -743,7 +767,12 @@ bool NextHopGroup::remove() { SWSS_LOG_ERROR("Failed to remove next hop group %s, rv: %d", m_key.to_string().c_str(), status); - return false; + + task_process_status handle_status = gNhgOrch->handleSaiRemoveStatus(SAI_API_NEXT_HOP_GROUP, status); + if (handle_status != task_success) + { + return gNhgOrch->parseHandleSaiStatusFailure(handle_status); + } } /* If the operation is successful, release the resources. */ @@ -920,90 +949,73 @@ bool NextHopGroup::update(const NextHopGroupKey& nhg_key) { SWSS_LOG_ENTER(); - /* If we are converting a temporary NHG into a permanent one, just create a new NHG. */ - if (m_is_temp) - { - /* - * The previous NHG is going to be destroyed as a consequence of the - * move assignment oprator being invoked on a temporary object. The - * destructor will be invoked right after the evaluation of the line - * below. - */ - *this = NextHopGroup(nhg_key, false); + /* Update the key. */ + m_key = nhg_key; - /* Sync the group only if it was synced before. */ - return sync(); - } - else + std::set new_nh_keys = nhg_key.getNextHops(); + std::set removed_nh_keys; + + /* Mark the members that need to be removed. */ + for (auto& mbr_it : m_members) { - /* Update the key. */ - m_key = nhg_key; + const NextHopKey& nh_key = mbr_it.first; - std::set new_nh_keys = nhg_key.getNextHops(); - std::set removed_nh_keys; + /* Look for the existing member inside the new ones. */ + const auto& new_nh_key_it = new_nh_keys.find(nh_key); - /* Mark the members that need to be removed. */ - for (auto& mbr_it : m_members) + /* If the member is not found, then it needs to be removed. */ + if (new_nh_key_it == new_nh_keys.end()) { - const NextHopKey& nh_key = mbr_it.first; - - /* Look for the existing member inside the new ones. */ - const auto& new_nh_key_it = new_nh_keys.find(nh_key); - - /* If the member is not found, then it needs to be removed. */ - if (new_nh_key_it == new_nh_keys.end()) - { - removed_nh_keys.insert(nh_key); - } - /* If the member is updated, update it's weight. */ - else + removed_nh_keys.insert(nh_key); + } + /* If the member is updated, update it's weight. */ + else + { + if (!mbr_it.second.updateWeight(new_nh_key_it->weight)) { - if (!mbr_it.second.updateWeight(new_nh_key_it->weight)) - { - SWSS_LOG_WARN("Failed to update member %s weight", nh_key.to_string().c_str()); - return false; - } - - /* - * Erase the member from the new members list as it already - * exists. - */ - new_nh_keys.erase(new_nh_key_it); + SWSS_LOG_WARN("Failed to update member %s weight", nh_key.to_string().c_str()); + return false; } - } - /* Remove the removed members. */ - if (!removeMembers(removed_nh_keys)) - { - SWSS_LOG_WARN("Failed to remove members from group %s", to_string().c_str()); - return false; + /* + * Erase the member from the new members list as it already + * exists. + */ + new_nh_keys.erase(new_nh_key_it); } + } - /* Remove the removed members. */ - for (const auto& nh_key : removed_nh_keys) - { - m_members.erase(nh_key); - } + /* Remove the removed members. */ + if (!removeMembers(removed_nh_keys)) + { + SWSS_LOG_WARN("Failed to remove members from group %s", to_string().c_str()); + return false; + } - /* Add any new members to the group. */ - for (const auto& it : new_nh_keys) - { - m_members.emplace(it, NextHopGroupMember(it)); - } + /* Remove the removed members. */ + for (const auto& nh_key : removed_nh_keys) + { + m_members.erase(nh_key); + } - /* - * Sync all the members of the group. We sync all of them because - * there may be previous members that were not successfully synced - * before the update, so we must make sure we sync those as well. - */ - if (!syncMembers(m_key.getNextHops())) - { - SWSS_LOG_WARN("Failed to sync new members for group %s", to_string().c_str()); - return false; - } + /* Add any new members to the group. */ + for (const auto& it : new_nh_keys) + { + m_members.emplace(it, NextHopGroupMember(it)); + } - return true; + /* + * Sync all the members of the group. We sync all of them because + * there may be previous members that were not successfully synced + * before the update, so we must make sure we sync those as well. + */ + if (!syncMembers(m_key.getNextHops())) + { + SWSS_LOG_WARN("Failed to sync new members for group %s", to_string().c_str()); + return false; } + + return true; } /* diff --git a/orchagent/nhgorch.h b/orchagent/nhgorch.h index 86825a2d85..c16ce40f83 100644 --- a/orchagent/nhgorch.h +++ b/orchagent/nhgorch.h @@ -229,6 +229,14 @@ class NhgOrch : public Orch void incNhgRefCount(const std::string& index); void decNhgRefCount(const std::string& index); + /* Handling SAI status*/ + task_process_status handleSaiCreateStatus(sai_api_t api, sai_status_t status, void *context = nullptr) + { return Orch::handleSaiCreateStatus(api, status, context); } + task_process_status handleSaiRemoveStatus(sai_api_t api, sai_status_t status, void *context = nullptr) + { return Orch::handleSaiRemoveStatus(api, status, context); } + bool parseHandleSaiStatusFailure(task_process_status status) + { return Orch::parseHandleSaiStatusFailure(status); } + private: /* diff --git a/tests/test_nhg.py b/tests/test_nhg.py index 84a7025704..1bc14dca1c 100644 --- a/tests/test_nhg.py +++ b/tests/test_nhg.py @@ -988,8 +988,6 @@ def nhg_exists(nhg_index): # Save the number of group members binary_count = binary.count('1') - dvs.runcmd('swssloglevel -c orchagent -l DEBUG') - # Update the temporary group with a different number of members while True: r += 1 @@ -1493,8 +1491,7 @@ def test_nhgorch_single_nh_group(self, dvs, testlog): rt_ps.set('2.2.2.0/24', fvs) asic_db.wait_for_n_keys(self.ASIC_RT_STR, asic_routes_count + 1) - # Update the NHG to a single next hop - should fail as it is being - # referenced + # Update the NHG to a single next hop nhgid = self.get_nhg_id('group1', dvs) fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1'), ("ifname", "Ethernet0")]) nhg_ps.set("group1", fvs) @@ -1505,9 +1502,6 @@ def test_nhgorch_single_nh_group(self, dvs, testlog): rt_ps._del("2.2.2.0/24") asic_db.wait_for_n_keys(self.ASIC_RT_STR, asic_routes_count) - # The group is not referenced anymore, so it should be updated - assert nhgid != self.get_nhg_id('group1', dvs) - # Remove group1 nhg_ps._del("group1") From 747fa54231277587ef3dd67b264eba369b44701f Mon Sep 17 00:00:00 2001 From: Alexandru Banu Date: Fri, 1 Oct 2021 17:26:25 +0300 Subject: [PATCH 14/51] Refactor UT structure --- tests/test_nhg.py | 1708 +++++++++++++++++++++------------------------ 1 file changed, 798 insertions(+), 910 deletions(-) diff --git a/tests/test_nhg.py b/tests/test_nhg.py index 1bc14dca1c..9f34bff583 100644 --- a/tests/test_nhg.py +++ b/tests/test_nhg.py @@ -8,65 +8,61 @@ from swsscommon import swsscommon - -class TestNextHopGroup(object): +class TestNextHopGroupBase(object): ASIC_NHS_STR = "ASIC_STATE:SAI_OBJECT_TYPE_NEXT_HOP" ASIC_NHG_STR = "ASIC_STATE:SAI_OBJECT_TYPE_NEXT_HOP_GROUP" ASIC_NHGM_STR = ASIC_NHG_STR + "_MEMBER" ASIC_RT_STR = "ASIC_STATE:SAI_OBJECT_TYPE_ROUTE_ENTRY" ASIC_INSEG_STR = "ASIC_STATE:SAI_OBJECT_TYPE_INSEG_ENTRY" - def get_route_id(self, prefix, dvs): - for k in dvs.get_asic_db().get_keys(self.ASIC_RT_STR): + def get_route_id(self, prefix): + for k in self.asic_db.get_keys(self.ASIC_RT_STR): if json.loads(k)['dest'] == prefix: return k return None - def get_inseg_id(self, label, dvs): - for k in dvs.get_asic_db().get_keys(self.ASIC_INSEG_STR): + def get_inseg_id(self, label): + for k in self.asic_db.get_keys(self.ASIC_INSEG_STR): if json.loads(k)['label'] == label: return k return None - def get_nhg_id(self, nhg_index, dvs): + def get_nhg_id(self, nhg_index): # Add a route with the given index, then retrieve the next hop group ID # from that route - asic_db = dvs.get_asic_db() - asic_rts_count = len(asic_db.get_keys(self.ASIC_RT_STR)) + asic_rts_count = len(self.asic_db.get_keys(self.ASIC_RT_STR)) fvs = swsscommon.FieldValuePairs([('nexthop_group', nhg_index)]) prefix = '255.255.255.255/24' - ps = swsscommon.ProducerStateTable(dvs.get_app_db().db_connection, - "ROUTE_TABLE") + ps = swsscommon.ProducerStateTable(self.dvs.get_app_db().db_connection, swsscommon.APP_ROUTE_TABLE_NAME) ps.set(prefix, fvs) # Assert the route is created try: - asic_db.wait_for_n_keys(self.ASIC_RT_STR, asic_rts_count + 1) + self.asic_db.wait_for_n_keys(self.ASIC_RT_STR, asic_rts_count + 1) except Exception as e: ps._del(prefix) return None else: # Get the route ID for the created route - rt_id = self.get_route_id(prefix, dvs) + rt_id = self.get_route_id(prefix) assert rt_id != None # Get the NHGID - nhgid = asic_db.get_entry(self.ASIC_RT_STR, rt_id)["SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID"] + nhgid = self.asic_db.get_entry(self.ASIC_RT_STR, rt_id)["SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID"] ps._del(prefix) - asic_db.wait_for_deleted_entry(self.ASIC_RT_STR, rt_id) + self.asic_db.wait_for_deleted_entry(self.ASIC_RT_STR, rt_id) return nhgid - def get_nhgm_ids(self, nhg_index, dvs): - nhgid = self.get_nhg_id(nhg_index, dvs) + def get_nhgm_ids(self, nhg_index): + nhgid = self.get_nhg_id(nhg_index) nhgms = [] - asic_db = dvs.get_asic_db() - for k in asic_db.get_keys(self.ASIC_NHGM_STR): - fvs = asic_db.get_entry(self.ASIC_NHGM_STR, k) + for k in self.asic_db.get_keys(self.ASIC_NHGM_STR): + fvs = self.asic_db.get_entry(self.ASIC_NHGM_STR, k) # Sometimes some of the NHGMs have no fvs for some reason, so # we skip those @@ -93,62 +89,559 @@ def peer_ip(self, i): def port_mac(self, i): return "00:00:00:00:00:0" + str(i) - def config_intf(self, i, dvs): - config_db = dvs.get_config_db() + def config_intf(self, i): fvs = {'NULL': 'NULL'} - config_db.create_entry("INTERFACE", self.port_name(i), fvs) - config_db.create_entry("INTERFACE", "{}|{}".format(self.port_name(i), self.port_ipprefix(i)), fvs) - dvs.runcmd("config interface startup " + self.port_name(i)) - dvs.runcmd("arp -s {} {}".format(self.peer_ip(i), self.port_mac(i))) - assert dvs.servers[i].runcmd("ip link set down dev eth0") == 0 - assert dvs.servers[i].runcmd("ip link set up dev eth0") == 0 + self.config_db.create_entry("INTERFACE", self.port_name(i), fvs) + self.config_db.create_entry("INTERFACE", "{}|{}".format(self.port_name(i), self.port_ipprefix(i)), fvs) + self.dvs.runcmd("config interface startup " + self.port_name(i)) + self.dvs.runcmd("arp -s {} {}".format(self.peer_ip(i), self.port_mac(i))) + assert self.dvs.servers[i].runcmd("ip link set down dev eth0") == 0 + assert self.dvs.servers[i].runcmd("ip link set up dev eth0") == 0 - def flap_intf(self, i, status, dvs): + def flap_intf(self, i, status): assert status in ['up', 'down'] - dvs.servers[i].runcmd("ip link set {} dev eth0".format(status)) == 0 + self.dvs.servers[i].runcmd("ip link set {} dev eth0".format(status)) == 0 time.sleep(2) - fvs = dvs.get_app_db().get_entry("PORT_TABLE", "Ethernet%d" % (i * 4)) + fvs = self.dvs.get_app_db().get_entry("PORT_TABLE", "Ethernet%d" % (i * 4)) assert bool(fvs) assert fvs["oper_status"] == status - def test_route_nhg(self, dvs, dvs_route, testlog): - for i in range(3): - self.config_intf(i, dvs) + def init_test(self, dvs, num_intfs): + self.dvs = dvs + self.app_db = self.dvs.get_app_db() + self.asic_db = self.dvs.get_asic_db() + self.config_db = self.dvs.get_config_db() + self.nhg_ps = swsscommon.ProducerStateTable(self.app_db.db_connection, swsscommon.APP_NEXTHOP_GROUP_TABLE_NAME) + self.rt_ps = swsscommon.ProducerStateTable(self.app_db.db_connection, swsscommon.APP_ROUTE_TABLE_NAME) + self.lr_ps = swsscommon.ProducerStateTable(self.app_db.db_connection, swsscommon.APP_LABEL_ROUTE_TABLE_NAME) - rtprefix = "2.2.2.0/24" + for i in range(num_intfs): + self.config_intf(i) + + self.asic_nhgs_count = len(self.asic_db.get_keys(self.ASIC_NHG_STR)) + self.asic_nhgms_count = len(self.asic_db.get_keys(self.ASIC_NHGM_STR)) + self.asic_insgs_count = len(self.asic_db.get_keys(self.ASIC_INSEG_STR)) + self.asic_nhs_count = len(self.asic_db.get_keys(self.ASIC_NHS_STR)) + self.asic_rts_count = len(self.asic_db.get_keys(self.ASIC_RT_STR)) + + def nhg_exists(self, nhg_index): + return self.get_nhg_id(nhg_index) is not None + +class TestNextHopGroupExhaust(TestNextHopGroupBase): + MAX_ECMP_COUNT = 512 + MAX_PORT_COUNT = 10 + + def init_test(self, dvs): + super().init_test(dvs, self.MAX_PORT_COUNT) + self.r = 0 + + def gen_nhg_fvs(self, binary): + nexthop = [] + ifname = [] + + for i in range(self.MAX_PORT_COUNT): + if binary[i] == '1': + nexthop.append(self.peer_ip(i)) + ifname.append(self.port_name(i)) + + nexthop = ','.join(nexthop) + ifname = ','.join(ifname) + fvs = swsscommon.FieldValuePairs([("nexthop", nexthop), ("ifname", ifname)]) + + return fvs + + def gen_valid_binary(self): + while True: + self.r += 1 + binary = self.gen_valid_binary.fmt.format(self.r) + # We need at least 2 ports for a nexthop group + if binary.count('1') <= 1: + continue + return binary + gen_valid_binary.fmt = '{{0:0{}b}}'.format(MAX_PORT_COUNT) + + def test_nhgorch_nhg_exhaust(self, dvs, testlog): + def gen_nhg_index(nhg_number): + return "group{}".format(nhg_number) + + def create_temp_nhg(): + binary = self.gen_valid_binary() + nhg_fvs = self.gen_nhg_fvs(binary) + nhg_index = gen_nhg_index(self.nhg_count) + self.nhg_ps.set(nhg_index, nhg_fvs) + self.nhg_count += 1 + + return nhg_index, binary + + def delete_nhg(): + del_nhg_index = gen_nhg_index(self.first_valid_nhg) + del_nhg_id = self.asic_nhgs[del_nhg_index] + + self.nhg_ps._del(del_nhg_index) + self.asic_nhgs.pop(del_nhg_index) + self.first_valid_nhg += 1 + + return del_nhg_id + + def test_temporary_group_promotion(): + # Add a new next hop group - it should create a temporary one instead + prev_nhgs = self.asic_db.get_keys(self.ASIC_NHG_STR) + nhg_index, _ = create_temp_nhg() + + # Save the temporary NHG's SAI ID + time.sleep(1) + nhg_id = self.get_nhg_id(nhg_index) + + # Assert no new group has been added + self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.MAX_ECMP_COUNT) + + # Assert the same NHGs are in ASIC DB + assert prev_nhgs == self.asic_db.get_keys(self.ASIC_NHG_STR) + + # Delete an existing next hop group + del_nhg_id = delete_nhg() + + # Wait for the key to be deleted + self.asic_db.wait_for_deleted_entry(self.ASIC_NHG_STR, del_nhg_id) + + # Wait for the temporary group to be promoted and replace the deleted + # NHG + self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.MAX_ECMP_COUNT) + + # Assert the SAI ID of the previously temporary NHG has been updated + assert nhg_id != self.get_nhg_id(nhg_index) + + # Save the promoted NHG index/ID + self.asic_nhgs[nhg_index] = self.get_nhg_id(nhg_index) + + def test_group_update(): + # Update a group + binary = self.gen_valid_binary() + nhg_fvs = self.gen_nhg_fvs(binary) + nhg_index = gen_nhg_index(self.first_valid_nhg) + + # Save the previous members + prev_nhg_members = self.get_nhgm_ids(nhg_index) + self.nhg_ps.set(nhg_index, nhg_fvs) + + # Wait a second so the NHG members get updated + time.sleep(1) + + # Assert the group was updated by checking it's members + assert self.get_nhgm_ids(nhg_index) != prev_nhg_members + + def test_create_delete_temporary(): + # Create a new temporary group + nhg_index, _ = create_temp_nhg() + time.sleep(1) + + # Delete the temporary group + self.nhg_ps._del(nhg_index) + + # Assert the NHG does not exist anymore + assert not self.nhg_exists(nhg_index) + + # Assert the number of groups is the same + self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.MAX_ECMP_COUNT) + + def test_update_temporary_group(): + # Create a new temporary group + nhg_index, binary = create_temp_nhg() + + # Save the number of group members + binary_count = binary.count('1') + + # Update the temporary group with a different number of members + while True: + binary = self.gen_valid_binary() + if binary.count('1') == binary_count: + continue + binary_count = binary.count('1') + break + nhg_fvs = self.gen_nhg_fvs(binary) + self.nhg_ps.set(nhg_index, nhg_fvs) + + # Delete a group + del_nhg_id = delete_nhg() + + # Wait for the group to be deleted + self.asic_db.wait_for_deleted_entry(self.ASIC_NHG_STR, del_nhg_id) + + # The temporary group should be promoted + self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.MAX_ECMP_COUNT) + + # Save the promoted NHG index/ID + self.asic_nhgs[nhg_index] = self.get_nhg_id(nhg_index) + + # Assert it has the updated details by checking the number of members + assert len(self.get_nhgm_ids(nhg_index)) == binary_count - app_db = dvs.get_app_db() - ps = swsscommon.ProducerStateTable(app_db.db_connection, "ROUTE_TABLE") + def test_route_nhg_update(): + # Add a route + nhg_index = gen_nhg_index(self.first_valid_nhg) + rt_fvs = swsscommon.FieldValuePairs([('nexthop_group', nhg_index)]) + self.rt_ps.set('2.2.2.0/24', rt_fvs) - asic_db = dvs.get_asic_db() + # Assert the route is created + self.asic_db.wait_for_n_keys(self.ASIC_RT_STR, self.asic_rts_count + 1) + + # Save the previous NHG ID + prev_nhg_id = self.asic_nhgs[nhg_index] + + # Create a new temporary group + nhg_index, binary = create_temp_nhg() + + # Get the route ID + rt_id = self.get_route_id('2.2.2.0/24') + assert rt_id != None + + # Update the route to point to the temporary NHG + rt_fvs = swsscommon.FieldValuePairs([('nexthop_group', nhg_index)]) + self.rt_ps.set('2.2.2.0/24', rt_fvs) + + # Wait for the route to change its NHG ID + self.asic_db.wait_for_field_negative_match(self.ASIC_RT_STR, + rt_id, + {'SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID': prev_nhg_id}) + + # Save the new route NHG ID + prev_nhg_id = self.asic_db.get_entry(self.ASIC_RT_STR, rt_id)['SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID'] + + # Update the temporary NHG with one that has different NHs + + # Create a new binary that uses the other interfaces than the previous + # binary was using + new_binary = [] + + for i in range(len(binary)): + if binary[i] == '1': + new_binary.append('0') + else: + new_binary.append('1') + + binary = ''.join(new_binary) + assert binary.count('1') > 1 + + nhg_fvs = self.gen_nhg_fvs(binary) + self.nhg_ps.set(nhg_index, nhg_fvs) + + # The NHG ID of the route should change + self.asic_db.wait_for_field_negative_match(self.ASIC_RT_STR, + rt_id, + {'SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID': prev_nhg_id}) + + # Delete a NHG. + del_nhg_id = delete_nhg() + + # Wait for the NHG to be deleted + self.asic_db.wait_for_deleted_entry(self.ASIC_NHG_STR, del_nhg_id) + + # The temporary group should get promoted. + self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.MAX_ECMP_COUNT) + + # Save the promoted NHG index/ID + self.asic_nhgs[nhg_index] = self.get_nhg_id(nhg_index) + + # Assert the NHGID of the route changed due to temporary group being + # promoted. + self.asic_db.wait_for_field_match(self.ASIC_RT_STR, + rt_id, + {'SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID': self.asic_nhgs[nhg_index]}) + + def test_labeled_nhg_temporary_promotion(): + # Create a next hop group that contains labeled NHs that do not exist + # in NeighOrch + self.asic_nhs_count = len(self.asic_db.get_keys(self.ASIC_NHS_STR)) + fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1,10.0.0.3'), + ('mpls_nh', 'push1,push3'), + ('ifname', 'Ethernet0,Ethernet4')]) + nhg_index = gen_nhg_index(self.nhg_count) + self.nhg_ps.set(nhg_index, fvs) + self.nhg_count += 1 + + # A temporary next hop should be elected to represent the group and + # thus a new labeled next hop should be created + self.asic_db.wait_for_n_keys(self.ASIC_NHS_STR, self.asic_nhs_count + 1) + + # Delete a next hop group + delete_nhg() + + # The group should be promoted and the other labeled NH should also get + # created + self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.MAX_ECMP_COUNT) + self.asic_db.wait_for_n_keys(self.ASIC_NHS_STR, self.asic_nhs_count + 2) + + # Save the promoted NHG index/ID + self.asic_nhgs[nhg_index] = self.get_nhg_id(nhg_index) + + def test_back_compatibility(): + # Update the route with a RouteOrch's owned NHG + binary = self.gen_valid_binary() + nhg_fvs = self.gen_nhg_fvs(binary) + self.rt_ps.set('2.2.2.0/24', nhg_fvs) + + # Assert no new group has been added + self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.MAX_ECMP_COUNT) + + # Delete a next hop group + del_nhg_id = delete_nhg() + self.asic_db.wait_for_deleted_entry(self.ASIC_NHG_STR, del_nhg_id) + + # The temporary group should be promoted + self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.MAX_ECMP_COUNT) + + def test_invalid_temporary(): + # Create a temporary NHG that contains only NHs that do not exist + nhg_fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.21,10.0.0.23'), + ('ifname', 'Ethernet40,Ethernet44')]) + nhg_index = gen_nhg_index(self.nhg_count) + self.nhg_count += 1 + self.nhg_ps.set(nhg_index, nhg_fvs) + + # Assert the group is not created + assert not self.nhg_exists(nhg_index) + + # Update the temporary NHG to a valid one + binary = self.gen_valid_binary() + nhg_fvs = self.gen_nhg_fvs(binary) + self.nhg_ps.set(nhg_index, nhg_fvs) + + # Assert the temporary group was updated and the group got created + nhg_id = self.get_nhg_id(nhg_index) + assert nhg_id is not None + + # Update the temporary NHG to an invalid one again + nhg_fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.21,10.0.0.23'), + ('ifname', 'Ethernet40,Ethernet44')]) + self.nhg_ps.set(nhg_index, nhg_fvs) + + # The update should fail and the temporary NHG should still be pointing + # to the old valid NH + assert self.get_nhg_id(nhg_index) == nhg_id + + # Delete the temporary group + self.nhg_ps._del(nhg_index) + + self.init_test(dvs) + + self.nhg_count = self.asic_nhgs_count + self.first_valid_nhg = self.nhg_count + self.asic_nhgs = {} + + # Add first batch of next hop groups to reach the NHG limit + while self.nhg_count < self.MAX_ECMP_COUNT: + binary = self.gen_valid_binary() + nhg_fvs = self.gen_nhg_fvs(binary) + nhg_index = gen_nhg_index(self.nhg_count) + self.nhg_ps.set(nhg_index, nhg_fvs) + + # Save the NHG index/ID pair + self.asic_nhgs[nhg_index] = self.get_nhg_id(nhg_index) + + # Increase the number of NHGs in ASIC DB + self.nhg_count += 1 + self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.MAX_ECMP_COUNT) + + test_temporary_group_promotion() + test_group_update() + test_create_delete_temporary() + test_update_temporary_group() + test_route_nhg_update() + test_labeled_nhg_temporary_promotion() + test_back_compatibility() + test_invalid_temporary() + + # Cleanup + + # Delete the route + self.rt_ps._del('2.2.2.0/24') + self.asic_db.wait_for_n_keys(self.ASIC_RT_STR, self.asic_rts_count) + + # Delete the next hop groups + for k in self.asic_nhgs: + self.nhg_ps._del(k) + self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count) + + + def test_route_nhg_exhaust(self, dvs, testlog): + """ + Test the situation of exhausting ECMP group, assume SAI_SWITCH_ATTR_NUMBER_OF_ECMP_GROUPS is 512 + + In order to achieve that, we will config + 1. 9 ports + 2. 512 routes with different nexthop group + + See Also + -------- + SwitchStateBase::set_number_of_ecmp_groups() + https://github.com/Azure/sonic-sairedis/blob/master/vslib/src/SwitchStateBase.cpp + + """ + + # TODO: check ECMP 512 + + def gen_ipprefix(r): + """ Construct route like 2.X.X.0/24 """ + ip = ipaddress.IPv4Address(IP_INTEGER_BASE + r * 256) + ip = str(ip) + ipprefix = ip + "/24" + return ipprefix + + def asic_route_nhg_fvs(k): + fvs = self.asic_db.get_entry(self.ASIC_RT_STR, k) + if not fvs: + return None + + nhgid = fvs.get("SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID") + if nhgid is None: + return None + + fvs = self.asic_db.get_entry(self.ASIC_NHG_STR, nhgid) + return fvs + + if sys.version_info < (3, 0): + IP_INTEGER_BASE = int(ipaddress.IPv4Address(unicode("2.2.2.0"))) + else: + IP_INTEGER_BASE = int(ipaddress.IPv4Address(str("2.2.2.0"))) + + self.init_test(dvs) + + # Add first batch of routes with unique nexthop groups in AppDB + route_count = 0 + while route_count < self.MAX_ECMP_COUNT: + binary = self.gen_valid_binary() + fvs = self.gen_nhg_fvs(binary) + route_ipprefix = gen_ipprefix(route_count) + self.rt_ps.set(route_ipprefix, fvs) + route_count += 1 + + # Wait and check ASIC DB the count of nexthop groups used + self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.MAX_ECMP_COUNT) + + # Wait and check ASIC DB the count of routes + self.asic_db.wait_for_n_keys(self.ASIC_RT_STR, self.asic_rts_count + self.MAX_ECMP_COUNT) + self.asic_rts_count += self.MAX_ECMP_COUNT + + # Add a route with labeled NHs + self.asic_nhs_count = len(self.asic_db.get_keys(self.ASIC_NHS_STR)) + route_ipprefix = gen_ipprefix(route_count) + fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1,10.0.0.3'), + ('mpls_nh', 'push1,push3'), + ('ifname', 'Ethernet0,Ethernet4')]) + self.rt_ps.set(route_ipprefix, fvs) + route_count += 1 + + # A temporary route should be created + self.asic_db.wait_for_n_keys(self.ASIC_RT_STR, self.asic_rts_count + 1) + + # A NH should be elected as the temporary NHG and it should be created + # as it doesn't exist. + self.asic_db.wait_for_n_keys(self.ASIC_NHS_STR, self.asic_nhs_count + 1) + + # Delete the route. The route and the added labeled NH should be + # removed. + self.rt_ps._del(route_ipprefix) + route_count -= 1 + self.asic_db.wait_for_n_keys(self.ASIC_RT_STR, self.asic_rts_count) + self.asic_db.wait_for_n_keys(self.ASIC_NHS_STR, self.asic_nhs_count) + + # Add second batch of routes with unique nexthop groups in AppDB + # Add more routes with new nexthop group in AppDBdd + route_ipprefix = gen_ipprefix(route_count) + base_ipprefix = route_ipprefix + base = route_count + route_count = 0 + while route_count < 10: + binary = self.gen_valid_binary() + fvs = self.gen_nhg_fvs(binary) + route_ipprefix = gen_ipprefix(base + route_count) + self.rt_ps.set(route_ipprefix, fvs) + route_count += 1 + last_ipprefix = route_ipprefix + + # Wait until we get expected routes and check ASIC DB on the count of nexthop groups used, and it should not increase + self.asic_db.wait_for_n_keys(self.ASIC_RT_STR, self.asic_rts_count + 10) + self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.MAX_ECMP_COUNT) + + # Check the route points to next hop group + # Note: no need to wait here + k = self.get_route_id("2.2.2.0/24") + assert k is not None + fvs = asic_route_nhg_fvs(k) + assert fvs is not None + + # Check the second batch does not point to next hop group + k = self.get_route_id(base_ipprefix) + assert k is not None + fvs = asic_route_nhg_fvs(k) + assert not(fvs) + + # Remove first batch of routes with unique nexthop groups in AppDB + route_count = 0 + self.r = 0 + while route_count < self.MAX_ECMP_COUNT: + binary = self.gen_valid_binary() + route_ipprefix = gen_ipprefix(route_count) + self.rt_ps._del(route_ipprefix) + route_count += 1 + self.asic_rts_count -= self.MAX_ECMP_COUNT + + # Wait and check the second batch points to next hop group + # Check ASIC DB on the count of nexthop groups used, and it should not increase or decrease + self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, 10) + k = self.get_route_id(base_ipprefix) + assert k is not None + fvs = asic_route_nhg_fvs(k) + assert fvs is not None + k = self.get_route_id(last_ipprefix) + assert k is not None + fvs = asic_route_nhg_fvs(k) + assert fvs is not None + + # Cleanup + + # Remove second batch of routes + for i in range(10): + route_ipprefix = gen_ipprefix(self.MAX_ECMP_COUNT + i) + self.rt_ps._del(route_ipprefix) + + self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, 0) + self.asic_db.wait_for_n_keys(self.ASIC_RT_STR, self.asic_rts_count) + +class TestNextHopGroup(TestNextHopGroupBase): + + def test_route_nhg(self, dvs, dvs_route, testlog): + self.init_test(dvs, 3) + + rtprefix = "2.2.2.0/24" dvs_route.check_asicdb_deleted_route_entries([rtprefix]) # nexthop group without weight fvs = swsscommon.FieldValuePairs([("nexthop","10.0.0.1,10.0.0.3,10.0.0.5"), ("ifname", "Ethernet0,Ethernet4,Ethernet8")]) - ps.set(rtprefix, fvs) + self.rt_ps.set(rtprefix, fvs) # check if route was propagated to ASIC DB rtkeys = dvs_route.check_asicdb_route_entries([rtprefix]) # assert the route points to next hop group - fvs = asic_db.get_entry(self.ASIC_RT_STR, rtkeys[0]) + fvs = self.asic_db.get_entry(self.ASIC_RT_STR, rtkeys[0]) nhgid = fvs["SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID"] - fvs = asic_db.get_entry(self.ASIC_NHG_STR, nhgid) + fvs = self.asic_db.get_entry(self.ASIC_NHG_STR, nhgid) assert bool(fvs) - keys = asic_db.get_keys(self.ASIC_NHGM_STR) + keys = self.asic_db.get_keys(self.ASIC_NHGM_STR) assert len(keys) == 3 for k in keys: - fvs = asic_db.get_entry(self.ASIC_NHGM_STR, k) + fvs = self.asic_db.get_entry(self.ASIC_NHGM_STR, k) assert fvs["SAI_NEXT_HOP_GROUP_MEMBER_ATTR_NEXT_HOP_GROUP_ID"] == nhgid @@ -156,7 +649,7 @@ def test_route_nhg(self, dvs, dvs_route, testlog): assert fvs.get("SAI_NEXT_HOP_GROUP_MEMBER_ATTR_WEIGHT") is None # Remove route 2.2.2.0/24 - ps._del(rtprefix) + self.rt_ps._del(rtprefix) # Wait for route 2.2.2.0/24 to be removed dvs_route.check_asicdb_deleted_route_entries([rtprefix]) @@ -165,26 +658,26 @@ def test_route_nhg(self, dvs, dvs_route, testlog): fvs = swsscommon.FieldValuePairs([("nexthop","10.0.0.1,10.0.0.3,10.0.0.5"), ("ifname", "Ethernet0,Ethernet4,Ethernet8"), ("weight", "10,30")]) - ps.set(rtprefix, fvs) + self.rt_ps.set(rtprefix, fvs) # check if route was propagated to ASIC DB rtkeys = dvs_route.check_asicdb_route_entries([rtprefix]) # assert the route points to next hop group - fvs = asic_db.get_entry("ASIC_STATE:SAI_OBJECT_TYPE_ROUTE_ENTRY", rtkeys[0]) + fvs = self.asic_db.get_entry("ASIC_STATE:SAI_OBJECT_TYPE_ROUTE_ENTRY", rtkeys[0]) nhgid = fvs["SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID"] - fvs = asic_db.get_entry("ASIC_STATE:SAI_OBJECT_TYPE_NEXT_HOP_GROUP", nhgid) + fvs = self.asic_db.get_entry("ASIC_STATE:SAI_OBJECT_TYPE_NEXT_HOP_GROUP", nhgid) assert bool(fvs) - keys = asic_db.get_keys("ASIC_STATE:SAI_OBJECT_TYPE_NEXT_HOP_GROUP_MEMBER") + keys = self.asic_db.get_keys("ASIC_STATE:SAI_OBJECT_TYPE_NEXT_HOP_GROUP_MEMBER") assert len(keys) == 3 for k in keys: - fvs = asic_db.get_entry("ASIC_STATE:SAI_OBJECT_TYPE_NEXT_HOP_GROUP_MEMBER", k) + fvs = self.asic_db.get_entry("ASIC_STATE:SAI_OBJECT_TYPE_NEXT_HOP_GROUP_MEMBER", k) assert fvs["SAI_NEXT_HOP_GROUP_MEMBER_ATTR_NEXT_HOP_GROUP_ID"] == nhgid @@ -192,7 +685,7 @@ def test_route_nhg(self, dvs, dvs_route, testlog): assert fvs.get("SAI_NEXT_HOP_GROUP_MEMBER_ATTR_WEIGHT") is None # Remove route 2.2.2.0/24 - ps._del(rtprefix) + self.rt_ps._del(rtprefix) # Wait for route 2.2.2.0/24 to be removed dvs_route.check_asicdb_deleted_route_entries([rtprefix]) @@ -200,26 +693,26 @@ def test_route_nhg(self, dvs, dvs_route, testlog): fvs = swsscommon.FieldValuePairs([("nexthop","10.0.0.1,10.0.0.3,10.0.0.5"), ("ifname", "Ethernet0,Ethernet4,Ethernet8"), ("weight", "10,30,50")]) - ps.set(rtprefix, fvs) + self.rt_ps.set(rtprefix, fvs) # check if route was propagated to ASIC DB rtkeys = dvs_route.check_asicdb_route_entries([rtprefix]) # assert the route points to next hop group - fvs = asic_db.get_entry("ASIC_STATE:SAI_OBJECT_TYPE_ROUTE_ENTRY", rtkeys[0]) + fvs = self.asic_db.get_entry("ASIC_STATE:SAI_OBJECT_TYPE_ROUTE_ENTRY", rtkeys[0]) nhgid = fvs["SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID"] - fvs = asic_db.get_entry("ASIC_STATE:SAI_OBJECT_TYPE_NEXT_HOP_GROUP", nhgid) + fvs = self.asic_db.get_entry("ASIC_STATE:SAI_OBJECT_TYPE_NEXT_HOP_GROUP", nhgid) assert bool(fvs) - keys = asic_db.get_keys("ASIC_STATE:SAI_OBJECT_TYPE_NEXT_HOP_GROUP_MEMBER") + keys = self.asic_db.get_keys("ASIC_STATE:SAI_OBJECT_TYPE_NEXT_HOP_GROUP_MEMBER") assert len(keys) == 3 for k in keys: - fvs = asic_db.get_entry("ASIC_STATE:SAI_OBJECT_TYPE_NEXT_HOP_GROUP_MEMBER", k) + fvs = self.asic_db.get_entry("ASIC_STATE:SAI_OBJECT_TYPE_NEXT_HOP_GROUP_MEMBER", k) assert fvs["SAI_NEXT_HOP_GROUP_MEMBER_ATTR_NEXT_HOP_GROUP_ID"] == nhgid @@ -227,7 +720,7 @@ def test_route_nhg(self, dvs, dvs_route, testlog): nhid = fvs["SAI_NEXT_HOP_GROUP_MEMBER_ATTR_NEXT_HOP_ID"] weight = fvs["SAI_NEXT_HOP_GROUP_MEMBER_ATTR_WEIGHT"] - fvs = asic_db.get_entry("ASIC_STATE:SAI_OBJECT_TYPE_NEXT_HOP", nhid) + fvs = self.asic_db.get_entry("ASIC_STATE:SAI_OBJECT_TYPE_NEXT_HOP", nhid) nhip = fvs["SAI_NEXT_HOP_ATTR_IP"].split('.') expected_weight = int(nhip[3]) * 10 @@ -238,21 +731,21 @@ def test_route_nhg(self, dvs, dvs_route, testlog): fvs = swsscommon.FieldValuePairs([("nexthop","10.0.0.1,10.0.0.3,10.0.0.5"), ("ifname", "Ethernet0,Ethernet4,Ethernet8"), ("weight", "20,30,40")]) - ps.set(rtprefix2, fvs) + self.rt_ps.set(rtprefix2, fvs) # wait for route to be programmed time.sleep(1) - keys = asic_db.get_keys("ASIC_STATE:SAI_OBJECT_TYPE_NEXT_HOP_GROUP") + keys = self.asic_db.get_keys("ASIC_STATE:SAI_OBJECT_TYPE_NEXT_HOP_GROUP") assert len(keys) == 2 - keys = asic_db.get_keys("ASIC_STATE:SAI_OBJECT_TYPE_NEXT_HOP_GROUP_MEMBER") + keys = self.asic_db.get_keys("ASIC_STATE:SAI_OBJECT_TYPE_NEXT_HOP_GROUP_MEMBER") assert len(keys) == 6 # Remove route 3.3.3.0/24 - ps._del(rtprefix2) + self.rt_ps._del(rtprefix2) # Wait for route 3.3.3.0/24 to be removed dvs_route.check_asicdb_deleted_route_entries([rtprefix2]) @@ -260,124 +753,109 @@ def test_route_nhg(self, dvs, dvs_route, testlog): # bring links down one-by-one for i in [0, 1, 2]: - self.flap_intf(i, 'down', dvs) + self.flap_intf(i, 'down') - keys = asic_db.get_keys(self.ASIC_NHGM_STR) + keys = self.asic_db.get_keys(self.ASIC_NHGM_STR) assert len(keys) == 2 - i # bring links up one-by-one for i in [0, 1, 2]: - self.flap_intf(i, 'up', dvs) + self.flap_intf(i, 'up') - keys = asic_db.get_keys(self.ASIC_NHGM_STR) + keys = self.asic_db.get_keys(self.ASIC_NHGM_STR) assert len(keys) == i + 1 for k in keys: - fvs = asic_db.get_entry(self.ASIC_NHGM_STR, k) + fvs = self.asic_db.get_entry(self.ASIC_NHGM_STR, k) assert fvs["SAI_NEXT_HOP_GROUP_MEMBER_ATTR_NEXT_HOP_GROUP_ID"] == nhgid # Remove route 2.2.2.0/24 - ps._del(rtprefix) + self.rt_ps._del(rtprefix) # Wait for route 2.2.2.0/24 to be removed dvs_route.check_asicdb_deleted_route_entries([rtprefix]) def test_label_route_nhg(self, dvs, testlog): - for i in range(3): - self.config_intf(i, dvs) - - app_db = dvs.get_app_db() - lr_ps = swsscommon.ProducerStateTable(app_db.db_connection, "LABEL_ROUTE_TABLE") - - asic_db = dvs.get_asic_db() - asic_insegs_count = len(asic_db.get_keys(self.ASIC_INSEG_STR)) + self.init_test(dvs, 3) # add label route fvs = swsscommon.FieldValuePairs([("nexthop","10.0.0.1,10.0.0.3,10.0.0.5"), ("ifname", "Ethernet0,Ethernet4,Ethernet8")]) - lr_ps.set("10", fvs) + self.lr_ps.set("10", fvs) # check if route was propagated to ASIC DB - asic_db.wait_for_n_keys(self.ASIC_INSEG_STR, asic_insegs_count + 1) + self.asic_db.wait_for_n_keys(self.ASIC_INSEG_STR, self.asic_insgs_count + 1) - k = self.get_inseg_id('10', dvs) + k = self.get_inseg_id('10') assert k is not None # assert the route points to next hop group - fvs = asic_db.get_entry(self.ASIC_INSEG_STR, k) + fvs = self.asic_db.get_entry(self.ASIC_INSEG_STR, k) nhgid = fvs["SAI_INSEG_ENTRY_ATTR_NEXT_HOP_ID"] - fvs = asic_db.get_entry(self.ASIC_NHG_STR, nhgid) + fvs = self.asic_db.get_entry(self.ASIC_NHG_STR, nhgid) assert bool(fvs) - keys = asic_db.get_keys(self.ASIC_NHGM_STR) + keys = self.asic_db.get_keys(self.ASIC_NHGM_STR) assert len(keys) == 3 for k in keys: - fvs = asic_db.get_entry(self.ASIC_NHGM_STR, k) + fvs = self.asic_db.get_entry(self.ASIC_NHGM_STR, k) assert fvs["SAI_NEXT_HOP_GROUP_MEMBER_ATTR_NEXT_HOP_GROUP_ID"] == nhgid # bring links down one-by-one for i in [0, 1, 2]: - self.flap_intf(i, 'down', dvs) + self.flap_intf(i, 'down') - keys = asic_db.get_keys(self.ASIC_NHGM_STR) + keys = self.asic_db.get_keys(self.ASIC_NHGM_STR) assert len(keys) == 2 - i # bring links up one-by-one for i in [0, 1, 2]: - self.flap_intf(i, 'up', dvs) + self.flap_intf(i, 'up') - keys = asic_db.get_keys(self.ASIC_NHGM_STR) + keys = self.asic_db.get_keys(self.ASIC_NHGM_STR) assert len(keys) == i + 1 for k in keys: - fvs = asic_db.get_entry(self.ASIC_NHGM_STR, k) + fvs = self.asic_db.get_entry(self.ASIC_NHGM_STR, k) assert fvs["SAI_NEXT_HOP_GROUP_MEMBER_ATTR_NEXT_HOP_GROUP_ID"] == nhgid # Remove label route 10 - lr_ps._del("10") + self.lr_ps._del("10") # Wait for label route 10 to be removed - asic_db.wait_for_n_keys(self.ASIC_INSEG_STR, asic_insegs_count) + self.asic_db.wait_for_n_keys(self.ASIC_INSEG_STR, self.asic_insgs_count) def test_nhgorch_labeled_nhs(self, dvs, testlog): - for i in range(2): - self.config_intf(i, dvs) - - app_db = dvs.get_app_db() - asic_db = dvs.get_asic_db() - nhg_ps = swsscommon.ProducerStateTable(app_db.db_connection, "NEXTHOP_GROUP_TABLE") - asic_nhgs_count = len(asic_db.get_keys(self.ASIC_NHG_STR)) - asic_nhgms_count = len(asic_db.get_keys(self.ASIC_NHGM_STR)) - asic_nhs_count = len(asic_db.get_keys(self.ASIC_NHS_STR)) + self.init_test(dvs, 2) # Add a group containing labeled weighted NHs fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1,10.0.0.3'), ('mpls_nh', 'push1,push3'), ('ifname', 'Ethernet0,Ethernet4'), ('weight', '2,4')]) - nhg_ps.set('group1', fvs) - asic_db.wait_for_n_keys(self.ASIC_NHG_STR, asic_nhgs_count + 1) - asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, asic_nhgms_count + 2) + self.nhg_ps.set('group1', fvs) + self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count + 1) + self.asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, self.asic_nhgms_count + 2) # NhgOrch should create two next hops for the labeled ones - asic_db.wait_for_n_keys(self.ASIC_NHS_STR, asic_nhs_count + 2) + self.asic_db.wait_for_n_keys(self.ASIC_NHS_STR, self.asic_nhs_count + 2) # Assert the weights are properly set - nhgm_ids = self.get_nhgm_ids('group1', dvs) + nhgm_ids = self.get_nhgm_ids('group1') weights = [] for k in nhgm_ids: - fvs = asic_db.get_entry(self.ASIC_NHGM_STR, k) + fvs = self.asic_db.get_entry(self.ASIC_NHGM_STR, k) weights.append(fvs['SAI_NEXT_HOP_GROUP_MEMBER_ATTR_WEIGHT']) assert set(weights) == set(['2', '4']) @@ -385,125 +863,119 @@ def test_nhgorch_labeled_nhs(self, dvs, testlog): fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1'), ('mpls_nh', 'push1'), ('ifname', 'Ethernet0')]) - nhg_ps.set('group2', fvs) + self.nhg_ps.set('group2', fvs) # No new next hop should be added time.sleep(1) - assert len(asic_db.get_keys(self.ASIC_NHS_STR)) == asic_nhs_count + 2 + assert len(self.asic_db.get_keys(self.ASIC_NHS_STR)) == self.asic_nhs_count + 2 # Create a new single next hop with a different label fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1'), ('mpls_nh', 'push2'), ('ifname', 'Ethernet0')]) - nhg_ps.set('group3', fvs) + self.nhg_ps.set('group3', fvs) # A new next hop should be added - asic_db.wait_for_n_keys(self.ASIC_NHS_STR, asic_nhs_count + 3) + self.asic_db.wait_for_n_keys(self.ASIC_NHS_STR, self.asic_nhs_count + 3) # Delete group3 - nhg_ps._del('group3') + self.nhg_ps._del('group3') # Group3's NH should be deleted - asic_db.wait_for_n_keys(self.ASIC_NHS_STR, asic_nhs_count + 2) + self.asic_db.wait_for_n_keys(self.ASIC_NHS_STR, self.asic_nhs_count + 2) # Delete group2 - nhg_ps._del('group2') + self.nhg_ps._del('group2') # The number of NHs should be the same as they are still referenced by # group1 time.sleep(1) - asic_db.wait_for_n_keys(self.ASIC_NHS_STR, asic_nhs_count + 2) + self.asic_db.wait_for_n_keys(self.ASIC_NHS_STR, self.asic_nhs_count + 2) # Update group1 with no weights and both labeled and unlabeled NHs fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1,10.0.0.3'), ('mpls_nh', 'push2,na'), ('ifname', 'Ethernet0,Ethernet4')]) - nhg_ps.set('group1', fvs) + self.nhg_ps.set('group1', fvs) # Group members should be replaced and one NH should get deleted - asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, asic_nhgms_count + 2) - asic_db.wait_for_n_keys(self.ASIC_NHS_STR, asic_nhs_count + 1) + self.asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, self.asic_nhgms_count + 2) + self.asic_db.wait_for_n_keys(self.ASIC_NHS_STR, self.asic_nhs_count + 1) # Assert the weights of the NHGMs are the expected ones - nhgm_ids = self.get_nhgm_ids('group1', dvs) + nhgm_ids = self.get_nhgm_ids('group1') weights = [] for nhgm_id in nhgm_ids: - fvs = asic_db.get_entry(self.ASIC_NHGM_STR, nhgm_id) + fvs = self.asic_db.get_entry(self.ASIC_NHGM_STR, nhgm_id) weights.append(fvs['SAI_NEXT_HOP_GROUP_MEMBER_ATTR_WEIGHT']) assert weights == ['0', '0'] # Delete group1 - nhg_ps._del('group1') + self.nhg_ps._del('group1') # Wait for the group and it's members to be deleted - asic_db.wait_for_n_keys(self.ASIC_NHG_STR, asic_nhgs_count) - asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, asic_nhgms_count) + self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count) + self.asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, self.asic_nhgms_count) # The two next hops should also get deleted - asic_db.wait_for_n_keys(self.ASIC_NHS_STR, asic_nhs_count) + self.asic_db.wait_for_n_keys(self.ASIC_NHS_STR, self.asic_nhs_count) def test_nhgorch_excp_group_cases(self, dvs, testlog): - for i in range(3): - self.config_intf(i, dvs) - - app_db = dvs.get_app_db() - asic_db = dvs.get_asic_db() - nhg_ps = swsscommon.ProducerStateTable(app_db.db_connection, "NEXTHOP_GROUP_TABLE") - rt_ps = swsscommon.ProducerStateTable(app_db.db_connection, "ROUTE_TABLE") + self.init_test(dvs, 3) def get_nhg_keys(): - return asic_db.get_keys(self.ASIC_NHG_STR) + return self.asic_db.get_keys(self.ASIC_NHG_STR) def get_nhgm_keys(): - return asic_db.get_keys(self.ASIC_NHGM_STR) + return self.asic_db.get_keys(self.ASIC_NHGM_STR) def get_rt_keys(): - return asic_db.get_keys(self.ASIC_RT_STR) + return self.asic_db.get_keys(self.ASIC_RT_STR) # Count existing objects prev_nhg_keys = get_nhg_keys() - asic_nhgs_count = len(prev_nhg_keys) - asic_nhgms_count = len(get_nhgm_keys()) - asic_nhs_count = len(asic_db.get_keys(self.ASIC_NHS_STR)) + self.asic_nhgs_count = len(prev_nhg_keys) + self.asic_nhgms_count = len(get_nhgm_keys()) + self.asic_nhs_count = len(self.asic_db.get_keys(self.ASIC_NHS_STR)) # Remove a group that does not exist - nhg_ps._del("group1") + self.nhg_ps._del("group1") time.sleep(1) - assert len(get_nhg_keys()) == asic_nhgs_count + assert len(get_nhg_keys()) == self.asic_nhgs_count # Create a next hop group with a member that does not exist - should fail fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1,10.0.0.3,10.0.0.63'), ("ifname", "Ethernet0,Ethernet4,Ethernet124")]) - nhg_ps.set("group1", fvs) + self.nhg_ps.set("group1", fvs) time.sleep(1) - assert len(asic_db.get_keys(self.ASIC_NHG_STR)) == asic_nhgs_count + assert len(self.asic_db.get_keys(self.ASIC_NHG_STR)) == self.asic_nhgs_count # Issue an update for this next hop group that doesn't yet exist, # which contains only valid NHs. This will overwrite the previous # operation and create the group. fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1,10.0.0.5'), ("ifname", "Ethernet0,Ethernet8")]) - nhg_ps.set("group1", fvs) - asic_db.wait_for_n_keys(self.ASIC_NHG_STR, asic_nhgs_count + 1) - asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, asic_nhgms_count + 2) + self.nhg_ps.set("group1", fvs) + self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count + 1) + self.asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, self.asic_nhgms_count + 2) # Check the group has it's two members - for nhgid in asic_db.get_keys(self.ASIC_NHG_STR): + for nhgid in self.asic_db.get_keys(self.ASIC_NHG_STR): if nhgid not in prev_nhg_keys: break count = 0 - for k in asic_db.get_keys(self.ASIC_NHGM_STR): - fvs = asic_db.get_entry(self.ASIC_NHGM_STR, k) + for k in self.asic_db.get_keys(self.ASIC_NHGM_STR): + fvs = self.asic_db.get_entry(self.ASIC_NHGM_STR, k) if fvs['SAI_NEXT_HOP_GROUP_MEMBER_ATTR_NEXT_HOP_GROUP_ID'] == nhgid: count += 1 assert count == 2 # Add a route referencing the new group - asic_rts_count = len(get_rt_keys()) + self.asic_rts_count = len(get_rt_keys()) fvs = swsscommon.FieldValuePairs([('nexthop_group', 'group1')]) - rt_ps.set('2.2.2.0/24', fvs) - asic_db.wait_for_n_keys(self.ASIC_RT_STR, asic_rts_count + 1) + self.rt_ps.set('2.2.2.0/24', fvs) + self.asic_db.wait_for_n_keys(self.ASIC_RT_STR, self.asic_rts_count + 1) # Get the route key for rt_key in get_rt_keys(): @@ -513,699 +985,154 @@ def get_rt_keys(): break # Try removing the group while it still has references - should fail - nhg_ps._del('group1') + self.nhg_ps._del('group1') time.sleep(1) - assert len(get_nhg_keys()) == asic_nhgs_count + 1 + assert len(get_nhg_keys()) == self.asic_nhgs_count + 1 # Create a new group fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1,10.0.0.3'), ('ifname', 'Ethernet0,Ethernet4')]) - nhg_ps.set("group2", fvs) - asic_db.wait_for_n_keys(self.ASIC_NHG_STR, asic_nhgs_count + 2) - asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, asic_nhgms_count + 4) + self.nhg_ps.set("group2", fvs) + self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count + 2) + self.asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, self.asic_nhgms_count + 4) # Update the route to point to the new group fvs = swsscommon.FieldValuePairs([('nexthop_group', 'group2')]) - rt_ps.set('2.2.2.0/24', fvs) + self.rt_ps.set('2.2.2.0/24', fvs) # The first group should have got deleted - asic_db.wait_for_n_keys(self.ASIC_NHG_STR, asic_nhgs_count + 1) - asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, asic_nhgms_count + 2) + self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count + 1) + self.asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, self.asic_nhgms_count + 2) # The route's group should have changed to the new one - assert asic_db.get_entry(self.ASIC_RT_STR, rt_key)['SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID'] != nhgid + assert self.asic_db.get_entry(self.ASIC_RT_STR, rt_key)['SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID'] != nhgid # Update the route with routeOrch's owned next hop group - nhgid = asic_db.get_entry(self.ASIC_RT_STR, rt_key)['SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID'] + nhgid = self.asic_db.get_entry(self.ASIC_RT_STR, rt_key)['SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID'] fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1,10.0.0.3'), ('ifname', 'Ethernet0,Ethernet4')]) - rt_ps.set('2.2.2.0/24', fvs) + self.rt_ps.set('2.2.2.0/24', fvs) - asic_db.wait_for_n_keys(self.ASIC_NHG_STR, asic_nhgs_count + 2) - asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, asic_nhgms_count + 4) + self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count + 2) + self.asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, self.asic_nhgms_count + 4) # Assert the next hop group ID changed time.sleep(1) - assert asic_db.get_entry(self.ASIC_RT_STR, rt_key)['SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID'] != nhgid - nhgid = asic_db.get_entry(self.ASIC_RT_STR, rt_key)['SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID'] + assert self.asic_db.get_entry(self.ASIC_RT_STR, rt_key)['SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID'] != nhgid + nhgid = self.asic_db.get_entry(self.ASIC_RT_STR, rt_key)['SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID'] # Update the route to point back to group2 fvs = swsscommon.FieldValuePairs([('nexthop_group', 'group2')]) - rt_ps.set('2.2.2.0/24', fvs) + self.rt_ps.set('2.2.2.0/24', fvs) # The routeOrch's owned next hop group should get deleted - asic_db.wait_for_n_keys(self.ASIC_NHG_STR, asic_nhgs_count + 1) - asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, asic_nhgms_count + 2) + self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count + 1) + self.asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, self.asic_nhgms_count + 2) # Assert the route points back to group2 - assert asic_db.get_entry(self.ASIC_RT_STR, rt_key)['SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID'] != nhgid + assert self.asic_db.get_entry(self.ASIC_RT_STR, rt_key)['SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID'] != nhgid # Create a new group with the same members as group2 - nhgid = asic_db.get_entry(self.ASIC_RT_STR, rt_key)['SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID'] + nhgid = self.asic_db.get_entry(self.ASIC_RT_STR, rt_key)['SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID'] fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1,10.0.0.3'), ('ifname', 'Ethernet0,Ethernet4')]) - nhg_ps.set("group1", fvs) - asic_db.wait_for_n_keys(self.ASIC_NHG_STR, asic_nhgs_count + 2) - asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, asic_nhgms_count + 4) + self.nhg_ps.set("group1", fvs) + self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count + 2) + self.asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, self.asic_nhgms_count + 4) # Update the route to point to the new group fvs = swsscommon.FieldValuePairs([('nexthop_group', 'group1')]) - rt_ps.set('2.2.2.0/24', fvs) + self.rt_ps.set('2.2.2.0/24', fvs) time.sleep(1) # Assert the next hop group ID changed - assert asic_db.get_entry(self.ASIC_RT_STR, rt_key)['SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID'] != nhgid + assert self.asic_db.get_entry(self.ASIC_RT_STR, rt_key)['SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID'] != nhgid # Remove the route - rt_ps._del('2.2.2.0/24') - asic_db.wait_for_n_keys(self.ASIC_RT_STR, asic_rts_count) + self.rt_ps._del('2.2.2.0/24') + self.asic_db.wait_for_n_keys(self.ASIC_RT_STR, self.asic_rts_count) # Remove the groups - nhg_ps._del('group1') - nhg_ps._del('group2') + self.nhg_ps._del('group1') + self.nhg_ps._del('group2') - asic_db.wait_for_n_keys(self.ASIC_NHG_STR, asic_nhgs_count) - asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, asic_nhgms_count) + self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count) + self.asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, self.asic_nhgms_count) # Create a route with labeled NHs fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1,10.0.0.3'), ('mpls_nh', 'push1,push3'), ('ifname', 'Ethernet0,Ethernet4'), ('weight', '2,4')]) - rt_ps.set('2.2.2.0/24', fvs) - asic_db.wait_for_n_keys(self.ASIC_RT_STR, asic_rts_count + 1) + self.rt_ps.set('2.2.2.0/24', fvs) + self.asic_db.wait_for_n_keys(self.ASIC_RT_STR, self.asic_rts_count + 1) # A NHG should be created - asic_db.wait_for_n_keys(self.ASIC_NHG_STR, asic_nhgs_count + 1) + self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count + 1) # Two new next hops should be created - asic_db.wait_for_n_keys(self.ASIC_NHS_STR, asic_nhs_count + 2) + self.asic_db.wait_for_n_keys(self.ASIC_NHS_STR, self.asic_nhs_count + 2) # Create a NHG with the same details - nhg_ps.set('group1', fvs) - asic_db.wait_for_n_keys(self.ASIC_NHG_STR, asic_nhgs_count + 2) + self.nhg_ps.set('group1', fvs) + self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count + 2) # No new next hops should be created - assert len(asic_db.get_keys(self.ASIC_NHS_STR)) == asic_nhs_count + 2 + assert len(self.asic_db.get_keys(self.ASIC_NHS_STR)) == self.asic_nhs_count + 2 # Update the group with a different NH fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1,10.0.0.3'), ('mpls_nh', 'push2,push3'), ('ifname', 'Ethernet0,Ethernet4'), ('weight', '2,4')]) - nhg_ps.set('group1', fvs) + self.nhg_ps.set('group1', fvs) # A new next hop should be created - asic_db.wait_for_n_keys(self.ASIC_NHS_STR, asic_nhs_count + 3) + self.asic_db.wait_for_n_keys(self.ASIC_NHS_STR, self.asic_nhs_count + 3) # group1 should be updated and a new NHG shouldn't be created time.sleep(1) - assert len(get_nhg_keys()) == asic_nhgs_count + 2 + assert len(get_nhg_keys()) == self.asic_nhgs_count + 2 # Remove the route - rt_ps._del('2.2.2.0/24') - asic_db.wait_for_n_keys(self.ASIC_NHG_STR, asic_nhgs_count + 1) + self.rt_ps._del('2.2.2.0/24') + self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count + 1) # One NH should become unreferenced and should be deleted. The other # one is still referenced by NhgOrch's owned NHG. - asic_db.wait_for_n_keys(self.ASIC_NHS_STR, asic_nhs_count + 2) + self.asic_db.wait_for_n_keys(self.ASIC_NHS_STR, self.asic_nhs_count + 2) # Remove the group - nhg_ps._del('group1') - asic_db.wait_for_n_keys(self.ASIC_NHG_STR, asic_nhgs_count) + self.nhg_ps._del('group1') + self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count) # Both new next hops should be deleted - asic_db.wait_for_n_keys(self.ASIC_NHS_STR, asic_nhs_count) + self.asic_db.wait_for_n_keys(self.ASIC_NHS_STR, self.asic_nhs_count) # Add a route with a NHG that does not exist fvs = swsscommon.FieldValuePairs([('nexthop_group', 'group1')]) - rt_ps.set('2.2.2.0/24', fvs) + self.rt_ps.set('2.2.2.0/24', fvs) time.sleep(1) - assert asic_rts_count == len(asic_db.get_keys(self.ASIC_RT_STR)) + assert self.asic_rts_count == len(self.asic_db.get_keys(self.ASIC_RT_STR)) # Remove the pending route - rt_ps._del('2.2.2.0/24') - - def test_route_nhg_exhaust(self, dvs, testlog): - """ - Test the situation of exhausting ECMP group, assume SAI_SWITCH_ATTR_NUMBER_OF_ECMP_GROUPS is 512 - - In order to achieve that, we will config - 1. 9 ports - 2. 512 routes with different nexthop group - - See Also - -------- - SwitchStateBase::set_number_of_ecmp_groups() - https://github.com/Azure/sonic-sairedis/blob/master/vslib/src/SwitchStateBase.cpp - - """ - - # TODO: check ECMP 512 - - def gen_ipprefix(r): - """ Construct route like 2.X.X.0/24 """ - ip = ipaddress.IPv4Address(IP_INTEGER_BASE + r * 256) - ip = str(ip) - ipprefix = ip + "/24" - return ipprefix - - def gen_nhg_fvs(binary): - nexthop = [] - ifname = [] - for i in range(MAX_PORT_COUNT): - if binary[i] == '1': - nexthop.append(self.peer_ip(i)) - ifname.append(self.port_name(i)) - - nexthop = ','.join(nexthop) - ifname = ','.join(ifname) - fvs = swsscommon.FieldValuePairs([("nexthop", nexthop), ("ifname", ifname)]) - return fvs - - def asic_route_nhg_fvs(k): - fvs = asic_db.get_entry(self.ASIC_RT_STR, k) - if not fvs: - return None - - print(fvs) - nhgid = fvs.get("SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID") - if nhgid is None: - return None - - fvs = asic_db.get_entry(self.ASIC_NHG_STR, nhgid) - return fvs - - MAX_ECMP_COUNT = 512 - MAX_PORT_COUNT = 10 - if sys.version_info < (3, 0): - IP_INTEGER_BASE = int(ipaddress.IPv4Address(unicode("2.2.2.0"))) - else: - IP_INTEGER_BASE = int(ipaddress.IPv4Address(str("2.2.2.0"))) - - for i in range(MAX_PORT_COUNT): - self.config_intf(i, dvs) - - app_db = dvs.get_app_db() - asic_db = dvs.get_asic_db() - ps = swsscommon.ProducerStateTable(app_db.db_connection, "ROUTE_TABLE") - - # Add first batch of routes with unique nexthop groups in AppDB - route_count = 0 - r = 0 - asic_routes_count = len(asic_db.get_keys(self.ASIC_RT_STR)) - while route_count < MAX_ECMP_COUNT: - r += 1 - fmt = '{{0:0{}b}}'.format(MAX_PORT_COUNT) - binary = fmt.format(r) - # We need at least 2 ports for a nexthop group - if binary.count('1') <= 1: - continue - fvs = gen_nhg_fvs(binary) - route_ipprefix = gen_ipprefix(route_count) - ps.set(route_ipprefix, fvs) - route_count += 1 - - # Wait and check ASIC DB the count of nexthop groups used - asic_db.wait_for_n_keys(self.ASIC_NHG_STR, MAX_ECMP_COUNT) - - # Wait and check ASIC DB the count of routes - asic_db.wait_for_n_keys(self.ASIC_RT_STR, asic_routes_count + MAX_ECMP_COUNT) - asic_routes_count += MAX_ECMP_COUNT - - # Add a route with labeled NHs - asic_nhs_count = len(asic_db.get_keys(self.ASIC_NHS_STR)) - route_ipprefix = gen_ipprefix(route_count) - fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1,10.0.0.3'), - ('mpls_nh', 'push1,push3'), - ('ifname', 'Ethernet0,Ethernet4')]) - ps.set(route_ipprefix, fvs) - route_count += 1 - - # A temporary route should be created - asic_db.wait_for_n_keys(self.ASIC_RT_STR, asic_routes_count + 1) - - # A NH should be elected as the temporary NHG and it should be created - # as it doesn't exist. - asic_db.wait_for_n_keys(self.ASIC_NHS_STR, asic_nhs_count + 1) - - # Delete the route. The route and the added labeled NH should be - # removed. - ps._del(route_ipprefix) - route_count -= 1 - asic_db.wait_for_n_keys(self.ASIC_RT_STR, asic_routes_count) - asic_db.wait_for_n_keys(self.ASIC_NHS_STR, asic_nhs_count) - - # Add second batch of routes with unique nexthop groups in AppDB - # Add more routes with new nexthop group in AppDBdd - route_ipprefix = gen_ipprefix(route_count) - base_ipprefix = route_ipprefix - base = route_count - route_count = 0 - while route_count < 10: - r += 1 - fmt = '{{0:0{}b}}'.format(MAX_PORT_COUNT) - binary = fmt.format(r) - # We need at least 2 ports for a nexthop group - if binary.count('1') <= 1: - continue - fvs = gen_nhg_fvs(binary) - route_ipprefix = gen_ipprefix(base + route_count) - ps.set(route_ipprefix, fvs) - route_count += 1 - last_ipprefix = route_ipprefix - - # Wait until we get expected routes and check ASIC DB on the count of nexthop groups used, and it should not increase - asic_db.wait_for_n_keys(self.ASIC_RT_STR, asic_routes_count + 10) - asic_db.wait_for_n_keys(self.ASIC_NHG_STR, MAX_ECMP_COUNT) - - # Check the route points to next hop group - # Note: no need to wait here - k = self.get_route_id("2.2.2.0/24", dvs) - assert k is not None - fvs = asic_route_nhg_fvs(k) - assert fvs is not None - - # Check the second batch does not point to next hop group - k = self.get_route_id(base_ipprefix, dvs) - assert k is not None - fvs = asic_route_nhg_fvs(k) - assert not(fvs) - - # Remove first batch of routes with unique nexthop groups in AppDB - route_count = 0 - r = 0 - while route_count < MAX_ECMP_COUNT: - r += 1 - fmt = '{{0:0{}b}}'.format(MAX_PORT_COUNT) - binary = fmt.format(r) - # We need at least 2 ports for a nexthop group - if binary.count('1') <= 1: - continue - route_ipprefix = gen_ipprefix(route_count) - ps._del(route_ipprefix) - route_count += 1 - asic_routes_count -= MAX_ECMP_COUNT - - # Wait and check the second batch points to next hop group - # Check ASIC DB on the count of nexthop groups used, and it should not increase or decrease - asic_db.wait_for_n_keys(self.ASIC_NHG_STR, 10) - k = self.get_route_id(base_ipprefix, dvs) - assert k is not None - fvs = asic_route_nhg_fvs(k) - assert fvs is not None - k = self.get_route_id(last_ipprefix, dvs) - assert k is not None - fvs = asic_route_nhg_fvs(k) - assert fvs is not None - - # Cleanup - - # Remove second batch of routes - for i in range(10): - route_ipprefix = gen_ipprefix(MAX_ECMP_COUNT + i) - ps._del(route_ipprefix) - - asic_db.wait_for_n_keys(self.ASIC_NHG_STR, 0) - asic_db.wait_for_n_keys(self.ASIC_RT_STR, asic_routes_count) - - def test_nhgorch_nhg_exhaust(self, dvs, testlog): - app_db = dvs.get_app_db() - asic_db = dvs.get_asic_db() - nhg_ps = swsscommon.ProducerStateTable(app_db.db_connection, "NEXTHOP_GROUP_TABLE") - rt_ps = swsscommon.ProducerStateTable(app_db.db_connection, "ROUTE_TABLE") - - MAX_ECMP_COUNT = 512 - MAX_PORT_COUNT = 10 - - r = 0 - fmt = '{{0:0{}b}}'.format(MAX_PORT_COUNT) - asic_nhgs_count = len(asic_db.get_keys(self.ASIC_NHG_STR)) - nhg_count = asic_nhgs_count - first_valid_nhg = nhg_count - - def gen_nhg_fvs(binary): - nexthop = [] - ifname = [] - - for i in range(MAX_PORT_COUNT): - if binary[i] == '1': - nexthop.append(self.peer_ip(i)) - ifname.append(self.port_name(i)) - - nexthop = ','.join(nexthop) - ifname = ','.join(ifname) - fvs = swsscommon.FieldValuePairs([("nexthop", nexthop), ("ifname", ifname)]) - - return fvs - - def gen_nhg_index(nhg_number): - return "group{}".format(nhg_number) - - def gen_valid_binary(): - nonlocal r - - while True: - r += 1 - binary = fmt.format(r) - # We need at least 2 ports for a nexthop group - if binary.count('1') <= 1: - continue - return binary - - def create_temp_nhg(): - nonlocal nhg_count - - binary = gen_valid_binary() - nhg_fvs = gen_nhg_fvs(binary) - nhg_index = gen_nhg_index(nhg_count) - nhg_ps.set(nhg_index, nhg_fvs) - nhg_count += 1 - - return nhg_index, binary - - def delete_nhg(): - nonlocal first_valid_nhg - - del_nhg_index = gen_nhg_index(first_valid_nhg) - del_nhg_id = asic_nhgs[del_nhg_index] - - nhg_ps._del(del_nhg_index) - asic_nhgs.pop(del_nhg_index) - first_valid_nhg += 1 - - return del_nhg_id - - def nhg_exists(nhg_index): - return self.get_nhg_id(nhg_index, dvs) is not None - - # Create interfaces - for i in range(MAX_PORT_COUNT): - self.config_intf(i, dvs) - - asic_nhgs = {} - - # Add first batch of next hop groups to reach the NHG limit - while nhg_count < MAX_ECMP_COUNT: - r += 1 - binary = fmt.format(r) - # We need at least 2 ports for a nexthop group - if binary.count('1') <= 1: - continue - nhg_fvs = gen_nhg_fvs(binary) - nhg_index = gen_nhg_index(nhg_count) - nhg_ps.set(nhg_index, nhg_fvs) - - # Wait for the group to be added - asic_db.wait_for_n_keys(self.ASIC_NHG_STR, asic_nhgs_count + nhg_count + 1) - - # Save the NHG index/ID pair - asic_nhgs[nhg_index] = self.get_nhg_id(nhg_index, dvs) - - nhg_count += 1 - - # Add a new next hop group - it should create a temporary one instead - prev_nhgs = asic_db.get_keys(self.ASIC_NHG_STR) - nhg_index, _ = create_temp_nhg() - - # Save the temporary NHG's SAI ID - time.sleep(1) - nhg_id = self.get_nhg_id(nhg_index, dvs) - - # Assert no new group has been added - asic_db.wait_for_n_keys(self.ASIC_NHG_STR, MAX_ECMP_COUNT) - - # Assert the same NHGs are in ASIC DB - assert prev_nhgs == asic_db.get_keys(self.ASIC_NHG_STR) - - # Delete an existing next hop group - del_nhg_id = delete_nhg() - - # Wait for the key to be deleted - asic_db.wait_for_deleted_entry(self.ASIC_NHG_STR, del_nhg_id) - - # Wait for the temporary group to be promoted and replace the deleted - # NHG - asic_db.wait_for_n_keys(self.ASIC_NHG_STR, MAX_ECMP_COUNT) - - # Assert the SAI ID of the previously temporary NHG has been updated - assert nhg_id != self.get_nhg_id(nhg_index, dvs) - - # Save the promoted NHG index/ID - asic_nhgs[nhg_index] = self.get_nhg_id(nhg_index, dvs) - - # Update a group - binary = gen_valid_binary() - nhg_fvs = gen_nhg_fvs(binary) - nhg_index = gen_nhg_index(first_valid_nhg) - - # Save the previous members - prev_nhg_members = self.get_nhgm_ids(nhg_index, dvs) - nhg_ps.set(nhg_index, nhg_fvs) - - # Wait a second so the NHG members get updated - time.sleep(1) - - # Assert the group was updated by checking it's members - assert self.get_nhgm_ids(nhg_index, dvs) != prev_nhg_members - - # Create a new temporary group - nhg_index, _ = create_temp_nhg() - time.sleep(1) - - # Delete the temporary group - nhg_ps._del(nhg_index) - - # Assert the NHG does not exist anymore - assert not nhg_exists(nhg_index) - - # Assert the number of groups is the same - asic_db.wait_for_n_keys(self.ASIC_NHG_STR, MAX_ECMP_COUNT) - - # Create a new temporary group - nhg_index, binary = create_temp_nhg() - - # Save the number of group members - binary_count = binary.count('1') - - # Update the temporary group with a different number of members - while True: - r += 1 - binary = fmt.format(r) - # We need at least 2 ports for a nexthop group - if binary.count('1') <= 1 or binary.count('1') == binary_count: - continue - binary_count = binary.count('1') - break - nhg_fvs = gen_nhg_fvs(binary) - nhg_ps.set(nhg_index, nhg_fvs) - - # Delete a group - del_nhg_id = delete_nhg() - - # Wait for the group to be deleted - asic_db.wait_for_deleted_entry(self.ASIC_NHG_STR, del_nhg_id) - - # The temporary group should be promoted - asic_db.wait_for_n_keys(self.ASIC_NHG_STR, MAX_ECMP_COUNT) - - # Save the promoted NHG index/ID - asic_nhgs[nhg_index] = self.get_nhg_id(nhg_index, dvs) - - # Assert it has the updated details by checking the number of members - assert len(self.get_nhgm_ids(nhg_index, dvs)) == binary_count - - # Add a route - asic_rts_count = len(asic_db.get_keys(self.ASIC_RT_STR)) - - rt_fvs = swsscommon.FieldValuePairs([('nexthop_group', nhg_index)]) - rt_ps.set('2.2.2.0/24', rt_fvs) - - # Assert the route is created - asic_db.wait_for_n_keys(self.ASIC_RT_STR, asic_rts_count + 1) - - # Save the previous NHG ID - prev_nhg_id = asic_nhgs[nhg_index] - - # Create a new temporary group - nhg_index, binary = create_temp_nhg() - - # Get the route ID - rt_id = self.get_route_id('2.2.2.0/24', dvs) - assert rt_id != None - - # Update the route to point to the temporary NHG - rt_fvs = swsscommon.FieldValuePairs([('nexthop_group', nhg_index)]) - rt_ps.set('2.2.2.0/24', rt_fvs) - - # Wait for the route to change it's NHG ID - asic_db.wait_for_field_negative_match(self.ASIC_RT_STR, - rt_id, - {'SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID': prev_nhg_id}) - - # Save the new route NHG ID - prev_nhg_id = asic_db.get_entry(self.ASIC_RT_STR, rt_id)['SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID'] - - # Update the temporary NHG with one that has different NHs - - # Create a new binary that uses the other interfaces than the previous - # binary was using - new_binary = [] - - for i in range(len(binary)): - if binary[i] == '1': - new_binary.append('0') - else: - new_binary.append('1') - - binary = ''.join(new_binary) - assert binary.count('1') > 1 - - nhg_fvs = gen_nhg_fvs(binary) - nhg_ps.set(nhg_index, nhg_fvs) - - # The NHG ID of the route should change - asic_db.wait_for_field_negative_match(self.ASIC_RT_STR, - rt_id, - {'SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID': prev_nhg_id}) - - # Delete a NHG. - del_nhg_id = delete_nhg() - - # Wait for the NHG to be deleted - asic_db.wait_for_deleted_entry(self.ASIC_NHG_STR, del_nhg_id) - - # The temporary group should get promoted. - asic_db.wait_for_n_keys(self.ASIC_NHG_STR, MAX_ECMP_COUNT) - - # Save the promoted NHG index/ID - asic_nhgs[nhg_index] = self.get_nhg_id(nhg_index, dvs) - - # Assert the NHGID of the route changed due to temporary group being - # promoted. - asic_db.wait_for_field_match(self.ASIC_RT_STR, - rt_id, - {'SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID': asic_nhgs[nhg_index]}) - - # Create a next hop group that contains labeled NHs that do not exist - # in NeighOrch - asic_nhs_count = len(asic_db.get_keys(self.ASIC_NHS_STR)) - fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1,10.0.0.3'), - ('mpls_nh', 'push1,push3'), - ('ifname', 'Ethernet0,Ethernet4')]) - nhg_index = gen_nhg_index(nhg_count) - nhg_ps.set(nhg_index, fvs) - nhg_count += 1 - - # A temporary next hop should be elected to represent the group and - # thus a new labeled next hop should be created - asic_db.wait_for_n_keys(self.ASIC_NHS_STR, asic_nhs_count + 1) - - # Delete a next hop group - delete_nhg() - - # The group should be promoted and the other labeled NH should also get - # created - asic_db.wait_for_n_keys(self.ASIC_NHG_STR, MAX_ECMP_COUNT) - asic_db.wait_for_n_keys(self.ASIC_NHS_STR, asic_nhs_count + 2) - - # Save the promoted NHG index/ID - asic_nhgs[nhg_index] = self.get_nhg_id(nhg_index, dvs) - - # Update the route with a RouteOrch's owned NHG - binary = gen_valid_binary() - nhg_fvs = gen_nhg_fvs(binary) - rt_ps.set('2.2.2.0/24', nhg_fvs) - - # Assert no new group has been added - asic_db.wait_for_n_keys(self.ASIC_NHG_STR, MAX_ECMP_COUNT) - - # Delete a next hop group - delete_nhg() - - # The temporary group should be promoted - asic_db.wait_for_n_keys(self.ASIC_NHG_STR, MAX_ECMP_COUNT) - - # Create a temporary NHG owned by NhgOrch - nhg_index, _ = create_temp_nhg() - - # Assert no new group has been added - asic_db.wait_for_n_keys(self.ASIC_NHG_STR, MAX_ECMP_COUNT) - - # Delete a next hop group - delete_nhg() - - # Check that the temporary group is promoted - asic_db.wait_for_n_keys(self.ASIC_NHG_STR, MAX_ECMP_COUNT) - - # Save the promoted NHG index/ID - asic_nhgs[nhg_index] = self.get_nhg_id(nhg_index, dvs) - - # Create a temporary NHG that contains only NHs that do not exist - nhg_fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.21,10.0.0.23'), - ('ifname', 'Ethernet40,Ethernet44')]) - nhg_index = gen_nhg_index(nhg_count) - nhg_count += 1 - nhg_ps.set(nhg_index, nhg_fvs) - - # Assert the group is not created - assert not nhg_exists(nhg_index) - - # Update the temporary NHG to a valid one - binary = gen_valid_binary() - nhg_fvs = gen_nhg_fvs(binary) - nhg_ps.set(nhg_index, nhg_fvs) - - # Assert the temporary group was updated and the group got created - nhg_id = self.get_nhg_id(nhg_index, dvs) - assert nhg_id is not None - - # Update the temporary NHG to an invalid one again - nhg_fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.21,10.0.0.23'), - ('ifname', 'Ethernet40,Ethernet44')]) - nhg_ps.set(nhg_index, nhg_fvs) - - # The update should fail and the temporary NHG should still be pointing - # to the old valid NH - assert self.get_nhg_id(nhg_index, dvs) == nhg_id - - # Delete the temporary group - nhg_ps._del(nhg_index) - - # Cleanup - - # Delete the route - rt_ps._del('2.2.2.0/24') - asic_db.wait_for_deleted_entry(self.ASIC_RT_STR, rt_id) - - # Delete the next hop groups - for k in asic_nhgs: - nhg_ps._del(k) - asic_db.wait_for_n_keys(self.ASIC_NHG_STR, asic_nhgs_count) + self.rt_ps._del('2.2.2.0/24') def test_nhgorch_multi_nh_group(self, dvs, testlog): - for i in range(4): - self.config_intf(i, dvs) - - app_db = dvs.get_app_db() - asic_db = dvs.get_asic_db() + self.init_test(dvs, 4) # create next hop group in APPL DB - nhg_ps = swsscommon.ProducerStateTable(app_db.db_connection, "NEXTHOP_GROUP_TABLE") - - prev_nhg_keys = asic_db.get_keys(self.ASIC_NHG_STR) - asic_nhgs_count = len(prev_nhg_keys) - asic_nhgms_count = len(asic_db.get_keys(self.ASIC_NHGM_STR)) + prev_nhg_keys = self.asic_db.get_keys(self.ASIC_NHG_STR) fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1,10.0.0.3,10.0.0.5'), ("ifname", "Ethernet0,Ethernet4,Ethernet8")]) - nhg_ps.set("group1", fvs) + self.nhg_ps.set("group1", fvs) # check if group was propagated to ASIC DB - asic_db.wait_for_n_keys(self.ASIC_NHG_STR, asic_nhgs_count + 1) - keys = asic_db.get_keys(self.ASIC_NHG_STR) + self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count + 1) + keys = self.asic_db.get_keys(self.ASIC_NHG_STR) found_nhg = False @@ -1218,12 +1145,12 @@ def test_nhgorch_multi_nh_group(self, dvs, testlog): # check if members were propagated to ASIC DB - asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, asic_nhgms_count + 3) - keys = asic_db.get_keys(self.ASIC_NHGM_STR) + self.asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, self.asic_nhgms_count + 3) + keys = self.asic_db.get_keys(self.ASIC_NHGM_STR) count = 0 for k in keys: - fvs = asic_db.get_entry(self.ASIC_NHGM_STR, k) + fvs = self.asic_db.get_entry(self.ASIC_NHGM_STR, k) if fvs["SAI_NEXT_HOP_GROUP_MEMBER_ATTR_NEXT_HOP_GROUP_ID"] == nhgid: count += 1 @@ -1232,302 +1159,269 @@ def test_nhgorch_multi_nh_group(self, dvs, testlog): # create route in APPL DB - rt_ps = swsscommon.ProducerStateTable(app_db.db_connection, "ROUTE_TABLE") - - asic_routes_count = len(asic_db.get_keys(self.ASIC_RT_STR)) - fvs = swsscommon.FieldValuePairs([("nexthop_group", "group1")]) - rt_ps.set("2.2.2.0/24", fvs) + self.rt_ps.set("2.2.2.0/24", fvs) # check if route was propagated to ASIC DB - asic_db.wait_for_n_keys(self.ASIC_RT_STR, asic_routes_count + 1) + self.asic_db.wait_for_n_keys(self.ASIC_RT_STR, self.asic_rts_count + 1) - k = self.get_route_id('2.2.2.0/24', dvs) + k = self.get_route_id('2.2.2.0/24') assert k is not None # assert the route points to next hop group - fvs = asic_db.get_entry(self.ASIC_RT_STR, k) + fvs = self.asic_db.get_entry(self.ASIC_RT_STR, k) assert fvs["SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID"] == nhgid # bring links down one-by-one for i in [0, 1, 2]: - self.flap_intf(i, 'down', dvs) + self.flap_intf(i, 'down') - keys = asic_db.get_keys(self.ASIC_NHGM_STR) + keys = self.asic_db.get_keys(self.ASIC_NHGM_STR) - assert len(keys) == (asic_nhgms_count + 2 - i) - assert bool(asic_db.get_entry(self.ASIC_NHG_STR, nhgid)) + assert len(keys) == (self.asic_nhgms_count + 2 - i) + assert bool(self.asic_db.get_entry(self.ASIC_NHG_STR, nhgid)) # bring links up one-by-one for i in [0, 1, 2]: - self.flap_intf(i, 'up', dvs) + self.flap_intf(i, 'up') - keys = asic_db.get_keys(self.ASIC_NHGM_STR) + keys = self.asic_db.get_keys(self.ASIC_NHGM_STR) - assert len(keys) == (asic_nhgms_count + i + 1) + assert len(keys) == (self.asic_nhgms_count + i + 1) count = 0 for k in keys: - fvs = asic_db.get_entry(self.ASIC_NHGM_STR, k) + fvs = self.asic_db.get_entry(self.ASIC_NHGM_STR, k) if fvs["SAI_NEXT_HOP_GROUP_MEMBER_ATTR_NEXT_HOP_GROUP_ID"] == nhgid: count += 1 assert count == i + 1 # Bring an interface down - self.flap_intf(1, 'down', dvs) + self.flap_intf(1, 'down') # One group member will get deleted - asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, 2) + self.asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, 2) # Create a group that contains a NH that uses the down link fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1,10.0.0.3'), ("ifname", "Ethernet0,Ethernet4")]) - nhg_ps.set('group2', fvs) + self.nhg_ps.set('group2', fvs) # The group should get created, but it will not contained the NH that # has the link down - asic_db.wait_for_n_keys(self.ASIC_NHG_STR, asic_nhgs_count + 2) - asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, asic_nhgms_count + 3) + self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count + 2) + self.asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, self.asic_nhgms_count + 3) # Update the NHG with one interface down fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.3,10.0.0.1'), ("ifname", "Ethernet4,Ethernet0")]) - nhg_ps.set("group1", fvs) + self.nhg_ps.set("group1", fvs) # Wait for group members to update - the group will contain only the # members that have their links up - asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, asic_nhgms_count + 2) + self.asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, self.asic_nhgms_count + 2) # Bring the interface up - self.flap_intf(1, 'up', dvs) + self.flap_intf(1, 'up') # Check that the missing member of group1 and group2 is being added - asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, asic_nhgms_count + 4) + self.asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, self.asic_nhgms_count + 4) # Remove group2 - nhg_ps._del('group2') - asic_db.wait_for_n_keys(self.ASIC_NHG_STR, asic_nhgs_count + 1) - asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, asic_nhgms_count + 2) + self.nhg_ps._del('group2') + self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count + 1) + self.asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, self.asic_nhgms_count + 2) # Create group2 with a NH that does not exist fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.3,10.0.0.63'), ("ifname", "Ethernet4,Ethernet124")]) - nhg_ps.set("group2", fvs) + self.nhg_ps.set("group2", fvs) # The groups should not be created time.sleep(1) - asic_db.wait_for_n_keys(self.ASIC_NHG_STR, asic_nhgs_count + 1) + self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count + 1) # Update group1 with a NH that does not exist fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.3,10.0.0.63'), ("ifname", "Ethernet4,Ethernet124")]) - nhg_ps.set("group1", fvs) + self.nhg_ps.set("group1", fvs) # The update should fail, leaving group1 with only the unremoved # members - asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, asic_nhgms_count + 1) + self.asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, self.asic_nhgms_count + 1) # Configure the missing NH's interface - self.config_intf(31, dvs) + self.config_intf(31) - # A couple more routes will be added to ASIC_DB - asic_routes_count += 2 + # A couple more routes will be added to ASIC DB + self.asic_rts_count += 2 # Group2 should get created and group1 should be updated - asic_db.wait_for_n_keys(self.ASIC_NHG_STR, asic_nhgs_count + 2) - asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, asic_nhgms_count + 4) + self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count + 2) + self.asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, self.asic_nhgms_count + 4) # Delete group2 - nhg_ps._del('group2') - asic_db.wait_for_n_keys(self.ASIC_NHG_STR, asic_nhgs_count + 1) + self.nhg_ps._del('group2') + self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count + 1) # Update the NHG, adding two new members fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1,10.0.0.3,10.0.0.5,10.0.0.7'), ("ifname", "Ethernet0,Ethernet4,Ethernet8,Ethernet12")]) - nhg_ps.set("group1", fvs) + self.nhg_ps.set("group1", fvs) # Wait for members to be added - asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, asic_nhgms_count + 4) + self.asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, self.asic_nhgms_count + 4) # Update the group to one NH only fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1'), ("ifname", "Ethernet0")]) - nhg_ps.set("group1", fvs) + self.nhg_ps.set("group1", fvs) time.sleep(1) - assert bool(asic_db.get_entry(self.ASIC_NHG_STR, nhgid)) - asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, asic_nhgms_count + 1) + assert bool(self.asic_db.get_entry(self.ASIC_NHG_STR, nhgid)) + self.asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, self.asic_nhgms_count + 1) # Remove route 2.2.2.0/24 - rt_ps._del("2.2.2.0/24") + self.rt_ps._del("2.2.2.0/24") # Remove group1 - nhg_ps._del("group1") + self.nhg_ps._del("group1") def test_nhgorch_single_nh_group(self, dvs, testlog): - for i in range(2): - self.config_intf(i, dvs) - - app_db = dvs.get_app_db() - asic_db = dvs.get_asic_db() - - # Count existing objects - asic_nhgs_count = len(asic_db.get_keys(self.ASIC_NHG_STR)) + self.init_test(dvs, 2) # Create single next hop group in APPL DB - nhg_ps = swsscommon.ProducerStateTable(app_db.db_connection, "NEXTHOP_GROUP_TABLE") fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1'), ("ifname", "Ethernet0")]) - nhg_ps.set("group1", fvs) + self.nhg_ps.set("group1", fvs) time.sleep(1) # Check that the group was created in ASIC DB - asic_db.wait_for_n_keys(self.ASIC_NHG_STR, asic_nhgs_count + 1) + self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count + 1) # Get the NHG ID. - nhgid = self.get_nhg_id("group1", dvs) + nhgid = self.get_nhg_id("group1") # create route in APPL DB - rt_ps = swsscommon.ProducerStateTable(app_db.db_connection, "ROUTE_TABLE") - - asic_routes_count = len(asic_db.get_keys(self.ASIC_RT_STR)) - fvs = swsscommon.FieldValuePairs([("nexthop_group", "group1")]) - rt_ps.set("2.2.2.0/24", fvs) + self.rt_ps.set("2.2.2.0/24", fvs) # check if route was propagated to ASIC DB - asic_db.wait_for_n_keys(self.ASIC_RT_STR, asic_routes_count + 1) + self.asic_db.wait_for_n_keys(self.ASIC_RT_STR, self.asic_rts_count + 1) - k = self.get_route_id('2.2.2.0/24', dvs) + k = self.get_route_id('2.2.2.0/24') # assert the route points to next hop group - fvs = asic_db.get_entry(self.ASIC_RT_STR, k) + fvs = self.asic_db.get_entry(self.ASIC_RT_STR, k) assert fvs["SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID"] == nhgid # Bring group's link down - self.flap_intf(0, 'down', dvs) + self.flap_intf(0, 'down') # Check that the group still has the same ID and everything works - assert nhgid == self.get_nhg_id('group1', dvs) + assert nhgid == self.get_nhg_id('group1') # Bring group's link back up - self.flap_intf(0, 'up', dvs) + self.flap_intf(0, 'up') # Check that the group still has the same ID and everything works - assert nhgid == self.get_nhg_id('group1', dvs) + assert nhgid == self.get_nhg_id('group1') # Bring an interface down - self.flap_intf(1, 'down', dvs) + self.flap_intf(1, 'down') # Create group2 pointing to the NH which's link is down fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.3'), ("ifname", "Ethernet4")]) - nhg_ps.set("group2", fvs) + self.nhg_ps.set("group2", fvs) - # The group should be created. To test this, add a route pointing to - # it. If the group exists, the route will be created as well. - fvs = swsscommon.FieldValuePairs([("nexthop_group", "group2")]) - rt_ps.set('2.2.4.0/24', fvs) - asic_db.wait_for_n_keys(self.ASIC_RT_STR, asic_routes_count + 2) + # The group should be created. + assert self.nhg_exists('group2') - # Delete the route and the group - rt_ps._del('2.2.4.0/24') - nhg_ps._del('group2') + # Delete the group + self.nhg_ps._del('group2') # Bring the interface back up - self.flap_intf(1, 'up', dvs) + self.flap_intf(1, 'up') # Create group2 pointing to a NH that does not exist fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.61'), ("ifname", "Ethernet120")]) - nhg_ps.set("group2", fvs) + self.nhg_ps.set("group2", fvs) - # The group should fail to be created. To test this, we add a route - # pointing to it. The route should not be created. - fvs = swsscommon.FieldValuePairs([("nexthop_group", "group2")]) - rt_ps.set('2.2.4.0/24', fvs) - time.sleep(1) - asic_db.wait_for_n_keys(self.ASIC_RT_STR, asic_routes_count + 1) + # The group should fail to be created. + assert not self.nhg_exists('group2') # Configure the NH's interface - self.config_intf(30, dvs) + self.config_intf(30) - # A couple of more routes will be added to ASIC_DB - asic_routes_count += 2 + # A couple more routes will be added to ASIC DB + self.asic_rts_count += 2 - # The group should be created, so the route should be added - asic_db.wait_for_n_keys(self.ASIC_RT_STR, asic_routes_count + 2) + # The group should be created + assert self.nhg_exists('group2') - # Delete the route and the group - rt_ps._del('2.2.4.0/24') - nhg_ps._del('group2') - asic_db.wait_for_n_keys(self.ASIC_RT_STR, asic_routes_count + 1) + # Delete the group + self.nhg_ps._del('group2') # Update the group to a multiple NH group fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1,10.0.0.3'), ("ifname", "Ethernet0,Ethernet4")]) - nhg_ps.set("group1", fvs) + self.nhg_ps.set("group1", fvs) time.sleep(1) # Update group1 to point to another NH - nhgid = self.get_nhg_id('group1', dvs) + nhgid = self.get_nhg_id('group1') fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.3'), ("ifname", "Ethernet4")]) - nhg_ps.set("group1", fvs) - assert nhgid == self.get_nhg_id('group1', dvs) + self.nhg_ps.set("group1", fvs) + assert nhgid == self.get_nhg_id('group1') # Remove route 2.2.2.0/24 - rt_ps._del("2.2.2.0/24") + self.rt_ps._del("2.2.2.0/24") # Wait for route 2.2.2.0/24 to be removed - asic_db.wait_for_n_keys(self.ASIC_RT_STR, asic_routes_count) + self.asic_db.wait_for_n_keys(self.ASIC_RT_STR, self.asic_rts_count) # Update the group to a multiple NH group fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1,10.0.0.3'), ("ifname", "Ethernet0,Ethernet4")]) - nhg_ps.set("group1", fvs) - asic_db.wait_for_n_keys(self.ASIC_NHG_STR, asic_nhgs_count + 1) + self.nhg_ps.set("group1", fvs) + self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count + 1) # Create a route referencing group1 fvs = swsscommon.FieldValuePairs([("nexthop_group", "group1")]) - rt_ps.set('2.2.2.0/24', fvs) - asic_db.wait_for_n_keys(self.ASIC_RT_STR, asic_routes_count + 1) + self.rt_ps.set('2.2.2.0/24', fvs) + self.asic_db.wait_for_n_keys(self.ASIC_RT_STR, self.asic_rts_count + 1) - # Update the NHG to a single next hop - nhgid = self.get_nhg_id('group1', dvs) + # Update the NHG to a single next hop - should fail as it is being + # referenced + nhgid = self.get_nhg_id('group1') fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1'), ("ifname", "Ethernet0")]) - nhg_ps.set("group1", fvs) + self.nhg_ps.set("group1", fvs) time.sleep(1) - assert nhgid == self.get_nhg_id('group1', dvs) + assert nhgid == self.get_nhg_id('group1') # Remove route 2.2.2.0/24 - rt_ps._del("2.2.2.0/24") - asic_db.wait_for_n_keys(self.ASIC_RT_STR, asic_routes_count) + self.rt_ps._del("2.2.2.0/24") + self.asic_db.wait_for_n_keys(self.ASIC_RT_STR, self.asic_rts_count) # Remove group1 - nhg_ps._del("group1") + self.nhg_ps._del("group1") def test_nhgorch_label_route(self, dvs, testlog): - for i in range(4): - self.config_intf(i, dvs) - - app_db = dvs.get_app_db() - asic_db = dvs.get_asic_db() + self.init_test(dvs, 4) # create next hop group in APPL DB - nhg_ps = swsscommon.ProducerStateTable(app_db.db_connection, "NEXTHOP_GROUP_TABLE") - - prev_nhg_keys = asic_db.get_keys(self.ASIC_NHG_STR) - asic_nhgs_count = len(prev_nhg_keys) - asic_nhgms_count = len(asic_db.get_keys(self.ASIC_NHGM_STR)) + prev_nhg_keys = self.asic_db.get_keys(self.ASIC_NHG_STR) fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1,10.0.0.3,10.0.0.5'), ("ifname", "Ethernet0,Ethernet4,Ethernet8")]) - nhg_ps.set("group1", fvs) + self.nhg_ps.set("group1", fvs) # check if group was propagated to ASIC DB - asic_db.wait_for_n_keys(self.ASIC_NHG_STR, asic_nhgs_count + 1) - keys = asic_db.get_keys(self.ASIC_NHG_STR) + self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count + 1) + keys = self.asic_db.get_keys(self.ASIC_NHG_STR) found_nhg = False @@ -1540,12 +1434,12 @@ def test_nhgorch_label_route(self, dvs, testlog): # check if members were propagated to ASIC DB - asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, asic_nhgms_count + 3) - keys = asic_db.get_keys(self.ASIC_NHGM_STR) + self.asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, self.asic_nhgms_count + 3) + keys = self.asic_db.get_keys(self.ASIC_NHGM_STR) count = 0 for k in keys: - fvs = asic_db.get_entry(self.ASIC_NHGM_STR, k) + fvs = self.asic_db.get_entry(self.ASIC_NHGM_STR, k) if fvs["SAI_NEXT_HOP_GROUP_MEMBER_ATTR_NEXT_HOP_GROUP_ID"] == nhgid: count += 1 @@ -1554,87 +1448,81 @@ def test_nhgorch_label_route(self, dvs, testlog): # create prefix route in APPL DB - rt_ps = swsscommon.ProducerStateTable(app_db.db_connection, "ROUTE_TABLE") - - asic_routes_count = len(asic_db.get_keys(self.ASIC_RT_STR)) + self.asic_rts_count = len(self.asic_db.get_keys(self.ASIC_RT_STR)) fvs = swsscommon.FieldValuePairs([("nexthop_group", "group1")]) - rt_ps.set("2.2.2.0/24", fvs) + self.rt_ps.set("2.2.2.0/24", fvs) # check if route was propagated to ASIC DB - asic_db.wait_for_n_keys(self.ASIC_RT_STR, asic_routes_count + 1) + self.asic_db.wait_for_n_keys(self.ASIC_RT_STR, self.asic_rts_count + 1) - k = self.get_route_id('2.2.2.0/24', dvs) + k = self.get_route_id('2.2.2.0/24') # assert the route points to next hop group - fvs = asic_db.get_entry(self.ASIC_RT_STR, k) + fvs = self.asic_db.get_entry(self.ASIC_RT_STR, k) assert fvs["SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID"] == nhgid - # create label route in APPL DB pointing to the same next hop group - - lrt_ps = swsscommon.ProducerStateTable(app_db.db_connection, "LABEL_ROUTE_TABLE") - - asic_insegs_count = len(asic_db.get_keys(self.ASIC_INSEG_STR)) + # create label route in APPL DB pointing to the same next hop grou fvs = swsscommon.FieldValuePairs([("nexthop_group", "group1")]) - lrt_ps.set("20", fvs) + self.lr_ps.set("20", fvs) # check if label route was propagated to ASIC DB - asic_db.wait_for_n_keys(self.ASIC_INSEG_STR, asic_insegs_count + 1) + self.asic_db.wait_for_n_keys(self.ASIC_INSEG_STR, self.asic_insgs_count + 1) - k = self.get_inseg_id('20', dvs) + k = self.get_inseg_id('20') assert k is not None # assert the route points to next hop group - fvs = asic_db.get_entry(self.ASIC_INSEG_STR, k) + fvs = self.asic_db.get_entry(self.ASIC_INSEG_STR, k) assert fvs["SAI_INSEG_ENTRY_ATTR_NEXT_HOP_ID"] == nhgid # bring links down one-by-one for i in [0, 1, 2]: - self.flap_intf(i, 'down', dvs) + self.flap_intf(i, 'down') - keys = asic_db.get_keys(self.ASIC_NHGM_STR) + keys = self.asic_db.get_keys(self.ASIC_NHGM_STR) - assert len(keys) == (asic_nhgms_count + 2 - i) - assert bool(asic_db.get_entry(self.ASIC_NHG_STR, nhgid)) + assert len(keys) == (self.asic_nhgms_count + 2 - i) + assert bool(self.asic_db.get_entry(self.ASIC_NHG_STR, nhgid)) # bring links up one-by-one for i in [0, 1, 2]: - self.flap_intf(i, 'up', dvs) + self.flap_intf(i, 'up') - keys = asic_db.get_keys(self.ASIC_NHGM_STR) + keys = self.asic_db.get_keys(self.ASIC_NHGM_STR) - assert len(keys) == (asic_nhgms_count + i + 1) + assert len(keys) == (self.asic_nhgms_count + i + 1) count = 0 for k in keys: - fvs = asic_db.get_entry(self.ASIC_NHGM_STR, k) + fvs = self.asic_db.get_entry(self.ASIC_NHGM_STR, k) if fvs["SAI_NEXT_HOP_GROUP_MEMBER_ATTR_NEXT_HOP_GROUP_ID"] == nhgid: count += 1 assert count == i + 1 # Bring an interface down - self.flap_intf(1, 'down', dvs) + self.flap_intf(1, 'down') # One group member will get deleted - asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, 2) + self.asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, 2) # Remove route 2.2.2.0/24 - rt_ps._del("2.2.2.0/24") + self.rt_ps._del("2.2.2.0/24") # Wait for route 2.2.2.0/24 to be removed - asic_db.wait_for_n_keys(self.ASIC_RT_STR, asic_routes_count) + self.asic_db.wait_for_n_keys(self.ASIC_RT_STR, self.asic_rts_count) # Remove label route 20 - lrt_ps._del("20") + self.lr_ps._del("20") # Wait for route 20 to be removed - asic_db.wait_for_n_keys(self.ASIC_INSEG_STR, asic_insegs_count) + self.asic_db.wait_for_n_keys(self.ASIC_INSEG_STR, self.asic_insgs_count) # Remove group1 - nhg_ps._del("group1") + self.nhg_ps._del("group1") # Add Dummy always-pass test at end as workaroud # for issue when Flaky fail on final test it invokes module tear-down before retrying From e25e6a4f7bd21e050ef5d6c8b8b86be06bd950f6 Mon Sep 17 00:00:00 2001 From: Alexandru Banu Date: Sat, 2 Oct 2021 21:19:16 +0300 Subject: [PATCH 15/51] Split test cases and remove redundant checks --- tests/test_nhg.py | 1066 ++++++++++++++++++++------------------------- 1 file changed, 461 insertions(+), 605 deletions(-) diff --git a/tests/test_nhg.py b/tests/test_nhg.py index 9f34bff583..ce3beccd6b 100644 --- a/tests/test_nhg.py +++ b/tests/test_nhg.py @@ -185,6 +185,9 @@ def delete_nhg(): return del_nhg_id + # Test scenario: + # - create a NHG and assert a NHG object doesn't get added to ASIC DB + # - delete a NHG and assert the newly created one is created in ASIC DB and its SAI ID changed def test_temporary_group_promotion(): # Add a new next hop group - it should create a temporary one instead prev_nhgs = self.asic_db.get_keys(self.ASIC_NHG_STR) @@ -216,6 +219,8 @@ def test_temporary_group_promotion(): # Save the promoted NHG index/ID self.asic_nhgs[nhg_index] = self.get_nhg_id(nhg_index) + # Test scenario: + # - update an existing NHG and assert the update is performed def test_group_update(): # Update a group binary = self.gen_valid_binary() @@ -232,6 +237,8 @@ def test_group_update(): # Assert the group was updated by checking it's members assert self.get_nhgm_ids(nhg_index) != prev_nhg_members + # Test scenario: + # - create and delete a NHG while the ASIC DB is full and assert nothing changes def test_create_delete_temporary(): # Create a new temporary group nhg_index, _ = create_temp_nhg() @@ -246,6 +253,10 @@ def test_create_delete_temporary(): # Assert the number of groups is the same self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.MAX_ECMP_COUNT) + # Test scenario: + # - create a temporary NHG + # - update the NHG with a different number of members + # - delete a NHG and assert the new one is added and it has the updated number of members def test_update_temporary_group(): # Create a new temporary group nhg_index, binary = create_temp_nhg() @@ -278,6 +289,11 @@ def test_update_temporary_group(): # Assert it has the updated details by checking the number of members assert len(self.get_nhgm_ids(nhg_index)) == binary_count + # Test scenario: + # - create a route pointing to a NHG and assert it is added + # - create a temporary NHG and update the route to point to it, asserting the route's SAI NHG ID changes + # - update the temporary NHG to contain completely different members and assert the SAI ID changes + # - delete a NHG and assert the temporary NHG is promoted and its SAI ID also changes def test_route_nhg_update(): # Add a route nhg_index = gen_nhg_index(self.first_valid_nhg) @@ -350,6 +366,9 @@ def test_route_nhg_update(): rt_id, {'SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID': self.asic_nhgs[nhg_index]}) + # Test scenario: + # - create a temporary NHG containing labeled NHs and assert a new NH is added to represent the group + # - delete a NHG and assert the temporary NHG is promoted and all its NHs are added def test_labeled_nhg_temporary_promotion(): # Create a next hop group that contains labeled NHs that do not exist # in NeighOrch @@ -376,6 +395,9 @@ def test_labeled_nhg_temporary_promotion(): # Save the promoted NHG index/ID self.asic_nhgs[nhg_index] = self.get_nhg_id(nhg_index) + # Test scenario: + # - update route to own its NHG and assert no new NHG is added + # - remove a NHG and assert the temporary NHG is promoted and added to ASIC DB def test_back_compatibility(): # Update the route with a RouteOrch's owned NHG binary = self.gen_valid_binary() @@ -383,6 +405,7 @@ def test_back_compatibility(): self.rt_ps.set('2.2.2.0/24', nhg_fvs) # Assert no new group has been added + time.sleep(1) self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.MAX_ECMP_COUNT) # Delete a next hop group @@ -392,6 +415,12 @@ def test_back_compatibility(): # The temporary group should be promoted self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.MAX_ECMP_COUNT) + # Test scenario: + # - create a NHG with all NHs not existing and assert the NHG is not created + # - update the NHG to have valid NHs and assert a temporary NHG is created + # - update the NHG to all invalid NHs again and assert the update is not performed and thus it has the same SAI + # ID + # - delete the temporary NHG def test_invalid_temporary(): # Create a temporary NHG that contains only NHs that do not exist nhg_fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.21,10.0.0.23'), @@ -401,6 +430,7 @@ def test_invalid_temporary(): self.nhg_ps.set(nhg_index, nhg_fvs) # Assert the group is not created + time.sleep(1) assert not self.nhg_exists(nhg_index) # Update the temporary NHG to a valid one @@ -784,48 +814,36 @@ def test_label_route_nhg(self, dvs, testlog): fvs = swsscommon.FieldValuePairs([("nexthop","10.0.0.1,10.0.0.3,10.0.0.5"), ("ifname", "Ethernet0,Ethernet4,Ethernet8")]) self.lr_ps.set("10", fvs) - - # check if route was propagated to ASIC DB - self.asic_db.wait_for_n_keys(self.ASIC_INSEG_STR, self.asic_insgs_count + 1) + self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count + 1) + self.asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, self.asic_nhgms_count + 3) k = self.get_inseg_id('10') assert k is not None # assert the route points to next hop group fvs = self.asic_db.get_entry(self.ASIC_INSEG_STR, k) - nhgid = fvs["SAI_INSEG_ENTRY_ATTR_NEXT_HOP_ID"] - fvs = self.asic_db.get_entry(self.ASIC_NHG_STR, nhgid) - assert bool(fvs) keys = self.asic_db.get_keys(self.ASIC_NHGM_STR) - assert len(keys) == 3 - for k in keys: fvs = self.asic_db.get_entry(self.ASIC_NHGM_STR, k) - assert fvs["SAI_NEXT_HOP_GROUP_MEMBER_ATTR_NEXT_HOP_GROUP_ID"] == nhgid # bring links down one-by-one for i in [0, 1, 2]: self.flap_intf(i, 'down') - keys = self.asic_db.get_keys(self.ASIC_NHGM_STR) - assert len(keys) == 2 - i # bring links up one-by-one for i in [0, 1, 2]: self.flap_intf(i, 'up') - keys = self.asic_db.get_keys(self.ASIC_NHGM_STR) - assert len(keys) == i + 1 - for k in keys: fvs = self.asic_db.get_entry(self.ASIC_NHGM_STR, k) assert fvs["SAI_NEXT_HOP_GROUP_MEMBER_ATTR_NEXT_HOP_GROUP_ID"] == nhgid @@ -835,218 +853,308 @@ def test_label_route_nhg(self, dvs, testlog): # Wait for label route 10 to be removed self.asic_db.wait_for_n_keys(self.ASIC_INSEG_STR, self.asic_insgs_count) + self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count) def test_nhgorch_labeled_nhs(self, dvs, testlog): - self.init_test(dvs, 2) - - # Add a group containing labeled weighted NHs - fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1,10.0.0.3'), - ('mpls_nh', 'push1,push3'), - ('ifname', 'Ethernet0,Ethernet4'), - ('weight', '2,4')]) - self.nhg_ps.set('group1', fvs) - self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count + 1) - self.asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, self.asic_nhgms_count + 2) + # Test scenario: + # - create a NHG with all labeled and weighted NHs and assert 2 new NHs are created + # - create a NHG with an existing label and assert no new NHs are created + # - create a NHG with a new label and assert a new NH is created + # - remove the third NHG and assert the NH is deleted + # - delete the second group and assert no NH is deleted because it is still referenced by the first group + # - remove the weights from the first NHG and change the labels, leaving one NH unlabeled; assert one NH is + # deleted + # - delete the first NHG and perform cleanup + def test_mainline_labeled_nhs(): + # Add a group containing labeled weighted NHs + fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1,10.0.0.3'), + ('mpls_nh', 'push1,push3'), + ('ifname', 'Ethernet0,Ethernet4'), + ('weight', '2,4')]) + self.nhg_ps.set('group1', fvs) + self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count + 1) + self.asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, self.asic_nhgms_count + 2) - # NhgOrch should create two next hops for the labeled ones - self.asic_db.wait_for_n_keys(self.ASIC_NHS_STR, self.asic_nhs_count + 2) + # NhgOrch should create two next hops for the labeled ones + self.asic_db.wait_for_n_keys(self.ASIC_NHS_STR, self.asic_nhs_count + 2) - # Assert the weights are properly set - nhgm_ids = self.get_nhgm_ids('group1') - weights = [] - for k in nhgm_ids: - fvs = self.asic_db.get_entry(self.ASIC_NHGM_STR, k) - weights.append(fvs['SAI_NEXT_HOP_GROUP_MEMBER_ATTR_WEIGHT']) - assert set(weights) == set(['2', '4']) + # Assert the weights are properly set + nhgm_ids = self.get_nhgm_ids('group1') + weights = [] + for k in nhgm_ids: + fvs = self.asic_db.get_entry(self.ASIC_NHGM_STR, k) + weights.append(fvs['SAI_NEXT_HOP_GROUP_MEMBER_ATTR_WEIGHT']) + assert set(weights) == set(['2', '4']) - # Create a new single next hop with the same label - fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1'), - ('mpls_nh', 'push1'), - ('ifname', 'Ethernet0')]) - self.nhg_ps.set('group2', fvs) + # Create a new single next hop with the same label + fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1'), + ('mpls_nh', 'push1'), + ('ifname', 'Ethernet0')]) + self.nhg_ps.set('group2', fvs) - # No new next hop should be added - time.sleep(1) - assert len(self.asic_db.get_keys(self.ASIC_NHS_STR)) == self.asic_nhs_count + 2 + # No new next hop should be added + time.sleep(1) + assert len(self.asic_db.get_keys(self.ASIC_NHS_STR)) == self.asic_nhs_count + 2 - # Create a new single next hop with a different label - fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1'), - ('mpls_nh', 'push2'), - ('ifname', 'Ethernet0')]) - self.nhg_ps.set('group3', fvs) + # Create a new single next hop with a different label + fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1'), + ('mpls_nh', 'push2'), + ('ifname', 'Ethernet0')]) + self.nhg_ps.set('group3', fvs) - # A new next hop should be added - self.asic_db.wait_for_n_keys(self.ASIC_NHS_STR, self.asic_nhs_count + 3) + # A new next hop should be added + self.asic_db.wait_for_n_keys(self.ASIC_NHS_STR, self.asic_nhs_count + 3) - # Delete group3 - self.nhg_ps._del('group3') + # Delete group3 + self.nhg_ps._del('group3') - # Group3's NH should be deleted - self.asic_db.wait_for_n_keys(self.ASIC_NHS_STR, self.asic_nhs_count + 2) + # Group3's NH should be deleted + self.asic_db.wait_for_n_keys(self.ASIC_NHS_STR, self.asic_nhs_count + 2) - # Delete group2 - self.nhg_ps._del('group2') + # Delete group2 + self.nhg_ps._del('group2') - # The number of NHs should be the same as they are still referenced by - # group1 - time.sleep(1) - self.asic_db.wait_for_n_keys(self.ASIC_NHS_STR, self.asic_nhs_count + 2) + # The number of NHs should be the same as they are still referenced by + # group1 + time.sleep(1) + self.asic_db.wait_for_n_keys(self.ASIC_NHS_STR, self.asic_nhs_count + 2) - # Update group1 with no weights and both labeled and unlabeled NHs - fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1,10.0.0.3'), - ('mpls_nh', 'push2,na'), - ('ifname', 'Ethernet0,Ethernet4')]) - self.nhg_ps.set('group1', fvs) + # Update group1 with no weights and both labeled and unlabeled NHs + fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1,10.0.0.3'), + ('mpls_nh', 'push2,na'), + ('ifname', 'Ethernet0,Ethernet4')]) + self.nhg_ps.set('group1', fvs) - # Group members should be replaced and one NH should get deleted - self.asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, self.asic_nhgms_count + 2) - self.asic_db.wait_for_n_keys(self.ASIC_NHS_STR, self.asic_nhs_count + 1) + # Group members should be replaced and one NH should get deleted + self.asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, self.asic_nhgms_count + 2) + self.asic_db.wait_for_n_keys(self.ASIC_NHS_STR, self.asic_nhs_count + 1) - # Assert the weights of the NHGMs are the expected ones - nhgm_ids = self.get_nhgm_ids('group1') - weights = [] - for nhgm_id in nhgm_ids: - fvs = self.asic_db.get_entry(self.ASIC_NHGM_STR, nhgm_id) - weights.append(fvs['SAI_NEXT_HOP_GROUP_MEMBER_ATTR_WEIGHT']) - assert weights == ['0', '0'] + # Assert the weights of the NHGMs are the expected ones + nhgm_ids = self.get_nhgm_ids('group1') + weights = [] + for nhgm_id in nhgm_ids: + fvs = self.asic_db.get_entry(self.ASIC_NHGM_STR, nhgm_id) + weights.append(fvs['SAI_NEXT_HOP_GROUP_MEMBER_ATTR_WEIGHT']) + assert weights == ['0', '0'] + + # Delete group1 + self.nhg_ps._del('group1') + + # Wait for the group and it's members to be deleted + self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count) + self.asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, self.asic_nhgms_count) + + # The two next hops should also get deleted + self.asic_db.wait_for_n_keys(self.ASIC_NHS_STR, self.asic_nhs_count) + + # Test scenario: + # - create a route with labeled and weighted NHs and assert a NHG and 2 NHs are created + # - create a NHG with the same details as the one being used by the route and assert a NHG is created and no + # new NHs are added + # - update the NHG by changing the first NH's label and assert a new NH is created + # - remove the route and assert that only one (now unreferenced) NH is removed + # - remove the NHG and perform cleanup + def test_routeorch_nhgorch_interop(): + # Create a route with labeled NHs + fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1,10.0.0.3'), + ('mpls_nh', 'push1,push3'), + ('ifname', 'Ethernet0,Ethernet4'), + ('weight', '2,4')]) + self.rt_ps.set('2.2.2.0/24', fvs) + self.asic_db.wait_for_n_keys(self.ASIC_RT_STR, self.asic_rts_count + 1) - # Delete group1 - self.nhg_ps._del('group1') + # A NHG should be created + self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count + 1) - # Wait for the group and it's members to be deleted - self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count) - self.asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, self.asic_nhgms_count) + # Two new next hops should be created + self.asic_db.wait_for_n_keys(self.ASIC_NHS_STR, self.asic_nhs_count + 2) - # The two next hops should also get deleted - self.asic_db.wait_for_n_keys(self.ASIC_NHS_STR, self.asic_nhs_count) + # Create a NHG with the same details + self.nhg_ps.set('group1', fvs) + self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count + 2) - def test_nhgorch_excp_group_cases(self, dvs, testlog): - self.init_test(dvs, 3) + # No new next hops should be created + assert len(self.asic_db.get_keys(self.ASIC_NHS_STR)) == self.asic_nhs_count + 2 - def get_nhg_keys(): - return self.asic_db.get_keys(self.ASIC_NHG_STR) + # Update the group with a different NH + fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1,10.0.0.3'), + ('mpls_nh', 'push2,push3'), + ('ifname', 'Ethernet0,Ethernet4'), + ('weight', '2,4')]) + self.nhg_ps.set('group1', fvs) - def get_nhgm_keys(): - return self.asic_db.get_keys(self.ASIC_NHGM_STR) + # A new next hop should be created + self.asic_db.wait_for_n_keys(self.ASIC_NHS_STR, self.asic_nhs_count + 3) - def get_rt_keys(): - return self.asic_db.get_keys(self.ASIC_RT_STR) + # group1 should be updated and a new NHG shouldn't be created + time.sleep(1) + assert len(self.asic_db.get_keys(self.ASIC_NHG_STR)) == self.asic_nhgs_count + 2 - # Count existing objects - prev_nhg_keys = get_nhg_keys() - self.asic_nhgs_count = len(prev_nhg_keys) - self.asic_nhgms_count = len(get_nhgm_keys()) - self.asic_nhs_count = len(self.asic_db.get_keys(self.ASIC_NHS_STR)) + # Remove the route + self.rt_ps._del('2.2.2.0/24') + self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count + 1) - # Remove a group that does not exist - self.nhg_ps._del("group1") - time.sleep(1) - assert len(get_nhg_keys()) == self.asic_nhgs_count + # One NH should become unreferenced and should be deleted. The other + # one is still referenced by NhgOrch's owned NHG. + self.asic_db.wait_for_n_keys(self.ASIC_NHS_STR, self.asic_nhs_count + 2) - # Create a next hop group with a member that does not exist - should fail - fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1,10.0.0.3,10.0.0.63'), - ("ifname", "Ethernet0,Ethernet4,Ethernet124")]) - self.nhg_ps.set("group1", fvs) - time.sleep(1) - assert len(self.asic_db.get_keys(self.ASIC_NHG_STR)) == self.asic_nhgs_count + # Remove the group + self.nhg_ps._del('group1') + self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count) - # Issue an update for this next hop group that doesn't yet exist, - # which contains only valid NHs. This will overwrite the previous - # operation and create the group. - fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1,10.0.0.5'), - ("ifname", "Ethernet0,Ethernet8")]) - self.nhg_ps.set("group1", fvs) - self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count + 1) - self.asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, self.asic_nhgms_count + 2) + # Both new next hops should be deleted + self.asic_db.wait_for_n_keys(self.ASIC_NHS_STR, self.asic_nhs_count) - # Check the group has it's two members - for nhgid in self.asic_db.get_keys(self.ASIC_NHG_STR): - if nhgid not in prev_nhg_keys: - break + self.init_test(dvs, 2) - count = 0 - for k in self.asic_db.get_keys(self.ASIC_NHGM_STR): - fvs = self.asic_db.get_entry(self.ASIC_NHGM_STR, k) - if fvs['SAI_NEXT_HOP_GROUP_MEMBER_ATTR_NEXT_HOP_GROUP_ID'] == nhgid: - count += 1 - assert count == 2 + test_mainline_labeled_nhs() + test_routeorch_nhgorch_interop() + def test_nhgorch_excp_group_cases(self, dvs, testlog): + # Test scenario: + # - remove a NHG that does not exist and assert the number of NHGs in ASIC DB remains the same + def test_remove_inexistent_nhg(): + # Remove a group that does not exist + self.nhg_ps._del("group1") + time.sleep(1) + assert len(self.asic_db.get_keys(self.ASIC_NHG_STR)) == self.asic_nhgs_count + + # Test scenario: + # - create a NHG with a member which does not exist and assert no NHG is created + # - update the NHG to contain all valid members and assert the NHG is created and it has 2 members + def test_nhg_members_validation(): + # Create a next hop group with a member that does not exist - should fail + fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1,10.0.0.3,10.0.0.63'), + ("ifname", "Ethernet0,Ethernet4,Ethernet124")]) + self.nhg_ps.set("group1", fvs) + time.sleep(1) + assert len(self.asic_db.get_keys(self.ASIC_NHG_STR)) == self.asic_nhgs_count + + # Issue an update for this next hop group that doesn't yet exist, + # which contains only valid NHs. This will overwrite the previous + # operation and create the group. + fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1,10.0.0.5'), + ("ifname", "Ethernet0,Ethernet8")]) + self.nhg_ps.set("group1", fvs) + self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count + 1) + self.asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, self.asic_nhgms_count + 2) + + # Check the group has its two members + assert len(self.get_nhgm_ids('group1')) == 2 + + # Test scenario: + # - create a route pointing to the NHG created in `test_nhg_members_validation` and assert it is being created + # - remove the NHG and assert it fails as it is being referenced + # - create a new NHG and assert it and its members are being created + # - update the route to point to the new NHG and assert the first NHG is now deleted as it's not referenced + # anymore + def test_remove_referenced_nhg(): # Add a route referencing the new group - self.asic_rts_count = len(get_rt_keys()) - fvs = swsscommon.FieldValuePairs([('nexthop_group', 'group1')]) - self.rt_ps.set('2.2.2.0/24', fvs) - self.asic_db.wait_for_n_keys(self.ASIC_RT_STR, self.asic_rts_count + 1) + fvs = swsscommon.FieldValuePairs([('nexthop_group', 'group1')]) + self.rt_ps.set('2.2.2.0/24', fvs) + self.asic_db.wait_for_n_keys(self.ASIC_RT_STR, self.asic_rts_count + 1) - # Get the route key - for rt_key in get_rt_keys(): - k = json.loads(rt_key) + # Try removing the group while it still has references - should fail + self.nhg_ps._del('group1') + time.sleep(1) + assert len(self.asic_db.get_keys(self.ASIC_NHG_STR)) == self.asic_nhgs_count + 1 - if k['dest'] == "2.2.2.0/24": - break + # Create a new group + fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1,10.0.0.3'), + ('ifname', 'Ethernet0,Ethernet4')]) + self.nhg_ps.set("group2", fvs) + self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count + 2) + self.asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, self.asic_nhgms_count + 4) + + # Update the route to point to the new group + fvs = swsscommon.FieldValuePairs([('nexthop_group', 'group2')]) + self.rt_ps.set('2.2.2.0/24', fvs) + + # The first group should have got deleted + self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count + 1) + self.asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, self.asic_nhgms_count + 2) + + # The route's group should have changed to the new one + assert self.asic_db.get_entry(self.ASIC_RT_STR, self.get_route_id('2.2.2.0/24'))['SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID'] == self.get_nhg_id('group2') + + # Test scenario: + # - update the route created in `test_remove_referenced_nhg` to own the NHG with the same details as the + # previous one and assert a new NHG and 2 new NHGMs are added + # - update the route to point back to the original NHG and assert the routeOrch's owned NHG is deleted + def test_routeorch_nhgorch_interop(): + rt_id = self.get_route_id('2.2.2.0/24') + assert rt_id is not None - # Try removing the group while it still has references - should fail - self.nhg_ps._del('group1') - time.sleep(1) - assert len(get_nhg_keys()) == self.asic_nhgs_count + 1 + # Update the route with routeOrch's owned next hop group + nhgid = self.asic_db.get_entry(self.ASIC_RT_STR, rt_id)['SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID'] + fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1,10.0.0.3'), + ('ifname', 'Ethernet0,Ethernet4')]) + self.rt_ps.set('2.2.2.0/24', fvs) - # Create a new group - fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1,10.0.0.3'), - ('ifname', 'Ethernet0,Ethernet4')]) - self.nhg_ps.set("group2", fvs) - self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count + 2) - self.asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, self.asic_nhgms_count + 4) + self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count + 2) + self.asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, self.asic_nhgms_count + 4) - # Update the route to point to the new group - fvs = swsscommon.FieldValuePairs([('nexthop_group', 'group2')]) - self.rt_ps.set('2.2.2.0/24', fvs) + # Assert the next hop group ID changed + time.sleep(1) + assert self.asic_db.get_entry(self.ASIC_RT_STR, rt_id)['SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID'] != nhgid + nhgid = self.asic_db.get_entry(self.ASIC_RT_STR, rt_id)['SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID'] - # The first group should have got deleted - self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count + 1) - self.asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, self.asic_nhgms_count + 2) + # Update the route to point back to group2 + fvs = swsscommon.FieldValuePairs([('nexthop_group', 'group2')]) + self.rt_ps.set('2.2.2.0/24', fvs) - # The route's group should have changed to the new one - assert self.asic_db.get_entry(self.ASIC_RT_STR, rt_key)['SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID'] != nhgid + # The routeOrch's owned next hop group should get deleted + self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count + 1) + self.asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, self.asic_nhgms_count + 2) - # Update the route with routeOrch's owned next hop group - nhgid = self.asic_db.get_entry(self.ASIC_RT_STR, rt_key)['SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID'] - fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1,10.0.0.3'), - ('ifname', 'Ethernet0,Ethernet4')]) - self.rt_ps.set('2.2.2.0/24', fvs) + # Assert the route points back to group2 + assert self.asic_db.get_entry(self.ASIC_RT_STR, rt_id)['SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID'] != nhgid - self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count + 2) - self.asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, self.asic_nhgms_count + 4) + # Test scenario: + # - create a new NHG with the same details as the previous NHG and assert a new NHG and 2 new NHGMs are created + # - update the route to point to the new NHG and assert its SAI NHG ID changes + def test_identical_nhgs(): + rt_id = self.get_route_id('2.2.2.0/24') + assert rt_id is not None - # Assert the next hop group ID changed - time.sleep(1) - assert self.asic_db.get_entry(self.ASIC_RT_STR, rt_key)['SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID'] != nhgid - nhgid = self.asic_db.get_entry(self.ASIC_RT_STR, rt_key)['SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID'] + # Create a new group with the same members as group2 + nhgid = self.asic_db.get_entry(self.ASIC_RT_STR, rt_id)['SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID'] + fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1,10.0.0.3'), + ('ifname', 'Ethernet0,Ethernet4')]) + self.nhg_ps.set("group1", fvs) + self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count + 2) + self.asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, self.asic_nhgms_count + 4) - # Update the route to point back to group2 - fvs = swsscommon.FieldValuePairs([('nexthop_group', 'group2')]) - self.rt_ps.set('2.2.2.0/24', fvs) + # Update the route to point to the new group + fvs = swsscommon.FieldValuePairs([('nexthop_group', 'group1')]) + self.rt_ps.set('2.2.2.0/24', fvs) + time.sleep(1) - # The routeOrch's owned next hop group should get deleted - self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count + 1) - self.asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, self.asic_nhgms_count + 2) + # Assert the next hop group ID changed + assert self.asic_db.get_entry(self.ASIC_RT_STR, rt_id)['SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID'] != nhgid - # Assert the route points back to group2 - assert self.asic_db.get_entry(self.ASIC_RT_STR, rt_key)['SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID'] != nhgid + # Test scenario: + # - create a route referencing a NHG that does not exist and assert it is not created + def test_create_route_inexistent_nhg(): + # Add a route with a NHG that does not exist + fvs = swsscommon.FieldValuePairs([('nexthop_group', 'group3')]) + self.rt_ps.set('2.2.3.0/24', fvs) + time.sleep(1) + assert self.get_route_id('2.2.3.0/24') is None - # Create a new group with the same members as group2 - nhgid = self.asic_db.get_entry(self.ASIC_RT_STR, rt_key)['SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID'] - fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1,10.0.0.3'), - ('ifname', 'Ethernet0,Ethernet4')]) - self.nhg_ps.set("group1", fvs) - self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count + 2) - self.asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, self.asic_nhgms_count + 4) + # Remove the pending route + self.rt_ps._del('2.2.3.0/24') - # Update the route to point to the new group - fvs = swsscommon.FieldValuePairs([('nexthop_group', 'group1')]) - self.rt_ps.set('2.2.2.0/24', fvs) - time.sleep(1) + self.init_test(dvs, 3) - # Assert the next hop group ID changed - assert self.asic_db.get_entry(self.ASIC_RT_STR, rt_key)['SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID'] != nhgid + test_remove_inexistent_nhg() + test_nhg_members_validation() + test_remove_referenced_nhg() + test_routeorch_nhgorch_interop() + test_identical_nhgs() + test_create_route_inexistent_nhg() + + # Cleanup # Remove the route self.rt_ps._del('2.2.2.0/24') @@ -1055,421 +1163,206 @@ def get_rt_keys(): # Remove the groups self.nhg_ps._del('group1') self.nhg_ps._del('group2') - self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count) self.asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, self.asic_nhgms_count) - # Create a route with labeled NHs - fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1,10.0.0.3'), - ('mpls_nh', 'push1,push3'), - ('ifname', 'Ethernet0,Ethernet4'), - ('weight', '2,4')]) - self.rt_ps.set('2.2.2.0/24', fvs) - self.asic_db.wait_for_n_keys(self.ASIC_RT_STR, self.asic_rts_count + 1) - - # A NHG should be created - self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count + 1) - - # Two new next hops should be created - self.asic_db.wait_for_n_keys(self.ASIC_NHS_STR, self.asic_nhs_count + 2) - - # Create a NHG with the same details - self.nhg_ps.set('group1', fvs) - self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count + 2) - - # No new next hops should be created - assert len(self.asic_db.get_keys(self.ASIC_NHS_STR)) == self.asic_nhs_count + 2 - - # Update the group with a different NH - fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1,10.0.0.3'), - ('mpls_nh', 'push2,push3'), - ('ifname', 'Ethernet0,Ethernet4'), - ('weight', '2,4')]) - self.nhg_ps.set('group1', fvs) + def test_nhgorch_nh_group(self, dvs, testlog): + # Test scenario: + # - create NHG 'group1' and assert it is being added to ASIC DB along with its members + def test_create_nhg(): + # create next hop group in APPL DB + fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1,10.0.0.3,10.0.0.5'), + ("ifname", "Ethernet0,Ethernet4,Ethernet8")]) + self.nhg_ps.set("group1", fvs) - # A new next hop should be created - self.asic_db.wait_for_n_keys(self.ASIC_NHS_STR, self.asic_nhs_count + 3) + # check if group was propagated to ASIC DB + self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count + 1) + assert self.nhg_exists('group1') - # group1 should be updated and a new NHG shouldn't be created - time.sleep(1) - assert len(get_nhg_keys()) == self.asic_nhgs_count + 2 + # check if members were propagated to ASIC DB + self.asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, self.asic_nhgms_count + 3) + assert len(self.get_nhgm_ids('group1')) == 3 - # Remove the route - self.rt_ps._del('2.2.2.0/24') - self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count + 1) + # Test scenario: + # - create a route pointing to `group1` and assert it is being added to ASIC DB and pointing to its SAI ID + # - delete the route and assert it is being removed + def test_create_route_nhg(): + # create route in APPL DB + fvs = swsscommon.FieldValuePairs([("nexthop_group", "group1")]) + self.rt_ps.set("2.2.2.0/24", fvs) - # One NH should become unreferenced and should be deleted. The other - # one is still referenced by NhgOrch's owned NHG. - self.asic_db.wait_for_n_keys(self.ASIC_NHS_STR, self.asic_nhs_count + 2) - - # Remove the group - self.nhg_ps._del('group1') - self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count) - - # Both new next hops should be deleted - self.asic_db.wait_for_n_keys(self.ASIC_NHS_STR, self.asic_nhs_count) + # check if route was propagated to ASIC DB + self.asic_db.wait_for_n_keys(self.ASIC_RT_STR, self.asic_rts_count + 1) - # Add a route with a NHG that does not exist - fvs = swsscommon.FieldValuePairs([('nexthop_group', 'group1')]) - self.rt_ps.set('2.2.2.0/24', fvs) - time.sleep(1) - assert self.asic_rts_count == len(self.asic_db.get_keys(self.ASIC_RT_STR)) + k = self.get_route_id('2.2.2.0/24') + assert k is not None - # Remove the pending route - self.rt_ps._del('2.2.2.0/24') + # assert the route points to next hop group + fvs = self.asic_db.get_entry(self.ASIC_RT_STR, k) + assert fvs["SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID"] == self.get_nhg_id('group1') + + # Remove route 2.2.2.0/24 + self.rt_ps._del("2.2.2.0/24") + self.asic_db.wait_for_n_keys(self.ASIC_RT_STR, self.asic_rts_count) + + # Test scenario: + # - bring the links down one by one and assert the group1's members are subsequently removed and the group + # still exists + # - bring the liks up one by one and assert the group1's members are subsequently added back + def test_link_flap(): + # bring links down one-by-one + for i in [0, 1, 2]: + self.flap_intf(i, 'down') + self.asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, self.asic_nhgms_count + 2 - i) + assert len(self.get_nhgm_ids('group1')) == 2 - i + assert self.nhg_exists('group1') + + # bring links up one-by-one + for i in [0, 1, 2]: + self.flap_intf(i, 'up') + self.asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, self.asic_nhgms_count + i + 1) + assert len(self.get_nhgm_ids('group1')) == i + 1 + + # Test scenario: + # - bring a link down and assert a NHGM of `group1` is removed + # - create NHG `group2` which has a member pointing to the link being down and assert the group gets created + # but the member referencing the link is not added + # - update `group1` by removing a member while having another member referencing the link which is down and + # assert it'll only have a member added in ASIC DB + # - bring the link back up and assert the missing 2 members of `group1` and `group2` are added + # - remove `group2` and assert it and its members are removed + def test_validate_invalidate_group_member(): + # Bring an interface down + self.flap_intf(1, 'down') + + # One group member will get deleted + self.asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, self.asic_nhgms_count + 2) + + # Create a group that contains a NH that uses the down link + fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1,10.0.0.3'), + ("ifname", "Ethernet0,Ethernet4")]) + self.nhg_ps.set('group2', fvs) + + # The group should get created, but it will not contained the NH that + # has the link down + self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count + 2) + self.asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, self.asic_nhgms_count + 3) + assert len(self.get_nhgm_ids('group2')) == 1 + + # Update the NHG with one interface down + fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.3,10.0.0.1'), + ("ifname", "Ethernet4,Ethernet0")]) + self.nhg_ps.set("group1", fvs) + + # Wait for group members to update - the group will contain only the + # members that have their links up + self.asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, self.asic_nhgms_count + 2) + assert len(self.get_nhgm_ids('group1')) == 1 + + # Bring the interface up + self.flap_intf(1, 'up') + + # Check that the missing member of group1 and group2 is being added + self.asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, self.asic_nhgms_count + 4) + + # Remove group2 + self.nhg_ps._del('group2') + self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count + 1) + self.asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, self.asic_nhgms_count + 2) + + # Test scenario: + # - create NHG `group2` with a NH that does not exist and assert it isn't created + # - update `group1` to contain the invalid NH and assert it remains only with the unremoved members + # - configure the invalid NH's interface and assert `group2` gets created and `group1`'s NH is added + # - delete `group` and assert it is being removed + def test_inexistent_group_member(): + # Create group2 with a NH that does not exist + fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.3,10.0.0.63'), + ("ifname", "Ethernet4,Ethernet124")]) + self.nhg_ps.set("group2", fvs) + + # The groups should not be created + time.sleep(1) + self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count + 1) + + # Update group1 with a NH that does not exist + fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.3,10.0.0.63'), + ("ifname", "Ethernet4,Ethernet124")]) + self.nhg_ps.set("group1", fvs) + + # The update should fail, leaving group1 with only the unremoved + # members + self.asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, self.asic_nhgms_count + 1) + assert len(self.get_nhgm_ids('group1')) == 1 + + # Configure the missing NH's interface + self.config_intf(31) + + # A couple more routes will be added to ASIC DB + self.asic_rts_count += 2 + + # Group2 should get created and group1 should be updated + self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count + 2) + self.asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, self.asic_nhgms_count + 4) + assert len(self.get_nhgm_ids('group1')) == 2 + assert len(self.get_nhgm_ids('group2')) == 2 + + # Delete group2 + self.nhg_ps._del('group2') + self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count + 1) + + # Test scenario: + # - update `group1` to have 4 members and assert they are all added + # - update `group1` to have only 1 member and assert the other 3 are removed + # - update `group1` to have 2 members and assert a new one is added + def test_update_nhgm_count(): + # Update the NHG, adding two new members + fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1,10.0.0.3,10.0.0.5,10.0.0.7'), + ("ifname", "Ethernet0,Ethernet4,Ethernet8,Ethernet12")]) + self.nhg_ps.set("group1", fvs) + self.asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, self.asic_nhgms_count + 4) + assert len(self.get_nhgm_ids('group1')) == 4 + + # Update the group to one NH only + fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1'), ("ifname", "Ethernet0")]) + self.nhg_ps.set("group1", fvs) + self.asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, self.asic_nhgms_count + 1) + assert len(self.get_nhgm_ids('group1')) == 1 + + # Update the group to 2 NHs + fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1,10.0.0.3'), ("ifname", "Ethernet0,Ethernet4")]) + self.nhg_ps.set("group1", fvs) + self.asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, self.asic_nhgms_count + 2) + assert len(self.get_nhgm_ids('group1')) == 2 - def test_nhgorch_multi_nh_group(self, dvs, testlog): self.init_test(dvs, 4) - # create next hop group in APPL DB - - prev_nhg_keys = self.asic_db.get_keys(self.ASIC_NHG_STR) - - fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1,10.0.0.3,10.0.0.5'), - ("ifname", "Ethernet0,Ethernet4,Ethernet8")]) - self.nhg_ps.set("group1", fvs) - - # check if group was propagated to ASIC DB - - self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count + 1) - keys = self.asic_db.get_keys(self.ASIC_NHG_STR) - - found_nhg = False - - for nhgid in keys: - if nhgid not in prev_nhg_keys: - found_nhg = True - break - - assert found_nhg == True - - # check if members were propagated to ASIC DB - - self.asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, self.asic_nhgms_count + 3) - keys = self.asic_db.get_keys(self.ASIC_NHGM_STR) - count = 0 - - for k in keys: - fvs = self.asic_db.get_entry(self.ASIC_NHGM_STR, k) - - if fvs["SAI_NEXT_HOP_GROUP_MEMBER_ATTR_NEXT_HOP_GROUP_ID"] == nhgid: - count += 1 - - assert count == 3 - - # create route in APPL DB - - fvs = swsscommon.FieldValuePairs([("nexthop_group", "group1")]) - self.rt_ps.set("2.2.2.0/24", fvs) - - # check if route was propagated to ASIC DB - - self.asic_db.wait_for_n_keys(self.ASIC_RT_STR, self.asic_rts_count + 1) - - k = self.get_route_id('2.2.2.0/24') - assert k is not None - - # assert the route points to next hop group - fvs = self.asic_db.get_entry(self.ASIC_RT_STR, k) - assert fvs["SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID"] == nhgid - - # bring links down one-by-one - for i in [0, 1, 2]: - self.flap_intf(i, 'down') - - keys = self.asic_db.get_keys(self.ASIC_NHGM_STR) - - assert len(keys) == (self.asic_nhgms_count + 2 - i) - assert bool(self.asic_db.get_entry(self.ASIC_NHG_STR, nhgid)) + test_create_nhg() + test_create_route_nhg() + test_link_flap() + test_validate_invalidate_group_member() + test_inexistent_group_member() + test_update_nhgm_count() - # bring links up one-by-one - for i in [0, 1, 2]: - self.flap_intf(i, 'up') - - keys = self.asic_db.get_keys(self.ASIC_NHGM_STR) - - assert len(keys) == (self.asic_nhgms_count + i + 1) - - count = 0 - for k in keys: - fvs = self.asic_db.get_entry(self.ASIC_NHGM_STR, k) - if fvs["SAI_NEXT_HOP_GROUP_MEMBER_ATTR_NEXT_HOP_GROUP_ID"] == nhgid: - count += 1 - assert count == i + 1 - - # Bring an interface down - self.flap_intf(1, 'down') - - # One group member will get deleted - self.asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, 2) - - # Create a group that contains a NH that uses the down link - fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1,10.0.0.3'), - ("ifname", "Ethernet0,Ethernet4")]) - self.nhg_ps.set('group2', fvs) - - # The group should get created, but it will not contained the NH that - # has the link down - self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count + 2) - self.asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, self.asic_nhgms_count + 3) - - # Update the NHG with one interface down - fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.3,10.0.0.1'), - ("ifname", "Ethernet4,Ethernet0")]) - self.nhg_ps.set("group1", fvs) - - # Wait for group members to update - the group will contain only the - # members that have their links up - self.asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, self.asic_nhgms_count + 2) - - # Bring the interface up - self.flap_intf(1, 'up') - - # Check that the missing member of group1 and group2 is being added - self.asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, self.asic_nhgms_count + 4) - - # Remove group2 - self.nhg_ps._del('group2') - self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count + 1) - self.asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, self.asic_nhgms_count + 2) - - # Create group2 with a NH that does not exist - fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.3,10.0.0.63'), - ("ifname", "Ethernet4,Ethernet124")]) - self.nhg_ps.set("group2", fvs) - - # The groups should not be created - time.sleep(1) - self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count + 1) - - # Update group1 with a NH that does not exist - fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.3,10.0.0.63'), - ("ifname", "Ethernet4,Ethernet124")]) - self.nhg_ps.set("group1", fvs) - - # The update should fail, leaving group1 with only the unremoved - # members - self.asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, self.asic_nhgms_count + 1) - - # Configure the missing NH's interface - self.config_intf(31) - - # A couple more routes will be added to ASIC DB - self.asic_rts_count += 2 - - # Group2 should get created and group1 should be updated - self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count + 2) - self.asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, self.asic_nhgms_count + 4) - - # Delete group2 - self.nhg_ps._del('group2') - self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count + 1) - - # Update the NHG, adding two new members - fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1,10.0.0.3,10.0.0.5,10.0.0.7'), - ("ifname", "Ethernet0,Ethernet4,Ethernet8,Ethernet12")]) - self.nhg_ps.set("group1", fvs) - - # Wait for members to be added - self.asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, self.asic_nhgms_count + 4) - - # Update the group to one NH only - fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1'), ("ifname", "Ethernet0")]) - self.nhg_ps.set("group1", fvs) - time.sleep(1) - - assert bool(self.asic_db.get_entry(self.ASIC_NHG_STR, nhgid)) - self.asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, self.asic_nhgms_count + 1) - - # Remove route 2.2.2.0/24 - self.rt_ps._del("2.2.2.0/24") - - # Remove group1 - self.nhg_ps._del("group1") - - def test_nhgorch_single_nh_group(self, dvs, testlog): - self.init_test(dvs, 2) - - # Create single next hop group in APPL DB - fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1'), ("ifname", "Ethernet0")]) - self.nhg_ps.set("group1", fvs) - time.sleep(1) - - # Check that the group was created in ASIC DB - self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count + 1) - - # Get the NHG ID. - nhgid = self.get_nhg_id("group1") - - # create route in APPL DB - - fvs = swsscommon.FieldValuePairs([("nexthop_group", "group1")]) - self.rt_ps.set("2.2.2.0/24", fvs) - - # check if route was propagated to ASIC DB - - self.asic_db.wait_for_n_keys(self.ASIC_RT_STR, self.asic_rts_count + 1) - - k = self.get_route_id('2.2.2.0/24') - - # assert the route points to next hop group - fvs = self.asic_db.get_entry(self.ASIC_RT_STR, k) - assert fvs["SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID"] == nhgid - - # Bring group's link down - self.flap_intf(0, 'down') - - # Check that the group still has the same ID and everything works - assert nhgid == self.get_nhg_id('group1') - - # Bring group's link back up - self.flap_intf(0, 'up') - - # Check that the group still has the same ID and everything works - assert nhgid == self.get_nhg_id('group1') - - # Bring an interface down - self.flap_intf(1, 'down') - - # Create group2 pointing to the NH which's link is down - fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.3'), ("ifname", "Ethernet4")]) - self.nhg_ps.set("group2", fvs) - - # The group should be created. - assert self.nhg_exists('group2') - - # Delete the group - self.nhg_ps._del('group2') - - # Bring the interface back up - self.flap_intf(1, 'up') - - # Create group2 pointing to a NH that does not exist - fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.61'), ("ifname", "Ethernet120")]) - self.nhg_ps.set("group2", fvs) - - # The group should fail to be created. - assert not self.nhg_exists('group2') - - # Configure the NH's interface - self.config_intf(30) - - # A couple more routes will be added to ASIC DB - self.asic_rts_count += 2 - - # The group should be created - assert self.nhg_exists('group2') - - # Delete the group - self.nhg_ps._del('group2') - - # Update the group to a multiple NH group - fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1,10.0.0.3'), - ("ifname", "Ethernet0,Ethernet4")]) - self.nhg_ps.set("group1", fvs) - time.sleep(1) - - # Update group1 to point to another NH - nhgid = self.get_nhg_id('group1') - fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.3'), ("ifname", "Ethernet4")]) - self.nhg_ps.set("group1", fvs) - assert nhgid == self.get_nhg_id('group1') - - # Remove route 2.2.2.0/24 - self.rt_ps._del("2.2.2.0/24") - - # Wait for route 2.2.2.0/24 to be removed - self.asic_db.wait_for_n_keys(self.ASIC_RT_STR, self.asic_rts_count) - - # Update the group to a multiple NH group - fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1,10.0.0.3'), - ("ifname", "Ethernet0,Ethernet4")]) - self.nhg_ps.set("group1", fvs) - self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count + 1) - - # Create a route referencing group1 - fvs = swsscommon.FieldValuePairs([("nexthop_group", "group1")]) - self.rt_ps.set('2.2.2.0/24', fvs) - self.asic_db.wait_for_n_keys(self.ASIC_RT_STR, self.asic_rts_count + 1) - - # Update the NHG to a single next hop - should fail as it is being - # referenced - nhgid = self.get_nhg_id('group1') - fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1'), ("ifname", "Ethernet0")]) - self.nhg_ps.set("group1", fvs) - time.sleep(1) - assert nhgid == self.get_nhg_id('group1') - - # Remove route 2.2.2.0/24 - self.rt_ps._del("2.2.2.0/24") - self.asic_db.wait_for_n_keys(self.ASIC_RT_STR, self.asic_rts_count) + # Cleanup # Remove group1 self.nhg_ps._del("group1") + self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count) def test_nhgorch_label_route(self, dvs, testlog): self.init_test(dvs, 4) # create next hop group in APPL DB - - prev_nhg_keys = self.asic_db.get_keys(self.ASIC_NHG_STR) - fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1,10.0.0.3,10.0.0.5'), ("ifname", "Ethernet0,Ethernet4,Ethernet8")]) self.nhg_ps.set("group1", fvs) - - # check if group was propagated to ASIC DB - self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count + 1) - keys = self.asic_db.get_keys(self.ASIC_NHG_STR) - - found_nhg = False - - for nhgid in keys: - if nhgid not in prev_nhg_keys: - found_nhg = True - break - - assert found_nhg == True - - # check if members were propagated to ASIC DB - self.asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, self.asic_nhgms_count + 3) - keys = self.asic_db.get_keys(self.ASIC_NHGM_STR) - count = 0 - - for k in keys: - fvs = self.asic_db.get_entry(self.ASIC_NHGM_STR, k) - - if fvs["SAI_NEXT_HOP_GROUP_MEMBER_ATTR_NEXT_HOP_GROUP_ID"] == nhgid: - count += 1 - - assert count == 3 - - # create prefix route in APPL DB - - self.asic_rts_count = len(self.asic_db.get_keys(self.ASIC_RT_STR)) - - fvs = swsscommon.FieldValuePairs([("nexthop_group", "group1")]) - self.rt_ps.set("2.2.2.0/24", fvs) - - # check if route was propagated to ASIC DB - - self.asic_db.wait_for_n_keys(self.ASIC_RT_STR, self.asic_rts_count + 1) - - k = self.get_route_id('2.2.2.0/24') - - # assert the route points to next hop group - fvs = self.asic_db.get_entry(self.ASIC_RT_STR, k) - assert fvs["SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID"] == nhgid - - # create label route in APPL DB pointing to the same next hop grou + # create label route in APPL DB pointing to the NHG fvs = swsscommon.FieldValuePairs([("nexthop_group", "group1")]) self.lr_ps.set("20", fvs) - - # check if label route was propagated to ASIC DB - self.asic_db.wait_for_n_keys(self.ASIC_INSEG_STR, self.asic_insgs_count + 1) k = self.get_inseg_id('20') @@ -1477,52 +1370,15 @@ def test_nhgorch_label_route(self, dvs, testlog): # assert the route points to next hop group fvs = self.asic_db.get_entry(self.ASIC_INSEG_STR, k) - assert fvs["SAI_INSEG_ENTRY_ATTR_NEXT_HOP_ID"] == nhgid - - # bring links down one-by-one - for i in [0, 1, 2]: - self.flap_intf(i, 'down') - - keys = self.asic_db.get_keys(self.ASIC_NHGM_STR) - - assert len(keys) == (self.asic_nhgms_count + 2 - i) - assert bool(self.asic_db.get_entry(self.ASIC_NHG_STR, nhgid)) - - # bring links up one-by-one - for i in [0, 1, 2]: - self.flap_intf(i, 'up') - - keys = self.asic_db.get_keys(self.ASIC_NHGM_STR) - - assert len(keys) == (self.asic_nhgms_count + i + 1) - - count = 0 - for k in keys: - fvs = self.asic_db.get_entry(self.ASIC_NHGM_STR, k) - if fvs["SAI_NEXT_HOP_GROUP_MEMBER_ATTR_NEXT_HOP_GROUP_ID"] == nhgid: - count += 1 - assert count == i + 1 - - # Bring an interface down - self.flap_intf(1, 'down') - - # One group member will get deleted - self.asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, 2) - - # Remove route 2.2.2.0/24 - self.rt_ps._del("2.2.2.0/24") - - # Wait for route 2.2.2.0/24 to be removed - self.asic_db.wait_for_n_keys(self.ASIC_RT_STR, self.asic_rts_count) + assert fvs["SAI_INSEG_ENTRY_ATTR_NEXT_HOP_ID"] == self.get_nhg_id('group1') # Remove label route 20 self.lr_ps._del("20") - - # Wait for route 20 to be removed self.asic_db.wait_for_n_keys(self.ASIC_INSEG_STR, self.asic_insgs_count) # Remove group1 self.nhg_ps._del("group1") + self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count) # Add Dummy always-pass test at end as workaroud # for issue when Flaky fail on final test it invokes module tear-down before retrying From 56878dbfb6f808f5986f301d35ddef1d4f285773 Mon Sep 17 00:00:00 2001 From: Alexandru Banu Date: Sat, 2 Oct 2021 21:33:19 +0300 Subject: [PATCH 16/51] Rename sub-test cases for easier filtering --- tests/test_nhg.py | 88 +++++++++++++++++++++++------------------------ 1 file changed, 44 insertions(+), 44 deletions(-) diff --git a/tests/test_nhg.py b/tests/test_nhg.py index ce3beccd6b..f04b4133c2 100644 --- a/tests/test_nhg.py +++ b/tests/test_nhg.py @@ -188,7 +188,7 @@ def delete_nhg(): # Test scenario: # - create a NHG and assert a NHG object doesn't get added to ASIC DB # - delete a NHG and assert the newly created one is created in ASIC DB and its SAI ID changed - def test_temporary_group_promotion(): + def temporary_group_promotion_test(): # Add a new next hop group - it should create a temporary one instead prev_nhgs = self.asic_db.get_keys(self.ASIC_NHG_STR) nhg_index, _ = create_temp_nhg() @@ -221,7 +221,7 @@ def test_temporary_group_promotion(): # Test scenario: # - update an existing NHG and assert the update is performed - def test_group_update(): + def group_update_test(): # Update a group binary = self.gen_valid_binary() nhg_fvs = self.gen_nhg_fvs(binary) @@ -239,7 +239,7 @@ def test_group_update(): # Test scenario: # - create and delete a NHG while the ASIC DB is full and assert nothing changes - def test_create_delete_temporary(): + def create_delete_temporary_test(): # Create a new temporary group nhg_index, _ = create_temp_nhg() time.sleep(1) @@ -257,7 +257,7 @@ def test_create_delete_temporary(): # - create a temporary NHG # - update the NHG with a different number of members # - delete a NHG and assert the new one is added and it has the updated number of members - def test_update_temporary_group(): + def update_temporary_group_test(): # Create a new temporary group nhg_index, binary = create_temp_nhg() @@ -294,7 +294,7 @@ def test_update_temporary_group(): # - create a temporary NHG and update the route to point to it, asserting the route's SAI NHG ID changes # - update the temporary NHG to contain completely different members and assert the SAI ID changes # - delete a NHG and assert the temporary NHG is promoted and its SAI ID also changes - def test_route_nhg_update(): + def route_nhg_update_test(): # Add a route nhg_index = gen_nhg_index(self.first_valid_nhg) rt_fvs = swsscommon.FieldValuePairs([('nexthop_group', nhg_index)]) @@ -369,7 +369,7 @@ def test_route_nhg_update(): # Test scenario: # - create a temporary NHG containing labeled NHs and assert a new NH is added to represent the group # - delete a NHG and assert the temporary NHG is promoted and all its NHs are added - def test_labeled_nhg_temporary_promotion(): + def labeled_nhg_temporary_promotion_test(): # Create a next hop group that contains labeled NHs that do not exist # in NeighOrch self.asic_nhs_count = len(self.asic_db.get_keys(self.ASIC_NHS_STR)) @@ -398,7 +398,7 @@ def test_labeled_nhg_temporary_promotion(): # Test scenario: # - update route to own its NHG and assert no new NHG is added # - remove a NHG and assert the temporary NHG is promoted and added to ASIC DB - def test_back_compatibility(): + def back_compatibility_test(): # Update the route with a RouteOrch's owned NHG binary = self.gen_valid_binary() nhg_fvs = self.gen_nhg_fvs(binary) @@ -421,7 +421,7 @@ def test_back_compatibility(): # - update the NHG to all invalid NHs again and assert the update is not performed and thus it has the same SAI # ID # - delete the temporary NHG - def test_invalid_temporary(): + def invalid_temporary_test(): # Create a temporary NHG that contains only NHs that do not exist nhg_fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.21,10.0.0.23'), ('ifname', 'Ethernet40,Ethernet44')]) @@ -474,14 +474,14 @@ def test_invalid_temporary(): self.nhg_count += 1 self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.MAX_ECMP_COUNT) - test_temporary_group_promotion() - test_group_update() - test_create_delete_temporary() - test_update_temporary_group() - test_route_nhg_update() - test_labeled_nhg_temporary_promotion() - test_back_compatibility() - test_invalid_temporary() + temporary_group_promotion_test() + group_update_test() + create_delete_temporary_test() + update_temporary_group_test() + route_nhg_update_test() + labeled_nhg_temporary_promotion_test() + back_compatibility_test() + invalid_temporary_test() # Cleanup @@ -865,7 +865,7 @@ def test_nhgorch_labeled_nhs(self, dvs, testlog): # - remove the weights from the first NHG and change the labels, leaving one NH unlabeled; assert one NH is # deleted # - delete the first NHG and perform cleanup - def test_mainline_labeled_nhs(): + def mainline_labeled_nhs_test(): # Add a group containing labeled weighted NHs fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1,10.0.0.3'), ('mpls_nh', 'push1,push3'), @@ -954,7 +954,7 @@ def test_mainline_labeled_nhs(): # - update the NHG by changing the first NH's label and assert a new NH is created # - remove the route and assert that only one (now unreferenced) NH is removed # - remove the NHG and perform cleanup - def test_routeorch_nhgorch_interop(): + def routeorch_nhgorch_interop_test(): # Create a route with labeled NHs fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1,10.0.0.3'), ('mpls_nh', 'push1,push3'), @@ -1007,13 +1007,13 @@ def test_routeorch_nhgorch_interop(): self.init_test(dvs, 2) - test_mainline_labeled_nhs() - test_routeorch_nhgorch_interop() + mainline_labeled_nhs_test() + routeorch_nhgorch_interop_test() def test_nhgorch_excp_group_cases(self, dvs, testlog): # Test scenario: # - remove a NHG that does not exist and assert the number of NHGs in ASIC DB remains the same - def test_remove_inexistent_nhg(): + def remove_inexistent_nhg_test(): # Remove a group that does not exist self.nhg_ps._del("group1") time.sleep(1) @@ -1022,7 +1022,7 @@ def test_remove_inexistent_nhg(): # Test scenario: # - create a NHG with a member which does not exist and assert no NHG is created # - update the NHG to contain all valid members and assert the NHG is created and it has 2 members - def test_nhg_members_validation(): + def nhg_members_validation_test(): # Create a next hop group with a member that does not exist - should fail fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1,10.0.0.3,10.0.0.63'), ("ifname", "Ethernet0,Ethernet4,Ethernet124")]) @@ -1048,7 +1048,7 @@ def test_nhg_members_validation(): # - create a new NHG and assert it and its members are being created # - update the route to point to the new NHG and assert the first NHG is now deleted as it's not referenced # anymore - def test_remove_referenced_nhg(): + def remove_referenced_nhg_test(): # Add a route referencing the new group fvs = swsscommon.FieldValuePairs([('nexthop_group', 'group1')]) self.rt_ps.set('2.2.2.0/24', fvs) @@ -1081,7 +1081,7 @@ def test_remove_referenced_nhg(): # - update the route created in `test_remove_referenced_nhg` to own the NHG with the same details as the # previous one and assert a new NHG and 2 new NHGMs are added # - update the route to point back to the original NHG and assert the routeOrch's owned NHG is deleted - def test_routeorch_nhgorch_interop(): + def routeorch_nhgorch_interop_test(): rt_id = self.get_route_id('2.2.2.0/24') assert rt_id is not None @@ -1113,7 +1113,7 @@ def test_routeorch_nhgorch_interop(): # Test scenario: # - create a new NHG with the same details as the previous NHG and assert a new NHG and 2 new NHGMs are created # - update the route to point to the new NHG and assert its SAI NHG ID changes - def test_identical_nhgs(): + def identical_nhgs_test(): rt_id = self.get_route_id('2.2.2.0/24') assert rt_id is not None @@ -1135,7 +1135,7 @@ def test_identical_nhgs(): # Test scenario: # - create a route referencing a NHG that does not exist and assert it is not created - def test_create_route_inexistent_nhg(): + def create_route_inexistent_nhg_test(): # Add a route with a NHG that does not exist fvs = swsscommon.FieldValuePairs([('nexthop_group', 'group3')]) self.rt_ps.set('2.2.3.0/24', fvs) @@ -1147,12 +1147,12 @@ def test_create_route_inexistent_nhg(): self.init_test(dvs, 3) - test_remove_inexistent_nhg() - test_nhg_members_validation() - test_remove_referenced_nhg() - test_routeorch_nhgorch_interop() - test_identical_nhgs() - test_create_route_inexistent_nhg() + remove_inexistent_nhg_test() + nhg_members_validation_test() + remove_referenced_nhg_test() + routeorch_nhgorch_interop_test() + identical_nhgs_test() + create_route_inexistent_nhg_test() # Cleanup @@ -1169,7 +1169,7 @@ def test_create_route_inexistent_nhg(): def test_nhgorch_nh_group(self, dvs, testlog): # Test scenario: # - create NHG 'group1' and assert it is being added to ASIC DB along with its members - def test_create_nhg(): + def create_nhg_test(): # create next hop group in APPL DB fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1,10.0.0.3,10.0.0.5'), ("ifname", "Ethernet0,Ethernet4,Ethernet8")]) @@ -1186,7 +1186,7 @@ def test_create_nhg(): # Test scenario: # - create a route pointing to `group1` and assert it is being added to ASIC DB and pointing to its SAI ID # - delete the route and assert it is being removed - def test_create_route_nhg(): + def create_route_nhg_test(): # create route in APPL DB fvs = swsscommon.FieldValuePairs([("nexthop_group", "group1")]) self.rt_ps.set("2.2.2.0/24", fvs) @@ -1209,7 +1209,7 @@ def test_create_route_nhg(): # - bring the links down one by one and assert the group1's members are subsequently removed and the group # still exists # - bring the liks up one by one and assert the group1's members are subsequently added back - def test_link_flap(): + def link_flap_test(): # bring links down one-by-one for i in [0, 1, 2]: self.flap_intf(i, 'down') @@ -1231,7 +1231,7 @@ def test_link_flap(): # assert it'll only have a member added in ASIC DB # - bring the link back up and assert the missing 2 members of `group1` and `group2` are added # - remove `group2` and assert it and its members are removed - def test_validate_invalidate_group_member(): + def validate_invalidate_group_member_test(): # Bring an interface down self.flap_intf(1, 'down') @@ -1275,7 +1275,7 @@ def test_validate_invalidate_group_member(): # - update `group1` to contain the invalid NH and assert it remains only with the unremoved members # - configure the invalid NH's interface and assert `group2` gets created and `group1`'s NH is added # - delete `group` and assert it is being removed - def test_inexistent_group_member(): + def inexistent_group_member_test(): # Create group2 with a NH that does not exist fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.3,10.0.0.63'), ("ifname", "Ethernet4,Ethernet124")]) @@ -1315,7 +1315,7 @@ def test_inexistent_group_member(): # - update `group1` to have 4 members and assert they are all added # - update `group1` to have only 1 member and assert the other 3 are removed # - update `group1` to have 2 members and assert a new one is added - def test_update_nhgm_count(): + def update_nhgm_count_test(): # Update the NHG, adding two new members fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1,10.0.0.3,10.0.0.5,10.0.0.7'), ("ifname", "Ethernet0,Ethernet4,Ethernet8,Ethernet12")]) @@ -1337,12 +1337,12 @@ def test_update_nhgm_count(): self.init_test(dvs, 4) - test_create_nhg() - test_create_route_nhg() - test_link_flap() - test_validate_invalidate_group_member() - test_inexistent_group_member() - test_update_nhgm_count() + create_nhg_test() + create_route_nhg_test() + link_flap_test() + validate_invalidate_group_member_test() + inexistent_group_member_test() + update_nhgm_count_test() # Cleanup From 77da6fa9ad3311999c8231733d76937864a148cb Mon Sep 17 00:00:00 2001 From: Tom cappleman Date: Mon, 4 Oct 2021 09:44:08 +0100 Subject: [PATCH 17/51] Minor fixes --- orchagent/nhgorch.h | 4 ---- orchagent/routeorch.cpp | 26 ++++++++++++++------------ tests/test_nhg.py | 1 - 3 files changed, 14 insertions(+), 17 deletions(-) diff --git a/orchagent/nhgorch.h b/orchagent/nhgorch.h index c16ce40f83..ce99ef85e3 100644 --- a/orchagent/nhgorch.h +++ b/orchagent/nhgorch.h @@ -3,10 +3,6 @@ #include "orch.h" #include "nexthopgroupkey.h" -/* Default maximum number of next hop groups */ -#define DEFAULT_NUMBER_OF_ECMP_GROUPS 128 -#define DEFAULT_MAX_ECMP_GROUP_SIZE 32 - class NextHopGroupMember { public: diff --git a/orchagent/routeorch.cpp b/orchagent/routeorch.cpp index 00c66872f6..9fc1c6a31d 100644 --- a/orchagent/routeorch.cpp +++ b/orchagent/routeorch.cpp @@ -23,6 +23,10 @@ extern NhgOrch *gNhgOrch; extern size_t gMaxBulkSize; +/* Default maximum number of next hop groups */ +#define DEFAULT_NUMBER_OF_ECMP_GROUPS 128 +#define DEFAULT_MAX_ECMP_GROUP_SIZE 32 + RouteOrch::RouteOrch(DBConnector *db, vector &tableNames, SwitchOrch *switchOrch, NeighOrch *neighOrch, IntfsOrch *intfsOrch, VRFOrch *vrfOrch, FgNhgOrch *fgNhgOrch) : gRouteBulker(sai_route_api, gMaxBulkSize), gLabelRouteBulker(sai_mpls_api, gMaxBulkSize), @@ -1675,7 +1679,7 @@ bool RouteOrch::addRoute(RouteBulkContext& ctx, const NextHopGroupKey &nextHops) /* Failed to create the next hop group and check if a temporary route is needed */ /* If the current next hop is part of the next hop group to sync, - * then return false and no need to add another temporary route. */ + * then return false and no need to add another temporary route. */ if (it_route != m_syncdRoutes.at(vrf_id).end() && it_route->second.nhg_key.getSize() == 1) { const NextHopKey& nexthop = *it_route->second.nhg_key.getNextHops().begin(); @@ -1686,8 +1690,8 @@ bool RouteOrch::addRoute(RouteBulkContext& ctx, const NextHopGroupKey &nextHops) } /* Add a temporary route when a next hop group cannot be added, - * and there is no temporary route right now or the current temporary - * route is not pointing to a member of the next hop group to sync. */ + * and there is no temporary route right now or the current temporary + * route is not pointing to a member of the next hop group to sync. */ addTempRoute(ctx, nextHops); /* Return false since the original route is not successfully added */ return false; @@ -1842,8 +1846,6 @@ bool RouteOrch::addRoutePost(const RouteBulkContext& ctx, const NextHopGroupKey { if (!hasNextHopGroup(nextHops)) { - SWSS_LOG_DEBUG("Next hop group is temporary, represented by %s", - ctx.tmp_next_hop.to_string().c_str()); // Previous added an temporary route auto& tmp_next_hop = ctx.tmp_next_hop; addRoutePost(ctx, tmp_next_hop); @@ -2190,8 +2192,13 @@ bool RouteOrch::removeRoutePost(const RouteBulkContext& ctx) /* Delete Fine Grained nhg if the revmoved route pointed to it */ m_fgNhgOrch->removeFgNhg(vrf_id, ipPrefix); } - /* Check that the next hop group is not owned by NhgOrch. */ - else if (it_route->second.nhg_index.empty()) + /* Check if the next hop group is not owned by NhgOrch. */ + else if (!it_route->second.nhg_index.empty()) + { + gNhgOrch->decNhgRefCount(it_route->second.nhg_index); + } + /* The NHG is owned by RouteOrch */ + else { auto ol_nextHops = it_route->second.nhg_key; @@ -2224,11 +2231,6 @@ bool RouteOrch::removeRoutePost(const RouteBulkContext& ctx) } } } - /* The NHG is owned by NhgOrch */ - else - { - gNhgOrch->decNhgRefCount(it_route->second.nhg_index); - } SWSS_LOG_INFO("Remove route %s with next hop(s) %s", ipPrefix.to_string().c_str(), it_route->second.nhg_key.to_string().c_str()); diff --git a/tests/test_nhg.py b/tests/test_nhg.py index f04b4133c2..f1b600a22b 100644 --- a/tests/test_nhg.py +++ b/tests/test_nhg.py @@ -612,7 +612,6 @@ def asic_route_nhg_fvs(k): route_count = 0 self.r = 0 while route_count < self.MAX_ECMP_COUNT: - binary = self.gen_valid_binary() route_ipprefix = gen_ipprefix(route_count) self.rt_ps._del(route_ipprefix) route_count += 1 From 0aac13fbd980f39a592787afc21dd2ed2375449b Mon Sep 17 00:00:00 2001 From: Tom cappleman Date: Tue, 5 Oct 2021 09:34:40 +0100 Subject: [PATCH 18/51] Pipeline fix --- .azure-pipelines/build-template.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.azure-pipelines/build-template.yml b/.azure-pipelines/build-template.yml index 2dbec2ffc7..251d2631a8 100644 --- a/.azure-pipelines/build-template.yml +++ b/.azure-pipelines/build-template.yml @@ -95,14 +95,14 @@ jobs: runBranch: 'refs/heads/master' displayName: "Download sonic buildimage deb packages" - script: | - sudo dpkg -i target/debs/buster/libnl-3-200_*.deb - sudo dpkg -i target/debs/buster/libnl-3-dev_*.deb - sudo dpkg -i target/debs/buster/libnl-genl-3-200_*.deb - sudo dpkg -i target/debs/buster/libnl-genl-3-dev_*.deb - sudo dpkg -i target/debs/buster/libnl-route-3-200_*.deb - sudo dpkg -i target/debs/buster/libnl-route-3-dev_*.deb - sudo dpkg -i target/debs/buster/libnl-nf-3-200_*.deb - sudo dpkg -i target/debs/buster/libnl-nf-3-dev_*.deb + sudo dpkg -i libnl-3-200_*_${{ parameters.arch }}.deb + sudo dpkg -i libnl-3-dev_*_${{ parameters.arch }}.deb + sudo dpkg -i libnl-genl-3-200_*_${{ parameters.arch }}.deb + sudo dpkg -i libnl-genl-3-dev_*_${{ parameters.arch }}.deb + sudo dpkg -i libnl-route-3-200_*_${{ parameters.arch }}.deb + sudo dpkg -i libnl-route-3-dev_*_${{ parameters.arch }}.deb + sudo dpkg -i libnl-nf-3-200_*_${{ parameters.arch }}.deb + sudo dpkg -i libnl-nf-3-dev_*_${{ parameters.arch }}.deb sudo dpkg -i libswsscommon_1.0.0_${{ parameters.arch }}.deb sudo dpkg -i libswsscommon-dev_1.0.0_${{ parameters.arch }}.deb sudo dpkg -i libsaivs_*.deb From 0b46048a5d1694b6d9fde6a86ccd0a2c2914fb18 Mon Sep 17 00:00:00 2001 From: Tom cappleman Date: Tue, 5 Oct 2021 10:28:41 +0100 Subject: [PATCH 19/51] Pipeline fix --- .azure-pipelines/build-template.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.azure-pipelines/build-template.yml b/.azure-pipelines/build-template.yml index 251d2631a8..036803a302 100644 --- a/.azure-pipelines/build-template.yml +++ b/.azure-pipelines/build-template.yml @@ -95,14 +95,14 @@ jobs: runBranch: 'refs/heads/master' displayName: "Download sonic buildimage deb packages" - script: | - sudo dpkg -i libnl-3-200_*_${{ parameters.arch }}.deb - sudo dpkg -i libnl-3-dev_*_${{ parameters.arch }}.deb - sudo dpkg -i libnl-genl-3-200_*_${{ parameters.arch }}.deb - sudo dpkg -i libnl-genl-3-dev_*_${{ parameters.arch }}.deb - sudo dpkg -i libnl-route-3-200_*_${{ parameters.arch }}.deb - sudo dpkg -i libnl-route-3-dev_*_${{ parameters.arch }}.deb - sudo dpkg -i libnl-nf-3-200_*_${{ parameters.arch }}.deb - sudo dpkg -i libnl-nf-3-dev_*_${{ parameters.arch }}.deb + sudo dpkg -i libnl-3-200_3.5.0-1_${{ parameters.arch }}.deb + sudo dpkg -i libnl-3-dev_3.5.0-1_${{ parameters.arch }}.deb + sudo dpkg -i libnl-genl-3-200_3.5.0-1_${{ parameters.arch }}.deb + sudo dpkg -i libnl-genl-3-dev_3.5.0-1_${{ parameters.arch }}.deb + sudo dpkg -i libnl-route-3-200_3.5.0-1_${{ parameters.arch }}.deb + sudo dpkg -i libnl-route-3-dev_3.5.0-1_${{ parameters.arch }}.deb + sudo dpkg -i libnl-nf-3-200_3.5.0-1_${{ parameters.arch }}.deb + sudo dpkg -i libnl-nf-3-dev_3.5.0-1_${{ parameters.arch }}.deb sudo dpkg -i libswsscommon_1.0.0_${{ parameters.arch }}.deb sudo dpkg -i libswsscommon-dev_1.0.0_${{ parameters.arch }}.deb sudo dpkg -i libsaivs_*.deb From 60b4793fc63a9b8415bf584df13e469463334534 Mon Sep 17 00:00:00 2001 From: Tom cappleman Date: Tue, 5 Oct 2021 10:39:44 +0100 Subject: [PATCH 20/51] Revert "Pipeline fix" This reverts commit 0b46048a5d1694b6d9fde6a86ccd0a2c2914fb18. --- .azure-pipelines/build-template.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.azure-pipelines/build-template.yml b/.azure-pipelines/build-template.yml index 036803a302..251d2631a8 100644 --- a/.azure-pipelines/build-template.yml +++ b/.azure-pipelines/build-template.yml @@ -95,14 +95,14 @@ jobs: runBranch: 'refs/heads/master' displayName: "Download sonic buildimage deb packages" - script: | - sudo dpkg -i libnl-3-200_3.5.0-1_${{ parameters.arch }}.deb - sudo dpkg -i libnl-3-dev_3.5.0-1_${{ parameters.arch }}.deb - sudo dpkg -i libnl-genl-3-200_3.5.0-1_${{ parameters.arch }}.deb - sudo dpkg -i libnl-genl-3-dev_3.5.0-1_${{ parameters.arch }}.deb - sudo dpkg -i libnl-route-3-200_3.5.0-1_${{ parameters.arch }}.deb - sudo dpkg -i libnl-route-3-dev_3.5.0-1_${{ parameters.arch }}.deb - sudo dpkg -i libnl-nf-3-200_3.5.0-1_${{ parameters.arch }}.deb - sudo dpkg -i libnl-nf-3-dev_3.5.0-1_${{ parameters.arch }}.deb + sudo dpkg -i libnl-3-200_*_${{ parameters.arch }}.deb + sudo dpkg -i libnl-3-dev_*_${{ parameters.arch }}.deb + sudo dpkg -i libnl-genl-3-200_*_${{ parameters.arch }}.deb + sudo dpkg -i libnl-genl-3-dev_*_${{ parameters.arch }}.deb + sudo dpkg -i libnl-route-3-200_*_${{ parameters.arch }}.deb + sudo dpkg -i libnl-route-3-dev_*_${{ parameters.arch }}.deb + sudo dpkg -i libnl-nf-3-200_*_${{ parameters.arch }}.deb + sudo dpkg -i libnl-nf-3-dev_*_${{ parameters.arch }}.deb sudo dpkg -i libswsscommon_1.0.0_${{ parameters.arch }}.deb sudo dpkg -i libswsscommon-dev_1.0.0_${{ parameters.arch }}.deb sudo dpkg -i libsaivs_*.deb From fbae0294b442e8c8b950f8758766d09d0cd4227e Mon Sep 17 00:00:00 2001 From: Tom cappleman Date: Tue, 5 Oct 2021 10:39:50 +0100 Subject: [PATCH 21/51] Revert "Pipeline fix" This reverts commit 0aac13fbd980f39a592787afc21dd2ed2375449b. --- .azure-pipelines/build-template.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.azure-pipelines/build-template.yml b/.azure-pipelines/build-template.yml index 251d2631a8..2dbec2ffc7 100644 --- a/.azure-pipelines/build-template.yml +++ b/.azure-pipelines/build-template.yml @@ -95,14 +95,14 @@ jobs: runBranch: 'refs/heads/master' displayName: "Download sonic buildimage deb packages" - script: | - sudo dpkg -i libnl-3-200_*_${{ parameters.arch }}.deb - sudo dpkg -i libnl-3-dev_*_${{ parameters.arch }}.deb - sudo dpkg -i libnl-genl-3-200_*_${{ parameters.arch }}.deb - sudo dpkg -i libnl-genl-3-dev_*_${{ parameters.arch }}.deb - sudo dpkg -i libnl-route-3-200_*_${{ parameters.arch }}.deb - sudo dpkg -i libnl-route-3-dev_*_${{ parameters.arch }}.deb - sudo dpkg -i libnl-nf-3-200_*_${{ parameters.arch }}.deb - sudo dpkg -i libnl-nf-3-dev_*_${{ parameters.arch }}.deb + sudo dpkg -i target/debs/buster/libnl-3-200_*.deb + sudo dpkg -i target/debs/buster/libnl-3-dev_*.deb + sudo dpkg -i target/debs/buster/libnl-genl-3-200_*.deb + sudo dpkg -i target/debs/buster/libnl-genl-3-dev_*.deb + sudo dpkg -i target/debs/buster/libnl-route-3-200_*.deb + sudo dpkg -i target/debs/buster/libnl-route-3-dev_*.deb + sudo dpkg -i target/debs/buster/libnl-nf-3-200_*.deb + sudo dpkg -i target/debs/buster/libnl-nf-3-dev_*.deb sudo dpkg -i libswsscommon_1.0.0_${{ parameters.arch }}.deb sudo dpkg -i libswsscommon-dev_1.0.0_${{ parameters.arch }}.deb sudo dpkg -i libsaivs_*.deb From 0f7a3f8e66e902d9bb44ebcdc72e5e2951ff3dbe Mon Sep 17 00:00:00 2001 From: Tom cappleman Date: Tue, 5 Oct 2021 09:34:40 +0100 Subject: [PATCH 22/51] Pipeline fix --- .azure-pipelines/build-template.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.azure-pipelines/build-template.yml b/.azure-pipelines/build-template.yml index 2dbec2ffc7..251d2631a8 100644 --- a/.azure-pipelines/build-template.yml +++ b/.azure-pipelines/build-template.yml @@ -95,14 +95,14 @@ jobs: runBranch: 'refs/heads/master' displayName: "Download sonic buildimage deb packages" - script: | - sudo dpkg -i target/debs/buster/libnl-3-200_*.deb - sudo dpkg -i target/debs/buster/libnl-3-dev_*.deb - sudo dpkg -i target/debs/buster/libnl-genl-3-200_*.deb - sudo dpkg -i target/debs/buster/libnl-genl-3-dev_*.deb - sudo dpkg -i target/debs/buster/libnl-route-3-200_*.deb - sudo dpkg -i target/debs/buster/libnl-route-3-dev_*.deb - sudo dpkg -i target/debs/buster/libnl-nf-3-200_*.deb - sudo dpkg -i target/debs/buster/libnl-nf-3-dev_*.deb + sudo dpkg -i libnl-3-200_*_${{ parameters.arch }}.deb + sudo dpkg -i libnl-3-dev_*_${{ parameters.arch }}.deb + sudo dpkg -i libnl-genl-3-200_*_${{ parameters.arch }}.deb + sudo dpkg -i libnl-genl-3-dev_*_${{ parameters.arch }}.deb + sudo dpkg -i libnl-route-3-200_*_${{ parameters.arch }}.deb + sudo dpkg -i libnl-route-3-dev_*_${{ parameters.arch }}.deb + sudo dpkg -i libnl-nf-3-200_*_${{ parameters.arch }}.deb + sudo dpkg -i libnl-nf-3-dev_*_${{ parameters.arch }}.deb sudo dpkg -i libswsscommon_1.0.0_${{ parameters.arch }}.deb sudo dpkg -i libswsscommon-dev_1.0.0_${{ parameters.arch }}.deb sudo dpkg -i libsaivs_*.deb From 7d4cfa8231c9c7639ebf0d9e77e1839ca4a5e910 Mon Sep 17 00:00:00 2001 From: Tom cappleman Date: Tue, 5 Oct 2021 10:39:50 +0100 Subject: [PATCH 23/51] Revert "Pipeline fix" This reverts commit 0aac13fbd980f39a592787afc21dd2ed2375449b. --- .azure-pipelines/build-template.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.azure-pipelines/build-template.yml b/.azure-pipelines/build-template.yml index 251d2631a8..2dbec2ffc7 100644 --- a/.azure-pipelines/build-template.yml +++ b/.azure-pipelines/build-template.yml @@ -95,14 +95,14 @@ jobs: runBranch: 'refs/heads/master' displayName: "Download sonic buildimage deb packages" - script: | - sudo dpkg -i libnl-3-200_*_${{ parameters.arch }}.deb - sudo dpkg -i libnl-3-dev_*_${{ parameters.arch }}.deb - sudo dpkg -i libnl-genl-3-200_*_${{ parameters.arch }}.deb - sudo dpkg -i libnl-genl-3-dev_*_${{ parameters.arch }}.deb - sudo dpkg -i libnl-route-3-200_*_${{ parameters.arch }}.deb - sudo dpkg -i libnl-route-3-dev_*_${{ parameters.arch }}.deb - sudo dpkg -i libnl-nf-3-200_*_${{ parameters.arch }}.deb - sudo dpkg -i libnl-nf-3-dev_*_${{ parameters.arch }}.deb + sudo dpkg -i target/debs/buster/libnl-3-200_*.deb + sudo dpkg -i target/debs/buster/libnl-3-dev_*.deb + sudo dpkg -i target/debs/buster/libnl-genl-3-200_*.deb + sudo dpkg -i target/debs/buster/libnl-genl-3-dev_*.deb + sudo dpkg -i target/debs/buster/libnl-route-3-200_*.deb + sudo dpkg -i target/debs/buster/libnl-route-3-dev_*.deb + sudo dpkg -i target/debs/buster/libnl-nf-3-200_*.deb + sudo dpkg -i target/debs/buster/libnl-nf-3-dev_*.deb sudo dpkg -i libswsscommon_1.0.0_${{ parameters.arch }}.deb sudo dpkg -i libswsscommon-dev_1.0.0_${{ parameters.arch }}.deb sudo dpkg -i libsaivs_*.deb From 8caed56ea01294cab1d7c3f492efc3336ccf2e4e Mon Sep 17 00:00:00 2001 From: Tom cappleman Date: Tue, 5 Oct 2021 09:34:40 +0100 Subject: [PATCH 24/51] Pipeline fix --- .azure-pipelines/build-template.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.azure-pipelines/build-template.yml b/.azure-pipelines/build-template.yml index 2dbec2ffc7..251d2631a8 100644 --- a/.azure-pipelines/build-template.yml +++ b/.azure-pipelines/build-template.yml @@ -95,14 +95,14 @@ jobs: runBranch: 'refs/heads/master' displayName: "Download sonic buildimage deb packages" - script: | - sudo dpkg -i target/debs/buster/libnl-3-200_*.deb - sudo dpkg -i target/debs/buster/libnl-3-dev_*.deb - sudo dpkg -i target/debs/buster/libnl-genl-3-200_*.deb - sudo dpkg -i target/debs/buster/libnl-genl-3-dev_*.deb - sudo dpkg -i target/debs/buster/libnl-route-3-200_*.deb - sudo dpkg -i target/debs/buster/libnl-route-3-dev_*.deb - sudo dpkg -i target/debs/buster/libnl-nf-3-200_*.deb - sudo dpkg -i target/debs/buster/libnl-nf-3-dev_*.deb + sudo dpkg -i libnl-3-200_*_${{ parameters.arch }}.deb + sudo dpkg -i libnl-3-dev_*_${{ parameters.arch }}.deb + sudo dpkg -i libnl-genl-3-200_*_${{ parameters.arch }}.deb + sudo dpkg -i libnl-genl-3-dev_*_${{ parameters.arch }}.deb + sudo dpkg -i libnl-route-3-200_*_${{ parameters.arch }}.deb + sudo dpkg -i libnl-route-3-dev_*_${{ parameters.arch }}.deb + sudo dpkg -i libnl-nf-3-200_*_${{ parameters.arch }}.deb + sudo dpkg -i libnl-nf-3-dev_*_${{ parameters.arch }}.deb sudo dpkg -i libswsscommon_1.0.0_${{ parameters.arch }}.deb sudo dpkg -i libswsscommon-dev_1.0.0_${{ parameters.arch }}.deb sudo dpkg -i libsaivs_*.deb From 5f8d52e1264df7999e91e158b3b7adf001edcf62 Mon Sep 17 00:00:00 2001 From: Tom cappleman Date: Tue, 5 Oct 2021 10:39:50 +0100 Subject: [PATCH 25/51] Revert "Pipeline fix" This reverts commit 0aac13fbd980f39a592787afc21dd2ed2375449b. --- .azure-pipelines/build-template.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.azure-pipelines/build-template.yml b/.azure-pipelines/build-template.yml index 251d2631a8..2dbec2ffc7 100644 --- a/.azure-pipelines/build-template.yml +++ b/.azure-pipelines/build-template.yml @@ -95,14 +95,14 @@ jobs: runBranch: 'refs/heads/master' displayName: "Download sonic buildimage deb packages" - script: | - sudo dpkg -i libnl-3-200_*_${{ parameters.arch }}.deb - sudo dpkg -i libnl-3-dev_*_${{ parameters.arch }}.deb - sudo dpkg -i libnl-genl-3-200_*_${{ parameters.arch }}.deb - sudo dpkg -i libnl-genl-3-dev_*_${{ parameters.arch }}.deb - sudo dpkg -i libnl-route-3-200_*_${{ parameters.arch }}.deb - sudo dpkg -i libnl-route-3-dev_*_${{ parameters.arch }}.deb - sudo dpkg -i libnl-nf-3-200_*_${{ parameters.arch }}.deb - sudo dpkg -i libnl-nf-3-dev_*_${{ parameters.arch }}.deb + sudo dpkg -i target/debs/buster/libnl-3-200_*.deb + sudo dpkg -i target/debs/buster/libnl-3-dev_*.deb + sudo dpkg -i target/debs/buster/libnl-genl-3-200_*.deb + sudo dpkg -i target/debs/buster/libnl-genl-3-dev_*.deb + sudo dpkg -i target/debs/buster/libnl-route-3-200_*.deb + sudo dpkg -i target/debs/buster/libnl-route-3-dev_*.deb + sudo dpkg -i target/debs/buster/libnl-nf-3-200_*.deb + sudo dpkg -i target/debs/buster/libnl-nf-3-dev_*.deb sudo dpkg -i libswsscommon_1.0.0_${{ parameters.arch }}.deb sudo dpkg -i libswsscommon-dev_1.0.0_${{ parameters.arch }}.deb sudo dpkg -i libsaivs_*.deb From 240bc43a600827d67c7d94563a4284bea19fcc62 Mon Sep 17 00:00:00 2001 From: Tom cappleman Date: Tue, 5 Oct 2021 09:34:40 +0100 Subject: [PATCH 26/51] Pipeline fix --- .azure-pipelines/build-template.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.azure-pipelines/build-template.yml b/.azure-pipelines/build-template.yml index 2dbec2ffc7..251d2631a8 100644 --- a/.azure-pipelines/build-template.yml +++ b/.azure-pipelines/build-template.yml @@ -95,14 +95,14 @@ jobs: runBranch: 'refs/heads/master' displayName: "Download sonic buildimage deb packages" - script: | - sudo dpkg -i target/debs/buster/libnl-3-200_*.deb - sudo dpkg -i target/debs/buster/libnl-3-dev_*.deb - sudo dpkg -i target/debs/buster/libnl-genl-3-200_*.deb - sudo dpkg -i target/debs/buster/libnl-genl-3-dev_*.deb - sudo dpkg -i target/debs/buster/libnl-route-3-200_*.deb - sudo dpkg -i target/debs/buster/libnl-route-3-dev_*.deb - sudo dpkg -i target/debs/buster/libnl-nf-3-200_*.deb - sudo dpkg -i target/debs/buster/libnl-nf-3-dev_*.deb + sudo dpkg -i libnl-3-200_*_${{ parameters.arch }}.deb + sudo dpkg -i libnl-3-dev_*_${{ parameters.arch }}.deb + sudo dpkg -i libnl-genl-3-200_*_${{ parameters.arch }}.deb + sudo dpkg -i libnl-genl-3-dev_*_${{ parameters.arch }}.deb + sudo dpkg -i libnl-route-3-200_*_${{ parameters.arch }}.deb + sudo dpkg -i libnl-route-3-dev_*_${{ parameters.arch }}.deb + sudo dpkg -i libnl-nf-3-200_*_${{ parameters.arch }}.deb + sudo dpkg -i libnl-nf-3-dev_*_${{ parameters.arch }}.deb sudo dpkg -i libswsscommon_1.0.0_${{ parameters.arch }}.deb sudo dpkg -i libswsscommon-dev_1.0.0_${{ parameters.arch }}.deb sudo dpkg -i libsaivs_*.deb From cc7fd155c9c372a0872b82453541cb4edbda0cdc Mon Sep 17 00:00:00 2001 From: Tom cappleman Date: Tue, 5 Oct 2021 10:39:50 +0100 Subject: [PATCH 27/51] Revert "Pipeline fix" This reverts commit 0aac13fbd980f39a592787afc21dd2ed2375449b. --- .azure-pipelines/build-template.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.azure-pipelines/build-template.yml b/.azure-pipelines/build-template.yml index 251d2631a8..2dbec2ffc7 100644 --- a/.azure-pipelines/build-template.yml +++ b/.azure-pipelines/build-template.yml @@ -95,14 +95,14 @@ jobs: runBranch: 'refs/heads/master' displayName: "Download sonic buildimage deb packages" - script: | - sudo dpkg -i libnl-3-200_*_${{ parameters.arch }}.deb - sudo dpkg -i libnl-3-dev_*_${{ parameters.arch }}.deb - sudo dpkg -i libnl-genl-3-200_*_${{ parameters.arch }}.deb - sudo dpkg -i libnl-genl-3-dev_*_${{ parameters.arch }}.deb - sudo dpkg -i libnl-route-3-200_*_${{ parameters.arch }}.deb - sudo dpkg -i libnl-route-3-dev_*_${{ parameters.arch }}.deb - sudo dpkg -i libnl-nf-3-200_*_${{ parameters.arch }}.deb - sudo dpkg -i libnl-nf-3-dev_*_${{ parameters.arch }}.deb + sudo dpkg -i target/debs/buster/libnl-3-200_*.deb + sudo dpkg -i target/debs/buster/libnl-3-dev_*.deb + sudo dpkg -i target/debs/buster/libnl-genl-3-200_*.deb + sudo dpkg -i target/debs/buster/libnl-genl-3-dev_*.deb + sudo dpkg -i target/debs/buster/libnl-route-3-200_*.deb + sudo dpkg -i target/debs/buster/libnl-route-3-dev_*.deb + sudo dpkg -i target/debs/buster/libnl-nf-3-200_*.deb + sudo dpkg -i target/debs/buster/libnl-nf-3-dev_*.deb sudo dpkg -i libswsscommon_1.0.0_${{ parameters.arch }}.deb sudo dpkg -i libswsscommon-dev_1.0.0_${{ parameters.arch }}.deb sudo dpkg -i libsaivs_*.deb From cbc36ff8492063f5d53a656c6e91191655b77323 Mon Sep 17 00:00:00 2001 From: Tom cappleman Date: Tue, 5 Oct 2021 09:34:40 +0100 Subject: [PATCH 28/51] Pipeline fix --- .azure-pipelines/build-template.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.azure-pipelines/build-template.yml b/.azure-pipelines/build-template.yml index 2dbec2ffc7..251d2631a8 100644 --- a/.azure-pipelines/build-template.yml +++ b/.azure-pipelines/build-template.yml @@ -95,14 +95,14 @@ jobs: runBranch: 'refs/heads/master' displayName: "Download sonic buildimage deb packages" - script: | - sudo dpkg -i target/debs/buster/libnl-3-200_*.deb - sudo dpkg -i target/debs/buster/libnl-3-dev_*.deb - sudo dpkg -i target/debs/buster/libnl-genl-3-200_*.deb - sudo dpkg -i target/debs/buster/libnl-genl-3-dev_*.deb - sudo dpkg -i target/debs/buster/libnl-route-3-200_*.deb - sudo dpkg -i target/debs/buster/libnl-route-3-dev_*.deb - sudo dpkg -i target/debs/buster/libnl-nf-3-200_*.deb - sudo dpkg -i target/debs/buster/libnl-nf-3-dev_*.deb + sudo dpkg -i libnl-3-200_*_${{ parameters.arch }}.deb + sudo dpkg -i libnl-3-dev_*_${{ parameters.arch }}.deb + sudo dpkg -i libnl-genl-3-200_*_${{ parameters.arch }}.deb + sudo dpkg -i libnl-genl-3-dev_*_${{ parameters.arch }}.deb + sudo dpkg -i libnl-route-3-200_*_${{ parameters.arch }}.deb + sudo dpkg -i libnl-route-3-dev_*_${{ parameters.arch }}.deb + sudo dpkg -i libnl-nf-3-200_*_${{ parameters.arch }}.deb + sudo dpkg -i libnl-nf-3-dev_*_${{ parameters.arch }}.deb sudo dpkg -i libswsscommon_1.0.0_${{ parameters.arch }}.deb sudo dpkg -i libswsscommon-dev_1.0.0_${{ parameters.arch }}.deb sudo dpkg -i libsaivs_*.deb From e5331e5c167b69d02a66bd8a845148a3fdb8735f Mon Sep 17 00:00:00 2001 From: Tom cappleman Date: Tue, 5 Oct 2021 10:39:50 +0100 Subject: [PATCH 29/51] Revert "Pipeline fix" This reverts commit 0aac13fbd980f39a592787afc21dd2ed2375449b. --- .azure-pipelines/build-template.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.azure-pipelines/build-template.yml b/.azure-pipelines/build-template.yml index 251d2631a8..2dbec2ffc7 100644 --- a/.azure-pipelines/build-template.yml +++ b/.azure-pipelines/build-template.yml @@ -95,14 +95,14 @@ jobs: runBranch: 'refs/heads/master' displayName: "Download sonic buildimage deb packages" - script: | - sudo dpkg -i libnl-3-200_*_${{ parameters.arch }}.deb - sudo dpkg -i libnl-3-dev_*_${{ parameters.arch }}.deb - sudo dpkg -i libnl-genl-3-200_*_${{ parameters.arch }}.deb - sudo dpkg -i libnl-genl-3-dev_*_${{ parameters.arch }}.deb - sudo dpkg -i libnl-route-3-200_*_${{ parameters.arch }}.deb - sudo dpkg -i libnl-route-3-dev_*_${{ parameters.arch }}.deb - sudo dpkg -i libnl-nf-3-200_*_${{ parameters.arch }}.deb - sudo dpkg -i libnl-nf-3-dev_*_${{ parameters.arch }}.deb + sudo dpkg -i target/debs/buster/libnl-3-200_*.deb + sudo dpkg -i target/debs/buster/libnl-3-dev_*.deb + sudo dpkg -i target/debs/buster/libnl-genl-3-200_*.deb + sudo dpkg -i target/debs/buster/libnl-genl-3-dev_*.deb + sudo dpkg -i target/debs/buster/libnl-route-3-200_*.deb + sudo dpkg -i target/debs/buster/libnl-route-3-dev_*.deb + sudo dpkg -i target/debs/buster/libnl-nf-3-200_*.deb + sudo dpkg -i target/debs/buster/libnl-nf-3-dev_*.deb sudo dpkg -i libswsscommon_1.0.0_${{ parameters.arch }}.deb sudo dpkg -i libswsscommon-dev_1.0.0_${{ parameters.arch }}.deb sudo dpkg -i libsaivs_*.deb From cb5d3ad2043f2d061a3154bd0e80a251877d570f Mon Sep 17 00:00:00 2001 From: Tom cappleman Date: Tue, 5 Oct 2021 09:34:40 +0100 Subject: [PATCH 30/51] Pipeline fix --- .azure-pipelines/build-template.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.azure-pipelines/build-template.yml b/.azure-pipelines/build-template.yml index 2dbec2ffc7..251d2631a8 100644 --- a/.azure-pipelines/build-template.yml +++ b/.azure-pipelines/build-template.yml @@ -95,14 +95,14 @@ jobs: runBranch: 'refs/heads/master' displayName: "Download sonic buildimage deb packages" - script: | - sudo dpkg -i target/debs/buster/libnl-3-200_*.deb - sudo dpkg -i target/debs/buster/libnl-3-dev_*.deb - sudo dpkg -i target/debs/buster/libnl-genl-3-200_*.deb - sudo dpkg -i target/debs/buster/libnl-genl-3-dev_*.deb - sudo dpkg -i target/debs/buster/libnl-route-3-200_*.deb - sudo dpkg -i target/debs/buster/libnl-route-3-dev_*.deb - sudo dpkg -i target/debs/buster/libnl-nf-3-200_*.deb - sudo dpkg -i target/debs/buster/libnl-nf-3-dev_*.deb + sudo dpkg -i libnl-3-200_*_${{ parameters.arch }}.deb + sudo dpkg -i libnl-3-dev_*_${{ parameters.arch }}.deb + sudo dpkg -i libnl-genl-3-200_*_${{ parameters.arch }}.deb + sudo dpkg -i libnl-genl-3-dev_*_${{ parameters.arch }}.deb + sudo dpkg -i libnl-route-3-200_*_${{ parameters.arch }}.deb + sudo dpkg -i libnl-route-3-dev_*_${{ parameters.arch }}.deb + sudo dpkg -i libnl-nf-3-200_*_${{ parameters.arch }}.deb + sudo dpkg -i libnl-nf-3-dev_*_${{ parameters.arch }}.deb sudo dpkg -i libswsscommon_1.0.0_${{ parameters.arch }}.deb sudo dpkg -i libswsscommon-dev_1.0.0_${{ parameters.arch }}.deb sudo dpkg -i libsaivs_*.deb From 77818c8f9b4ca90fe0e1e1a6c15f32a695d5424d Mon Sep 17 00:00:00 2001 From: Tom cappleman Date: Tue, 5 Oct 2021 10:39:50 +0100 Subject: [PATCH 31/51] Revert "Pipeline fix" This reverts commit 0aac13fbd980f39a592787afc21dd2ed2375449b. --- .azure-pipelines/build-template.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.azure-pipelines/build-template.yml b/.azure-pipelines/build-template.yml index 251d2631a8..2dbec2ffc7 100644 --- a/.azure-pipelines/build-template.yml +++ b/.azure-pipelines/build-template.yml @@ -95,14 +95,14 @@ jobs: runBranch: 'refs/heads/master' displayName: "Download sonic buildimage deb packages" - script: | - sudo dpkg -i libnl-3-200_*_${{ parameters.arch }}.deb - sudo dpkg -i libnl-3-dev_*_${{ parameters.arch }}.deb - sudo dpkg -i libnl-genl-3-200_*_${{ parameters.arch }}.deb - sudo dpkg -i libnl-genl-3-dev_*_${{ parameters.arch }}.deb - sudo dpkg -i libnl-route-3-200_*_${{ parameters.arch }}.deb - sudo dpkg -i libnl-route-3-dev_*_${{ parameters.arch }}.deb - sudo dpkg -i libnl-nf-3-200_*_${{ parameters.arch }}.deb - sudo dpkg -i libnl-nf-3-dev_*_${{ parameters.arch }}.deb + sudo dpkg -i target/debs/buster/libnl-3-200_*.deb + sudo dpkg -i target/debs/buster/libnl-3-dev_*.deb + sudo dpkg -i target/debs/buster/libnl-genl-3-200_*.deb + sudo dpkg -i target/debs/buster/libnl-genl-3-dev_*.deb + sudo dpkg -i target/debs/buster/libnl-route-3-200_*.deb + sudo dpkg -i target/debs/buster/libnl-route-3-dev_*.deb + sudo dpkg -i target/debs/buster/libnl-nf-3-200_*.deb + sudo dpkg -i target/debs/buster/libnl-nf-3-dev_*.deb sudo dpkg -i libswsscommon_1.0.0_${{ parameters.arch }}.deb sudo dpkg -i libswsscommon-dev_1.0.0_${{ parameters.arch }}.deb sudo dpkg -i libsaivs_*.deb From 1b69823fa847a5591d507d2eaaa50f52264a775e Mon Sep 17 00:00:00 2001 From: Tom cappleman Date: Tue, 5 Oct 2021 09:34:40 +0100 Subject: [PATCH 32/51] Pipeline fix --- .azure-pipelines/build-template.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.azure-pipelines/build-template.yml b/.azure-pipelines/build-template.yml index 2dbec2ffc7..251d2631a8 100644 --- a/.azure-pipelines/build-template.yml +++ b/.azure-pipelines/build-template.yml @@ -95,14 +95,14 @@ jobs: runBranch: 'refs/heads/master' displayName: "Download sonic buildimage deb packages" - script: | - sudo dpkg -i target/debs/buster/libnl-3-200_*.deb - sudo dpkg -i target/debs/buster/libnl-3-dev_*.deb - sudo dpkg -i target/debs/buster/libnl-genl-3-200_*.deb - sudo dpkg -i target/debs/buster/libnl-genl-3-dev_*.deb - sudo dpkg -i target/debs/buster/libnl-route-3-200_*.deb - sudo dpkg -i target/debs/buster/libnl-route-3-dev_*.deb - sudo dpkg -i target/debs/buster/libnl-nf-3-200_*.deb - sudo dpkg -i target/debs/buster/libnl-nf-3-dev_*.deb + sudo dpkg -i libnl-3-200_*_${{ parameters.arch }}.deb + sudo dpkg -i libnl-3-dev_*_${{ parameters.arch }}.deb + sudo dpkg -i libnl-genl-3-200_*_${{ parameters.arch }}.deb + sudo dpkg -i libnl-genl-3-dev_*_${{ parameters.arch }}.deb + sudo dpkg -i libnl-route-3-200_*_${{ parameters.arch }}.deb + sudo dpkg -i libnl-route-3-dev_*_${{ parameters.arch }}.deb + sudo dpkg -i libnl-nf-3-200_*_${{ parameters.arch }}.deb + sudo dpkg -i libnl-nf-3-dev_*_${{ parameters.arch }}.deb sudo dpkg -i libswsscommon_1.0.0_${{ parameters.arch }}.deb sudo dpkg -i libswsscommon-dev_1.0.0_${{ parameters.arch }}.deb sudo dpkg -i libsaivs_*.deb From 4315d7cf118c765834ede7b0e89dbcc32d197784 Mon Sep 17 00:00:00 2001 From: Tom cappleman Date: Tue, 5 Oct 2021 10:39:50 +0100 Subject: [PATCH 33/51] Revert "Pipeline fix" This reverts commit 0aac13fbd980f39a592787afc21dd2ed2375449b. --- .azure-pipelines/build-template.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.azure-pipelines/build-template.yml b/.azure-pipelines/build-template.yml index 251d2631a8..2dbec2ffc7 100644 --- a/.azure-pipelines/build-template.yml +++ b/.azure-pipelines/build-template.yml @@ -95,14 +95,14 @@ jobs: runBranch: 'refs/heads/master' displayName: "Download sonic buildimage deb packages" - script: | - sudo dpkg -i libnl-3-200_*_${{ parameters.arch }}.deb - sudo dpkg -i libnl-3-dev_*_${{ parameters.arch }}.deb - sudo dpkg -i libnl-genl-3-200_*_${{ parameters.arch }}.deb - sudo dpkg -i libnl-genl-3-dev_*_${{ parameters.arch }}.deb - sudo dpkg -i libnl-route-3-200_*_${{ parameters.arch }}.deb - sudo dpkg -i libnl-route-3-dev_*_${{ parameters.arch }}.deb - sudo dpkg -i libnl-nf-3-200_*_${{ parameters.arch }}.deb - sudo dpkg -i libnl-nf-3-dev_*_${{ parameters.arch }}.deb + sudo dpkg -i target/debs/buster/libnl-3-200_*.deb + sudo dpkg -i target/debs/buster/libnl-3-dev_*.deb + sudo dpkg -i target/debs/buster/libnl-genl-3-200_*.deb + sudo dpkg -i target/debs/buster/libnl-genl-3-dev_*.deb + sudo dpkg -i target/debs/buster/libnl-route-3-200_*.deb + sudo dpkg -i target/debs/buster/libnl-route-3-dev_*.deb + sudo dpkg -i target/debs/buster/libnl-nf-3-200_*.deb + sudo dpkg -i target/debs/buster/libnl-nf-3-dev_*.deb sudo dpkg -i libswsscommon_1.0.0_${{ parameters.arch }}.deb sudo dpkg -i libswsscommon-dev_1.0.0_${{ parameters.arch }}.deb sudo dpkg -i libsaivs_*.deb From b8b19e846389906a659c6d355c72983f78b549c6 Mon Sep 17 00:00:00 2001 From: Tom cappleman Date: Tue, 5 Oct 2021 09:34:40 +0100 Subject: [PATCH 34/51] Pipeline fix --- .azure-pipelines/build-template.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.azure-pipelines/build-template.yml b/.azure-pipelines/build-template.yml index 2dbec2ffc7..251d2631a8 100644 --- a/.azure-pipelines/build-template.yml +++ b/.azure-pipelines/build-template.yml @@ -95,14 +95,14 @@ jobs: runBranch: 'refs/heads/master' displayName: "Download sonic buildimage deb packages" - script: | - sudo dpkg -i target/debs/buster/libnl-3-200_*.deb - sudo dpkg -i target/debs/buster/libnl-3-dev_*.deb - sudo dpkg -i target/debs/buster/libnl-genl-3-200_*.deb - sudo dpkg -i target/debs/buster/libnl-genl-3-dev_*.deb - sudo dpkg -i target/debs/buster/libnl-route-3-200_*.deb - sudo dpkg -i target/debs/buster/libnl-route-3-dev_*.deb - sudo dpkg -i target/debs/buster/libnl-nf-3-200_*.deb - sudo dpkg -i target/debs/buster/libnl-nf-3-dev_*.deb + sudo dpkg -i libnl-3-200_*_${{ parameters.arch }}.deb + sudo dpkg -i libnl-3-dev_*_${{ parameters.arch }}.deb + sudo dpkg -i libnl-genl-3-200_*_${{ parameters.arch }}.deb + sudo dpkg -i libnl-genl-3-dev_*_${{ parameters.arch }}.deb + sudo dpkg -i libnl-route-3-200_*_${{ parameters.arch }}.deb + sudo dpkg -i libnl-route-3-dev_*_${{ parameters.arch }}.deb + sudo dpkg -i libnl-nf-3-200_*_${{ parameters.arch }}.deb + sudo dpkg -i libnl-nf-3-dev_*_${{ parameters.arch }}.deb sudo dpkg -i libswsscommon_1.0.0_${{ parameters.arch }}.deb sudo dpkg -i libswsscommon-dev_1.0.0_${{ parameters.arch }}.deb sudo dpkg -i libsaivs_*.deb From 5d8ed464ca92b7dcf25be4fefd38a4efde52ea5a Mon Sep 17 00:00:00 2001 From: Tom cappleman Date: Tue, 5 Oct 2021 10:39:50 +0100 Subject: [PATCH 35/51] Revert "Pipeline fix" This reverts commit 0aac13fbd980f39a592787afc21dd2ed2375449b. --- .azure-pipelines/build-template.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.azure-pipelines/build-template.yml b/.azure-pipelines/build-template.yml index 251d2631a8..2dbec2ffc7 100644 --- a/.azure-pipelines/build-template.yml +++ b/.azure-pipelines/build-template.yml @@ -95,14 +95,14 @@ jobs: runBranch: 'refs/heads/master' displayName: "Download sonic buildimage deb packages" - script: | - sudo dpkg -i libnl-3-200_*_${{ parameters.arch }}.deb - sudo dpkg -i libnl-3-dev_*_${{ parameters.arch }}.deb - sudo dpkg -i libnl-genl-3-200_*_${{ parameters.arch }}.deb - sudo dpkg -i libnl-genl-3-dev_*_${{ parameters.arch }}.deb - sudo dpkg -i libnl-route-3-200_*_${{ parameters.arch }}.deb - sudo dpkg -i libnl-route-3-dev_*_${{ parameters.arch }}.deb - sudo dpkg -i libnl-nf-3-200_*_${{ parameters.arch }}.deb - sudo dpkg -i libnl-nf-3-dev_*_${{ parameters.arch }}.deb + sudo dpkg -i target/debs/buster/libnl-3-200_*.deb + sudo dpkg -i target/debs/buster/libnl-3-dev_*.deb + sudo dpkg -i target/debs/buster/libnl-genl-3-200_*.deb + sudo dpkg -i target/debs/buster/libnl-genl-3-dev_*.deb + sudo dpkg -i target/debs/buster/libnl-route-3-200_*.deb + sudo dpkg -i target/debs/buster/libnl-route-3-dev_*.deb + sudo dpkg -i target/debs/buster/libnl-nf-3-200_*.deb + sudo dpkg -i target/debs/buster/libnl-nf-3-dev_*.deb sudo dpkg -i libswsscommon_1.0.0_${{ parameters.arch }}.deb sudo dpkg -i libswsscommon-dev_1.0.0_${{ parameters.arch }}.deb sudo dpkg -i libsaivs_*.deb From c64b8cfdf5f4aa3893f089e04685bf22520f8a50 Mon Sep 17 00:00:00 2001 From: Tom cappleman Date: Thu, 7 Oct 2021 09:12:49 +0100 Subject: [PATCH 36/51] Fix removed code --- orchagent/routeorch.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/orchagent/routeorch.cpp b/orchagent/routeorch.cpp index 9fc1c6a31d..e39b05c343 100644 --- a/orchagent/routeorch.cpp +++ b/orchagent/routeorch.cpp @@ -2230,6 +2230,9 @@ bool RouteOrch::removeRoutePost(const RouteBulkContext& ctx) m_neighOrch->removeMplsNextHop(nexthop); } } + + RouteKey r_key = { vrf_id, ipPrefix }; + removeNextHopRoute(nexthop, r_key); } SWSS_LOG_INFO("Remove route %s with next hop(s) %s", From 207e4c0809cdef6af139cb8bde34ae95e454eaee Mon Sep 17 00:00:00 2001 From: Tom cappleman Date: Thu, 7 Oct 2021 09:24:20 +0100 Subject: [PATCH 37/51] Fix removed code --- orchagent/routeorch.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/orchagent/routeorch.cpp b/orchagent/routeorch.cpp index e39b05c343..4dd3a0fff1 100644 --- a/orchagent/routeorch.cpp +++ b/orchagent/routeorch.cpp @@ -2229,10 +2229,10 @@ bool RouteOrch::removeRoutePost(const RouteBulkContext& ctx) { m_neighOrch->removeMplsNextHop(nexthop); } - } - RouteKey r_key = { vrf_id, ipPrefix }; - removeNextHopRoute(nexthop, r_key); + RouteKey r_key = { vrf_id, ipPrefix }; + removeNextHopRoute(nexthop, r_key); + } } SWSS_LOG_INFO("Remove route %s with next hop(s) %s", From 5f14a7c5c2c121cb225f26a06da62881d35cdcaf Mon Sep 17 00:00:00 2001 From: Tom cappleman Date: Fri, 8 Oct 2021 09:18:24 +0100 Subject: [PATCH 38/51] Minor fix --- orchagent/routeorch.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/orchagent/routeorch.cpp b/orchagent/routeorch.cpp index 4dd3a0fff1..453d5d3c86 100644 --- a/orchagent/routeorch.cpp +++ b/orchagent/routeorch.cpp @@ -2200,17 +2200,17 @@ bool RouteOrch::removeRoutePost(const RouteBulkContext& ctx) /* The NHG is owned by RouteOrch */ else { - auto ol_nextHops = it_route->second.nhg_key; - /* * Decrease the reference count only when the route is pointing to a next hop. */ decreaseNextHopRefCount(ol_nextHops); - if (ol_nextHops.getSize() > 1 - && m_syncdNextHopGroups[ol_nextHops].ref_count == 0) + auto ol_nextHops = it_route->second.nhg_key; + + if (it_route->second.nhg_key.getSize() > 1 + && m_syncdNextHopGroups[it_route->second.nhg_key].ref_count == 0) { - m_bulkNhgReducedRefCnt.emplace(ol_nextHops, 0); + m_bulkNhgReducedRefCnt.emplace(it_route->second.nhg_key, 0); } else if (ol_nextHops.is_overlay_nexthop()) { @@ -2221,7 +2221,7 @@ bool RouteOrch::removeRoutePost(const RouteBulkContext& ctx) * Additionally check if the NH has label and its ref count == 0, then * remove the label next hop. */ - else if (ol_nextHops.getSize() == 1) + else if (it_route->second.nhg_key.getSize() == 1) { const NextHopKey& nexthop = *it_route->second.nhg_key.getNextHops().begin(); if (nexthop.isMplsNextHop() && From e670d6fe3ef9a0efde203859c1a6c0391b264928 Mon Sep 17 00:00:00 2001 From: Tom cappleman Date: Fri, 8 Oct 2021 09:23:10 +0100 Subject: [PATCH 39/51] Minor fix --- orchagent/routeorch.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/orchagent/routeorch.cpp b/orchagent/routeorch.cpp index 453d5d3c86..769229cba7 100644 --- a/orchagent/routeorch.cpp +++ b/orchagent/routeorch.cpp @@ -2201,9 +2201,9 @@ bool RouteOrch::removeRoutePost(const RouteBulkContext& ctx) else { /* - * Decrease the reference count only when the route is pointing to a next hop. - */ - decreaseNextHopRefCount(ol_nextHops); + * Decrease the reference count only when the route is pointing to a next hop. + */ + decreaseNextHopRefCount(it_route->second.nhg_key); auto ol_nextHops = it_route->second.nhg_key; @@ -2218,9 +2218,9 @@ bool RouteOrch::removeRoutePost(const RouteBulkContext& ctx) m_bulkNhgReducedRefCnt.emplace(ol_nextHops, vrf_id); } /* - * Additionally check if the NH has label and its ref count == 0, then - * remove the label next hop. - */ + * Additionally check if the NH has label and its ref count == 0, then + * remove the label next hop. + */ else if (it_route->second.nhg_key.getSize() == 1) { const NextHopKey& nexthop = *it_route->second.nhg_key.getNextHops().begin(); From bed44e33be7ca61965345105c6f9546766812e3c Mon Sep 17 00:00:00 2001 From: Tom cappleman Date: Tue, 5 Oct 2021 09:34:40 +0100 Subject: [PATCH 40/51] Pipeline fix --- .azure-pipelines/build-template.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.azure-pipelines/build-template.yml b/.azure-pipelines/build-template.yml index 2dbec2ffc7..251d2631a8 100644 --- a/.azure-pipelines/build-template.yml +++ b/.azure-pipelines/build-template.yml @@ -95,14 +95,14 @@ jobs: runBranch: 'refs/heads/master' displayName: "Download sonic buildimage deb packages" - script: | - sudo dpkg -i target/debs/buster/libnl-3-200_*.deb - sudo dpkg -i target/debs/buster/libnl-3-dev_*.deb - sudo dpkg -i target/debs/buster/libnl-genl-3-200_*.deb - sudo dpkg -i target/debs/buster/libnl-genl-3-dev_*.deb - sudo dpkg -i target/debs/buster/libnl-route-3-200_*.deb - sudo dpkg -i target/debs/buster/libnl-route-3-dev_*.deb - sudo dpkg -i target/debs/buster/libnl-nf-3-200_*.deb - sudo dpkg -i target/debs/buster/libnl-nf-3-dev_*.deb + sudo dpkg -i libnl-3-200_*_${{ parameters.arch }}.deb + sudo dpkg -i libnl-3-dev_*_${{ parameters.arch }}.deb + sudo dpkg -i libnl-genl-3-200_*_${{ parameters.arch }}.deb + sudo dpkg -i libnl-genl-3-dev_*_${{ parameters.arch }}.deb + sudo dpkg -i libnl-route-3-200_*_${{ parameters.arch }}.deb + sudo dpkg -i libnl-route-3-dev_*_${{ parameters.arch }}.deb + sudo dpkg -i libnl-nf-3-200_*_${{ parameters.arch }}.deb + sudo dpkg -i libnl-nf-3-dev_*_${{ parameters.arch }}.deb sudo dpkg -i libswsscommon_1.0.0_${{ parameters.arch }}.deb sudo dpkg -i libswsscommon-dev_1.0.0_${{ parameters.arch }}.deb sudo dpkg -i libsaivs_*.deb From 5961551daa0f1c3a03652073e1755766b3f2183e Mon Sep 17 00:00:00 2001 From: Tom cappleman Date: Tue, 5 Oct 2021 10:39:50 +0100 Subject: [PATCH 41/51] Revert "Pipeline fix" This reverts commit 0aac13fbd980f39a592787afc21dd2ed2375449b. --- .azure-pipelines/build-template.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.azure-pipelines/build-template.yml b/.azure-pipelines/build-template.yml index 251d2631a8..2dbec2ffc7 100644 --- a/.azure-pipelines/build-template.yml +++ b/.azure-pipelines/build-template.yml @@ -95,14 +95,14 @@ jobs: runBranch: 'refs/heads/master' displayName: "Download sonic buildimage deb packages" - script: | - sudo dpkg -i libnl-3-200_*_${{ parameters.arch }}.deb - sudo dpkg -i libnl-3-dev_*_${{ parameters.arch }}.deb - sudo dpkg -i libnl-genl-3-200_*_${{ parameters.arch }}.deb - sudo dpkg -i libnl-genl-3-dev_*_${{ parameters.arch }}.deb - sudo dpkg -i libnl-route-3-200_*_${{ parameters.arch }}.deb - sudo dpkg -i libnl-route-3-dev_*_${{ parameters.arch }}.deb - sudo dpkg -i libnl-nf-3-200_*_${{ parameters.arch }}.deb - sudo dpkg -i libnl-nf-3-dev_*_${{ parameters.arch }}.deb + sudo dpkg -i target/debs/buster/libnl-3-200_*.deb + sudo dpkg -i target/debs/buster/libnl-3-dev_*.deb + sudo dpkg -i target/debs/buster/libnl-genl-3-200_*.deb + sudo dpkg -i target/debs/buster/libnl-genl-3-dev_*.deb + sudo dpkg -i target/debs/buster/libnl-route-3-200_*.deb + sudo dpkg -i target/debs/buster/libnl-route-3-dev_*.deb + sudo dpkg -i target/debs/buster/libnl-nf-3-200_*.deb + sudo dpkg -i target/debs/buster/libnl-nf-3-dev_*.deb sudo dpkg -i libswsscommon_1.0.0_${{ parameters.arch }}.deb sudo dpkg -i libswsscommon-dev_1.0.0_${{ parameters.arch }}.deb sudo dpkg -i libsaivs_*.deb From 260abf18ac5e1d07bc46524d8fb2f07512892384 Mon Sep 17 00:00:00 2001 From: Tom cappleman Date: Tue, 5 Oct 2021 09:34:40 +0100 Subject: [PATCH 42/51] Pipeline fix --- .azure-pipelines/build-template.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.azure-pipelines/build-template.yml b/.azure-pipelines/build-template.yml index 2dbec2ffc7..251d2631a8 100644 --- a/.azure-pipelines/build-template.yml +++ b/.azure-pipelines/build-template.yml @@ -95,14 +95,14 @@ jobs: runBranch: 'refs/heads/master' displayName: "Download sonic buildimage deb packages" - script: | - sudo dpkg -i target/debs/buster/libnl-3-200_*.deb - sudo dpkg -i target/debs/buster/libnl-3-dev_*.deb - sudo dpkg -i target/debs/buster/libnl-genl-3-200_*.deb - sudo dpkg -i target/debs/buster/libnl-genl-3-dev_*.deb - sudo dpkg -i target/debs/buster/libnl-route-3-200_*.deb - sudo dpkg -i target/debs/buster/libnl-route-3-dev_*.deb - sudo dpkg -i target/debs/buster/libnl-nf-3-200_*.deb - sudo dpkg -i target/debs/buster/libnl-nf-3-dev_*.deb + sudo dpkg -i libnl-3-200_*_${{ parameters.arch }}.deb + sudo dpkg -i libnl-3-dev_*_${{ parameters.arch }}.deb + sudo dpkg -i libnl-genl-3-200_*_${{ parameters.arch }}.deb + sudo dpkg -i libnl-genl-3-dev_*_${{ parameters.arch }}.deb + sudo dpkg -i libnl-route-3-200_*_${{ parameters.arch }}.deb + sudo dpkg -i libnl-route-3-dev_*_${{ parameters.arch }}.deb + sudo dpkg -i libnl-nf-3-200_*_${{ parameters.arch }}.deb + sudo dpkg -i libnl-nf-3-dev_*_${{ parameters.arch }}.deb sudo dpkg -i libswsscommon_1.0.0_${{ parameters.arch }}.deb sudo dpkg -i libswsscommon-dev_1.0.0_${{ parameters.arch }}.deb sudo dpkg -i libsaivs_*.deb From 54234b45c9577b090cfbdec91dd1a2f87a530916 Mon Sep 17 00:00:00 2001 From: Tom cappleman Date: Tue, 5 Oct 2021 10:39:50 +0100 Subject: [PATCH 43/51] Revert "Pipeline fix" This reverts commit 0aac13fbd980f39a592787afc21dd2ed2375449b. --- .azure-pipelines/build-template.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.azure-pipelines/build-template.yml b/.azure-pipelines/build-template.yml index 251d2631a8..2dbec2ffc7 100644 --- a/.azure-pipelines/build-template.yml +++ b/.azure-pipelines/build-template.yml @@ -95,14 +95,14 @@ jobs: runBranch: 'refs/heads/master' displayName: "Download sonic buildimage deb packages" - script: | - sudo dpkg -i libnl-3-200_*_${{ parameters.arch }}.deb - sudo dpkg -i libnl-3-dev_*_${{ parameters.arch }}.deb - sudo dpkg -i libnl-genl-3-200_*_${{ parameters.arch }}.deb - sudo dpkg -i libnl-genl-3-dev_*_${{ parameters.arch }}.deb - sudo dpkg -i libnl-route-3-200_*_${{ parameters.arch }}.deb - sudo dpkg -i libnl-route-3-dev_*_${{ parameters.arch }}.deb - sudo dpkg -i libnl-nf-3-200_*_${{ parameters.arch }}.deb - sudo dpkg -i libnl-nf-3-dev_*_${{ parameters.arch }}.deb + sudo dpkg -i target/debs/buster/libnl-3-200_*.deb + sudo dpkg -i target/debs/buster/libnl-3-dev_*.deb + sudo dpkg -i target/debs/buster/libnl-genl-3-200_*.deb + sudo dpkg -i target/debs/buster/libnl-genl-3-dev_*.deb + sudo dpkg -i target/debs/buster/libnl-route-3-200_*.deb + sudo dpkg -i target/debs/buster/libnl-route-3-dev_*.deb + sudo dpkg -i target/debs/buster/libnl-nf-3-200_*.deb + sudo dpkg -i target/debs/buster/libnl-nf-3-dev_*.deb sudo dpkg -i libswsscommon_1.0.0_${{ parameters.arch }}.deb sudo dpkg -i libswsscommon-dev_1.0.0_${{ parameters.arch }}.deb sudo dpkg -i libsaivs_*.deb From 2b0a5338daab0d2f517d56f5da4020d1b9fec6da Mon Sep 17 00:00:00 2001 From: Tom cappleman Date: Tue, 12 Oct 2021 09:19:34 +0100 Subject: [PATCH 44/51] Fixes --- orchagent/mplsrouteorch.cpp | 208 +++++++++++++++++------------------- orchagent/routeorch.h | 2 +- 2 files changed, 101 insertions(+), 109 deletions(-) diff --git a/orchagent/mplsrouteorch.cpp b/orchagent/mplsrouteorch.cpp index 8afdc1af02..276c7e243e 100644 --- a/orchagent/mplsrouteorch.cpp +++ b/orchagent/mplsrouteorch.cpp @@ -474,97 +474,93 @@ bool RouteOrch::addLabelRoute(LabelRouteBulkContext& ctx, const NextHopGroupKey auto it_route = m_syncdLabelRoutes.at(vrf_id).find(label); - if (ctx.nhg_index.empty()) + if (!ctx.nhg_index.empty()) { - if (nextHops.getSize() == 0) + try { - /* The route is pointing to a blackhole */ - blackhole = true; + const NextHopGroup& nhg = gNhgOrch->getNhg(ctx.nhg_index); + next_hop_id = nhg.getId(); } - /* The route is pointing to a next hop */ - else if (nextHops.getSize() == 1) + catch(const std::out_of_range& e) { - const NextHopKey& nexthop = *nextHops.getNextHops().begin(); - if (nexthop.isIntfNextHop()) + SWSS_LOG_WARN("Next hop group key %s does not exist", ctx.nhg_index.c_str()); + return false; + } + } + else if (nextHops.getSize() == 0) + { + /* The route is pointing to a blackhole */ + blackhole = true; + } + /* The route is pointing to a next hop */ + else if (nextHops.getSize() == 1) + { + const NextHopKey& nexthop = *nextHops.getNextHops().begin(); + if (nexthop.isIntfNextHop()) + { + next_hop_id = m_intfsOrch->getRouterIntfsId(nexthop.alias); + /* rif is not created yet */ + if (next_hop_id == SAI_NULL_OBJECT_ID) { - next_hop_id = m_intfsOrch->getRouterIntfsId(nexthop.alias); - /* rif is not created yet */ - if (next_hop_id == SAI_NULL_OBJECT_ID) - { - SWSS_LOG_INFO("Failed to get next hop %s for %u", - nextHops.to_string().c_str(), label); - return false; - } + SWSS_LOG_INFO("Failed to get next hop %s for %u", + nextHops.to_string().c_str(), label); + return false; + } + } + else + { + if (m_neighOrch->hasNextHop(nexthop)) + { + next_hop_id = m_neighOrch->getNextHopId(nexthop); + } + /* See if there is an IP neighbor nexthop */ + else if (nexthop.isMplsNextHop() && + m_neighOrch->isNeighborResolved(nexthop)) + { + m_neighOrch->addNextHop(nexthop); + next_hop_id = m_neighOrch->getNextHopId(nexthop); } else { - if (m_neighOrch->hasNextHop(nexthop)) - { - next_hop_id = m_neighOrch->getNextHopId(nexthop); - } - /* See if there is an IP neighbor nexthop */ - else if (nexthop.isMplsNextHop() && - m_neighOrch->isNeighborResolved(nexthop)) - { - m_neighOrch->addNextHop(nexthop); - next_hop_id = m_neighOrch->getNextHopId(nexthop); - } - else - { - SWSS_LOG_INFO("Failed to get next hop %s for %u", - nextHops.to_string().c_str(), label); - return false; - } + SWSS_LOG_INFO("Failed to get next hop %s for %u", + nextHops.to_string().c_str(), label); + return false; } } - /* The route is pointing to a next hop group */ - else + } + /* The route is pointing to a next hop group */ + else + { + /* Check if there is already an existing next hop group */ + if (!hasNextHopGroup(nextHops)) { - /* Check if there is already an existing next hop group */ - if (!hasNextHopGroup(nextHops)) + /* Try to create a new next hop group */ + if (!addNextHopGroup(nextHops)) { - /* Try to create a new next hop group */ - if (!addNextHopGroup(nextHops)) - { - /* Failed to create the next hop group and check if a temporary route is needed */ + /* Failed to create the next hop group and check if a temporary route is needed */ - /* If the current next hop is part of the next hop group to sync, - * then return false and no need to add another temporary route. */ - if (it_route != m_syncdLabelRoutes.at(vrf_id).end() && - it_route->second.nhg_key.getSize() == 1) + /* If the current next hop is part of the next hop group to sync, + * then return false and no need to add another temporary route. */ + if (it_route != m_syncdLabelRoutes.at(vrf_id).end() && + it_route->second.nhg_key.getSize() == 1) + { + const NextHopKey& nexthop = *it_route->second.nhg_key.getNextHops().begin(); + if (nextHops.contains(nexthop)) { - const NextHopKey& nexthop = *it_route->second.nhg_key.getNextHops().begin(); - if (nextHops.contains(nexthop)) - { - return false; - } + return false; } - - /* Add a temporary route when a next hop group cannot be added, - * and there is no temporary route right now or the current temporary - * route is not pointing to a member of the next hop group to sync. */ - addTempLabelRoute(ctx, nextHops); - /* Return false since the original route is not successfully added */ - return false; } - } - next_hop_id = m_syncdNextHopGroups[nextHops].next_hop_group_id; - } - } - else - { - SWSS_LOG_DEBUG("Next hop group is owned by NhgOrch with index %s", ctx.nhg_index.c_str()); - try - { - const NextHopGroup& nhg = gNhgOrch->getNhg(ctx.nhg_index); - next_hop_id = nhg.getId(); - } - catch(const std::out_of_range& e) - { - SWSS_LOG_WARN("Next hop group key %s does not exist", ctx.nhg_index.c_str()); - return false; + /* Add a temporary route when a next hop group cannot be added, + * and there is no temporary route right now or the current temporary + * route is not pointing to a member of the next hop group to sync. */ + addTempLabelRoute(ctx, nextHops); + /* Return false since the original route is not successfully added */ + return false; + } } + + next_hop_id = m_syncdNextHopGroups[nextHops].next_hop_group_id; } /* Sync the inseg entry */ @@ -661,57 +657,53 @@ bool RouteOrch::addLabelRoutePost(const LabelRouteBulkContext& ctx, const NextHo sai_object_id_t next_hop_id; /* Check that the next hop group is not owned by NhgOrch. */ - if (ctx.nhg_index.empty()) + if (!ctx.nhg_index.empty()) { - if (nextHops.getSize() == 0) + if (!gNhgOrch->hasNhg(ctx.nhg_index)) { - /* The route is pointing to a blackhole */ - blackhole = true; + SWSS_LOG_WARN("Failed to get next hop group with index %s", ctx.nhg_index.c_str()); + return false; } - /* The route is pointing to a next hop */ - else if (nextHops.getSize() == 1) + } + else if (nextHops.getSize() == 0) + { + /* The route is pointing to a blackhole */ + blackhole = true; + } + /* The route is pointing to a next hop */ + else if (nextHops.getSize() == 1) + { + const NextHopKey& nexthop = *nextHops.getNextHops().begin(); + if (nexthop.isIntfNextHop()) { - const NextHopKey& nexthop = *nextHops.getNextHops().begin(); - if (nexthop.isIntfNextHop()) - { - next_hop_id = m_intfsOrch->getRouterIntfsId(nexthop.alias); - /* rif is not created yet */ - if (next_hop_id == SAI_NULL_OBJECT_ID) - { - SWSS_LOG_INFO("Failed to get next hop %s for label %u", - nextHops.to_string().c_str(), label); - return false; - } - } - else + next_hop_id = m_intfsOrch->getRouterIntfsId(nexthop.alias); + /* rif is not created yet */ + if (next_hop_id == SAI_NULL_OBJECT_ID) { - if (!m_neighOrch->hasNextHop(nexthop)) - { - SWSS_LOG_INFO("Failed to get next hop %s for label %u", - nextHops.to_string().c_str(), label); - return false; - } + SWSS_LOG_INFO("Failed to get next hop %s for label %u", + nextHops.to_string().c_str(), label); + return false; } } - /* The route is pointing to a next hop group */ - else if (nextHops.getSize() > 1) + else { - if (!hasNextHopGroup(nextHops)) + if (!m_neighOrch->hasNextHop(nexthop)) { - // Previous added an temporary route - auto& tmp_next_hop = ctx.tmp_next_hop; - addLabelRoutePost(ctx, tmp_next_hop); + SWSS_LOG_INFO("Failed to get next hop %s for label %u", + nextHops.to_string().c_str(), label); return false; } } } - else + /* The route is pointing to a next hop group */ + else if (nextHops.getSize() > 1) { - SWSS_LOG_DEBUG("NhgOrch owns the next hop group with index %s", ctx.nhg_index.c_str()); - if (!gNhgOrch->hasNhg(ctx.nhg_index)) + if (!hasNextHopGroup(nextHops)) { - SWSS_LOG_WARN("Failed to get next hop group with index %s", ctx.nhg_index.c_str()); + // Previous added an temporary route + auto& tmp_next_hop = ctx.tmp_next_hop; + addLabelRoutePost(ctx, tmp_next_hop); return false; } } diff --git a/orchagent/routeorch.h b/orchagent/routeorch.h index a619eab428..d28ba4322e 100644 --- a/orchagent/routeorch.h +++ b/orchagent/routeorch.h @@ -168,7 +168,7 @@ struct LabelRouteBulkContext class RouteOrch : public Orch, public Subject { public: - RouteOrch(DBConnector *db, vector &tableNames, SwitchOrch *switchOrc, NeighOrch *neighOrch, IntfsOrch *intfsOrch, VRFOrch *vrfOrch, FgNhgOrch *fgNhgOrch); + RouteOrch(DBConnector *db, vector &tableNames, SwitchOrch *switchOrch, NeighOrch *neighOrch, IntfsOrch *intfsOrch, VRFOrch *vrfOrch, FgNhgOrch *fgNhgOrch); bool hasNextHopGroup(const NextHopGroupKey&) const; sai_object_id_t getNextHopGroupId(const NextHopGroupKey&); From 792a4bd26a322485b75f60ee7bc2916e64d9db08 Mon Sep 17 00:00:00 2001 From: Tom cappleman Date: Tue, 12 Oct 2021 09:23:00 +0100 Subject: [PATCH 45/51] Fixes --- orchagent/mplsrouteorch.cpp | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/orchagent/mplsrouteorch.cpp b/orchagent/mplsrouteorch.cpp index 276c7e243e..97ea75dfd8 100644 --- a/orchagent/mplsrouteorch.cpp +++ b/orchagent/mplsrouteorch.cpp @@ -537,6 +537,17 @@ bool RouteOrch::addLabelRoute(LabelRouteBulkContext& ctx, const NextHopGroupKey /* Try to create a new next hop group */ if (!addNextHopGroup(nextHops)) { + for (auto it = nextHops.getNextHops().begin(); it != nextHops.getNextHops().end(); ++it) + { + const NextHopKey& nextHop = *it; + if (!m_neighOrch->hasNextHop(nextHop)) + { + SWSS_LOG_INFO("Failed to get next hop %s in %s, resolving neighbor", + nextHop.to_string().c_str(), nextHops.to_string().c_str()); + m_neighOrch->resolveNeighbor(nextHop); + } + } + /* Failed to create the next hop group and check if a temporary route is needed */ /* If the current next hop is part of the next hop group to sync, From 660bc90e4017869cc93263e7336e0b71063de812 Mon Sep 17 00:00:00 2001 From: Tom cappleman Date: Tue, 12 Oct 2021 17:14:06 +0100 Subject: [PATCH 46/51] Fixes --- orchagent/mplsrouteorch.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/orchagent/mplsrouteorch.cpp b/orchagent/mplsrouteorch.cpp index 97ea75dfd8..148eee5c75 100644 --- a/orchagent/mplsrouteorch.cpp +++ b/orchagent/mplsrouteorch.cpp @@ -708,7 +708,7 @@ bool RouteOrch::addLabelRoutePost(const LabelRouteBulkContext& ctx, const NextHo } } /* The route is pointing to a next hop group */ - else if (nextHops.getSize() > 1) + else { if (!hasNextHopGroup(nextHops)) { @@ -747,8 +747,8 @@ bool RouteOrch::addLabelRoutePost(const LabelRouteBulkContext& ctx, const NextHo gNhgOrch->incNhgRefCount(ctx.nhg_index); } - SWSS_LOG_INFO("Create label route %u with next hop(s) %s", - label, nextHops.to_string().c_str()); + SWSS_LOG_INFO("Post create label %u with next hop(s) %s", + label, nextHops.to_string().c_str()); } else { @@ -825,7 +825,7 @@ bool RouteOrch::addLabelRoutePost(const LabelRouteBulkContext& ctx, const NextHo } SWSS_LOG_INFO("Post set label %u with next hop(s) %s", - label, nextHops.to_string().c_str()); + label, nextHops.to_string().c_str()); } m_syncdLabelRoutes[vrf_id][label] = RouteNhg(nextHops, ctx.nhg_index); From dbd96f4f893a8fbd30e8e5c3ba343054b52e7678 Mon Sep 17 00:00:00 2001 From: Tom cappleman Date: Tue, 12 Oct 2021 17:16:03 +0100 Subject: [PATCH 47/51] Fixes --- orchagent/mplsrouteorch.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/orchagent/mplsrouteorch.cpp b/orchagent/mplsrouteorch.cpp index 148eee5c75..71dcce285a 100644 --- a/orchagent/mplsrouteorch.cpp +++ b/orchagent/mplsrouteorch.cpp @@ -931,7 +931,7 @@ bool RouteOrch::removeLabelRoutePost(const LabelRouteBulkContext& ctx) } SWSS_LOG_INFO("Remove label route %u with next hop(s) %s", - label, it_route->second.nhg_key.to_string().c_str()); + label, it_route->second.nhg_key.to_string().c_str()); it_route_table->second.erase(label); From 979d34931c9d9de44ee7532eae0b7e13c51343b6 Mon Sep 17 00:00:00 2001 From: Tom cappleman Date: Tue, 12 Oct 2021 17:27:42 +0100 Subject: [PATCH 48/51] Fixes --- orchagent/mplsrouteorch.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/orchagent/mplsrouteorch.cpp b/orchagent/mplsrouteorch.cpp index 71dcce285a..1bf37e711c 100644 --- a/orchagent/mplsrouteorch.cpp +++ b/orchagent/mplsrouteorch.cpp @@ -513,13 +513,15 @@ bool RouteOrch::addLabelRoute(LabelRouteBulkContext& ctx, const NextHopGroupKey { next_hop_id = m_neighOrch->getNextHopId(nexthop); } - /* See if there is an IP neighbor nexthop */ + /* For non-existent MPLS NH, check if IP neighbor NH exists */ else if (nexthop.isMplsNextHop() && m_neighOrch->isNeighborResolved(nexthop)) { + /* since IP neighbor NH exists, neighbor is resolved, add MPLS NH */ m_neighOrch->addNextHop(nexthop); next_hop_id = m_neighOrch->getNextHopId(nexthop); } + /* IP neighbor is not yet resolved */ else { SWSS_LOG_INFO("Failed to get next hop %s for %u", From 13ef025fee454e415fa278f8bc02d55ee3562f6f Mon Sep 17 00:00:00 2001 From: Tom cappleman Date: Tue, 12 Oct 2021 17:44:28 +0100 Subject: [PATCH 49/51] Fixes --- orchagent/routeorch.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/orchagent/routeorch.cpp b/orchagent/routeorch.cpp index 769229cba7..0c7cb5dbd7 100644 --- a/orchagent/routeorch.cpp +++ b/orchagent/routeorch.cpp @@ -1602,13 +1602,15 @@ bool RouteOrch::addRoute(RouteBulkContext& ctx, const NextHopGroupKey &nextHops) { next_hop_id = m_neighOrch->getNextHopId(nexthop); } - /* See if there is an IP neighbor nexthop */ + /* For non-existent MPLS NH, check if IP neighbor NH exists */ else if (nexthop.isMplsNextHop() && - m_neighOrch->isNeighborResolved(nexthop)) + m_neighOrch->isNeighborResolved(nexthop)) { + /* since IP neighbor NH exists, neighbor is resolved, add MPLS NH */ m_neighOrch->addNextHop(nexthop); next_hop_id = m_neighOrch->getNextHopId(nexthop); } + /* IP neighbor is not yet resolved */ else { if(overlay_nh) From 3789b5bb8b74f29abc8d609680eb06e14d0a9599 Mon Sep 17 00:00:00 2001 From: Tom cappleman Date: Tue, 12 Oct 2021 17:46:01 +0100 Subject: [PATCH 50/51] Fixes --- orchagent/routeorch.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/orchagent/routeorch.cpp b/orchagent/routeorch.cpp index 0c7cb5dbd7..f634872614 100644 --- a/orchagent/routeorch.cpp +++ b/orchagent/routeorch.cpp @@ -1610,7 +1610,7 @@ bool RouteOrch::addRoute(RouteBulkContext& ctx, const NextHopGroupKey &nextHops) m_neighOrch->addNextHop(nexthop); next_hop_id = m_neighOrch->getNextHopId(nexthop); } - /* IP neighbor is not yet resolved */ + /* IP neighbor is not yet resolved */ else { if(overlay_nh) From a0c982e55328af7bfb38aad463e86e8b400175ad Mon Sep 17 00:00:00 2001 From: Tom cappleman Date: Tue, 12 Oct 2021 17:47:31 +0100 Subject: [PATCH 51/51] Fixes --- orchagent/routeorch.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/orchagent/routeorch.cpp b/orchagent/routeorch.cpp index f634872614..80b2b1e571 100644 --- a/orchagent/routeorch.cpp +++ b/orchagent/routeorch.cpp @@ -1595,7 +1595,6 @@ bool RouteOrch::addRoute(RouteBulkContext& ctx, const NextHopGroupKey &nextHops) return false; } } - /* IP neighbor is not yet resolved */ else { if (m_neighOrch->hasNextHop(nexthop))