zebra: Do not delete nhg's when retain_mode is engaged

When `-r` is specified to zebra, on shutdown we should
not remove any routes from the fib.  This was a problem
with nhg's on shutdown due to their ref-count behavior.

Introduce a methodology where on shutdown we don't mess
with the nexthop groups in the kernel.  That way on
next startup things will be ok.

Signed-off-by: Donald Sharp <[email protected]>

zebra/main.c

@@ -174,12 +174,14 @@ static void sigint(void)
 
 	zebra_ptm_finish();
 
-	if (retain_mode)
+	if (retain_mode) {
+		zebra_nhg_mark_keep();
 		RB_FOREACH (vrf, vrf_name_head, &vrfs_by_name) {
 			zvrf = vrf->info;
 			if (zvrf)
 				SET_FLAG(zvrf->flags, ZEBRA_VRF_RETAIN);
 		}
+	}
 	if (zrouter.lsp_process_q)
 		work_queue_free_and_null(&zrouter.lsp_process_q);
 

zebra/zebra_nhg.c

@@ -2646,6 +2646,30 @@ void zebra_nhg_sweep_table(struct hash *hash)
 	hash_iterate(hash, zebra_nhg_sweep_entry, NULL);
 }
 
+static void zebra_nhg_mark_keep_entry(struct hash_bucket *bucket, void *arg)
+{
+	struct nhg_hash_entry *nhe = bucket->data;
+
+	UNSET_FLAG(nhe->flags, NEXTHOP_GROUP_INSTALLED);
+}
+
+/*
+ * When we are shutting down and we have retain mode enabled
+ * in zebra the process is to mark each vrf that it's
+ * routes should not be deleted.  The problem with that
+ * is that shutdown actually free's up memory which
+ * causes the nexthop group's ref counts to go to zero
+ * we need a way to subtly tell the system to not remove
+ * the nexthop groups from the kernel at the same time.
+ * The easiest just looks like that we should not mark
+ * the nhg's as installed any more and when the ref count
+ * goes to zero we'll attempt to delete and do nothing
+ */
+void zebra_nhg_mark_keep(void)
+{
+	hash_iterate(zrouter.nhgs_id, zebra_nhg_mark_keep_entry, NULL);
+}
+
 /* Global control to disable use of kernel nexthops, if available. We can't
  * force the kernel to support nexthop ids, of course, but we can disable
  * zebra's use of them, for testing e.g. By default, if the kernel supports

zebra/zebra_nhg.h

@@ -333,9 +333,16 @@ struct zebra_dplane_ctx;
 extern void zebra_nhg_dplane_result(struct zebra_dplane_ctx *ctx);
 
 
-/* Sweet the nhg hash tables for old entries on restart */
+/* Sweep the nhg hash tables for old entries on restart */
 extern void zebra_nhg_sweep_table(struct hash *hash);
 
+/*
+ * We are shutting down but the nexthops should be kept
+ * as that -r has been specified and we don't want to delete
+ * the routes unintentionally
+ */
+extern void zebra_nhg_mark_keep(void);
+
 /* Nexthop resolution processing */
 struct route_entry; /* Forward ref to avoid circular includes */
 extern int nexthop_active_update(struct route_node *rn, struct route_entry *re);