Feature/parallel loading

Makefile.in

@@ -76,13 +76,14 @@ MANUALS=nsd.8 nsd-checkconf.8 nsd-checkzone.8 nsd-control.8 nsd.conf.5
 
 COMMON_OBJ=answer.o axfr.o buffer.o configlexer.o configparser.o dname.o dns.o edns.o iterated_hash.o lookup3.o namedb.o nsec3.o options.o packet.o query.o rbtree.o radtree.o rdata.o region-allocator.o rrl.o tsig.o tsig-openssl.o udb.o udbradtree.o udbzone.o util.o
 XFRD_OBJ=xfrd-disk.o xfrd-notify.o xfrd-tcp.o xfrd.o remote.o $(DNSTAP_OBJ)
-NSD_OBJ=$(COMMON_OBJ) $(XFRD_OBJ) difffile.o ipc.o mini_event.o netio.o nsd.o server.o dbaccess.o dbcreate.o zlexer.o zonec.o zparser.o
+PZL_OBJ=pzl4nsd.o zonefile_processor.o presentation.o dnsextlang.o rrtypes.o
+NSD_OBJ=$(COMMON_OBJ) $(XFRD_OBJ) difffile.o ipc.o mini_event.o netio.o nsd.o server.o dbaccess.o dbcreate.o zlexer.o zonec.o zparser.o $(PZL_OBJ)
 ALL_OBJ=$(NSD_OBJ) nsd-checkconf.o nsd-checkzone.o nsd-control.o nsd-mem.o
 NSD_CHECKCONF_OBJ=$(COMMON_OBJ) nsd-checkconf.o
-NSD_CHECKZONE_OBJ=$(COMMON_OBJ) $(XFRD_OBJ) dbaccess.o dbcreate.o difffile.o ipc.o mini_event.o netio.o server.o zonec.o zparser.o zlexer.o nsd-checkzone.o
+NSD_CHECKZONE_OBJ=$(COMMON_OBJ) $(XFRD_OBJ) dbaccess.o dbcreate.o difffile.o ipc.o mini_event.o netio.o server.o zonec.o zparser.o zlexer.o nsd-checkzone.o $(PZL_OBJ)
 NSD_CONTROL_OBJ=$(COMMON_OBJ) nsd-control.o
-CUTEST_OBJ=$(COMMON_OBJ) $(XFRD_OBJ) dbaccess.o dbcreate.o difffile.o ipc.o mini_event.o netio.o server.o zonec.o zparser.o zlexer.o cutest_dname.o cutest_dns.o cutest_iterated_hash.o cutest_run.o cutest_radtree.o cutest_rbtree.o cutest_namedb.o cutest_options.o cutest_region.o cutest_rrl.o cutest_udb.o cutest_udbrad.o cutest_util.o cutest.o qtest.o
-NSD_MEM_OBJ=$(COMMON_OBJ) $(XFRD_OBJ) dbaccess.o dbcreate.o difffile.o ipc.o mini_event.o netio.o server.o zonec.o zparser.o zlexer.o nsd-mem.o
+CUTEST_OBJ=$(COMMON_OBJ) $(XFRD_OBJ) dbaccess.o dbcreate.o difffile.o ipc.o mini_event.o netio.o server.o zonec.o zparser.o zlexer.o cutest_dname.o cutest_dns.o cutest_iterated_hash.o cutest_run.o cutest_radtree.o cutest_rbtree.o cutest_namedb.o cutest_options.o cutest_region.o cutest_rrl.o cutest_udb.o cutest_udbrad.o cutest_util.o cutest.o qtest.o $(PZL_OBJ)
+NSD_MEM_OBJ=$(COMMON_OBJ) $(XFRD_OBJ) dbaccess.o dbcreate.o difffile.o ipc.o mini_event.o netio.o server.o zonec.o zparser.o zlexer.o nsd-mem.o $(PZL_OBJ)
 all:	$(TARGETS) $(MANUALS)
 
 $(ALL_OBJ):
@@ -323,6 +324,29 @@ dnstap/dnstap.pb-c.c dnstap/dnstap.pb-c.h: $(srcdir)/dnstap/dnstap.proto
 	@-if test ! -d dnstap; then $(INSTALL) -d dnstap; fi
 	$(PROTOC_C) --c_out=. --proto_path=$(srcdir) $(srcdir)/dnstap/dnstap.proto
 
+# Fats Zone Loading
+pzl4nsd.o: $(srcdir)/pzl/pzl4nsd.c  $(srcdir)/pzl/pzl4nsd.h \
+	$(srcdir)/pzl/zonefile_processor.h \
+	$(srcdir)/pzl/presentation.h $(srcdir)/pzl/dns_config.h \
+	$(srcdir)/pzl/mmap_parser.h $(srcdir)/pzl/return_status.h
+
+zonefile_processor.o: $(srcdir)/pzl/zonefile_processor.c \
+	$(srcdir)/pzl/zonefile_processor.h \
+	$(srcdir)/pzl/presentation.h $(srcdir)/pzl/dns_config.h \
+	$(srcdir)/pzl/mmap_parser.h $(srcdir)/pzl/return_status.h
+
+presentation.o: $(srcdir)/pzl/presentation.c \
+	$(srcdir)/pzl/presentation.h $(srcdir)/pzl/dns_config.h \
+	$(srcdir)/pzl/mmap_parser.h $(srcdir)/pzl/return_status.h
+
+rrtypes.o: $(srcdir)/pzl/rrtypes.c \
+	$(srcdir)/pzl/dnsextlang.h $(srcdir)/pzl/dns_config.h \
+	$(srcdir)/pzl/return_status.h
+
+dnsextlang.o: $(srcdir)/pzl/dnsextlang.c \
+	$(srcdir)/pzl/dnsextlang.h $(srcdir)/pzl/dns_config.h \
+	$(srcdir)/pzl/mmap_parser.h $(srcdir)/pzl/return_status.h
+
 # autoconf rules
 config.h.in:	configure.ac
 	autoheader

configure.ac

@@ -913,8 +913,10 @@ esac
 AC_ARG_ENABLE(radix-tree, AC_HELP_STRING([--disable-radix-tree], [You can disable the radix tree and use the red-black tree for the main lookups, the red-black tree uses less memory, but uses some more CPU.]))
 case "$enable_radix_tree" in
         no)
+	USE_RBTREE="yes"
 	;;
 	yes|*)
+	USE_RBTREE="no"
 	AC_DEFINE_UNQUOTED([USE_RADIX_TREE], [], [Define this to configure to use the radix tree.])
 	;;
 esac
@@ -929,6 +931,19 @@ case "$enable_packed" in
 	;;
 esac
 
+AC_ARG_ENABLE(parallel-loading, AC_HELP_STRING([--enable-parallel-loading], [Enable parallel loading of zonefiles]))
+case "$enable_parallel_loading" in
+	yes)
+	if test x$USE_RBTREE != x"yes"; then
+		AC_MSG_ERROR(Parallel loading currently requires the red-black tree backend. Please reconfigure with --disable-radix-tree)
+	fi
+	AC_DEFINE_UNQUOTED([PARALLEL_LOADING], [], [Define this to use parallel loading of zone files.])
+	AC_CHECK_LIB(pthread, pthread_join)
+	;;
+	no|*)
+	;;
+esac
+
 # check for dnstap if requested
 dt_DNSTAP([${localstatedir}/run/nsd-dnstap.sock],
     [

dbaccess.c

@@ -642,7 +642,9 @@ namedb_read_zonefile(struct nsd* nsd, struct zone* zone, udb_base* taskudb,
 	}
 	if(taskudb) task_new_soainfo(taskudb, last_task, zone, 0);
 #ifdef NSEC3
+# ifndef PARALLEL_LOADING
 	prehash_zone_complete(nsd->db, zone);
+# endif
 #endif
 }
 

dbcreate.c

@@ -180,7 +180,10 @@ write_zone_to_udb(udb_base* udb, zone_type* zone, struct timespec* mtime,
 	return 1;
 }
 
-static int
+#ifdef NDEBUG
+static
+#endif
+int
 print_rrs(FILE* out, struct zone* zone)
 {
 	rrset_type *rrset;

namedb.c

@@ -49,6 +49,9 @@ allocate_domain_info(domain_table_type* table,
 #endif
 	result->is_existing = 0;
 	result->is_apex = 0;
+#ifdef PARALLEL_LOADING
+	result->rrsets2merge = 0;
+#endif
 	assert(table->numlist_last); /* it exists because root exists */
 	/* push this domain at the end of the numlist */
 	result->number = table->numlist_last->number+1;
@@ -524,7 +527,7 @@ domain_add_rrset(domain_type* domain, rrset_type* rrset)
 
 
 rrset_type *
-domain_find_rrset(domain_type* domain, zone_type* zone, uint16_t type)
+domain_find_rrset(domain_type* domain, const zone_type* zone, uint16_t type)
 {
 	rrset_type* result = domain->rrsets;
 
@@ -668,7 +671,7 @@ namedb_find_zone(namedb_type* db, const dname_type* dname)
 }
 
 rrset_type *
-domain_find_non_cname_rrset(domain_type* domain, zone_type* zone)
+domain_find_non_cname_rrset(domain_type* domain, const zone_type* zone)
 {
 	/* find any rrset type that is not allowed next to a CNAME */
 	/* nothing is allowed next to a CNAME, except RRSIG, NSEC, NSEC3 */

namedb.h

@@ -118,6 +118,9 @@ struct domain
 	 */
 	unsigned     is_existing : 1;
 	unsigned     is_apex : 1;
+#ifdef PARALLEL_LOADING
+	unsigned     rrsets2merge: 1;
+#endif
 } ATTR_PACKED;
 
 struct zone
@@ -244,7 +247,7 @@ int domain_is_prehash(domain_table_type* table, domain_type* domain);
  */
 void domain_add_rrset(domain_type* domain, rrset_type* rrset);
 
-rrset_type* domain_find_rrset(domain_type* domain, zone_type* zone, uint16_t type);
+rrset_type* domain_find_rrset(domain_type* domain, const zone_type* zone, uint16_t type);
 rrset_type* domain_find_any_rrset(domain_type* domain, zone_type* zone);
 
 zone_type* domain_find_zone(namedb_type* db, domain_type* domain);
@@ -256,7 +259,7 @@ domain_type * find_dname_above(domain_type* domain, zone_type* zone);
 
 int domain_is_glue(domain_type* domain, zone_type* zone);
 
-rrset_type* domain_find_non_cname_rrset(domain_type* domain, zone_type* zone);
+rrset_type* domain_find_non_cname_rrset(domain_type* domain, const zone_type* zone);
 
 domain_type* domain_wildcard_child(domain_type* domain);
 domain_type *domain_previous_existing_child(domain_type* domain);

pzl/dns_config.h

@@ -0,0 +1,61 @@
+/* Copyright (c) 2019, NLnet Labs. All rights reserved.
+ * 
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+ * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef DNS_CONFIG_H_ 
+#define DNS_CONFIG_H_
+#include <stdint.h>
+
+#ifndef DNS_DEFAULT_TTL
+#define DNS_DEFAULT_TTL     3600
+#endif
+#ifndef DNS_DEFAULT_CLASS
+#define DNS_DEFAULT_CLASS   1
+#endif
+#ifndef DNS_DEFAULT_ORIGIN
+#define DNS_DEFAULT_ORIGIN  ""
+#endif
+#ifndef DNS_DEFAULT_RRTYPES
+struct dnsextlang_def;
+#define DNS_DEFAULT_RRTYPES NULL
+#endif
+
+#define DNS_CONFIG_DEFAULTS { DNS_DEFAULT_TTL   , DNS_DEFAULT_CLASS \
+                            , DNS_DEFAULT_ORIGIN, DNS_DEFAULT_RRTYPES }
+
+typedef struct dns_config {
+	uint32_t               default_ttl;
+	uint16_t               default_class;
+	const char            *default_origin;
+	struct dnsextlang_def *rrtypes;
+} dns_config;
+
+#endif /* #ifndef DNS_CONFIG_H_ */