reduce processing memory and stable hash (#70)

In large datasets we had a problem with 
* updating dependencies and necessary adjustments
* updating conan dependencies
* updated hypertrie to 0.6.0-rc10
  * stable hash based on xxh3
  * operator graph clear() function
  * minimized memory footprint of Cartesian product operator
* async clear used operator graphs 
* introduced SAX SPARQL JSON Result writer with support for chunked responses.
* limit parallel connections to thread pool size
* changed clang version in Dockerfile to 11

CMakeLists.txt

@@ -68,7 +68,6 @@ if (TENTRIS_STATIC)
     SET(RESTINIO_USE_BOOST_ASIO=static)
 endif ()
 find_package(restinio REQUIRED)
-find_package(http-parser REQUIRED)
 find_package(string-view-lite REQUIRED)
 find_package(optional-lite REQUIRED)
 
@@ -116,7 +115,6 @@ target_link_libraries(tentris_server
         ${TENTRIS_PTHREAD_DEF}
         tentris
         restinio::restinio
-        http-parser::http-parser
         nonstd::string-view-lite
         nonstd::optional-lite
         nonstd::variant-lite

Dockerfile

@@ -1,11 +1,11 @@
-FROM ubuntu:focal AS builder
+FROM ubuntu:groovy AS builder
 ARG DEBIAN_FRONTEND=noninteractive
 ARG CXXFLAGS="${CXXFLAGS} -march=x86-64"
-ARG CMAKE_EXE_LINKER_FLAGS="-fuse-ld=lld-10"
+ARG CMAKE_EXE_LINKER_FLAGS="-fuse-ld=lld-11"
 ARG TENTRIS_MARCH="x86-64"
 
 RUN apt-get -qq update && \
-    apt-get -qq install -y make cmake uuid-dev git openjdk-11-jdk python3-pip python3-setuptools python3-wheel libstdc++-10-dev clang-10 g++-10 pkg-config google-perftools libgoogle-perftools-dev
+    apt-get -qq install -y make cmake uuid-dev git openjdk-11-jdk python3-pip python3-setuptools python3-wheel libstdc++-10-dev clang-11 g++-10 pkg-config google-perftools libgoogle-perftools-dev
 # we need serd as static library. Not available from ubuntu repos
 RUN ln -s /usr/bin/python3 /usr/bin/python && \
     git clone --quiet --branch v0.30.2 https://gitlab.com/drobilla/serd.git && \
@@ -25,10 +25,10 @@ RUN pip3 install conan && \
     conan profile update env.sparql-parser-base:CXX=/usr/bin/g++-10 default && \
     conan profile update env.sparql-parser-base:CC=/usr/bin/gcc-10 default && \
     conan profile update settings.compiler=clang default &&\
-    conan profile update settings.compiler.version=10 default && \
+    conan profile update settings.compiler.version=11 default && \
     conan profile update settings.compiler.libcxx=libstdc++11 default && \
-    conan profile update env.CXX=/usr/bin/clang++-10 default && \
-    conan profile update env.CC=/usr/bin/clang-10 default
+    conan profile update env.CXX=/usr/bin/clang++-11 default && \
+    conan profile update env.CC=/usr/bin/clang-11 default
 
 # add conan repositories
 RUN conan remote add tsl https://api.bintray.com/conan/tessil/tsl
@@ -50,8 +50,8 @@ COPY conanfile.txt /tentris/conanfile.txt
 # import and build depenedencies via conan
 RUN mkdir /tentris/build && cd /tentris/build && \
     conan install .. --build=missing
-# build tentris_server with clang++-10 again
-RUN export CXX="clang++-10" && export CC="clang-10" && \
+# build tentris_server with clang++-11 again
+RUN export CXX="clang++-11" && export CC="clang-11" && \
     cd /tentris/build && \
     cmake -DCMAKE_BUILD_TYPE=Release -DTENTRIS_BUILD_WITH_TCMALLOC=true -DTENTRIS_STATIC=true .. && \
     make -j $(nproc)

conanfile.txt

@@ -1,16 +1,16 @@
 [requires]
 boost/1.74.0
-fmt/7.0.3
+fmt/7.1.2
 gtest/1.8.1
 abseil/20200225.2
-http-parser/2.8.1@bincrafters/stable  # for restinio
-restinio/0.6.10@stiffstream/stable
-hypertrie/0.6.0-rc9@dice-group/stable
+restinio/0.6.12
+hypertrie/0.6.0-rc10@dice-group/stable
 rdf-parser/0.10@dice-group/stable
 sparql-parser-base/0.1.0@dice-group/stable
 
 [options]
-restinio:boost_libs=static
+restinio:asio=boost
+restinio:with_zlib=True
 boost:shared=False
 *:shared=False
 

src/exec/TentrisServer.cpp

@@ -12,7 +12,7 @@
 
 
 
-void bulkload(std::string triple_file, size_t bulksize) {
+void bulkload(const std::string &triple_file, size_t bulksize) {
 	namespace fs = std::filesystem;
 	using namespace fmt::literals;
 	using namespace tentris::logging;
@@ -34,6 +34,19 @@ void bulkload(std::string triple_file, size_t bulksize) {
 	log_duration(loading_start_time, loading_end_time);
 }
 
+struct tentris_restinio_traits : public	restinio::traits_t<
+		restinio::null_timer_manager_t,
+#ifdef DEBUG
+		restinio::shared_ostream_logger_t,
+#else
+		restinio::null_logger_t,
+#endif
+		restinio::router::express_router_t<>
+>{
+	static constexpr bool use_connection_count_limiter = true;
+};
+
+
 int main(int argc, char *argv[]) {
 	using namespace tentris::http;
 	using namespace tentris::store::config;
@@ -49,6 +62,7 @@ int main(int argc, char *argv[]) {
 	store_cfg.rdf_file = cfg.rdf_file;
 	store_cfg.timeout = cfg.timeout;
 	store_cfg.cache_size = cfg.cache_size;
+	store_cfg.threads = cfg.threads;
 
 	// bulkload file
 	if (not cfg.rdf_file.empty()) {
@@ -62,7 +76,10 @@ int main(int argc, char *argv[]) {
 	auto router = std::make_unique<router::express_router_t<>>();
 	router->http_get(
 			R"(/sparql)",
-			tentris::http::sparql_endpoint::sparql_endpoint);
+			tentris::http::sparql_endpoint::SparqlEndpoint<restinio::restinio_controlled_output_t>{});
+	router->http_get(
+			R"(/stream)",
+			tentris::http::sparql_endpoint::SparqlEndpoint<restinio::chunked_output_t>{});
 
 	router->non_matched_request_handler(
 			[](auto req) -> restinio::request_handling_status_t {
@@ -71,21 +88,11 @@ int main(int argc, char *argv[]) {
 
 	// Launching a server with custom traits.
 
-	using traits_t =
-	restinio::traits_t<
-			restinio::null_timer_manager_t,
-#ifdef DEBUG
-			restinio::shared_ostream_logger_t,
-#else
-			null_logger_t,
-#endif
-			restinio::router::express_router_t<>
-	>;
-
 	log("SPARQL endpoint serving sparkling linked data treasures on {} threads at http://0.0.0.0:{}/sparql?query="_format(cfg.threads, cfg.port));
 
 	restinio::run(
-			restinio::on_thread_pool<traits_t>(cfg.threads)
+			restinio::on_thread_pool<tentris_restinio_traits>(cfg.threads)
+			        .max_parallel_connections(cfg.threads)
 					.address("0.0.0.0")
 					.port(cfg.port)
 					.request_handler(std::move(router))