change the append to triple store to be done by chunk of 500 000 lines

lib/goo/sparql/client.rb

@@ -83,41 +83,28 @@ def delete_data_graph(graph)
       def append_triples_no_bnodes(graph,file_path,mime_type_in)
         bnodes_filter = nil
         dir = nil
-
-        if file_path.end_with?("ttl")
+        response = nil
+        if file_path.end_with?('ttl')
           bnodes_filter = file_path
         else
-          bnodes_filter,dir = bnodes_filter_file(file_path,mime_type_in)
+          bnodes_filter, dir = bnodes_filter_file(file_path, mime_type_in)
+        end
+        chunk_lines = 500_000 # number of line
+        file = File.foreach(bnodes_filter)
+        lines = []
+        file.each_entry do |line|
+          lines << line
+          if lines.size == chunk_lines
+            response = execute_append_request graph, lines.join, mime_type_in
+            lines.clear
+          end
         end
-        mime_type = "text/turtle"
 
-        if mime_type_in == "text/x-nquads"
-          mime_type = "text/x-nquads"
-          graph = "http://data.bogus.graph/uri"
-        end
+        response = execute_append_request graph, lines.join, mime_type_in unless lines.empty?
 
-        data_file = File.read(bnodes_filter)
-        params = {method: :post, url: "#{url.to_s}", headers: {"content-type" => mime_type, "mime-type" => mime_type}, timeout: nil}
-        backend_name = Goo.sparql_backend_name
-
-        if backend_name == BACKEND_4STORE
-          params[:payload] = {
-            graph: graph.to_s,
-            data: data_file,
-            "mime-type" => mime_type
-          }
-          #for some reason \\\\ breaks parsing
-          params[:payload][:data] = params[:payload][:data].split("\n").map { |x| x.sub("\\\\","") }.join("\n")
-        else
-          params[:url] << "?context=#{CGI.escape("<#{graph.to_s}>")}"
-          params[:payload] = data_file
-        end
-
-        response = RestClient::Request.execute(params)
 
         unless dir.nil?
           File.delete(bnodes_filter)
-
           begin
             FileUtils.rm_rf(dir)
           rescue => e