Add xio_gdal_handler

CMakeLists.txt

@@ -62,7 +62,11 @@ set(XTENSOR_IO_HEADERS
     ${XTENSOR_IO_INCLUDE_DIR}/xtensor-io/xio_aws_handler.hpp
     ${XTENSOR_IO_INCLUDE_DIR}/xtensor-io/xio_disk_handler.hpp
     ${XTENSOR_IO_INCLUDE_DIR}/xtensor-io/xio_gcs_handler.hpp
+    ${XTENSOR_IO_INCLUDE_DIR}/xtensor-io/xio_gdal_handler.hpp
     ${XTENSOR_IO_INCLUDE_DIR}/xtensor-io/xio_gzip.hpp
+    ${XTENSOR_IO_INCLUDE_DIR}/xtensor-io/xio_file_wrapper.hpp
+    ${XTENSOR_IO_INCLUDE_DIR}/xtensor-io/xio_vsilfile_wrapper.hpp
+    ${XTENSOR_IO_INCLUDE_DIR}/xtensor-io/xio_stream_wrapper.hpp
     ${XTENSOR_IO_INCLUDE_DIR}/xtensor-io/xnpz.hpp
     ${XTENSOR_IO_INCLUDE_DIR}/xtensor-io/xtensor-io.hpp
     ${XTENSOR_IO_INCLUDE_DIR}/xtensor-io/xtensor_io_config.hpp

include/xtensor-io/xgdal.hpp

@@ -46,7 +46,7 @@ namespace xt
     }
 
     /**
-     * Get a band interleaved by yixel layout; index order = [row, column, band].
+     * Get a band interleaved by pixel layout; index order = [row, column, band].
      */
     inline layout layout_band_interleaved_pixel()
     {
@@ -346,7 +346,7 @@ namespace xt
         std::string driver_name;
 
         /**
-         * Options passed to to GDAL when the dataset is created (like COMPRESS=JPEG).
+         * Options passed to GDAL when the dataset is created (like COMPRESS=JPEG).
          */
         std::vector<std::string> creation_options;
 
@@ -541,4 +541,4 @@ namespace xt
 }  // namespace xt
 
 
-#endif // XTENSOR_IO_XGDAL_HPP
+#endif // XTENSOR_IO_XGDAL_HPP

include/xtensor-io/xio_aws_handler.hpp

@@ -4,6 +4,7 @@
 #include "xtensor/xarray.hpp"
 #include "xtensor/xexpression.hpp"
 #include "xfile_array.hpp"
+#include "xio_stream_wrapper.hpp"
 #include <aws/core/Aws.h>
 #include <aws/s3/S3Client.h>
 #include <aws/s3/model/GetObjectRequest.h>
@@ -70,7 +71,8 @@ namespace xt
             request.SetKey(path2);
 
             std::shared_ptr<Aws::IOStream> writer = Aws::MakeShared<Aws::FStream>("SampleAllocationTag", path, std::ios_base::in | std::ios_base::binary);
-            dump_file(*writer, expression, m_format_config);
+            auto s = xostream_wrapper(*writer);
+            dump_file(s, expression, m_format_config);
 
             request.SetBody(writer);
 
@@ -109,7 +111,8 @@ namespace xt
         }
 
         auto& reader = outcome.GetResultWithOwnership().GetBody();
-        load_file<ET>(reader, array, m_format_config);
+        auto s = xistream_wrapper(reader);
+        load_file<ET>(s, array, m_format_config);
     }
 
     template <class C>

include/xtensor-io/xio_binary.hpp

@@ -15,15 +15,17 @@
 #include "xtensor/xadapt.hpp"
 #include "xtensor-io.hpp"
 #include "xfile_array.hpp"
+#include "xio_stream_wrapper.hpp"
 
 namespace xt
 {
     namespace detail
     {
-        template <typename T>
-        inline xt::svector<T> load_bin_file(std::istream& stream, bool as_big_endian)
+        template <typename T, class I>
+        inline xt::svector<T> load_bin(I& stream, bool as_big_endian)
         {
-            std::string buffer{std::istreambuf_iterator<char>{stream}, {}};
+            std::string buffer;
+            stream.read_all(buffer);
             std::size_t uncompressed_size = buffer.size() / sizeof(T);
             xt::svector<T> uncompressed_buffer(uncompressed_size);
             std::copy((const T*)(buffer.data()), (const T*)(buffer.data()) + uncompressed_size, uncompressed_buffer.begin());
@@ -35,7 +37,7 @@ namespace xt
         }
 
         template <class O, class E>
-        inline void dump_bin_stream(O& stream, const xexpression<E>& e, bool as_big_endian)
+        inline void dump_bin(O& stream, const xexpression<E>& e, bool as_big_endian)
         {
             using value_type = typename E::value_type;
             const E& ex = e.derived_cast();
@@ -56,7 +58,7 @@ namespace xt
             {
                 uncompressed_buffer = reinterpret_cast<const char*>(eval_ex.data());
             }
-            stream.write(uncompressed_buffer, std::streamsize(uncompressed_size));
+            stream.write(uncompressed_buffer, uncompressed_size);
             stream.flush();
         }
     }  // namespace detail
@@ -67,10 +69,17 @@ namespace xt
      * @param stream An output stream to which to dump the data
      * @param e the xexpression
      */
+    template <typename E, class O>
+    inline void dump_bin(O& stream, const xexpression<E>& e, bool as_big_endian=is_big_endian())
+    {
+        detail::dump_bin(stream, e, as_big_endian);
+    }
+
     template <typename E>
-    inline void dump_bin(std::ostream& stream, const xexpression<E>& e, bool as_big_endian=is_big_endian())
+    inline void dump_bin(std::ofstream& stream, const xexpression<E>& e, bool as_big_endian=is_big_endian())
     {
-        detail::dump_bin_stream(stream, e, as_big_endian);
+        auto s = xostream_wrapper(stream);
+        detail::dump_bin(s, e, as_big_endian);
     }
 
     /**
@@ -80,14 +89,21 @@ namespace xt
      * @param e the xexpression
      */
     template <typename E>
-    inline void dump_bin(const std::string& filename, const xexpression<E>& e, bool as_big_endian=is_big_endian())
+    inline void dump_bin(const char* filename, const xexpression<E>& e, bool as_big_endian=is_big_endian())
     {
         std::ofstream stream(filename, std::ofstream::binary);
         if (!stream.is_open())
         {
             std::runtime_error("IO Error: failed to open file");
         }
-        detail::dump_bin_stream(stream, e, as_big_endian);
+        auto s = xostream_wrapper(stream);
+        detail::dump_bin(s, e, as_big_endian);
+    }
+
+    template <typename E>
+    inline void dump_bin(const std::string& filename, const xexpression<E>& e, bool as_big_endian=is_big_endian())
+    {
+        dump_bin<E>(filename.c_str(), e, as_big_endian);
     }
 
     /**
@@ -99,7 +115,8 @@ namespace xt
     inline std::string dump_bin(const xexpression<E>& e, bool as_big_endian=is_big_endian())
     {
         std::stringstream stream;
-        detail::dump_bin_stream(stream, e, as_big_endian);
+        auto s = xostream_wrapper(stream);
+        detail::dump_bin(s, e, as_big_endian);
         return stream.str();
     }
 
@@ -112,10 +129,10 @@ namespace xt
      *           Fortran format
      * @return xarray with contents from binary file
      */
-    template <typename T, layout_type L = layout_type::dynamic>
-    inline auto load_bin(std::istream& stream, bool as_big_endian=is_big_endian())
+    template <typename T, layout_type L = layout_type::dynamic, class I>
+    inline auto load_bin(I& stream, bool as_big_endian=is_big_endian())
     {
-        xt::svector<T> uncompressed_buffer = detail::load_bin_file<T>(stream, as_big_endian);
+        xt::svector<T> uncompressed_buffer = detail::load_bin<T>(stream, as_big_endian);
         std::vector<std::size_t> shape = {uncompressed_buffer.size()};
         auto array = adapt(std::move(uncompressed_buffer), shape);
         return array;
@@ -131,14 +148,21 @@ namespace xt
      * @return xarray with contents from binary file
      */
     template <typename T, layout_type L = layout_type::dynamic>
-    inline auto load_bin(const std::string& filename, bool as_big_endian=is_big_endian())
+    inline auto load_bin(const char* filename, bool as_big_endian=is_big_endian())
     {
         std::ifstream stream(filename, std::ifstream::binary);
         if (!stream.is_open())
         {
-            std::runtime_error("load_bin: failed to open file " + filename);
+            std::runtime_error(std::string("load_bin: failed to open file ") + filename);
         }
-        return load_bin<T, L>(stream, as_big_endian);
+        auto s = xistream_wrapper(stream);
+        return load_bin<T, L>(s, as_big_endian);
+    }
+
+    template <typename T, layout_type L = layout_type::dynamic>
+    inline auto load_bin(const std::string& filename, bool as_big_endian=is_big_endian())
+    {
+        return load_bin<T, L>(filename.c_str(), as_big_endian);
     }
 
     struct xio_binary_config
@@ -170,8 +194,8 @@ namespace xt
         }
     };
 
-    template <class E>
-    void load_file(std::istream& stream, xexpression<E>& e, const xio_binary_config& config)
+    template <class E, class I>
+    void load_file(I& stream, xexpression<E>& e, const xio_binary_config& config)
     {
         E& ex = e.derived_cast();
         auto shape = ex.shape();
@@ -186,8 +210,8 @@ namespace xt
         }
     }
 
-    template <class E>
-    void dump_file(std::ostream& stream, const xexpression<E> &e, const xio_binary_config& config)
+    template <class E, class O>
+    void dump_file(O& stream, const xexpression<E> &e, const xio_binary_config& config)
     {
         dump_bin(stream, e, config.big_endian);
     }

include/xtensor-io/xio_blosc.hpp

@@ -16,6 +16,7 @@
 #include "xtensor-io.hpp"
 #include "xfile_array.hpp"
 #include "blosc.h"
+#include "xio_stream_wrapper.hpp"
 
 namespace xt
 {
@@ -31,11 +32,12 @@ namespace xt
             }
         }
 
-        template <typename T>
-        inline xt::svector<T> load_blosc_file(std::istream& stream, bool as_big_endian)
+        template <typename T, class I>
+        inline xt::svector<T> load_blosc(I& stream, bool as_big_endian)
         {
             init_blosc();
-            std::string compressed_buffer{std::istreambuf_iterator<char>{stream}, {}};
+            std::string compressed_buffer;
+            stream.read_all(compressed_buffer);
             auto compressed_size = compressed_buffer.size();
             std::size_t uncompressed_size = 0;
             int res = blosc_cbuffer_validate(compressed_buffer.data(), compressed_size, &uncompressed_size);
@@ -60,7 +62,7 @@ namespace xt
         }
 
         template <class O, class E>
-        inline void dump_blosc_stream(O& stream, const xexpression<E>& e, bool as_big_endian, int clevel, int shuffle, const char* cname, std::size_t blocksize)
+        inline void dump_blosc(O& stream, const xexpression<E>& e, bool as_big_endian, int clevel, int shuffle, const char* cname, std::size_t blocksize)
         {
             init_blosc();
             using value_type = typename E::value_type;
@@ -99,8 +101,8 @@ namespace xt
             {
                 XTENSOR_THROW(std::runtime_error, "Blosc: compression error");
             }
-            stream.write(compressed_buffer,
-                         std::streamsize(true_compressed_size));
+            stream.write(compressed_buffer, std::streamsize(true_compressed_size));
+            stream.flush();
             char_allocator.deallocate(compressed_buffer, max_compressed_size);
         }
     }  // namespace detail
@@ -111,10 +113,17 @@ namespace xt
      * @param stream An output stream to which to dump the data
      * @param e the xexpression
      */
+    template <typename E, class O>
+    inline void dump_blosc(O& stream, const xexpression<E>& e, bool as_big_endian=is_big_endian(), int clevel=5, int shuffle=1, const char* cname="blosclz", std::size_t blocksize=0)
+    {
+        detail::dump_blosc(stream, e, as_big_endian, clevel, shuffle, cname, blocksize);
+    }
+
     template <typename E>
     inline void dump_blosc(std::ostream& stream, const xexpression<E>& e, bool as_big_endian=is_big_endian(), int clevel=5, int shuffle=1, const char* cname="blosclz", std::size_t blocksize=0)
     {
-        detail::dump_blosc_stream(stream, e, as_big_endian, clevel, shuffle, cname, blocksize);
+        auto s = xostream_wrapper(stream);
+        detail::dump_blosc(s, e, as_big_endian, clevel, shuffle, cname, blocksize);
     }
 
     /**
@@ -124,14 +133,21 @@ namespace xt
      * @param e the xexpression
      */
     template <typename E>
-    inline void dump_blosc(const std::string& filename, const xexpression<E>& e, bool as_big_endian=is_big_endian(), int clevel=5, int shuffle=1, const char* cname="blosclz", std::size_t blocksize=0)
+    inline void dump_blosc(const char* filename, const xexpression<E>& e, bool as_big_endian=is_big_endian(), int clevel=5, int shuffle=1, const char* cname="blosclz", std::size_t blocksize=0)
     {
         std::ofstream stream(filename, std::ofstream::binary);
         if (!stream.is_open())
         {
-            XTENSOR_THROW(std::runtime_error, "Blosc: failed to open file " + filename);
+            XTENSOR_THROW(std::runtime_error, std::string("Blosc: failed to open file ") + filename);
         }
-        detail::dump_blosc_stream(stream, e, as_big_endian, clevel, shuffle, cname, blocksize);
+        auto s = xostream_wrapper(stream);
+        detail::dump_blosc(s, e, as_big_endian, clevel, shuffle, cname, blocksize);
+    }
+
+    template <typename E>
+    inline void dump_blosc(const std::string& filename, const xexpression<E>& e, bool as_big_endian=is_big_endian(), int clevel=5, int shuffle=1, const char* cname="blosclz", std::size_t blocksize=0)
+    {
+        dump_blosc<E>(filename.c_str(), e, as_big_endian, clevel, shuffle, cname, blocksize);
     }
 
     /**
@@ -143,7 +159,8 @@ namespace xt
     inline std::string dump_blosc(const xexpression<E>& e, bool as_big_endian=is_big_endian(), int clevel=5, int shuffle=1, const char* cname="blosclz", std::size_t blocksize=0)
     {
         std::stringstream stream;
-        detail::dump_blosc_stream(stream, e, as_big_endian, clevel, shuffle, cname, blocksize);
+        auto s = xostream_wrapper(stream);
+        detail::dump_blosc(s, e, as_big_endian, clevel, shuffle, cname, blocksize);
         return stream.str();
     }
 
@@ -156,10 +173,10 @@ namespace xt
      *           Fortran format
      * @return xarray with contents from blosc file
      */
-    template <typename T, layout_type L = layout_type::dynamic>
-    inline auto load_blosc(std::istream& stream, bool as_big_endian=is_big_endian())
+    template <typename T, layout_type L = layout_type::dynamic, class I>
+    inline auto load_blosc(I& stream, bool as_big_endian=is_big_endian())
     {
-        xt::svector<T> uncompressed_buffer = detail::load_blosc_file<T>(stream, as_big_endian);
+        xt::svector<T> uncompressed_buffer = detail::load_blosc<T>(stream, as_big_endian);
         std::vector<std::size_t> shape = {uncompressed_buffer.size()};
         auto array = adapt(std::move(uncompressed_buffer), shape);
         return array;
@@ -175,14 +192,21 @@ namespace xt
      * @return xarray with contents from blosc file
      */
     template <typename T, layout_type L = layout_type::dynamic>
-    inline auto load_blosc(const std::string& filename, bool as_big_endian=is_big_endian())
+    inline auto load_blosc(const char* filename, bool as_big_endian=is_big_endian())
     {
         std::ifstream stream(filename, std::ifstream::binary);
         if (!stream.is_open())
         {
-            XTENSOR_THROW(std::runtime_error, "Blosc: failed to open file " + filename);
+            XTENSOR_THROW(std::runtime_error, std::string("Blosc: failed to open file ") + filename);
         }
-        return load_blosc<T, L>(stream, as_big_endian);
+        auto s = xistream_wrapper(stream);;
+        return load_blosc<T, L>(s, as_big_endian);
+    }
+
+    template <typename T, layout_type L = layout_type::dynamic>
+    inline auto load_blosc(const std::string& filename, bool as_big_endian=is_big_endian())
+    {
+        return load_blosc<T, L>(filename.c_str(), as_big_endian);
     }
 
     struct xio_blosc_config
@@ -230,8 +254,8 @@ namespace xt
         }
     };
 
-    template <class E>
-    void load_file(std::istream& stream, xexpression<E>& e, const xio_blosc_config& config)
+    template <class E, class I>
+    void load_file(I& stream, xexpression<E>& e, const xio_blosc_config& config)
     {
         E& ex = e.derived_cast();
         auto shape = ex.shape();
@@ -246,8 +270,8 @@ namespace xt
         }
     }
 
-    template <class E>
-    void dump_file(std::ostream& stream, const xexpression<E> &e, const xio_blosc_config& config)
+    template <class E, class O>
+    void dump_file(O& stream, const xexpression<E> &e, const xio_blosc_config& config)
     {
         dump_blosc(stream, e, config.big_endian, config.clevel, config.shuffle, config.cname.c_str(), config.blocksize);
     }