From 156617aa82affee1c26892196b0457f8238d48c3 Mon Sep 17 00:00:00 2001 From: David Brochart Date: Thu, 24 Dec 2020 21:27:18 +0100 Subject: [PATCH 1/3] Add file-like support in xio_binary --- include/xtensor-io/xgdal.hpp | 6 +-- include/xtensor-io/xio_binary.hpp | 82 ++++++++++++++++++++++++------- test/test_xio_binary.cpp | 22 ++++++++- 3 files changed, 88 insertions(+), 22 deletions(-) diff --git a/include/xtensor-io/xgdal.hpp b/include/xtensor-io/xgdal.hpp index 03586e2..26cecb6 100644 --- a/include/xtensor-io/xgdal.hpp +++ b/include/xtensor-io/xgdal.hpp @@ -46,7 +46,7 @@ namespace xt } /** - * Get a band interleaved by yixel layout; index order = [row, column, band]. + * Get a band interleaved by pixel layout; index order = [row, column, band]. */ inline layout layout_band_interleaved_pixel() { @@ -346,7 +346,7 @@ namespace xt std::string driver_name; /** - * Options passed to to GDAL when the dataset is created (like COMPRESS=JPEG). + * Options passed to GDAL when the dataset is created (like COMPRESS=JPEG). */ std::vector creation_options; @@ -541,4 +541,4 @@ namespace xt } // namespace xt -#endif // XTENSOR_IO_XGDAL_HPP \ No newline at end of file +#endif // XTENSOR_IO_XGDAL_HPP diff --git a/include/xtensor-io/xio_binary.hpp b/include/xtensor-io/xio_binary.hpp index b95418f..116d612 100644 --- a/include/xtensor-io/xio_binary.hpp +++ b/include/xtensor-io/xio_binary.hpp @@ -20,10 +20,33 @@ namespace xt { namespace detail { - template - inline xt::svector load_bin_file(std::istream& stream, bool as_big_endian) + // load_bin "overload" for file-like objects + // we check that `fclose` can be called on them! + template + auto load_bin_imp(I& file, std::string& buffer) + -> decltype(fclose(file), void()) { - std::string buffer{std::istreambuf_iterator{stream}, {}}; + fseek(file, 0, SEEK_END); + std::size_t size = ftell(file); + buffer.resize(size); + rewind(file); + fread(&buffer[0], 1, size, file); + } + + // load_bin "overload" for stream-like objects + // we check that they have a `tellg` method! + template + auto load_bin_imp(I& stream, std::string& buffer) + -> decltype(stream.tellg(), void()) + { + buffer = {std::istreambuf_iterator{stream}, {}}; + } + + template + inline xt::svector load_bin(I& stream, bool as_big_endian) + { + std::string buffer; + load_bin_imp(stream, buffer); std::size_t uncompressed_size = buffer.size() / sizeof(T); xt::svector uncompressed_buffer(uncompressed_size); std::copy((const T*)(buffer.data()), (const T*)(buffer.data()) + uncompressed_size, uncompressed_buffer.begin()); @@ -34,8 +57,28 @@ namespace xt return uncompressed_buffer; } + // dump_bin "overload" for file-like objects + // we check that `fclose` can be called on them! + template + auto dump_bin_imp(O& file, const char* uncompressed_buffer, std::size_t uncompressed_size) + -> decltype(fclose(file), void()) + { + fwrite(uncompressed_buffer, 1, uncompressed_size, file); + fflush(file); + } + + // dump_bin "overload" for stream-like objects + // we check that they have a `tellp` method! + template + auto dump_bin_imp(O& stream, const char* uncompressed_buffer, std::size_t uncompressed_size) + -> decltype(stream.tellp(), void()) + { + stream.write(uncompressed_buffer, std::streamsize(uncompressed_size)); + stream.flush(); + } + template - inline void dump_bin_stream(O& stream, const xexpression& e, bool as_big_endian) + inline void dump_bin(O& stream, const xexpression& e, bool as_big_endian) { using value_type = typename E::value_type; const E& ex = e.derived_cast(); @@ -56,11 +99,16 @@ namespace xt { uncompressed_buffer = reinterpret_cast(eval_ex.data()); } - stream.write(uncompressed_buffer, std::streamsize(uncompressed_size)); - stream.flush(); + dump_bin_imp(stream, uncompressed_buffer, uncompressed_size); } } // namespace detail + template + inline void dump_bin(O& stream, const xexpression& e, bool as_big_endian=is_big_endian()) + { + detail::dump_bin(stream, e, as_big_endian); + } + /** * Save xexpression to binary format * @@ -68,9 +116,9 @@ namespace xt * @param e the xexpression */ template - inline void dump_bin(std::ostream& stream, const xexpression& e, bool as_big_endian=is_big_endian()) + inline void dump_bin(std::ofstream& stream, const xexpression& e, bool as_big_endian=is_big_endian()) { - detail::dump_bin_stream(stream, e, as_big_endian); + detail::dump_bin(stream, e, as_big_endian); } /** @@ -87,7 +135,7 @@ namespace xt { std::runtime_error("IO Error: failed to open file"); } - detail::dump_bin_stream(stream, e, as_big_endian); + detail::dump_bin(stream, e, as_big_endian); } /** @@ -99,7 +147,7 @@ namespace xt inline std::string dump_bin(const xexpression& e, bool as_big_endian=is_big_endian()) { std::stringstream stream; - detail::dump_bin_stream(stream, e, as_big_endian); + detail::dump_bin(stream, e, as_big_endian); return stream.str(); } @@ -112,10 +160,10 @@ namespace xt * Fortran format * @return xarray with contents from binary file */ - template - inline auto load_bin(std::istream& stream, bool as_big_endian=is_big_endian()) + template + inline auto load_bin(I& stream, bool as_big_endian=is_big_endian()) { - xt::svector uncompressed_buffer = detail::load_bin_file(stream, as_big_endian); + xt::svector uncompressed_buffer = detail::load_bin(stream, as_big_endian); std::vector shape = {uncompressed_buffer.size()}; auto array = adapt(std::move(uncompressed_buffer), shape); return array; @@ -170,8 +218,8 @@ namespace xt } }; - template - void load_file(std::istream& stream, xexpression& e, const xio_binary_config& config) + template + void load_file(I& stream, xexpression& e, const xio_binary_config& config) { E& ex = e.derived_cast(); auto shape = ex.shape(); @@ -186,8 +234,8 @@ namespace xt } } - template - void dump_file(std::ostream& stream, const xexpression &e, const xio_binary_config& config) + template + void dump_file(O& stream, const xexpression &e, const xio_binary_config& config) { dump_bin(stream, e, config.big_endian); } diff --git a/test/test_xio_binary.cpp b/test/test_xio_binary.cpp index 48160a0..01f7e15 100644 --- a/test/test_xio_binary.cpp +++ b/test/test_xio_binary.cpp @@ -13,13 +13,13 @@ namespace xt { - TEST(xio_binary, dump_load) + TEST(xio_binary, dump_load_stream) { xtensor data {{ 1.0, 2.0, 3.0, 4.0}, {10.0, 12.0, 15.0, 18.0}}; - const char* fname = "data.bin"; + const char* fname = "data_stream.bin"; std::ofstream out_file(fname, std::ofstream::binary); dump_file(out_file, data, xio_binary_config()); @@ -30,4 +30,22 @@ namespace xt ASSERT_TRUE(all(equal(a, data))); } + + TEST(xio_binary, dump_load_file) + { + xtensor data + {{ 1.0, 2.0, 3.0, 4.0}, + {10.0, 12.0, 15.0, 18.0}}; + + const char* fname = "data_file.bin"; + FILE* out_file = fopen(fname, "wb"); + dump_file(out_file, data, xio_binary_config()); + + xarray a; + FILE* in_file = fopen(fname, "rb"); + load_file(in_file, a, xio_binary_config()); + a.reshape({2, 4}); + + ASSERT_TRUE(all(equal(a, data))); + } } From 86ec45eb154d2ee41fdf303a1502157bff44ddb8 Mon Sep 17 00:00:00 2001 From: David Brochart Date: Sat, 26 Dec 2020 01:12:54 +0100 Subject: [PATCH 2/3] Add xio_gdal_handler --- CMakeLists.txt | 3 + include/xtensor-io/xio_binary.hpp | 17 ++--- include/xtensor-io/xio_file.hpp | 58 +++++++++++++++++ include/xtensor-io/xio_gdal_handler.hpp | 85 +++++++++++++++++++++++++ include/xtensor-io/xio_vsilfile.hpp | 58 +++++++++++++++++ test/CMakeLists.txt | 1 + test/test_xio_gdal_handler.cpp | 29 +++++++++ 7 files changed, 243 insertions(+), 8 deletions(-) create mode 100644 include/xtensor-io/xio_file.hpp create mode 100644 include/xtensor-io/xio_gdal_handler.hpp create mode 100644 include/xtensor-io/xio_vsilfile.hpp create mode 100644 test/test_xio_gdal_handler.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index cdc9aff..62a101e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -62,7 +62,10 @@ set(XTENSOR_IO_HEADERS ${XTENSOR_IO_INCLUDE_DIR}/xtensor-io/xio_aws_handler.hpp ${XTENSOR_IO_INCLUDE_DIR}/xtensor-io/xio_disk_handler.hpp ${XTENSOR_IO_INCLUDE_DIR}/xtensor-io/xio_gcs_handler.hpp + ${XTENSOR_IO_INCLUDE_DIR}/xtensor-io/xio_gdal_handler.hpp ${XTENSOR_IO_INCLUDE_DIR}/xtensor-io/xio_gzip.hpp + ${XTENSOR_IO_INCLUDE_DIR}/xtensor-io/xio_file.hpp + ${XTENSOR_IO_INCLUDE_DIR}/xtensor-io/xio_vsilfile.hpp ${XTENSOR_IO_INCLUDE_DIR}/xtensor-io/xnpz.hpp ${XTENSOR_IO_INCLUDE_DIR}/xtensor-io/xtensor-io.hpp ${XTENSOR_IO_INCLUDE_DIR}/xtensor-io/xtensor_io_config.hpp diff --git a/include/xtensor-io/xio_binary.hpp b/include/xtensor-io/xio_binary.hpp index 116d612..b94c3da 100644 --- a/include/xtensor-io/xio_binary.hpp +++ b/include/xtensor-io/xio_binary.hpp @@ -15,6 +15,7 @@ #include "xtensor/xadapt.hpp" #include "xtensor-io.hpp" #include "xfile_array.hpp" +#include "xio_file.hpp" namespace xt { @@ -24,13 +25,13 @@ namespace xt // we check that `fclose` can be called on them! template auto load_bin_imp(I& file, std::string& buffer) - -> decltype(fclose(file), void()) + -> decltype(file.ftell(), void()) { - fseek(file, 0, SEEK_END); - std::size_t size = ftell(file); + file.fseek(0, SEEK_END); + std::size_t size = file.ftell(); buffer.resize(size); - rewind(file); - fread(&buffer[0], 1, size, file); + file.rewind(); + file.fread(&buffer[0], 1, size); } // load_bin "overload" for stream-like objects @@ -61,10 +62,10 @@ namespace xt // we check that `fclose` can be called on them! template auto dump_bin_imp(O& file, const char* uncompressed_buffer, std::size_t uncompressed_size) - -> decltype(fclose(file), void()) + -> decltype(file.ftell(), void()) { - fwrite(uncompressed_buffer, 1, uncompressed_size, file); - fflush(file); + file.fwrite(uncompressed_buffer, 1, uncompressed_size); + file.fflush(); } // dump_bin "overload" for stream-like objects diff --git a/include/xtensor-io/xio_file.hpp b/include/xtensor-io/xio_file.hpp new file mode 100644 index 0000000..bd09974 --- /dev/null +++ b/include/xtensor-io/xio_file.hpp @@ -0,0 +1,58 @@ +#ifndef XTENSOR_IO_FILE_HPP +#define XTENSOR_IO_FILE_HPP + +#include + +namespace xt +{ + class xfile + { + public: + xfile(FILE* stream); + size_t fwrite(const void* ptr, size_t size, size_t count); + size_t fread(void* ptr, size_t size, size_t count); + long int ftell(); + int fseek(long int offset, int origin); + void rewind(); + int fflush(); + private: + FILE* m_stream; + }; + + inline xfile::xfile(FILE* stream) + : m_stream(stream) + { + } + + inline size_t xfile::fwrite(const void* ptr, size_t size, size_t count) + { + return ::fwrite(ptr, size, count, m_stream); + } + + inline size_t xfile::fread(void* ptr, size_t size, size_t count) + { + return ::fread(ptr, size, count, m_stream); + } + + inline long int xfile::ftell() + { + return ::ftell(m_stream); + } + + inline int xfile::fseek(long int offset, int origin) + { + return ::fseek(m_stream, offset, origin); + } + + inline void xfile::rewind() + { + ::rewind(m_stream); + } + + inline int xfile::fflush() + { + return ::fflush(m_stream); + } +} + +#endif diff --git a/include/xtensor-io/xio_gdal_handler.hpp b/include/xtensor-io/xio_gdal_handler.hpp new file mode 100644 index 0000000..cd4eb31 --- /dev/null +++ b/include/xtensor-io/xio_gdal_handler.hpp @@ -0,0 +1,85 @@ +#ifndef XTENSOR_IO_GDAL_HANDLER_HPP +#define XTENSOR_IO_GDAL_HANDLER_HPP + +#include +#include +#include +#include + +#include + +namespace xt +{ + struct xio_gdal_config + { + }; + + template + class xio_gdal_handler + { + public: + using io_config = xio_gdal_config; + + template + void write(const xexpression& expression, const std::string& path, xfile_dirty dirty); + + template + void read(ET& array, const std::string& path); + + void configure(const C& format_config, const xio_gdal_config& io_config); + void configure_io(const xio_gdal_config& io_config); + + private: + + C m_format_config; + }; + + template + template + inline void xio_gdal_handler::write(const xexpression& expression, const std::string& path, xfile_dirty dirty) + { + if (m_format_config.will_dump(dirty)) + { + VSILFILE* out_file = VSIFOpenL(path.c_str(), "wb"); + if (out_file != NULL) + { + auto f = xvsilfile(out_file); + dump_file(f, expression, m_format_config); + } + else + { + XTENSOR_THROW(std::runtime_error, "write: failed to open file " + path); + } + } + } + + template + template + inline void xio_gdal_handler::read(ET& array, const std::string& path) + { + VSILFILE* in_file = VSIFOpenL(path.c_str(), "rb"); + if (in_file != NULL) + { + auto f = xvsilfile(in_file); + load_file(f, array, m_format_config); + } + else + { + XTENSOR_THROW(std::runtime_error, "read: failed to open file " + path); + } + } + + template + inline void xio_gdal_handler::configure(const C& format_config, const xio_gdal_config& io_config) + { + m_format_config = format_config; + } + + template + inline void xio_gdal_handler::configure_io(const xio_gdal_config& io_config) + { + } + +} + +#endif diff --git a/include/xtensor-io/xio_vsilfile.hpp b/include/xtensor-io/xio_vsilfile.hpp new file mode 100644 index 0000000..f093042 --- /dev/null +++ b/include/xtensor-io/xio_vsilfile.hpp @@ -0,0 +1,58 @@ +#ifndef XTENSOR_IO_VSILFILE_HPP +#define XTENSOR_IO_VSILFILE_HPP + +#include + +namespace xt +{ + class xvsilfile + { + public: + xvsilfile(VSILFILE* stream); + size_t fwrite(const void* ptr, size_t size, size_t count); + size_t fread(void* ptr, size_t size, size_t count); + long int ftell(); + int fseek(long int offset, int origin); + void rewind(); + int fflush(); + private: + VSILFILE* m_stream; + }; + + inline xvsilfile::xvsilfile(VSILFILE* stream) + : m_stream(stream) + { + } + + inline size_t xvsilfile::fwrite(const void* ptr, size_t size, size_t count) + { + return VSIFWriteL(ptr, size, count, m_stream); + } + + inline size_t xvsilfile::fread(void* ptr, size_t size, size_t count) + { + return VSIFReadL(ptr, size, count, m_stream); + } + + inline long int xvsilfile::ftell() + { + return VSIFTellL(m_stream); + } + + inline int xvsilfile::fseek(long int offset, int origin) + { + return VSIFSeekL(m_stream, offset, origin); + } + + inline void xvsilfile::rewind() + { + VSIRewindL(m_stream); + } + + inline int xvsilfile::fflush() + { + return VSIFFlushL(m_stream); + } +} + +#endif diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 263a63d..dc129c6 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -85,6 +85,7 @@ set(XTENSOR_IO_TESTS test_xio_binary.cpp test_xio_gcs_handler.cpp test_xio_aws_handler.cpp + test_xio_gdal_handler.cpp ) set(XTENSOR_IO_HO_TESTS diff --git a/test/test_xio_gdal_handler.cpp b/test/test_xio_gdal_handler.cpp new file mode 100644 index 0000000..cc6a9fd --- /dev/null +++ b/test/test_xio_gdal_handler.cpp @@ -0,0 +1,29 @@ +/*************************************************************************** +* Copyright (c) Johan Mabille, Sylvain Corlay and Wolf Vollprecht * +* Copyright (c) QuantStack * +* * +* Distributed under the terms of the BSD 3-Clause License. * +* * +* The full license is in the file LICENSE, distributed with this software. * +****************************************************************************/ + +#include "gtest/gtest.h" + +#include +#include "xtensor-io/xio_vsilfile.hpp" +#include "xtensor-io/xio_binary.hpp" +#include "xtensor-io/xio_gdal_handler.hpp" + +namespace xt +{ + TEST(xio_gdal_handler, write_read) + { + VSIInstallMemFileHandler(); + xio_gdal_handler h; + xarray a0 = {3, 2, 1, 4}; + h.write(a0, "/vsimem/test_file.tif", xfile_dirty(true)); + xarray a1; + h.read(a1, "/vsimem/test_file.tif"); + EXPECT_TRUE(xt::all(xt::equal(a0, a1))); + } +} From b0fd65771ad849906ab5663e86d731222c4d0995 Mon Sep 17 00:00:00 2001 From: David Brochart Date: Sat, 26 Dec 2020 04:26:49 +0100 Subject: [PATCH 3/3] Refactor, add file and stream wrapper classes --- CMakeLists.txt | 5 +- include/xtensor-io/xio_aws_handler.hpp | 7 +- include/xtensor-io/xio_binary.hpp | 91 ++++++++------------- include/xtensor-io/xio_blosc.hpp | 66 ++++++++++----- include/xtensor-io/xio_disk_handler.hpp | 7 +- include/xtensor-io/xio_file.hpp | 58 ------------- include/xtensor-io/xio_file_wrapper.hpp | 71 ++++++++++++++++ include/xtensor-io/xio_gcs_handler.hpp | 7 +- include/xtensor-io/xio_gdal_handler.hpp | 6 +- include/xtensor-io/xio_gzip.hpp | 57 +++++++++---- include/xtensor-io/xio_stream_wrapper.hpp | 76 +++++++++++++++++ include/xtensor-io/xio_vsilfile.hpp | 58 ------------- include/xtensor-io/xio_vsilfile_wrapper.hpp | 71 ++++++++++++++++ test/test_xchunk_store_manager.cpp | 9 +- test/test_xfile_array.cpp | 6 +- test/test_xio_aws_handler.cpp | 12 +-- test/test_xio_binary.cpp | 14 +++- test/test_xio_gdal_handler.cpp | 25 +++++- 18 files changed, 406 insertions(+), 240 deletions(-) delete mode 100644 include/xtensor-io/xio_file.hpp create mode 100644 include/xtensor-io/xio_file_wrapper.hpp create mode 100644 include/xtensor-io/xio_stream_wrapper.hpp delete mode 100644 include/xtensor-io/xio_vsilfile.hpp create mode 100644 include/xtensor-io/xio_vsilfile_wrapper.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 62a101e..d72d4d6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -64,8 +64,9 @@ set(XTENSOR_IO_HEADERS ${XTENSOR_IO_INCLUDE_DIR}/xtensor-io/xio_gcs_handler.hpp ${XTENSOR_IO_INCLUDE_DIR}/xtensor-io/xio_gdal_handler.hpp ${XTENSOR_IO_INCLUDE_DIR}/xtensor-io/xio_gzip.hpp - ${XTENSOR_IO_INCLUDE_DIR}/xtensor-io/xio_file.hpp - ${XTENSOR_IO_INCLUDE_DIR}/xtensor-io/xio_vsilfile.hpp + ${XTENSOR_IO_INCLUDE_DIR}/xtensor-io/xio_file_wrapper.hpp + ${XTENSOR_IO_INCLUDE_DIR}/xtensor-io/xio_vsilfile_wrapper.hpp + ${XTENSOR_IO_INCLUDE_DIR}/xtensor-io/xio_stream_wrapper.hpp ${XTENSOR_IO_INCLUDE_DIR}/xtensor-io/xnpz.hpp ${XTENSOR_IO_INCLUDE_DIR}/xtensor-io/xtensor-io.hpp ${XTENSOR_IO_INCLUDE_DIR}/xtensor-io/xtensor_io_config.hpp diff --git a/include/xtensor-io/xio_aws_handler.hpp b/include/xtensor-io/xio_aws_handler.hpp index 3eeec76..8411e93 100644 --- a/include/xtensor-io/xio_aws_handler.hpp +++ b/include/xtensor-io/xio_aws_handler.hpp @@ -4,6 +4,7 @@ #include "xtensor/xarray.hpp" #include "xtensor/xexpression.hpp" #include "xfile_array.hpp" +#include "xio_stream_wrapper.hpp" #include #include #include @@ -70,7 +71,8 @@ namespace xt request.SetKey(path2); std::shared_ptr writer = Aws::MakeShared("SampleAllocationTag", path, std::ios_base::in | std::ios_base::binary); - dump_file(*writer, expression, m_format_config); + auto s = xostream_wrapper(*writer); + dump_file(s, expression, m_format_config); request.SetBody(writer); @@ -109,7 +111,8 @@ namespace xt } auto& reader = outcome.GetResultWithOwnership().GetBody(); - load_file(reader, array, m_format_config); + auto s = xistream_wrapper(reader); + load_file(s, array, m_format_config); } template diff --git a/include/xtensor-io/xio_binary.hpp b/include/xtensor-io/xio_binary.hpp index b94c3da..7a6f953 100644 --- a/include/xtensor-io/xio_binary.hpp +++ b/include/xtensor-io/xio_binary.hpp @@ -15,39 +15,17 @@ #include "xtensor/xadapt.hpp" #include "xtensor-io.hpp" #include "xfile_array.hpp" -#include "xio_file.hpp" +#include "xio_stream_wrapper.hpp" namespace xt { namespace detail { - // load_bin "overload" for file-like objects - // we check that `fclose` can be called on them! - template - auto load_bin_imp(I& file, std::string& buffer) - -> decltype(file.ftell(), void()) - { - file.fseek(0, SEEK_END); - std::size_t size = file.ftell(); - buffer.resize(size); - file.rewind(); - file.fread(&buffer[0], 1, size); - } - - // load_bin "overload" for stream-like objects - // we check that they have a `tellg` method! - template - auto load_bin_imp(I& stream, std::string& buffer) - -> decltype(stream.tellg(), void()) - { - buffer = {std::istreambuf_iterator{stream}, {}}; - } - template inline xt::svector load_bin(I& stream, bool as_big_endian) { std::string buffer; - load_bin_imp(stream, buffer); + stream.read_all(buffer); std::size_t uncompressed_size = buffer.size() / sizeof(T); xt::svector uncompressed_buffer(uncompressed_size); std::copy((const T*)(buffer.data()), (const T*)(buffer.data()) + uncompressed_size, uncompressed_buffer.begin()); @@ -58,26 +36,6 @@ namespace xt return uncompressed_buffer; } - // dump_bin "overload" for file-like objects - // we check that `fclose` can be called on them! - template - auto dump_bin_imp(O& file, const char* uncompressed_buffer, std::size_t uncompressed_size) - -> decltype(file.ftell(), void()) - { - file.fwrite(uncompressed_buffer, 1, uncompressed_size); - file.fflush(); - } - - // dump_bin "overload" for stream-like objects - // we check that they have a `tellp` method! - template - auto dump_bin_imp(O& stream, const char* uncompressed_buffer, std::size_t uncompressed_size) - -> decltype(stream.tellp(), void()) - { - stream.write(uncompressed_buffer, std::streamsize(uncompressed_size)); - stream.flush(); - } - template inline void dump_bin(O& stream, const xexpression& e, bool as_big_endian) { @@ -100,26 +58,28 @@ namespace xt { uncompressed_buffer = reinterpret_cast(eval_ex.data()); } - dump_bin_imp(stream, uncompressed_buffer, uncompressed_size); + stream.write(uncompressed_buffer, uncompressed_size); + stream.flush(); } } // namespace detail - template - inline void dump_bin(O& stream, const xexpression& e, bool as_big_endian=is_big_endian()) - { - detail::dump_bin(stream, e, as_big_endian); - } - /** * Save xexpression to binary format * * @param stream An output stream to which to dump the data * @param e the xexpression */ + template + inline void dump_bin(O& stream, const xexpression& e, bool as_big_endian=is_big_endian()) + { + detail::dump_bin(stream, e, as_big_endian); + } + template inline void dump_bin(std::ofstream& stream, const xexpression& e, bool as_big_endian=is_big_endian()) { - detail::dump_bin(stream, e, as_big_endian); + auto s = xostream_wrapper(stream); + detail::dump_bin(s, e, as_big_endian); } /** @@ -129,14 +89,21 @@ namespace xt * @param e the xexpression */ template - inline void dump_bin(const std::string& filename, const xexpression& e, bool as_big_endian=is_big_endian()) + inline void dump_bin(const char* filename, const xexpression& e, bool as_big_endian=is_big_endian()) { std::ofstream stream(filename, std::ofstream::binary); if (!stream.is_open()) { std::runtime_error("IO Error: failed to open file"); } - detail::dump_bin(stream, e, as_big_endian); + auto s = xostream_wrapper(stream); + detail::dump_bin(s, e, as_big_endian); + } + + template + inline void dump_bin(const std::string& filename, const xexpression& e, bool as_big_endian=is_big_endian()) + { + dump_bin(filename.c_str(), e, as_big_endian); } /** @@ -148,7 +115,8 @@ namespace xt inline std::string dump_bin(const xexpression& e, bool as_big_endian=is_big_endian()) { std::stringstream stream; - detail::dump_bin(stream, e, as_big_endian); + auto s = xostream_wrapper(stream); + detail::dump_bin(s, e, as_big_endian); return stream.str(); } @@ -180,14 +148,21 @@ namespace xt * @return xarray with contents from binary file */ template - inline auto load_bin(const std::string& filename, bool as_big_endian=is_big_endian()) + inline auto load_bin(const char* filename, bool as_big_endian=is_big_endian()) { std::ifstream stream(filename, std::ifstream::binary); if (!stream.is_open()) { - std::runtime_error("load_bin: failed to open file " + filename); + std::runtime_error(std::string("load_bin: failed to open file ") + filename); } - return load_bin(stream, as_big_endian); + auto s = xistream_wrapper(stream); + return load_bin(s, as_big_endian); + } + + template + inline auto load_bin(const std::string& filename, bool as_big_endian=is_big_endian()) + { + return load_bin(filename.c_str(), as_big_endian); } struct xio_binary_config diff --git a/include/xtensor-io/xio_blosc.hpp b/include/xtensor-io/xio_blosc.hpp index b7ded7b..78f7b3e 100644 --- a/include/xtensor-io/xio_blosc.hpp +++ b/include/xtensor-io/xio_blosc.hpp @@ -16,6 +16,7 @@ #include "xtensor-io.hpp" #include "xfile_array.hpp" #include "blosc.h" +#include "xio_stream_wrapper.hpp" namespace xt { @@ -31,11 +32,12 @@ namespace xt } } - template - inline xt::svector load_blosc_file(std::istream& stream, bool as_big_endian) + template + inline xt::svector load_blosc(I& stream, bool as_big_endian) { init_blosc(); - std::string compressed_buffer{std::istreambuf_iterator{stream}, {}}; + std::string compressed_buffer; + stream.read_all(compressed_buffer); auto compressed_size = compressed_buffer.size(); std::size_t uncompressed_size = 0; int res = blosc_cbuffer_validate(compressed_buffer.data(), compressed_size, &uncompressed_size); @@ -60,7 +62,7 @@ namespace xt } template - inline void dump_blosc_stream(O& stream, const xexpression& e, bool as_big_endian, int clevel, int shuffle, const char* cname, std::size_t blocksize) + inline void dump_blosc(O& stream, const xexpression& e, bool as_big_endian, int clevel, int shuffle, const char* cname, std::size_t blocksize) { init_blosc(); using value_type = typename E::value_type; @@ -99,8 +101,8 @@ namespace xt { XTENSOR_THROW(std::runtime_error, "Blosc: compression error"); } - stream.write(compressed_buffer, - std::streamsize(true_compressed_size)); + stream.write(compressed_buffer, std::streamsize(true_compressed_size)); + stream.flush(); char_allocator.deallocate(compressed_buffer, max_compressed_size); } } // namespace detail @@ -111,10 +113,17 @@ namespace xt * @param stream An output stream to which to dump the data * @param e the xexpression */ + template + inline void dump_blosc(O& stream, const xexpression& e, bool as_big_endian=is_big_endian(), int clevel=5, int shuffle=1, const char* cname="blosclz", std::size_t blocksize=0) + { + detail::dump_blosc(stream, e, as_big_endian, clevel, shuffle, cname, blocksize); + } + template inline void dump_blosc(std::ostream& stream, const xexpression& e, bool as_big_endian=is_big_endian(), int clevel=5, int shuffle=1, const char* cname="blosclz", std::size_t blocksize=0) { - detail::dump_blosc_stream(stream, e, as_big_endian, clevel, shuffle, cname, blocksize); + auto s = xostream_wrapper(stream); + detail::dump_blosc(s, e, as_big_endian, clevel, shuffle, cname, blocksize); } /** @@ -124,14 +133,21 @@ namespace xt * @param e the xexpression */ template - inline void dump_blosc(const std::string& filename, const xexpression& e, bool as_big_endian=is_big_endian(), int clevel=5, int shuffle=1, const char* cname="blosclz", std::size_t blocksize=0) + inline void dump_blosc(const char* filename, const xexpression& e, bool as_big_endian=is_big_endian(), int clevel=5, int shuffle=1, const char* cname="blosclz", std::size_t blocksize=0) { std::ofstream stream(filename, std::ofstream::binary); if (!stream.is_open()) { - XTENSOR_THROW(std::runtime_error, "Blosc: failed to open file " + filename); + XTENSOR_THROW(std::runtime_error, std::string("Blosc: failed to open file ") + filename); } - detail::dump_blosc_stream(stream, e, as_big_endian, clevel, shuffle, cname, blocksize); + auto s = xostream_wrapper(stream); + detail::dump_blosc(s, e, as_big_endian, clevel, shuffle, cname, blocksize); + } + + template + inline void dump_blosc(const std::string& filename, const xexpression& e, bool as_big_endian=is_big_endian(), int clevel=5, int shuffle=1, const char* cname="blosclz", std::size_t blocksize=0) + { + dump_blosc(filename.c_str(), e, as_big_endian, clevel, shuffle, cname, blocksize); } /** @@ -143,7 +159,8 @@ namespace xt inline std::string dump_blosc(const xexpression& e, bool as_big_endian=is_big_endian(), int clevel=5, int shuffle=1, const char* cname="blosclz", std::size_t blocksize=0) { std::stringstream stream; - detail::dump_blosc_stream(stream, e, as_big_endian, clevel, shuffle, cname, blocksize); + auto s = xostream_wrapper(stream); + detail::dump_blosc(s, e, as_big_endian, clevel, shuffle, cname, blocksize); return stream.str(); } @@ -156,10 +173,10 @@ namespace xt * Fortran format * @return xarray with contents from blosc file */ - template - inline auto load_blosc(std::istream& stream, bool as_big_endian=is_big_endian()) + template + inline auto load_blosc(I& stream, bool as_big_endian=is_big_endian()) { - xt::svector uncompressed_buffer = detail::load_blosc_file(stream, as_big_endian); + xt::svector uncompressed_buffer = detail::load_blosc(stream, as_big_endian); std::vector shape = {uncompressed_buffer.size()}; auto array = adapt(std::move(uncompressed_buffer), shape); return array; @@ -175,14 +192,21 @@ namespace xt * @return xarray with contents from blosc file */ template - inline auto load_blosc(const std::string& filename, bool as_big_endian=is_big_endian()) + inline auto load_blosc(const char* filename, bool as_big_endian=is_big_endian()) { std::ifstream stream(filename, std::ifstream::binary); if (!stream.is_open()) { - XTENSOR_THROW(std::runtime_error, "Blosc: failed to open file " + filename); + XTENSOR_THROW(std::runtime_error, std::string("Blosc: failed to open file ") + filename); } - return load_blosc(stream, as_big_endian); + auto s = xistream_wrapper(stream);; + return load_blosc(s, as_big_endian); + } + + template + inline auto load_blosc(const std::string& filename, bool as_big_endian=is_big_endian()) + { + return load_blosc(filename.c_str(), as_big_endian); } struct xio_blosc_config @@ -230,8 +254,8 @@ namespace xt } }; - template - void load_file(std::istream& stream, xexpression& e, const xio_blosc_config& config) + template + void load_file(I& stream, xexpression& e, const xio_blosc_config& config) { E& ex = e.derived_cast(); auto shape = ex.shape(); @@ -246,8 +270,8 @@ namespace xt } } - template - void dump_file(std::ostream& stream, const xexpression &e, const xio_blosc_config& config) + template + void dump_file(O& stream, const xexpression &e, const xio_blosc_config& config) { dump_blosc(stream, e, config.big_endian, config.clevel, config.shuffle, config.cname.c_str(), config.blocksize); } diff --git a/include/xtensor-io/xio_disk_handler.hpp b/include/xtensor-io/xio_disk_handler.hpp index 8fba150..f9f8612 100644 --- a/include/xtensor-io/xio_disk_handler.hpp +++ b/include/xtensor-io/xio_disk_handler.hpp @@ -3,6 +3,7 @@ #include #include +#include namespace xt { @@ -39,7 +40,8 @@ namespace xt std::ofstream out_file(path, std::ofstream::binary); if (out_file.is_open()) { - dump_file(out_file, expression, m_format_config); + auto s = xostream_wrapper(out_file); + dump_file(s, expression, m_format_config); } else { @@ -55,7 +57,8 @@ namespace xt std::ifstream in_file(path, std::ifstream::binary); if (in_file.is_open()) { - load_file(in_file, array, m_format_config); + auto s = xistream_wrapper(in_file); + load_file(s, array, m_format_config); } else { diff --git a/include/xtensor-io/xio_file.hpp b/include/xtensor-io/xio_file.hpp deleted file mode 100644 index bd09974..0000000 --- a/include/xtensor-io/xio_file.hpp +++ /dev/null @@ -1,58 +0,0 @@ -#ifndef XTENSOR_IO_FILE_HPP -#define XTENSOR_IO_FILE_HPP - -#include - -namespace xt -{ - class xfile - { - public: - xfile(FILE* stream); - size_t fwrite(const void* ptr, size_t size, size_t count); - size_t fread(void* ptr, size_t size, size_t count); - long int ftell(); - int fseek(long int offset, int origin); - void rewind(); - int fflush(); - private: - FILE* m_stream; - }; - - inline xfile::xfile(FILE* stream) - : m_stream(stream) - { - } - - inline size_t xfile::fwrite(const void* ptr, size_t size, size_t count) - { - return ::fwrite(ptr, size, count, m_stream); - } - - inline size_t xfile::fread(void* ptr, size_t size, size_t count) - { - return ::fread(ptr, size, count, m_stream); - } - - inline long int xfile::ftell() - { - return ::ftell(m_stream); - } - - inline int xfile::fseek(long int offset, int origin) - { - return ::fseek(m_stream, offset, origin); - } - - inline void xfile::rewind() - { - ::rewind(m_stream); - } - - inline int xfile::fflush() - { - return ::fflush(m_stream); - } -} - -#endif diff --git a/include/xtensor-io/xio_file_wrapper.hpp b/include/xtensor-io/xio_file_wrapper.hpp new file mode 100644 index 0000000..646b770 --- /dev/null +++ b/include/xtensor-io/xio_file_wrapper.hpp @@ -0,0 +1,71 @@ +#ifndef XTENSOR_IO_FILE_WRAPPER_HPP +#define XTENSOR_IO_FILE_WRAPPER_HPP + +#include + +namespace xt +{ + class xfile_wrapper + { + public: + xfile_wrapper(FILE* pfile); + std::streampos tellg(); + xfile_wrapper& read_all(std::string& s); + xfile_wrapper& read(char* s, std::streamsize n); + void write(const char* s, std::streamsize size); + std::streamsize gcount(); + void flush(); + bool eof(); + private: + FILE* m_pfile; + std::size_t m_gcount; + }; + + inline xfile_wrapper::xfile_wrapper(FILE* pfile) + : m_pfile(pfile) + { + } + + inline std::streampos xfile_wrapper::tellg() + { + return ftell(m_pfile); + } + + inline xfile_wrapper& xfile_wrapper::read_all(std::string& s) + { + fseek(m_pfile, 0, SEEK_END); + std::size_t size = ftell(m_pfile); + s.resize(size); + rewind(m_pfile); + m_gcount = fread(&s[0], 1, size, m_pfile); + return *this; + } + + inline xfile_wrapper& xfile_wrapper::read(char* s, std::streamsize n) + { + m_gcount = fread(&s[0], 1, n, m_pfile); + return *this; + } + + inline std::streamsize xfile_wrapper::gcount() + { + return m_gcount; + } + + inline void xfile_wrapper::write(const char* s, std::streamsize size) + { + fwrite(s, 1, size, m_pfile); + } + + inline void xfile_wrapper::flush() + { + fflush(m_pfile); + } + + inline bool xfile_wrapper::eof() + { + return feof(m_pfile); + } +} + +#endif diff --git a/include/xtensor-io/xio_gcs_handler.hpp b/include/xtensor-io/xio_gcs_handler.hpp index d349598..ec1e879 100644 --- a/include/xtensor-io/xio_gcs_handler.hpp +++ b/include/xtensor-io/xio_gcs_handler.hpp @@ -1,6 +1,7 @@ #ifndef XTENSOR_IO_GCS_HANDLER_HPP #define XTENSOR_IO_GCS_HANDLER_HPP +#include "xio_stream_wrapper.hpp" #include "xtensor/xarray.hpp" #include "xtensor/xexpression.hpp" #include "google/cloud/storage/client.h" @@ -52,7 +53,8 @@ namespace xt if (m_format_config.will_dump(dirty)) { auto writer = m_client.WriteObject(m_bucket, path); - dump_file(writer, expression, m_format_config); + auto s = xt::xostream_wrapper(writer); + dump_file(s, expression, m_format_config); } } @@ -61,7 +63,8 @@ namespace xt inline void xio_gcs_handler::read(ET& array, const std::string& path) { auto reader = m_client.ReadObject(m_bucket, path); - load_file(reader, array, m_format_config); + auto s = xt::xistream_wrapper(reader); + load_file(s, array, m_format_config); } template diff --git a/include/xtensor-io/xio_gdal_handler.hpp b/include/xtensor-io/xio_gdal_handler.hpp index cd4eb31..acd7267 100644 --- a/include/xtensor-io/xio_gdal_handler.hpp +++ b/include/xtensor-io/xio_gdal_handler.hpp @@ -4,7 +4,7 @@ #include #include #include -#include +#include #include @@ -43,7 +43,7 @@ namespace xt VSILFILE* out_file = VSIFOpenL(path.c_str(), "wb"); if (out_file != NULL) { - auto f = xvsilfile(out_file); + auto f = xvsilfile_wrapper(out_file); dump_file(f, expression, m_format_config); } else @@ -60,7 +60,7 @@ namespace xt VSILFILE* in_file = VSIFOpenL(path.c_str(), "rb"); if (in_file != NULL) { - auto f = xvsilfile(in_file); + auto f = xvsilfile_wrapper(in_file); load_file(f, array, m_format_config); } else diff --git a/include/xtensor-io/xio_gzip.hpp b/include/xtensor-io/xio_gzip.hpp index 901d181..0e62af3 100644 --- a/include/xtensor-io/xio_gzip.hpp +++ b/include/xtensor-io/xio_gzip.hpp @@ -17,6 +17,7 @@ #include "xtensor/xadapt.hpp" #include "xtensor-io.hpp" #include "xfile_array.hpp" +#include "xio_stream_wrapper.hpp" #define GZIP_CHUNK 0x4000 #define GZIP_WINDOWBITS 15 @@ -27,8 +28,8 @@ namespace xt { namespace detail { - template - inline xt::svector load_gzip_file(std::istream& stream, bool as_big_endian) + template + inline xt::svector load_gzip(I& stream, bool as_big_endian) { xt::svector uncompressed_buffer; z_stream zs = {0}; @@ -88,7 +89,7 @@ namespace xt } template - inline void dump_gzip_stream(O& stream, const xexpression& e, bool as_big_endian, int level) + inline void dump_gzip(O& stream, const xexpression& e, bool as_big_endian, int level) { using value_type = typename E::value_type; const E& ex = e.derived_cast(); @@ -138,10 +139,17 @@ namespace xt * @param stream An output stream to which to dump the data * @param e the xexpression */ + template + inline void dump_gzip(O& stream, const xexpression& e, bool as_big_endian=is_big_endian(), int level=1) + { + detail::dump_gzip(stream, e, as_big_endian, level); + } + template inline void dump_gzip(std::ostream& stream, const xexpression& e, bool as_big_endian=is_big_endian(), int level=1) { - detail::dump_gzip_stream(stream, e, as_big_endian, level); + auto s = xostream_wrapper(stream); + detail::dump_gzip(s, e, as_big_endian, level); } /** @@ -151,14 +159,21 @@ namespace xt * @param e the xexpression */ template - inline void dump_gzip(const std::string& filename, const xexpression& e, bool as_big_endian=is_big_endian(), int level=1) + inline void dump_gzip(const char* filename, const xexpression& e, bool as_big_endian=is_big_endian(), int level=1) { std::ofstream stream(filename, std::ofstream::binary); if (!stream.is_open()) { std::runtime_error("IO Error: failed to open file"); } - detail::dump_gzip_stream(stream, e, as_big_endian, level); + auto s = xostream_wrapper(stream); + detail::dump_gzip(s, e, as_big_endian, level); + } + + template + inline void dump_gzip(const std::string& filename, const xexpression& e, bool as_big_endian=is_big_endian(), int level=1) + { + dump_gzip(filename.c_str(), e, as_big_endian, level); } /** @@ -170,7 +185,8 @@ namespace xt inline std::string dump_gzip(const xexpression& e, bool as_big_endian=is_big_endian(), int level=1) { std::stringstream stream; - detail::dump_gzip_stream(stream, e, as_big_endian, level); + auto s = xostream_wrapper(stream); + detail::dump_gzip(s, e, as_big_endian, level); return stream.str(); } @@ -183,10 +199,10 @@ namespace xt * Fortran format * @return xarray with contents from GZIP file */ - template - inline auto load_gzip(std::istream& stream, bool as_big_endian=is_big_endian()) + template + inline auto load_gzip(I& stream, bool as_big_endian=is_big_endian()) { - xt::svector uncompressed_buffer = detail::load_gzip_file(stream, as_big_endian); + xt::svector uncompressed_buffer = detail::load_gzip(stream, as_big_endian); std::vector shape = {uncompressed_buffer.size()}; auto array = adapt(std::move(uncompressed_buffer), shape); return array; @@ -202,14 +218,21 @@ namespace xt * @return xarray with contents from GZIP file */ template - inline auto load_gzip(const std::string& filename, bool as_big_endian=is_big_endian()) + inline auto load_gzip(const char* filename, bool as_big_endian=is_big_endian()) { std::ifstream stream(filename, std::ifstream::binary); if (!stream.is_open()) { - std::runtime_error("load_gzip: failed to open file " + filename); + std::runtime_error(std::string("load_gzip: failed to open file ") + filename); } - return load_gzip(stream, as_big_endian); + auto s = xistream_wrapper(stream); + return load_gzip(s, as_big_endian); + } + + template + inline auto load_gzip(const std::string& filename, bool as_big_endian=is_big_endian()) + { + return load_gzip(filename.c_str(), as_big_endian); } struct xio_gzip_config @@ -245,8 +268,8 @@ namespace xt } }; - template - void load_file(std::istream& stream, xexpression& e, const xio_gzip_config& config) + template + void load_file(I& stream, xexpression& e, const xio_gzip_config& config) { E& ex = e.derived_cast(); auto shape = ex.shape(); @@ -261,8 +284,8 @@ namespace xt } } - template - void dump_file(std::ostream& stream, const xexpression &e, const xio_gzip_config& config) + template + void dump_file(O& stream, const xexpression &e, const xio_gzip_config& config) { dump_gzip(stream, e, config.big_endian, config.level); } diff --git a/include/xtensor-io/xio_stream_wrapper.hpp b/include/xtensor-io/xio_stream_wrapper.hpp new file mode 100644 index 0000000..47af548 --- /dev/null +++ b/include/xtensor-io/xio_stream_wrapper.hpp @@ -0,0 +1,76 @@ +#ifndef XTENSOR_IO_STREAM_WRAPPER_HPP +#define XTENSOR_IO_STREAM_WRAPPER_HPP + +#include + +namespace xt +{ + class xistream_wrapper + { + public: + xistream_wrapper(std::istream& stream); + xistream_wrapper& read_all(std::string& s); + xistream_wrapper& read(char* s, std::streamsize n); + bool eof(); + std::streamsize gcount(); + private: + std::istream& m_stream; + }; + + inline xistream_wrapper::xistream_wrapper(std::istream& stream) + : m_stream(stream) + { + } + + inline xistream_wrapper& xistream_wrapper::read_all(std::string& s) + { + s = {std::istreambuf_iterator{m_stream}, {}}; + return *this; + } + + inline xistream_wrapper& xistream_wrapper::read(char* s, std::streamsize n) + { + m_stream.read(s, n); + return *this; + } + + inline bool xistream_wrapper::eof() + { + return m_stream.eof(); + } + + inline std::streamsize xistream_wrapper::gcount() + { + return m_stream.gcount(); + } + + + + class xostream_wrapper + { + public: + xostream_wrapper(std::ostream& stream); + xostream_wrapper& write(const char* s, std::streamsize n); + void flush(); + private: + std::ostream& m_stream; + }; + + inline xostream_wrapper::xostream_wrapper(std::ostream& stream) + : m_stream(stream) + { + } + + inline xostream_wrapper& xostream_wrapper::write(const char* s, std::streamsize n) + { + m_stream.write(s, std::streamsize(n)); + return *this; + } + + inline void xostream_wrapper::flush() + { + m_stream.flush(); + } +} + +#endif diff --git a/include/xtensor-io/xio_vsilfile.hpp b/include/xtensor-io/xio_vsilfile.hpp deleted file mode 100644 index f093042..0000000 --- a/include/xtensor-io/xio_vsilfile.hpp +++ /dev/null @@ -1,58 +0,0 @@ -#ifndef XTENSOR_IO_VSILFILE_HPP -#define XTENSOR_IO_VSILFILE_HPP - -#include - -namespace xt -{ - class xvsilfile - { - public: - xvsilfile(VSILFILE* stream); - size_t fwrite(const void* ptr, size_t size, size_t count); - size_t fread(void* ptr, size_t size, size_t count); - long int ftell(); - int fseek(long int offset, int origin); - void rewind(); - int fflush(); - private: - VSILFILE* m_stream; - }; - - inline xvsilfile::xvsilfile(VSILFILE* stream) - : m_stream(stream) - { - } - - inline size_t xvsilfile::fwrite(const void* ptr, size_t size, size_t count) - { - return VSIFWriteL(ptr, size, count, m_stream); - } - - inline size_t xvsilfile::fread(void* ptr, size_t size, size_t count) - { - return VSIFReadL(ptr, size, count, m_stream); - } - - inline long int xvsilfile::ftell() - { - return VSIFTellL(m_stream); - } - - inline int xvsilfile::fseek(long int offset, int origin) - { - return VSIFSeekL(m_stream, offset, origin); - } - - inline void xvsilfile::rewind() - { - VSIRewindL(m_stream); - } - - inline int xvsilfile::fflush() - { - return VSIFFlushL(m_stream); - } -} - -#endif diff --git a/include/xtensor-io/xio_vsilfile_wrapper.hpp b/include/xtensor-io/xio_vsilfile_wrapper.hpp new file mode 100644 index 0000000..7464405 --- /dev/null +++ b/include/xtensor-io/xio_vsilfile_wrapper.hpp @@ -0,0 +1,71 @@ +#ifndef XTENSOR_IO_VSILFILE_WRAPPER_HPP +#define XTENSOR_IO_VSILFILE_WRAPPER_HPP + +#include + +namespace xt +{ + class xvsilfile_wrapper + { + public: + xvsilfile_wrapper(VSILFILE* pfile); + std::streampos tellg(); + xvsilfile_wrapper& read_all(std::string& s); + xvsilfile_wrapper& read(char* s, std::streamsize n); + void write(const char* s, std::size_t size); + std::streamsize gcount(); + void flush(); + bool eof(); + private: + VSILFILE* m_pfile; + std::size_t m_gcount; + }; + + inline xvsilfile_wrapper::xvsilfile_wrapper(VSILFILE* pfile) + : m_pfile(pfile) + { + } + + inline std::streampos xvsilfile_wrapper::tellg() + { + return VSIFTellL(m_pfile); + } + + inline xvsilfile_wrapper& xvsilfile_wrapper::read_all(std::string& s) + { + VSIFSeekL(m_pfile, 0, SEEK_END); + std::size_t size = VSIFTellL(m_pfile); + s.resize(size); + VSIRewindL(m_pfile); + m_gcount = VSIFReadL(&s[0], 1, size, m_pfile); + return *this; + } + + inline xvsilfile_wrapper& xvsilfile_wrapper::read(char* s, std::streamsize n) + { + m_gcount = VSIFReadL(&s[0], 1, n, m_pfile); + return *this; + } + + inline std::streamsize xvsilfile_wrapper::gcount() + { + return m_gcount; + } + + inline void xvsilfile_wrapper::write(const char* s, std::size_t size) + { + VSIFWriteL(s, 1, size, m_pfile); + } + + inline void xvsilfile_wrapper::flush() + { + VSIFFlushL(m_pfile); + } + + inline bool xvsilfile_wrapper::eof() + { + return VSIFEofL(m_pfile); + } +} + +#endif diff --git a/test/test_xchunk_store_manager.cpp b/test/test_xchunk_store_manager.cpp index e10bbb6..1bb837c 100644 --- a/test/test_xchunk_store_manager.cpp +++ b/test/test_xchunk_store_manager.cpp @@ -59,18 +59,21 @@ namespace xt std::ifstream in_file; xt::xarray data; in_file.open(chunk_dir + "/1.0"); - data = xt::load_bin(in_file); + auto i1 = xt::xistream_wrapper(in_file); + data = xt::load_bin(i1); EXPECT_EQ(data(1), v1); in_file.close(); a1.chunks().flush(); in_file.open(chunk_dir + "/0.1"); - data = xt::load_bin(in_file); + auto i2 = xt::xistream_wrapper(in_file); + data = xt::load_bin(i2); EXPECT_EQ(data(2), v2); in_file.close(); in_file.open(chunk_dir + "/0.0"); - data = xt::load_bin(in_file); + auto i3 = xt::xistream_wrapper(in_file); + data = xt::load_bin(i3); EXPECT_EQ(data(0), v3); in_file.close(); } diff --git a/test/test_xfile_array.cpp b/test/test_xfile_array.cpp index d870351..23268bc 100644 --- a/test/test_xfile_array.cpp +++ b/test/test_xfile_array.cpp @@ -49,13 +49,15 @@ namespace xt std::ifstream in_file; in_file.open("a1"); - auto data = load_bin(in_file); + auto i1 = xt::xistream_wrapper(in_file); + auto data = load_bin(i1); xarray ref = {v1, v1, v1, v1}; EXPECT_TRUE(xt::all(xt::equal(data, ref))); in_file.close(); in_file.open("a2"); - data = load_bin(in_file); + auto i2 = xt::xistream_wrapper(in_file); + data = load_bin(i2); ref = {v2, v2, v2, v2}; EXPECT_TRUE(xt::all(xt::equal(data, ref))); in_file.close(); diff --git a/test/test_xio_aws_handler.cpp b/test/test_xio_aws_handler.cpp index 9603b4a..679bd88 100644 --- a/test/test_xio_aws_handler.cpp +++ b/test/test_xio_aws_handler.cpp @@ -12,7 +12,6 @@ #include #include "gtest/gtest.h" -#include "xtensor/xview.hpp" #include "xtensor-io/xfile_array.hpp" #include "xtensor-io/xio_binary.hpp" #include "xtensor-io/xio_aws_handler.hpp" @@ -33,8 +32,9 @@ namespace xt xarray a0; std::string path = "main.js"; h.read(a0, path); - xarray a1 = {'/', '*', 'j', 's', 'l', 'i', 'n', 't', ' ', 'b', 'r' ,'o', 'w', 's', 'e', 'r', ':', ' ', 't', 'r', 'u', 'e', '*', '/'}; - EXPECT_TRUE(xt::all(xt::equal(xt::view(a0, xt::range(0, 24)), a1))); + std::string ref = "/*jslint browser: true*/"; + std::string res = a0.data(); + EXPECT_EQ(ref, res.substr(0, ref.size())); Aws::ShutdownAPI(options); } @@ -47,9 +47,11 @@ namespace xt Aws::S3::S3Client client; xio_aws_config c = {client, "elevation-tiles-prod"}; xfile_array> a0("main.js", c); + xarray a1 = a0; - xarray a1 = {'/', '*', 'j', 's', 'l', 'i', 'n', 't', ' ', 'b', 'r' ,'o', 'w', 's', 'e', 'r', ':', ' ', 't', 'r', 'u', 'e', '*', '/'}; - EXPECT_TRUE(xt::all(xt::equal(xt::view(a0, xt::range(0, 24)), a1))); + std::string ref = "/*jslint browser: true*/"; + std::string res = a1.data(); + EXPECT_EQ(ref, res.substr(0, ref.size())); Aws::ShutdownAPI(options); } diff --git a/test/test_xio_binary.cpp b/test/test_xio_binary.cpp index 01f7e15..fe47b5f 100644 --- a/test/test_xio_binary.cpp +++ b/test/test_xio_binary.cpp @@ -10,6 +10,8 @@ #include "gtest/gtest.h" #include "xtensor-io/xio_binary.hpp" +#include "xtensor-io/xio_stream_wrapper.hpp" +#include "xtensor-io/xio_file_wrapper.hpp" namespace xt { @@ -21,11 +23,13 @@ namespace xt const char* fname = "data_stream.bin"; std::ofstream out_file(fname, std::ofstream::binary); - dump_file(out_file, data, xio_binary_config()); + auto o = xt::xostream_wrapper(out_file); + dump_file(o, data, xio_binary_config()); xarray a; std::ifstream in_file(fname, std::ifstream::binary); - load_file(in_file, a, xio_binary_config()); + auto i = xt::xistream_wrapper(in_file); + load_file(i, a, xio_binary_config()); a.reshape({2, 4}); ASSERT_TRUE(all(equal(a, data))); @@ -39,11 +43,13 @@ namespace xt const char* fname = "data_file.bin"; FILE* out_file = fopen(fname, "wb"); - dump_file(out_file, data, xio_binary_config()); + auto o = xt::xfile_wrapper(out_file); + dump_file(o, data, xio_binary_config()); xarray a; FILE* in_file = fopen(fname, "rb"); - load_file(in_file, a, xio_binary_config()); + auto i = xt::xfile_wrapper(in_file); + load_file(i, a, xio_binary_config()); a.reshape({2, 4}); ASSERT_TRUE(all(equal(a, data))); diff --git a/test/test_xio_gdal_handler.cpp b/test/test_xio_gdal_handler.cpp index cc6a9fd..c7f1a49 100644 --- a/test/test_xio_gdal_handler.cpp +++ b/test/test_xio_gdal_handler.cpp @@ -10,15 +10,15 @@ #include "gtest/gtest.h" #include -#include "xtensor-io/xio_vsilfile.hpp" +#include "xtensor-io/xio_vsilfile_wrapper.hpp" #include "xtensor-io/xio_binary.hpp" +#include "xtensor-io/xio_gzip.hpp" #include "xtensor-io/xio_gdal_handler.hpp" namespace xt { - TEST(xio_gdal_handler, write_read) + TEST(xio_gdal_handler, write_read_vsimem) { - VSIInstallMemFileHandler(); xio_gdal_handler h; xarray a0 = {3, 2, 1, 4}; h.write(a0, "/vsimem/test_file.tif", xfile_dirty(true)); @@ -26,4 +26,23 @@ namespace xt h.read(a1, "/vsimem/test_file.tif"); EXPECT_TRUE(xt::all(xt::equal(a0, a1))); } + + TEST(xio_gdal_handler, read_vsigs) + { + xio_gdal_handler h; + xarray a0; + h.read(a0, "/vsigs/zarr-demo/v3/test.zr3/data/root/arthur/dent/c0/0"); + xarray a1 = {0, 1, 2, 3, 4, 10, 11, 12, 13, 14}; + EXPECT_TRUE(xt::all(xt::equal(a0, a1))); + } + + TEST(xio_gdal_handler, read_vsicurl) + { + xio_gdal_handler h; + xarray a0; + h.read(a0, "/vsicurl/https://raw.githubusercontent.com/xtensor-stack/xtensor-io/master/LICENSE"); + std::string ref = "Copyright"; + std::string res = a0.data(); + EXPECT_EQ(ref, res.substr(0, ref.size())); + } }