Merge pull request #527 from lukemartinlogan/master

Hermes 1.0.0

.github/workflows/main.yml

@@ -74,34 +74,34 @@ jobs:
       - name: Build
         if: steps.hermes-cache.outputs.cache-hit != 'true'
         run: ci/build_hermes.sh
-
-      - name: Test
-        run: cd build && ctest -VV
-
+#
+#      - name: Test
+#        run: cd build && ctest -VV -R test_hermes_posix_basic_small
+#
       - name: Install
         run: pushd build && make install && popd
 
       # Enable tmate debugging of manually-triggered workflows if the input option was provided
-      - name: Setup tmate session
-        uses: mxschmitt/action-tmate@v3
-        if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.debug_enabled && (failure() || !failure()) }}
+#      - name: Setup tmate session
+#        uses: mxschmitt/action-tmate@v3
+#        if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.debug_enabled && (failure() || !failure()) }}
 
 #      - name: Multi-node Test
 #        run: pushd ci/cluster && ./multi_node_ci_test.sh
 
-      - name: Generate coverage file
-        run: |
-          COVERAGE_DIR=${GITHUB_WORKSPACE}/coverage
-          mkdir -p ${COVERAGE_DIR}
-          pushd ${GITHUB_WORKSPACE}/build
-          lcov -c -d . -o "${COVERAGE_DIR}/tmp.info"
-          lcov --remove "${COVERAGE_DIR}/tmp.info"   \
-                        "/usr/include/*"             \
-                        "${HOME}/${LOCAL}/*"         \
-                        "*/stb_ds.h"                 \
-                        -o ${COVERAGE_DIR}/lcov.info
-
-      - name: Coveralls
-        uses: coverallsapp/github-action@master
-        with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
+#      - name: Generate coverage file
+#        run: |
+#          COVERAGE_DIR=${GITHUB_WORKSPACE}/coverage
+#          mkdir -p ${COVERAGE_DIR}
+#          pushd ${GITHUB_WORKSPACE}/build
+#          lcov -c -d . -o "${COVERAGE_DIR}/tmp.info"
+#          lcov --remove "${COVERAGE_DIR}/tmp.info"   \
+#                        "/usr/include/*"             \
+#                        "${HOME}/${LOCAL}/*"         \
+#                        "*/stb_ds.h"                 \
+#                        -o ${COVERAGE_DIR}/lcov.info
+
+#      - name: Coveralls
+#        uses: coverallsapp/github-action@master
+#        with:
+#          github-token: ${{ secrets.GITHUB_TOKEN }}

CMakeLists.txt

@@ -27,7 +27,7 @@ if (CMAKE_BUILD_TYPE STREQUAL "Debug")
   add_compile_definitions(HERMES_LOG_VERBOSITY=10)
   message("This is NOT a release build: ${CMAKE_BUILD_TYPE} ${CMAKE_CXX_FLAGS}")
 elseif(CMAKE_BUILD_TYPE STREQUAL "RelWithDebInfo")
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -fPIC")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O2 -fPIC")
   # Ensure that debug logging is enabled
   # This will keep logs which have beneath priority 10
   add_compile_definitions(HERMES_LOG_VERBOSITY=10)
@@ -323,6 +323,12 @@ if(yaml-cpp_FOUND)
   message(STATUS "found yaml-cpp at ${yaml-cpp_DIR}")
 endif()
 
+#Cereal
+find_package(cereal REQUIRED)
+if(cereal)
+  message(STATUS "found cereal")
+endif()
+
 # GOTCHA
 if(HERMES_INTERCEPT_IO)
   find_package(gotcha REQUIRED)
@@ -362,7 +368,7 @@ endif()
 
 # HDF5
 if(HERMES_ENABLE_VFD)
-  set(HERMES_REQUIRED_HDF5_VERSION 1.13.0)
+  set(HERMES_REQUIRED_HDF5_VERSION 1.14.0)
   set(HERMES_REQUIRED_HDF5_COMPONENTS C)
   find_package(HDF5 ${HERMES_REQUIRED_HDF5_VERSION} CONFIG NAMES hdf5
     COMPONENTS ${HERMES_REQUIRED_HDF5_COMPONENTS} shared)

README.md

@@ -1,3 +1,9 @@
+# Fork Branch Description
+This branch is for developing a customized Hermes VFD adaptor.
+## Dependencies
+*   HDF5 1.14.0
+
+---
 # Hermes
 
 Hermes is a heterogeneous-aware, multi-tiered, dynamic, and distributed I/O buffering system that aims to significantly accelerate I/O performance. See the [official site](http://www.cs.iit.edu/~scs/assets/projects/Hermes/Hermes.html) for more information. For design documents, architecture description, performance data, and individual component design, see the [wiki](https://github.com/HDFGroup/hermes/wiki).

adapter/filesystem/filesystem.cc

@@ -62,8 +62,8 @@ void Filesystem::Open(AdapterStat &stat, File &f, const std::string &path) {
       stat.bkt_id_.Clear();
     } else {
       // The file was opened regularly
-      size_t file_size = io_client_->GetSize(*path_shm);
-      stat.bkt_id_ = HERMES->GetBucket(stat.path_, ctx, file_size);
+      stat.file_size_ = io_client_->GetSize(*path_shm);
+      stat.bkt_id_ = HERMES->GetBucket(stat.path_, ctx, stat.file_size_);
     }
     HILOG(kDebug, "File has size: {}", stat.bkt_id_.GetSize());
     // Attach trait to bucket (if not scratch mode)
@@ -116,6 +116,7 @@ static inline size_t GetBackendSize(size_t file_off,
  * @param blob_name the semantic name of the blob
  * @param blob the buffer to put final data in
  * @param blob_off the offset within the blob to begin the Put
+ * @param page_size the page size of the adapter
  * @param io_ctx which adapter to route I/O request if blob DNE
  * @param opts which adapter to route I/O request if blob DNE
  * @param ctx any additional information
@@ -124,11 +125,12 @@ Status Filesystem::PartialPutOrCreate(hapi::Bucket &bkt,
                                       const std::string &blob_name,
                                       const Blob &blob,
                                       size_t blob_off,
+                                      size_t page_size,
                                       BlobId &blob_id,
                                       IoStatus &status,
                                       const FsIoOptions &opts,
                                       Context &ctx) {
-  Blob full_blob;
+  Blob full_blob(page_size);
   if (bkt.ContainsBlob(blob_name, blob_id)) {
     // Case 1: The blob already exists (read from hermes)
     // Read blob from Hermes
@@ -154,10 +156,12 @@ Status Filesystem::PartialPutOrCreate(hapi::Bucket &bkt,
     io_client_->ReadBlob(bkt.GetName(),
                         full_blob, opts, status);
     if (!status.success_) {
-      HILOG(kDebug, "Failed to read blob from backend (PartialPut)."
+      HELOG(kFatal, "Failed to read blob from {} (PartialPut)."
             " cur_size: {}"
+            " backend_off: {}"
             " backend_size: {}",
-            full_blob.size(), opts.backend_size_)
+            bkt.GetName(), full_blob.size(),
+            opts.backend_off_, opts.backend_size_)
       // return PARTIAL_PUT_OR_CREATE_OVERFLOW;
     }
   }
@@ -233,7 +237,7 @@ size_t Filesystem::Write(File &f, AdapterStat &stat, const void *ptr,
     BlobId blob_id;
     opts.backend_off_ = p.page_ * kPageSize;
     opts.backend_size_ = GetBackendSize(opts.backend_off_,
-                                        backend_size,
+                                        stat.file_size_,
                                         kPageSize);
     opts.adapter_mode_ = stat.adapter_mode_;
     bkt.TryCreateBlob(blob_name.str(), blob_id, ctx);
@@ -242,6 +246,7 @@ size_t Filesystem::Write(File &f, AdapterStat &stat, const void *ptr,
                                      blob_name.str(),
                                      blob_wrap,
                                      p.blob_off_,
+                                     kPageSize,
                                      blob_id,
                                      io_status,
                                      opts,
@@ -273,6 +278,7 @@ size_t Filesystem::Write(File &f, AdapterStat &stat, const void *ptr,
  * @param blob the buffer to put final data in
  * @param blob_off the offset within the blob to begin the Put
  * @param blob_size the total amount of data to read
+ * @param page_size the page size of the adapter
  * @param blob_id [out] the blob id corresponding to blob_name
  * @param io_ctx information required to perform I/O to the backend
  * @param opts specific configuration of the I/O to perform
@@ -283,11 +289,12 @@ Status Filesystem::PartialGetOrCreate(hapi::Bucket &bkt,
                                       Blob &blob,
                                       size_t blob_off,
                                       size_t blob_size,
+                                      size_t page_size,
                                       BlobId &blob_id,
                                       IoStatus &status,
                                       const FsIoOptions &opts,
                                       Context &ctx) {
-  Blob full_blob;
+  Blob full_blob(page_size);
   if (bkt.ContainsBlob(blob_name, blob_id)) {
     // Case 1: The blob already exists (read from hermes)
     // Read blob from Hermes
@@ -377,15 +384,14 @@ size_t Filesystem::Read(File &f, AdapterStat &stat, void *ptr,
   size_t data_offset = 0;
   auto mapper = MapperFactory().Get(MapperType::kBalancedMapper);
   mapper->map(off, total_size, kPageSize, mapping);
-  size_t backend_size = stat.bkt_id_.GetSize();
 
   for (const auto &p : mapping) {
     Blob blob_wrap((const char*)ptr + data_offset, p.blob_size_);
     hshm::charbuf blob_name(p.CreateBlobName());
     BlobId blob_id;
     opts.backend_off_ = p.page_ * kPageSize;
     opts.backend_size_ = GetBackendSize(opts.backend_off_,
-                                        backend_size,
+                                        stat.file_size_,
                                         kPageSize);
     opts.adapter_mode_ = stat.adapter_mode_;
     bkt.TryCreateBlob(blob_name.str(), blob_id, ctx);
@@ -395,6 +401,7 @@ size_t Filesystem::Read(File &f, AdapterStat &stat, void *ptr,
                                      blob_wrap,
                                      p.blob_off_,
                                      p.blob_size_,
+                                     kPageSize,
                                      blob_id,
                                      io_status,
                                      opts,

adapter/filesystem/filesystem.h

@@ -69,6 +69,7 @@ class Filesystem {
    * @param blob_name the semantic name of the blob
    * @param blob the buffer to put final data in
    * @param blob_off the offset within the blob to begin the Put
+   * @param page_size the page size of the adapter
    * @param blob_id [out] the blob id corresponding to blob_name
    * @param io_ctx information required to perform I/O to the backend
    * @param opts specific configuration of the I/O to perform
@@ -78,6 +79,7 @@ class Filesystem {
                             const std::string &blob_name,
                             const Blob &blob,
                             size_t blob_off,
+                            size_t page_size,
                             BlobId &blob_id,
                             IoStatus &status,
                             const FsIoOptions &opts,
@@ -96,6 +98,7 @@ class Filesystem {
    * @param blob_name the semantic name of the blob
    * @param blob the buffer to put final data in
    * @param blob_off the offset within the blob to begin the Put
+   * @param page_size the page size of the adapter
    * @param blob_id [out] the blob id corresponding to blob_name
    * @param io_ctx information required to perform I/O to the backend
    * @param opts specific configuration of the I/O to perform
@@ -106,6 +109,7 @@ class Filesystem {
                             Blob &blob,
                             size_t blob_off,
                             size_t blob_size,
+                            size_t page_size,
                             BlobId &blob_id,
                             IoStatus &status,
                             const FsIoOptions &opts,

adapter/filesystem/filesystem_io_client.h

@@ -192,6 +192,7 @@ struct AdapterStat {
   uid_t st_uid_;         /**< user ID of owner */
   gid_t st_gid_;         /**< group ID of owner */
   size_t st_ptr_;        /**< current ptr of FILE */
+  size_t file_size_;     /**< Size of file at backend at time of open */
   timespec st_atim_;     /**< time of last access */
   timespec st_mtim_;     /**< time of last modification */
   timespec st_ctim_;     /**< time of last status change */
@@ -217,6 +218,7 @@ struct AdapterStat {
         hflags_(),
         st_mode_(),
         st_ptr_(0),
+        file_size_(0),
         st_atim_(),
         st_mtim_(),
         st_ctim_(),