Implement the compressor inside OpenEXRCore (#5)

* Whitespaces and licensing

* WIP OpenEXRCore implementation

* Brand new spanking blosc build.

* Switch to single Scanline zstd compression and Single implementation

* Fixed the tests

* Undo whitespace changes

* Last touches

* Revert extra build changes

src/lib/OpenEXR/CMakeLists.txt

@@ -217,7 +217,6 @@ openexr_define_library(OpenEXR
     ImfXdr.h
   DEPENDENCIES
     Imath::Imath
-    Blosc2::blosc2_static
     OpenEXR::Config
     OpenEXR::Iex
     OpenEXR::IlmThread

src/lib/OpenEXR/ImfCompressor.cpp

@@ -19,6 +19,7 @@
 #include "ImfRleCompressor.h"
 #include "ImfZipCompressor.h"
 #include "ImfZstdCompressor.h"
+#include "openexr_compression.h"
 
 OPENEXR_IMF_INTERNAL_NAMESPACE_SOURCE_ENTER
 
@@ -145,7 +146,7 @@ newCompressor (Compression c, size_t maxScanLineSize, const Header& hdr)
                 DwaCompressor::STATIC_HUFFMAN);
 
         case ZSTD_COMPRESSION:
-            return new ZstdCompressor (hdr, maxScanLineSize, 32);
+            return new ZstdCompressor (hdr);
         default: return 0;
     }
 }
@@ -167,7 +168,7 @@ numLinesInBuffer (Compression comp)
         case B44_COMPRESSION:
         case B44A_COMPRESSION:
         case DWAA_COMPRESSION: return 32;
-        case ZSTD_COMPRESSION: return 32;
+        case ZSTD_COMPRESSION: return (int)exr_get_zstd_lines_per_chunk();
         case DWAB_COMPRESSION: return 256;
 
         default: throw IEX_NAMESPACE::ArgExc ("Unknown compression type");
@@ -190,7 +191,7 @@ newTileCompressor (
             return new ZipCompressor (hdr, tileLineSize, numTileLines);
         case ZSTD_COMPRESSION:
 
-            return new ZstdCompressor (hdr, tileLineSize, numTileLines);
+            return new ZstdCompressor (hdr);
 
         case PIZ_COMPRESSION:
 

src/lib/OpenEXR/ImfMultiPartInputFile.cpp

@@ -22,6 +22,7 @@
 #include "ImfTiledMisc.h"
 #include "ImfTimeCodeAttribute.h"
 #include "ImfVersion.h"
+#include "openexr_compression.h"
 
 #include <OpenEXRConfig.h>
 
@@ -547,7 +548,7 @@ MultiPartInputFile::Data::chunkOffsetReconstruction (
             // (TODO) fix this so that it doesn't need to be revised for future compression types.
             switch (parts[i]->header.compression ())
             {
-                case ZSTD_COMPRESSION: rowsizes[i] = 32; break;
+                case ZSTD_COMPRESSION: rowsizes[i] = (int)exr_get_zstd_lines_per_chunk(); break;
                 case DWAB_COMPRESSION: rowsizes[i] = 256; break;
                 case PIZ_COMPRESSION:
                 case B44_COMPRESSION:

src/lib/OpenEXR/ImfZstdCompressor.cpp

@@ -4,143 +4,58 @@
 //
 
 #include <cstring>
+#include <mutex>
+#include "openexr_compression.h"
 #include "ImfZstdCompressor.h"
 
-#include "blosc2.h"
 #include "IlmThreadPool.h"
 #include "ImfChannelList.h"
 #include "ImfMisc.h"
+
 namespace
 {
-class BloscInit
-{
-public:
-    static void Init () { getInstance (); }
-    BloscInit (const BloscInit&)            = delete;
-    BloscInit& operator= (const BloscInit&) = delete;
-
-private:
-    BloscInit () { blosc2_init (); }
-    ~BloscInit () { blosc2_destroy (); }
-    static BloscInit& getInstance ()
-    {
-        static BloscInit instance;
-        return instance;
-    }
-};
-} // namespace
-
+std::mutex g_mutex;
+}
+
 OPENEXR_IMF_INTERNAL_NAMESPACE_SOURCE_ENTER
-ZstdCompressor::ZstdCompressor (
-    const Header& hdr, size_t maxScanlineSize, size_t numScanLines)
-    : Compressor (hdr)
-    , _maxScanlineSize (maxScanlineSize)
-    , _numScanLines (numScanLines)
-    , _outBuffer (nullptr, &free)
-    , _schunk (nullptr, &blosc2_schunk_free)
+
+ZstdCompressor::ZstdCompressor (const Header& hdr)
+    : Compressor (hdr), _outBuffer ()
 {}
 
 int
 ZstdCompressor::numScanLines () const
 {
-    return _numScanLines; // Needs to be in sync with ImfCompressor::numLinesInBuffer
+    return (int)exr_get_zstd_lines_per_chunk(); // Needs to be in sync with ImfCompressor::numLinesInBuffer
 }
 
 int
 ZstdCompressor::compress (
     const char* inPtr, int inSize, int minY, const char*& outPtr)
 {
-    int typeSize = std::numeric_limits<int>::min ();
-    for (auto it = header ().channels ().begin ();
-         it != header ().channels ().end ();
-         ++it)
+    outPtr = (char*) malloc (inSize);
     {
-        // BLOSC prefilter is affected by the typesize. Initializing to max will ensure that a channel is not split in 2 and filtered separately.
-        // probably compression can be improved for non-deep images by compressing every channel separately with the correct typeSize
-        // (much harder to do for Deep-Data).
-        typeSize = std::max (typeSize, Imf::pixelTypeSize (it.channel ().type));
+        std::lock_guard<std::mutex> lock (g_mutex);
+        _outBuffer.push_back (raw_ptr ((char*) outPtr, &free));
     }
-
-    auto ret      = BLOSC_compress_impl (inPtr, inSize, typeSize, outPtr);
-    auto fullSize = Xdr::size<int> () + ret;
-    auto data     = malloc (fullSize);
-    auto write    = (char*) data;
-
-    Xdr::write<CharPtrIO> (write, Versions::LATEST);
-
-    memcpy (write, outPtr, ret);
-    outPtr = (char*) data;
-
-    _outBuffer = raw_ptr ((char*) data, &free);
-
+    auto fullSize =
+        exr_compress_zstd ((char*) (inPtr), inSize, (void*) outPtr, inSize);
     return fullSize;
 }
 
 int
 ZstdCompressor::uncompress (
     const char* inPtr, int inSize, int minY, const char*& outPtr)
 {
-    auto read = (const char*) inPtr;
-    int  v;
-    Xdr::read<CharPtrIO> (read, v);
-    if (v == Versions::SINGLE_BLOB)
+    auto  read  = (const char*) inPtr;
+    void* write = nullptr;
+    auto  ret   = exr_uncompress_zstd (read, inSize, &write, 0);
     {
-        return BLOSC_uncompress_impl_single_blob (
-            read, inSize - Xdr::size<int> (), outPtr);
+        std::lock_guard<std::mutex> lock (g_mutex);
+        _outBuffer.push_back (raw_ptr ((char*) write, &free));
     }
-    else { throw Iex::InputExc ("Unsupported ZstdCompressor version"); }
-}
-
-int
-ZstdCompressor::BLOSC_compress_impl (
-    const char* inPtr, int inSize, int typeSize, const char*& out)
-{
-    BloscInit::Init ();
-    blosc2_cparams cparams = BLOSC2_CPARAMS_DEFAULTS;
-
-    cparams.typesize = typeSize;
-    // clevel 9 is about a 20% increase in compression compared to 5.
-    // Decompression speed is unchanged.
-    cparams.clevel   = header ().zstdCompressionLevel ();
-    cparams.nthreads = 1;
-    cparams.compcode = BLOSC_ZSTD; // Codec
-    cparams.splitmode =
-        BLOSC_NEVER_SPLIT; // Split => multithreading, not split better compression
-
-    blosc2_storage storage = BLOSC2_STORAGE_DEFAULTS;
-    storage.cparams        = &cparams;
-    storage.contiguous     = true;
-
-    _schunk = schunk_ptr (blosc2_schunk_new (&storage), &blosc2_schunk_free);
-
-    auto in = const_cast<char*> (inPtr);
-    blosc2_schunk_append_buffer (_schunk.get (), in, inSize);
-
-    uint8_t* buffer;
-    bool     shouldFree = true;
-    auto size = blosc2_schunk_to_buffer (_schunk.get (), &buffer, &shouldFree);
-    out       = (char*) buffer;
-    if (shouldFree) { _outBuffer = raw_ptr ((char*) buffer, &free); }
-    return size;
-}
-
-int
-ZstdCompressor::BLOSC_uncompress_impl_single_blob (
-    const char* inPtr, int inSize, const char*& out)
-{
-    auto in = const_cast<char*> (inPtr);
-    _schunk = schunk_ptr (
-        blosc2_schunk_from_buffer (
-            reinterpret_cast<uint8_t*> (in), inSize, true),
-        &blosc2_schunk_free);
-
-    auto buffSize = _maxScanlineSize * numScanLines ();
-    _outBuffer =
-        Imf::ZstdCompressor::raw_ptr ((char*) malloc (buffSize), &free);
-    auto size = blosc2_schunk_decompress_chunk (
-        _schunk.get (), 0, _outBuffer.get (), buffSize);
-    out = _outBuffer.get ();
-    return size;
+    outPtr = (const char*) write;
+    return ret;
 }
 
 OPENEXR_IMF_INTERNAL_NAMESPACE_SOURCE_EXIT

src/lib/OpenEXR/ImfZstdCompressor.h

@@ -1,45 +1,30 @@
-#pragma once
-
 //
 // SPDX-License-Identifier: BSD-3-Clause
 // Copyright (c) Contributors to the OpenEXR Project.
 //
 
+#pragma once
+
 #include <memory>
 #include "ImfNamespace.h"
 #include "ImfCompressor.h"
 #include "ImfHeader.h"
 #include "blosc2.h"
+#include "vector"
 
 OPENEXR_IMF_INTERNAL_NAMESPACE_HEADER_ENTER
 class ZstdCompressor : public Compressor
 {
 public:
-    ZstdCompressor (
-        const Header& hdr, size_t maxScanLines, size_t numScanLines);
-
+    explicit ZstdCompressor (const Header& hdr);
 private:
-    using schunk_ptr =
-        std::unique_ptr<blosc2_schunk, decltype (&blosc2_schunk_free)>;
     using raw_ptr = std::unique_ptr<char, decltype (&free)>;
-    raw_ptr    _outBuffer;
-    schunk_ptr _schunk;
-    size_t     _maxScanlineSize;
-    size_t     _numScanLines;
+    std::vector<raw_ptr>    _outBuffer;
     int        numScanLines () const override; // max
     int        compress (
                const char* inPtr, int inSize, int minY, const char*& outPtr) override;
     int uncompress (
         const char* inPtr, int inSize, int minY, const char*& outPtr) override;
-    int BLOSC_compress_impl (
-        const char* inPtr, int inSize, int typeSize, const char*& out);
-    int BLOSC_uncompress_impl_single_blob (
-        const char* inPtr, int inSize, const char*& out);
-    enum Versions : int
-    {
-        SINGLE_BLOB = 1,
-        LATEST      = SINGLE_BLOB
-    };
 };
 
 OPENEXR_IMF_INTERNAL_NAMESPACE_HEADER_EXIT

src/lib/OpenEXRCore/CMakeLists.txt

@@ -45,6 +45,7 @@ openexr_define_library(OpenEXRCore
     internal_piz.c
     internal_dwa.c
     internal_huf.c
+    internal_zstd.c
 
     attributes.c
     string.c
@@ -102,6 +103,7 @@ openexr_define_library(OpenEXRCore
 
   DEPENDENCIES
     Imath::Imath
+    Blosc2::blosc2_static
   )
 
 if (DEFINED EXR_DEFLATE_LIB)
@@ -110,4 +112,4 @@ if (DEFINED EXR_DEFLATE_LIB)
   else()
     target_link_libraries(OpenEXRCore PUBLIC ${EXR_DEFLATE_LIB})
   endif()
-endif()
+endif()

src/lib/OpenEXRCore/decoding.c

@@ -263,6 +263,10 @@ decompress_data (
             rv = internal_exr_undo_dwab (
                 decode, packbufptr, packsz, unpackbufptr, unpacksz);
             break;
+        case EXR_COMPRESSION_ZSTD:
+            rv = internal_exr_undo_zstd (
+                decode, packbufptr, packsz, unpackbufptr, unpacksz);
+            break;
         case EXR_COMPRESSION_LAST_TYPE:
         default:
             return pctxt->print_error (

src/lib/OpenEXRCore/encoding.c

@@ -54,6 +54,7 @@ default_compress_chunk (exr_encode_pipeline_t* encode)
         case EXR_COMPRESSION_B44A: rv = internal_exr_apply_b44a (encode); break;
         case EXR_COMPRESSION_DWAA: rv = internal_exr_apply_dwaa (encode); break;
         case EXR_COMPRESSION_DWAB: rv = internal_exr_apply_dwab (encode); break;
+        case EXR_COMPRESSION_ZSTD: rv = internal_exr_apply_zstd (encode); break;
         case EXR_COMPRESSION_LAST_TYPE:
         default:
             return pctxt->print_error (

src/lib/OpenEXRCore/internal_compress.h

@@ -33,4 +33,5 @@ exr_result_t internal_exr_apply_dwaa (exr_encode_pipeline_t* encode);
 
 exr_result_t internal_exr_apply_dwab (exr_encode_pipeline_t* encode);
 
+exr_result_t internal_exr_apply_zstd (exr_encode_pipeline_t* encode);
 #endif /* OPENEXR_CORE_COMPRESS_H */

src/lib/OpenEXRCore/internal_decompress.h

@@ -73,4 +73,11 @@ exr_result_t internal_exr_undo_dwab (
     void*                  uncompressed_data,
     uint64_t               uncompressed_size);
 
+exr_result_t internal_exr_undo_zstd (
+    exr_decode_pipeline_t* decode,
+    const void*            compressed_data,
+    uint64_t               comp_buf_size,
+    void*                  uncompressed_data,
+    uint64_t               uncompressed_size);
+
 #endif /* OPENEXR_CORE_DECOMPRESS_H */