Zstd compressor dummy

.gitignore

@@ -57,3 +57,7 @@ docs/_test_images/
 
 # Ignore Bazel generated files
 bazel-* 
+
+cmake-build-debug/
+
+.idea/

src/bin/exrheader/main.cpp

@@ -65,6 +65,8 @@ printCompression (Compression c)
 
         case DWAB_COMPRESSION: cout << "dwa, medium scanline blocks"; break;
 
+        case ZSTD_COMPRESSION: cout << "zstd, single scanline"; break;
+
         default: cout << int (c); break;
     }
 }

src/examples/deepExamples.cpp

@@ -24,6 +24,7 @@
 #include "drawImage.h"
 
 #include "namespaceAlias.h"
+
 using namespace IMF;
 using namespace std;
 using namespace IMATH_NAMESPACE;
@@ -147,7 +148,8 @@ writeDeepScanlineFile (
 
     Array2D<half*>& dataA,
 
-    Array2D<unsigned int>& sampleCount)
+    Array2D<unsigned int>& sampleCount,
+    Compression compression = Compression::ZIPS_COMPRESSION )
 
 {
     //
@@ -169,7 +171,7 @@ writeDeepScanlineFile (
     header.channels ().insert ("Z", Channel (FLOAT));
     header.channels ().insert ("A", Channel (HALF));
     header.setType (DEEPSCANLINE);
-    header.compression () = ZIPS_COMPRESSION;
+    header.compression () = compression;
 
     DeepScanLineOutputFile file (filename, header);
 
@@ -250,7 +252,33 @@ void deepExamples()
     testDataA.resizeErase(h, w);
     testDataZ.resizeErase(h, w);
     drawImage2(testDataA, testDataZ, w, h);
-
-    writeDeepScanlineFile("test.deep.exr", window, window, dataZ, dataA, sampleCount);
-    readDeepScanlineFile ("test.deep.exr", window, window, dataZ, dataA, sampleCount);
+
+    {
+        writeDeepScanlineFile (
+            "test.deep.exr",
+            window,
+            window,
+            dataZ,
+            dataA,
+            sampleCount,
+            Compression::ZSTD_COMPRESSION);
+    }
+    {
+        writeDeepScanlineFile (
+            "test.zips.exr",
+            window,
+            window,
+            dataZ,
+            dataA,
+            sampleCount,
+            Compression::ZIPS_COMPRESSION);
+    }
+    {
+        readDeepScanlineFile (
+            "test.deep.exr", window, window, dataZ, dataA, sampleCount);
+    }
+    {
+        readDeepScanlineFile (
+            "test.zips.exr", window, window, dataZ, dataA, sampleCount);
+    }
 }

src/lib/OpenEXR/CMakeLists.txt

@@ -29,6 +29,7 @@ openexr_define_library(OpenEXR
     ImfTiledMisc.h
     ImfZip.h
     ImfZipCompressor.h
+    ImfZstdCompressor.h
     b44ExpLogTable.h
     dwaLookups.h
     ImfAcesFile.cpp
@@ -122,6 +123,7 @@ openexr_define_library(OpenEXR
     ImfWav.cpp
     ImfZip.cpp
     ImfZipCompressor.cpp
+    ImfZstdCompressor.cpp
   HEADERS
     ImfAcesFile.h
     ImfArray.h
@@ -220,3 +222,7 @@ openexr_define_library(OpenEXR
     OpenEXR::IlmThread
     OpenEXR::OpenEXRCore
   )
+
+target_include_directories(OpenEXR PUBLIC "/home/vladal/bin/include/")
+target_link_directories(OpenEXR PUBLIC "/home/vladal/bin/lib")
+target_link_libraries(OpenEXR PUBLIC "dl" "blosc2")

src/lib/OpenEXR/ImfCompression.h

@@ -45,6 +45,8 @@ enum IMF_EXPORT_ENUM Compression
                           // wise and faster to decode full frames
                           // than DWAA_COMPRESSION.
 
+    ZSTD_COMPRESSION = 10,
+
     NUM_COMPRESSION_METHODS // number of different compression methods
 };
 

src/lib/OpenEXR/ImfCompressionAttribute.cpp

@@ -57,7 +57,7 @@ CompressionAttribute::readValueFrom (
         tmp != ZIPS_COMPRESSION && tmp != ZIP_COMPRESSION &&
         tmp != PIZ_COMPRESSION && tmp != PXR24_COMPRESSION &&
         tmp != B44_COMPRESSION && tmp != B44A_COMPRESSION &&
-        tmp != DWAA_COMPRESSION && tmp != DWAB_COMPRESSION)
+        tmp != DWAA_COMPRESSION && tmp != DWAB_COMPRESSION && tmp != ZSTD_COMPRESSION)
     {
         tmp = NUM_COMPRESSION_METHODS;
     }

src/lib/OpenEXR/ImfCompressor.cpp

@@ -18,6 +18,7 @@
 #include "ImfPxr24Compressor.h"
 #include "ImfRleCompressor.h"
 #include "ImfZipCompressor.h"
+#include "ImfZstdCompressor.h"
 
 OPENEXR_IMF_INTERNAL_NAMESPACE_SOURCE_ENTER
 
@@ -63,6 +64,7 @@ isValidCompression (Compression c)
         case B44_COMPRESSION:
         case B44A_COMPRESSION:
         case DWAA_COMPRESSION:
+        case ZSTD_COMPRESSION:
         case DWAB_COMPRESSION: return true;
 
         default: return false;
@@ -89,7 +91,8 @@ isValidDeepCompression (Compression c)
     {
         case NO_COMPRESSION:
         case RLE_COMPRESSION:
-        case ZIPS_COMPRESSION: return true;
+        case ZIPS_COMPRESSION:
+        case ZSTD_COMPRESSION: return true;
         default: return false;
     }
 }
@@ -140,7 +143,8 @@ newCompressor (Compression c, size_t maxScanLineSize, const Header& hdr)
                 static_cast<int> (maxScanLineSize),
                 256,
                 DwaCompressor::STATIC_HUFFMAN);
-
+        case ZSTD_COMPRESSION:
+            return new ZstdCompressor (hdr, maxScanLineSize);
         default: return 0;
     }
 }
@@ -162,6 +166,7 @@ numLinesInBuffer (Compression comp)
         case B44_COMPRESSION:
         case B44A_COMPRESSION:
         case DWAA_COMPRESSION: return 32;
+        case ZSTD_COMPRESSION: return 1;// FIXME
         case DWAB_COMPRESSION: return 256;
 
         default: throw IEX_NAMESPACE::ArgExc ("Unknown compression type");

src/lib/OpenEXR/ImfHeader.cpp

@@ -70,9 +70,11 @@ struct CompressionRecord
     {
         exr_get_default_zip_compression_level(&zip_level);
         exr_get_default_dwa_compression_quality(&dwa_level);
+        exr_get_default_zstd_compression_level(&zstd_level);
     }
     int   zip_level;
     float dwa_level;
+    int zstd_level;
 };
 // NB: This is extra complicated than one would normally write to
 // handle scenario that seems to happen on MacOS/Windows (probably
@@ -696,6 +698,18 @@ Header::zipCompressionLevel () const
     return retrieveCompressionRecord (this).zip_level;
 }
 
+int&
+Header::zstdCompressionLevel ()
+{
+    return retrieveCompressionRecord (this).zstd_level;
+}
+
+int
+Header::zstdCompressionLevel () const
+{
+    return retrieveCompressionRecord (this).zstd_level;
+}
+
 float&
 Header::dwaCompressionLevel ()
 {

src/lib/OpenEXR/ImfHeader.h

@@ -285,6 +285,11 @@ class IMF_EXPORT_TYPE Header
     float& dwaCompressionLevel ();
     IMF_EXPORT
     float dwaCompressionLevel () const;
+    IMF_EXPORT
+    int& zstdCompressionLevel ();
+    IMF_EXPORT
+    int zstdCompressionLevel () const;
+    IMF_EXPORT
 
     //-----------------------------------------------------
     // Access to required attributes for multipart files

src/lib/OpenEXR/ImfMultiPartInputFile.cpp

@@ -547,6 +547,7 @@ MultiPartInputFile::Data::chunkOffsetReconstruction (
             // (TODO) fix this so that it doesn't need to be revised for future compression types.
             switch (parts[i]->header.compression ())
             {
+                case ZSTD_COMPRESSION: rowsizes[i] = 1; break;
                 case DWAB_COMPRESSION: rowsizes[i] = 256; break;
                 case PIZ_COMPRESSION:
                 case B44_COMPRESSION:

src/lib/OpenEXR/ImfZstdCompressor.cpp

@@ -0,0 +1,123 @@
+//
+// Created by vladal on 27/10/23.
+//
+
+#include <cstring>
+#include "ImfZstdCompressor.h"
+
+//#include <zstd.h>
+#include "blosc2.h"
+#include "IlmThreadPool.h"
+#include "ImfChannelList.h"
+#include "ImfMisc.h"
+namespace
+{
+/*
+int
+ZSTD_compress_impl (const char* inPtr, int inSize, Imf::ZstdCompressor::raw_ptr& outPtr)
+{
+    auto const cBuffSize = ZSTD_compressBound(inSize);
+    outPtr = Imf::ZstdCompressor::raw_ptr((char*)malloc (cBuffSize), &free);
+    auto const cSize = ZSTD_compress(outPtr.get(), cBuffSize, inPtr, inSize, 15);
+    return cSize;
+}
+
+int
+ZSTD_uncompress_impl (const char* inPtr, int inSize, Imf::ZstdCompressor::raw_ptr& outPtr)
+{
+    auto rSize = ZSTD_getFrameContentSize(inPtr, inSize);
+    outPtr = Imf::ZstdCompressor::raw_ptr((char*)malloc (rSize), &free);
+    auto const dSize = ZSTD_decompress(outPtr.get(), rSize, inPtr, inSize);
+    return dSize;
+}
+*/
+class BloscInit
+{
+public:
+    static void Init () { getInstance (); }
+    BloscInit (const BloscInit&) = delete;
+    BloscInit& operator= (const BloscInit&) = delete;
+private:
+    BloscInit () { blosc2_init (); }
+    ~BloscInit () { blosc2_destroy (); }
+    static BloscInit& getInstance ()
+    {
+        static BloscInit instance;
+        return instance;
+    }
+};
+}
+
+OPENEXR_IMF_INTERNAL_NAMESPACE_SOURCE_ENTER
+ZstdCompressor::ZstdCompressor ( const Header& hdr, size_t maxScanlineSize): Compressor(hdr),
+    _maxScanlineSize (maxScanlineSize),_outBuffer (nullptr, &free), _schunk (nullptr,&blosc2_schunk_free)
+{}
+
+int
+ZstdCompressor::numScanLines () const
+{
+    return 1; // max ? // Needs to be in sync with ImfCompressor::numLinesInBuffer
+}
+int
+ZstdCompressor::compress (
+    const char* inPtr, int inSize, int minY, const char*& outPtr)
+{
+    auto ret = BLOSC_compress_impl(inPtr, inSize, outPtr);
+    return ret;
+}
+int
+ZstdCompressor::uncompress (
+    const char* inPtr, int inSize, int minY, const char*& outPtr)
+{
+    auto ret = BLOSC_uncompress_impl(inPtr, inSize, outPtr);
+    return ret;
+}
+
+
+int
+ZstdCompressor::BLOSC_compress_impl (const char* inPtr, int inSize, const char*& out)
+{
+    BloscInit::Init();
+    blosc2_cparams cparams = BLOSC2_CPARAMS_DEFAULTS;
+
+    cparams.typesize = pixelTypeSize(PixelType::FLOAT); // Expect Float values
+    cparams.clevel = header().zstdCompressionLevel();  // 9 is about a 20% increase in compression compared to 5. Decompression speed is unchanged.
+    cparams.nthreads = 1;
+    cparams.compcode = BLOSC_ZSTD; // Codec
+    cparams.splitmode = BLOSC_NEVER_SPLIT;  // Split => multithreading, not split better compression
+
+
+    blosc2_storage storage= BLOSC2_STORAGE_DEFAULTS;
+    storage.cparams=&cparams;
+    storage.contiguous = true;
+
+    _schunk = schunk_ptr (blosc2_schunk_new(&storage), &blosc2_schunk_free);
+
+    auto in = const_cast<char*>(inPtr);
+    blosc2_schunk_append_buffer(_schunk.get(), in, inSize);
+
+    uint8_t *buffer;
+    bool shouldFree = true;
+    auto size = blosc2_schunk_to_buffer(_schunk.get(), &buffer, &shouldFree);
+    out = (char*)buffer;
+    if (shouldFree)
+    {
+        _outBuffer = raw_ptr((char*)buffer, &free);
+    }
+    return size;
+}
+
+int
+ZstdCompressor::BLOSC_uncompress_impl (const char* inPtr, int inSize, const char*& out)
+{
+    auto in = const_cast<char*>(inPtr);
+    _schunk = schunk_ptr (blosc2_schunk_from_buffer(reinterpret_cast<uint8_t*>(in), inSize, true), &blosc2_schunk_free);
+
+    auto buffSize = _maxScanlineSize * numScanLines();
+    _outBuffer = Imf::ZstdCompressor::raw_ptr((char*)malloc (buffSize), &free);
+    auto size = blosc2_schunk_decompress_chunk(_schunk.get(), 0, _outBuffer.get(), buffSize);
+    out = _outBuffer.get();
+    return size;
+}
+
+OPENEXR_IMF_INTERNAL_NAMESPACE_SOURCE_EXIT

src/lib/OpenEXR/ImfZstdCompressor.h

@@ -0,0 +1,29 @@
+#pragma once
+#include <memory>
+#include "ImfNamespace.h"
+#include "ImfCompressor.h"
+#include "ImfHeader.h"
+#include "blosc2.h"
+
+OPENEXR_IMF_INTERNAL_NAMESPACE_HEADER_ENTER
+class ZstdCompressor : public Compressor
+{
+public:
+    ZstdCompressor (const Header& hdr, size_t maxScanLines);
+
+private:
+    using schunk_ptr = std::unique_ptr<blosc2_schunk, decltype(&blosc2_schunk_free)> ;
+    using raw_ptr = std::unique_ptr<char, decltype(&free)> ;
+    raw_ptr _outBuffer;
+    schunk_ptr _schunk;
+    size_t  _maxScanlineSize;
+    int numScanLines () const override; // max
+    int compress (
+        const char* inPtr, int inSize, int minY, const char*& outPtr) override;
+    int uncompress (
+        const char* inPtr, int inSize, int minY, const char*& outPtr) override;
+    int BLOSC_compress_impl (const char* inPtr, int inSize, const char*& out);
+    int BLOSC_uncompress_impl (const char* inPtr, int inSize, const char*& out);
+};
+
+OPENEXR_IMF_INTERNAL_NAMESPACE_HEADER_EXIT

src/lib/OpenEXRCore/base.c

@@ -208,3 +208,19 @@ exr_get_default_dwa_compression_quality (float* q)
 {
     if (q) *q = sDefaultDwaLevel;
 }
+
+static int sDefaultZstdLevel = 5;
+
+void
+exr_set_default_zstd_compression_level (int q)
+{
+    if (q < 0) q = 0;
+    if (q > 9) q = 9;
+    sDefaultDwaLevel = q;
+}
+
+void
+exr_get_default_zstd_compression_level (int* q)
+{
+    if (q) *q = sDefaultZstdLevel;
+}