blake3 hash implementation merge commit

This commit contains the blake3 implementation to be used for libfyaml.
It is considerably faster than the base BLAKE3 implementation since
it is fully accelerated and with configurable backends.

Signed-off-by: Pantelis Antoniou <[email protected]>

.gitignore

@@ -42,6 +42,8 @@ sphinx/
 src/fy-tool
 src/libfyaml-parser
 src/fy-thread
+src/fy-b3sum
+src/*.a
 stamp-h1
 tags
 TAGS

include/libfyaml.h

@@ -8079,6 +8079,175 @@ fy_thread_arg_join(struct fy_thread_pool *tp,
 		   void *arg, size_t count)
 	FY_EXPORT;
 
+/*
+ * Minimal exposing of internal BLAKE3 implementation
+ *
+ */
+
+/* BLAKE3 key length */
+#define FY_BLAKE3_KEY_LEN 32
+/* BLAKE3 output length */
+#define FY_BLAKE3_OUT_LEN 32
+
+/* opaque BLAKE3 hasher type for the user*/
+struct fy_blake3_hasher;
+
+/**
+ * fy_blake3_backend_iterate() - Iterate over the supported BLAKE3 backends
+ *
+ * This method iterates over the supported BLAKE3 backends.
+ * The start of the iteration is signalled by a NULL in \*prevp.
+ *
+ * The default backend is always the last in sequence, so for
+ * example if the order is [ "portable", "sse2", NULL ] the
+ * default is "sse2".
+ *
+ * @prevp: The previous backend pointer, or NULL at start
+ *
+ * Returns:
+ * The next backend or NULL at the end.
+ */
+const char *
+fy_blake3_backend_iterate(const char **prevp)
+	FY_EXPORT;
+
+/**
+ * struct fy_blake3_hasher_cfg - BLAKE3 hasher configuration
+ *
+ * Argument to the fy_blake3_hasher_create() method which
+ * is the fyaml's user facing BLAKE3 API.
+ * It is very minimal, on purpose, since it's meant to be
+ * exposing a full blown BLAKE3 API.
+ *
+ * @backend: NULL for default, or a specific backend name
+ * @file_buffer: Use this amount of buffer for buffering, zero for default
+ * @mmap_min_chunk: Minimum chunk size for mmap case
+ * @mmap_max_chunk: Maximum chunk size for mmap case
+ * @no_mmap: Disable mmap for file access
+ * @key: pointer to a FY_BLAKE3_KEY_LEN area when in keyed mode.
+ *       NULL otherwise.
+ * @context: pointer to a context when in key derivation mode.
+ *           NULL otherwise.
+ * @context_len: The size of the context when in key derivation mode.
+ *               0 otherwise.
+ * @tp: The thread pool to use, if NULL, create a private one
+ * @num_threads: Number of threads to use
+ *               - 0 means default: NUM_CPUS * 3 / 2
+ *               - > 0 specific number of threads
+ *               - -1 disable threading entirely
+ */
+struct fy_blake3_hasher_cfg {
+	const char *backend;
+	size_t file_buffer;
+	size_t mmap_min_chunk;
+	size_t mmap_max_chunk;
+	bool no_mmap;
+	const uint8_t *key;
+	const void *context;
+	size_t context_len;
+	struct fy_thread_pool *tp;
+	int num_threads;
+};
+
+/**
+ * fy_blake3_hasher_create() - Create a BLAKE3 hasher object.
+ *
+ * Creates a BLAKE3 hasher with its configuration @cfg
+ * The hasher may be destroyed by a corresponding call to
+ * fy_blake3_hasher_destroy().
+ *
+ * @cfg: The configuration for the BLAKE3 hasher
+ *
+ * Returns:
+ * A pointer to the BLAKE3 hasher or NULL in case of an error.
+ */
+struct fy_blake3_hasher *
+fy_blake3_hasher_create(const struct fy_blake3_hasher_cfg *cfg)
+	FY_EXPORT;
+
+/**
+ * fy_blake3_hasher_destroy() - Destroy the given BLAKE3 hasher
+ *
+ * Destroy a BLAKE3 hasher created earlier via fy_blake3_hasher_create().
+ *
+ * @fyh: The BLAKE3 hasher to destroy
+ */
+void
+fy_blake3_hasher_destroy(struct fy_blake3_hasher *fyh)
+	FY_EXPORT;
+
+/**
+ * fy_blake3_hasher_update() - Update the BLAKE3 hasher state with the given input
+ *
+ * Updates the BLAKE3 hasher state by hashing the given input.
+ *
+ * @fyh: The BLAKE3 hasher
+ * @input: Pointer to the input
+ * @input_len: Size of the input in bytes
+ */
+void
+fy_blake3_hasher_update(struct fy_blake3_hasher *fyh, const void *input, size_t input_len)
+	FY_EXPORT;
+
+/**
+ * fy_blake3_hasher_finalize() - Finalize the hash and get output
+ *
+ * Finalizes the BLAKE3 hasher and returns the output
+ *
+ * @fyh: The BLAKE3 hasher
+ *
+ * Returns:
+ * A pointer to the BLAKE3 output (sized FY_BLAKE3_OUT_LEN), or NULL
+ * in case of an error.
+ */
+const uint8_t *
+fy_blake3_hasher_finalize(struct fy_blake3_hasher *fyh)
+	FY_EXPORT;
+
+/**
+ * fy_blake3_hasher_reset() - Resets the hasher
+ *
+ * Resets the hasher for re-use
+ *
+ * @fyh: The BLAKE3 hasher
+ */
+void
+fy_blake3_hasher_reset(struct fy_blake3_hasher *fyh)
+	FY_EXPORT;
+
+/**
+ * fy_blake3_hash() - BLAKE3 hash a memory area
+ *
+ * Hash a memory area and return the BLAKE3 output.
+ *
+ * @fyh: The BLAKE3 hasher
+ * @mem: Pointer to the memory to use
+ * @size: The size of the memory in bytes
+ *
+ * Returns:
+ * A pointer to the BLAKE3 output (sized FY_BLAKE3_OUT_LEN), or NULL
+ * in case of an error.
+ */
+const uint8_t *
+fy_blake3_hash(struct fy_blake3_hasher *fyh, const void *mem, size_t size)
+	FY_EXPORT;
+
+/**
+ * fy_blake3_hash_file() - BLAKE3 hash a file.
+ *
+ * Hash the given file (possibly using mmap)
+ *
+ * @fyh: The BLAKE3 hasher
+ * @filename: The filename
+ *
+ * Returns:
+ * A pointer to the BLAKE3 output (sized FY_BLAKE3_OUT_LEN), or NULL
+ * in case of an error.
+ */
+const uint8_t *
+fy_blake3_hash_file(struct fy_blake3_hasher *fyh, const char *filename)
+	FY_EXPORT;
+
 #ifdef __cplusplus
 }
 #endif

src/Makefile.am

@@ -50,6 +50,69 @@ libfyaml_la_LDFLAGS = -no-undefined $(AM_LDFLAGS) $(AM_LIBLDFLAGS) \
 		      -version $(LIBTOOL_VERSION)
 libfyaml_la_LIBADD =
 
+# blake3 section
+libfyaml_la_CPPFLAGS += -I$(top_srcdir)/src/blake3
+libfyaml_la_SOURCES += blake3/blake3.h \
+		       blake3/blake3_impl.h \
+		       blake3/blake3_internal.h \
+		       blake3/blake3_host_state.c \
+		       blake3/blake3_backend.c \
+		       blake3/blake3_be_cpusimd.c \
+		       blake3/fy-blake3.c
+
+LIBB3_COMMON_CPPFLAGS=$(AM_CPPFLAGS) -I$(top_srcdir)/src/util -I$(top_srcdir)/src/thread
+
+# the portable implementation
+noinst_LTLIBRARIES += libb3portable.la
+libb3portable_la_SOURCES = blake3/blake3_portable.c blake3/blake3.c
+libb3portable_la_CPPFLAGS = $(LIBB3_COMMON_CPPFLAGS) -DHASHER_SUFFIX=portable -DSIMD_DEGREE=1
+libb3portable_la_CFLAGS = $(AM_CPPFLAGS)
+libfyaml_la_LIBADD += libb3portable.la
+
+# note that each target lib has blake3 compiled again but with a suffix
+if TARGET_HAS_SSE2
+noinst_LTLIBRARIES += libb3sse2.la
+libb3sse2_la_SOURCES = blake3/blake3_sse2.c blake3/blake3_sse2_x86-64_unix.S blake3/blake3.c
+libb3sse2_la_CPPFLAGS = $(LIBB3_COMMON_CPPFLAGS) -msse2 -DHASHER_SUFFIX=sse2 -DSIMD_DEGREE=4
+libb3sse2_la_CFLAGS = $(AM_CPPFLAGS) -msse2
+libfyaml_la_LIBADD += libb3sse2.la
+endif
+
+if TARGET_HAS_SSE41
+noinst_LTLIBRARIES += libb3sse41.la
+libb3sse41_la_SOURCES = blake3/blake3_sse41.c blake3/blake3_sse41_x86-64_unix.S blake3/blake3.c
+libb3sse41_la_CPPFLAGS = $(LIBB3_COMMON_CPPFLAGS) -msse4.1 -DHASHER_SUFFIX=sse41 -DSIMD_DEGREE=4
+libb3sse41_la_CFLAGS = $(AM_CPPFLAGS) -msse4.1
+libfyaml_la_LIBADD += libb3sse41.la
+endif
+
+if TARGET_HAS_AVX2
+noinst_LTLIBRARIES += libb3avx2.la
+libb3avx2_la_SOURCES = blake3/blake3_avx2.c blake3/blake3_avx2_x86-64_unix.S blake3/blake3.c
+libb3avx2_la_CPPFLAGS = $(LIBB3_COMMON_CPPFLAGS) -mavx2 -DHASHER_SUFFIX=avx2 -DSIMD_DEGREE=8
+libb3avx2_la_CFLAGS = $(AM_CPPFLAGS) -mavx2
+libfyaml_la_LIBADD += libb3avx2.la
+endif
+
+if TARGET_HAS_AVX512
+noinst_LTLIBRARIES += libb3avx512.la
+libb3avx512_la_SOURCES = blake3/blake3_avx512.c blake3/blake3_avx512_x86-64_unix.S blake3/blake3.c
+libb3avx512_la_CPPFLAGS = $(LIBB3_COMMON_CPPFLAGS) -mavx512f -mavx512vl -DHASHER_SUFFIX=avx512 -DSIMD_DEGREE=16
+libb3avx512_la_CFLAGS = $(AM_CPPFLAGS) -mavx512f -mavx512vl
+libfyaml_la_LIBADD += libb3avx512.la
+endif
+
+if TARGET_HAS_NEON
+noinst_LTLIBRARIES += libb3neon.la
+libb3neon_la_SOURCES = blake3/blake3_neon.c blake3/blake3.c
+libb3neon_la_CPPFLAGS = $(LIBB3_COMMON_CPPFLAGS) -DHASHER_SUFFIX=neon -DSIMD_DEGREE=4
+libb3neon_la_CFLAGS = $(AM_CPPFLAGS)
+if TARGET_CPU_ARM
+libb3neon_la_CPPFLAGS += -mfpu=neon
+libb3neon_la_CFLAGS += -mfpu=neon
+endif
+libfyaml_la_LIBADD += libb3neon.la
+endif
 
 # libfyaml-parser needs both LIBYAML and static
 if HAVE_LIBYAML
@@ -95,6 +158,28 @@ fy_thread_CFLAGS = $(AM_CFLAGS) $(LIBYAML_CFLAGS)
 fy_thread_LDFLAGS = $(AM_LDFLAGS) -static
 endif
 
+# fy-b3sum
+if HAVE_STATIC
+
+noinst_PROGRAMS += fy-b3sum
+
+fy_b3sum_SOURCES = \
+	internal/fy-b3sum.c \
+	valgrind/fy-valgrind.h
+
+fy_b3sum_CPPFLAGS = $(AM_CPPFLAGS) \
+			   -I$(top_srcdir)/src/valgrind \
+			   -I$(top_srcdir)/src/lib \
+			   -I$(top_srcdir)/src/xxhash \
+			   -I$(top_srcdir)/src/util \
+			   -I$(top_srcdir)/src/thread \
+			   -I$(top_srcdir)/src/blake3
+fy_b3sum_LDADD = $(AM_LDADD) $(LIBYAML_LIBS) libfyaml.la
+fy_b3sum_CFLAGS = $(AM_CFLAGS) $(LIBYAML_CFLAGS)
+
+fy_b3sum_LDFLAGS = $(AM_LDFLAGS) -static
+endif
+
 bin_PROGRAMS += fy-tool
 
 fy_tool_SOURCES = \