riscv64: Implement optimised crc using zbc and zbb extensions

.clang-format-ignore

@@ -1,3 +1,5 @@
 include/aarch64_multibinary.h
 include/aarch64_label.h
 **/aarch64/*.h
+
+**/riscv64/*.h

Makefile.am

@@ -29,10 +29,12 @@ other_tests_x86_64=
 other_tests_x86_32=
 other_tests_aarch64=
 other_tests_ppc64le=
+other_tests_riscv64=
 lsrc_x86_64=
 lsrc_x86_32=
 lsrc_aarch64=
 lsrc_ppc64le=
+lsrc_riscv64=
 lsrc_base_aliases=
 lsrc32=
 unit_tests32=
@@ -83,6 +85,12 @@ libisal_la_SOURCES += ${lsrc_ppc64le}
 other_tests += ${other_tests_ppc64le}
 endif
 
+if CPU_RISCV64
+ARCH=-Driscv64
+libisal_la_SOURCES += ${lsrc_riscv64}
+other_tests += ${other_tests_riscv64}
+endif
+
 if CPU_UNDEFINED
 libisal_la_SOURCES += ${lsrc_base_aliases}
 endif
@@ -131,6 +139,9 @@ endif
 if CPU_AARCH64
   as_filter = $(CC) -D__ASSEMBLY__
 endif
+if CPU_RISCV64
+  as_filter = $(CC) -D__ASSEMBLY__
+endif
 
 CCAS = $(as_filter)
 EXTRA_DIST += tools/yasm-filter.sh tools/nasm-filter.sh
@@ -142,6 +153,9 @@ AM_CCASFLAGS = ${AM_CFLAGS}
 else
 AM_CCASFLAGS = ${yasm_args} ${INCLUDE} ${src_include} ${DEFS} ${D}
 endif
+if CPU_RISCV64
+AM_CCASFLAGS = ${AM_CFLAGS}
+endif
 
 .asm.s:
 	@echo "  MKTMP   " $@;

configure.ac

@@ -31,11 +31,13 @@ AS_CASE([$host_cpu],
   [arm64], [CPU="aarch64"],
   [powerpc64le], [CPU="ppc64le"],
   [ppc64le], [CPU="ppc64le"],
+  [riscv64], [CPU="riscv64"],
 )
 AM_CONDITIONAL([CPU_X86_64], [test "$CPU" = "x86_64"])
 AM_CONDITIONAL([CPU_X86_32], [test "$CPU" = "x86_32"])
 AM_CONDITIONAL([CPU_AARCH64], [test "$CPU" = "aarch64"])
 AM_CONDITIONAL([CPU_PPC64LE], [test "$CPU" = "ppc64le"])
+AM_CONDITIONAL([CPU_RISCV64], [test "$CPU" = "riscv64"])
 AM_CONDITIONAL([CPU_UNDEFINED], [test "x$CPU" = "x"])
 
 if test "$CPU" = "x86_64"; then

crc/Makefile.am

@@ -28,6 +28,7 @@
 ########################################################################
 
 include crc/aarch64/Makefile.am
+include crc/riscv64/Makefile.am
 
 lsrc  += \
 	crc/crc_base.c \

crc/riscv64/Makefile.am

@@ -0,0 +1,43 @@
+########################################################################
+#  Copyright(c) 2024 ByteDance All rights reserved.
+#
+#  Redistribution and use in source and binary forms, with or without
+#  modification, are permitted provided that the following conditions
+#  are met:
+#    * Redistributions of source code must retain the above copyright
+#      notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above copyright
+#      notice, this list of conditions and the following disclaimer in
+#      the documentation and/or other materials provided with the
+#      distribution.
+#    * Neither the name of ByteDance Corporation nor the names of its
+#      contributors may be used to endorse or promote products derived
+#      from this software without specific prior written permission.
+#
+#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+#  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+#  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+#  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+#  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+#  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+#  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+#  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+#  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+#  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+########################################################################
+
+lsrc_riscv64 += \
+	crc/riscv64/crc16_t10dif.S \
+	crc/riscv64/crc32_gzip_refl.S \
+	crc/riscv64/crc32_ieee.S \
+	crc/riscv64/crc32_iscsi.S \
+	crc/riscv64/crc64_ecma_norm.S \
+	crc/riscv64/crc64_ecma_refl.S \
+	crc/riscv64/crc64_iso_norm.S \
+	crc/riscv64/crc64_iso_refl.S \
+	crc/riscv64/crc64_jones_norm.S \
+	crc/riscv64/crc64_jones_refl.S \
+	crc/riscv64/crc64_rocksoft_norm.S \
+	crc/riscv64/crc64_rocksoft_refl.S
+

crc/riscv64/crc16_t10dif.S

@@ -0,0 +1,97 @@
+########################################################################
+#  Copyright(c) 2024 ByteDance All rights reserved.
+#
+#  Redistribution and use in source and binary forms, with or without
+#  modification, are permitted provided that the following conditions
+#  are met:
+#    * Redistributions of source code must retain the above copyright
+#      notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above copyright
+#      notice, this list of conditions and the following disclaimer in
+#      the documentation and/or other materials provided with the
+#      distribution.
+#    * Neither the name of ByteDance Corporation nor the names of its
+#      contributors may be used to endorse or promote products derived
+#      from this software without specific prior written permission.
+#
+#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+#  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+#  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+#  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+#  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+#  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+#  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+#  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+#  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+#  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+########################################################################
+
+#include "crc32_norm_common_clmul.h"
+
+/* uint16_t crc16_t10dif(uint16_t init_crc, uint8_t *buf, uint64_t len) */
+.text
+.align 1
+.global crc16_t10dif
+.type crc16_t10dif, %function
+crc16_t10dif:
+	/* load precomputed constants */
+	ld POLY, .poly
+	ld MU, .mu
+
+	/* shift 16-bit seed into upper nibble */
+	slli SEED, SEED, 16
+
+	/* align and fold as though we're calculating a 32-bit crc */
+	crc32_norm_align
+
+	crc_fold_loop 32 1 0
+	crc32_norm_fold_reduction
+
+	crc32_norm_excess
+
+	/* shift back down result */
+	srli SEED, SEED, 16
+	ret
+
+/* precomputed constants */
+.poly:
+	.dword 0x000000018bb70000
+.mu:
+	.dword 0x00000001f65a57f8
+.k1:
+	.dword 0x00000000371d0000
+.k2:
+	.dword 0x0000000087e70000
+.k3:
+	.dword 0x000000004c1a0000
+.k4:
+	.dword 0x00000000fb0b0000
+.k5:
+	.dword 0x000000002d560000
+.k6:
+	.dword 0x0000000013680000
+
+
+/* uint16_t crc16_t10dif_copy(uint16_t seed, uint8_t * dst, uint8_t * src, uint64_t len) */
+/* in addition to calculating crc, also copies from src to dst */
+.text
+.align 1
+.global crc16_t10dif_copy
+.type crc16_t10dif_copy, %function
+crc16_t10dif_copy:
+	beqz a3, .memcpy_done
+	add t0, a2, a3
+	mv t1, a2
+.memcpy_loop:
+	lb t2, 0(t1)
+	sb t2, 0(a1)
+	addi t1, t1, 1
+	addi a1, a1, 1
+	bne t1, t0, .memcpy_loop
+
+.memcpy_done:
+	/* tail-call crc function */
+	mv a1, a2
+	mv a2, a3
+	tail crc16_t10dif

crc/riscv64/crc32_gzip_refl.S

@@ -0,0 +1,75 @@
+########################################################################
+#  Copyright(c) 2024 ByteDance All rights reserved.
+#
+#  Redistribution and use in source and binary forms, with or without
+#  modification, are permitted provided that the following conditions
+#  are met:
+#    * Redistributions of source code must retain the above copyright
+#      notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above copyright
+#      notice, this list of conditions and the following disclaimer in
+#      the documentation and/or other materials provided with the
+#      distribution.
+#    * Neither the name of ByteDance Corporation nor the names of its
+#      contributors may be used to endorse or promote products derived
+#      from this software without specific prior written permission.
+#
+#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+#  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+#  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+#  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+#  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+#  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+#  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+#  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+#  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+#  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+########################################################################
+
+#include "crc32_refl_common_clmul.h"
+
+/* uint32_t crc32_gzip_refl(uint32_t seed, uint8_t *buf, uint64_t len) */
+.text
+.align 1
+.global crc32_gzip_refl
+.type crc32_gzip_refl, %function
+crc32_gzip_refl:
+	/* load precomputed constants */
+	ld POLY, .poly_refl
+	ld MU, .mu
+
+	/* invert and zero-extend seed */
+	not SEED, SEED
+	slli SEED, SEED, 32
+	srli SEED, SEED, 32
+
+	/* align buffer to 128-bits, then fold */
+	crc32_refl_align
+
+	crc_fold_loop 32 0 1
+	crc32_refl_fold_reduction
+
+	/* handle any excess */
+	crc32_refl_excess
+
+	/* sign-extend and reflect result */
+	sext.w SEED, SEED
+	not SEED, SEED
+	ret
+
+/* precomputed constants */
+.poly_refl:
+	.dword 0x00000001db710641
+.mu:
+	.dword 0xb4e5b025f7011641
+.k1:
+	.dword 0x0000000154442bd4
+.k2:
+	.dword 0x00000001c6e41596
+.k3:
+	.dword 0x00000001751997d0
+.k4:
+	.dword 0x00000000ccaa009e
+.k5:
+	.dword 0x0000000163cd6124

crc/riscv64/crc32_ieee.S

@@ -0,0 +1,78 @@
+########################################################################
+#  Copyright(c) 2024 ByteDance All rights reserved.
+#
+#  Redistribution and use in source and binary forms, with or without
+#  modification, are permitted provided that the following conditions
+#  are met:
+#    * Redistributions of source code must retain the above copyright
+#      notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above copyright
+#      notice, this list of conditions and the following disclaimer in
+#      the documentation and/or other materials provided with the
+#      distribution.
+#    * Neither the name of ByteDance Corporation nor the names of its
+#      contributors may be used to endorse or promote products derived
+#      from this software without specific prior written permission.
+#
+#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+#  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+#  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+#  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+#  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+#  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+#  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+#  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+#  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+#  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+########################################################################
+
+#include "crc32_norm_common_clmul.h"
+
+/* uint32_t crc32_ieee(uint32_t init_crc, uint8_t *buf, uint64_t len) */
+.text
+.align 1
+.global crc32_ieee
+.type crc32_ieee, %function
+crc32_ieee:
+	/* load precomputed constants */
+	ld POLY, .poly
+	ld MU, .mu
+
+	/* reflect and zero-extend seed (riscv calling convention has uint32_t
+         * passed in and returned sign-extended)
+         */
+	not SEED, SEED
+	slli SEED, SEED, 32
+	srli SEED, SEED, 32
+
+	/* align and fold buffer */
+	crc32_norm_align
+
+	crc_fold_loop 32 1 0
+	crc32_norm_fold_reduction
+
+	crc32_norm_excess
+
+	/* sign-extend and reflect result */
+	sext.w SEED, SEED
+	not SEED, SEED
+	ret
+
+/* precomputed constants */
+.poly:
+	.dword 0x0000000104c11db7
+.mu:
+	.dword 0x0000000104d101df
+.k1:
+	.dword 0x000000008833794c
+.k2:
+	.dword 0x00000000e6228b11
+.k3:
+	.dword 0x00000000c5b9cd4c
+.k4:
+	.dword 0x00000000e8a45605
+.k5:
+	.dword 0x00000000f200aa66
+.k6:
+	.dword 0x00000000490d678d