feat: refactor asm2asm (#393)

.github/workflows/fuzzing-linux-x64.yml

@@ -0,0 +1,33 @@
+name: Fuzz Linux-X64
+
+on: pull_request
+
+jobs:
+  build:
+    runs-on: [self-hosted, X64]
+    steps:
+      - name: Clear repository
+        run: sudo rm -fr $GITHUB_WORKSPACE && mkdir $GITHUB_WORKSPACE
+
+      - uses: actions/checkout@v2
+
+      - name: Check Branch
+        run: ./check_branch_name.sh ${{ github.head_ref }}
+
+      - name: Set up Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.18
+
+      - uses: actions/cache@v2
+        with:
+          path: ~/go/pkg/mod
+          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-
+
+      - name: Fuzz sonic
+        run: |
+          cd ./fuzz
+          make fuzz
+          make run 

.github/workflows/push-check-linux-arm64.yml

@@ -6,7 +6,7 @@ jobs:
   build:
     strategy:
       matrix:
-        go-version: [1.15.x, 1.20.x]
+        go-version: [1.15.x, 1.16.x, 1.20.x]
         os: [arm]
     runs-on: ${{ matrix.os }}
     steps:

.github/workflows/push-check-linux-x64.yml

@@ -6,7 +6,7 @@ jobs:
   build:
     strategy:
       matrix:
-        go-version: [1.15.x, 1.16.x, 1.17.x, 1.19.x, 1.20.x]
+        go-version: [1.16.x, 1.17.x, 1.19.x, 1.20.x]
     runs-on: [self-hosted, X64]
     steps:
       - name: Clear repository

.github/workflows/push-check-qemu.yml

@@ -6,7 +6,7 @@ jobs:
   build:
     strategy:
       matrix:
-        go-version: [1.15.x, 1.20.x]
+        go-version: [1.16.x, 1.20.x]
         os: [arm]
     runs-on: ${{ matrix.os }}
     steps:

.github/workflows/push-check-windows.yml

@@ -6,7 +6,7 @@ jobs:
   build:
     strategy:
       matrix:
-        go-version: [1.15.x, 1.20.x]
+        go-version: [1.17.x, 1.20.x]
     runs-on: windows-latest
     steps:
       - uses: actions/checkout@v2

Makefile

@@ -23,23 +23,25 @@ CPU_avx			:= amd64
 CPU_avx2		:= amd64
 CPU_sse  		:= amd64
 
-TMPL_avx		:= fastint_amd64_test fastfloat_amd64_test native_amd64_test native_export_amd64
-TMPL_avx2		:= fastint_amd64_test fastfloat_amd64_test native_amd64_test native_export_amd64
-TMPL_sse 		:= fastint_amd64_test fastfloat_amd64_test native_amd64_test native_export_amd64
-
-CFLAGS_avx		:= -msse -mssse3 -mno-sse4 -mavx -mpclmul -mno-avx2 -DUSE_AVX=1 -DUSE_AVX2=0
-CFLAGS_avx2		:= -msse -mssse3 -mno-sse4 -mavx -mpclmul -mavx2    -DUSE_AVX=1 -DUSE_AVX2=1 
-CFLAGS_sse		:= -msse -mssse3 -mno-sse4 -mno-avx -mno-avx2 -mpclmul
-TARGETFLAGS		:= -target x86_64-apple-macos11 -nostdlib -fno-builtin -fno-asynchronous-unwind-tables
+TMPL_avx		:= fastint_amd64_test fastfloat_amd64_test native_amd64_test recover_amd64_test
+TMPL_avx2		:= fastint_amd64_test fastfloat_amd64_test native_amd64_test recover_amd64_test
+TMPL_sse 		:= fastint_amd64_test fastfloat_amd64_test native_amd64_test recover_amd64_test
 
+CFLAGS_avx		:= -msse -mno-sse4 -mavx -mpclmul -mno-avx2 -mstack-alignment=0 -DUSE_AVX=1 -DUSE_AVX2=0
+CFLAGS_avx2		:= -msse -mno-sse4 -mavx -mpclmul -mavx2 -mstack-alignment=0 -DUSE_AVX=1 -DUSE_AVX2=1 
+CFLAGS_sse		:= -msse -mno-sse4 -mno-avx -mno-avx2 -mpclmul
 
 CC_amd64		:= clang
 ASM2ASM_amd64	:= tools/asm2asm/asm2asm.py
 
 CFLAGS			:= -mno-red-zone
+CFLAGS			+= -target x86_64-apple-macos11
+CFLAGS			+= -fno-asynchronous-unwind-tables
+CFLAGS			+= -fno-builtin
 CFLAGS			+= -fno-exceptions
 CFLAGS			+= -fno-rtti
 CFLAGS			+= -fno-stack-protector
+CFLAGS			+= -nostdlib
 CFLAGS			+= -O3
 CFLAGS			+= -Wall -Werror
 
@@ -64,22 +66,16 @@ define build_arch
 	$(eval @cpu		:= $(value CPU_$(1)))
 	$(eval @deps	:= $(foreach tmpl,$(value TMPL_$(1)),${OUT_DIR}/$(1)/${tmpl}.go))
 	$(eval @asmin	:= ${TMP_DIR}/$(1)/native.s)
-	$(eval @asmout	:= ${OUT_DIR}/$(1)/native_${@cpu}.s)
+	$(eval @asmout	:= ${OUT_DIR}/$(1)/native_text_${@cpu}.go)
 	$(eval @stubin	:= ${OUT_DIR}/native_${@cpu}.tmpl)
 	$(eval @stubout	:= ${OUT_DIR}/$(1)/native_${@cpu}.go)
 
 $(1): ${@asmout} ${@deps}
 
 ${@asmout}: ${@stubout} ${NATIVE_SRC}
 	mkdir -p ${TMP_DIR}/$(1)
-	$${CC_${@cpu}} $${CFLAGS} $${CFLAGS_$(1)} ${TARGETFLAGS} -S -o ${TMP_DIR}/$(1)/native.s ${SRC_FILE}
-	$(foreach file, 
-		$(wildcard native/unittest/*), 
-		$${CC_${@cpu}} $${CFLAGS} $${CFLAGS_$(1)} -I./native -o ${TMP_DIR}/$(1)/test $(file)
-		./${TMP_DIR}/$(1)/test
-	)
-	python3 $${ASM2ASM_${@cpu}} ${@asmout} ${TMP_DIR}/$(1)/native.s
-	asmfmt -w ${@asmout}
+	$${CC_${@cpu}} $${CFLAGS} $${CFLAGS_$(1)} -S -o ${TMP_DIR}/$(1)/native.s ${SRC_FILE}
+	python3 $${ASM2ASM_${@cpu}} -r ${@stubout} ${TMP_DIR}/$(1)/native.s
 
 $(eval $(call 	\
 	build_tmpl,	\
@@ -113,4 +109,3 @@ $(foreach 								\
 	${ARCH},							\
 	$(eval $(call build_arch,${arch}))	\
 )
-

README.md

@@ -5,8 +5,8 @@ English | [中文](README_ZH_CN.md)
 A blazingly fast JSON serializing & deserializing library, accelerated by JIT (just-in-time compiling) and SIMD (single-instruction-multiple-data).
 
 ## Requirement
-- Go 1.15~1.20
-- Linux/MacOS/Windows
+- Go 1.16~1.20
+- Linux/MacOS/Windows(需要go1.17以上)
 - Amd64 ARCH
 
 ## Features

ast/api_amd64.go

@@ -1,4 +1,4 @@
-// +build amd64,go1.15,!go1.21
+// +build amd64,go1.16,!go1.21
 
 /*
  * Copyright 2022 ByteDance Inc.

ast/api_amd64_test.go

@@ -1,5 +1,5 @@
-//go:build amd64 && go1.15 && !go1.21
-// +build amd64,go1.15,!go1.21
+//go:build amd64 && go1.16 && !go1.21
+// +build amd64,go1.16,!go1.21
 
 /*
  * Copyright 2022 ByteDance Inc.

ast/api_compat.go

@@ -1,4 +1,4 @@
-// +build !amd64 go1.21
+// +build !amd64 !go1.16 go1.21
 
 /*
  * Copyright 2022 ByteDance Inc.
@@ -27,6 +27,10 @@ import (
     `github.com/bytedance/sonic/internal/rt`
 )
 
+func init() {
+    println("WARNING: sonic only supports Go1.16~1.20 && CPU amd64, but your environment is not suitable")
+}
+
 func quote(buf *[]byte, val string) {
     quoteString(buf, val)
 }

compat.go

@@ -1,4 +1,4 @@
-// +build !amd64 go1.21
+// +build !amd64 !go1.16 go1.21 
 
 /*
  * Copyright 2021 ByteDance Inc.

decode_float_test.go

@@ -1,4 +1,4 @@
-// +build amd64,go1.15,!go1.21
+// +build amd64,go1.16,!go1.21
 
 /*
  * Copyright 2021 ByteDance Inc.

decode_test.go

@@ -1,4 +1,4 @@
-// +build amd64,go1.15,!go1.21
+// +build amd64,go1.16,!go1.21
 
 /*
  * Copyright 2021 ByteDance Inc.

decoder/decoder_amd64.go

@@ -1,4 +1,4 @@
-// +build amd64,go1.15,!go1.21
+// +build amd64,go1.16,!go1.21
 
 /*
 * Copyright 2023 ByteDance Inc.
@@ -25,6 +25,10 @@ import (
 // Decoder is the decoder context object
 type Decoder = decoder.Decoder
 
+// SyntaxError represents json syntax error
+type SyntaxError = decoder.SyntaxError
+
+// MismatchTypeError represents dismatching between json and object
 type MismatchTypeError = decoder.MismatchTypeError
 
 // Options for decode.
@@ -42,8 +46,6 @@ const (
 // StreamDecoder is the decoder context object for streaming input.
 type StreamDecoder = decoder.StreamDecoder
 
-type SyntaxError = decoder.SyntaxError
-
 var (
     // NewDecoder creates a new decoder instance.
     NewDecoder = decoder.NewDecoder

decoder/decoder_amd64_test.go

@@ -1,4 +1,4 @@
-// +build amd64,go1.15,!go1.21
+// +build amd64,go1.16,!go1.21
 
 /*
  * Copyright 2021 ByteDance Inc.

decoder/decoder_compat.go

@@ -1,4 +1,5 @@
-// +build !amd64 go1.21
+//go:build !amd64 || !go1.16 || go1.21
+// +build !amd64 !go1.16 go1.21
 
 /*
 * Copyright 2023 ByteDance Inc.
@@ -14,19 +15,25 @@
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
-*/
+ */
 
 package decoder
 
 import (
-     `encoding/json`
-     `bytes`
-     `reflect`
-     `github.com/bytedance/sonic/internal/native/types`
-     `github.com/bytedance/sonic/option`
-     `io`
+    `bytes`
+    `encoding/json`
+    `io`
+    `reflect`
+    `unsafe`
+
+    `github.com/bytedance/sonic/internal/native/types`
+    `github.com/bytedance/sonic/option`
 )
 
+func init() {
+     println("WARNING: sonic only supports Go1.16~1.20 && CPU amd64, but your environment is not suitable")
+}
+
 const (
      _F_use_int64 = iota
      _F_use_number
@@ -194,3 +201,17 @@ func (self *StreamDecoder) Decode(val interface{}) (err error) {
    return dec.Decode(val)
 }
 
+// SyntaxError represents json syntax error
+type SyntaxError json.SyntaxError
+
+// Description
+func (s SyntaxError) Description() string {
+     return (*json.SyntaxError)(unsafe.Pointer(&s)).Error()
+}
+// Error
+func (s SyntaxError) Error() string {
+     return (*json.SyntaxError)(unsafe.Pointer(&s)).Error()
+}
+
+// MismatchTypeError represents dismatching between json and object
+type MismatchTypeError json.UnmarshalTypeError

encode_test.go

@@ -1,4 +1,4 @@
-// +build amd64,go1.15,!go1.21
+// +build amd64,go1.16,!go1.21
 
 /*
  * Copyright 2021 ByteDance Inc.

encoder/encoder_amd64.go

@@ -1,4 +1,4 @@
-// +build amd64,go1.15,!go1.21
+// +build amd64,go1.16,!go1.21
 
 /*
  * Copyright 2023 ByteDance Inc.

encoder/encoder_amd64_test.go

@@ -1,4 +1,4 @@
-// +build amd64,go1.15,!go1.21
+// +build amd64,go1.16,!go1.21
 
 /*
  * Copyright 2021 ByteDance Inc.

encoder/encoder_compat.go

@@ -1,4 +1,4 @@
-// +build !amd64 go1.21
+// +build !amd64 !go1.16 go1.21
 
 /*
 * Copyright 2023 ByteDance Inc.
@@ -27,6 +27,10 @@ import (
     `github.com/bytedance/sonic/option`
 )
 
+func init() {
+    println("WARNING: sonic only supports Go1.16~1.20 && CPU amd64, but your environment is not suitable")
+}
+
 // Options is a set of encoding options.
 type Options uint64
 

fuzz/Makefile

@@ -5,10 +5,11 @@ fuzz:
 	mkdir -p ${corpusdir}
 	rm -rf ./go-fuzz-corpus
 	git clone https://github.com/dvyukov/go-fuzz-corpus.git ./go-fuzz-corpus/
+	go install golang.org/x/tools/cmd/file2fuzz@latest
 	file2fuzz -o ${corpusdir} ./go-fuzz-corpus/json/corpus/*  ./corpus/* 
 
 run:
-	GOARCH=amd64 go test -fuzz=${testname} -v
+	SONIC_FUZZ_MEM_LIMIT=2 GOMAXPROCS=4 GOARCH=amd64 go test -fuzz=${testname} -v -fuzztime 5m
 
 clean:
 	rm -rf ./go-fuzz-corpus/

go.mod

@@ -4,6 +4,7 @@ go 1.15
 
 require (
 	github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311
+	github.com/chenzhuoyu/iasm v0.9.0
 	github.com/davecgh/go-spew v1.1.1
 	github.com/klauspost/cpuid/v2 v2.0.9
 	github.com/stretchr/testify v1.8.1

go.sum

@@ -2,11 +2,14 @@ github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1
 github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY=
 github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 h1:qSGYFH7+jGhDF8vLC+iwCD4WpbV1EBDSzWkJODFLams=
 github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk=
+github.com/chenzhuoyu/iasm v0.9.0 h1:9fhXjVzq5hUy2gkhhgHl95zG2cEAhw9OSGs8toWWAwo=
+github.com/chenzhuoyu/iasm v0.9.0/go.mod h1:Xjy2NpN3h7aUqeqM+woSuuvxmIe6+DDsiNLIrkAmYog=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4=
 github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
+github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -26,4 +29,5 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50=
 rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=