diff --git a/dumpling/export/config.go b/dumpling/export/config.go index 548535ed486ac..c5f9e7c41d426 100644 --- a/dumpling/export/config.go +++ b/dumpling/export/config.go @@ -291,7 +291,7 @@ func (*Config) DefineFlags(flags *pflag.FlagSet) { _ = flags.MarkHidden(flagReadTimeout) flags.Bool(flagTransactionalConsistency, true, "Only support transactional consistency") _ = flags.MarkHidden(flagTransactionalConsistency) - flags.StringP(flagCompress, "c", "", "Compress output file type, support 'gzip', 'no-compression' now") + flags.StringP(flagCompress, "c", "", "Compress output file type, support 'gzip', 'snappy', 'zstd', 'no-compression' now") } // ParseFromFlags parses dumpling's export.Config from flags @@ -596,6 +596,10 @@ func ParseCompressType(compressType string) (storage.CompressType, error) { return storage.NoCompression, nil case "gzip", "gz": return storage.Gzip, nil + case "snappy": + return storage.Snappy, nil + case "zstd", "zst": + return storage.Zstd, nil default: return storage.NoCompression, errors.Errorf("unknown compress type %s", compressType) } diff --git a/dumpling/export/writer_util.go b/dumpling/export/writer_util.go index 453585d43dec3..809f82e59c9b5 100644 --- a/dumpling/export/writer_util.go +++ b/dumpling/export/writer_util.go @@ -591,6 +591,10 @@ func compressFileSuffix(compressType storage.CompressType) string { return "" case storage.Gzip: return ".gz" + case storage.Snappy: + return ".snappy" + case storage.Zstd: + return ".zst" default: return "" } diff --git a/dumpling/install.sh b/dumpling/install.sh index d322b41242231..f24d54b499dea 100644 --- a/dumpling/install.sh +++ b/dumpling/install.sh @@ -15,3 +15,11 @@ chmod a+x bin/minio wget https://dl.minio.io/client/mc/release/linux-amd64/mc -O bin/mc chmod a+x bin/mc + +go get github.com/ma6174/snappy +go install github.com/ma6174/snappy + +wget https://github.com/facebook/zstd/releases/download/v1.5.2/zstd-1.5.2.tar.gz +tar xvfz zstd-1.5.2.tar.gz +cd zstd-1.5.2 +make diff --git a/dumpling/tests/s3/run.sh b/dumpling/tests/s3/run.sh index ede8c316106ed..c82d65f5ab16e 100755 --- a/dumpling/tests/s3/run.sh +++ b/dumpling/tests/s3/run.sh @@ -80,21 +80,46 @@ echo "1st records count is ${cnt}" # run dumpling with compress option export DUMPLING_OUTPUT_DIR=s3://mybucket/dump-compress -run_dumpling --s3.endpoint="http://$S3_ENDPOINT/" --compress "gzip" - -mkdir -p "${HOST_DIR}/compress" - -bin/mc cp minio/mybucket/dump-compress/s3-schema-create.sql.gz "${HOST_DIR}/compress/s3-schema-create.sql.gz" -bin/mc cp minio/mybucket/dump-compress/s3.t-schema.sql.gz "${HOST_DIR}/compress/s3.t-schema.sql.gz" -bin/mc cp minio/mybucket/dump-compress/s3.t.000000000.sql.gz "${HOST_DIR}/compress/s3.t.000000000.sql.gz" - -gzip "${HOST_DIR}/compress/s3-schema-create.sql.gz" -d -diff "${HOST_DIR}/local/s3-schema-create.sql" "${HOST_DIR}/compress/s3-schema-create.sql" - -gzip "${HOST_DIR}/compress/s3.t-schema.sql.gz" -d -diff "${HOST_DIR}/local/s3.t-schema.sql" "${HOST_DIR}/compress/s3.t-schema.sql" - -gzip "${HOST_DIR}/compress/s3.t.000000000.sql.gz" -d -diff "${HOST_DIR}/local/s3.t.000000000.sql" "${HOST_DIR}/compress/s3.t.000000000.sql" +for compressType in "gzip" "snappy" "zst" +do + echo -e "\nstart ${compressType} test" + run_dumpling --s3.endpoint="http://$S3_ENDPOINT/" --compress ${compressType} + + mkdir -p "${HOST_DIR}/compress" + + case $compressType in + "gzip") + suffix="gz" + binary=gzip + ;; + + "snappy") + suffix="snappy" + binary=snappy + ;; + + "zst") + suffix="zst" + binary=zstd + ;; + esac + + bin/mc cp minio/mybucket/dump-compress/s3-schema-create.sql.${suffix} "${HOST_DIR}/compress/s3-schema-create.sql.${suffix}" + bin/mc cp minio/mybucket/dump-compress/s3.t-schema.sql.${suffix} "${HOST_DIR}/compress/s3.t-schema.sql.${suffix}" + bin/mc cp minio/mybucket/dump-compress/s3.t.000000000.sql.${suffix} "${HOST_DIR}/compress/s3.t.000000000.sql.${suffix}" + + ${binary} -d "${HOST_DIR}/compress/s3-schema-create.sql.${suffix}" + diff "${HOST_DIR}/local/s3-schema-create.sql" "${HOST_DIR}/compress/s3-schema-create.sql" + + ${binary} -d "${HOST_DIR}/compress/s3.t-schema.sql.${suffix}" + diff "${HOST_DIR}/local/s3.t-schema.sql" "${HOST_DIR}/compress/s3.t-schema.sql" + + ${binary} -d "${HOST_DIR}/compress/s3.t.000000000.sql.${suffix}" + diff "${HOST_DIR}/local/s3.t.000000000.sql" "${HOST_DIR}/compress/s3.t.000000000.sql" + + rm "${HOST_DIR}/compress/s3-schema-create.sql" + rm "${HOST_DIR}/compress/s3.t-schema.sql" + rm "${HOST_DIR}/compress/s3.t.000000000.sql" +done run_sql "drop database if exists \`$DB_NAME\`;"