dumpling: compress supports snappy/zstd

dumpling/export/config.go

@@ -291,7 +291,7 @@ func (*Config) DefineFlags(flags *pflag.FlagSet) {
 	_ = flags.MarkHidden(flagReadTimeout)
 	flags.Bool(flagTransactionalConsistency, true, "Only support transactional consistency")
 	_ = flags.MarkHidden(flagTransactionalConsistency)
-	flags.StringP(flagCompress, "c", "", "Compress output file type, support 'gzip', 'no-compression' now")
+	flags.StringP(flagCompress, "c", "", "Compress output file type, support 'gzip', 'snappy', 'zstd', 'no-compression' now")
 }
 
 // ParseFromFlags parses dumpling's export.Config from flags
@@ -596,6 +596,10 @@ func ParseCompressType(compressType string) (storage.CompressType, error) {
 		return storage.NoCompression, nil
 	case "gzip", "gz":
 		return storage.Gzip, nil
+	case "snappy":
+		return storage.Snappy, nil
+	case "zstd", "zst":
+		return storage.Zstd, nil
 	default:
 		return storage.NoCompression, errors.Errorf("unknown compress type %s", compressType)
 	}

dumpling/export/writer_util.go

@@ -591,6 +591,10 @@ func compressFileSuffix(compressType storage.CompressType) string {
 		return ""
 	case storage.Gzip:
 		return ".gz"
+	case storage.Snappy:
+		return ".snappy"
+	case storage.Zstd:
+		return ".zst"
 	default:
 		return ""
 	}

dumpling/install.sh

@@ -15,3 +15,10 @@ chmod a+x bin/minio
 
 wget https://dl.minio.io/client/mc/release/linux-amd64/mc -O bin/mc
 chmod a+x bin/mc
+
+go install github.com/ma6174/snappy
+
+wget https://github.com/facebook/zstd/releases/download/v1.5.2/zstd-1.5.2.tar.gz
+tar xvfz zstd-1.5.2.tar.gz
+cd zstd-1.5.2
+make

dumpling/tests/s3/run.sh

@@ -80,21 +80,46 @@ echo "1st records count is ${cnt}"
 # run dumpling with compress option
 export DUMPLING_OUTPUT_DIR=s3://mybucket/dump-compress
 
-run_dumpling --s3.endpoint="http://$S3_ENDPOINT/" --compress "gzip"
-
-mkdir -p "${HOST_DIR}/compress"
-
-bin/mc cp minio/mybucket/dump-compress/s3-schema-create.sql.gz "${HOST_DIR}/compress/s3-schema-create.sql.gz"
-bin/mc cp minio/mybucket/dump-compress/s3.t-schema.sql.gz "${HOST_DIR}/compress/s3.t-schema.sql.gz"
-bin/mc cp minio/mybucket/dump-compress/s3.t.000000000.sql.gz "${HOST_DIR}/compress/s3.t.000000000.sql.gz"
-
-gzip "${HOST_DIR}/compress/s3-schema-create.sql.gz" -d
-diff "${HOST_DIR}/local/s3-schema-create.sql" "${HOST_DIR}/compress/s3-schema-create.sql"
-
-gzip "${HOST_DIR}/compress/s3.t-schema.sql.gz" -d
-diff "${HOST_DIR}/local/s3.t-schema.sql" "${HOST_DIR}/compress/s3.t-schema.sql"
-
-gzip "${HOST_DIR}/compress/s3.t.000000000.sql.gz" -d
-diff "${HOST_DIR}/local/s3.t.000000000.sql" "${HOST_DIR}/compress/s3.t.000000000.sql"
+for compressType in "gzip" "snappy" "zst"
+do
+  echo -e "\nstart ${compressType} test"
+  run_dumpling --s3.endpoint="http://$S3_ENDPOINT/" --compress ${compressType}
+
+  mkdir -p "${HOST_DIR}/compress"
+
+  case $compressType in
+  "gzip")
+    suffix="gz"
+    binary=gzip
+  ;;
+
+  "snappy")
+    suffix="snappy"
+    binary=snappy
+  ;;
+
+  "zst")
+    suffix="zst"
+    binary=zstd
+  ;;
+  esac
+
+  bin/mc cp minio/mybucket/dump-compress/s3-schema-create.sql.${suffix} "${HOST_DIR}/compress/s3-schema-create.sql.${suffix}"
+  bin/mc cp minio/mybucket/dump-compress/s3.t-schema.sql.${suffix} "${HOST_DIR}/compress/s3.t-schema.sql.${suffix}"
+  bin/mc cp minio/mybucket/dump-compress/s3.t.000000000.sql.${suffix} "${HOST_DIR}/compress/s3.t.000000000.sql.${suffix}"
+
+  ${binary} -d "${HOST_DIR}/compress/s3-schema-create.sql.${suffix}"
+  diff "${HOST_DIR}/local/s3-schema-create.sql" "${HOST_DIR}/compress/s3-schema-create.sql"
+
+  ${binary} -d "${HOST_DIR}/compress/s3.t-schema.sql.${suffix}"
+  diff "${HOST_DIR}/local/s3.t-schema.sql" "${HOST_DIR}/compress/s3.t-schema.sql"
+
+  ${binary} -d "${HOST_DIR}/compress/s3.t.000000000.sql.${suffix}"
+  diff "${HOST_DIR}/local/s3.t.000000000.sql" "${HOST_DIR}/compress/s3.t.000000000.sql"
+
+  rm "${HOST_DIR}/compress/s3-schema-create.sql"
+  rm "${HOST_DIR}/compress/s3.t-schema.sql"
+  rm "${HOST_DIR}/compress/s3.t.000000000.sql"
+done
 
 run_sql "drop database if exists \`$DB_NAME\`;"