From d4a3bf99b22cda273d551f9424db579836e8846a Mon Sep 17 00:00:00 2001 From: John Kerl Date: Sat, 19 Aug 2023 15:22:59 -0400 Subject: [PATCH] Support ZSTD compression in-process (#1360) * Support ZSTD compression in-process * doc mods * unit-test cases * doc-gen artifacts --- docs/src/customization.md | 2 +- docs/src/customization.md.in | 2 +- docs/src/glossary.md | 5 +++ docs/src/glossary.md.in | 5 +++ docs/src/manpage.md | 8 +++-- docs/src/manpage.txt | 8 +++-- docs/src/new-in-miller-6.md | 2 +- docs/src/new-in-miller-6.md.in | 2 +- docs/src/reference-main-compressed-data.md | 12 +++---- docs/src/reference-main-compressed-data.md.in | 12 +++---- docs/src/reference-main-flag-list.md | 6 ++-- go.mod | 1 + go.sum | 2 ++ internal/pkg/cli/option_parse.go | 24 +++++++++++-- internal/pkg/lib/file_readers.go | 33 ++++++++++++++++++ man/manpage.txt | 8 +++-- man/mlr.1 | 8 +++-- test/cases/io-compressed-input/0014/cmd | 1 + test/cases/io-compressed-input/0014/experr | 0 test/cases/io-compressed-input/0014/expout | 5 +++ test/cases/io-compressed-input/0015/cmd | 1 + test/cases/io-compressed-input/0015/experr | 0 test/cases/io-compressed-input/0015/expout | 5 +++ test/cases/io-compressed-input/0016/cmd | 1 + test/cases/io-compressed-input/0016/experr | 0 test/cases/io-compressed-input/0016/expout | 5 +++ test/input/medium.zst | Bin 0 -> 957 bytes 27 files changed, 130 insertions(+), 28 deletions(-) create mode 100644 test/cases/io-compressed-input/0014/cmd create mode 100644 test/cases/io-compressed-input/0014/experr create mode 100644 test/cases/io-compressed-input/0014/expout create mode 100644 test/cases/io-compressed-input/0015/cmd create mode 100644 test/cases/io-compressed-input/0015/experr create mode 100644 test/cases/io-compressed-input/0015/expout create mode 100644 test/cases/io-compressed-input/0016/cmd create mode 100644 test/cases/io-compressed-input/0016/experr create mode 100644 test/cases/io-compressed-input/0016/expout create mode 100644 test/input/medium.zst diff --git a/docs/src/customization.md b/docs/src/customization.md index 5a787ad4fc..cbc69928f3 100644 --- a/docs/src/customization.md +++ b/docs/src/customization.md @@ -50,7 +50,7 @@ and the `--csv` part will automatically be understood. If you do want to process * You can include any command-line flags, except the "terminal" ones such as `--help`. -* The `--prepipe`, `--load`, and `--mload` flags aren't allowed in `.mlrrc` as they control code execution, and could result in your scripts running things you don't expect if you receive data from someone with a `./.mlrrc` in it. You can use `--prepipe-bz2`, `--prepipe-gunzip`, and `--prepipe-zcat` in `.mlrrc`, though. +* The `--prepipe`, `--load`, and `--mload` flags aren't allowed in `.mlrrc` as they control code execution, and could result in your scripts running things you don't expect if you receive data from someone with a `./.mlrrc` in it. You can use `--prepipe-bz2`, `--prepipe-gunzip`, `--prepipe-zcat`, and `--prepipe-zstdcat` in `.mlrrc`, though. * The formatting rule is you need to put one flag beginning with `--` per line: for example, `--csv` on one line and `--nr-progress-mod 1000` on a separate line. diff --git a/docs/src/customization.md.in b/docs/src/customization.md.in index 9a1d2894b7..00367b2f76 100644 --- a/docs/src/customization.md.in +++ b/docs/src/customization.md.in @@ -34,7 +34,7 @@ and the `--csv` part will automatically be understood. If you do want to process * You can include any command-line flags, except the "terminal" ones such as `--help`. -* The `--prepipe`, `--load`, and `--mload` flags aren't allowed in `.mlrrc` as they control code execution, and could result in your scripts running things you don't expect if you receive data from someone with a `./.mlrrc` in it. You can use `--prepipe-bz2`, `--prepipe-gunzip`, and `--prepipe-zcat` in `.mlrrc`, though. +* The `--prepipe`, `--load`, and `--mload` flags aren't allowed in `.mlrrc` as they control code execution, and could result in your scripts running things you don't expect if you receive data from someone with a `./.mlrrc` in it. You can use `--prepipe-bz2`, `--prepipe-gunzip`, `--prepipe-zcat`, and `--prepipe-zstdcat` in `.mlrrc`, though. * The formatting rule is you need to put one flag beginning with `--` per line: for example, `--csv` on one line and `--nr-progress-mod 1000` on a separate line. diff --git a/docs/src/glossary.md b/docs/src/glossary.md index bb731297b7..774975c41e 100644 --- a/docs/src/glossary.md +++ b/docs/src/glossary.md @@ -905,3 +905,8 @@ See also the [arrays page](reference-main-arrays.md), as well as the page on A [data-compression format supported by Miller](reference-main-compressed-data.md). Files compressed using ZLIB compression normally end in `.z`. + +## ZSTD / .zst + +A [data-compression format supported by Miller](reference-main-compressed-data.md). +Files compressed using ZSTD compression normally end in`.zst`. diff --git a/docs/src/glossary.md.in b/docs/src/glossary.md.in index 7e03b7d11b..b8eb8f4177 100644 --- a/docs/src/glossary.md.in +++ b/docs/src/glossary.md.in @@ -889,3 +889,8 @@ See also the [arrays page](reference-main-arrays.md), as well as the page on A [data-compression format supported by Miller](reference-main-compressed-data.md). Files compressed using ZLIB compression normally end in `.z`. + +## ZSTD / .zst + +A [data-compression format supported by Miller](reference-main-compressed-data.md). +Files compressed using ZSTD compression normally end in`.zst`. diff --git a/docs/src/manpage.md b/docs/src/manpage.md index 74d7c6c9ac..aad8a4f50f 100644 --- a/docs/src/manpage.md +++ b/docs/src/manpage.md @@ -262,7 +262,7 @@ MILLER(1) MILLER(1) Miller offers a few different ways to handle reading data files which have been compressed. - * Decompression done within the Miller process itself: `--bz2in` `--gzin` `--zin` + * Decompression done within the Miller process itself: `--bz2in` `--gzin` `--zin``--zstdin` * Decompression done outside the Miller process: `--prepipe` `--prepipex` Using `--prepipe` and `--prepipex` you can specify an action to be @@ -285,7 +285,7 @@ MILLER(1) MILLER(1) Lastly, note that if `--prepipe` or `--prepipex` is specified, it replaces any decisions that might have been made based on the file suffix. Likewise, - `--gzin`/`--bz2in`/`--zin` are ignored if `--prepipe` is also specified. + `--gzin`/`--bz2in`/`--zin``--zin` are ignored if `--prepipe` is also specified. --bz2in Uncompress bzip2 within the Miller process. Done by default if file ends in `.bz2`. @@ -302,6 +302,8 @@ MILLER(1) MILLER(1) `.mlrrc`. --prepipe-zcat Same as `--prepipe zcat`, except this is allowed in `.mlrrc`. + --prepipe-zstdcat Same as `--prepipe zstdcat`, except this is allowed + in `.mlrrc`. --prepipex {decompression command} Like `--prepipe` with one exception: doesn't insert `<` between command and filename at runtime. Useful @@ -310,6 +312,8 @@ MILLER(1) MILLER(1) in `.mlrrc` to avoid unexpected code execution. --zin Uncompress zlib within the Miller process. Done by default if file ends in `.z`. + --zstdin Uncompress zstd within the Miller process. Done by + default if file ends in `.zstd`. 1mCSV/TSV-ONLY FLAGS0m These are flags which are applicable to CSV format. diff --git a/docs/src/manpage.txt b/docs/src/manpage.txt index 8d79e4f608..1d59128536 100644 --- a/docs/src/manpage.txt +++ b/docs/src/manpage.txt @@ -241,7 +241,7 @@ MILLER(1) MILLER(1) Miller offers a few different ways to handle reading data files which have been compressed. - * Decompression done within the Miller process itself: `--bz2in` `--gzin` `--zin` + * Decompression done within the Miller process itself: `--bz2in` `--gzin` `--zin``--zstdin` * Decompression done outside the Miller process: `--prepipe` `--prepipex` Using `--prepipe` and `--prepipex` you can specify an action to be @@ -264,7 +264,7 @@ MILLER(1) MILLER(1) Lastly, note that if `--prepipe` or `--prepipex` is specified, it replaces any decisions that might have been made based on the file suffix. Likewise, - `--gzin`/`--bz2in`/`--zin` are ignored if `--prepipe` is also specified. + `--gzin`/`--bz2in`/`--zin``--zin` are ignored if `--prepipe` is also specified. --bz2in Uncompress bzip2 within the Miller process. Done by default if file ends in `.bz2`. @@ -281,6 +281,8 @@ MILLER(1) MILLER(1) `.mlrrc`. --prepipe-zcat Same as `--prepipe zcat`, except this is allowed in `.mlrrc`. + --prepipe-zstdcat Same as `--prepipe zstdcat`, except this is allowed + in `.mlrrc`. --prepipex {decompression command} Like `--prepipe` with one exception: doesn't insert `<` between command and filename at runtime. Useful @@ -289,6 +291,8 @@ MILLER(1) MILLER(1) in `.mlrrc` to avoid unexpected code execution. --zin Uncompress zlib within the Miller process. Done by default if file ends in `.z`. + --zstdin Uncompress zstd within the Miller process. Done by + default if file ends in `.zstd`. 1mCSV/TSV-ONLY FLAGS0m These are flags which are applicable to CSV format. diff --git a/docs/src/new-in-miller-6.md b/docs/src/new-in-miller-6.md index 3170819c9d..32633b6f8e 100644 --- a/docs/src/new-in-miller-6.md +++ b/docs/src/new-in-miller-6.md @@ -143,7 +143,7 @@ the `TZ` environment variable. Please see [DSL datetime/timezone functions](refe ### In-process support for compressed input -In addition to `--prepipe gunzip`, you can now use the `--gzin` flag. In fact, if your files end in `.gz` you don't even need to do that -- Miller will autodetect by file extension and automatically uncompress `mlr --csv cat foo.csv.gz`. Similarly for `.z` and `.bz2` files. Please see the page on [Compressed data](reference-main-compressed-data.md) for more information. +In addition to `--prepipe gunzip`, you can now use the `--gzin` flag. In fact, if your files end in `.gz` you don't even need to do that -- Miller will autodetect by file extension and automatically uncompress `mlr --csv cat foo.csv.gz`. Similarly for `.z`, `.bz2`, and `.zst` files. Please see the page on [Compressed data](reference-main-compressed-data.md) for more information. ### Support for reading web URLs diff --git a/docs/src/new-in-miller-6.md.in b/docs/src/new-in-miller-6.md.in index 43ea44d905..c450a96224 100644 --- a/docs/src/new-in-miller-6.md.in +++ b/docs/src/new-in-miller-6.md.in @@ -125,7 +125,7 @@ the `TZ` environment variable. Please see [DSL datetime/timezone functions](refe ### In-process support for compressed input -In addition to `--prepipe gunzip`, you can now use the `--gzin` flag. In fact, if your files end in `.gz` you don't even need to do that -- Miller will autodetect by file extension and automatically uncompress `mlr --csv cat foo.csv.gz`. Similarly for `.z` and `.bz2` files. Please see the page on [Compressed data](reference-main-compressed-data.md) for more information. +In addition to `--prepipe gunzip`, you can now use the `--gzin` flag. In fact, if your files end in `.gz` you don't even need to do that -- Miller will autodetect by file extension and automatically uncompress `mlr --csv cat foo.csv.gz`. Similarly for `.z`, `.bz2`, and `.zst` files. Please see the page on [Compressed data](reference-main-compressed-data.md) for more information. ### Support for reading web URLs diff --git a/docs/src/reference-main-compressed-data.md b/docs/src/reference-main-compressed-data.md index a54ed8026b..729cf5bbcc 100644 --- a/docs/src/reference-main-compressed-data.md +++ b/docs/src/reference-main-compressed-data.md @@ -16,13 +16,13 @@ Quick links: # Compressed data -As of [Miller 6](new-in-miller-6.md), Miller supports reading GZIP, BZIP2, and -ZLIB formats transparently, and in-process. And (as before Miller 6) you have a +As of [Miller 6](new-in-miller-6.md), Miller supports reading GZIP, BZIP2, ZLIB, and +ZSTD formats transparently, and in-process. And (as before Miller 6) you have a more general `--prepipe` option to support other decompression programs. ## Automatic detection on input -If your files end in `.gz`, `.bz2`, or `.z` then Miller will autodetect by file extension: +If your files end in `.gz`, `.bz2`, `.z`, or `.zst` then Miller will autodetect by file extension: 
 file gz-example.csv.gz
@@ -52,7 +52,7 @@ This will decompress the input data on the fly, while leaving the disk file unmo
 
 ## Manual detection on input
 
-If the filename doesn't in in `.gz`, `.bz2`, or `.z` then you can use the flags `--gzin`, `--bz2in`, or `--zin` to let Miller know:
+If the filename doesn't in in `.gz`, `.bz2`, `-z`, or `.zst` then you can use the flags `--gzin`, `--bz2in`, `--zin`, or `--zstdin` to let Miller know:
 
 
 mlr --csv --gzin sort -f color myfile.bin # myfile.bin has gzip contents
@@ -94,7 +94,7 @@ If the command has flags, quote them: e.g. `mlr --prepipe 'zcat -cf'`.
 
 In your [.mlrrc file](customization.md), `--prepipe` and `--prepipex` are not
 allowed as they could be used for unexpected code execution. You can use
-`--prepipe-bz2`, `--prepipe-gunzip`, and `--prepipe-zcat` in `.mlrrc`, though.
+`--prepipe-bz2`, `--prepipe-gunzip`, `--prepipe-zcat`, and `--prepipe-zstdcat` in `.mlrrc`, though.
 
 Note that this feature is quite general and is not limited to decompression
 utilities. You can use it to apply per-file filters of your choice: e.g. `mlr
@@ -107,7 +107,7 @@ There is a `--prepipe` and a `--prepipex`:
 
 Lastly, note that if `--prepipe` or `--prepipex` is specified on the Miller
 command line, it replaces any autodetect decisions that might have been made
-based on the filename extension. Likewise, `--gzin`/`--bz2in`/`--zin` are ignored if
+based on the filename extension. Likewise, `--gzin`/`--bz2in`/`--zin`/`--zstdin` are ignored if
 `--prepipe` or `--prepipex` is also specified.
 
 ## Compressed output
diff --git a/docs/src/reference-main-compressed-data.md.in b/docs/src/reference-main-compressed-data.md.in
index b13e5e7327..cbca6a3c34 100644
--- a/docs/src/reference-main-compressed-data.md.in
+++ b/docs/src/reference-main-compressed-data.md.in
@@ -1,12 +1,12 @@
 # Compressed data
 
-As of [Miller 6](new-in-miller-6.md), Miller supports reading GZIP, BZIP2, and
-ZLIB formats transparently, and in-process. And (as before Miller 6) you have a
+As of [Miller 6](new-in-miller-6.md), Miller supports reading GZIP, BZIP2, ZLIB, and
+ZSTD formats transparently, and in-process. And (as before Miller 6) you have a
 more general `--prepipe` option to support other decompression programs.
 
 ## Automatic detection on input
 
-If your files end in `.gz`, `.bz2`, or `.z` then Miller will autodetect by file extension:
+If your files end in `.gz`, `.bz2`, `.z`, or `.zst` then Miller will autodetect by file extension:
 
 GENMD-CARDIFY-HIGHLIGHT-ONE
 file gz-example.csv.gz
@@ -21,7 +21,7 @@ This will decompress the input data on the fly, while leaving the disk file unmo
 
 ## Manual detection on input
 
-If the filename doesn't in in `.gz`, `.bz2`, or `.z` then you can use the flags `--gzin`, `--bz2in`, or `--zin` to let Miller know:
+If the filename doesn't in in `.gz`, `.bz2`, `-z`, or `.zst` then you can use the flags `--gzin`, `--bz2in`, `--zin`, or `--zstdin` to let Miller know:
 
 GENMD-CARDIFY-HIGHLIGHT-ONE
 mlr --csv --gzin sort -f color myfile.bin # myfile.bin has gzip contents
@@ -50,7 +50,7 @@ If the command has flags, quote them: e.g. `mlr --prepipe 'zcat -cf'`.
 
 In your [.mlrrc file](customization.md), `--prepipe` and `--prepipex` are not
 allowed as they could be used for unexpected code execution. You can use
-`--prepipe-bz2`, `--prepipe-gunzip`, and `--prepipe-zcat` in `.mlrrc`, though.
+`--prepipe-bz2`, `--prepipe-gunzip`, `--prepipe-zcat`, and `--prepipe-zstdcat` in `.mlrrc`, though.
 
 Note that this feature is quite general and is not limited to decompression
 utilities. You can use it to apply per-file filters of your choice: e.g. `mlr
@@ -63,7 +63,7 @@ There is a `--prepipe` and a `--prepipex`:
 
 Lastly, note that if `--prepipe` or `--prepipex` is specified on the Miller
 command line, it replaces any autodetect decisions that might have been made
-based on the filename extension. Likewise, `--gzin`/`--bz2in`/`--zin` are ignored if
+based on the filename extension. Likewise, `--gzin`/`--bz2in`/`--zin`/`--zstdin` are ignored if
 `--prepipe` or `--prepipex` is also specified.
 
 ## Compressed output
diff --git a/docs/src/reference-main-flag-list.md b/docs/src/reference-main-flag-list.md
index b07c0847e0..8e2daf9d02 100644
--- a/docs/src/reference-main-flag-list.md
+++ b/docs/src/reference-main-flag-list.md
@@ -72,7 +72,7 @@ Notes:
 Miller offers a few different ways to handle reading data files
 	which have been compressed.
 
-* Decompression done within the Miller process itself: `--bz2in` `--gzin` `--zin`
+* Decompression done within the Miller process itself: `--bz2in` `--gzin` `--zin``--zstdin`
 * Decompression done outside the Miller process: `--prepipe` `--prepipex`
 
 Using `--prepipe` and `--prepipex` you can specify an action to be
@@ -95,7 +95,7 @@ compression (or other) utilities, simply pipe the output:
 
 Lastly, note that if `--prepipe` or `--prepipex` is specified, it replaces any
 decisions that might have been made based on the file suffix. Likewise,
-`--gzin`/`--bz2in`/`--zin` are ignored if `--prepipe` is also specified.
+`--gzin`/`--bz2in`/`--zin``--zin` are ignored if `--prepipe` is also specified.
 
 
 **Flags:**
@@ -106,8 +106,10 @@ decisions that might have been made based on the file suffix. Likewise,
 * `--prepipe-bz2`: Same as  `--prepipe bz2`, except this is allowed in `.mlrrc`.
 * `--prepipe-gunzip`: Same as  `--prepipe gunzip`, except this is allowed in `.mlrrc`.
 * `--prepipe-zcat`: Same as  `--prepipe zcat`, except this is allowed in `.mlrrc`.
+* `--prepipe-zstdcat`: Same as  `--prepipe zstdcat`, except this is allowed in `.mlrrc`.
 * `--prepipex {decompression command}`: Like `--prepipe` with one exception: doesn't insert `<` between command and filename at runtime. Useful for some commands like `unzip -qc` which don't read standard input.  Allowed at the command line, but not in `.mlrrc` to avoid unexpected code execution.
 * `--zin`: Uncompress zlib within the Miller process. Done by default if file ends in `.z`.
+* `--zstdin`: Uncompress zstd within the Miller process. Done by default if file ends in `.zstd`.
 
 ## CSV/TSV-only flags
 
diff --git a/go.mod b/go.mod
index b9e11f7eb3..2373dea148 100644
--- a/go.mod
+++ b/go.mod
@@ -34,6 +34,7 @@ require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/felixge/fgprof v0.9.3 // indirect
 	github.com/google/pprof v0.0.0-20211214055906-6f57359322fd // indirect
+	github.com/klauspost/compress v1.16.7 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
diff --git a/go.sum b/go.sum
index e896c8f4e9..84593de572 100644
--- a/go.sum
+++ b/go.sum
@@ -15,6 +15,8 @@ github.com/johnkerl/lumin v1.0.0 h1:CV34cHZOJ92Y02RbQ0rd4gA0C06Qck9q8blOyaPoWpU=
 github.com/johnkerl/lumin v1.0.0/go.mod h1:eLf5AdQOaLvzZ2zVy4REr/DSeEwG+CZreHwNLICqv9E=
 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=
 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
+github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I=
+github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
 github.com/lestrrat-go/envload v0.0.0-20180220234015-a3eb8ddeffcc h1:RKf14vYWi2ttpEmkA4aQ3j4u9dStX2t4M8UM6qqNsG8=
 github.com/lestrrat-go/envload v0.0.0-20180220234015-a3eb8ddeffcc/go.mod h1:kopuH9ugFRkIXf3YoqHKyrJ9YfUFsckUU9S7B+XP+is=
 github.com/lestrrat-go/strftime v1.0.6 h1:CFGsDEt1pOpFNU+TJB0nhz9jl+K0hZSLE205AhTIGQQ=
diff --git a/internal/pkg/cli/option_parse.go b/internal/pkg/cli/option_parse.go
index 034f3a6902..0ee362f2b3 100644
--- a/internal/pkg/cli/option_parse.go
+++ b/internal/pkg/cli/option_parse.go
@@ -2200,7 +2200,8 @@ func CompressedDataPrintInfo() {
 	fmt.Print(`Miller offers a few different ways to handle reading data files
 	which have been compressed.
 
-* Decompression done within the Miller process itself: ` + "`--bz2in`" + ` ` + "`--gzin`" + ` ` + "`--zin`" + `
+* Decompression done within the Miller process itself: ` + "`--bz2in`" + ` ` + "`--gzin`" + ` ` + "`--zin`" + "`--zstdin`" +
+		`
 * Decompression done outside the Miller process: ` + "`--prepipe`" + ` ` + "`--prepipex`" + `
 
 Using ` + "`--prepipe`" + ` and ` + "`--prepipex`" + ` you can specify an action to be
@@ -2223,7 +2224,7 @@ compression (or other) utilities, simply pipe the output:
 
 Lastly, note that if ` + "`--prepipe`" + ` or ` + "`--prepipex`" + ` is specified, it replaces any
 decisions that might have been made based on the file suffix. Likewise,
-` + "`--gzin`" + `/` + "`--bz2in`" + `/` + "`--zin`" + ` are ignored if ` + "`--prepipe`" + ` is also specified.
+` + "`--gzin`" + `/` + "`--bz2in`" + `/` + "`--zin`" + "`--zin`" + ` are ignored if ` + "`--prepipe`" + ` is also specified.
 `)
 }
 
@@ -2278,6 +2279,16 @@ var CompressedDataFlagSection = FlagSection{
 			},
 		},
 
+		{
+			name: "--prepipe-zstdcat",
+			help: "Same as  `--prepipe zstdcat`, except this is allowed in `.mlrrc`.",
+			parser: func(args []string, argc int, pargi *int, options *TOptions) {
+				options.ReaderOptions.Prepipe = "zstdcat"
+				options.ReaderOptions.PrepipeIsRaw = false
+				*pargi += 1
+			},
+		},
+
 		{
 			name: "--prepipe-bz2",
 			help: "Same as  `--prepipe bz2`, except this is allowed in `.mlrrc`.",
@@ -2314,6 +2325,15 @@ var CompressedDataFlagSection = FlagSection{
 				*pargi += 1
 			},
 		},
+
+		{
+			name: "--zstdin",
+			help: "Uncompress zstd within the Miller process. Done by default if file ends in `.zstd`.",
+			parser: func(args []string, argc int, pargi *int, options *TOptions) {
+				options.ReaderOptions.FileInputEncoding = lib.FileInputEncodingZstd
+				*pargi += 1
+			},
+		},
 	},
 }
 
diff --git a/internal/pkg/lib/file_readers.go b/internal/pkg/lib/file_readers.go
index fa42688efd..1511200406 100644
--- a/internal/pkg/lib/file_readers.go
+++ b/internal/pkg/lib/file_readers.go
@@ -25,6 +25,7 @@ import (
 	"compress/gzip"
 	"compress/zlib"
 	"fmt"
+	"github.com/klauspost/compress/zstd"
 	"io"
 	"net/http"
 	"os"
@@ -38,6 +39,7 @@ const (
 	FileInputEncodingBzip2
 	FileInputEncodingGzip
 	FileInputEncodingZlib
+	FileInputEncodingZstd
 )
 
 // OpenFileForRead: If prepipe is non-empty, popens "{prepipe} < {filename}"
@@ -160,6 +162,8 @@ func openEncodedHandleForRead(
 		return gzip.NewReader(handle)
 	case FileInputEncodingZlib:
 		return zlib.NewReader(handle)
+	case FileInputEncodingZstd:
+		return NewZstdReadCloser(handle)
 	}
 
 	InternalCodingErrorIf(encoding != FileInputEncodingDefault)
@@ -173,6 +177,9 @@ func openEncodedHandleForRead(
 	if strings.HasSuffix(filename, ".z") {
 		return zlib.NewReader(handle)
 	}
+	if strings.HasSuffix(filename, ".zst") {
+		return NewZstdReadCloser(handle)
+	}
 
 	// Pass along os.Stdin or os.Open(filename)
 	return handle, nil
@@ -200,6 +207,32 @@ func (rc *BZip2ReadCloser) Close() error {
 	return rc.originalHandle.Close()
 }
 
+// ----------------------------------------------------------------
+// ZstdReadCloser remedies the fact that zstd.NewReader does not implement io.ReadCloser.
+type ZstdReadCloser struct {
+	originalHandle io.ReadCloser
+	zstdHandle     io.Reader
+}
+
+func NewZstdReadCloser(handle io.ReadCloser) (*ZstdReadCloser, error) {
+	zstdHandle, err := zstd.NewReader(handle)
+	if err != nil {
+		return nil, err
+	}
+	return &ZstdReadCloser{
+		originalHandle: handle,
+		zstdHandle:     zstdHandle,
+	}, nil
+}
+
+func (rc *ZstdReadCloser) Read(p []byte) (n int, err error) {
+	return rc.zstdHandle.Read(p)
+}
+
+func (rc *ZstdReadCloser) Close() error {
+	return rc.originalHandle.Close()
+}
+
 // ----------------------------------------------------------------
 
 // IsEOF handles the following problem: reading past end of files opened with
diff --git a/man/manpage.txt b/man/manpage.txt
index 8d79e4f608..1d59128536 100644
--- a/man/manpage.txt
+++ b/man/manpage.txt
@@ -241,7 +241,7 @@ MILLER(1)                                                            MILLER(1)
        Miller offers a few different ways to handle reading data files
             which have been compressed.
 
-       * Decompression done within the Miller process itself: `--bz2in` `--gzin` `--zin`
+       * Decompression done within the Miller process itself: `--bz2in` `--gzin` `--zin``--zstdin`
        * Decompression done outside the Miller process: `--prepipe` `--prepipex`
 
        Using `--prepipe` and `--prepipex` you can specify an action to be
@@ -264,7 +264,7 @@ MILLER(1)                                                            MILLER(1)
 
        Lastly, note that if `--prepipe` or `--prepipex` is specified, it replaces any
        decisions that might have been made based on the file suffix. Likewise,
-       `--gzin`/`--bz2in`/`--zin` are ignored if `--prepipe` is also specified.
+       `--gzin`/`--bz2in`/`--zin``--zin` are ignored if `--prepipe` is also specified.
 
        --bz2in                  Uncompress bzip2 within the Miller process. Done by
                                 default if file ends in `.bz2`.
@@ -281,6 +281,8 @@ MILLER(1)                                                            MILLER(1)
                                 `.mlrrc`.
        --prepipe-zcat           Same as `--prepipe zcat`, except this is allowed in
                                 `.mlrrc`.
+       --prepipe-zstdcat        Same as `--prepipe zstdcat`, except this is allowed
+                                in `.mlrrc`.
        --prepipex {decompression command}
                                 Like `--prepipe` with one exception: doesn't insert
                                 `<` between command and filename at runtime. Useful
@@ -289,6 +291,8 @@ MILLER(1)                                                            MILLER(1)
                                 in `.mlrrc` to avoid unexpected code execution.
        --zin                    Uncompress zlib within the Miller process. Done by
                                 default if file ends in `.z`.
+       --zstdin                 Uncompress zstd within the Miller process. Done by
+                                default if file ends in `.zstd`.
 
 1mCSV/TSV-ONLY FLAGS0m
        These are flags which are applicable to CSV format.
diff --git a/man/mlr.1 b/man/mlr.1
index 1d25bb6c02..583b5dc11f 100644
--- a/man/mlr.1
+++ b/man/mlr.1
@@ -304,7 +304,7 @@ Notes:
 Miller offers a few different ways to handle reading data files
 	which have been compressed.
 
-* Decompression done within the Miller process itself: `--bz2in` `--gzin` `--zin`
+* Decompression done within the Miller process itself: `--bz2in` `--gzin` `--zin``--zstdin`
 * Decompression done outside the Miller process: `--prepipe` `--prepipex`
 
 Using `--prepipe` and `--prepipex` you can specify an action to be
@@ -327,7 +327,7 @@ compression (or other) utilities, simply pipe the output:
 
 Lastly, note that if `--prepipe` or `--prepipex` is specified, it replaces any
 decisions that might have been made based on the file suffix. Likewise,
-`--gzin`/`--bz2in`/`--zin` are ignored if `--prepipe` is also specified.
+`--gzin`/`--bz2in`/`--zin``--zin` are ignored if `--prepipe` is also specified.
 
 --bz2in                  Uncompress bzip2 within the Miller process. Done by
                          default if file ends in `.bz2`.
@@ -344,6 +344,8 @@ decisions that might have been made based on the file suffix. Likewise,
                          `.mlrrc`.
 --prepipe-zcat           Same as `--prepipe zcat`, except this is allowed in
                          `.mlrrc`.
+--prepipe-zstdcat        Same as `--prepipe zstdcat`, except this is allowed
+                         in `.mlrrc`.
 --prepipex {decompression command}
                          Like `--prepipe` with one exception: doesn't insert
                          `<` between command and filename at runtime. Useful
@@ -352,6 +354,8 @@ decisions that might have been made based on the file suffix. Likewise,
                          in `.mlrrc` to avoid unexpected code execution.
 --zin                    Uncompress zlib within the Miller process. Done by
                          default if file ends in `.z`.
+--zstdin                 Uncompress zstd within the Miller process. Done by
+                         default if file ends in `.zstd`.
 .fi
 .if n \{\
 .RE
diff --git a/test/cases/io-compressed-input/0014/cmd b/test/cases/io-compressed-input/0014/cmd
new file mode 100644
index 0000000000..f6141361ef
--- /dev/null
+++ b/test/cases/io-compressed-input/0014/cmd
@@ -0,0 +1 @@
+mlr count -g a test/input/medium.zst
diff --git a/test/cases/io-compressed-input/0014/experr b/test/cases/io-compressed-input/0014/experr
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/test/cases/io-compressed-input/0014/expout b/test/cases/io-compressed-input/0014/expout
new file mode 100644
index 0000000000..7dcf142127
--- /dev/null
+++ b/test/cases/io-compressed-input/0014/expout
@@ -0,0 +1,5 @@
+a=pan,count=8
+a=eks,count=10
+a=wye,count=7
+a=zee,count=8
+a=hat,count=7
diff --git a/test/cases/io-compressed-input/0015/cmd b/test/cases/io-compressed-input/0015/cmd
new file mode 100644
index 0000000000..8a6e18c1e2
--- /dev/null
+++ b/test/cases/io-compressed-input/0015/cmd
@@ -0,0 +1 @@
+mlr --zstdin count -g a < test/input/medium.zst
diff --git a/test/cases/io-compressed-input/0015/experr b/test/cases/io-compressed-input/0015/experr
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/test/cases/io-compressed-input/0015/expout b/test/cases/io-compressed-input/0015/expout
new file mode 100644
index 0000000000..7dcf142127
--- /dev/null
+++ b/test/cases/io-compressed-input/0015/expout
@@ -0,0 +1,5 @@
+a=pan,count=8
+a=eks,count=10
+a=wye,count=7
+a=zee,count=8
+a=hat,count=7
diff --git a/test/cases/io-compressed-input/0016/cmd b/test/cases/io-compressed-input/0016/cmd
new file mode 100644
index 0000000000..7d38bc22ac
--- /dev/null
+++ b/test/cases/io-compressed-input/0016/cmd
@@ -0,0 +1 @@
+mlr --zstdin count -g a test/input/medium.zst
diff --git a/test/cases/io-compressed-input/0016/experr b/test/cases/io-compressed-input/0016/experr
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/test/cases/io-compressed-input/0016/expout b/test/cases/io-compressed-input/0016/expout
new file mode 100644
index 0000000000..7dcf142127
--- /dev/null
+++ b/test/cases/io-compressed-input/0016/expout
@@ -0,0 +1,5 @@
+a=pan,count=8
+a=eks,count=10
+a=wye,count=7
+a=zee,count=8
+a=hat,count=7
diff --git a/test/input/medium.zst b/test/input/medium.zst
new file mode 100644
index 0000000000000000000000000000000000000000..f7b5c9a0d729b4df5c16d7bd6767108b4eab3005
GIT binary patch
literal 957
zcmV;u148^LwJ-eyScM$`N>daH7@#T30iBn9E!->i4=+nfDN{;GwEE#G7_I=70FnT2
zg;<7WuTcjK0tyC#L{!PCqsbI4v5N3db#lpQtCfj{=9@+y5~kT?3I_&#v=Q9TB&U`
zDl(5Y#K9I00vH?=ASf6JQ{%Q2+ecn1RWz2W5&F=;fJ4DRXzzL`#311Sh=KwGlY&}%
zQ6nyZC?qgg;UItk0Yia;Awk;Y5Rz45UGv={l;P`kRq}bs5*p5fWPUp_8yfq;;YQ-+
z8bxu10|NsC4x{N?GXF!?NP?;->0DmQ%ZCL$G!UJ)loqv5K?5Cu=+Wj2XhuUOqRlhigtwW5%JQJfT~@(o!Px42*BY0x~U
zm&dd-G2B-D2#cwTi}(e_ip`N32R;VB|mlMy9>n9OyXk;c}|~bc$k_|l?^ynLovO0
f=>7aYPLKWGbLsdn?+i^K;A1?UQbidh3(Fx>)!MFg

literal 0
HcmV?d00001