released 6.5

- faster binary file checking with SIMD (SSE2/AVX2/NEON) UTF-8 validation #421 - new option `--context-separator=SEP` - options `-W` and `-X` now also apply to `--format` fields `%o` and `%O` to output hex - ugrep-indexer option `-z` to index zip/7z/tar/cpio/pax archives no longer indexes hidden directories and files stored in the archive - fix ugrep-indexer option `-z`, which was broken due to a code refactoring mistake in one line of code
Genivia · Aug 23, 2024 · 13fa077 · 13fa077
1 parent d3addf1
commit 13fa077
Show file tree

Hide file tree

Showing 32 changed files with 553 additions and 176 deletions.
diff --git a/README.md b/README.md
@@ -454,9 +454,8 @@ You can build static executables by specifying:
 
     $ ./build.sh --enable-static
 
->[!TIP]
->This may fail if libraries don't link statically, such as brotli.  In that
->case try `./build.sh --enable-static --without-brotli`.
+This may fail if libraries don't link statically, such as brotli.  In that case
+try `./build.sh --enable-static --without-brotli`.
 
 You can build `ugrep` with customized defaults enabled, such as a pager:
 

diff --git a/bin/win32/ug.exe b/bin/win32/ug.exe
diff --git a/bin/win32/ugrep-indexer.exe b/bin/win32/ugrep-indexer.exe
diff --git a/bin/win32/ugrep.exe b/bin/win32/ugrep.exe
diff --git a/bin/win64/ug.exe b/bin/win64/ug.exe
diff --git a/bin/win64/ugrep-indexer.exe b/bin/win64/ugrep-indexer.exe
diff --git a/bin/win64/ugrep.exe b/bin/win64/ugrep.exe
diff --git a/completions/bash/ug b/completions/bash/ug
@@ -314,7 +314,7 @@ cat <<'END'
 -S, --dereference-files When -r is specified, follow symbolic links to files, but not to directories.
 -s, --no-messages Silent mode: nonexistent and unreadable files are ignored and their error messages and warnings are suppressed.
 --save-config[=FILE] [OPTIONS] Save configuration FILE to include OPTIONS.
---separator[=SEP] Use SEP as field separator between file name, line number, column number, byte offset and the matched line.
+--separator[=SEP], --context-separator=SEP Use SEP as field separator between file name, line number, column number, byte offset and the matched line.
 --split Split the -Q query TUI screen on startup.
 --sort[=KEY] Displays matching files in the order specified by KEY in recursive searches.
 --stats Output statistics on the number of files and directories searched and the inclusion and exclusion constraints applied.

diff --git a/completions/fish/ug+.fish b/completions/fish/ug+.fish
@@ -92,7 +92,7 @@ complete -c ug+ -l replace -d 'Replace matching patterns in the output by FORMAT
 complete -c ug+ -s S -l dereference-files -d 'When -r is specified, follow symbolic links to files, but not to directories'
 complete -c ug+ -s s -l no-messages -d 'Silent mode: nonexistent and unreadable files are ignored and their error messages and warnings are suppressed'
 complete -c ug+ -l save-config -d 'Save configuration FILE to include OPTIONS'
-complete -c ug+ -l separator -d 'Use SEP as field separator between file name, line number, column number, byte offset and the matched line'
+complete -c ug+ -l separator -l context-separator -d 'Use SEP as field separator between file name, line number, column number, byte offset and the matched line'
 complete -c ug+ -l split -d 'Split the -Q query TUI screen on startup'
 complete -c ug+ -l sort -xa 'name best size changed created used rname rbest rsize rchanged rcreated rused' -d 'Displays matching files in the order specified by KEY in recursive searches'
 complete -c ug+ -l stats -d 'Output statistics on the number of files and directories searched and the inclusion and exclusion constraints applied'

diff --git a/completions/fish/ug.fish b/completions/fish/ug.fish
@@ -92,7 +92,7 @@ complete -c ug -l replace -d 'Replace matching patterns in the output by FORMAT
 complete -c ug -s S -l dereference-files -d 'When -r is specified, follow symbolic links to files, but not to directories'
 complete -c ug -s s -l no-messages -d 'Silent mode: nonexistent and unreadable files are ignored and their error messages and warnings are suppressed'
 complete -c ug -l save-config -d 'Save configuration FILE to include OPTIONS'
-complete -c ug -l separator -d 'Use SEP as field separator between file name, line number, column number, byte offset and the matched line'
+complete -c ug -l separator -l context-separator -d 'Use SEP as field separator between file name, line number, column number, byte offset and the matched line'
 complete -c ug -l split -d 'Split the -Q query TUI screen on startup'
 complete -c ug -l sort -xa 'name best size changed created used rname rbest rsize rchanged rcreated rused' -d 'Displays matching files in the order specified by KEY in recursive searches'
 complete -c ug -l stats -d 'Output statistics on the number of files and directories searched and the inclusion and exclusion constraints applied'

diff --git a/completions/fish/ugrep+.fish b/completions/fish/ugrep+.fish
@@ -92,7 +92,7 @@ complete -c ugrep+ -l replace -d 'Replace matching patterns in the output by FOR
 complete -c ugrep+ -s S -l dereference-files -d 'When -r is specified, follow symbolic links to files, but not to directories'
 complete -c ugrep+ -s s -l no-messages -d 'Silent mode: nonexistent and unreadable files are ignored and their error messages and warnings are suppressed'
 complete -c ugrep+ -l save-config -d 'Save configuration FILE to include OPTIONS'
-complete -c ugrep+ -l separator -d 'Use SEP as field separator between file name, line number, column number, byte offset and the matched line'
+complete -c ugrep+ -l separator -l context-separator -d 'Use SEP as field separator between file name, line number, column number, byte offset and the matched line'
 complete -c ugrep+ -l split -d 'Split the -Q query TUI screen on startup'
 complete -c ugrep+ -l sort -xa 'name best size changed created used rname rbest rsize rchanged rcreated rused' -d 'Displays matching files in the order specified by KEY in recursive searches'
 complete -c ugrep+ -l stats -d 'Output statistics on the number of files and directories searched and the inclusion and exclusion constraints applied'

diff --git a/completions/fish/ugrep.fish b/completions/fish/ugrep.fish
@@ -92,7 +92,7 @@ complete -c ugrep -l replace -d 'Replace matching patterns in the output by FORM
 complete -c ugrep -s S -l dereference-files -d 'When -r is specified, follow symbolic links to files, but not to directories'
 complete -c ugrep -s s -l no-messages -d 'Silent mode: nonexistent and unreadable files are ignored and their error messages and warnings are suppressed'
 complete -c ugrep -l save-config -d 'Save configuration FILE to include OPTIONS'
-complete -c ugrep -l separator -d 'Use SEP as field separator between file name, line number, column number, byte offset and the matched line'
+complete -c ugrep -l separator -l context-separator -d 'Use SEP as field separator between file name, line number, column number, byte offset and the matched line'
 complete -c ugrep -l split -d 'Split the -Q query TUI screen on startup'
 complete -c ugrep -l sort -xa 'name best size changed created used rname rbest rsize rchanged rcreated rused' -d 'Displays matching files in the order specified by KEY in recursive searches'
 complete -c ugrep -l stats -d 'Output statistics on the number of files and directories searched and the inclusion and exclusion constraints applied'

diff --git a/completions/zsh/_ug b/completions/zsh/_ug
@@ -99,7 +99,7 @@ $matchers{-P,--perl-regexp}'[Interpret PATTERN as a Perl regular expression usin
 {-S,--dereference-files}'[When -r is specified, follow symbolic links to files, but not to directories]'
 {-s,--no-messages}'[Silent mode: nonexistent and unreadable files are ignored and their error messages and warnings are suppressed]'
 --save-config'[Save configuration FILE to include OPTIONS]'
---separator'[Use SEP as field separator between file name, line number, column number, byte offset and the matched line]'
+{--separator,--context-separator=-}'[Use SEP as field separator between file name, line number, column number, byte offset and the matched line]'
 --split'[Split the -Q query TUI screen on startup]'
 --sort=-'[Displays matching files in the order specified by KEY in recursive searches]:KEY:(name best size changed created used rname rbest rsize rchanged rcreated rused)'
 --stats'[Output statistics on the number of files and directories searched and the inclusion and exclusion constraints applied]'

diff --git a/completions/zsh/_ug+ b/completions/zsh/_ug+
@@ -99,7 +99,7 @@ $matchers{-P,--perl-regexp}'[Interpret PATTERN as a Perl regular expression usin
 {-S,--dereference-files}'[When -r is specified, follow symbolic links to files, but not to directories]'
 {-s,--no-messages}'[Silent mode: nonexistent and unreadable files are ignored and their error messages and warnings are suppressed]'
 --save-config'[Save configuration FILE to include OPTIONS]'
---separator'[Use SEP as field separator between file name, line number, column number, byte offset and the matched line]'
+{--separator,--context-separator=-}'[Use SEP as field separator between file name, line number, column number, byte offset and the matched line]'
 --split'[Split the -Q query TUI screen on startup]'
 --sort=-'[Displays matching files in the order specified by KEY in recursive searches]:KEY:(name best size changed created used rname rbest rsize rchanged rcreated rused)'
 --stats'[Output statistics on the number of files and directories searched and the inclusion and exclusion constraints applied]'

diff --git a/completions/zsh/_ugrep b/completions/zsh/_ugrep
@@ -99,7 +99,7 @@ $matchers{-P,--perl-regexp}'[Interpret PATTERN as a Perl regular expression usin
 {-S,--dereference-files}'[When -r is specified, follow symbolic links to files, but not to directories]'
 {-s,--no-messages}'[Silent mode: nonexistent and unreadable files are ignored and their error messages and warnings are suppressed]'
 --save-config'[Save configuration FILE to include OPTIONS]'
---separator'[Use SEP as field separator between file name, line number, column number, byte offset and the matched line]'
+{--separator,--context-separator=-}'[Use SEP as field separator between file name, line number, column number, byte offset and the matched line]'
 --split'[Split the -Q query TUI screen on startup]'
 --sort=-'[Displays matching files in the order specified by KEY in recursive searches]:KEY:(name best size changed created used rname rbest rsize rchanged rcreated rused)'
 --stats'[Output statistics on the number of files and directories searched and the inclusion and exclusion constraints applied]'

diff --git a/completions/zsh/_ugrep+ b/completions/zsh/_ugrep+
@@ -99,7 +99,7 @@ $matchers{-P,--perl-regexp}'[Interpret PATTERN as a Perl regular expression usin
 {-S,--dereference-files}'[When -r is specified, follow symbolic links to files, but not to directories]'
 {-s,--no-messages}'[Silent mode: nonexistent and unreadable files are ignored and their error messages and warnings are suppressed]'
 --save-config'[Save configuration FILE to include OPTIONS]'
---separator'[Use SEP as field separator between file name, line number, column number, byte offset and the matched line]'
+{--separator,--context-separator=-}'[Use SEP as field separator between file name, line number, column number, byte offset and the matched line]'
 --split'[Split the -Q query TUI screen on startup]'
 --sort=-'[Displays matching files in the order specified by KEY in recursive searches]:KEY:(name best size changed created used rname rbest rsize rchanged rcreated rused)'
 --stats'[Output statistics on the number of files and directories searched and the inclusion and exclusion constraints applied]'

diff --git a/configure b/configure
@@ -1,6 +1,6 @@
 #! /bin/sh
 # Guess values for system-dependent variables and create Makefiles.
-# Generated by GNU Autoconf 2.72 for ugrep 6.4.1.
+# Generated by GNU Autoconf 2.72 for ugrep 6.5.0.
 #
 # Report bugs to <https://github.com/Genivia/ugrep/issues>.
 #
@@ -606,8 +606,8 @@ MAKEFLAGS=
 # Identity of this package.
 PACKAGE_NAME='ugrep'
 PACKAGE_TARNAME='ugrep'
-PACKAGE_VERSION='6.4.1'
-PACKAGE_STRING='ugrep 6.4.1'
+PACKAGE_VERSION='6.5.0'
+PACKAGE_STRING='ugrep 6.5.0'
 PACKAGE_BUGREPORT='https://github.com/Genivia/ugrep/issues'
 PACKAGE_URL='https://ugrep.com'
 
@@ -1382,7 +1382,7 @@ if test "$ac_init_help" = "long"; then
   # Omit some internal or obsolete options to make the list less imposing.
   # This message is too long to be a string in the A/UX 3.1 sh.
   cat <<_ACEOF
-'configure' configures ugrep 6.4.1 to adapt to many kinds of systems.
+'configure' configures ugrep 6.5.0 to adapt to many kinds of systems.
 
 Usage: $0 [OPTION]... [VAR=VALUE]...
 
@@ -1453,7 +1453,7 @@ fi
 
 if test -n "$ac_init_help"; then
   case $ac_init_help in
-     short | recursive ) echo "Configuration of ugrep 6.4.1:";;
+     short | recursive ) echo "Configuration of ugrep 6.5.0:";;
    esac
   cat <<\_ACEOF
 
@@ -1489,14 +1489,14 @@ Optional Features:
 Optional Packages:
   --with-PACKAGE[=ARG]    use PACKAGE [ARG=yes]
   --without-PACKAGE       do not use PACKAGE (same as --with-PACKAGE=no)
-  --with-pcre2=DIR        root directory path of PCRE2 installation [defaults to
-                          /usr/local or /usr if not found in /usr/local]
+  --with-pcre2=DIR        root directory path of PCRE2 installation [defaults
+                          to /usr/local or /usr if not found in /usr/local]
   --without-pcre2         to disable PCRE2 usage completely
   --with-boost-regex[=special-lib]
-                          use the Regex library from boost - it is possible to
-                          specify a path to include/boost and
-                          lib/libboost_regex-mt e.g.
+                          use the Boost Regex library, optionally specify a
+                          path to include/boost and lib/libboost_regex-mt e.g.
                           --with-boost-regex=/opt/local
+  --without-boost-regex   to disable Boost Regex usage completely
   --with-zlib=DIR         root directory path of zlib installation [defaults to
                           /usr/local or /usr if not found in /usr/local]
   --without-zlib          to disable zlib usage completely
@@ -1630,7 +1630,7 @@ fi
 test -n "$ac_init_help" && exit $ac_status
 if $ac_init_version; then
   cat <<\_ACEOF
-ugrep configure 6.4.1
+ugrep configure 6.5.0
 generated by GNU Autoconf 2.72
 
 Copyright (C) 2023 Free Software Foundation, Inc.
@@ -2184,7 +2184,7 @@ cat >config.log <<_ACEOF
 This file contains any messages produced by compilers while
 running configure, to aid debugging if configure makes a mistake.
 
-It was created by ugrep $as_me 6.4.1, which was
+It was created by ugrep $as_me 6.5.0, which was
 generated by GNU Autoconf 2.72.  Invocation command line was
 
   $ $0$ac_configure_args_raw
@@ -3694,7 +3694,7 @@ fi
 
 # Define the identity of the package.
  PACKAGE='ugrep'
- VERSION='6.4.1'
+ VERSION='6.5.0'
 
 
 printf "%s\n" "#define PACKAGE \"$PACKAGE\"" >>confdefs.h
@@ -11790,7 +11790,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
 # report actual input values of CONFIG_FILES etc. instead of their
 # values after options handling.
 ac_log="
-This file was extended by ugrep $as_me 6.4.1, which was
+This file was extended by ugrep $as_me 6.5.0, which was
 generated by GNU Autoconf 2.72.  Invocation command line was
 
   CONFIG_FILES    = $CONFIG_FILES
@@ -11863,7 +11863,7 @@ ac_cs_config_escaped=`printf "%s\n" "$ac_cs_config" | sed "s/^ //; s/'/'\\\\\\\\
 cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
 ac_cs_config='$ac_cs_config_escaped'
 ac_cs_version="\\
-ugrep config.status 6.4.1
+ugrep config.status 6.5.0
 configured by $0, generated by GNU Autoconf 2.72,
   with options \\"\$ac_cs_config\\"
 

diff --git a/configure.ac b/configure.ac
@@ -1,4 +1,4 @@
-AC_INIT([ugrep],[6.4.1],[https://github.com/Genivia/ugrep/issues],[ugrep],[https://ugrep.com])
+AC_INIT([ugrep],[6.5.0],[https://github.com/Genivia/ugrep/issues],[ugrep],[https://ugrep.com])
 AM_INIT_AUTOMAKE([foreign subdir-objects dist-xz no-dist-gzip])
 AC_CONFIG_HEADERS([config.h])
 AC_COPYRIGHT([Copyright (C) 2019-2024 Robert van Engelen, Genivia Inc.])

diff --git a/include/reflex/simd.h b/include/reflex/simd.h
@@ -140,8 +140,11 @@ inline uint32_t popcountl(uint64_t x)
 #endif
 
 // Partially count newlines in string b up to e, updates b close to e with uncounted part
-extern size_t simd_nlcount_avx2(const char *&b, const char *e);
-extern size_t simd_nlcount_avx512bw(const char *&b, const char *e);
+extern size_t simd_nlcount_avx2(const char *& b, const char *e);
+extern size_t simd_nlcount_avx512bw(const char *& b, const char *e);
+
+// Partially check if valid UTF-8 encoding
+extern bool simd_isutf8_avx2(const char *& b, const char *e);
 
 } // namespace reflex
 
@@ -152,6 +155,9 @@ namespace reflex {
 // Count newlines in string s up to t
 extern size_t nlcount(const char *s, const char *t);
 
+// Check if valid UTF-8 encoding and does not include a NUL, but accept surrogates and 3/4 byte overlongs
+extern bool isutf8(const char *s, const char *e);
+
 } // namespace reflex
 
 #endif