Merge remote-tracking branch 'refs/remotes/origin/dev'

inikep · Dec 3, 2015 · 9e94145 · 9e94145
2 parents 502898e + 4301f27
commit 9e94145
Show file tree

Hide file tree

Showing 17 changed files with 1,640 additions and 920 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1,4 +1,5 @@
 # Directories
+_lz5bench/
 _codelite/
 cmake_unofficial/
 contrib/
@@ -7,6 +8,9 @@ examples/
 versionsTest/
 visual/
 
+# Archives
+*.zip 
+
 # Object files
 *.o
 *.ko

diff --git a/Makefile b/Makefile
@@ -31,7 +31,7 @@
 # ################################################################
 
 # Version number
-export VERSION=131
+export VERSION=132
 export RELEASE=r$(VERSION)
 
 DESTDIR?=

diff --git a/NEWS b/NEWS
@@ -1,3 +1,8 @@
+r132
+- improved compression ratio
+- added: new parsers: LZ5HC_fast, LZ5HC_price_fast, LZ5HC_lowest_price
+- added: a special 1-byte codeword for the last occured offset
+- added: support for 3-byte long matches (MINMATCH = 3)
+
 r131
 The first release based on LZ4 r132 dev
-
diff --git a/README.md b/README.md
@@ -7,17 +7,18 @@ This is caused mainly because of 22-bit dictionary instead of 16-bit in LZ4.
 LZ5 uses different output codewords and is not compatible with LZ4. LZ4 output codewords are 3 byte long (24-bit) and look as follows:
 - LLLL_MMMM OOOOOOOO OOOOOOOO - 16-bit offset, 4-bit match length, 4-bit literal length 
 
-LZ5 uses 3 types of codewords from 2 to 4 bytes long:
+LZ5 uses 4 types of codewords from 1 to 4 bytes long:
 - 1_OO_LL_MMM OOOOOOOO - 10-bit offset, 3-bit match length, 2-bit literal length
 - 00_LLL_MMM OOOOOOOO OOOOOOOO - 16-bit offset, 3-bit match length, 3-bit literal length
-- 01_LLL_MMM OOOOOOOO OOOOOOOO OOOOOOOO - 24-bit offset, 3-bit match length, 3-bit literal length 
+- 010_LL_MMM OOOOOOOO OOOOOOOO OOOOOOOO - 24-bit offset, 3-bit match length, 2-bit literal length 
+- 011_LL_MMM - last offset, 3-bit match length, 2-bit literal length
 
 [LZ4]: https://github.com/Cyan4973/lz4
 
 Benchmarks
 -------------------------
 
-In our experiments decompression speed of LZ5 is from 650-950 MB/s. It's slower than LZ4 but much faster than zstd and brotli.
+In our experiments decompression speed of LZ5 is from 600-1600 MB/s. It's slower than LZ4 but much faster than zstd and brotli.
 With the compresion ratio is opposite: LZ5 is better than LZ4 but worse than zstd and brotli.
 
 | Compressor name             | Compression| Decompress.| Compr. size | Ratio |
@@ -32,22 +33,28 @@ With the compresion ratio is opposite: LZ5 is better than LZ4 but worse than zst
 | lz4hc r131 -11              |    20 MB/s |  1969 MB/s |    54751363 | 52.21 |
 | lz4hc r131 -13              |    17 MB/s |  1969 MB/s |    54744790 | 52.21 |
 | lz4hc r131 -15              |    14 MB/s |  2007 MB/s |    54741827 | 52.21 |
-| lz5 r131                    |   195 MB/s |   939 MB/s |    55884927 | 53.30 |
-| lz5hc r131 -1               |    32 MB/s |   742 MB/s |    52927122 | 50.48 |
-| lz5hc r131 -3               |    20 MB/s |   716 MB/s |    50970192 | 48.61 |
-| lz5hc r131 -5               |    10 MB/s |   701 MB/s |    49970285 | 47.66 |
-| lz5hc r131 -7               |  5.54 MB/s |   682 MB/s |    49541511 | 47.25 |
-| lz5hc r131 -9               |  2.69 MB/s |   673 MB/s |    49346894 | 47.06 |
-| lz5hc r131 -11              |  1.36 MB/s |   664 MB/s |    49266526 | 46.98 |
-| zstd v0.3                   |   257 MB/s |   547 MB/s |    51231016 | 48.86 |
-| zstd_HC v0.3 -1             |   257 MB/s |   553 MB/s |    51231016 | 48.86 |
-| zstd_HC v0.3 -3             |    76 MB/s |   417 MB/s |    46774383 | 44.61 |
-| zstd_HC v0.3 -5             |    40 MB/s |   476 MB/s |    45628362 | 43.51 |
-| zstd_HC v0.3 -9             |    14 MB/s |   485 MB/s |    44840562 | 42.76 |
-| zstd_HC v0.3 -13            |  9.34 MB/s |   469 MB/s |    43114895 | 41.12 |
-| zstd_HC v0.3 -17            |  6.02 MB/s |   463 MB/s |    42989971 | 41.00 |
-| zstd_HC v0.3 -21            |  3.35 MB/s |   461 MB/s |    42956964 | 40.97 |
-| zstd_HC v0.3 -23            |  2.33 MB/s |   463 MB/s |    42934217 | 40.95 |
+| lz5 r132                    |   180 MB/s |   877 MB/s |    56183327 | 53.58 |
+| lz5hc r132 level 1          |   453 MB/s |  1649 MB/s |    68770655 | 65.58 |
+| lz5hc r132 level 2          |   341 MB/s |  1533 MB/s |    65201626 | 62.18 |
+| lz5hc r132 level 3          |   222 MB/s |  1267 MB/s |    61423270 | 58.58 |
+| lz5hc r132 level 4          |   122 MB/s |   892 MB/s |    55011906 | 52.46 |
+| lz5hc r132 level 5          |    92 MB/s |   784 MB/s |    52790905 | 50.35 |
+| lz5hc r132 level 6          |    40 MB/s |   872 MB/s |    52561673 | 50.13 |
+| lz5hc r132 level 7          |    30 MB/s |   825 MB/s |    50947061 | 48.59 |
+| lz5hc r132 level 8          |    21 MB/s |   771 MB/s |    50049555 | 47.73 |
+| lz5hc r132 level 9          |    16 MB/s |   702 MB/s |    48718531 | 46.46 |
+| lz5hc r132 level 10         |    12 MB/s |   670 MB/s |    48109030 | 45.88 |
+| lz5hc r132 level 11         |  6.60 MB/s |   592 MB/s |    47639520 | 45.43 |
+| lz5hc r132 level 12         |  3.22 MB/s |   670 MB/s |    47461368 | 45.26 |
+| zstd_HC v0.3.6 level 1      |   250 MB/s |   529 MB/s |    51230550 | 48.86 |
+| zstd_HC v0.3.6 level 2      |   186 MB/s |   498 MB/s |    49678572 | 47.38 |
+| zstd_HC v0.3.6 level 3      |    90 MB/s |   484 MB/s |    48838293 | 46.58 |
+| zstd_HC v0.3.6 level 5      |    61 MB/s |   467 MB/s |    46480999 | 44.33 |
+| zstd_HC v0.3.6 level 7      |    28 MB/s |   480 MB/s |    44803941 | 42.73 |
+| zstd_HC v0.3.6 level 9      |    15 MB/s |   497 MB/s |    43899996 | 41.87 |
+| zstd_HC v0.3.6 level 12     |    11 MB/s |   505 MB/s |    42402232 | 40.44 |
+| zstd_HC v0.3.6 level 16     |  2.29 MB/s |   499 MB/s |    42122327 | 40.17 |
+| zstd_HC v0.3.6 level 20     |  1.65 MB/s |   454 MB/s |    41884658 | 39.94 |
 | brotli 2015-10-29 -1        |    86 MB/s |   208 MB/s |    47882059 | 45.66 |
 | brotli 2015-10-29 -3        |    60 MB/s |   214 MB/s |    47451223 | 45.25 |
 | brotli 2015-10-29 -5        |    17 MB/s |   217 MB/s |    43363897 | 41.36 |

diff --git a/lib/Makefile b/lib/Makefile
@@ -31,7 +31,7 @@
 # ################################################################
 
 # Version numbers
-VERSION?= 131
+VERSION?= 132
 LIBVER_MAJOR:=`sed -n '/define LZ5_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < lz5.h`
 LIBVER_MINOR:=`sed -n '/define LZ5_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < lz5.h`
 LIBVER_PATCH:=`sed -n '/define LZ5_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < lz5.h`