Skip to content

Commit

Permalink
flate: Improve level 5+6 compression (#656)
Browse files Browse the repository at this point in the history
Skip 2 bytes when doing end-of-match search.

This allows better matches to be picked up further forward, adding up to 2 literals, but giving a better match.

Level 5:

nyc: 779171549 -> 767765165
github-june: 943488021 -> 938777887
github-ranks: 435111686 -> 435008334
sofia: 3083510553 -> 3048677443

level6, nyc: 768119006 -> 753545897

Performance unchanged.
  • Loading branch information
klauspost authored Sep 7, 2022
1 parent 7bab791 commit b95ee96
Show file tree
Hide file tree
Showing 2 changed files with 26 additions and 10 deletions.
15 changes: 11 additions & 4 deletions flate/level5.go
Original file line number Diff line number Diff line change
Expand Up @@ -191,14 +191,21 @@ func (e *fastEncL5) Encode(dst *tokens, src []byte) {

// Try to locate a better match by checking the end of best match...
if sAt := s + l; l < 30 && sAt < sLimit {
// Allow some bytes at the beginning to mismatch.
// Sweet spot is 2/3 bytes depending on input.
// 3 is only a little better when it is but sometimes a lot worse.
// The skipped bytes are tested in Extend backwards,
// and still picked up as part of the match if they do.
const skipBeginning = 2
eLong := e.bTable[hash7(load6432(src, sAt), tableBits)].Cur.offset
// Test current
t2 := eLong - e.cur - l
off := s - t2
t2 := eLong - e.cur - l + skipBeginning
s2 := s + skipBeginning
off := s2 - t2
if t2 >= 0 && off < maxMatchOffset && off > 0 {
if l2 := e.matchlenLong(s, t2, src); l2 > l {
if l2 := e.matchlenLong(s2, t2, src); l2 > l {
t = t2
l = l2
s = s2
}
}
}
Expand Down
21 changes: 15 additions & 6 deletions flate/level6.go
Original file line number Diff line number Diff line change
Expand Up @@ -213,24 +213,33 @@ func (e *fastEncL6) Encode(dst *tokens, src []byte) {

// Try to locate a better match by checking the end-of-match...
if sAt := s + l; sAt < sLimit {
// Allow some bytes at the beginning to mismatch.
// Sweet spot is 2/3 bytes depending on input.
// 3 is only a little better when it is but sometimes a lot worse.
// The skipped bytes are tested in Extend backwards,
// and still picked up as part of the match if they do.
const skipBeginning = 2
eLong := &e.bTable[hash7(load6432(src, sAt), tableBits)]
// Test current
t2 := eLong.Cur.offset - e.cur - l
off := s - t2
t2 := eLong.Cur.offset - e.cur - l + skipBeginning
s2 := s + skipBeginning
off := s2 - t2
if off < maxMatchOffset {
if off > 0 && t2 >= 0 {
if l2 := e.matchlenLong(s, t2, src); l2 > l {
if l2 := e.matchlenLong(s2, t2, src); l2 > l {
t = t2
l = l2
s = s2
}
}
// Test next:
t2 = eLong.Prev.offset - e.cur - l
off := s - t2
t2 = eLong.Prev.offset - e.cur - l + skipBeginning
off := s2 - t2
if off > 0 && off < maxMatchOffset && t2 >= 0 {
if l2 := e.matchlenLong(s, t2, src); l2 > l {
if l2 := e.matchlenLong(s2, t2, src); l2 > l {
t = t2
l = l2
s = s2
}
}
}
Expand Down

0 comments on commit b95ee96

Please sign in to comment.