mirror of https://github.com/grafana/loki
Bump github.com/minio/minio-go/v7 from 7.0.45 to 7.0.52 (#9208)
Bumps [github.com/minio/minio-go/v7](https://github.com/minio/minio-go) from 7.0.45 to 7.0.52. <details> <summary>Release notes</summary> <p><em>Sourced from <a href="https://github.com/minio/minio-go/releases">github.com/minio/minio-go/v7's releases</a>.</em></p> <blockquote> <h2>Bugfix release</h2> <h2>What's Changed</h2> <ul> <li>Revert "Add missing aws-chunked transfer encoding (<a href="https://redirect.github.com/minio/minio-go/issues/1803">#1803</a>)" by <a href="https://github.com/harshavardhana"><code>@harshavardhana</code></a> in <a href="https://redirect.github.com/minio/minio-go/pull/1804">minio/minio-go#1804</a></li> </ul> <p><strong>Full Changelog</strong>: <a href="https://github.com/minio/minio-go/compare/v7.0.51...v7.0.52">https://github.com/minio/minio-go/compare/v7.0.51...v7.0.52</a></p> <h2>Bugfix release</h2> <h2>What's Changed</h2> <ul> <li>Fix listobjects with metadata by <a href="https://github.com/klauspost"><code>@klauspost</code></a> in <a href="https://redirect.github.com/minio/minio-go/pull/1795">minio/minio-go#1795</a></li> <li>Add missing aws-chunked transfer encoding by <a href="https://github.com/klauspost"><code>@klauspost</code></a> in <a href="https://redirect.github.com/minio/minio-go/pull/1803">minio/minio-go#1803</a></li> </ul> <p><strong>Full Changelog</strong>: <a href="https://github.com/minio/minio-go/compare/v7.0.50...v7.0.51">https://github.com/minio/minio-go/compare/v7.0.50...v7.0.51</a></p> <h2>Bugfix release</h2> <h2>What's Changed</h2> <ul> <li>Update minio.Core API by <a href="https://github.com/donatello"><code>@donatello</code></a> in <a href="https://redirect.github.com/minio/minio-go/pull/1764">minio/minio-go#1764</a></li> <li>remove sse-s3 header in completeMultipartUpload by <a href="https://github.com/harshavardhana"><code>@harshavardhana</code></a> in <a href="https://redirect.github.com/minio/minio-go/pull/1777">minio/minio-go#1777</a></li> <li>upgrade all deps by <a href="https://github.com/harshavardhana"><code>@harshavardhana</code></a> in <a href="https://redirect.github.com/minio/minio-go/pull/1778">minio/minio-go#1778</a></li> <li>allow setting region via custom function by <a href="https://github.com/harshavardhana"><code>@harshavardhana</code></a> in <a href="https://redirect.github.com/minio/minio-go/pull/1786">minio/minio-go#1786</a></li> <li>allow virtual host style for GetBucketLocation API, if requested by <a href="https://github.com/harshavardhana"><code>@harshavardhana</code></a> in <a href="https://redirect.github.com/minio/minio-go/pull/1785">minio/minio-go#1785</a></li> <li>Add ap-south-2 to endpoints by <a href="https://github.com/Prajithp"><code>@Prajithp</code></a> in <a href="https://redirect.github.com/minio/minio-go/pull/1781">minio/minio-go#1781</a></li> </ul> <h2>New Contributors</h2> <ul> <li><a href="https://github.com/Prajithp"><code>@Prajithp</code></a> made their first contribution in <a href="https://redirect.github.com/minio/minio-go/pull/1781">minio/minio-go#1781</a></li> </ul> <p><strong>Full Changelog</strong>: <a href="https://github.com/minio/minio-go/compare/v7.0.49...v7.0.50">https://github.com/minio/minio-go/compare/v7.0.49...v7.0.50</a></p> <h2>Bugfix Release</h2> <h2>What's Changed</h2> <ul> <li>support If-Match/If-None-Match for PUT operations by <a href="https://github.com/harshavardhana"><code>@harshavardhana</code></a> in <a href="https://redirect.github.com/minio/minio-go/pull/1772">minio/minio-go#1772</a></li> <li>upgrade minio-go deps by <a href="https://github.com/harshavardhana"><code>@harshavardhana</code></a> in <a href="https://redirect.github.com/minio/minio-go/pull/1774">minio/minio-go#1774</a></li> </ul> <p><strong>Full Changelog</strong>: <a href="https://github.com/minio/minio-go/compare/v7.0.48...v7.0.49">https://github.com/minio/minio-go/compare/v7.0.48...v7.0.49</a></p> <h2>Bugfix Release</h2> <h2>What's Changed</h2> <ul> <li>Fix sharing of MD5 hasher by <a href="https://github.com/klauspost"><code>@klauspost</code></a> in <a href="https://redirect.github.com/minio/minio-go/pull/1753">minio/minio-go#1753</a></li> <li>feat(<a href="https://redirect.github.com/minio/minio-go/issues/1755">#1755</a>): GetObject supports overriding response header values by <a href="https://github.com/reedchan7"><code>@reedchan7</code></a> in <a href="https://redirect.github.com/minio/minio-go/pull/1756">minio/minio-go#1756</a></li> <li>Snowball: If Modtime unset, use current time by <a href="https://github.com/klauspost"><code>@klauspost</code></a> in <a href="https://redirect.github.com/minio/minio-go/pull/1763">minio/minio-go#1763</a></li> <li>Bump Go versions by <a href="https://github.com/klauspost"><code>@klauspost</code></a> in <a href="https://redirect.github.com/minio/minio-go/pull/1768">minio/minio-go#1768</a></li> <li>feat: add aws ap-southeast-4 region by <a href="https://github.com/ianmuge"><code>@ianmuge</code></a> in <a href="https://redirect.github.com/minio/minio-go/pull/1767">minio/minio-go#1767</a></li> <li>STSCertificateIdentity: Init request form by <a href="https://github.com/klauspost"><code>@klauspost</code></a> in <a href="https://redirect.github.com/minio/minio-go/pull/1770">minio/minio-go#1770</a></li> </ul> <h2>New Contributors</h2> <ul> <li><a href="https://github.com/reedchan7"><code>@reedchan7</code></a> made their first contribution in <a href="https://redirect.github.com/minio/minio-go/pull/1756">minio/minio-go#1756</a></li> <li><a href="https://github.com/ianmuge"><code>@ianmuge</code></a> made their first contribution in <a href="https://redirect.github.com/minio/minio-go/pull/1767">minio/minio-go#1767</a></li> </ul> <!-- raw HTML omitted --> </blockquote> <p>... (truncated)</p> </details> <details> <summary>Commits</summary> <ul> <li><a href="pull/9245/head11eb82d85f"><code>11eb82d</code></a> Revert "Add missing aws-chunked transfer encoding (<a href="https://redirect.github.com/minio/minio-go/issues/1803">#1803</a>)" (<a href="https://redirect.github.com/minio/minio-go/issues/1804">#1804</a>)</li> <li><a href="7bf8079be6"><code>7bf8079</code></a> Update version to next release</li> <li><a href="8175bf7bda"><code>8175bf7</code></a> Add missing aws-chunked transfer encoding (<a href="https://redirect.github.com/minio/minio-go/issues/1803">#1803</a>)</li> <li><a href="5b15988473"><code>5b15988</code></a> Fix listobjects with metadata (<a href="https://redirect.github.com/minio/minio-go/issues/1795">#1795</a>)</li> <li><a href="6fcbe6de97"><code>6fcbe6d</code></a> Update version to next release</li> <li><a href="7c9b5ff1c0"><code>7c9b5ff</code></a> fix: all the linter issues reported</li> <li><a href="5d09d8f78b"><code>5d09d8f</code></a> Add ap-south-2 to endpoints (<a href="https://redirect.github.com/minio/minio-go/issues/1781">#1781</a>)</li> <li><a href="24cdd7fc94"><code>24cdd7f</code></a> allow virtual host style for GetBucketLocation API, if requested (<a href="https://redirect.github.com/minio/minio-go/issues/1785">#1785</a>)</li> <li><a href="5384d69d15"><code>5384d69</code></a> allow setting region via custom function (<a href="https://redirect.github.com/minio/minio-go/issues/1786">#1786</a>)</li> <li><a href="9948a7b56c"><code>9948a7b</code></a> upgrade all deps (<a href="https://redirect.github.com/minio/minio-go/issues/1778">#1778</a>)</li> <li>Additional commits viewable in <a href="https://github.com/minio/minio-go/compare/v7.0.45...v7.0.52">compare view</a></li> </ul> </details> <br /> [](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) --- <details> <summary>Dependabot commands and options</summary> <br /> You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself) </details> Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Michel Hollands <42814411+MichelHollands@users.noreply.github.com>
dependabot[bot]
3 years ago
committed by
GitHub
parent
8c0e4fbe83
commit
c91bfdf6fd
@ -0,0 +1,331 @@ |
||||
// Copyright (c) 2022+ Klaus Post. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package s2 |
||||
|
||||
import ( |
||||
"bytes" |
||||
"encoding/binary" |
||||
"sync" |
||||
) |
||||
|
||||
const ( |
||||
// MinDictSize is the minimum dictionary size when repeat has been read.
|
||||
MinDictSize = 16 |
||||
|
||||
// MaxDictSize is the maximum dictionary size when repeat has been read.
|
||||
MaxDictSize = 65536 |
||||
|
||||
// MaxDictSrcOffset is the maximum offset where a dictionary entry can start.
|
||||
MaxDictSrcOffset = 65535 |
||||
) |
||||
|
||||
// Dict contains a dictionary that can be used for encoding and decoding s2
|
||||
type Dict struct { |
||||
dict []byte |
||||
repeat int // Repeat as index of dict
|
||||
|
||||
fast, better, best sync.Once |
||||
fastTable *[1 << 14]uint16 |
||||
|
||||
betterTableShort *[1 << 14]uint16 |
||||
betterTableLong *[1 << 17]uint16 |
||||
|
||||
bestTableShort *[1 << 16]uint32 |
||||
bestTableLong *[1 << 19]uint32 |
||||
} |
||||
|
||||
// NewDict will read a dictionary.
|
||||
// It will return nil if the dictionary is invalid.
|
||||
func NewDict(dict []byte) *Dict { |
||||
if len(dict) == 0 { |
||||
return nil |
||||
} |
||||
var d Dict |
||||
// Repeat is the first value of the dict
|
||||
r, n := binary.Uvarint(dict) |
||||
if n <= 0 { |
||||
return nil |
||||
} |
||||
dict = dict[n:] |
||||
d.dict = dict |
||||
if cap(d.dict) < len(d.dict)+16 { |
||||
d.dict = append(make([]byte, 0, len(d.dict)+16), d.dict...) |
||||
} |
||||
if len(dict) < MinDictSize || len(dict) > MaxDictSize { |
||||
return nil |
||||
} |
||||
d.repeat = int(r) |
||||
if d.repeat > len(dict) { |
||||
return nil |
||||
} |
||||
return &d |
||||
} |
||||
|
||||
// Bytes will return a serialized version of the dictionary.
|
||||
// The output can be sent to NewDict.
|
||||
func (d *Dict) Bytes() []byte { |
||||
dst := make([]byte, binary.MaxVarintLen16+len(d.dict)) |
||||
return append(dst[:binary.PutUvarint(dst, uint64(d.repeat))], d.dict...) |
||||
} |
||||
|
||||
// MakeDict will create a dictionary.
|
||||
// 'data' must be at least MinDictSize.
|
||||
// If data is longer than MaxDictSize only the last MaxDictSize bytes will be used.
|
||||
// If searchStart is set the start repeat value will be set to the last
|
||||
// match of this content.
|
||||
// If no matches are found, it will attempt to find shorter matches.
|
||||
// This content should match the typical start of a block.
|
||||
// If at least 4 bytes cannot be matched, repeat is set to start of block.
|
||||
func MakeDict(data []byte, searchStart []byte) *Dict { |
||||
if len(data) == 0 { |
||||
return nil |
||||
} |
||||
if len(data) > MaxDictSize { |
||||
data = data[len(data)-MaxDictSize:] |
||||
} |
||||
var d Dict |
||||
dict := data |
||||
d.dict = dict |
||||
if cap(d.dict) < len(d.dict)+16 { |
||||
d.dict = append(make([]byte, 0, len(d.dict)+16), d.dict...) |
||||
} |
||||
if len(dict) < MinDictSize { |
||||
return nil |
||||
} |
||||
|
||||
// Find the longest match possible, last entry if multiple.
|
||||
for s := len(searchStart); s > 4; s-- { |
||||
if idx := bytes.LastIndex(data, searchStart[:s]); idx >= 0 && idx <= len(data)-8 { |
||||
d.repeat = idx |
||||
break |
||||
} |
||||
} |
||||
|
||||
return &d |
||||
} |
||||
|
||||
// Encode returns the encoded form of src. The returned slice may be a sub-
|
||||
// slice of dst if dst was large enough to hold the entire encoded block.
|
||||
// Otherwise, a newly allocated slice will be returned.
|
||||
//
|
||||
// The dst and src must not overlap. It is valid to pass a nil dst.
|
||||
//
|
||||
// The blocks will require the same amount of memory to decode as encoding,
|
||||
// and does not make for concurrent decoding.
|
||||
// Also note that blocks do not contain CRC information, so corruption may be undetected.
|
||||
//
|
||||
// If you need to encode larger amounts of data, consider using
|
||||
// the streaming interface which gives all of these features.
|
||||
func (d *Dict) Encode(dst, src []byte) []byte { |
||||
if n := MaxEncodedLen(len(src)); n < 0 { |
||||
panic(ErrTooLarge) |
||||
} else if cap(dst) < n { |
||||
dst = make([]byte, n) |
||||
} else { |
||||
dst = dst[:n] |
||||
} |
||||
|
||||
// The block starts with the varint-encoded length of the decompressed bytes.
|
||||
dstP := binary.PutUvarint(dst, uint64(len(src))) |
||||
|
||||
if len(src) == 0 { |
||||
return dst[:dstP] |
||||
} |
||||
if len(src) < minNonLiteralBlockSize { |
||||
dstP += emitLiteral(dst[dstP:], src) |
||||
return dst[:dstP] |
||||
} |
||||
n := encodeBlockDictGo(dst[dstP:], src, d) |
||||
if n > 0 { |
||||
dstP += n |
||||
return dst[:dstP] |
||||
} |
||||
// Not compressible
|
||||
dstP += emitLiteral(dst[dstP:], src) |
||||
return dst[:dstP] |
||||
} |
||||
|
||||
// EncodeBetter returns the encoded form of src. The returned slice may be a sub-
|
||||
// slice of dst if dst was large enough to hold the entire encoded block.
|
||||
// Otherwise, a newly allocated slice will be returned.
|
||||
//
|
||||
// EncodeBetter compresses better than Encode but typically with a
|
||||
// 10-40% speed decrease on both compression and decompression.
|
||||
//
|
||||
// The dst and src must not overlap. It is valid to pass a nil dst.
|
||||
//
|
||||
// The blocks will require the same amount of memory to decode as encoding,
|
||||
// and does not make for concurrent decoding.
|
||||
// Also note that blocks do not contain CRC information, so corruption may be undetected.
|
||||
//
|
||||
// If you need to encode larger amounts of data, consider using
|
||||
// the streaming interface which gives all of these features.
|
||||
func (d *Dict) EncodeBetter(dst, src []byte) []byte { |
||||
if n := MaxEncodedLen(len(src)); n < 0 { |
||||
panic(ErrTooLarge) |
||||
} else if len(dst) < n { |
||||
dst = make([]byte, n) |
||||
} |
||||
|
||||
// The block starts with the varint-encoded length of the decompressed bytes.
|
||||
dstP := binary.PutUvarint(dst, uint64(len(src))) |
||||
|
||||
if len(src) == 0 { |
||||
return dst[:dstP] |
||||
} |
||||
if len(src) < minNonLiteralBlockSize { |
||||
dstP += emitLiteral(dst[dstP:], src) |
||||
return dst[:dstP] |
||||
} |
||||
n := encodeBlockBetterDict(dst[dstP:], src, d) |
||||
if n > 0 { |
||||
dstP += n |
||||
return dst[:dstP] |
||||
} |
||||
// Not compressible
|
||||
dstP += emitLiteral(dst[dstP:], src) |
||||
return dst[:dstP] |
||||
} |
||||
|
||||
// EncodeBest returns the encoded form of src. The returned slice may be a sub-
|
||||
// slice of dst if dst was large enough to hold the entire encoded block.
|
||||
// Otherwise, a newly allocated slice will be returned.
|
||||
//
|
||||
// EncodeBest compresses as good as reasonably possible but with a
|
||||
// big speed decrease.
|
||||
//
|
||||
// The dst and src must not overlap. It is valid to pass a nil dst.
|
||||
//
|
||||
// The blocks will require the same amount of memory to decode as encoding,
|
||||
// and does not make for concurrent decoding.
|
||||
// Also note that blocks do not contain CRC information, so corruption may be undetected.
|
||||
//
|
||||
// If you need to encode larger amounts of data, consider using
|
||||
// the streaming interface which gives all of these features.
|
||||
func (d *Dict) EncodeBest(dst, src []byte) []byte { |
||||
if n := MaxEncodedLen(len(src)); n < 0 { |
||||
panic(ErrTooLarge) |
||||
} else if len(dst) < n { |
||||
dst = make([]byte, n) |
||||
} |
||||
|
||||
// The block starts with the varint-encoded length of the decompressed bytes.
|
||||
dstP := binary.PutUvarint(dst, uint64(len(src))) |
||||
|
||||
if len(src) == 0 { |
||||
return dst[:dstP] |
||||
} |
||||
if len(src) < minNonLiteralBlockSize { |
||||
dstP += emitLiteral(dst[dstP:], src) |
||||
return dst[:dstP] |
||||
} |
||||
n := encodeBlockBest(dst[dstP:], src, d) |
||||
if n > 0 { |
||||
dstP += n |
||||
return dst[:dstP] |
||||
} |
||||
// Not compressible
|
||||
dstP += emitLiteral(dst[dstP:], src) |
||||
return dst[:dstP] |
||||
} |
||||
|
||||
// Decode returns the decoded form of src. The returned slice may be a sub-
|
||||
// slice of dst if dst was large enough to hold the entire decoded block.
|
||||
// Otherwise, a newly allocated slice will be returned.
|
||||
//
|
||||
// The dst and src must not overlap. It is valid to pass a nil dst.
|
||||
func (d *Dict) Decode(dst, src []byte) ([]byte, error) { |
||||
dLen, s, err := decodedLen(src) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
if dLen <= cap(dst) { |
||||
dst = dst[:dLen] |
||||
} else { |
||||
dst = make([]byte, dLen) |
||||
} |
||||
if s2DecodeDict(dst, src[s:], d) != 0 { |
||||
return nil, ErrCorrupt |
||||
} |
||||
return dst, nil |
||||
} |
||||
|
||||
func (d *Dict) initFast() { |
||||
d.fast.Do(func() { |
||||
const ( |
||||
tableBits = 14 |
||||
maxTableSize = 1 << tableBits |
||||
) |
||||
|
||||
var table [maxTableSize]uint16 |
||||
// We stop so any entry of length 8 can always be read.
|
||||
for i := 0; i < len(d.dict)-8-2; i += 3 { |
||||
x0 := load64(d.dict, i) |
||||
h0 := hash6(x0, tableBits) |
||||
h1 := hash6(x0>>8, tableBits) |
||||
h2 := hash6(x0>>16, tableBits) |
||||
table[h0] = uint16(i) |
||||
table[h1] = uint16(i + 1) |
||||
table[h2] = uint16(i + 2) |
||||
} |
||||
d.fastTable = &table |
||||
}) |
||||
} |
||||
|
||||
func (d *Dict) initBetter() { |
||||
d.better.Do(func() { |
||||
const ( |
||||
// Long hash matches.
|
||||
lTableBits = 17 |
||||
maxLTableSize = 1 << lTableBits |
||||
|
||||
// Short hash matches.
|
||||
sTableBits = 14 |
||||
maxSTableSize = 1 << sTableBits |
||||
) |
||||
|
||||
var lTable [maxLTableSize]uint16 |
||||
var sTable [maxSTableSize]uint16 |
||||
|
||||
// We stop so any entry of length 8 can always be read.
|
||||
for i := 0; i < len(d.dict)-8; i++ { |
||||
cv := load64(d.dict, i) |
||||
lTable[hash7(cv, lTableBits)] = uint16(i) |
||||
sTable[hash4(cv, sTableBits)] = uint16(i) |
||||
} |
||||
d.betterTableShort = &sTable |
||||
d.betterTableLong = &lTable |
||||
}) |
||||
} |
||||
|
||||
func (d *Dict) initBest() { |
||||
d.best.Do(func() { |
||||
const ( |
||||
// Long hash matches.
|
||||
lTableBits = 19 |
||||
maxLTableSize = 1 << lTableBits |
||||
|
||||
// Short hash matches.
|
||||
sTableBits = 16 |
||||
maxSTableSize = 1 << sTableBits |
||||
) |
||||
|
||||
var lTable [maxLTableSize]uint32 |
||||
var sTable [maxSTableSize]uint32 |
||||
|
||||
// We stop so any entry of length 8 can always be read.
|
||||
for i := 0; i < len(d.dict)-8; i++ { |
||||
cv := load64(d.dict, i) |
||||
hashL := hash8(cv, lTableBits) |
||||
hashS := hash4(cv, sTableBits) |
||||
candidateL := lTable[hashL] |
||||
candidateS := sTable[hashS] |
||||
lTable[hashL] = uint32(i) | candidateL<<16 |
||||
sTable[hashS] = uint32(i) | candidateS<<16 |
||||
} |
||||
d.bestTableShort = &sTable |
||||
d.bestTableLong = &lTable |
||||
}) |
||||
} |
||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,585 @@ |
||||
// Copyright (c) 2022 Klaus Post. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package s2 |
||||
|
||||
import ( |
||||
"encoding/binary" |
||||
"errors" |
||||
"fmt" |
||||
) |
||||
|
||||
// LZ4Converter provides conversion from LZ4 blocks as defined here:
|
||||
// https://github.com/lz4/lz4/blob/dev/doc/lz4_Block_format.md
|
||||
type LZ4Converter struct { |
||||
} |
||||
|
||||
// ErrDstTooSmall is returned when provided destination is too small.
|
||||
var ErrDstTooSmall = errors.New("s2: destination too small") |
||||
|
||||
// ConvertBlock will convert an LZ4 block and append it as an S2
|
||||
// block without block length to dst.
|
||||
// The uncompressed size is returned as well.
|
||||
// dst must have capacity to contain the entire compressed block.
|
||||
func (l *LZ4Converter) ConvertBlock(dst, src []byte) ([]byte, int, error) { |
||||
if len(src) == 0 { |
||||
return dst, 0, nil |
||||
} |
||||
const debug = false |
||||
const inline = true |
||||
const lz4MinMatch = 4 |
||||
|
||||
s, d := 0, len(dst) |
||||
dst = dst[:cap(dst)] |
||||
if !debug && hasAmd64Asm { |
||||
res, sz := cvtLZ4BlockAsm(dst[d:], src) |
||||
if res < 0 { |
||||
const ( |
||||
errCorrupt = -1 |
||||
errDstTooSmall = -2 |
||||
) |
||||
switch res { |
||||
case errCorrupt: |
||||
return nil, 0, ErrCorrupt |
||||
case errDstTooSmall: |
||||
return nil, 0, ErrDstTooSmall |
||||
default: |
||||
return nil, 0, fmt.Errorf("unexpected result: %d", res) |
||||
} |
||||
} |
||||
if d+sz > len(dst) { |
||||
return nil, 0, ErrDstTooSmall |
||||
} |
||||
return dst[:d+sz], res, nil |
||||
} |
||||
|
||||
dLimit := len(dst) - 10 |
||||
var lastOffset uint16 |
||||
var uncompressed int |
||||
if debug { |
||||
fmt.Printf("convert block start: len(src): %d, len(dst):%d \n", len(src), len(dst)) |
||||
} |
||||
|
||||
for { |
||||
if s >= len(src) { |
||||
return dst[:d], 0, ErrCorrupt |
||||
} |
||||
// Read literal info
|
||||
token := src[s] |
||||
ll := int(token >> 4) |
||||
ml := int(lz4MinMatch + (token & 0xf)) |
||||
|
||||
// If upper nibble is 15, literal length is extended
|
||||
if token >= 0xf0 { |
||||
for { |
||||
s++ |
||||
if s >= len(src) { |
||||
if debug { |
||||
fmt.Printf("error reading ll: s (%d) >= len(src) (%d)\n", s, len(src)) |
||||
} |
||||
return dst[:d], 0, ErrCorrupt |
||||
} |
||||
val := src[s] |
||||
ll += int(val) |
||||
if val != 255 { |
||||
break |
||||
} |
||||
} |
||||
} |
||||
// Skip past token
|
||||
if s+ll >= len(src) { |
||||
if debug { |
||||
fmt.Printf("error literals: s+ll (%d+%d) >= len(src) (%d)\n", s, ll, len(src)) |
||||
} |
||||
return nil, 0, ErrCorrupt |
||||
} |
||||
s++ |
||||
if ll > 0 { |
||||
if d+ll > dLimit { |
||||
return nil, 0, ErrDstTooSmall |
||||
} |
||||
if debug { |
||||
fmt.Printf("emit %d literals\n", ll) |
||||
} |
||||
d += emitLiteralGo(dst[d:], src[s:s+ll]) |
||||
s += ll |
||||
uncompressed += ll |
||||
} |
||||
|
||||
// Check if we are done...
|
||||
if s == len(src) && ml == lz4MinMatch { |
||||
break |
||||
} |
||||
// 2 byte offset
|
||||
if s >= len(src)-2 { |
||||
if debug { |
||||
fmt.Printf("s (%d) >= len(src)-2 (%d)", s, len(src)-2) |
||||
} |
||||
return nil, 0, ErrCorrupt |
||||
} |
||||
offset := binary.LittleEndian.Uint16(src[s:]) |
||||
s += 2 |
||||
if offset == 0 { |
||||
if debug { |
||||
fmt.Printf("error: offset 0, ml: %d, len(src)-s: %d\n", ml, len(src)-s) |
||||
} |
||||
return nil, 0, ErrCorrupt |
||||
} |
||||
if int(offset) > uncompressed { |
||||
if debug { |
||||
fmt.Printf("error: offset (%d)> uncompressed (%d)\n", offset, uncompressed) |
||||
} |
||||
return nil, 0, ErrCorrupt |
||||
} |
||||
|
||||
if ml == lz4MinMatch+15 { |
||||
for { |
||||
if s >= len(src) { |
||||
if debug { |
||||
fmt.Printf("error reading ml: s (%d) >= len(src) (%d)\n", s, len(src)) |
||||
} |
||||
return nil, 0, ErrCorrupt |
||||
} |
||||
val := src[s] |
||||
s++ |
||||
ml += int(val) |
||||
if val != 255 { |
||||
if s >= len(src) { |
||||
if debug { |
||||
fmt.Printf("error reading ml: s (%d) >= len(src) (%d)\n", s, len(src)) |
||||
} |
||||
return nil, 0, ErrCorrupt |
||||
} |
||||
break |
||||
} |
||||
} |
||||
} |
||||
if offset == lastOffset { |
||||
if debug { |
||||
fmt.Printf("emit repeat, length: %d, offset: %d\n", ml, offset) |
||||
} |
||||
if !inline { |
||||
d += emitRepeat16(dst[d:], offset, ml) |
||||
} else { |
||||
length := ml |
||||
dst := dst[d:] |
||||
for len(dst) > 5 { |
||||
// Repeat offset, make length cheaper
|
||||
length -= 4 |
||||
if length <= 4 { |
||||
dst[0] = uint8(length)<<2 | tagCopy1 |
||||
dst[1] = 0 |
||||
d += 2 |
||||
break |
||||
} |
||||
if length < 8 && offset < 2048 { |
||||
// Encode WITH offset
|
||||
dst[1] = uint8(offset) |
||||
dst[0] = uint8(offset>>8)<<5 | uint8(length)<<2 | tagCopy1 |
||||
d += 2 |
||||
break |
||||
} |
||||
if length < (1<<8)+4 { |
||||
length -= 4 |
||||
dst[2] = uint8(length) |
||||
dst[1] = 0 |
||||
dst[0] = 5<<2 | tagCopy1 |
||||
d += 3 |
||||
break |
||||
} |
||||
if length < (1<<16)+(1<<8) { |
||||
length -= 1 << 8 |
||||
dst[3] = uint8(length >> 8) |
||||
dst[2] = uint8(length >> 0) |
||||
dst[1] = 0 |
||||
dst[0] = 6<<2 | tagCopy1 |
||||
d += 4 |
||||
break |
||||
} |
||||
const maxRepeat = (1 << 24) - 1 |
||||
length -= 1 << 16 |
||||
left := 0 |
||||
if length > maxRepeat { |
||||
left = length - maxRepeat + 4 |
||||
length = maxRepeat - 4 |
||||
} |
||||
dst[4] = uint8(length >> 16) |
||||
dst[3] = uint8(length >> 8) |
||||
dst[2] = uint8(length >> 0) |
||||
dst[1] = 0 |
||||
dst[0] = 7<<2 | tagCopy1 |
||||
if left > 0 { |
||||
d += 5 + emitRepeat16(dst[5:], offset, left) |
||||
break |
||||
} |
||||
d += 5 |
||||
break |
||||
} |
||||
} |
||||
} else { |
||||
if debug { |
||||
fmt.Printf("emit copy, length: %d, offset: %d\n", ml, offset) |
||||
} |
||||
if !inline { |
||||
d += emitCopy16(dst[d:], offset, ml) |
||||
} else { |
||||
length := ml |
||||
dst := dst[d:] |
||||
for len(dst) > 5 { |
||||
// Offset no more than 2 bytes.
|
||||
if length > 64 { |
||||
off := 3 |
||||
if offset < 2048 { |
||||
// emit 8 bytes as tagCopy1, rest as repeats.
|
||||
dst[1] = uint8(offset) |
||||
dst[0] = uint8(offset>>8)<<5 | uint8(8-4)<<2 | tagCopy1 |
||||
length -= 8 |
||||
off = 2 |
||||
} else { |
||||
// Emit a length 60 copy, encoded as 3 bytes.
|
||||
// Emit remaining as repeat value (minimum 4 bytes).
|
||||
dst[2] = uint8(offset >> 8) |
||||
dst[1] = uint8(offset) |
||||
dst[0] = 59<<2 | tagCopy2 |
||||
length -= 60 |
||||
} |
||||
// Emit remaining as repeats, at least 4 bytes remain.
|
||||
d += off + emitRepeat16(dst[off:], offset, length) |
||||
break |
||||
} |
||||
if length >= 12 || offset >= 2048 { |
||||
// Emit the remaining copy, encoded as 3 bytes.
|
||||
dst[2] = uint8(offset >> 8) |
||||
dst[1] = uint8(offset) |
||||
dst[0] = uint8(length-1)<<2 | tagCopy2 |
||||
d += 3 |
||||
break |
||||
} |
||||
// Emit the remaining copy, encoded as 2 bytes.
|
||||
dst[1] = uint8(offset) |
||||
dst[0] = uint8(offset>>8)<<5 | uint8(length-4)<<2 | tagCopy1 |
||||
d += 2 |
||||
break |
||||
} |
||||
} |
||||
lastOffset = offset |
||||
} |
||||
uncompressed += ml |
||||
if d > dLimit { |
||||
return nil, 0, ErrDstTooSmall |
||||
} |
||||
} |
||||
|
||||
return dst[:d], uncompressed, nil |
||||
} |
||||
|
||||
// ConvertBlockSnappy will convert an LZ4 block and append it
|
||||
// as a Snappy block without block length to dst.
|
||||
// The uncompressed size is returned as well.
|
||||
// dst must have capacity to contain the entire compressed block.
|
||||
func (l *LZ4Converter) ConvertBlockSnappy(dst, src []byte) ([]byte, int, error) { |
||||
if len(src) == 0 { |
||||
return dst, 0, nil |
||||
} |
||||
const debug = false |
||||
const lz4MinMatch = 4 |
||||
|
||||
s, d := 0, len(dst) |
||||
dst = dst[:cap(dst)] |
||||
// Use assembly when possible
|
||||
if !debug && hasAmd64Asm { |
||||
res, sz := cvtLZ4BlockSnappyAsm(dst[d:], src) |
||||
if res < 0 { |
||||
const ( |
||||
errCorrupt = -1 |
||||
errDstTooSmall = -2 |
||||
) |
||||
switch res { |
||||
case errCorrupt: |
||||
return nil, 0, ErrCorrupt |
||||
case errDstTooSmall: |
||||
return nil, 0, ErrDstTooSmall |
||||
default: |
||||
return nil, 0, fmt.Errorf("unexpected result: %d", res) |
||||
} |
||||
} |
||||
if d+sz > len(dst) { |
||||
return nil, 0, ErrDstTooSmall |
||||
} |
||||
return dst[:d+sz], res, nil |
||||
} |
||||
|
||||
dLimit := len(dst) - 10 |
||||
var uncompressed int |
||||
if debug { |
||||
fmt.Printf("convert block start: len(src): %d, len(dst):%d \n", len(src), len(dst)) |
||||
} |
||||
|
||||
for { |
||||
if s >= len(src) { |
||||
return nil, 0, ErrCorrupt |
||||
} |
||||
// Read literal info
|
||||
token := src[s] |
||||
ll := int(token >> 4) |
||||
ml := int(lz4MinMatch + (token & 0xf)) |
||||
|
||||
// If upper nibble is 15, literal length is extended
|
||||
if token >= 0xf0 { |
||||
for { |
||||
s++ |
||||
if s >= len(src) { |
||||
if debug { |
||||
fmt.Printf("error reading ll: s (%d) >= len(src) (%d)\n", s, len(src)) |
||||
} |
||||
return nil, 0, ErrCorrupt |
||||
} |
||||
val := src[s] |
||||
ll += int(val) |
||||
if val != 255 { |
||||
break |
||||
} |
||||
} |
||||
} |
||||
// Skip past token
|
||||
if s+ll >= len(src) { |
||||
if debug { |
||||
fmt.Printf("error literals: s+ll (%d+%d) >= len(src) (%d)\n", s, ll, len(src)) |
||||
} |
||||
return nil, 0, ErrCorrupt |
||||
} |
||||
s++ |
||||
if ll > 0 { |
||||
if d+ll > dLimit { |
||||
return nil, 0, ErrDstTooSmall |
||||
} |
||||
if debug { |
||||
fmt.Printf("emit %d literals\n", ll) |
||||
} |
||||
d += emitLiteralGo(dst[d:], src[s:s+ll]) |
||||
s += ll |
||||
uncompressed += ll |
||||
} |
||||
|
||||
// Check if we are done...
|
||||
if s == len(src) && ml == lz4MinMatch { |
||||
break |
||||
} |
||||
// 2 byte offset
|
||||
if s >= len(src)-2 { |
||||
if debug { |
||||
fmt.Printf("s (%d) >= len(src)-2 (%d)", s, len(src)-2) |
||||
} |
||||
return nil, 0, ErrCorrupt |
||||
} |
||||
offset := binary.LittleEndian.Uint16(src[s:]) |
||||
s += 2 |
||||
if offset == 0 { |
||||
if debug { |
||||
fmt.Printf("error: offset 0, ml: %d, len(src)-s: %d\n", ml, len(src)-s) |
||||
} |
||||
return nil, 0, ErrCorrupt |
||||
} |
||||
if int(offset) > uncompressed { |
||||
if debug { |
||||
fmt.Printf("error: offset (%d)> uncompressed (%d)\n", offset, uncompressed) |
||||
} |
||||
return nil, 0, ErrCorrupt |
||||
} |
||||
|
||||
if ml == lz4MinMatch+15 { |
||||
for { |
||||
if s >= len(src) { |
||||
if debug { |
||||
fmt.Printf("error reading ml: s (%d) >= len(src) (%d)\n", s, len(src)) |
||||
} |
||||
return nil, 0, ErrCorrupt |
||||
} |
||||
val := src[s] |
||||
s++ |
||||
ml += int(val) |
||||
if val != 255 { |
||||
if s >= len(src) { |
||||
if debug { |
||||
fmt.Printf("error reading ml: s (%d) >= len(src) (%d)\n", s, len(src)) |
||||
} |
||||
return nil, 0, ErrCorrupt |
||||
} |
||||
break |
||||
} |
||||
} |
||||
} |
||||
if debug { |
||||
fmt.Printf("emit copy, length: %d, offset: %d\n", ml, offset) |
||||
} |
||||
length := ml |
||||
// d += emitCopyNoRepeat(dst[d:], int(offset), ml)
|
||||
for length > 0 { |
||||
if d >= dLimit { |
||||
return nil, 0, ErrDstTooSmall |
||||
} |
||||
|
||||
// Offset no more than 2 bytes.
|
||||
if length > 64 { |
||||
// Emit a length 64 copy, encoded as 3 bytes.
|
||||
dst[d+2] = uint8(offset >> 8) |
||||
dst[d+1] = uint8(offset) |
||||
dst[d+0] = 63<<2 | tagCopy2 |
||||
length -= 64 |
||||
d += 3 |
||||
continue |
||||
} |
||||
if length >= 12 || offset >= 2048 || length < 4 { |
||||
// Emit the remaining copy, encoded as 3 bytes.
|
||||
dst[d+2] = uint8(offset >> 8) |
||||
dst[d+1] = uint8(offset) |
||||
dst[d+0] = uint8(length-1)<<2 | tagCopy2 |
||||
d += 3 |
||||
break |
||||
} |
||||
// Emit the remaining copy, encoded as 2 bytes.
|
||||
dst[d+1] = uint8(offset) |
||||
dst[d+0] = uint8(offset>>8)<<5 | uint8(length-4)<<2 | tagCopy1 |
||||
d += 2 |
||||
break |
||||
} |
||||
uncompressed += ml |
||||
if d > dLimit { |
||||
return nil, 0, ErrDstTooSmall |
||||
} |
||||
} |
||||
|
||||
return dst[:d], uncompressed, nil |
||||
} |
||||
|
||||
// emitRepeat writes a repeat chunk and returns the number of bytes written.
|
||||
// Length must be at least 4 and < 1<<24
|
||||
func emitRepeat16(dst []byte, offset uint16, length int) int { |
||||
// Repeat offset, make length cheaper
|
||||
length -= 4 |
||||
if length <= 4 { |
||||
dst[0] = uint8(length)<<2 | tagCopy1 |
||||
dst[1] = 0 |
||||
return 2 |
||||
} |
||||
if length < 8 && offset < 2048 { |
||||
// Encode WITH offset
|
||||
dst[1] = uint8(offset) |
||||
dst[0] = uint8(offset>>8)<<5 | uint8(length)<<2 | tagCopy1 |
||||
return 2 |
||||
} |
||||
if length < (1<<8)+4 { |
||||
length -= 4 |
||||
dst[2] = uint8(length) |
||||
dst[1] = 0 |
||||
dst[0] = 5<<2 | tagCopy1 |
||||
return 3 |
||||
} |
||||
if length < (1<<16)+(1<<8) { |
||||
length -= 1 << 8 |
||||
dst[3] = uint8(length >> 8) |
||||
dst[2] = uint8(length >> 0) |
||||
dst[1] = 0 |
||||
dst[0] = 6<<2 | tagCopy1 |
||||
return 4 |
||||
} |
||||
const maxRepeat = (1 << 24) - 1 |
||||
length -= 1 << 16 |
||||
left := 0 |
||||
if length > maxRepeat { |
||||
left = length - maxRepeat + 4 |
||||
length = maxRepeat - 4 |
||||
} |
||||
dst[4] = uint8(length >> 16) |
||||
dst[3] = uint8(length >> 8) |
||||
dst[2] = uint8(length >> 0) |
||||
dst[1] = 0 |
||||
dst[0] = 7<<2 | tagCopy1 |
||||
if left > 0 { |
||||
return 5 + emitRepeat16(dst[5:], offset, left) |
||||
} |
||||
return 5 |
||||
} |
||||
|
||||
// emitCopy writes a copy chunk and returns the number of bytes written.
|
||||
//
|
||||
// It assumes that:
|
||||
//
|
||||
// dst is long enough to hold the encoded bytes
|
||||
// 1 <= offset && offset <= math.MaxUint16
|
||||
// 4 <= length && length <= math.MaxUint32
|
||||
func emitCopy16(dst []byte, offset uint16, length int) int { |
||||
// Offset no more than 2 bytes.
|
||||
if length > 64 { |
||||
off := 3 |
||||
if offset < 2048 { |
||||
// emit 8 bytes as tagCopy1, rest as repeats.
|
||||
dst[1] = uint8(offset) |
||||
dst[0] = uint8(offset>>8)<<5 | uint8(8-4)<<2 | tagCopy1 |
||||
length -= 8 |
||||
off = 2 |
||||
} else { |
||||
// Emit a length 60 copy, encoded as 3 bytes.
|
||||
// Emit remaining as repeat value (minimum 4 bytes).
|
||||
dst[2] = uint8(offset >> 8) |
||||
dst[1] = uint8(offset) |
||||
dst[0] = 59<<2 | tagCopy2 |
||||
length -= 60 |
||||
} |
||||
// Emit remaining as repeats, at least 4 bytes remain.
|
||||
return off + emitRepeat16(dst[off:], offset, length) |
||||
} |
||||
if length >= 12 || offset >= 2048 { |
||||
// Emit the remaining copy, encoded as 3 bytes.
|
||||
dst[2] = uint8(offset >> 8) |
||||
dst[1] = uint8(offset) |
||||
dst[0] = uint8(length-1)<<2 | tagCopy2 |
||||
return 3 |
||||
} |
||||
// Emit the remaining copy, encoded as 2 bytes.
|
||||
dst[1] = uint8(offset) |
||||
dst[0] = uint8(offset>>8)<<5 | uint8(length-4)<<2 | tagCopy1 |
||||
return 2 |
||||
} |
||||
|
||||
// emitLiteral writes a literal chunk and returns the number of bytes written.
|
||||
//
|
||||
// It assumes that:
|
||||
//
|
||||
// dst is long enough to hold the encoded bytes
|
||||
// 0 <= len(lit) && len(lit) <= math.MaxUint32
|
||||
func emitLiteralGo(dst, lit []byte) int { |
||||
if len(lit) == 0 { |
||||
return 0 |
||||
} |
||||
i, n := 0, uint(len(lit)-1) |
||||
switch { |
||||
case n < 60: |
||||
dst[0] = uint8(n)<<2 | tagLiteral |
||||
i = 1 |
||||
case n < 1<<8: |
||||
dst[1] = uint8(n) |
||||
dst[0] = 60<<2 | tagLiteral |
||||
i = 2 |
||||
case n < 1<<16: |
||||
dst[2] = uint8(n >> 8) |
||||
dst[1] = uint8(n) |
||||
dst[0] = 61<<2 | tagLiteral |
||||
i = 3 |
||||
case n < 1<<24: |
||||
dst[3] = uint8(n >> 16) |
||||
dst[2] = uint8(n >> 8) |
||||
dst[1] = uint8(n) |
||||
dst[0] = 62<<2 | tagLiteral |
||||
i = 4 |
||||
default: |
||||
dst[4] = uint8(n >> 24) |
||||
dst[3] = uint8(n >> 16) |
||||
dst[2] = uint8(n >> 8) |
||||
dst[1] = uint8(n) |
||||
dst[0] = 63<<2 | tagLiteral |
||||
i = 5 |
||||
} |
||||
return i + copy(dst[i:], lit) |
||||
} |
||||
Loading…
Reference in new issue