mirror of https://github.com/grafana/loki
Re-license and export `pkg/ingester` WAL code to be used in Promtail's WAL (#8315)
Pablo
3 years ago
committed by
GitHub
parent
66f7407f15
commit
fda14ca156
@ -0,0 +1,202 @@ |
||||
|
||||
Apache License |
||||
Version 2.0, January 2004 |
||||
http://www.apache.org/licenses/ |
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION |
||||
|
||||
1. Definitions. |
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction, |
||||
and distribution as defined by Sections 1 through 9 of this document. |
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by |
||||
the copyright owner that is granting the License. |
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all |
||||
other entities that control, are controlled by, or are under common |
||||
control with that entity. For the purposes of this definition, |
||||
"control" means (i) the power, direct or indirect, to cause the |
||||
direction or management of such entity, whether by contract or |
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the |
||||
outstanding shares, or (iii) beneficial ownership of such entity. |
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity |
||||
exercising permissions granted by this License. |
||||
|
||||
"Source" form shall mean the preferred form for making modifications, |
||||
including but not limited to software source code, documentation |
||||
source, and configuration files. |
||||
|
||||
"Object" form shall mean any form resulting from mechanical |
||||
transformation or translation of a Source form, including but |
||||
not limited to compiled object code, generated documentation, |
||||
and conversions to other media types. |
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or |
||||
Object form, made available under the License, as indicated by a |
||||
copyright notice that is included in or attached to the work |
||||
(an example is provided in the Appendix below). |
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object |
||||
form, that is based on (or derived from) the Work and for which the |
||||
editorial revisions, annotations, elaborations, or other modifications |
||||
represent, as a whole, an original work of authorship. For the purposes |
||||
of this License, Derivative Works shall not include works that remain |
||||
separable from, or merely link (or bind by name) to the interfaces of, |
||||
the Work and Derivative Works thereof. |
||||
|
||||
"Contribution" shall mean any work of authorship, including |
||||
the original version of the Work and any modifications or additions |
||||
to that Work or Derivative Works thereof, that is intentionally |
||||
submitted to Licensor for inclusion in the Work by the copyright owner |
||||
or by an individual or Legal Entity authorized to submit on behalf of |
||||
the copyright owner. For the purposes of this definition, "submitted" |
||||
means any form of electronic, verbal, or written communication sent |
||||
to the Licensor or its representatives, including but not limited to |
||||
communication on electronic mailing lists, source code control systems, |
||||
and issue tracking systems that are managed by, or on behalf of, the |
||||
Licensor for the purpose of discussing and improving the Work, but |
||||
excluding communication that is conspicuously marked or otherwise |
||||
designated in writing by the copyright owner as "Not a Contribution." |
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity |
||||
on behalf of whom a Contribution has been received by Licensor and |
||||
subsequently incorporated within the Work. |
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of |
||||
this License, each Contributor hereby grants to You a perpetual, |
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable |
||||
copyright license to reproduce, prepare Derivative Works of, |
||||
publicly display, publicly perform, sublicense, and distribute the |
||||
Work and such Derivative Works in Source or Object form. |
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of |
||||
this License, each Contributor hereby grants to You a perpetual, |
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable |
||||
(except as stated in this section) patent license to make, have made, |
||||
use, offer to sell, sell, import, and otherwise transfer the Work, |
||||
where such license applies only to those patent claims licensable |
||||
by such Contributor that are necessarily infringed by their |
||||
Contribution(s) alone or by combination of their Contribution(s) |
||||
with the Work to which such Contribution(s) was submitted. If You |
||||
institute patent litigation against any entity (including a |
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work |
||||
or a Contribution incorporated within the Work constitutes direct |
||||
or contributory patent infringement, then any patent licenses |
||||
granted to You under this License for that Work shall terminate |
||||
as of the date such litigation is filed. |
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the |
||||
Work or Derivative Works thereof in any medium, with or without |
||||
modifications, and in Source or Object form, provided that You |
||||
meet the following conditions: |
||||
|
||||
(a) You must give any other recipients of the Work or |
||||
Derivative Works a copy of this License; and |
||||
|
||||
(b) You must cause any modified files to carry prominent notices |
||||
stating that You changed the files; and |
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works |
||||
that You distribute, all copyright, patent, trademark, and |
||||
attribution notices from the Source form of the Work, |
||||
excluding those notices that do not pertain to any part of |
||||
the Derivative Works; and |
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its |
||||
distribution, then any Derivative Works that You distribute must |
||||
include a readable copy of the attribution notices contained |
||||
within such NOTICE file, excluding those notices that do not |
||||
pertain to any part of the Derivative Works, in at least one |
||||
of the following places: within a NOTICE text file distributed |
||||
as part of the Derivative Works; within the Source form or |
||||
documentation, if provided along with the Derivative Works; or, |
||||
within a display generated by the Derivative Works, if and |
||||
wherever such third-party notices normally appear. The contents |
||||
of the NOTICE file are for informational purposes only and |
||||
do not modify the License. You may add Your own attribution |
||||
notices within Derivative Works that You distribute, alongside |
||||
or as an addendum to the NOTICE text from the Work, provided |
||||
that such additional attribution notices cannot be construed |
||||
as modifying the License. |
||||
|
||||
You may add Your own copyright statement to Your modifications and |
||||
may provide additional or different license terms and conditions |
||||
for use, reproduction, or distribution of Your modifications, or |
||||
for any such Derivative Works as a whole, provided Your use, |
||||
reproduction, and distribution of the Work otherwise complies with |
||||
the conditions stated in this License. |
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise, |
||||
any Contribution intentionally submitted for inclusion in the Work |
||||
by You to the Licensor shall be under the terms and conditions of |
||||
this License, without any additional terms or conditions. |
||||
Notwithstanding the above, nothing herein shall supersede or modify |
||||
the terms of any separate license agreement you may have executed |
||||
with Licensor regarding such Contributions. |
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade |
||||
names, trademarks, service marks, or product names of the Licensor, |
||||
except as required for reasonable and customary use in describing the |
||||
origin of the Work and reproducing the content of the NOTICE file. |
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or |
||||
agreed to in writing, Licensor provides the Work (and each |
||||
Contributor provides its Contributions) on an "AS IS" BASIS, |
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or |
||||
implied, including, without limitation, any warranties or conditions |
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A |
||||
PARTICULAR PURPOSE. You are solely responsible for determining the |
||||
appropriateness of using or redistributing the Work and assume any |
||||
risks associated with Your exercise of permissions under this License. |
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory, |
||||
whether in tort (including negligence), contract, or otherwise, |
||||
unless required by applicable law (such as deliberate and grossly |
||||
negligent acts) or agreed to in writing, shall any Contributor be |
||||
liable to You for damages, including any direct, indirect, special, |
||||
incidental, or consequential damages of any character arising as a |
||||
result of this License or out of the use or inability to use the |
||||
Work (including but not limited to damages for loss of goodwill, |
||||
work stoppage, computer failure or malfunction, or any and all |
||||
other commercial damages or losses), even if such Contributor |
||||
has been advised of the possibility of such damages. |
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing |
||||
the Work or Derivative Works thereof, You may choose to offer, |
||||
and charge a fee for, acceptance of support, warranty, indemnity, |
||||
or other liability obligations and/or rights consistent with this |
||||
License. However, in accepting such obligations, You may act only |
||||
on Your own behalf and on Your sole responsibility, not on behalf |
||||
of any other Contributor, and only if You agree to indemnify, |
||||
defend, and hold each Contributor harmless for any liability |
||||
incurred by, or claims asserted against, such Contributor by reason |
||||
of your accepting any such warranty or additional liability. |
||||
|
||||
END OF TERMS AND CONDITIONS |
||||
|
||||
APPENDIX: How to apply the Apache License to your work. |
||||
|
||||
To apply the Apache License to your work, attach the following |
||||
boilerplate notice, with the fields enclosed by brackets "[]" |
||||
replaced with your own identifying information. (Don't include |
||||
the brackets!) The text should be enclosed in the appropriate |
||||
comment syntax for the file format. We also recommend that a |
||||
file or class name and description of purpose be included on the |
||||
same "printed page" as the copyright notice for easier |
||||
identification within third-party archives. |
||||
|
||||
Copyright [yyyy] [name of copyright owner] |
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License"); |
||||
you may not use this file except in compliance with the License. |
||||
You may obtain a copy of the License at |
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0 |
||||
|
||||
Unless required by applicable law or agreed to in writing, software |
||||
distributed under the License is distributed on an "AS IS" BASIS, |
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
See the License for the specific language governing permissions and |
||||
limitations under the License. |
||||
@ -0,0 +1,208 @@ |
||||
package wal |
||||
|
||||
import ( |
||||
"fmt" |
||||
"testing" |
||||
"time" |
||||
|
||||
"github.com/prometheus/prometheus/model/labels" |
||||
"github.com/prometheus/prometheus/tsdb/record" |
||||
"github.com/stretchr/testify/require" |
||||
|
||||
"github.com/grafana/loki/pkg/logproto" |
||||
) |
||||
|
||||
var ( |
||||
recordPool = NewRecordPool() |
||||
) |
||||
|
||||
func Test_Encoding_Series(t *testing.T) { |
||||
record := &Record{ |
||||
entryIndexMap: make(map[uint64]int), |
||||
UserID: "123", |
||||
Series: []record.RefSeries{ |
||||
{ |
||||
Ref: 456, |
||||
Labels: labels.FromMap(map[string]string{ |
||||
"foo": "bar", |
||||
"bazz": "buzz", |
||||
}), |
||||
}, |
||||
{ |
||||
Ref: 789, |
||||
Labels: labels.FromMap(map[string]string{ |
||||
"abc": "123", |
||||
"def": "456", |
||||
}), |
||||
}, |
||||
}, |
||||
} |
||||
|
||||
buf := record.EncodeSeries(nil) |
||||
|
||||
decoded := recordPool.GetRecord() |
||||
|
||||
err := DecodeRecord(buf, decoded) |
||||
require.Nil(t, err) |
||||
|
||||
// Since we use a pool, there can be subtle differentiations between nil slices and len(0) slices.
|
||||
// Both are valid, so check length.
|
||||
require.Equal(t, 0, len(decoded.RefEntries)) |
||||
decoded.RefEntries = nil |
||||
require.Equal(t, record, decoded) |
||||
} |
||||
|
||||
func Test_Encoding_Entries(t *testing.T) { |
||||
for _, tc := range []struct { |
||||
desc string |
||||
rec *Record |
||||
version RecordType |
||||
}{ |
||||
{ |
||||
desc: "v1", |
||||
rec: &Record{ |
||||
entryIndexMap: make(map[uint64]int), |
||||
UserID: "123", |
||||
RefEntries: []RefEntries{ |
||||
{ |
||||
Ref: 456, |
||||
Entries: []logproto.Entry{ |
||||
{ |
||||
Timestamp: time.Unix(1000, 0), |
||||
Line: "first", |
||||
}, |
||||
{ |
||||
Timestamp: time.Unix(2000, 0), |
||||
Line: "second", |
||||
}, |
||||
}, |
||||
}, |
||||
{ |
||||
Ref: 789, |
||||
Entries: []logproto.Entry{ |
||||
{ |
||||
Timestamp: time.Unix(3000, 0), |
||||
Line: "third", |
||||
}, |
||||
{ |
||||
Timestamp: time.Unix(4000, 0), |
||||
Line: "fourth", |
||||
}, |
||||
}, |
||||
}, |
||||
}, |
||||
}, |
||||
version: WALRecordEntriesV1, |
||||
}, |
||||
{ |
||||
desc: "v2", |
||||
rec: &Record{ |
||||
entryIndexMap: make(map[uint64]int), |
||||
UserID: "123", |
||||
RefEntries: []RefEntries{ |
||||
{ |
||||
Ref: 456, |
||||
Counter: 1, // v2 uses counter for WAL replay
|
||||
Entries: []logproto.Entry{ |
||||
{ |
||||
Timestamp: time.Unix(1000, 0), |
||||
Line: "first", |
||||
}, |
||||
{ |
||||
Timestamp: time.Unix(2000, 0), |
||||
Line: "second", |
||||
}, |
||||
}, |
||||
}, |
||||
{ |
||||
Ref: 789, |
||||
Counter: 2, // v2 uses counter for WAL replay
|
||||
Entries: []logproto.Entry{ |
||||
{ |
||||
Timestamp: time.Unix(3000, 0), |
||||
Line: "third", |
||||
}, |
||||
{ |
||||
Timestamp: time.Unix(4000, 0), |
||||
Line: "fourth", |
||||
}, |
||||
}, |
||||
}, |
||||
}, |
||||
}, |
||||
version: WALRecordEntriesV2, |
||||
}, |
||||
} { |
||||
decoded := recordPool.GetRecord() |
||||
buf := tc.rec.EncodeEntries(tc.version, nil) |
||||
err := DecodeRecord(buf, decoded) |
||||
require.Nil(t, err) |
||||
require.Equal(t, tc.rec, decoded) |
||||
|
||||
} |
||||
} |
||||
|
||||
func Benchmark_EncodeEntries(b *testing.B) { |
||||
var entries []logproto.Entry |
||||
for i := int64(0); i < 10000; i++ { |
||||
entries = append(entries, logproto.Entry{ |
||||
Timestamp: time.Unix(0, i), |
||||
Line: fmt.Sprintf("long line with a lot of data like a log %d", i), |
||||
}) |
||||
} |
||||
record := &Record{ |
||||
entryIndexMap: make(map[uint64]int), |
||||
UserID: "123", |
||||
RefEntries: []RefEntries{ |
||||
{ |
||||
Ref: 456, |
||||
Entries: entries, |
||||
}, |
||||
{ |
||||
Ref: 789, |
||||
Entries: entries, |
||||
}, |
||||
}, |
||||
} |
||||
b.ReportAllocs() |
||||
b.ResetTimer() |
||||
buf := recordPool.GetBytes()[:0] |
||||
defer recordPool.PutBytes(buf) |
||||
|
||||
for n := 0; n < b.N; n++ { |
||||
record.EncodeEntries(CurrentEntriesRec, buf) |
||||
} |
||||
} |
||||
|
||||
func Benchmark_DecodeWAL(b *testing.B) { |
||||
var entries []logproto.Entry |
||||
for i := int64(0); i < 10000; i++ { |
||||
entries = append(entries, logproto.Entry{ |
||||
Timestamp: time.Unix(0, i), |
||||
Line: fmt.Sprintf("long line with a lot of data like a log %d", i), |
||||
}) |
||||
} |
||||
record := &Record{ |
||||
entryIndexMap: make(map[uint64]int), |
||||
UserID: "123", |
||||
RefEntries: []RefEntries{ |
||||
{ |
||||
Ref: 456, |
||||
Entries: entries, |
||||
}, |
||||
{ |
||||
Ref: 789, |
||||
Entries: entries, |
||||
}, |
||||
}, |
||||
} |
||||
|
||||
buf := record.EncodeEntries(CurrentEntriesRec, nil) |
||||
rec := recordPool.GetRecord() |
||||
b.ReportAllocs() |
||||
b.ResetTimer() |
||||
|
||||
for n := 0; n < b.N; n++ { |
||||
require.NoError(b, DecodeRecord(buf, rec)) |
||||
} |
||||
} |
||||
@ -0,0 +1,62 @@ |
||||
package wal |
||||
|
||||
import ( |
||||
"sync" |
||||
|
||||
"github.com/grafana/loki/pkg/logproto" |
||||
) |
||||
|
||||
type ResettingPool struct { |
||||
rPool *sync.Pool // records
|
||||
ePool *sync.Pool // entries
|
||||
bPool *sync.Pool // bytes
|
||||
} |
||||
|
||||
func NewRecordPool() *ResettingPool { |
||||
return &ResettingPool{ |
||||
rPool: &sync.Pool{ |
||||
New: func() interface{} { |
||||
return &Record{} |
||||
}, |
||||
}, |
||||
ePool: &sync.Pool{ |
||||
New: func() interface{} { |
||||
return make([]logproto.Entry, 0, 512) |
||||
}, |
||||
}, |
||||
bPool: &sync.Pool{ |
||||
New: func() interface{} { |
||||
return make([]byte, 0, 1<<10) // 1kb
|
||||
}, |
||||
}, |
||||
} |
||||
} |
||||
|
||||
func (p *ResettingPool) GetRecord() *Record { |
||||
rec := p.rPool.Get().(*Record) |
||||
rec.Reset() |
||||
for _, ref := range rec.RefEntries { |
||||
p.PutEntries(ref.Entries) |
||||
} |
||||
return rec |
||||
} |
||||
|
||||
func (p *ResettingPool) PutRecord(r *Record) { |
||||
p.rPool.Put(r) |
||||
} |
||||
|
||||
func (p *ResettingPool) GetEntries() []logproto.Entry { |
||||
return p.ePool.Get().([]logproto.Entry) |
||||
} |
||||
|
||||
func (p *ResettingPool) PutEntries(es []logproto.Entry) { |
||||
p.ePool.Put(es[:0]) // nolint:staticcheck
|
||||
} |
||||
|
||||
func (p *ResettingPool) GetBytes() []byte { |
||||
return p.bPool.Get().([]byte) |
||||
} |
||||
|
||||
func (p *ResettingPool) PutBytes(b []byte) { |
||||
p.bPool.Put(b[:0]) // nolint:staticcheck
|
||||
} |
||||
Loading…
Reference in new issue