apitech
/
prometheus
mirror of https://github.com/prometheus/prometheus
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Remove temporary patch for out-of-order (#283)

Browse Source 
* Remove temporary patch for out-of-order

Signed-off-by: Ganesh Vernekar <ganeshvern@gmail.com>

* Remove ooo_wbl patch and fix tests

Signed-off-by: Ganesh Vernekar <ganeshvern@gmail.com>
owilliams/utf8-02-mimir
Ganesh Vernekar 3 years ago committed by GitHub
parent 5e8406a1d4
commit c6f3d4ab33
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 42 additions and 99 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 16
      tsdb/db.go
  2. 125
      tsdb/db_test.go

16
tsdb/db.go
Unescape View File

@ -711,14 +711,6 @@ func open(dir string, l log.Logger, r prometheus.Registerer, opts *Options, rngs
walDir := filepath.Join(dir, "wal")
wblDir := filepath.Join(dir, wal.WblDirName)
// TODO(jesus.vazquez) Remove the block of code below, only necessary until all ooo_wbl dirs in prod have been replaced with wbl
oldWblDir := filepath.Join(dir, "ooo_wbl")
if _, err := os.Stat(oldWblDir); err == nil {
err = fileutil.Rename(oldWblDir, wblDir)
if err != nil {
return nil, errors.Wrap(err, "failed to move old wbl dir to new wbl dir")
}
}
// Migrate old WAL if one exists.
if err := MigrateWAL(l, walDir); err != nil {
@ -1653,14 +1645,6 @@ func (db *DB) inOrderBlocksMaxTime() (maxt int64, ok bool) {
maxt = b.meta.MaxTime
}
}
if !hasOOO && ok && db.opts.OutOfOrderTimeWindow > 0 {
// Temporary patch. To be removed by mid July 2022.
// Before this patch, blocks did not have "out_of_order" in their meta, so we cannot
// say which block has the out_of_order data. In that case the out-of-order block can be
// up to 2 block ranges ahead of the latest in-order block.
// Note: if hasOOO was true, it means the latest block has the new meta and is taken care in inOrderBlocksMaxTime().
maxt -= 2 * db.opts.MinBlockDuration
}
return maxt, ok
}

125
tsdb/db_test.go
Unescape View File

@ -4433,11 +4433,6 @@ func TestWBLAndMmapReplay(t *testing.T) {
require.NoError(t, os.RemoveAll(wblDir))
require.NoError(t, fileutil.CopyDirs(originalWblDir, wblDir))
}
// TODO(jesus.vazquez) Remove this function and the test below using it, only necessary until all ooo_wbl dirs in prod have been replaced with wbl
resetWBLToOldDir := func() {
require.NoError(t, os.RemoveAll(wblDir))
require.NoError(t, fileutil.CopyDirs(originalWblDir, filepath.Join(db.dir, "ooo_wbl")))
}
resetMmapToOriginal := func() {
require.NoError(t, os.RemoveAll(mmapDir))
require.NoError(t, fileutil.CopyDirs(originalMmapDir, mmapDir))
@ -4534,18 +4529,6 @@ func TestWBLAndMmapReplay(t *testing.T) {
require.Equal(t, oooMint, db.head.MinOOOTime())
require.Equal(t, oooMaxt, db.head.MaxOOOTime())
testQuery(expSamples)
require.NoError(t, db.Close())
})
t.Run("Restart DB with Old WBL Dir", func(t *testing.T) {
resetWBLToOldDir()
resetMmapToOriginal()
db, err = Open(db.dir, nil, nil, opts, nil)
require.NoError(t, err)
require.Equal(t, oooMint, db.head.MinOOOTime())
require.Equal(t, oooMaxt, db.head.MaxOOOTime())
testQuery(expSamples)
})
}
@ -5233,92 +5216,68 @@ func TestNoGapAfterRestartWithOOO(t *testing.T) {
// After compaction.
blockRanges [][2]int64
headMint, headMaxt int64
// Head time ranges after restart for old blocks.
legacyHeadMint, legacyHeadMaxt int64
}{
{
300, 490,
489, 489,
[][2]int64{{300, 360}, {480, 600}},
360, 490,
360, 490, // OOO blocks is already 2 ranges ahead of the in-order block.
},
{
300, 490,
479, 479,
[][2]int64{{300, 360}, {360, 480}},
360, 490,
240, 490, // OOO block was only 1 range ahead of in-order block.
},
}
for i, c := range cases {
// legacy = true means the out-of-order blocks don't have the `out_of_order: true` metadata.
for _, legacy := range []bool{false, true} {
t.Run(fmt.Sprintf("case=%d,legacy=%t", i, legacy), func(t *testing.T) {
dir := t.TempDir()
t.Run(fmt.Sprintf("case=%d", i), func(t *testing.T) {
dir := t.TempDir()
opts := DefaultOptions()
opts.OutOfOrderTimeWindow = 30 * time.Minute.Milliseconds()
db, err := Open(dir, nil, nil, opts, nil)
require.NoError(t, err)
db.DisableCompactions()
t.Cleanup(func() {
require.NoError(t, db.Close())
})
// 3h10m=190m worth in-order data.
addSamples(t, db, c.inOrderMint, c.inOrderMaxt, true)
verifySamples(t, db, c.inOrderMint, c.inOrderMaxt)
// One ooo samples.
addSamples(t, db, c.oooMint, c.oooMaxt, true)
verifySamples(t, db, c.inOrderMint, c.inOrderMaxt)
// We get 2 blocks. 1 from OOO, 1 from in-order.
require.NoError(t, db.Compact())
verifyBlockRanges := func() {
blocks := db.Blocks()
require.Equal(t, len(c.blockRanges), len(blocks))
for j, br := range c.blockRanges {
require.Equal(t, br[0]*time.Minute.Milliseconds(), blocks[j].MinTime())
require.Equal(t, br[1]*time.Minute.Milliseconds(), blocks[j].MaxTime())
}
}
verifyBlockRanges()
require.Equal(t, c.headMint*time.Minute.Milliseconds(), db.head.MinTime())
require.Equal(t, c.headMaxt*time.Minute.Milliseconds(), db.head.MaxTime())
if legacy {
// In the legacy version, the blocks from out-of-order data did not write a
// "out_of_order: true" to the meta. So we remove it here.
for _, b := range db.Blocks() {
m, _, err := readMetaFile(b.Dir())
require.NoError(t, err)
m.OutOfOrder = false
_, err = writeMetaFile(log.NewNopLogger(), b.Dir(), m)
require.NoError(t, err)
}
}
opts := DefaultOptions()
opts.OutOfOrderTimeWindow = 30 * time.Minute.Milliseconds()
// Restart and expect all samples to be present.
db, err := Open(dir, nil, nil, opts, nil)
require.NoError(t, err)
db.DisableCompactions()
t.Cleanup(func() {
require.NoError(t, db.Close())
})
db, err = Open(dir, nil, nil, opts, nil)
require.NoError(t, err)
db.DisableCompactions()
verifyBlockRanges()
if legacy {
require.Equal(t, c.legacyHeadMint*time.Minute.Milliseconds(), db.head.MinTime())
require.Equal(t, c.legacyHeadMaxt*time.Minute.Milliseconds(), db.head.MaxTime())
} else {
require.Equal(t, c.headMint*time.Minute.Milliseconds(), db.head.MinTime())
require.Equal(t, c.headMaxt*time.Minute.Milliseconds(), db.head.MaxTime())
// 3h10m=190m worth in-order data.
addSamples(t, db, c.inOrderMint, c.inOrderMaxt, true)
verifySamples(t, db, c.inOrderMint, c.inOrderMaxt)
// One ooo samples.
addSamples(t, db, c.oooMint, c.oooMaxt, true)
verifySamples(t, db, c.inOrderMint, c.inOrderMaxt)
// We get 2 blocks. 1 from OOO, 1 from in-order.
require.NoError(t, db.Compact())
verifyBlockRanges := func() {
blocks := db.Blocks()
require.Equal(t, len(c.blockRanges), len(blocks))
for j, br := range c.blockRanges {
require.Equal(t, br[0]*time.Minute.Milliseconds(), blocks[j].MinTime())
require.Equal(t, br[1]*time.Minute.Milliseconds(), blocks[j].MaxTime())
}
verifySamples(t, db, c.inOrderMint, c.inOrderMaxt)
})
}
}
verifyBlockRanges()
require.Equal(t, c.headMint*time.Minute.Milliseconds(), db.head.MinTime())
require.Equal(t, c.headMaxt*time.Minute.Milliseconds(), db.head.MaxTime())
// Restart and expect all samples to be present.
require.NoError(t, db.Close())
db, err = Open(dir, nil, nil, opts, nil)
require.NoError(t, err)
db.DisableCompactions()
verifyBlockRanges()
require.Equal(t, c.headMint*time.Minute.Milliseconds(), db.head.MinTime())
require.Equal(t, c.headMaxt*time.Minute.Milliseconds(), db.head.MaxTime())
verifySamples(t, db, c.inOrderMint, c.inOrderMaxt)
})
}
}

Write Preview
Loading…
Cancel
Save