apitech
/
loki
mirror of https://github.com/grafana/loki
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

fix broken boltdb-shipper test (#5222)

Browse Source
pull/5196/head
Sandeep Sukhani 3 years ago committed by GitHub
parent cd1537d06e
commit 0426870f3f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 276 additions and 167 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 54
      pkg/storage/stores/shipper/compactor/compactor_test.go
  2. 13
      pkg/storage/stores/shipper/compactor/table_test.go
  3. 26
      pkg/storage/stores/shipper/downloads/index_set_test.go
  4. 13
      pkg/storage/stores/shipper/downloads/table_manager_test.go
  5. 138
      pkg/storage/stores/shipper/downloads/table_test.go
  6. 72
      pkg/storage/stores/shipper/testutil/testutil.go
  7. 32
      pkg/storage/stores/shipper/uploads/table_manager_test.go
  8. 95
      pkg/storage/stores/shipper/uploads/table_test.go

54
pkg/storage/stores/shipper/compactor/compactor_test.go
Unescape View File

@ -48,50 +48,66 @@ func TestCompactor_RunCompaction(t *testing.T) {
tablesPath := filepath.Join(tempDir, "index")
tablesCopyPath := filepath.Join(tempDir, "index-copy")
tables := map[string]map[string]testutil.DBRecords{
tables := map[string]map[string]testutil.DBConfig{
"table1": {
"db1": {
Start: 0,
NumRecords: 10,
DBRecords: testutil.DBRecords{
Start: 0,
NumRecords: 10,
},
},
"db2": {
Start: 10,
NumRecords: 10,
DBRecords: testutil.DBRecords{
Start: 10,
NumRecords: 10,
},
},
"db3": {
Start: 20,
NumRecords: 10,
DBRecords: testutil.DBRecords{
Start: 20,
NumRecords: 10,
},
},
"db4": {
Start: 30,
NumRecords: 10,
DBRecords: testutil.DBRecords{
Start: 30,
NumRecords: 10,
},
},
},
"table2": {
"db1": {
Start: 40,
NumRecords: 10,
DBRecords: testutil.DBRecords{
Start: 40,
NumRecords: 10,
},
},
"db2": {
Start: 50,
NumRecords: 10,
DBRecords: testutil.DBRecords{
Start: 50,
NumRecords: 10,
},
},
"db3": {
Start: 60,
NumRecords: 10,
DBRecords: testutil.DBRecords{
Start: 60,
NumRecords: 10,
},
},
"db4": {
Start: 70,
NumRecords: 10,
DBRecords: testutil.DBRecords{
Start: 70,
NumRecords: 10,
},
},
},
}
for name, dbs := range tables {
testutil.SetupDBsAtPath(t, filepath.Join(tablesPath, name), dbs, false, nil)
testutil.SetupDBsAtPath(t, filepath.Join(tablesPath, name), dbs, nil)
// setup exact same copy of dbs for comparison.
testutil.SetupDBsAtPath(t, filepath.Join(tablesCopyPath, name), dbs, false, nil)
testutil.SetupDBsAtPath(t, filepath.Join(tablesCopyPath, name), dbs, nil)
}
compactor := setupTestCompactor(t, tempDir)

13
pkg/storage/stores/shipper/compactor/table_test.go
Unescape View File

@ -435,15 +435,18 @@ func TestTable_CompactionFailure(t *testing.T) {
numDBs := 10
numRecordsPerDB := 100
dbsToSetup := make(map[string]testutil.DBRecords)
dbsToSetup := make(map[string]testutil.DBConfig)
for i := 0; i < numDBs; i++ {
dbsToSetup[fmt.Sprint(i)] = testutil.DBRecords{
Start: i * numRecordsPerDB,
NumRecords: (i + 1) * numRecordsPerDB,
dbsToSetup[fmt.Sprint(i)] = testutil.DBConfig{
CompressFile: i%2 == 0,
DBRecords: testutil.DBRecords{
Start: i * numRecordsPerDB,
NumRecords: (i + 1) * numRecordsPerDB,
},
}
}
testutil.SetupDBsAtPath(t, filepath.Join(objectStoragePath, tableName), dbsToSetup, true, nil)
testutil.SetupDBsAtPath(t, filepath.Join(objectStoragePath, tableName), dbsToSetup, nil)
// put a non-boltdb file in the table which should cause the compaction to fail in the middle because it would fail to open that file with boltdb client.
require.NoError(t, ioutil.WriteFile(filepath.Join(tablePathInStorage, "fail.txt"), []byte("fail the compaction"), 0666))

26
pkg/storage/stores/shipper/downloads/index_set_test.go
Unescape View File

@ -37,7 +37,7 @@ func buildTestIndexSet(t *testing.T, userID, path string) (*indexSet, stopFunc)
func TestIndexSet_Init(t *testing.T) {
tempDir := t.TempDir()
objectStoragePath := filepath.Join(tempDir, objectsStorageDirName)
testDBs := map[string]testutil.DBRecords{}
testDBs := map[string]testutil.DBConfig{}
checkIndexSet := func() {
indexSet, stopFunc := buildTestIndexSet(t, userID, tempDir)
@ -51,13 +51,16 @@ func TestIndexSet_Init(t *testing.T) {
// setup some dbs in object storage
for i := 0; i < 10; i++ {
testDBs[fmt.Sprint(i)] = testutil.DBRecords{
Start: i * 10,
NumRecords: 10,
testDBs[fmt.Sprint(i)] = testutil.DBConfig{
CompressFile: i%2 == 0,
DBRecords: testutil.DBRecords{
Start: i * 10,
NumRecords: 10,
},
}
}
testutil.SetupDBsAtPath(t, filepath.Join(objectStoragePath, tableName, userID), testDBs, true, nil)
testutil.SetupDBsAtPath(t, filepath.Join(objectStoragePath, tableName, userID), testDBs, nil)
// check index set twice; first run to have new files to download, second run to test with no changes in storage.
for i := 0; i < 2; i++ {
@ -86,16 +89,19 @@ func TestIndexSet_doConcurrentDownload(t *testing.T) {
for _, tc := range []int{0, 10, maxDownloadConcurrency, maxDownloadConcurrency * 2} {
t.Run(fmt.Sprintf("%d dbs", tc), func(t *testing.T) {
userID := fmt.Sprint(tc)
testDBs := map[string]testutil.DBRecords{}
testDBs := map[string]testutil.DBConfig{}
for i := 0; i < tc; i++ {
testDBs[fmt.Sprint(i)] = testutil.DBRecords{
Start: i * 10,
NumRecords: 10,
testDBs[fmt.Sprint(i)] = testutil.DBConfig{
CompressFile: i%2 == 0,
DBRecords: testutil.DBRecords{
Start: i * 10,
NumRecords: 10,
},
}
}
testutil.SetupDBsAtPath(t, filepath.Join(objectStoragePath, tableName, userID), testDBs, true, nil)
testutil.SetupDBsAtPath(t, filepath.Join(objectStoragePath, tableName, userID), testDBs, nil)
indexSet, stopFunc := buildTestIndexSet(t, userID, tempDir)
defer func() {

13
pkg/storage/stores/shipper/downloads/table_manager_test.go
Unescape View File

@ -143,19 +143,22 @@ func TestTableManager_ensureQueryReadiness(t *testing.T) {
objectStoragePath := filepath.Join(tempDir, objectsStorageDirName)
tables := map[string]map[string]testutil.DBRecords{}
tables := map[string]map[string]testutil.DBConfig{}
activeTableNumber := getActiveTableNumber()
for i := 0; i < 10; i++ {
tables[fmt.Sprintf("table_%d", activeTableNumber-int64(i))] = map[string]testutil.DBRecords{
tables[fmt.Sprintf("table_%d", activeTableNumber-int64(i))] = map[string]testutil.DBConfig{
"db": {
Start: i * 10,
NumRecords: 10,
CompressFile: i%2 == 0,
DBRecords: testutil.DBRecords{
Start: i * 10,
NumRecords: 10,
},
},
}
}
for name, dbs := range tables {
testutil.SetupDBsAtPath(t, filepath.Join(objectStoragePath, name), dbs, true, nil)
testutil.SetupDBsAtPath(t, filepath.Join(objectStoragePath, name), dbs, nil)
}
boltDBIndexClient, indexStorageClient := buildTestClients(t, tempDir)

138
pkg/storage/stores/shipper/downloads/table_test.go
Unescape View File

@ -255,16 +255,19 @@ func TestTable_DropUnusedIndex(t *testing.T) {
func TestTable_EnsureQueryReadiness(t *testing.T) {
tempDir := t.TempDir()
dbsToSetup := map[string]testutil.DBRecords{
dbsToSetup := map[string]testutil.DBConfig{
"db1": {
Start: 0,
NumRecords: 10,
CompressFile: true,
DBRecords: testutil.DBRecords{
Start: 0,
NumRecords: 10,
},
},
}
objectStoragePath := filepath.Join(tempDir, objectsStorageDirName)
tablePathInStorage := filepath.Join(objectStoragePath, tableName)
testutil.SetupDBsAtPath(t, tablePathInStorage, dbsToSetup, true, nil)
testutil.SetupDBsAtPath(t, tablePathInStorage, dbsToSetup, nil)
table, _, stopFunc := buildTestTable(t, tempDir)
defer func() {
@ -281,7 +284,7 @@ func TestTable_EnsureQueryReadiness(t *testing.T) {
ensureIndexSetExistsInTable(t, table, "")
require.InDelta(t, time.Now().Unix(), table.indexSets[""].(*indexSet).lastUsedAt.Unix(), 5)
testutil.SetupDBsAtPath(t, filepath.Join(tablePathInStorage, userID), dbsToSetup, true, nil)
testutil.SetupDBsAtPath(t, filepath.Join(tablePathInStorage, userID), dbsToSetup, nil)
// Running EnsureQueryReadiness should initialize newly setup index for userID.
// Running it multiple times should behave similarly.
@ -306,19 +309,23 @@ func TestTable_Sync(t *testing.T) {
noUpdatesDB := "no-updates"
newDB := "new"
testDBs := map[string]testutil.DBRecords{
testDBs := map[string]testutil.DBConfig{
deleteDB: {
Start: 0,
NumRecords: 10,
DBRecords: testutil.DBRecords{
Start: 0,
NumRecords: 10,
},
},
noUpdatesDB: {
Start: 10,
NumRecords: 10,
DBRecords: testutil.DBRecords{
Start: 10,
NumRecords: 10,
},
},
}
// setup the table in storage with some records
testutil.SetupDBsAtPath(t, filepath.Join(objectStoragePath, tableName), testDBs, false, nil)
testutil.SetupDBsAtPath(t, filepath.Join(objectStoragePath, tableName), testDBs, nil)
// create table instance
table, boltdbClient, stopFunc := buildTestTable(t, tempDir)
@ -367,46 +374,67 @@ func TestTable_QueryResponse(t *testing.T) {
objectStoragePath := filepath.Join(tempDir, objectsStorageDirName)
tablePathInStorage := filepath.Join(objectStoragePath, tableName)
commonDBs := map[string]testutil.DBRecords{
commonDBs := map[string]testutil.DBConfig{
"db1": {
Start: 0,
NumRecords: 10,
CompressFile: true,
DBRecords: testutil.DBRecords{
Start: 0,
NumRecords: 10,
},
},
"duplicate_db1": {
Start: 0,
NumRecords: 10,
CompressFile: true,
DBRecords: testutil.DBRecords{
Start: 0,
NumRecords: 10,
},
},
"db2": {
Start: 10,
NumRecords: 10,
DBRecords: testutil.DBRecords{
Start: 10,
NumRecords: 10,
},
},
"partially_duplicate_db2": {
Start: 10,
NumRecords: 5,
CompressFile: true,
DBRecords: testutil.DBRecords{
Start: 10,
NumRecords: 5,
},
},
"db3": {
Start: 20,
NumRecords: 10,
DBRecords: testutil.DBRecords{
Start: 20,
NumRecords: 10,
},
},
}
userDBs := map[string]testutil.DBRecords{
userDBs := map[string]testutil.DBConfig{
"overlaps_with_common_dbs": {
Start: 10,
NumRecords: 30,
CompressFile: true,
DBRecords: testutil.DBRecords{
Start: 10,
NumRecords: 30,
},
},
"same_db_again": {
Start: 10,
NumRecords: 20,
DBRecords: testutil.DBRecords{
Start: 10,
NumRecords: 20,
},
},
"additional_records": {
Start: 30,
NumRecords: 10,
CompressFile: true,
DBRecords: testutil.DBRecords{
Start: 30,
NumRecords: 10,
},
},
}
testutil.SetupDBsAtPath(t, tablePathInStorage, commonDBs, true, nil)
testutil.SetupDBsAtPath(t, filepath.Join(tablePathInStorage, userID), userDBs, true, nil)
testutil.SetupDBsAtPath(t, tablePathInStorage, commonDBs, nil)
testutil.SetupDBsAtPath(t, filepath.Join(tablePathInStorage, userID), userDBs, nil)
table, _, stopFunc := buildTestTable(t, tempDir)
defer func() {
@ -439,22 +467,26 @@ func TestLoadTable(t *testing.T) {
objectStoragePath := filepath.Join(tempDir, objectsStorageDirName)
tablePathInStorage := filepath.Join(objectStoragePath, tableName)
commonDBs := make(map[string]testutil.DBRecords)
userDBs := make(map[string]testutil.DBRecords)
commonDBs := make(map[string]testutil.DBConfig)
userDBs := make(map[string]testutil.DBConfig)
for i := 0; i < 10; i++ {
commonDBs[fmt.Sprint(i)] = testutil.DBRecords{
Start: i,
NumRecords: 1,
commonDBs[fmt.Sprint(i)] = testutil.DBConfig{
DBRecords: testutil.DBRecords{
Start: i,
NumRecords: 1,
},
}
userDBs[fmt.Sprint(i+10)] = testutil.DBRecords{
Start: i + 10,
NumRecords: 1,
userDBs[fmt.Sprint(i+10)] = testutil.DBConfig{
DBRecords: testutil.DBRecords{
Start: i + 10,
NumRecords: 1,
},
}
}
// setup the table in storage with some records
testutil.SetupDBsAtPath(t, tablePathInStorage, commonDBs, false, nil)
testutil.SetupDBsAtPath(t, filepath.Join(tablePathInStorage, userID), userDBs, false, nil)
testutil.SetupDBsAtPath(t, tablePathInStorage, commonDBs, nil)
testutil.SetupDBsAtPath(t, filepath.Join(tablePathInStorage, userID), userDBs, nil)
boltDBIndexClient, storageClient := buildTestClients(t, tempDir)
tablePathInCache := filepath.Join(tempDir, cacheDirName, tableName)
@ -481,21 +513,25 @@ func TestLoadTable(t *testing.T) {
require.Error(t, err)
// add some more files to the storage.
commonDBs = make(map[string]testutil.DBRecords)
userDBs = make(map[string]testutil.DBRecords)
commonDBs = make(map[string]testutil.DBConfig)
userDBs = make(map[string]testutil.DBConfig)
for i := 20; i < 30; i++ {
commonDBs[fmt.Sprint(i)] = testutil.DBRecords{
Start: i,
NumRecords: 1,
commonDBs[fmt.Sprint(i)] = testutil.DBConfig{
DBRecords: testutil.DBRecords{
Start: i,
NumRecords: 1,
},
}
userDBs[fmt.Sprint(i+10)] = testutil.DBRecords{
Start: i + 10,
NumRecords: 1,
userDBs[fmt.Sprint(i+10)] = testutil.DBConfig{
DBRecords: testutil.DBRecords{
Start: i + 10,
NumRecords: 1,
},
}
}
testutil.SetupDBsAtPath(t, tablePathInStorage, commonDBs, false, nil)
testutil.SetupDBsAtPath(t, filepath.Join(tablePathInStorage, userID), userDBs, false, nil)
testutil.SetupDBsAtPath(t, tablePathInStorage, commonDBs, nil)
testutil.SetupDBsAtPath(t, filepath.Join(tablePathInStorage, userID), userDBs, nil)
// try loading the table, it should skip loading corrupt file and reload it from storage.
table, err = LoadTable(tableName, tablePathInCache, storageClient, boltDBIndexClient, newMetrics(nil))

72
pkg/storage/stores/shipper/testutil/testutil.go
Unescape View File

@ -140,11 +140,16 @@ func readDB(t *testing.T, db *bbolt.DB) map[string]map[string]string {
return dbRecords
}
type DBConfig struct {
CompressFile bool
DBRecords
}
type DBRecords struct {
Start, NumRecords int
}
func SetupDBsAtPath(t *testing.T, path string, dbs map[string]DBRecords, compressRandomFiles bool, bucketName []byte) string {
func SetupDBsAtPath(t *testing.T, path string, dbs map[string]DBConfig, bucketName []byte) string {
t.Helper()
boltIndexClient, err := local.NewBoltDBIndexClient(local.BoltDBConfig{Directory: path})
require.NoError(t, err)
@ -153,13 +158,11 @@ func SetupDBsAtPath(t *testing.T, path string, dbs map[string]DBRecords, compres
require.NoError(t, chunk_util.EnsureDirectory(path))
var i int
for name, dbRecords := range dbs {
AddRecordsToDB(t, filepath.Join(path, name), boltIndexClient, dbRecords.Start, dbRecords.NumRecords, bucketName)
if compressRandomFiles && i%2 == 0 {
for name, dbConfig := range dbs {
AddRecordsToDB(t, filepath.Join(path, name), boltIndexClient, dbConfig.Start, dbConfig.NumRecords, bucketName)
if dbConfig.CompressFile {
compressFile(t, filepath.Join(path, name))
}
i++
}
return path
@ -227,31 +230,39 @@ func (c PerUserDBsConfig) String() string {
func SetupTable(t *testing.T, path string, commonDBsConfig DBsConfig, perUserDBsConfig PerUserDBsConfig) {
numRecordsPerDB := 100
commonDBsWithDefaultBucket := map[string]DBRecords{}
commonDBsWithPerUserBucket := map[string]map[string]DBRecords{}
perUserDBs := map[string]map[string]DBRecords{}
commonDBsWithDefaultBucket := map[string]DBConfig{}
commonDBsWithPerUserBucket := map[string]map[string]DBConfig{}
perUserDBs := map[string]map[string]DBConfig{}
for i := 0; i < commonDBsConfig.NumUnCompactedDBs; i++ {
commonDBsWithDefaultBucket[fmt.Sprint(i)] = DBRecords{
Start: commonDBsConfig.DBRecordsStart + i*numRecordsPerDB,
NumRecords: numRecordsPerDB,
commonDBsWithDefaultBucket[fmt.Sprint(i)] = DBConfig{
CompressFile: i%2 == 0,
DBRecords: DBRecords{
Start: commonDBsConfig.DBRecordsStart + i*numRecordsPerDB,
NumRecords: numRecordsPerDB,
},
}
}
for i := 0; i < commonDBsConfig.NumCompactedDBs; i++ {
commonDBsWithDefaultBucket[fmt.Sprintf("compactor-%d", i)] = DBRecords{
Start: commonDBsConfig.DBRecordsStart + i*numRecordsPerDB,
NumRecords: ((i + 1) * numRecordsPerDB) * 2,
commonDBsWithDefaultBucket[fmt.Sprintf("compactor-%d", i)] = DBConfig{
CompressFile: i%2 == 0,
DBRecords: DBRecords{
Start: commonDBsConfig.DBRecordsStart + i*numRecordsPerDB,
NumRecords: ((i + 1) * numRecordsPerDB) * 2,
},
}
}
for i := 0; i < perUserDBsConfig.NumUnCompactedDBs; i++ {
dbName := fmt.Sprintf("per-user-bucket-db-%d", i)
commonDBsWithPerUserBucket[dbName] = map[string]DBRecords{}
commonDBsWithPerUserBucket[dbName] = map[string]DBConfig{}
for j := 0; j < perUserDBsConfig.NumUsers; j++ {
commonDBsWithPerUserBucket[dbName][BuildUserID(j)] = DBRecords{
Start: perUserDBsConfig.DBRecordsStart + i*numRecordsPerDB,
NumRecords: numRecordsPerDB,
commonDBsWithPerUserBucket[dbName][BuildUserID(j)] = DBConfig{
DBRecords: DBRecords{
Start: perUserDBsConfig.DBRecordsStart + i*numRecordsPerDB,
NumRecords: numRecordsPerDB,
},
}
}
}
@ -260,27 +271,30 @@ func SetupTable(t *testing.T, path string, commonDBsConfig DBsConfig, perUserDBs
for j := 0; j < perUserDBsConfig.NumUsers; j++ {
userID := BuildUserID(j)
if i == 0 {
perUserDBs[userID] = map[string]DBRecords{}
perUserDBs[userID] = map[string]DBConfig{}
}
perUserDBs[userID][fmt.Sprintf("compactor-%d", i)] = DBRecords{
Start: perUserDBsConfig.DBRecordsStart + i*numRecordsPerDB,
NumRecords: (i + 1) * numRecordsPerDB,
perUserDBs[userID][fmt.Sprintf("compactor-%d", i)] = DBConfig{
CompressFile: i%2 == 0,
DBRecords: DBRecords{
Start: perUserDBsConfig.DBRecordsStart + i*numRecordsPerDB,
NumRecords: (i + 1) * numRecordsPerDB,
},
}
}
}
SetupDBsAtPath(t, path, commonDBsWithDefaultBucket, true, local.IndexBucketName)
SetupDBsAtPath(t, path, commonDBsWithDefaultBucket, local.IndexBucketName)
for dbName, userRecords := range commonDBsWithPerUserBucket {
for userID, dbRecords := range userRecords {
SetupDBsAtPath(t, path, map[string]DBRecords{
dbName: dbRecords,
}, false, []byte(userID))
for userID, dbConfig := range userRecords {
SetupDBsAtPath(t, path, map[string]DBConfig{
dbName: dbConfig,
}, []byte(userID))
}
}
for userID, dbRecords := range perUserDBs {
SetupDBsAtPath(t, filepath.Join(path, userID), dbRecords, true, local.IndexBucketName)
SetupDBsAtPath(t, filepath.Join(path, userID), dbRecords, local.IndexBucketName)
}
}

32
pkg/storage/stores/shipper/uploads/table_manager_test.go
Unescape View File

@ -56,28 +56,36 @@ func TestLoadTables(t *testing.T) {
testutil.AddRecordsToDB(t, filepath.Join(indexPath, "table0"), boltDBIndexClient, 0, 10, nil)
// table1 with 2 dbs
testutil.SetupDBsAtPath(t, filepath.Join(indexPath, "table1"), map[string]testutil.DBRecords{
testutil.SetupDBsAtPath(t, filepath.Join(indexPath, "table1"), map[string]testutil.DBConfig{
"db1": {
Start: 10,
NumRecords: 10,
DBRecords: testutil.DBRecords{
Start: 10,
NumRecords: 10,
},
},
"db2": {
Start: 20,
NumRecords: 10,
DBRecords: testutil.DBRecords{
Start: 20,
NumRecords: 10,
},
},
}, false, nil)
}, nil)
// table2 with 2 dbs
testutil.SetupDBsAtPath(t, filepath.Join(indexPath, "table2"), map[string]testutil.DBRecords{
testutil.SetupDBsAtPath(t, filepath.Join(indexPath, "table2"), map[string]testutil.DBConfig{
"db1": {
Start: 30,
NumRecords: 10,
DBRecords: testutil.DBRecords{
Start: 30,
NumRecords: 10,
},
},
"db2": {
Start: 40,
NumRecords: 10,
DBRecords: testutil.DBRecords{
Start: 40,
NumRecords: 10,
},
},
}, false, nil)
}, nil)
expectedTables := map[string]struct {
start, numRecords int

95
pkg/storage/stores/shipper/uploads/table_test.go
Unescape View File

@ -73,26 +73,34 @@ func TestLoadTable(t *testing.T) {
// setup some dbs with default bucket and per tenant bucket for a table at a path.
tablePath := filepath.Join(indexPath, "test-table")
testutil.SetupDBsAtPath(t, tablePath, map[string]testutil.DBRecords{
testutil.SetupDBsAtPath(t, tablePath, map[string]testutil.DBConfig{
"db1": {
Start: 0,
NumRecords: 10,
DBRecords: testutil.DBRecords{
Start: 0,
NumRecords: 10,
},
},
"db2": {
Start: 10,
NumRecords: 10,
DBRecords: testutil.DBRecords{
Start: 10,
NumRecords: 10,
},
},
}, false, nil)
testutil.SetupDBsAtPath(t, tablePath, map[string]testutil.DBRecords{
}, nil)
testutil.SetupDBsAtPath(t, tablePath, map[string]testutil.DBConfig{
"db3": {
Start: 20,
NumRecords: 10,
DBRecords: testutil.DBRecords{
Start: 20,
NumRecords: 10,
},
},
"db4": {
Start: 30,
NumRecords: 10,
DBRecords: testutil.DBRecords{
Start: 30,
NumRecords: 10,
},
},
}, false, []byte(userID))
}, []byte(userID))
// change a boltdb file to text file which would fail to open.
invalidFilePath := filepath.Join(tablePath, "invalid")
@ -360,20 +368,26 @@ func Test_LoadBoltDBsFromDir(t *testing.T) {
}()
// setup some dbs with a snapshot file.
tablePath := testutil.SetupDBsAtPath(t, filepath.Join(indexPath, "test-table"), map[string]testutil.DBRecords{
tablePath := testutil.SetupDBsAtPath(t, filepath.Join(indexPath, "test-table"), map[string]testutil.DBConfig{
"db1": {
Start: 0,
NumRecords: 10,
DBRecords: testutil.DBRecords{
Start: 0,
NumRecords: 10,
},
},
"db1" + tempFileSuffix: { // a snapshot file which should be ignored.
Start: 0,
NumRecords: 10,
DBRecords: testutil.DBRecords{
Start: 0,
NumRecords: 10,
},
},
"db2": {
Start: 10,
NumRecords: 10,
DBRecords: testutil.DBRecords{
Start: 10,
NumRecords: 10,
},
},
}, false, nil)
}, nil)
// create a boltdb file without bucket which should get removed
db, err := local.OpenBoltdbFile(filepath.Join(tablePath, "no-bucket"))
@ -424,16 +438,18 @@ func TestTable_ImmutableUploads(t *testing.T) {
time.Now().Truncate(ShardDBsByDuration).Unix(), // active shard, should not upload
}
dbs := map[string]testutil.DBRecords{}
dbs := map[string]testutil.DBConfig{}
for _, dbName := range dbNames {
dbs[fmt.Sprint(dbName)] = testutil.DBRecords{
NumRecords: 10,
dbs[fmt.Sprint(dbName)] = testutil.DBConfig{
DBRecords: testutil.DBRecords{
NumRecords: 10,
},
}
}
// setup some dbs for a table at a path.
tableName := "test-table"
tablePath := testutil.SetupDBsAtPath(t, filepath.Join(indexPath, tableName), dbs, false, nil)
tablePath := testutil.SetupDBsAtPath(t, filepath.Join(indexPath, tableName), dbs, nil)
table, err := LoadTable(tablePath, "test", storageClient, boltDBIndexClient, false, newMetrics(nil))
require.NoError(t, err)
@ -507,26 +523,33 @@ func TestTable_MultiQueries(t *testing.T) {
// setup some dbs with default bucket and per tenant bucket for a table at a path.
tablePath := filepath.Join(indexPath, "test-table")
testutil.SetupDBsAtPath(t, tablePath, map[string]testutil.DBRecords{
testutil.SetupDBsAtPath(t, tablePath, map[string]testutil.DBConfig{
"db1": {
Start: 0,
NumRecords: 10,
DBRecords: testutil.DBRecords{
NumRecords: 10,
},
},
"db2": {
Start: 10,
NumRecords: 10,
DBRecords: testutil.DBRecords{
Start: 10,
NumRecords: 10,
},
},
}, false, nil)
testutil.SetupDBsAtPath(t, tablePath, map[string]testutil.DBRecords{
}, nil)
testutil.SetupDBsAtPath(t, tablePath, map[string]testutil.DBConfig{
"db3": {
Start: 20,
NumRecords: 10,
DBRecords: testutil.DBRecords{
Start: 20,
NumRecords: 10,
},
},
"db4": {
Start: 30,
NumRecords: 10,
DBRecords: testutil.DBRecords{
Start: 30,
NumRecords: 10,
},
},
}, false, []byte(user1))
}, []byte(user1))
// try loading the table.
table, err := LoadTable(tablePath, "test", nil, boltDBIndexClient, false, newMetrics(nil))

Write Preview
Loading…
Cancel
Save