diff --git a/pkg/storage/stores/shipper/compactor/compactor_test.go b/pkg/storage/stores/shipper/compactor/compactor_test.go index 2a62aa4246..165ad35cb7 100644 --- a/pkg/storage/stores/shipper/compactor/compactor_test.go +++ b/pkg/storage/stores/shipper/compactor/compactor_test.go @@ -48,50 +48,66 @@ func TestCompactor_RunCompaction(t *testing.T) { tablesPath := filepath.Join(tempDir, "index") tablesCopyPath := filepath.Join(tempDir, "index-copy") - tables := map[string]map[string]testutil.DBRecords{ + tables := map[string]map[string]testutil.DBConfig{ "table1": { "db1": { - Start: 0, - NumRecords: 10, + DBRecords: testutil.DBRecords{ + Start: 0, + NumRecords: 10, + }, }, "db2": { - Start: 10, - NumRecords: 10, + DBRecords: testutil.DBRecords{ + Start: 10, + NumRecords: 10, + }, }, "db3": { - Start: 20, - NumRecords: 10, + DBRecords: testutil.DBRecords{ + Start: 20, + NumRecords: 10, + }, }, "db4": { - Start: 30, - NumRecords: 10, + DBRecords: testutil.DBRecords{ + Start: 30, + NumRecords: 10, + }, }, }, "table2": { "db1": { - Start: 40, - NumRecords: 10, + DBRecords: testutil.DBRecords{ + Start: 40, + NumRecords: 10, + }, }, "db2": { - Start: 50, - NumRecords: 10, + DBRecords: testutil.DBRecords{ + Start: 50, + NumRecords: 10, + }, }, "db3": { - Start: 60, - NumRecords: 10, + DBRecords: testutil.DBRecords{ + Start: 60, + NumRecords: 10, + }, }, "db4": { - Start: 70, - NumRecords: 10, + DBRecords: testutil.DBRecords{ + Start: 70, + NumRecords: 10, + }, }, }, } for name, dbs := range tables { - testutil.SetupDBsAtPath(t, filepath.Join(tablesPath, name), dbs, false, nil) + testutil.SetupDBsAtPath(t, filepath.Join(tablesPath, name), dbs, nil) // setup exact same copy of dbs for comparison. - testutil.SetupDBsAtPath(t, filepath.Join(tablesCopyPath, name), dbs, false, nil) + testutil.SetupDBsAtPath(t, filepath.Join(tablesCopyPath, name), dbs, nil) } compactor := setupTestCompactor(t, tempDir) diff --git a/pkg/storage/stores/shipper/compactor/table_test.go b/pkg/storage/stores/shipper/compactor/table_test.go index 88dc3b08e3..6bcc5d4ea0 100644 --- a/pkg/storage/stores/shipper/compactor/table_test.go +++ b/pkg/storage/stores/shipper/compactor/table_test.go @@ -435,15 +435,18 @@ func TestTable_CompactionFailure(t *testing.T) { numDBs := 10 numRecordsPerDB := 100 - dbsToSetup := make(map[string]testutil.DBRecords) + dbsToSetup := make(map[string]testutil.DBConfig) for i := 0; i < numDBs; i++ { - dbsToSetup[fmt.Sprint(i)] = testutil.DBRecords{ - Start: i * numRecordsPerDB, - NumRecords: (i + 1) * numRecordsPerDB, + dbsToSetup[fmt.Sprint(i)] = testutil.DBConfig{ + CompressFile: i%2 == 0, + DBRecords: testutil.DBRecords{ + Start: i * numRecordsPerDB, + NumRecords: (i + 1) * numRecordsPerDB, + }, } } - testutil.SetupDBsAtPath(t, filepath.Join(objectStoragePath, tableName), dbsToSetup, true, nil) + testutil.SetupDBsAtPath(t, filepath.Join(objectStoragePath, tableName), dbsToSetup, nil) // put a non-boltdb file in the table which should cause the compaction to fail in the middle because it would fail to open that file with boltdb client. require.NoError(t, ioutil.WriteFile(filepath.Join(tablePathInStorage, "fail.txt"), []byte("fail the compaction"), 0666)) diff --git a/pkg/storage/stores/shipper/downloads/index_set_test.go b/pkg/storage/stores/shipper/downloads/index_set_test.go index 3d48b1b277..6afbcdb005 100644 --- a/pkg/storage/stores/shipper/downloads/index_set_test.go +++ b/pkg/storage/stores/shipper/downloads/index_set_test.go @@ -37,7 +37,7 @@ func buildTestIndexSet(t *testing.T, userID, path string) (*indexSet, stopFunc) func TestIndexSet_Init(t *testing.T) { tempDir := t.TempDir() objectStoragePath := filepath.Join(tempDir, objectsStorageDirName) - testDBs := map[string]testutil.DBRecords{} + testDBs := map[string]testutil.DBConfig{} checkIndexSet := func() { indexSet, stopFunc := buildTestIndexSet(t, userID, tempDir) @@ -51,13 +51,16 @@ func TestIndexSet_Init(t *testing.T) { // setup some dbs in object storage for i := 0; i < 10; i++ { - testDBs[fmt.Sprint(i)] = testutil.DBRecords{ - Start: i * 10, - NumRecords: 10, + testDBs[fmt.Sprint(i)] = testutil.DBConfig{ + CompressFile: i%2 == 0, + DBRecords: testutil.DBRecords{ + Start: i * 10, + NumRecords: 10, + }, } } - testutil.SetupDBsAtPath(t, filepath.Join(objectStoragePath, tableName, userID), testDBs, true, nil) + testutil.SetupDBsAtPath(t, filepath.Join(objectStoragePath, tableName, userID), testDBs, nil) // check index set twice; first run to have new files to download, second run to test with no changes in storage. for i := 0; i < 2; i++ { @@ -86,16 +89,19 @@ func TestIndexSet_doConcurrentDownload(t *testing.T) { for _, tc := range []int{0, 10, maxDownloadConcurrency, maxDownloadConcurrency * 2} { t.Run(fmt.Sprintf("%d dbs", tc), func(t *testing.T) { userID := fmt.Sprint(tc) - testDBs := map[string]testutil.DBRecords{} + testDBs := map[string]testutil.DBConfig{} for i := 0; i < tc; i++ { - testDBs[fmt.Sprint(i)] = testutil.DBRecords{ - Start: i * 10, - NumRecords: 10, + testDBs[fmt.Sprint(i)] = testutil.DBConfig{ + CompressFile: i%2 == 0, + DBRecords: testutil.DBRecords{ + Start: i * 10, + NumRecords: 10, + }, } } - testutil.SetupDBsAtPath(t, filepath.Join(objectStoragePath, tableName, userID), testDBs, true, nil) + testutil.SetupDBsAtPath(t, filepath.Join(objectStoragePath, tableName, userID), testDBs, nil) indexSet, stopFunc := buildTestIndexSet(t, userID, tempDir) defer func() { diff --git a/pkg/storage/stores/shipper/downloads/table_manager_test.go b/pkg/storage/stores/shipper/downloads/table_manager_test.go index 49b122cfda..a4f0a02941 100644 --- a/pkg/storage/stores/shipper/downloads/table_manager_test.go +++ b/pkg/storage/stores/shipper/downloads/table_manager_test.go @@ -143,19 +143,22 @@ func TestTableManager_ensureQueryReadiness(t *testing.T) { objectStoragePath := filepath.Join(tempDir, objectsStorageDirName) - tables := map[string]map[string]testutil.DBRecords{} + tables := map[string]map[string]testutil.DBConfig{} activeTableNumber := getActiveTableNumber() for i := 0; i < 10; i++ { - tables[fmt.Sprintf("table_%d", activeTableNumber-int64(i))] = map[string]testutil.DBRecords{ + tables[fmt.Sprintf("table_%d", activeTableNumber-int64(i))] = map[string]testutil.DBConfig{ "db": { - Start: i * 10, - NumRecords: 10, + CompressFile: i%2 == 0, + DBRecords: testutil.DBRecords{ + Start: i * 10, + NumRecords: 10, + }, }, } } for name, dbs := range tables { - testutil.SetupDBsAtPath(t, filepath.Join(objectStoragePath, name), dbs, true, nil) + testutil.SetupDBsAtPath(t, filepath.Join(objectStoragePath, name), dbs, nil) } boltDBIndexClient, indexStorageClient := buildTestClients(t, tempDir) diff --git a/pkg/storage/stores/shipper/downloads/table_test.go b/pkg/storage/stores/shipper/downloads/table_test.go index c302b448ff..985991df40 100644 --- a/pkg/storage/stores/shipper/downloads/table_test.go +++ b/pkg/storage/stores/shipper/downloads/table_test.go @@ -255,16 +255,19 @@ func TestTable_DropUnusedIndex(t *testing.T) { func TestTable_EnsureQueryReadiness(t *testing.T) { tempDir := t.TempDir() - dbsToSetup := map[string]testutil.DBRecords{ + dbsToSetup := map[string]testutil.DBConfig{ "db1": { - Start: 0, - NumRecords: 10, + CompressFile: true, + DBRecords: testutil.DBRecords{ + Start: 0, + NumRecords: 10, + }, }, } objectStoragePath := filepath.Join(tempDir, objectsStorageDirName) tablePathInStorage := filepath.Join(objectStoragePath, tableName) - testutil.SetupDBsAtPath(t, tablePathInStorage, dbsToSetup, true, nil) + testutil.SetupDBsAtPath(t, tablePathInStorage, dbsToSetup, nil) table, _, stopFunc := buildTestTable(t, tempDir) defer func() { @@ -281,7 +284,7 @@ func TestTable_EnsureQueryReadiness(t *testing.T) { ensureIndexSetExistsInTable(t, table, "") require.InDelta(t, time.Now().Unix(), table.indexSets[""].(*indexSet).lastUsedAt.Unix(), 5) - testutil.SetupDBsAtPath(t, filepath.Join(tablePathInStorage, userID), dbsToSetup, true, nil) + testutil.SetupDBsAtPath(t, filepath.Join(tablePathInStorage, userID), dbsToSetup, nil) // Running EnsureQueryReadiness should initialize newly setup index for userID. // Running it multiple times should behave similarly. @@ -306,19 +309,23 @@ func TestTable_Sync(t *testing.T) { noUpdatesDB := "no-updates" newDB := "new" - testDBs := map[string]testutil.DBRecords{ + testDBs := map[string]testutil.DBConfig{ deleteDB: { - Start: 0, - NumRecords: 10, + DBRecords: testutil.DBRecords{ + Start: 0, + NumRecords: 10, + }, }, noUpdatesDB: { - Start: 10, - NumRecords: 10, + DBRecords: testutil.DBRecords{ + Start: 10, + NumRecords: 10, + }, }, } // setup the table in storage with some records - testutil.SetupDBsAtPath(t, filepath.Join(objectStoragePath, tableName), testDBs, false, nil) + testutil.SetupDBsAtPath(t, filepath.Join(objectStoragePath, tableName), testDBs, nil) // create table instance table, boltdbClient, stopFunc := buildTestTable(t, tempDir) @@ -367,46 +374,67 @@ func TestTable_QueryResponse(t *testing.T) { objectStoragePath := filepath.Join(tempDir, objectsStorageDirName) tablePathInStorage := filepath.Join(objectStoragePath, tableName) - commonDBs := map[string]testutil.DBRecords{ + commonDBs := map[string]testutil.DBConfig{ "db1": { - Start: 0, - NumRecords: 10, + CompressFile: true, + DBRecords: testutil.DBRecords{ + Start: 0, + NumRecords: 10, + }, }, "duplicate_db1": { - Start: 0, - NumRecords: 10, + CompressFile: true, + DBRecords: testutil.DBRecords{ + Start: 0, + NumRecords: 10, + }, }, "db2": { - Start: 10, - NumRecords: 10, + DBRecords: testutil.DBRecords{ + Start: 10, + NumRecords: 10, + }, }, "partially_duplicate_db2": { - Start: 10, - NumRecords: 5, + CompressFile: true, + DBRecords: testutil.DBRecords{ + Start: 10, + NumRecords: 5, + }, }, "db3": { - Start: 20, - NumRecords: 10, + DBRecords: testutil.DBRecords{ + Start: 20, + NumRecords: 10, + }, }, } - userDBs := map[string]testutil.DBRecords{ + userDBs := map[string]testutil.DBConfig{ "overlaps_with_common_dbs": { - Start: 10, - NumRecords: 30, + CompressFile: true, + DBRecords: testutil.DBRecords{ + Start: 10, + NumRecords: 30, + }, }, "same_db_again": { - Start: 10, - NumRecords: 20, + DBRecords: testutil.DBRecords{ + Start: 10, + NumRecords: 20, + }, }, "additional_records": { - Start: 30, - NumRecords: 10, + CompressFile: true, + DBRecords: testutil.DBRecords{ + Start: 30, + NumRecords: 10, + }, }, } - testutil.SetupDBsAtPath(t, tablePathInStorage, commonDBs, true, nil) - testutil.SetupDBsAtPath(t, filepath.Join(tablePathInStorage, userID), userDBs, true, nil) + testutil.SetupDBsAtPath(t, tablePathInStorage, commonDBs, nil) + testutil.SetupDBsAtPath(t, filepath.Join(tablePathInStorage, userID), userDBs, nil) table, _, stopFunc := buildTestTable(t, tempDir) defer func() { @@ -439,22 +467,26 @@ func TestLoadTable(t *testing.T) { objectStoragePath := filepath.Join(tempDir, objectsStorageDirName) tablePathInStorage := filepath.Join(objectStoragePath, tableName) - commonDBs := make(map[string]testutil.DBRecords) - userDBs := make(map[string]testutil.DBRecords) + commonDBs := make(map[string]testutil.DBConfig) + userDBs := make(map[string]testutil.DBConfig) for i := 0; i < 10; i++ { - commonDBs[fmt.Sprint(i)] = testutil.DBRecords{ - Start: i, - NumRecords: 1, + commonDBs[fmt.Sprint(i)] = testutil.DBConfig{ + DBRecords: testutil.DBRecords{ + Start: i, + NumRecords: 1, + }, } - userDBs[fmt.Sprint(i+10)] = testutil.DBRecords{ - Start: i + 10, - NumRecords: 1, + userDBs[fmt.Sprint(i+10)] = testutil.DBConfig{ + DBRecords: testutil.DBRecords{ + Start: i + 10, + NumRecords: 1, + }, } } // setup the table in storage with some records - testutil.SetupDBsAtPath(t, tablePathInStorage, commonDBs, false, nil) - testutil.SetupDBsAtPath(t, filepath.Join(tablePathInStorage, userID), userDBs, false, nil) + testutil.SetupDBsAtPath(t, tablePathInStorage, commonDBs, nil) + testutil.SetupDBsAtPath(t, filepath.Join(tablePathInStorage, userID), userDBs, nil) boltDBIndexClient, storageClient := buildTestClients(t, tempDir) tablePathInCache := filepath.Join(tempDir, cacheDirName, tableName) @@ -481,21 +513,25 @@ func TestLoadTable(t *testing.T) { require.Error(t, err) // add some more files to the storage. - commonDBs = make(map[string]testutil.DBRecords) - userDBs = make(map[string]testutil.DBRecords) + commonDBs = make(map[string]testutil.DBConfig) + userDBs = make(map[string]testutil.DBConfig) for i := 20; i < 30; i++ { - commonDBs[fmt.Sprint(i)] = testutil.DBRecords{ - Start: i, - NumRecords: 1, + commonDBs[fmt.Sprint(i)] = testutil.DBConfig{ + DBRecords: testutil.DBRecords{ + Start: i, + NumRecords: 1, + }, } - userDBs[fmt.Sprint(i+10)] = testutil.DBRecords{ - Start: i + 10, - NumRecords: 1, + userDBs[fmt.Sprint(i+10)] = testutil.DBConfig{ + DBRecords: testutil.DBRecords{ + Start: i + 10, + NumRecords: 1, + }, } } - testutil.SetupDBsAtPath(t, tablePathInStorage, commonDBs, false, nil) - testutil.SetupDBsAtPath(t, filepath.Join(tablePathInStorage, userID), userDBs, false, nil) + testutil.SetupDBsAtPath(t, tablePathInStorage, commonDBs, nil) + testutil.SetupDBsAtPath(t, filepath.Join(tablePathInStorage, userID), userDBs, nil) // try loading the table, it should skip loading corrupt file and reload it from storage. table, err = LoadTable(tableName, tablePathInCache, storageClient, boltDBIndexClient, newMetrics(nil)) diff --git a/pkg/storage/stores/shipper/testutil/testutil.go b/pkg/storage/stores/shipper/testutil/testutil.go index 838f9d7502..8d54f07685 100644 --- a/pkg/storage/stores/shipper/testutil/testutil.go +++ b/pkg/storage/stores/shipper/testutil/testutil.go @@ -140,11 +140,16 @@ func readDB(t *testing.T, db *bbolt.DB) map[string]map[string]string { return dbRecords } +type DBConfig struct { + CompressFile bool + DBRecords +} + type DBRecords struct { Start, NumRecords int } -func SetupDBsAtPath(t *testing.T, path string, dbs map[string]DBRecords, compressRandomFiles bool, bucketName []byte) string { +func SetupDBsAtPath(t *testing.T, path string, dbs map[string]DBConfig, bucketName []byte) string { t.Helper() boltIndexClient, err := local.NewBoltDBIndexClient(local.BoltDBConfig{Directory: path}) require.NoError(t, err) @@ -153,13 +158,11 @@ func SetupDBsAtPath(t *testing.T, path string, dbs map[string]DBRecords, compres require.NoError(t, chunk_util.EnsureDirectory(path)) - var i int - for name, dbRecords := range dbs { - AddRecordsToDB(t, filepath.Join(path, name), boltIndexClient, dbRecords.Start, dbRecords.NumRecords, bucketName) - if compressRandomFiles && i%2 == 0 { + for name, dbConfig := range dbs { + AddRecordsToDB(t, filepath.Join(path, name), boltIndexClient, dbConfig.Start, dbConfig.NumRecords, bucketName) + if dbConfig.CompressFile { compressFile(t, filepath.Join(path, name)) } - i++ } return path @@ -227,31 +230,39 @@ func (c PerUserDBsConfig) String() string { func SetupTable(t *testing.T, path string, commonDBsConfig DBsConfig, perUserDBsConfig PerUserDBsConfig) { numRecordsPerDB := 100 - commonDBsWithDefaultBucket := map[string]DBRecords{} - commonDBsWithPerUserBucket := map[string]map[string]DBRecords{} - perUserDBs := map[string]map[string]DBRecords{} + commonDBsWithDefaultBucket := map[string]DBConfig{} + commonDBsWithPerUserBucket := map[string]map[string]DBConfig{} + perUserDBs := map[string]map[string]DBConfig{} for i := 0; i < commonDBsConfig.NumUnCompactedDBs; i++ { - commonDBsWithDefaultBucket[fmt.Sprint(i)] = DBRecords{ - Start: commonDBsConfig.DBRecordsStart + i*numRecordsPerDB, - NumRecords: numRecordsPerDB, + commonDBsWithDefaultBucket[fmt.Sprint(i)] = DBConfig{ + CompressFile: i%2 == 0, + DBRecords: DBRecords{ + Start: commonDBsConfig.DBRecordsStart + i*numRecordsPerDB, + NumRecords: numRecordsPerDB, + }, } } for i := 0; i < commonDBsConfig.NumCompactedDBs; i++ { - commonDBsWithDefaultBucket[fmt.Sprintf("compactor-%d", i)] = DBRecords{ - Start: commonDBsConfig.DBRecordsStart + i*numRecordsPerDB, - NumRecords: ((i + 1) * numRecordsPerDB) * 2, + commonDBsWithDefaultBucket[fmt.Sprintf("compactor-%d", i)] = DBConfig{ + CompressFile: i%2 == 0, + DBRecords: DBRecords{ + Start: commonDBsConfig.DBRecordsStart + i*numRecordsPerDB, + NumRecords: ((i + 1) * numRecordsPerDB) * 2, + }, } } for i := 0; i < perUserDBsConfig.NumUnCompactedDBs; i++ { dbName := fmt.Sprintf("per-user-bucket-db-%d", i) - commonDBsWithPerUserBucket[dbName] = map[string]DBRecords{} + commonDBsWithPerUserBucket[dbName] = map[string]DBConfig{} for j := 0; j < perUserDBsConfig.NumUsers; j++ { - commonDBsWithPerUserBucket[dbName][BuildUserID(j)] = DBRecords{ - Start: perUserDBsConfig.DBRecordsStart + i*numRecordsPerDB, - NumRecords: numRecordsPerDB, + commonDBsWithPerUserBucket[dbName][BuildUserID(j)] = DBConfig{ + DBRecords: DBRecords{ + Start: perUserDBsConfig.DBRecordsStart + i*numRecordsPerDB, + NumRecords: numRecordsPerDB, + }, } } } @@ -260,27 +271,30 @@ func SetupTable(t *testing.T, path string, commonDBsConfig DBsConfig, perUserDBs for j := 0; j < perUserDBsConfig.NumUsers; j++ { userID := BuildUserID(j) if i == 0 { - perUserDBs[userID] = map[string]DBRecords{} + perUserDBs[userID] = map[string]DBConfig{} } - perUserDBs[userID][fmt.Sprintf("compactor-%d", i)] = DBRecords{ - Start: perUserDBsConfig.DBRecordsStart + i*numRecordsPerDB, - NumRecords: (i + 1) * numRecordsPerDB, + perUserDBs[userID][fmt.Sprintf("compactor-%d", i)] = DBConfig{ + CompressFile: i%2 == 0, + DBRecords: DBRecords{ + Start: perUserDBsConfig.DBRecordsStart + i*numRecordsPerDB, + NumRecords: (i + 1) * numRecordsPerDB, + }, } } } - SetupDBsAtPath(t, path, commonDBsWithDefaultBucket, true, local.IndexBucketName) + SetupDBsAtPath(t, path, commonDBsWithDefaultBucket, local.IndexBucketName) for dbName, userRecords := range commonDBsWithPerUserBucket { - for userID, dbRecords := range userRecords { - SetupDBsAtPath(t, path, map[string]DBRecords{ - dbName: dbRecords, - }, false, []byte(userID)) + for userID, dbConfig := range userRecords { + SetupDBsAtPath(t, path, map[string]DBConfig{ + dbName: dbConfig, + }, []byte(userID)) } } for userID, dbRecords := range perUserDBs { - SetupDBsAtPath(t, filepath.Join(path, userID), dbRecords, true, local.IndexBucketName) + SetupDBsAtPath(t, filepath.Join(path, userID), dbRecords, local.IndexBucketName) } } diff --git a/pkg/storage/stores/shipper/uploads/table_manager_test.go b/pkg/storage/stores/shipper/uploads/table_manager_test.go index 879f992e79..5ec282448d 100644 --- a/pkg/storage/stores/shipper/uploads/table_manager_test.go +++ b/pkg/storage/stores/shipper/uploads/table_manager_test.go @@ -56,28 +56,36 @@ func TestLoadTables(t *testing.T) { testutil.AddRecordsToDB(t, filepath.Join(indexPath, "table0"), boltDBIndexClient, 0, 10, nil) // table1 with 2 dbs - testutil.SetupDBsAtPath(t, filepath.Join(indexPath, "table1"), map[string]testutil.DBRecords{ + testutil.SetupDBsAtPath(t, filepath.Join(indexPath, "table1"), map[string]testutil.DBConfig{ "db1": { - Start: 10, - NumRecords: 10, + DBRecords: testutil.DBRecords{ + Start: 10, + NumRecords: 10, + }, }, "db2": { - Start: 20, - NumRecords: 10, + DBRecords: testutil.DBRecords{ + Start: 20, + NumRecords: 10, + }, }, - }, false, nil) + }, nil) // table2 with 2 dbs - testutil.SetupDBsAtPath(t, filepath.Join(indexPath, "table2"), map[string]testutil.DBRecords{ + testutil.SetupDBsAtPath(t, filepath.Join(indexPath, "table2"), map[string]testutil.DBConfig{ "db1": { - Start: 30, - NumRecords: 10, + DBRecords: testutil.DBRecords{ + Start: 30, + NumRecords: 10, + }, }, "db2": { - Start: 40, - NumRecords: 10, + DBRecords: testutil.DBRecords{ + Start: 40, + NumRecords: 10, + }, }, - }, false, nil) + }, nil) expectedTables := map[string]struct { start, numRecords int diff --git a/pkg/storage/stores/shipper/uploads/table_test.go b/pkg/storage/stores/shipper/uploads/table_test.go index 8dcdaa8a8e..ca611f15fc 100644 --- a/pkg/storage/stores/shipper/uploads/table_test.go +++ b/pkg/storage/stores/shipper/uploads/table_test.go @@ -73,26 +73,34 @@ func TestLoadTable(t *testing.T) { // setup some dbs with default bucket and per tenant bucket for a table at a path. tablePath := filepath.Join(indexPath, "test-table") - testutil.SetupDBsAtPath(t, tablePath, map[string]testutil.DBRecords{ + testutil.SetupDBsAtPath(t, tablePath, map[string]testutil.DBConfig{ "db1": { - Start: 0, - NumRecords: 10, + DBRecords: testutil.DBRecords{ + Start: 0, + NumRecords: 10, + }, }, "db2": { - Start: 10, - NumRecords: 10, + DBRecords: testutil.DBRecords{ + Start: 10, + NumRecords: 10, + }, }, - }, false, nil) - testutil.SetupDBsAtPath(t, tablePath, map[string]testutil.DBRecords{ + }, nil) + testutil.SetupDBsAtPath(t, tablePath, map[string]testutil.DBConfig{ "db3": { - Start: 20, - NumRecords: 10, + DBRecords: testutil.DBRecords{ + Start: 20, + NumRecords: 10, + }, }, "db4": { - Start: 30, - NumRecords: 10, + DBRecords: testutil.DBRecords{ + Start: 30, + NumRecords: 10, + }, }, - }, false, []byte(userID)) + }, []byte(userID)) // change a boltdb file to text file which would fail to open. invalidFilePath := filepath.Join(tablePath, "invalid") @@ -360,20 +368,26 @@ func Test_LoadBoltDBsFromDir(t *testing.T) { }() // setup some dbs with a snapshot file. - tablePath := testutil.SetupDBsAtPath(t, filepath.Join(indexPath, "test-table"), map[string]testutil.DBRecords{ + tablePath := testutil.SetupDBsAtPath(t, filepath.Join(indexPath, "test-table"), map[string]testutil.DBConfig{ "db1": { - Start: 0, - NumRecords: 10, + DBRecords: testutil.DBRecords{ + Start: 0, + NumRecords: 10, + }, }, "db1" + tempFileSuffix: { // a snapshot file which should be ignored. - Start: 0, - NumRecords: 10, + DBRecords: testutil.DBRecords{ + Start: 0, + NumRecords: 10, + }, }, "db2": { - Start: 10, - NumRecords: 10, + DBRecords: testutil.DBRecords{ + Start: 10, + NumRecords: 10, + }, }, - }, false, nil) + }, nil) // create a boltdb file without bucket which should get removed db, err := local.OpenBoltdbFile(filepath.Join(tablePath, "no-bucket")) @@ -424,16 +438,18 @@ func TestTable_ImmutableUploads(t *testing.T) { time.Now().Truncate(ShardDBsByDuration).Unix(), // active shard, should not upload } - dbs := map[string]testutil.DBRecords{} + dbs := map[string]testutil.DBConfig{} for _, dbName := range dbNames { - dbs[fmt.Sprint(dbName)] = testutil.DBRecords{ - NumRecords: 10, + dbs[fmt.Sprint(dbName)] = testutil.DBConfig{ + DBRecords: testutil.DBRecords{ + NumRecords: 10, + }, } } // setup some dbs for a table at a path. tableName := "test-table" - tablePath := testutil.SetupDBsAtPath(t, filepath.Join(indexPath, tableName), dbs, false, nil) + tablePath := testutil.SetupDBsAtPath(t, filepath.Join(indexPath, tableName), dbs, nil) table, err := LoadTable(tablePath, "test", storageClient, boltDBIndexClient, false, newMetrics(nil)) require.NoError(t, err) @@ -507,26 +523,33 @@ func TestTable_MultiQueries(t *testing.T) { // setup some dbs with default bucket and per tenant bucket for a table at a path. tablePath := filepath.Join(indexPath, "test-table") - testutil.SetupDBsAtPath(t, tablePath, map[string]testutil.DBRecords{ + testutil.SetupDBsAtPath(t, tablePath, map[string]testutil.DBConfig{ "db1": { - Start: 0, - NumRecords: 10, + DBRecords: testutil.DBRecords{ + NumRecords: 10, + }, }, "db2": { - Start: 10, - NumRecords: 10, + DBRecords: testutil.DBRecords{ + Start: 10, + NumRecords: 10, + }, }, - }, false, nil) - testutil.SetupDBsAtPath(t, tablePath, map[string]testutil.DBRecords{ + }, nil) + testutil.SetupDBsAtPath(t, tablePath, map[string]testutil.DBConfig{ "db3": { - Start: 20, - NumRecords: 10, + DBRecords: testutil.DBRecords{ + Start: 20, + NumRecords: 10, + }, }, "db4": { - Start: 30, - NumRecords: 10, + DBRecords: testutil.DBRecords{ + Start: 30, + NumRecords: 10, + }, }, - }, false, []byte(user1)) + }, []byte(user1)) // try loading the table. table, err := LoadTable(tablePath, "test", nil, boltDBIndexClient, false, newMetrics(nil))