apitech
/
loki
mirror of https://github.com/grafana/loki
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Fix Flaky Integration Tests (#7360)

Browse Source 
Allow the OS kernel to decide what's an open port instead of trying to
guess
pull/7517/head
Travis Patterson 3 years ago committed by GitHub
parent 289190ee26
commit e89c30ce6b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 64 additions and 129 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 115
      integration/cluster/cluster.go
  2. 29
      integration/loki_micro_services_delete_test.go
  3. 23
      integration/loki_micro_services_test.go
  4. 4
      integration/loki_simple_scalable_test.go
  5. 2
      integration/loki_single_binary_test.go
  6. 20
      pkg/loki/modules.go

115
integration/cluster/cluster.go
Unescape View File

@ -8,7 +8,6 @@ import (
"net"
"net/http"
"net/http/httptest"
"net/url"
"os"
"strings"
"sync"
@ -29,8 +28,8 @@ var (
auth_enabled: true
server:
http_listen_port: {{.httpPort}}
grpc_listen_port: {{.grpcPort}}
http_listen_port: 0
grpc_listen_port: 0
common:
path_prefix: {{.dataPath}}
@ -72,12 +71,6 @@ ingester:
lifecycler:
min_ready_duration: 0s
frontend_worker:
scheduler_address: localhost:{{.schedulerPort}}
frontend:
scheduler_address: localhost:{{.schedulerPort}}
{{if .remoteWriteUrls}}
ruler:
wal:
@ -170,17 +163,6 @@ func (c *Cluster) Run() error {
continue
}
var err error
component.httpPort, err = getFreePort()
if err != nil {
panic(fmt.Errorf("error allocating HTTP port: %w", err))
}
component.grpcPort, err = getFreePort()
if err != nil {
panic(fmt.Errorf("error allocating GRPC port: %w", err))
}
if err := component.run(); err != nil {
return err
}
@ -242,9 +224,6 @@ type Component struct {
cluster *Cluster
flags []string
httpPort int
grpcPort int
configFile string
dataPath string
rulerWALPath string
@ -256,18 +235,17 @@ type Component struct {
RemoteWriteUrls []string
}
func (c *Component) HTTPURL() *url.URL {
return &url.URL{
Host: fmt.Sprintf("localhost:%d", c.httpPort),
Scheme: "http",
}
func (c *Component) HTTPURL() string {
return fmt.Sprintf("http://localhost:%s", port(c.loki.Server.HTTPListenAddr().String()))
}
func (c *Component) GRPCURL() *url.URL {
return &url.URL{
Host: fmt.Sprintf("localhost:%d", c.grpcPort),
Scheme: "grpc",
}
func (c *Component) GRPCURL() string {
return fmt.Sprintf("localhost:%s", port(c.loki.Server.GRPCListenAddr().String()))
}
func port(addr string) string {
parts := strings.Split(addr, ":")
return parts[len(parts)-1]
}
func (c *Component) writeConfig() error {
@ -317,9 +295,6 @@ func (c *Component) writeConfig() error {
if err := configTemplate.Execute(configFile, map[string]interface{}{
"dataPath": c.dataPath,
"sharedDataPath": c.cluster.sharedPath,
"grpcPort": c.grpcPort,
"httpPort": c.httpPort,
"schedulerPort": c.grpcPort,
"remoteWriteUrls": c.RemoteWriteUrls,
"rulesPath": c.rulesPath,
"rulerWALPath": c.rulerWALPath,
@ -331,7 +306,6 @@ func (c *Component) writeConfig() error {
return fmt.Errorf("error closing config file: %w", err)
}
c.configFile = configFile.Name()
return nil
}
@ -409,7 +383,7 @@ func (c *Component) run() error {
// cleanup calls the stop handler and returns files and directories to be cleaned up
func (c *Component) cleanup() (files []string, dirs []string) {
if c.loki != nil {
if c.loki != nil && c.loki.SignalHandler != nil {
c.loki.SignalHandler.Stop()
}
if c.configFile != "" {
@ -418,12 +392,6 @@ func (c *Component) cleanup() (files []string, dirs []string) {
if c.dataPath != "" {
dirs = append(dirs, c.dataPath)
}
if p := c.httpPort; p != 0 {
allocatedFreePorts.free(p)
}
if p := c.grpcPort; p != 0 {
allocatedFreePorts.free(p)
}
if c.rulerWALPath != "" {
dirs = append(dirs, c.rulerWALPath)
}
@ -434,68 +402,13 @@ func (c *Component) cleanup() (files []string, dirs []string) {
return files, dirs
}
// keep track of previously allocated random ports, to ensure them not to clash
var (
allocatedFreePorts = newAllocatedPorts()
)
type allocatedPorts struct {
m map[int]struct{}
lock sync.Mutex
}
func newAllocatedPorts() *allocatedPorts {
return &allocatedPorts{
m: make(map[int]struct{}),
}
}
func (a *allocatedPorts) reserve(p int) (ok bool) {
a.lock.Lock()
defer a.lock.Unlock()
if _, exists := a.m[p]; exists {
return false
}
a.m[p] = struct{}{}
return true
}
func (a *allocatedPorts) free(p int) {
a.lock.Lock()
defer a.lock.Unlock()
delete(a.m, p)
}
func getFreePort() (port int, err error) {
var a *net.TCPAddr
if a, err = net.ResolveTCPAddr("tcp", "localhost:0"); err == nil {
var l *net.TCPListener
if l, err = net.ListenTCP("tcp", a); err == nil {
defer l.Close()
port := l.Addr().(*net.TCPAddr).Port
if !allocatedFreePorts.reserve(port) {
// port has been allocated before, try your luck again
return getFreePort()
}
return port, nil
}
}
return
}
func NewRemoteWriteServer(handler *http.HandlerFunc) *httptest.Server {
server := httptest.NewUnstartedServer(*handler)
p, err := getFreePort()
if err != nil {
panic(fmt.Errorf("error allocating HTTP port: %w", err))
}
l, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", p))
l, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
panic(fmt.Errorf("failed to listen on: %v", err))
}
server := httptest.NewUnstartedServer(*handler)
server.Listener = l
server.Start()

29
integration/loki_micro_services_delete_test.go
Unescape View File

@ -46,12 +46,13 @@ func TestMicroServicesDeleteRequest(t *testing.T) {
tIngester = clu.AddComponent(
"ingester",
"-target=ingester",
"-boltdb.shipper.index-gateway-client.server-address="+tIndexGateway.GRPCURL().Host,
"-boltdb.shipper.index-gateway-client.server-address="+tIndexGateway.GRPCURL(),
)
tQueryScheduler = clu.AddComponent(
"query-scheduler",
"-target=query-scheduler",
"-boltdb.shipper.index-gateway-client.server-address="+tIndexGateway.GRPCURL().Host,
"-query-scheduler.use-scheduler-ring=false",
"-boltdb.shipper.index-gateway-client.server-address="+tIndexGateway.GRPCURL(),
)
)
require.NoError(t, clu.Run())
@ -61,17 +62,17 @@ func TestMicroServicesDeleteRequest(t *testing.T) {
tQueryFrontend = clu.AddComponent(
"query-frontend",
"-target=query-frontend",
"-frontend.scheduler-address="+tQueryScheduler.GRPCURL().Host,
"-frontend.scheduler-address="+tQueryScheduler.GRPCURL(),
"-frontend.default-validity=0s",
"-boltdb.shipper.index-gateway-client.server-address="+tIndexGateway.GRPCURL().Host,
"-common.compactor-address="+tCompactor.HTTPURL().String(),
"-boltdb.shipper.index-gateway-client.server-address="+tIndexGateway.GRPCURL(),
"-common.compactor-address="+tCompactor.HTTPURL(),
)
_ = clu.AddComponent(
"querier",
"-target=querier",
"-querier.scheduler-address="+tQueryScheduler.GRPCURL().Host,
"-boltdb.shipper.index-gateway-client.server-address="+tIndexGateway.GRPCURL().Host,
"-common.compactor-address="+tCompactor.HTTPURL().String(),
"-querier.scheduler-address="+tQueryScheduler.GRPCURL(),
"-boltdb.shipper.index-gateway-client.server-address="+tIndexGateway.GRPCURL(),
"-common.compactor-address="+tCompactor.HTTPURL(),
)
)
require.NoError(t, clu.Run())
@ -82,7 +83,7 @@ func TestMicroServicesDeleteRequest(t *testing.T) {
tRuler = clu.AddComponent(
"ruler",
"-target=ruler",
"-common.compactor-address="+tCompactor.HTTPURL().String(),
"-common.compactor-address="+tCompactor.HTTPURL(),
)
)
@ -118,15 +119,15 @@ func TestMicroServicesDeleteRequest(t *testing.T) {
tenantID := randStringRunes()
now := time.Now()
cliDistributor := client.New(tenantID, "", tDistributor.HTTPURL().String())
cliDistributor := client.New(tenantID, "", tDistributor.HTTPURL())
cliDistributor.Now = now
cliIngester := client.New(tenantID, "", tIngester.HTTPURL().String())
cliIngester := client.New(tenantID, "", tIngester.HTTPURL())
cliIngester.Now = now
cliQueryFrontend := client.New(tenantID, "", tQueryFrontend.HTTPURL().String())
cliQueryFrontend := client.New(tenantID, "", tQueryFrontend.HTTPURL())
cliQueryFrontend.Now = now
cliCompactor := client.New(tenantID, "", tCompactor.HTTPURL().String())
cliCompactor := client.New(tenantID, "", tCompactor.HTTPURL())
cliCompactor.Now = now
cliRuler := client.New(tRuler.RulesTenant, "", tRuler.HTTPURL().String())
cliRuler := client.New(tRuler.RulesTenant, "", tRuler.HTTPURL())
cliRuler.Now = now
t.Run("ingest-logs-store", func(t *testing.T) {

23
integration/loki_micro_services_test.go
Unescape View File

@ -45,12 +45,13 @@ func TestMicroServicesIngestQuery(t *testing.T) {
tIngester = clu.AddComponent(
"ingester",
"-target=ingester",
"-boltdb.shipper.index-gateway-client.server-address="+tIndexGateway.GRPCURL().Host,
"-boltdb.shipper.index-gateway-client.server-address="+tIndexGateway.GRPCURL(),
)
tQueryScheduler = clu.AddComponent(
"query-scheduler",
"-target=query-scheduler",
"-boltdb.shipper.index-gateway-client.server-address="+tIndexGateway.GRPCURL().Host,
"-query-scheduler.use-scheduler-ring=false",
"-boltdb.shipper.index-gateway-client.server-address="+tIndexGateway.GRPCURL(),
)
)
require.NoError(t, clu.Run())
@ -60,16 +61,16 @@ func TestMicroServicesIngestQuery(t *testing.T) {
tQueryFrontend = clu.AddComponent(
"query-frontend",
"-target=query-frontend",
"-frontend.scheduler-address="+tQueryScheduler.GRPCURL().Host,
"-boltdb.shipper.index-gateway-client.server-address="+tIndexGateway.GRPCURL().Host,
"-common.compactor-address="+tCompactor.HTTPURL().String(),
"-frontend.scheduler-address="+tQueryScheduler.GRPCURL(),
"-boltdb.shipper.index-gateway-client.server-address="+tIndexGateway.GRPCURL(),
"-common.compactor-address="+tCompactor.HTTPURL(),
)
_ = clu.AddComponent(
"querier",
"-target=querier",
"-querier.scheduler-address="+tQueryScheduler.GRPCURL().Host,
"-boltdb.shipper.index-gateway-client.server-address="+tIndexGateway.GRPCURL().Host,
"-common.compactor-address="+tCompactor.HTTPURL().String(),
"-querier.scheduler-address="+tQueryScheduler.GRPCURL(),
"-boltdb.shipper.index-gateway-client.server-address="+tIndexGateway.GRPCURL(),
"-common.compactor-address="+tCompactor.HTTPURL(),
)
)
require.NoError(t, clu.Run())
@ -77,11 +78,11 @@ func TestMicroServicesIngestQuery(t *testing.T) {
tenantID := randStringRunes()
now := time.Now()
cliDistributor := client.New(tenantID, "", tDistributor.HTTPURL().String())
cliDistributor := client.New(tenantID, "", tDistributor.HTTPURL())
cliDistributor.Now = now
cliIngester := client.New(tenantID, "", tIngester.HTTPURL().String())
cliIngester := client.New(tenantID, "", tIngester.HTTPURL())
cliIngester.Now = now
cliQueryFrontend := client.New(tenantID, "", tQueryFrontend.HTTPURL().String())
cliQueryFrontend := client.New(tenantID, "", tQueryFrontend.HTTPURL())
cliQueryFrontend.Now = now
t.Run("ingest-logs-store", func(t *testing.T) {

4
integration/loki_simple_scalable_test.go
Unescape View File

@ -34,9 +34,9 @@ func TestSimpleScalableIngestQuery(t *testing.T) {
tenantID := randStringRunes()
now := time.Now()
cliWrite := client.New(tenantID, "", tWrite.HTTPURL().String())
cliWrite := client.New(tenantID, "", tWrite.HTTPURL())
cliWrite.Now = now
cliRead := client.New(tenantID, "", tRead.HTTPURL().String())
cliRead := client.New(tenantID, "", tRead.HTTPURL())
cliRead.Now = now
t.Run("ingest logs", func(t *testing.T) {

2
integration/loki_single_binary_test.go
Unescape View File

@ -28,7 +28,7 @@ func TestSingleBinaryIngestQuery(t *testing.T) {
require.NoError(t, clu.Run())
tenantID := randStringRunes()
cli := client.New(tenantID, "", tAll.HTTPURL().String())
cli := client.New(tenantID, "", tAll.HTTPURL())
t.Run("ingest-logs-store", func(t *testing.T) {
// ingest some log lines

20
pkg/loki/modules.go
Unescape View File

@ -8,6 +8,8 @@ import (
"net/http/httputil"
"net/url"
"os"
"strconv"
"strings"
"time"
"github.com/NYTimes/gziphandler"
@ -145,9 +147,27 @@ func (t *Loki) initServer() (services.Service, error) {
t.Server.HTTPServer.Handler = middleware.Merge(serverutil.RecoveryHTTPMiddleware).Wrap(h)
if t.Cfg.Server.HTTPListenPort == 0 {
t.Cfg.Server.HTTPListenPort = portFromAddr(t.Server.HTTPListenAddr().String())
}
if t.Cfg.Server.GRPCListenPort == 0 {
t.Cfg.Server.GRPCListenPort = portFromAddr(t.Server.GRPCListenAddr().String())
}
return s, nil
}
func portFromAddr(addr string) int {
parts := strings.Split(addr, ":")
port := parts[len(parts)-1]
portNumber, err := strconv.Atoi(port)
if err != nil {
return 0
}
return portNumber
}
func (t *Loki) initInternalServer() (services.Service, error) {
// Loki handles signals on its own.
DisableSignalHandling(&t.Cfg.InternalServer.Config)

Write Preview
Loading…
Cancel
Save