moves timeout tests to an integration test

pkg/services/alerting/engine.go

@@ -205,8 +205,14 @@ func (e *AlertingService) processJob(attemptID int, attemptChan chan int, cancel
 			}
 		}
 
+		// create new context with timeout for notifications
 		resultHandleCtx, resultHandleCancelFn := context.WithTimeout(context.Background(), resultHandleTimeout)
 		cancelChan <- resultHandleCancelFn
+
+		// override the context used for evaluation with a new context for notifications.
+		// This makes it possible for notifiers to execute when datasources
+		// dont respond within the timeout limit. We should rewrite this so notifications
+		// dont reuse the evalContext and get its own context.
 		evalContext.Ctx = resultHandleCtx
 		evalContext.Rule.State = evalContext.GetNewState()
 		e.resultHandler.Handle(evalContext)

pkg/services/alerting/engine_integration_test.go

@@ -0,0 +1,148 @@
+// +build integration
+
+package alerting
+
+import (
+	"context"
+	"errors"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func TestEngineTimeouts(t *testing.T) {
+	Convey("Alerting engine timeout tests", t, func() {
+		engine := NewEngine()
+		engine.resultHandler = &FakeResultHandler{}
+		job := &Job{Running: true, Rule: &Rule{}}
+
+		Convey("Should trigger as many retries as needed", func() {
+			Convey("pended alert for datasource -> result handler should be worked", func() {
+				// reduce alert timeout to test quickly
+				originAlertTimeout := alertTimeout
+				alertTimeout = 2 * time.Second
+				transportTimeoutInterval := 2 * time.Second
+				serverBusySleepDuration := 1 * time.Second
+
+				evalHandler := NewFakeCommonTimeoutHandler(transportTimeoutInterval, serverBusySleepDuration)
+				resultHandler := NewFakeCommonTimeoutHandler(transportTimeoutInterval, serverBusySleepDuration)
+				engine.evalHandler = evalHandler
+				engine.resultHandler = resultHandler
+
+				engine.processJobWithRetry(context.TODO(), job)
+
+				So(evalHandler.EvalSucceed, ShouldEqual, true)
+				So(resultHandler.ResultHandleSucceed, ShouldEqual, true)
+
+				// initialize for other tests.
+				alertTimeout = originAlertTimeout
+				engine.resultHandler = &FakeResultHandler{}
+			})
+		})
+	})
+}
+
+type FakeCommonTimeoutHandler struct {
+	TransportTimeoutDuration time.Duration
+	ServerBusySleepDuration  time.Duration
+	EvalSucceed              bool
+	ResultHandleSucceed      bool
+}
+
+func NewFakeCommonTimeoutHandler(transportTimeoutDuration time.Duration, serverBusySleepDuration time.Duration) *FakeCommonTimeoutHandler {
+	return &FakeCommonTimeoutHandler{
+		TransportTimeoutDuration: transportTimeoutDuration,
+		ServerBusySleepDuration:  serverBusySleepDuration,
+		EvalSucceed:              false,
+		ResultHandleSucceed:      false,
+	}
+}
+
+func (handler *FakeCommonTimeoutHandler) Eval(evalContext *EvalContext) {
+	// 1. prepare mock server
+	path := "/evaltimeout"
+	srv := runBusyServer(path, handler.ServerBusySleepDuration)
+	defer srv.Close()
+
+	// 2. send requests
+	url := srv.URL + path
+	res, err := sendRequest(evalContext.Ctx, url, handler.TransportTimeoutDuration)
+	if res != nil {
+		defer res.Body.Close()
+	}
+
+	if err != nil {
+		evalContext.Error = errors.New("Fake evaluation timeout test failure")
+		return
+	}
+
+	if res.StatusCode == 200 {
+		handler.EvalSucceed = true
+	}
+
+	evalContext.Error = errors.New("Fake evaluation timeout test failure; wrong response")
+}
+
+func (handler *FakeCommonTimeoutHandler) Handle(evalContext *EvalContext) error {
+	// 1. prepare mock server
+	path := "/resulthandle"
+	srv := runBusyServer(path, handler.ServerBusySleepDuration)
+	defer srv.Close()
+
+	// 2. send requests
+	url := srv.URL + path
+	res, err := sendRequest(evalContext.Ctx, url, handler.TransportTimeoutDuration)
+	if res != nil {
+		defer res.Body.Close()
+	}
+
+	if err != nil {
+		evalContext.Error = errors.New("Fake result handle timeout test failure")
+		return evalContext.Error
+	}
+
+	if res.StatusCode == 200 {
+		handler.ResultHandleSucceed = true
+		return nil
+	}
+
+	evalContext.Error = errors.New("Fake result handle timeout test failure; wrong response")
+
+	return evalContext.Error
+}
+
+func runBusyServer(path string, serverBusySleepDuration time.Duration) *httptest.Server {
+	mux := http.NewServeMux()
+	server := httptest.NewServer(mux)
+
+	mux.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) {
+		time.Sleep(serverBusySleepDuration)
+	})
+
+	return server
+}
+
+func sendRequest(context context.Context, url string, transportTimeoutInterval time.Duration) (resp *http.Response, err error) {
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	req = req.WithContext(context)
+
+	transport := http.Transport{
+		Dial: (&net.Dialer{
+			Timeout:   transportTimeoutInterval,
+			KeepAlive: transportTimeoutInterval,
+		}).Dial,
+	}
+	client := http.Client{
+		Transport: &transport,
+	}
+
+	return client.Do(req)
+}

pkg/services/alerting/engine_test.go

@@ -4,11 +4,7 @@ import (
 	"context"
 	"errors"
 	"math"
-	"net"
-	"net/http"
-	"net/http/httptest"
 	"testing"
-	"time"
 
 	. "github.com/smartystreets/goconvey/convey"
 )
@@ -38,106 +34,6 @@ func (handler *FakeResultHandler) Handle(evalContext *EvalContext) error {
 	return nil
 }
 
-type FakeCommonTimeoutHandler struct {
-	TransportTimeoutDuration time.Duration
-	ServerBusySleepDuration  time.Duration
-	EvalSucceed              bool
-	ResultHandleSucceed      bool
-}
-
-func NewFakeCommonTimeoutHandler(transportTimeoutDuration time.Duration, serverBusySleepDuration time.Duration) *FakeCommonTimeoutHandler {
-	return &FakeCommonTimeoutHandler{
-		TransportTimeoutDuration: transportTimeoutDuration,
-		ServerBusySleepDuration:  serverBusySleepDuration,
-		EvalSucceed:              false,
-		ResultHandleSucceed:      false,
-	}
-}
-
-func (handler *FakeCommonTimeoutHandler) Eval(evalContext *EvalContext) {
-	// 1. prepare mock server
-	path := "/evaltimeout"
-	srv := runBusyServer(path, handler.ServerBusySleepDuration)
-	defer srv.Close()
-
-	// 2. send requests
-	url := srv.URL + path
-	res, err := sendRequest(evalContext.Ctx, url, handler.TransportTimeoutDuration)
-	if res != nil {
-		defer res.Body.Close()
-	}
-
-	if err != nil {
-		evalContext.Error = errors.New("Fake evaluation timeout test failure")
-		return
-	}
-
-	if res.StatusCode == 200 {
-		handler.EvalSucceed = true
-	}
-
-	evalContext.Error = errors.New("Fake evaluation timeout test failure; wrong response")
-}
-
-func (handler *FakeCommonTimeoutHandler) Handle(evalContext *EvalContext) error {
-	// 1. prepare mock server
-	path := "/resulthandle"
-	srv := runBusyServer(path, handler.ServerBusySleepDuration)
-	defer srv.Close()
-
-	// 2. send requests
-	url := srv.URL + path
-	res, err := sendRequest(evalContext.Ctx, url, handler.TransportTimeoutDuration)
-	if res != nil {
-		defer res.Body.Close()
-	}
-
-	if err != nil {
-		evalContext.Error = errors.New("Fake result handle timeout test failure")
-		return evalContext.Error
-	}
-
-	if res.StatusCode == 200 {
-		handler.ResultHandleSucceed = true
-		return nil
-	}
-
-	evalContext.Error = errors.New("Fake result handle timeout test failure; wrong response")
-
-	return evalContext.Error
-}
-
-func runBusyServer(path string, serverBusySleepDuration time.Duration) *httptest.Server {
-	mux := http.NewServeMux()
-	server := httptest.NewServer(mux)
-
-	mux.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) {
-		time.Sleep(serverBusySleepDuration)
-	})
-
-	return server
-}
-
-func sendRequest(context context.Context, url string, transportTimeoutInterval time.Duration) (resp *http.Response, err error) {
-	req, err := http.NewRequest("GET", url, nil)
-	if err != nil {
-		return nil, err
-	}
-
-	req = req.WithContext(context)
-
-	transport := http.Transport{
-		Dial: (&net.Dialer{
-			Timeout:   transportTimeoutInterval,
-			KeepAlive: transportTimeoutInterval,
-		}).Dial,
-	}
-	client := http.Client{
-		Transport: &transport,
-	}
-
-	return client.Do(req)
-}
 func TestEngineProcessJob(t *testing.T) {
 	Convey("Alerting engine job processing", t, func() {
 		engine := NewEngine()
@@ -217,29 +113,6 @@ func TestEngineProcessJob(t *testing.T) {
 				engine.processJobWithRetry(context.TODO(), job)
 				So(evalHandler.CallNb, ShouldEqual, expectedAttempts)
 			})
-
-			Convey("pended alert for datasource -> result handler should be worked", func() {
-				// reduce alert timeout to test quickly
-				originAlertTimeout := alertTimeout
-				alertTimeout = 5 * time.Second
-				transportTimeoutInterval := 5 * time.Second
-				serverBusySleepDuration := 4 * time.Second
-
-				evalHandler := NewFakeCommonTimeoutHandler(transportTimeoutInterval, serverBusySleepDuration)
-				resultHandler := NewFakeCommonTimeoutHandler(transportTimeoutInterval, serverBusySleepDuration)
-				engine.evalHandler = evalHandler
-				engine.resultHandler = resultHandler
-
-				engine.processJobWithRetry(context.TODO(), job)
-
-				So(evalHandler.EvalSucceed, ShouldEqual, true)
-				So(resultHandler.ResultHandleSucceed, ShouldEqual, true)
-
-				// initialize for other tests.
-				alertTimeout = originAlertTimeout
-				engine.resultHandler = &FakeResultHandler{}
-			})
-
 		})
 	})
 }