Alerting: Upload images to Slack via files.upload (#59163)

This commit makes a number of changes to how images work in Slack
notifications.

It adds support for uploading images to Slack via the files.upload
API when the contact point has a token. Images are no longer linked
via a URL if a token is present.

Each image uploaded to Slack is posted as a reply to the original
notification. Up to maxImagesPerThreadTs images can be posted as
replies before a final message is sent with:

  There are no images than can be shown here. To see the panels for
  all firing and resolved alerts please check Grafana

Incoming Webhooks cannot upload files via files.upload and so webhooks
require the image to be uploaded to cloud storage and linked via URL.

pkg/services/ngalert/notifier/channels/slack.go

@@ -8,9 +8,12 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"mime/multipart"
 	"net"
 	"net/http"
 	"net/url"
+	"os"
+	"path"
 	"strings"
 	"time"
 
@@ -25,8 +28,34 @@ import (
 	"github.com/grafana/grafana/pkg/setting"
 )
 
+const (
+	// maxImagesPerThreadTs is the maximum number of images that can be posted as
+	// replies to the same thread_ts. It should prevent tokens from exceeding the
+	// rate limits for files.upload https://api.slack.com/docs/rate-limits#tier_t2
+	maxImagesPerThreadTs        = 5
+	maxImagesPerThreadTsMessage = "There are more images than can be shown here. To see the panels for all firing and resolved alerts please check Grafana"
+)
+
+var (
+	slackClient = &http.Client{
+		Timeout: time.Second * 30,
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{
+				Renegotiation: tls.RenegotiateFreelyAsClient,
+			},
+			Proxy: http.ProxyFromEnvironment,
+			DialContext: (&net.Dialer{
+				Timeout: 30 * time.Second,
+			}).DialContext,
+			TLSHandshakeTimeout: 5 * time.Second,
+		},
+	}
+)
+
 var SlackAPIEndpoint = "https://slack.com/api/chat.postMessage"
 
+type sendFunc func(ctx context.Context, req *http.Request, logger log.Logger) (string, error)
+
 // SlackNotifier is responsible for sending
 // alert notification to Slack.
 type SlackNotifier struct {
@@ -35,6 +64,7 @@ type SlackNotifier struct {
 	tmpl          *template.Template
 	images        ImageStore
 	webhookSender notifications.WebhookSender
+	sendFn        sendFunc
 	settings      slackSettings
 }
 
@@ -53,6 +83,22 @@ type slackSettings struct {
 	MentionGroups  CommaSeparatedStrings `json:"mentionGroups,omitempty" yaml:"mentionGroups,omitempty"`
 }
 
+// isIncomingWebhook returns true if the settings are for an incoming webhook.
+func isIncomingWebhook(s slackSettings) bool {
+	return s.Token == ""
+}
+
+// uploadURL returns the upload URL for Slack.
+func uploadURL(s slackSettings) (string, error) {
+	u, err := url.Parse(s.URL)
+	if err != nil {
+		return "", fmt.Errorf("failed to parse URL: %w", err)
+	}
+	dir, _ := path.Split(u.Path)
+	u.Path = path.Join(dir, "files.upload")
+	return u.String(), nil
+}
+
 // SlackFactory creates a new NotificationChannel that sends notifications to Slack.
 func SlackFactory(fc FactoryConfig) (NotificationChannel, error) {
 	ch, err := buildSlackNotifier(fc)
@@ -80,6 +126,7 @@ func buildSlackNotifier(factoryConfig FactoryConfig) (*SlackNotifier, error) {
 	if slackURL == "" {
 		slackURL = settings.EndpointURL
 	}
+
 	apiURL, err := url.Parse(slackURL)
 	if err != nil {
 		return nil, fmt.Errorf("invalid URL %q", slackURL)
@@ -106,7 +153,6 @@ func buildSlackNotifier(factoryConfig FactoryConfig) (*SlackNotifier, error) {
 	if settings.Title == "" {
 		settings.Title = DefaultMessageTitleEmbed
 	}
-
 	return &SlackNotifier{
 		Base: NewBase(&models.AlertNotification{
 			Uid:                   factoryConfig.Config.UID,
@@ -119,6 +165,7 @@ func buildSlackNotifier(factoryConfig FactoryConfig) (*SlackNotifier, error) {
 
 		images:        factoryConfig.ImageStore,
 		webhookSender: factoryConfig.NotificationService,
+		sendFn:        sendSlackRequest,
 		log:           log.New("alerting.notifier.slack"),
 		tmpl:          factoryConfig.Template,
 	}, nil
@@ -133,6 +180,7 @@ type slackMessage struct {
 	IconURL     string                   `json:"icon_url,omitempty"`
 	Attachments []attachment             `json:"attachments"`
 	Blocks      []map[string]interface{} `json:"blocks,omitempty"`
+	ThreadTs    string                   `json:"thread_ts,omitempty"`
 }
 
 // attachment is used to display a richly-formatted message block.
@@ -153,36 +201,41 @@ type attachment struct {
 
 // Notify sends an alert notification to Slack.
 func (sn *SlackNotifier) Notify(ctx context.Context, alerts ...*types.Alert) (bool, error) {
-	sn.log.Debug("building slack message", "alerts", len(alerts))
-	msg, err := sn.buildSlackMessage(ctx, alerts)
-	if err != nil {
-		return false, fmt.Errorf("build slack message: %w", err)
-	}
+	sn.log.Debug("Creating slack message", "alerts", len(alerts))
 
-	b, err := json.Marshal(msg)
+	m, err := sn.createSlackMessage(ctx, alerts)
 	if err != nil {
-		return false, fmt.Errorf("marshal json: %w", err)
+		sn.log.Error("Failed to create Slack message", "err", err)
+		return false, fmt.Errorf("failed to create Slack message: %w", err)
 	}
 
-	sn.log.Debug("sending Slack API request", "url", sn.settings.URL, "data", string(b))
-	request, err := http.NewRequestWithContext(ctx, http.MethodPost, sn.settings.URL, bytes.NewReader(b))
+	thread_ts, err := sn.sendSlackMessage(ctx, m)
 	if err != nil {
-		return false, fmt.Errorf("failed to create HTTP request: %w", err)
+		sn.log.Error("Failed to send Slack message", "err", err)
+		return false, fmt.Errorf("failed to send Slack message: %w", err)
 	}
 
-	request.Header.Set("Content-Type", "application/json")
-	request.Header.Set("User-Agent", "Grafana")
-	if sn.settings.Token == "" {
-		if sn.settings.URL == SlackAPIEndpoint {
-			panic("Token should be set when using the Slack chat API")
+	// Do not upload images if using an incoming webhook as incoming webhooks cannot upload files
+	if !isIncomingWebhook(sn.settings) {
+		if err := withStoredImages(ctx, sn.log, sn.images, func(index int, image ngmodels.Image) error {
+			// If we have exceeded the maximum number of images for this thread_ts
+			// then tell the recipient and stop iterating subsequent images
+			if index >= maxImagesPerThreadTs {
+				if _, err := sn.sendSlackMessage(ctx, &slackMessage{
+					Channel:  sn.settings.Recipient,
+					Text:     maxImagesPerThreadTsMessage,
+					ThreadTs: thread_ts,
+				}); err != nil {
+					sn.log.Error("Failed to send Slack message", "err", err)
+				}
+				return ErrImagesDone
+			}
+			comment := initialCommentForImage(alerts[index])
+			return sn.uploadImage(ctx, image, sn.settings.Recipient, comment, thread_ts)
+		}, alerts...); err != nil {
+			// Do not return an error here as we might have exceeded the rate limit for uploading files
+			sn.log.Error("Failed to upload image", "err", err)
 		}
-	} else {
-		sn.log.Debug("adding authorization header to HTTP request")
-		request.Header.Set("Authorization", fmt.Sprintf("Bearer %s", sn.settings.Token))
-	}
-
-	if err := sendSlackRequest(request, sn.log); err != nil {
-		return false, err
 	}
 
 	return true, nil
@@ -190,74 +243,117 @@ func (sn *SlackNotifier) Notify(ctx context.Context, alerts ...*types.Alert) (bo
 
 // sendSlackRequest sends a request to the Slack API.
 // Stubbable by tests.
-var sendSlackRequest = func(request *http.Request, logger log.Logger) (retErr error) {
+var sendSlackRequest = func(ctx context.Context, req *http.Request, logger log.Logger) (string, error) {
+	resp, err := slackClient.Do(req)
+	if err != nil {
+		return "", fmt.Errorf("failed to send request: %w", err)
+	}
+
 	defer func() {
-		if retErr != nil {
-			logger.Warn("failed to send slack request", "error", retErr)
+		if err := resp.Body.Close(); err != nil {
+			logger.Warn("Failed to close response body", "err", err)
 		}
 	}()
 
-	netTransport := &http.Transport{
-		TLSClientConfig: &tls.Config{
-			Renegotiation: tls.RenegotiateFreelyAsClient,
-		},
-		Proxy: http.ProxyFromEnvironment,
-		DialContext: (&net.Dialer{
-			Timeout: 30 * time.Second,
-		}).DialContext,
-		TLSHandshakeTimeout: 5 * time.Second,
+	if resp.StatusCode < http.StatusOK {
+		logger.Error("Unexpected 1xx response", "status", resp.StatusCode)
+		return "", fmt.Errorf("unexpected 1xx status code: %d", resp.StatusCode)
+	} else if resp.StatusCode >= 300 && resp.StatusCode < 400 {
+		logger.Error("Unexpected 3xx response", "status", resp.StatusCode)
+		return "", fmt.Errorf("unexpected 3xx status code: %d", resp.StatusCode)
+	} else if resp.StatusCode >= http.StatusInternalServerError {
+		logger.Error("Unexpected 5xx response", "status", resp.StatusCode)
+		return "", fmt.Errorf("unexpected 5xx status code: %d", resp.StatusCode)
 	}
-	netClient := &http.Client{
-		Timeout:   time.Second * 30,
-		Transport: netTransport,
+
+	content := resp.Header.Get("Content-Type")
+	// If the response is text/html it could be the response to an incoming webhook
+	if strings.HasPrefix(content, "text/html") {
+		return handleSlackIncomingWebhookResponse(resp, logger)
+	} else {
+		return handleSlackJSONResponse(resp, logger)
 	}
-	resp, err := netClient.Do(request)
+}
+
+func handleSlackIncomingWebhookResponse(resp *http.Response, logger log.Logger) (string, error) {
+	b, err := io.ReadAll(resp.Body)
 	if err != nil {
-		return err
+		return "", fmt.Errorf("failed to read response: %w", err)
+	}
+
+	// Incoming webhooks return the string "ok" on success
+	if bytes.Equal(b, []byte("ok")) {
+		logger.Debug("The incoming webhook was successful")
+		return "", nil
+	}
+
+	logger.Debug("Incoming webhook was unsuccessful", "status", resp.StatusCode, "body", string(b))
+
+	// There are a number of known errors that we can check. The documentation incoming webhooks
+	// errors can be found at https://api.slack.com/messaging/webhooks#handling_errors and
+	// https://api.slack.com/changelog/2016-05-17-changes-to-errors-for-incoming-webhooks
+	if bytes.Equal(b, []byte("user_not_found")) {
+		return "", errors.New("the user does not exist or is invalid")
+	}
+
+	if bytes.Equal(b, []byte("channel_not_found")) {
+		return "", errors.New("the channel does not exist or is invalid")
+	}
+
+	if bytes.Equal(b, []byte("channel_is_archived")) {
+		return "", errors.New("cannot send an incoming webhook for an archived channel")
+	}
+
+	if bytes.Equal(b, []byte("posting_to_general_channel_denied")) {
+		return "", errors.New("cannot send an incoming webhook to the #general channel")
+	}
+
+	if bytes.Equal(b, []byte("no_service")) {
+		return "", errors.New("the incoming webhook is either disabled, removed, or invalid")
+	}
+
+	if bytes.Equal(b, []byte("no_text")) {
+		return "", errors.New("cannot send an incoming webhook without a message")
 	}
-	defer func() {
-		if err := resp.Body.Close(); err != nil {
-			logger.Warn("failed to close response body", "error", err)
-		}
-	}()
 
-	body, err := io.ReadAll(resp.Body)
+	return "", fmt.Errorf("failed incoming webhook: %s", string(b))
+}
+
+func handleSlackJSONResponse(resp *http.Response, logger log.Logger) (string, error) {
+	b, err := io.ReadAll(resp.Body)
 	if err != nil {
-		return fmt.Errorf("failed to read response body: %w", err)
+		return "", fmt.Errorf("failed to read response: %w", err)
 	}
 
-	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
-		logger.Error("Slack API request failed", "url", request.URL.String(), "statusCode", resp.Status, "body", string(body))
-		return fmt.Errorf("request to Slack API failed with status code %d", resp.StatusCode)
+	if len(b) == 0 {
+		logger.Error("Expected JSON but got empty response")
+		return "", errors.New("unexpected empty response")
 	}
 
 	// Slack responds to some requests with a JSON document, that might contain an error.
-	rslt := struct {
-		Ok  bool   `json:"ok"`
+	result := struct {
+		OK  bool   `json:"ok"`
+		Ts  string `json:"ts"`
 		Err string `json:"error"`
 	}{}
 
-	// Marshaling can fail if Slack's response body is plain text (e.g. "ok").
-	if err := json.Unmarshal(body, &rslt); err != nil && json.Valid(body) {
-		logger.Error("Failed to unmarshal Slack API response", "url", request.URL.String(), "statusCode", resp.Status,
-			"body", string(body))
-		return fmt.Errorf("failed to unmarshal Slack API response: %s", err)
+	if err := json.Unmarshal(b, &result); err != nil {
+		logger.Error("Failed to unmarshal response", "body", string(b), "err", err)
+		return "", fmt.Errorf("failed to unmarshal response: %w", err)
 	}
 
-	if !rslt.Ok && rslt.Err != "" {
-		logger.Error("Sending Slack API request failed", "url", request.URL.String(), "statusCode", resp.Status,
-			"error", rslt.Err)
-		return fmt.Errorf("failed to make Slack API request: %s", rslt.Err)
+	if !result.OK {
+		logger.Error("The request was unsuccessful", "body", string(b), "err", result.Err)
+		return "", fmt.Errorf("failed to send request: %s", result.Err)
 	}
 
-	logger.Debug("sending Slack API request succeeded", "url", request.URL.String(), "statusCode", resp.Status)
-	return nil
+	logger.Debug("The request was successful")
+	return result.Ts, nil
 }
 
-func (sn *SlackNotifier) buildSlackMessage(ctx context.Context, alrts []*types.Alert) (*slackMessage, error) {
-	alerts := types.Alerts(alrts...)
+func (sn *SlackNotifier) createSlackMessage(ctx context.Context, alerts []*types.Alert) (*slackMessage, error) {
 	var tmplErr error
-	tmpl, _ := TmplText(ctx, sn.tmpl, alrts, sn.log, &tmplErr)
+	tmpl, _ := TmplText(ctx, sn.tmpl, alerts, sn.log, &tmplErr)
 
 	ruleURL := joinUrlPath(sn.tmpl.ExternalURL.String(), "/alerting/list", sn.log)
 
@@ -270,7 +366,7 @@ func (sn *SlackNotifier) buildSlackMessage(ctx context.Context, alrts []*types.A
 		// https://api.slack.com/messaging/composing/layouts#when-to-use-attachments
 		Attachments: []attachment{
 			{
-				Color:      getAlertStatusColor(alerts.Status()),
+				Color:      getAlertStatusColor(types.Alerts(alerts...).Status()),
 				Title:      tmpl(sn.settings.Title),
 				Fallback:   tmpl(sn.settings.Title),
 				Footer:     "Grafana v" + setting.BuildVersion,
@@ -283,10 +379,16 @@ func (sn *SlackNotifier) buildSlackMessage(ctx context.Context, alrts []*types.A
 		},
 	}
 
-	_ = withStoredImages(ctx, sn.log, sn.images, func(index int, image ngmodels.Image) error {
-		req.Attachments[0].ImageURL = image.URL
-		return ErrImagesDone
-	}, alrts...)
+	if isIncomingWebhook(sn.settings) {
+		// Incoming webhooks cannot upload files, instead share images via their URL
+		_ = withStoredImages(ctx, sn.log, sn.images, func(index int, image ngmodels.Image) error {
+			if image.URL != "" {
+				req.Attachments[0].ImageURL = image.URL
+				return ErrImagesDone
+			}
+			return nil
+		}, alerts...)
+	}
 
 	if tmplErr != nil {
 		sn.log.Warn("failed to template Slack message", "error", tmplErr.Error())
@@ -298,16 +400,19 @@ func (sn *SlackNotifier) buildSlackMessage(ctx context.Context, alrts []*types.A
 			mentionsBuilder.WriteString(" ")
 		}
 	}
+
 	mentionChannel := strings.TrimSpace(sn.settings.MentionChannel)
 	if mentionChannel != "" {
 		mentionsBuilder.WriteString(fmt.Sprintf("<!%s|%s>", mentionChannel, mentionChannel))
 	}
+
 	if len(sn.settings.MentionGroups) > 0 {
 		appendSpace()
 		for _, g := range sn.settings.MentionGroups {
 			mentionsBuilder.WriteString(fmt.Sprintf("<!subteam^%s>", tmpl(g)))
 		}
 	}
+
 	if len(sn.settings.MentionUsers) > 0 {
 		appendSpace()
 		for _, u := range sn.settings.MentionUsers {
@@ -324,6 +429,159 @@ func (sn *SlackNotifier) buildSlackMessage(ctx context.Context, alrts []*types.A
 	return req, nil
 }
 
+func (sn *SlackNotifier) sendSlackMessage(ctx context.Context, m *slackMessage) (string, error) {
+	b, err := json.Marshal(m)
+	if err != nil {
+		return "", fmt.Errorf("failed to marshal Slack message: %w", err)
+	}
+
+	sn.log.Debug("sending Slack API request", "url", sn.settings.URL, "data", string(b))
+	request, err := http.NewRequestWithContext(ctx, http.MethodPost, sn.settings.URL, bytes.NewReader(b))
+	if err != nil {
+		return "", fmt.Errorf("failed to create HTTP request: %w", err)
+	}
+
+	request.Header.Set("Content-Type", "application/json")
+	request.Header.Set("User-Agent", "Grafana")
+	if sn.settings.Token == "" {
+		if sn.settings.URL == SlackAPIEndpoint {
+			panic("Token should be set when using the Slack chat API")
+		}
+		sn.log.Debug("Looks like we are using an incoming webhook, no Authorization header required")
+	} else {
+		sn.log.Debug("Looks like we are using the Slack API, have set the Bearer token for this request")
+		request.Header.Set("Authorization", "Bearer "+sn.settings.Token)
+	}
+
+	thread_ts, err := sn.sendFn(ctx, request, sn.log)
+	if err != nil {
+		return "", err
+	}
+
+	return thread_ts, nil
+}
+
+// createImageMultipart returns the mutlipart/form-data request and headers for files.upload.
+// It returns an error if the image does not exist or there was an error preparing the
+// multipart form.
+func (sn *SlackNotifier) createImageMultipart(image ngmodels.Image, channel, comment, thread_ts string) (http.Header, []byte, error) {
+	buf := bytes.Buffer{}
+	w := multipart.NewWriter(&buf)
+	defer func() {
+		if err := w.Close(); err != nil {
+			sn.log.Error("Failed to close multipart writer", "err", err)
+		}
+	}()
+
+	f, err := os.Open(image.Path)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	fw, err := w.CreateFormFile("file", image.Path)
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to create form file: %w", err)
+	}
+
+	if _, err := io.Copy(fw, f); err != nil {
+		return nil, nil, fmt.Errorf("failed to copy file to form: %w", err)
+	}
+
+	if err := w.WriteField("channels", channel); err != nil {
+		return nil, nil, fmt.Errorf("failed to write channels to form: %w", err)
+	}
+
+	if err := w.WriteField("initial_comment", comment); err != nil {
+		return nil, nil, fmt.Errorf("failed to write initial_comment to form: %w", err)
+	}
+
+	if err := w.WriteField("thread_ts", thread_ts); err != nil {
+		return nil, nil, fmt.Errorf("failed to write thread_ts to form: %w", err)
+	}
+
+	if err := w.Close(); err != nil {
+		return nil, nil, fmt.Errorf("failed to close multipart writer: %w", err)
+	}
+
+	b := buf.Bytes()
+	headers := http.Header{}
+	headers.Set("Content-Type", w.FormDataContentType())
+	return headers, b, nil
+}
+
+func (sn *SlackNotifier) sendMultipart(ctx context.Context, headers http.Header, data io.Reader) error {
+	sn.log.Debug("Sending multipart request to files.upload")
+
+	u, err := uploadURL(sn.settings)
+	if err != nil {
+		return fmt.Errorf("failed to get URL for files.upload: %w", err)
+	}
+
+	req, err := http.NewRequest(http.MethodPost, u, data)
+	if err != nil {
+		return fmt.Errorf("failed to create request: %w", err)
+	}
+	for k, v := range headers {
+		req.Header[k] = v
+	}
+	req.Header.Set("Authorization", "Bearer "+sn.settings.Token)
+
+	if _, err := sn.sendFn(ctx, req, sn.log); err != nil {
+		return fmt.Errorf("failed to send request: %w", err)
+	}
+
+	return nil
+}
+
+// uploadImage shares the image to the channel names or IDs. It returns an error if the file
+// does not exist, or if there was an error either preparing or sending the multipart/form-data
+// request.
+func (sn *SlackNotifier) uploadImage(ctx context.Context, image ngmodels.Image, channel, comment, thread_ts string) error {
+	sn.log.Debug("Uploadimg image", "image", image.Token)
+	headers, data, err := sn.createImageMultipart(image, channel, comment, thread_ts)
+	if err != nil {
+		return fmt.Errorf("failed to create multipart form: %w", err)
+	}
+
+	return sn.sendMultipart(ctx, headers, bytes.NewReader(data))
+}
+
 func (sn *SlackNotifier) SendResolved() bool {
 	return !sn.GetDisableResolveMessage()
 }
+
+// initialCommentForImage returns the initial comment for the image.
+// Here is an example of the initial comment for an alert called
+// AlertName with two labels:
+//
+//	Resolved|Firing: AlertName, Labels: A=B, C=D
+//
+// where Resolved|Firing and Labels is in bold text.
+func initialCommentForImage(alert *types.Alert) string {
+	sb := strings.Builder{}
+
+	if alert.Resolved() {
+		sb.WriteString("*Resolved*:")
+	} else {
+		sb.WriteString("*Firing*:")
+	}
+
+	sb.WriteString(" ")
+	sb.WriteString(alert.Name())
+	sb.WriteString(", ")
+
+	sb.WriteString("*Labels*: ")
+
+	var n int
+	for k, v := range alert.Labels {
+		sb.WriteString(string(k))
+		sb.WriteString(" = ")
+		sb.WriteString(string(v))
+		if n < len(alert.Labels)-1 {
+			sb.WriteString(", ")
+			n += 1
+		}
+	}
+
+	return sb.String()
+}

pkg/services/ngalert/notifier/channels/slack_test.go

@@ -3,342 +3,499 @@ package channels
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"io"
+	"mime"
+	"mime/multipart"
 	"net/http"
 	"net/http/httptest"
 	"net/url"
+	"os"
 	"testing"
 
+	"github.com/prometheus/alertmanager/notify"
+	"github.com/prometheus/alertmanager/types"
+	"github.com/prometheus/common/model"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
 	"github.com/grafana/grafana/pkg/components/simplejson"
 	"github.com/grafana/grafana/pkg/infra/log"
 	"github.com/grafana/grafana/pkg/services/ngalert/models"
 	"github.com/grafana/grafana/pkg/services/secrets/fakes"
 	secretsManager "github.com/grafana/grafana/pkg/services/secrets/manager"
 	"github.com/grafana/grafana/pkg/setting"
-
-	"github.com/prometheus/alertmanager/notify"
-	"github.com/prometheus/alertmanager/types"
-	"github.com/prometheus/common/model"
-	"github.com/stretchr/testify/require"
 )
 
-func TestSlackNotifier(t *testing.T) {
-	tmpl := templateForTests(t)
-
-	fakeImageStore := &fakeImageStore{
-		Images: []*models.Image{
-			{
-				Token: "test-with-url",
-				URL:   "https://www.example.com/image.jpg",
+func TestSlackIncomingWebhook(t *testing.T) {
+	tests := []struct {
+		name            string
+		alerts          []*types.Alert
+		expectedMessage *slackMessage
+		expectedError   string
+		settings        string
+	}{{
+		name: "Message is sent",
+		settings: `{
+			"icon_emoji": ":emoji:",
+			"recipient": "#test",
+			"url": "https://example.com/hooks/xxxx"
+		}`,
+		alerts: []*types.Alert{{
+			Alert: model.Alert{
+				Labels:      model.LabelSet{"alertname": "alert1", "lbl1": "val1"},
+				Annotations: model.LabelSet{"ann1": "annv1"},
 			},
-		},
-	}
-
-	externalURL, err := url.Parse("http://localhost")
-	require.NoError(t, err)
-	tmpl.ExternalURL = externalURL
-
-	cases := []struct {
-		name          string
-		settings      string
-		alerts        []*types.Alert
-		expMsg        *slackMessage
-		expInitError  string
-		expMsgError   error
-		expWebhookURL string
-	}{
-		{
-			name: "Correct config with one alert",
-			settings: `{
-				"token": "1234",
-				"recipient": "#testchannel",
-				"icon_emoji": ":emoji:"
-			}`,
-			alerts: []*types.Alert{
+		}},
+		expectedMessage: &slackMessage{
+			Channel:   "#test",
+			Username:  "Grafana",
+			IconEmoji: ":emoji:",
+			Attachments: []attachment{
 				{
-					Alert: model.Alert{
-						Labels:      model.LabelSet{"alertname": "alert1", "lbl1": "val1"},
-						Annotations: model.LabelSet{"ann1": "annv1", "__dashboardUid__": "abcd", "__panelId__": "efgh"},
-					},
-				},
-			},
-			expMsg: &slackMessage{
-				Channel:   "#testchannel",
-				Username:  "Grafana",
-				IconEmoji: ":emoji:",
-				Attachments: []attachment{
-					{
-						Title:      "[FIRING:1]  (val1)",
-						TitleLink:  "http://localhost/alerting/list",
-						Text:       "**Firing**\n\nValue: [no value]\nLabels:\n - alertname = alert1\n - lbl1 = val1\nAnnotations:\n - ann1 = annv1\nSilence: http://localhost/alerting/silence/new?alertmanager=grafana&matcher=alertname%3Dalert1&matcher=lbl1%3Dval1\nDashboard: http://localhost/d/abcd\nPanel: http://localhost/d/abcd?viewPanel=efgh\n",
-						Fallback:   "[FIRING:1]  (val1)",
-						Fields:     nil,
-						Footer:     "Grafana v" + setting.BuildVersion,
-						FooterIcon: "https://grafana.com/static/assets/img/fav32.png",
-						Color:      "#D63232",
-						Ts:         0,
-					},
+					Title:      "[FIRING:1]  (val1)",
+					TitleLink:  "http://localhost/alerting/list",
+					Text:       "**Firing**\n\nValue: [no value]\nLabels:\n - alertname = alert1\n - lbl1 = val1\nAnnotations:\n - ann1 = annv1\nSilence: http://localhost/alerting/silence/new?alertmanager=grafana&matcher=alertname%3Dalert1&matcher=lbl1%3Dval1\n",
+					Fallback:   "[FIRING:1]  (val1)",
+					Fields:     nil,
+					Footer:     "Grafana v" + setting.BuildVersion,
+					FooterIcon: "https://grafana.com/static/assets/img/fav32.png",
+					Color:      "#D63232",
 				},
 			},
-			expMsgError: nil,
 		},
-		{
-			name: "Correct config with webhook",
-			settings: `{
-				"url": "https://webhook.com",
-				"recipient": "#testchannel",
-				"icon_emoji": ":emoji:"
+	}, {
+		name: "Message is sent with image URL",
+		settings: `{
+				"icon_emoji": ":emoji:",
+				"recipient": "#test",
+				"url": "https://example.com/hooks/xxxx"
 			}`,
-			alerts: []*types.Alert{
-				{
-					Alert: model.Alert{
-						Labels:      model.LabelSet{"alertname": "alert1", "lbl1": "val1"},
-						Annotations: model.LabelSet{"ann1": "annv1"},
-					},
-				},
+		alerts: []*types.Alert{{
+			Alert: model.Alert{
+				Labels:      model.LabelSet{"alertname": "alert1", "lbl1": "val1"},
+				Annotations: model.LabelSet{"ann1": "annv1", "__dashboardUid__": "abcd", "__panelId__": "efgh", "__alertImageToken__": "image-with-url"},
 			},
-			expMsg: &slackMessage{
-				Channel:   "#testchannel",
-				Username:  "Grafana",
-				IconEmoji: ":emoji:",
-				Attachments: []attachment{
-					{
-						Title:      "[FIRING:1]  (val1)",
-						TitleLink:  "http://localhost/alerting/list",
-						Text:       "**Firing**\n\nValue: [no value]\nLabels:\n - alertname = alert1\n - lbl1 = val1\nAnnotations:\n - ann1 = annv1\nSilence: http://localhost/alerting/silence/new?alertmanager=grafana&matcher=alertname%3Dalert1&matcher=lbl1%3Dval1\n",
-						Fallback:   "[FIRING:1]  (val1)",
-						Fields:     nil,
-						Footer:     "Grafana v" + setting.BuildVersion,
-						FooterIcon: "https://grafana.com/static/assets/img/fav32.png",
-						Color:      "#D63232",
-						Ts:         0,
-					},
+		}},
+		expectedMessage: &slackMessage{
+			Channel:   "#test",
+			Username:  "Grafana",
+			IconEmoji: ":emoji:",
+			Attachments: []attachment{
+				{
+					Title:      "[FIRING:1]  (val1)",
+					TitleLink:  "http://localhost/alerting/list",
+					Text:       "**Firing**\n\nValue: [no value]\nLabels:\n - alertname = alert1\n - lbl1 = val1\nAnnotations:\n - ann1 = annv1\nSilence: http://localhost/alerting/silence/new?alertmanager=grafana&matcher=alertname%3Dalert1&matcher=lbl1%3Dval1\nDashboard: http://localhost/d/abcd\nPanel: http://localhost/d/abcd?viewPanel=efgh\n",
+					Fallback:   "[FIRING:1]  (val1)",
+					Fields:     nil,
+					Footer:     "Grafana v" + setting.BuildVersion,
+					FooterIcon: "https://grafana.com/static/assets/img/fav32.png",
+					Color:      "#D63232",
+					ImageURL:   "https://www.example.com/test.png",
 				},
 			},
-			expMsgError: nil,
 		},
-		{
-			name: "Image URL in alert appears in slack message",
-			settings: `{
-				"token": "1234",
-				"recipient": "#testchannel",
-				"icon_emoji": ":emoji:"
+	}, {
+		name: "Message is sent and image on local disk is ignored",
+		settings: `{
+				"icon_emoji": ":emoji:",
+				"recipient": "#test",
+				"url": "https://example.com/hooks/xxxx"
 			}`,
-			alerts: []*types.Alert{
+		alerts: []*types.Alert{{
+			Alert: model.Alert{
+				Labels:      model.LabelSet{"alertname": "alert1", "lbl1": "val1"},
+				Annotations: model.LabelSet{"ann1": "annv1", "__dashboardUid__": "abcd", "__panelId__": "efgh", "__alertImageToken__": "image-on-disk"},
+			},
+		}},
+		expectedMessage: &slackMessage{
+			Channel:   "#test",
+			Username:  "Grafana",
+			IconEmoji: ":emoji:",
+			Attachments: []attachment{
 				{
-					Alert: model.Alert{
-						Labels:      model.LabelSet{"alertname": "alert1", "lbl1": "val1"},
-						Annotations: model.LabelSet{"ann1": "annv1", "__dashboardUid__": "abcd", "__panelId__": "efgh", "__alertImageToken__": "test-with-url"},
-					},
+					Title:      "[FIRING:1]  (val1)",
+					TitleLink:  "http://localhost/alerting/list",
+					Text:       "**Firing**\n\nValue: [no value]\nLabels:\n - alertname = alert1\n - lbl1 = val1\nAnnotations:\n - ann1 = annv1\nSilence: http://localhost/alerting/silence/new?alertmanager=grafana&matcher=alertname%3Dalert1&matcher=lbl1%3Dval1\nDashboard: http://localhost/d/abcd\nPanel: http://localhost/d/abcd?viewPanel=efgh\n",
+					Fallback:   "[FIRING:1]  (val1)",
+					Fields:     nil,
+					Footer:     "Grafana v" + setting.BuildVersion,
+					FooterIcon: "https://grafana.com/static/assets/img/fav32.png",
+					Color:      "#D63232",
 				},
 			},
-			expMsg: &slackMessage{
-				Channel:   "#testchannel",
-				Username:  "Grafana",
-				IconEmoji: ":emoji:",
-				Attachments: []attachment{
-					{
-						Title:      "[FIRING:1]  (val1)",
-						TitleLink:  "http://localhost/alerting/list",
-						Text:       "**Firing**\n\nValue: [no value]\nLabels:\n - alertname = alert1\n - lbl1 = val1\nAnnotations:\n - ann1 = annv1\nSilence: http://localhost/alerting/silence/new?alertmanager=grafana&matcher=alertname%3Dalert1&matcher=lbl1%3Dval1\nDashboard: http://localhost/d/abcd\nPanel: http://localhost/d/abcd?viewPanel=efgh\n",
-						Fallback:   "[FIRING:1]  (val1)",
-						Fields:     nil,
-						Footer:     "Grafana v" + setting.BuildVersion,
-						FooterIcon: "https://grafana.com/static/assets/img/fav32.png",
-						Color:      "#D63232",
-						Ts:         0,
-						ImageURL:   "https://www.example.com/image.jpg",
-					},
+		},
+	}}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			notifier, recorder, err := setupSlackForTests(t, test.settings)
+			require.NoError(t, err)
+
+			ctx := context.Background()
+			ctx = notify.WithGroupKey(ctx, "alertname")
+			ctx = notify.WithGroupLabels(ctx, model.LabelSet{"alertname": ""})
+
+			ok, err := notifier.Notify(ctx, test.alerts...)
+			if test.expectedError != "" {
+				assert.EqualError(t, err, test.expectedError)
+				assert.False(t, ok)
+			} else {
+				assert.NoError(t, err)
+				assert.True(t, ok)
+
+				// When sending a notification to an Incoming Webhook there should a single request.
+				// This is different from PostMessage where some content, such as images, are sent
+				// as replies to the original message
+				require.Len(t, recorder.requests, 1)
+
+				// Get the request and check that it's sending to the URL of the Incoming Webhook
+				r := recorder.requests[0]
+				assert.Equal(t, notifier.settings.URL, r.URL.String())
+
+				// Check that the request contains the expected message
+				b, err := io.ReadAll(r.Body)
+				require.NoError(t, err)
+
+				message := slackMessage{}
+				require.NoError(t, json.Unmarshal(b, &message))
+				for i, v := range message.Attachments {
+					// Need to update the ts as these cannot be set in the test definition
+					test.expectedMessage.Attachments[i].Ts = v.Ts
+				}
+				assert.Equal(t, *test.expectedMessage, message)
+			}
+		})
+	}
+}
+
+func TestSlackPostMessage(t *testing.T) {
+	tests := []struct {
+		name            string
+		alerts          []*types.Alert
+		expectedMessage *slackMessage
+		expectedReplies []interface{} // can contain either slackMessage or map[string]struct{} for multipart/form-data
+		expectedError   string
+		settings        string
+	}{{
+		name: "Message is sent",
+		settings: `{
+			"icon_emoji": ":emoji:",
+			"recipient": "#test",
+			"token": "1234"
+		}`,
+		alerts: []*types.Alert{{
+			Alert: model.Alert{
+				Labels:      model.LabelSet{"alertname": "alert1", "lbl1": "val1"},
+				Annotations: model.LabelSet{"ann1": "annv1", "__dashboardUid__": "abcd", "__panelId__": "efgh"},
+			},
+		}},
+		expectedMessage: &slackMessage{
+			Channel:   "#test",
+			Username:  "Grafana",
+			IconEmoji: ":emoji:",
+			Attachments: []attachment{
+				{
+					Title:      "[FIRING:1]  (val1)",
+					TitleLink:  "http://localhost/alerting/list",
+					Text:       "**Firing**\n\nValue: [no value]\nLabels:\n - alertname = alert1\n - lbl1 = val1\nAnnotations:\n - ann1 = annv1\nSilence: http://localhost/alerting/silence/new?alertmanager=grafana&matcher=alertname%3Dalert1&matcher=lbl1%3Dval1\nDashboard: http://localhost/d/abcd\nPanel: http://localhost/d/abcd?viewPanel=efgh\n",
+					Fallback:   "[FIRING:1]  (val1)",
+					Fields:     nil,
+					Footer:     "Grafana v" + setting.BuildVersion,
+					FooterIcon: "https://grafana.com/static/assets/img/fav32.png",
+					Color:      "#D63232",
 				},
 			},
-			expMsgError: nil,
 		},
-		{
-			name: "Correct config with multiple alerts and template",
-			settings: `{
-				"token": "1234",
-				"recipient": "#testchannel",
-				"icon_emoji": ":emoji:",
-				"title": "{{ .Alerts.Firing | len }} firing, {{ .Alerts.Resolved | len }} resolved"
-			}`,
-			alerts: []*types.Alert{
+	}, {
+		name: "Message is sent with two firing alerts",
+		settings: `{
+			"title": "{{ .Alerts.Firing | len }} firing, {{ .Alerts.Resolved | len }} resolved",
+			"icon_emoji": ":emoji:",
+			"recipient": "#test",
+			"token": "1234"
+		}`,
+		alerts: []*types.Alert{{
+			Alert: model.Alert{
+				Labels:      model.LabelSet{"alertname": "alert1", "lbl1": "val1"},
+				Annotations: model.LabelSet{"ann1": "annv1"},
+			},
+		}, {
+			Alert: model.Alert{
+				Labels:      model.LabelSet{"alertname": "alert1", "lbl1": "val2"},
+				Annotations: model.LabelSet{"ann1": "annv2"},
+			},
+		}},
+		expectedMessage: &slackMessage{
+			Channel:   "#test",
+			Username:  "Grafana",
+			IconEmoji: ":emoji:",
+			Attachments: []attachment{
 				{
-					Alert: model.Alert{
-						Labels:      model.LabelSet{"alertname": "alert1", "lbl1": "val1"},
-						Annotations: model.LabelSet{"ann1": "annv1"},
-					},
+					Title:      "2 firing, 0 resolved",
+					TitleLink:  "http://localhost/alerting/list",
+					Text:       "**Firing**\n\nValue: [no value]\nLabels:\n - alertname = alert1\n - lbl1 = val1\nAnnotations:\n - ann1 = annv1\nSilence: http://localhost/alerting/silence/new?alertmanager=grafana&matcher=alertname%3Dalert1&matcher=lbl1%3Dval1\n\nValue: [no value]\nLabels:\n - alertname = alert1\n - lbl1 = val2\nAnnotations:\n - ann1 = annv2\nSilence: http://localhost/alerting/silence/new?alertmanager=grafana&matcher=alertname%3Dalert1&matcher=lbl1%3Dval2\n",
+					Fallback:   "2 firing, 0 resolved",
+					Fields:     nil,
+					Footer:     "Grafana v" + setting.BuildVersion,
+					FooterIcon: "https://grafana.com/static/assets/img/fav32.png",
+					Color:      "#D63232",
 				},
+			},
+		},
+	}, {
+		name: "Message is sent and image is uploaded",
+		settings: `{
+			"icon_emoji": ":emoji:",
+			"recipient": "#test",
+			"token": "1234"
+		}`,
+		alerts: []*types.Alert{{
+			Alert: model.Alert{
+				Labels:      model.LabelSet{"alertname": "alert1", "lbl1": "val1"},
+				Annotations: model.LabelSet{"ann1": "annv1", "__dashboardUid__": "abcd", "__panelId__": "efgh", "__alertImageToken__": "image-on-disk"},
+			},
+		}},
+		expectedMessage: &slackMessage{
+			Channel:   "#test",
+			Username:  "Grafana",
+			IconEmoji: ":emoji:",
+			Attachments: []attachment{
 				{
-					Alert: model.Alert{
-						Labels:      model.LabelSet{"alertname": "alert1", "lbl1": "val2"},
-						Annotations: model.LabelSet{"ann1": "annv2"},
-					},
+					Title:      "[FIRING:1]  (val1)",
+					TitleLink:  "http://localhost/alerting/list",
+					Text:       "**Firing**\n\nValue: [no value]\nLabels:\n - alertname = alert1\n - lbl1 = val1\nAnnotations:\n - ann1 = annv1\nSilence: http://localhost/alerting/silence/new?alertmanager=grafana&matcher=alertname%3Dalert1&matcher=lbl1%3Dval1\nDashboard: http://localhost/d/abcd\nPanel: http://localhost/d/abcd?viewPanel=efgh\n",
+					Fallback:   "[FIRING:1]  (val1)",
+					Fields:     nil,
+					Footer:     "Grafana v" + setting.BuildVersion,
+					FooterIcon: "https://grafana.com/static/assets/img/fav32.png",
+					Color:      "#D63232",
 				},
 			},
-			expMsg: &slackMessage{
-				Channel:   "#testchannel",
-				Username:  "Grafana",
-				IconEmoji: ":emoji:",
-				Attachments: []attachment{
-					{
-						Title:      "2 firing, 0 resolved",
-						TitleLink:  "http://localhost/alerting/list",
-						Text:       "**Firing**\n\nValue: [no value]\nLabels:\n - alertname = alert1\n - lbl1 = val1\nAnnotations:\n - ann1 = annv1\nSilence: http://localhost/alerting/silence/new?alertmanager=grafana&matcher=alertname%3Dalert1&matcher=lbl1%3Dval1\n\nValue: [no value]\nLabels:\n - alertname = alert1\n - lbl1 = val2\nAnnotations:\n - ann1 = annv2\nSilence: http://localhost/alerting/silence/new?alertmanager=grafana&matcher=alertname%3Dalert1&matcher=lbl1%3Dval2\n",
-						Fallback:   "2 firing, 0 resolved",
-						Fields:     nil,
-						Footer:     "Grafana v" + setting.BuildVersion,
-						FooterIcon: "https://grafana.com/static/assets/img/fav32.png",
-						Color:      "#D63232",
-						Ts:         0,
-					},
-				},
+		},
+		expectedReplies: []interface{}{
+			// check that the following parts are present in the multipart/form-data
+			map[string]struct{}{
+				"file":            {},
+				"channels":        {},
+				"initial_comment": {},
+				"thread_ts":       {},
 			},
-			expMsgError: nil,
-		}, {
-			name: "Missing token",
-			settings: `{
-				"recipient": "#testchannel"
-			}`,
-			expInitError: `token must be specified when using the Slack chat API`,
-		}, {
-			name: "Missing recipient",
-			settings: `{
-				"token": "1234"
-			}`,
-			expInitError: `recipient must be specified when using the Slack chat API`,
 		},
-		{
-			name: "Custom endpoint url",
-			settings: `{
-				"token": "1234",
-				"recipient": "#testchannel",
-				"endpointUrl": "https://slack-custom.com/api/",
-				"icon_emoji": ":emoji:"
-			}`,
-			alerts: []*types.Alert{
-				{
-					Alert: model.Alert{
-						Labels:      model.LabelSet{"alertname": "alert1", "lbl1": "val1"},
-						Annotations: model.LabelSet{"ann1": "annv1"},
-					},
-				},
+	}, {
+		name: "Message is sent to custom URL",
+		settings: `{
+			"icon_emoji": ":emoji:",
+			"recipient": "#test",
+			"endpointUrl": "https://example.com/api",
+			"token": "1234"
+		}`,
+		alerts: []*types.Alert{{
+			Alert: model.Alert{
+				Labels:      model.LabelSet{"alertname": "alert1", "lbl1": "val1"},
+				Annotations: model.LabelSet{"ann1": "annv1"},
 			},
-			expMsg: &slackMessage{
-				Channel:   "#testchannel",
-				Username:  "Grafana",
-				IconEmoji: ":emoji:",
-				Attachments: []attachment{
-					{
-						Title:      "[FIRING:1]  (val1)",
-						TitleLink:  "http://localhost/alerting/list",
-						Text:       "**Firing**\n\nValue: [no value]\nLabels:\n - alertname = alert1\n - lbl1 = val1\nAnnotations:\n - ann1 = annv1\nSilence: http://localhost/alerting/silence/new?alertmanager=grafana&matcher=alertname%3Dalert1&matcher=lbl1%3Dval1\n",
-						Fallback:   "[FIRING:1]  (val1)",
-						Fields:     nil,
-						Footer:     "Grafana v" + setting.BuildVersion,
-						FooterIcon: "https://grafana.com/static/assets/img/fav32.png",
-						Color:      "#D63232",
-						Ts:         0,
-					},
+		}},
+		expectedMessage: &slackMessage{
+			Channel:   "#test",
+			Username:  "Grafana",
+			IconEmoji: ":emoji:",
+			Attachments: []attachment{
+				{
+					Title:      "[FIRING:1]  (val1)",
+					TitleLink:  "http://localhost/alerting/list",
+					Text:       "**Firing**\n\nValue: [no value]\nLabels:\n - alertname = alert1\n - lbl1 = val1\nAnnotations:\n - ann1 = annv1\nSilence: http://localhost/alerting/silence/new?alertmanager=grafana&matcher=alertname%3Dalert1&matcher=lbl1%3Dval1\n",
+					Fallback:   "[FIRING:1]  (val1)",
+					Fields:     nil,
+					Footer:     "Grafana v" + setting.BuildVersion,
+					FooterIcon: "https://grafana.com/static/assets/img/fav32.png",
+					Color:      "#D63232",
 				},
 			},
-			expMsgError: nil,
 		},
-	}
+	}}
 
-	for _, c := range cases {
-		t.Run(c.name, func(t *testing.T) {
-			settingsJSON, err := simplejson.NewJson([]byte(c.settings))
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			notifier, recorder, err := setupSlackForTests(t, test.settings)
 			require.NoError(t, err)
-			secureSettings := make(map[string][]byte)
-
-			secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
-			decryptFn := secretsService.GetDecryptedValue
-			notificationService := mockNotificationService()
-
-			fc := FactoryConfig{
-				Config: &NotificationChannelConfig{
-					Name:           "slack_testing",
-					Type:           "slack",
-					Settings:       settingsJSON,
-					SecureSettings: secureSettings,
-				},
-				ImageStore: fakeImageStore,
-				// TODO: allow changing the associated values for different tests.
-				NotificationService: notificationService,
-				DecryptFunc:         decryptFn,
-				Template:            tmpl,
-			}
 
-			pn, err := buildSlackNotifier(fc)
-			if c.expInitError != "" {
-				require.Error(t, err)
-				require.Equal(t, c.expInitError, err.Error())
-				return
-			}
-			require.NoError(t, err)
+			ctx := context.Background()
+			ctx = notify.WithGroupKey(ctx, "alertname")
+			ctx = notify.WithGroupLabels(ctx, model.LabelSet{"alertname": ""})
 
-			body := ""
-			origSendSlackRequest := sendSlackRequest
-			t.Cleanup(func() {
-				sendSlackRequest = origSendSlackRequest
-			})
-			sendSlackRequest = func(request *http.Request, log log.Logger) error {
-				t.Helper()
-				defer func() {
-					_ = request.Body.Close()
-				}()
-
-				url := settingsJSON.Get("url").MustString()
-				if len(url) == 0 {
-					endpointUrl := settingsJSON.Get("endpointUrl").MustString(SlackAPIEndpoint)
-					require.Equal(t, endpointUrl, request.URL.String())
-				}
+			ok, err := notifier.Notify(ctx, test.alerts...)
+			if test.expectedError != "" {
+				assert.EqualError(t, err, test.expectedError)
+				assert.False(t, ok)
+			} else {
+				assert.NoError(t, err)
+				assert.True(t, ok)
+
+				// When sending a notification via PostMessage some content, such as images,
+				// are sent as replies to the original message
+				require.Len(t, recorder.requests, len(test.expectedReplies)+1)
+
+				// Get the request and check that it's sending to the URL
+				r := recorder.requests[0]
+				assert.Equal(t, notifier.settings.URL, r.URL.String())
 
-				b, err := io.ReadAll(request.Body)
+				// Check that the request contains the expected message
+				b, err := io.ReadAll(r.Body)
 				require.NoError(t, err)
-				body = string(b)
-				return nil
-			}
 
-			ctx := notify.WithGroupKey(context.Background(), "alertname")
-			ctx = notify.WithGroupLabels(ctx, model.LabelSet{"alertname": ""})
-			ok, err := pn.Notify(ctx, c.alerts...)
-			if c.expMsgError != nil {
-				require.Error(t, err)
-				require.False(t, ok)
-				require.Equal(t, c.expMsgError.Error(), err.Error())
-				return
+				message := slackMessage{}
+				require.NoError(t, json.Unmarshal(b, &message))
+				for i, v := range message.Attachments {
+					// Need to update the ts as these cannot be set in the test definition
+					test.expectedMessage.Attachments[i].Ts = v.Ts
+				}
+				assert.Equal(t, *test.expectedMessage, message)
+
+				// Check that the replies match expectations
+				for i := 1; i < len(recorder.requests); i++ {
+					r = recorder.requests[i]
+					assert.Equal(t, "https://slack.com/api/files.upload", r.URL.String())
+
+					media, params, err := mime.ParseMediaType(r.Header.Get("Content-Type"))
+					require.NoError(t, err)
+					if media == "multipart/form-data" {
+						// Some replies are file uploads, so check the multipart form
+						checkMultipart(t, test.expectedReplies[i-1].(map[string]struct{}), r.Body, params["boundary"])
+					} else {
+						b, err = io.ReadAll(r.Body)
+						require.NoError(t, err)
+						message = slackMessage{}
+						require.NoError(t, json.Unmarshal(b, &message))
+						assert.Equal(t, test.expectedReplies[i-1], message)
+					}
+				}
 			}
-			require.True(t, ok)
-			require.NoError(t, err)
+		})
+	}
+}
 
-			// Getting Ts from actual since that can't be predicted.
-			var obj slackMessage
-			require.NoError(t, json.Unmarshal([]byte(body), &obj))
-			c.expMsg.Attachments[0].Ts = obj.Attachments[0].Ts
+// slackRequestRecorder is used in tests to record all requests.
+type slackRequestRecorder struct {
+	requests []*http.Request
+}
 
-			expBody, err := json.Marshal(c.expMsg)
-			require.NoError(t, err)
+func (s *slackRequestRecorder) fn(_ context.Context, r *http.Request, _ log.Logger) (string, error) {
+	s.requests = append(s.requests, r)
+	return "", nil
+}
 
-			require.JSONEq(t, string(expBody), body)
+// checkMulipart checks that each part is present, but not its contents
+func checkMultipart(t *testing.T, expected map[string]struct{}, r io.Reader, boundary string) {
+	m := multipart.NewReader(r, boundary)
+	visited := make(map[string]struct{})
+	for {
+		part, err := m.NextPart()
+		if errors.Is(err, io.EOF) {
+			break
+		}
+		require.NoError(t, err)
+		visited[part.FormName()] = struct{}{}
+	}
+	assert.Equal(t, expected, visited)
+}
 
-			// If we should have sent to the webhook, the mock notification service
-			// will have a record of it.
-			require.Equal(t, c.expWebhookURL, notificationService.Webhook.Url)
+func setupSlackForTests(t *testing.T, settings string) (*SlackNotifier, *slackRequestRecorder, error) {
+	tmpl := templateForTests(t)
+	externalURL, err := url.Parse("http://localhost")
+	require.NoError(t, err)
+	tmpl.ExternalURL = externalURL
+
+	f, err := os.Create(t.TempDir() + "test.png")
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		if err := os.Remove(f.Name()); err != nil {
+			t.Logf("failed to delete test file: %s", err)
+		}
+	})
+
+	images := &fakeImageStore{
+		Images: []*models.Image{{
+			Token: "image-on-disk",
+			Path:  f.Name(),
+		}, {
+			Token: "image-with-url",
+			URL:   "https://www.example.com/test.png",
+		}},
+	}
+
+	settingsJSON, err := simplejson.NewJson([]byte(settings))
+	require.NoError(t, err)
+
+	secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
+	notificationService := mockNotificationService()
+
+	c := FactoryConfig{
+		Config: &NotificationChannelConfig{
+			Name:           "slack_testing",
+			Type:           "slack",
+			Settings:       settingsJSON,
+			SecureSettings: make(map[string][]byte),
+		},
+		ImageStore:          images,
+		NotificationService: notificationService,
+		DecryptFunc:         secretsService.GetDecryptedValue,
+		Template:            tmpl,
+	}
+
+	sn, err := buildSlackNotifier(c)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	sr := &slackRequestRecorder{}
+	sn.sendFn = sr.fn
+	return sn, sr, nil
+}
+
+func TestCreateSlackNotifierFromConfig(t *testing.T) {
+	tests := []struct {
+		name          string
+		settings      string
+		expectedError string
+	}{{
+		name: "Missing token",
+		settings: `{
+			"recipient": "#testchannel"
+		}`,
+		expectedError: "token must be specified when using the Slack chat API",
+	}, {
+		name: "Missing recipient",
+		settings: `{
+			"token": "1234"
+		}`,
+		expectedError: "recipient must be specified when using the Slack chat API",
+	}}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			n, _, err := setupSlackForTests(t, test.settings)
+			if test.expectedError != "" {
+				assert.Nil(t, n)
+				assert.EqualError(t, err, test.expectedError)
+			} else {
+				assert.NotNil(t, n)
+				assert.Nil(t, err)
+			}
 		})
 	}
 }
 
 func TestSendSlackRequest(t *testing.T) {
 	tests := []struct {
-		name          string
-		slackResponse string
-		statusCode    int
-		expectError   bool
+		name        string
+		response    string
+		statusCode  int
+		expectError bool
 	}{
 		{
 			name: "Example error",
-			slackResponse: `{
+			response: `{
 					"ok": false,
 					"error": "too_many_attachments"
 				}`,
@@ -352,7 +509,7 @@ func TestSendSlackRequest(t *testing.T) {
 		},
 		{
 			name: "Success case, normal response body",
-			slackResponse: `{
+			response: `{
 				"ok": true,
 				"channel": "C1H9RESGL",
 				"ts": "1503435956.000247",
@@ -376,26 +533,27 @@ func TestSendSlackRequest(t *testing.T) {
 			expectError: false,
 		},
 		{
-			name:       "No response body",
-			statusCode: http.StatusOK,
+			name:        "No response body",
+			statusCode:  http.StatusOK,
+			expectError: true,
 		},
 		{
-			name:          "Success case, unexpected response body",
-			statusCode:    http.StatusOK,
-			slackResponse: `{"test": true}`,
-			expectError:   false,
+			name:        "Success case, unexpected response body",
+			statusCode:  http.StatusOK,
+			response:    `{"test": true}`,
+			expectError: true,
 		},
 		{
-			name:          "Success case, ok: true",
-			statusCode:    http.StatusOK,
-			slackResponse: `{"ok": true}`,
-			expectError:   false,
+			name:        "Success case, ok: true",
+			statusCode:  http.StatusOK,
+			response:    `{"ok": true}`,
+			expectError: false,
 		},
 		{
-			name:          "200 status code, error in body",
-			statusCode:    http.StatusOK,
-			slackResponse: `{"ok": false, "error": "test error"}`,
-			expectError:   true,
+			name:        "200 status code, error in body",
+			statusCode:  http.StatusOK,
+			response:    `{"ok": false, "error": "test error"}`,
+			expectError: true,
 		},
 	}
 
@@ -403,14 +561,14 @@ func TestSendSlackRequest(t *testing.T) {
 		t.Run(test.name, func(tt *testing.T) {
 			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 				w.WriteHeader(test.statusCode)
-				_, err := w.Write([]byte(test.slackResponse))
+				_, err := w.Write([]byte(test.response))
 				require.NoError(tt, err)
 			}))
 			defer server.Close()
 			req, err := http.NewRequest(http.MethodGet, server.URL, nil)
 			require.NoError(tt, err)
 
-			err = sendSlackRequest(req, log.New("test"))
+			_, err = sendSlackRequest(context.Background(), req, log.New("test"))
 			if !test.expectError {
 				require.NoError(tt, err)
 			} else {

pkg/tests/api/alerting/api_notification_channel_test.go

@@ -707,6 +707,10 @@ func TestNotificationChannels(t *testing.T) {
 	amConfig := getAlertmanagerConfig(mockChannel.server.Addr)
 	mockEmail := &mockEmailHandler{}
 
+	// Set up responses
+	mockChannel.responses["slack_recv1"] = `{"ok": true}`
+	mockChannel.responses["slack_recvX"] = `{"ok": true}`
+
 	// Overriding some URLs to send to the mock channel.
 	os, opa, ot, opu, ogb, ol, oth := channels.SlackAPIEndpoint, channels.PagerdutyEventAPIURL,
 		channels.TelegramAPIURL, channels.PushoverEndpoint, channels.GetBoundary,
@@ -990,6 +994,7 @@ type mockNotificationChannel struct {
 	server *http.Server
 
 	receivedNotifications  map[string][]string
+	responses              map[string]string
 	notificationErrorCount int
 	notificationsMtx       sync.RWMutex
 }
@@ -1007,6 +1012,7 @@ func newMockNotificationChannel(t *testing.T, grafanaListedAddr string) *mockNot
 			Addr: listener.Addr().String(),
 		},
 		receivedNotifications: make(map[string][]string),
+		responses:             make(map[string]string),
 		t:                     t,
 	}
 
@@ -1036,6 +1042,7 @@ func (nc *mockNotificationChannel) ServeHTTP(res http.ResponseWriter, req *http.
 
 	nc.receivedNotifications[key] = append(nc.receivedNotifications[key], body)
 	res.WriteHeader(http.StatusOK)
+	fmt.Fprint(res, nc.responses[paths[0]])
 }
 
 func (nc *mockNotificationChannel) totalNotifications() int {
@@ -2635,7 +2642,7 @@ var expNonEmailNotifications = map[string][]string{
 
 // expNotificationErrors maps a receiver name with its expected error string.
 var expNotificationErrors = map[string]string{
-	"slack_failed_recv": "request to Slack API failed with status code 500",
+	"slack_failed_recv": "failed to send Slack message: unexpected 5xx status code: 500",
 }
 
 // expInactiveReceivers is a set of receivers expected to be unused.