Remote Alertmanager: Send SMTP config (#106337)

* (WIP) Remote Alertmanager: Send SMTP config

* send SMTP configs separately

* bring back deleted fields

* actually send stuff over

* remove redundant type, fix comments

* smtp -> smtpConfig

* also send SmtpFrom an StaticHeaders separately

* tests

* restore defaults.ini

pkg/services/ngalert/ngalert.go

@@ -38,6 +38,7 @@ import (
 	"github.com/grafana/grafana/pkg/services/ngalert/notifier/legacy_storage"
 	"github.com/grafana/grafana/pkg/services/ngalert/provisioning"
 	"github.com/grafana/grafana/pkg/services/ngalert/remote"
+	remoteClient "github.com/grafana/grafana/pkg/services/ngalert/remote/client"
 	"github.com/grafana/grafana/pkg/services/ngalert/schedule"
 	"github.com/grafana/grafana/pkg/services/ngalert/sender"
 	"github.com/grafana/grafana/pkg/services/ngalert/state"
@@ -187,15 +188,30 @@ func (ng *AlertNG) init() error {
 	remoteSecondary := ng.FeatureToggles.IsEnabled(initCtx, featuremgmt.FlagAlertmanagerRemoteSecondary)
 	if remotePrimary || remoteSecondary {
 		m := ng.Metrics.GetRemoteAlertmanagerMetrics()
+		smtpCfg := remoteClient.SmtpConfig{
+			FromAddress:    ng.Cfg.Smtp.FromAddress,
+			FromName:       ng.Cfg.Smtp.FromName,
+			Host:           ng.Cfg.Smtp.Host,
+			User:           ng.Cfg.Smtp.User,
+			Password:       ng.Cfg.Smtp.Password,
+			EhloIdentity:   ng.Cfg.Smtp.EhloIdentity,
+			StartTLSPolicy: ng.Cfg.Smtp.StartTLSPolicy,
+			SkipVerify:     ng.Cfg.Smtp.SkipVerify,
+			StaticHeaders:  ng.Cfg.Smtp.StaticHeaders,
+		}
+
 		cfg := remote.AlertmanagerConfig{
 			BasicAuthPassword: ng.Cfg.UnifiedAlerting.RemoteAlertmanager.Password,
 			DefaultConfig:     ng.Cfg.UnifiedAlerting.DefaultConfiguration,
 			TenantID:          ng.Cfg.UnifiedAlerting.RemoteAlertmanager.TenantID,
 			URL:               ng.Cfg.UnifiedAlerting.RemoteAlertmanager.URL,
 			ExternalURL:       ng.Cfg.AppURL,
+			SmtpConfig:        smtpCfg,
+			Timeout:           ng.Cfg.UnifiedAlerting.RemoteAlertmanager.Timeout,
+
+			// TODO: Remove once everything can be sent in the 'smtp_config' field.
 			SmtpFrom:      ng.Cfg.Smtp.FromAddress,
 			StaticHeaders: ng.Cfg.Smtp.StaticHeaders,
-			Timeout:           ng.Cfg.UnifiedAlerting.RemoteAlertmanager.Timeout,
 		}
 		autogenFn := func(ctx context.Context, logger log.Logger, orgID int64, cfg *definitions.PostableApiAlertingConfig, skipInvalid bool) error {
 			return notifier.AddAutogenConfig(ctx, logger, ng.store, orgID, cfg, skipInvalid)

pkg/services/ngalert/remote/alertmanager.go

@@ -70,6 +70,8 @@ type Alertmanager struct {
 
 	amClient    *remoteClient.Alertmanager
 	mimirClient remoteClient.MimirClient
+
+	smtp remoteClient.SmtpConfig
 }
 
 type AlertmanagerConfig struct {
@@ -87,15 +89,19 @@ type AlertmanagerConfig struct {
 	// The same flag is used for promoting state.
 	PromoteConfig bool
 
-	// SmtpFrom and StaticHeaders are used in email notifications sent by the remote Alertmanager.
+	// SmtpConfig has all the necessary settings for the remote Alertmanager to create an email sender.
-	SmtpFrom      string
+	SmtpConfig remoteClient.SmtpConfig
-	StaticHeaders map[string]string
 
 	// SyncInterval determines how often we should attempt to synchronize configuration.
 	SyncInterval time.Duration
 
 	// Timeout for the HTTP client.
 	Timeout time.Duration
+
+	// TODO: Remove once everything can be send in the 'smtp_config' field.
+	// SmtpFrom and StaticHeaders are used in email notifications sent by the remote Alertmanager.
+	SmtpFrom      string
+	StaticHeaders map[string]string
 }
 
 func (cfg *AlertmanagerConfig) Validate() error {
@@ -131,6 +137,10 @@ func NewAlertmanager(ctx context.Context, cfg AlertmanagerConfig, store stateSto
 		URL:           u,
 		PromoteConfig: cfg.PromoteConfig,
 		ExternalURL:   cfg.ExternalURL,
+
+		Smtp: cfg.SmtpConfig,
+
+		// TODO: Remove once everything can be sent in the 'smtp_config' field.
 		SmtpFrom:      cfg.SmtpFrom,
 		StaticHeaders: cfg.StaticHeaders,
 	}
@@ -200,12 +210,15 @@ func NewAlertmanager(ctx context.Context, cfg AlertmanagerConfig, store stateSto
 		metrics:           metrics,
 		mimirClient:       mc,
 		orgID:             cfg.OrgID,
-		smtpFrom:          cfg.SmtpFrom,
 		state:             store,
 		sender:            s,
 		syncInterval:      cfg.SyncInterval,
 		tenantID:          cfg.TenantID,
 		url:               cfg.URL,
+		smtp:              cfg.SmtpConfig,
+
+		// TODO: Remove once it can be sent only in the 'smtp_config' field.
+		smtpFrom: cfg.SmtpFrom,
 	}, nil
 }
 
@@ -686,11 +699,34 @@ func (am *Alertmanager) shouldSendConfig(ctx context.Context, hash [16]byte) boo
 		return true
 	}
 
+	// TODO: Remove when the from address can be sent only in the 'smtp_config' field.
 	if rc.SmtpFrom != am.smtpFrom {
 		am.log.Debug("SMTP 'from' address is different, sending the configuration to the remote Alertmanager", "remote", rc.SmtpFrom, "local", am.smtpFrom)
 		return true
 	}
 
+	// Compare SMTP configs.
+	if rc.SmtpConfig.EhloIdentity != am.smtp.EhloIdentity ||
+		rc.SmtpConfig.Password != am.smtp.Password ||
+		rc.SmtpConfig.FromAddress != am.smtp.FromAddress ||
+		rc.SmtpConfig.FromName != am.smtp.FromName ||
+		rc.SmtpConfig.Host != am.smtp.Host ||
+		rc.SmtpConfig.SkipVerify != am.smtp.SkipVerify ||
+		rc.SmtpConfig.StartTLSPolicy != am.smtp.StartTLSPolicy ||
+		len(rc.SmtpConfig.StaticHeaders) != len(am.smtp.StaticHeaders) ||
+		rc.SmtpConfig.User != am.smtp.User {
+		am.log.Debug("SMTP config is different, sending the configuration to the remote Alertmanager")
+		return true
+	}
+
+	for k, v := range rc.SmtpConfig.StaticHeaders {
+		if value, ok := am.smtp.StaticHeaders[k]; !ok || v != value {
+			am.log.Debug("SMTP static headers are different, sending the configuration to the remote Alertmanager")
+			return true
+		}
+	}
+
+	// Hash and compare Alertmanager configs.
 	rawRemote, err := json.Marshal(rc.GrafanaAlertmanagerConfig)
 	if err != nil {
 		am.log.Error("Unable to marshal the remote Alertmanager configuration for comparison", "err", err)

pkg/services/ngalert/remote/alertmanager_test.go

@@ -188,6 +188,10 @@ func TestApplyConfig(t *testing.T) {
 		PromoteConfig: true,
 		SyncInterval:  1 * time.Hour,
 		ExternalURL:   "https://test.grafana.com",
+		SmtpConfig: client.SmtpConfig{
+			FromAddress: "test-instance@grafana.net",
+		},
+
 		SmtpFrom:      "test-instance@grafana.net",
 		StaticHeaders: map[string]string{"Header-1": "Value-1", "Header-2": "Value-2"},
 	}
@@ -259,10 +263,28 @@ func TestApplyConfig(t *testing.T) {
 	require.Equal(t, 3, configSyncs)
 	require.Equal(t, am.smtpFrom, configSent.SmtpFrom)
 
+	// Changing fields in the SMTP config should result in the configuration being updated.
+	cfg.SmtpConfig = client.SmtpConfig{
+		EhloIdentity:   "test",
+		FromAddress:    "test@test.com",
+		FromName:       "Test Name",
+		Host:           "test:25",
+		Password:       "test",
+		SkipVerify:     true,
+		StartTLSPolicy: "test",
+		StaticHeaders:  map[string]string{"test": "true"},
+		User:           "Test User",
+	}
+	am, err = NewAlertmanager(context.Background(), cfg, fstore, secretsService.Decrypt, NoopAutogenFn, m, tracing.InitializeTracerForTest())
+	require.NoError(t, err)
+	require.NoError(t, am.ApplyConfig(ctx, config))
+	require.Equal(t, 4, configSyncs)
+	require.Equal(t, am.smtp, configSent.SmtpConfig)
+
 	// Failing to add the auto-generated routes should result in an error.
 	_, err = NewAlertmanager(context.Background(), cfg, fstore, secretsService.Decrypt, errAutogenFn, m, tracing.InitializeTracerForTest())
 	require.ErrorIs(t, err, errTest)
-	require.Equal(t, 3, configSyncs)
+	require.Equal(t, 4, configSyncs)
 }
 
 func TestCompareAndSendConfiguration(t *testing.T) {

pkg/services/ngalert/remote/client/alertmanager_configuration.go

@@ -22,6 +22,9 @@ type UserGrafanaConfig struct {
 	Default                   bool                          `json:"default"`
 	Promoted                  bool                          `json:"promoted"`
 	ExternalURL               string                        `json:"external_url"`
+	SmtpConfig                SmtpConfig                    `json:"smtp_config"`
+
+	// TODO: Remove once everything can be sent in the 'SmtpConfig' field.
 	SmtpFrom      string            `json:"smtp_from"`
 	StaticHeaders map[string]string `json:"static_headers"`
 }
@@ -57,6 +60,9 @@ func (mc *Mimir) CreateGrafanaAlertmanagerConfig(ctx context.Context, cfg *apimo
 		Default:                   isDefault,
 		Promoted:                  mc.promoteConfig,
 		ExternalURL:               mc.externalURL,
+		SmtpConfig:                mc.smtpConfig,
+
+		// TODO: Remove once everything can be sent only in the 'smtp_config' field.
 		SmtpFrom:      mc.smtpFrom,
 		StaticHeaders: mc.staticHeaders,
 	})

pkg/services/ngalert/remote/client/mimir.go

@@ -48,10 +48,25 @@ type Mimir struct {
 	metrics       *metrics.RemoteAlertmanager
 	promoteConfig bool
 	externalURL   string
+	smtpConfig    SmtpConfig
+
+	// TODO: Remove once everything can be sent in the 'smtp' field.
 	smtpFrom      string
 	staticHeaders map[string]string
 }
 
+type SmtpConfig struct {
+	EhloIdentity   string            `json:"ehlo_identity"`
+	FromAddress    string            `json:"from_address"`
+	FromName       string            `json:"from_name"`
+	Host           string            `json:"host"`
+	Password       string            `json:"password"`
+	SkipVerify     bool              `json:"skip_verify"`
+	StartTLSPolicy string            `json:"start_tls_policy"`
+	StaticHeaders  map[string]string `json:"static_headers"`
+	User           string            `json:"user"`
+}
+
 type Config struct {
 	URL      *url.URL
 	TenantID string
@@ -60,6 +75,9 @@ type Config struct {
 	Logger        log.Logger
 	PromoteConfig bool
 	ExternalURL   string
+	Smtp          SmtpConfig
+
+	// TODO: Remove once everything can be sent in the 'smtp_config' field.
 	SmtpFrom      string
 	StaticHeaders map[string]string
 }
@@ -105,6 +123,9 @@ func New(cfg *Config, metrics *metrics.RemoteAlertmanager, tracer tracing.Tracer
 		metrics:       metrics,
 		promoteConfig: cfg.PromoteConfig,
 		externalURL:   cfg.ExternalURL,
+		smtpConfig:    cfg.Smtp,
+
+		// TODO: Remove once everything can be sent in the 'smtp_config' field.
 		smtpFrom:      cfg.SmtpFrom,
 		staticHeaders: cfg.StaticHeaders,
 	}, nil