Alerting: Feature toggle to disallow sending alerts externally (#87982)

* Define feature toggle

* Implement feature toggle

packages/grafana-data/src/types/featureToggles.gen.ts

@@ -190,4 +190,5 @@ export interface FeatureToggles {
   notificationBanner?: boolean;
   dashboardRestore?: boolean;
   datasourceProxyDisableRBAC?: boolean;
+  alertingDisableSendAlertsExternal?: boolean;
 }

pkg/services/featuremgmt/registry.go

@@ -1282,6 +1282,15 @@ var (
 			Owner:        identityAccessTeam,
 			HideFromDocs: true,
 		},
+		{
+			Name:              "alertingDisableSendAlertsExternal",
+			Description:       "Disables the ability to send alerts to an external Alertmanager datasource.",
+			Stage:             FeatureStageExperimental,
+			Owner:             grafanaAlertingSquad,
+			AllowSelfServe:    false,
+			HideFromDocs:      true,
+			HideFromAdminPage: true,
+		},
 	}
 )
 

pkg/services/featuremgmt/toggles_gen.csv

@@ -171,3 +171,4 @@ newDashboardSharingComponent,experimental,@grafana/sharing-squad,false,false,tru
 notificationBanner,experimental,@grafana/grafana-frontend-platform,false,false,false
 dashboardRestore,experimental,@grafana/grafana-frontend-platform,false,false,false
 datasourceProxyDisableRBAC,GA,@grafana/identity-access-team,false,false,false
+alertingDisableSendAlertsExternal,experimental,@grafana/alerting-squad,false,false,false

pkg/services/featuremgmt/toggles_gen.go

@@ -694,4 +694,8 @@ const (
 	// FlagDatasourceProxyDisableRBAC
 	// Disables applying a plugin route's ReqAction field to authorization
 	FlagDatasourceProxyDisableRBAC = "datasourceProxyDisableRBAC"
+
+	// FlagAlertingDisableSendAlertsExternal
+	// Disables the ability to send alerts to an external Alertmanager datasource.
+	FlagAlertingDisableSendAlertsExternal = "alertingDisableSendAlertsExternal"
 )

pkg/services/ngalert/api/api.go

@@ -144,6 +144,7 @@ func (api *API) RegisterAPIEndpoints(m *metrics.API) {
 			store:                api.AdminConfigStore,
 			log:                  logger,
 			alertmanagerProvider: api.AlertsRouter,
+			featureManager:       api.FeatureManager,
 		},
 	), m)
 

pkg/services/ngalert/api/api_configuration.go

@@ -12,6 +12,7 @@ import (
 	"github.com/grafana/grafana/pkg/infra/log"
 	contextmodel "github.com/grafana/grafana/pkg/services/contexthandler/model"
 	"github.com/grafana/grafana/pkg/services/datasources"
+	"github.com/grafana/grafana/pkg/services/featuremgmt"
 	apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
 	ngmodels "github.com/grafana/grafana/pkg/services/ngalert/models"
 	"github.com/grafana/grafana/pkg/services/ngalert/store"
@@ -24,6 +25,7 @@ type ConfigSrv struct {
 	alertmanagerProvider ExternalAlertmanagerProvider
 	store                store.AdminConfigurationStore
 	log                  log.Logger
+	featureManager       featuremgmt.FeatureToggles
 }
 
 func (srv ConfigSrv) RouteGetAlertmanagers(c *contextmodel.ReqContext) response.Response {
@@ -75,6 +77,11 @@ func (srv ConfigSrv) RoutePostNGalertConfig(c *contextmodel.ReqContext, body api
 		return response.Error(http.StatusBadRequest, "Invalid alertmanager choice specified", err)
 	}
 
+	disableExternal := srv.featureManager.IsEnabled(c.Req.Context(), featuremgmt.FlagAlertingDisableSendAlertsExternal)
+	if disableExternal && sendAlertsTo != ngmodels.InternalAlertmanager {
+		return response.Error(http.StatusBadRequest, "Sending alerts to external alertmanagers is disallowed on this instance", err)
+	}
+
 	externalAlertmanagers, err := srv.externalAlertmanagers(c.Req.Context(), c.SignedInUser.GetOrgID())
 	if err != nil {
 		return response.Error(http.StatusInternalServerError, "Couldn't fetch the external Alertmanagers from datasources", err)

pkg/services/ngalert/api/api_configuration_test.go

@@ -10,6 +10,7 @@ import (
 	"github.com/grafana/grafana/pkg/components/simplejson"
 	"github.com/grafana/grafana/pkg/services/datasources"
 	fakeDatasources "github.com/grafana/grafana/pkg/services/datasources/fakes"
+	"github.com/grafana/grafana/pkg/services/featuremgmt"
 	"github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
 	"github.com/grafana/grafana/pkg/services/ngalert/store"
 	"github.com/grafana/grafana/pkg/services/org"
@@ -22,6 +23,7 @@ func TestExternalAlertmanagerChoice(t *testing.T) {
 		datasources        []*datasources.DataSource
 		statusCode         int
 		message            string
+		features           featuremgmt.FeatureToggles
 	}{
 		{
 			name:               "setting the choice to external by having a enabled external am datasource should succeed",
@@ -38,6 +40,7 @@ func TestExternalAlertmanagerChoice(t *testing.T) {
 			},
 			statusCode: http.StatusCreated,
 			message:    "admin configuration updated",
+			features:   featuremgmt.WithFeatures(),
 		},
 		{
 			name:               "setting the choice to external by having a disabled external am datasource should fail",
@@ -52,6 +55,7 @@ func TestExternalAlertmanagerChoice(t *testing.T) {
 			},
 			statusCode: http.StatusBadRequest,
 			message:    "At least one Alertmanager must be provided or configured as a datasource that handles alerts to choose this option",
+			features:   featuremgmt.WithFeatures(),
 		},
 		{
 			name:               "setting the choice to external and having no am configured should fail",
@@ -59,6 +63,7 @@ func TestExternalAlertmanagerChoice(t *testing.T) {
 			datasources:        []*datasources.DataSource{},
 			statusCode:         http.StatusBadRequest,
 			message:            "At least one Alertmanager must be provided or configured as a datasource that handles alerts to choose this option",
+			features:           featuremgmt.WithFeatures(),
 		},
 		{
 			name:               "setting the choice to all and having no external am configured should succeed",
@@ -66,6 +71,7 @@ func TestExternalAlertmanagerChoice(t *testing.T) {
 			datasources:        []*datasources.DataSource{},
 			statusCode:         http.StatusCreated,
 			message:            "admin configuration updated",
+			features:           featuremgmt.WithFeatures(),
 		},
 		{
 			name:               "setting the choice to internal should always succeed",
@@ -73,13 +79,38 @@ func TestExternalAlertmanagerChoice(t *testing.T) {
 			datasources:        []*datasources.DataSource{},
 			statusCode:         http.StatusCreated,
 			message:            "admin configuration updated",
+			features:           featuremgmt.WithFeatures(),
+		},
+		{
+			name:               "setting the choice to internal should succeed when external disallowed",
+			alertmanagerChoice: definitions.InternalAlertmanager,
+			datasources:        []*datasources.DataSource{},
+			statusCode:         http.StatusCreated,
+			message:            "admin configuration updated",
+			features:           featuremgmt.WithFeatures(featuremgmt.FlagAlertingDisableSendAlertsExternal),
+		},
+		{
+			name:               "setting the choice to all should fail when external disallowed",
+			alertmanagerChoice: definitions.AllAlertmanagers,
+			datasources:        []*datasources.DataSource{},
+			statusCode:         http.StatusBadRequest,
+			message:            "Sending alerts to external alertmanagers is disallowed on this instance",
+			features:           featuremgmt.WithFeatures(featuremgmt.FlagAlertingDisableSendAlertsExternal),
+		},
+		{
+			name:               "setting the choice to external should fail when external disallowed",
+			alertmanagerChoice: definitions.ExternalAlertmanagers,
+			datasources:        []*datasources.DataSource{},
+			statusCode:         http.StatusBadRequest,
+			message:            "Sending alerts to external alertmanagers is disallowed on this instance",
+			features:           featuremgmt.WithFeatures(featuremgmt.FlagAlertingDisableSendAlertsExternal),
 		},
 	}
 	ctx := createRequestCtxInOrg(1)
 	ctx.OrgRole = org.RoleAdmin
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			sut := createAPIAdminSut(t, test.datasources)
+			sut := createAPIAdminSut(t, test.datasources, test.features)
 			resp := sut.RoutePostNGalertConfig(ctx, definitions.PostableNGalertConfig{
 				AlertmanagersChoice: test.alertmanagerChoice,
 			})
@@ -93,11 +124,12 @@ func TestExternalAlertmanagerChoice(t *testing.T) {
 }
 
 func createAPIAdminSut(t *testing.T,
-	datasources []*datasources.DataSource) ConfigSrv {
+	datasources []*datasources.DataSource, features featuremgmt.FeatureToggles) ConfigSrv {
 	return ConfigSrv{
 		datasourceService: &fakeDatasources.FakeDataSourceService{
 			DataSources: datasources,
 		},
 		store:          store.NewFakeAdminConfigStore(t),
+		featureManager: features,
 	}
 }

pkg/services/ngalert/ngalert.go

@@ -250,10 +250,10 @@ func (ng *AlertNG) init() error {
 	clk := clock.New()
 
 	alertsRouter := sender.NewAlertsRouter(ng.MultiOrgAlertmanager, ng.store, clk, appUrl, ng.Cfg.UnifiedAlerting.DisabledOrgs,
-		ng.Cfg.UnifiedAlerting.AdminConfigPollInterval, ng.DataSourceService, ng.SecretsService)
+		ng.Cfg.UnifiedAlerting.AdminConfigPollInterval, ng.DataSourceService, ng.SecretsService, ng.FeatureToggles)
 
 	// Make sure we sync at least once as Grafana starts to get the router up and running before we start sending any alerts.
-	if err := alertsRouter.SyncAndApplyConfigFromDatabase(); err != nil {
+	if err := alertsRouter.SyncAndApplyConfigFromDatabase(initCtx); err != nil {
 		return fmt.Errorf("failed to initialize alerting because alert notifications router failed to warm up: %w", err)
 	}
 

pkg/services/ngalert/sender/router.go

@@ -16,6 +16,7 @@ import (
 	"github.com/grafana/grafana/pkg/api/datasource"
 	"github.com/grafana/grafana/pkg/infra/log"
 	"github.com/grafana/grafana/pkg/services/datasources"
+	"github.com/grafana/grafana/pkg/services/featuremgmt"
 	"github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
 	"github.com/grafana/grafana/pkg/services/ngalert/models"
 	"github.com/grafana/grafana/pkg/services/ngalert/notifier"
@@ -48,11 +49,12 @@ type AlertsRouter struct {
 
 	datasourceService datasources.DataSourceService
 	secretService     secrets.Service
+	featureManager    featuremgmt.FeatureToggles
 }
 
 func NewAlertsRouter(multiOrgNotifier *notifier.MultiOrgAlertmanager, store store.AdminConfigurationStore,
 	clk clock.Clock, appURL *url.URL, disabledOrgs map[int64]struct{}, configPollInterval time.Duration,
-	datasourceService datasources.DataSourceService, secretService secrets.Service) *AlertsRouter {
+	datasourceService datasources.DataSourceService, secretService secrets.Service, featureManager featuremgmt.FeatureToggles) *AlertsRouter {
 	d := &AlertsRouter{
 		logger:           log.New("ngalert.sender.router"),
 		clock:            clk,
@@ -71,13 +73,14 @@ func NewAlertsRouter(multiOrgNotifier *notifier.MultiOrgAlertmanager, store stor
 
 		datasourceService: datasourceService,
 		secretService:     secretService,
+		featureManager:    featureManager,
 	}
 	return d
 }
 
 // SyncAndApplyConfigFromDatabase looks for the admin configuration in the database
 // and adjusts the sender(s) and alert handling mechanism accordingly.
-func (d *AlertsRouter) SyncAndApplyConfigFromDatabase() error {
+func (d *AlertsRouter) SyncAndApplyConfigFromDatabase(ctx context.Context) error {
 	cfgs, err := d.adminConfigStore.GetAdminConfigurations()
 	if err != nil {
 		return err
@@ -85,6 +88,8 @@ func (d *AlertsRouter) SyncAndApplyConfigFromDatabase() error {
 
 	d.logger.Debug("Attempting to sync admin configs", "count", len(cfgs))
 
+	disableExternal := d.featureManager.IsEnabled(ctx, featuremgmt.FlagAlertingDisableSendAlertsExternal)
+
 	orgsFound := make(map[int64]struct{}, len(cfgs))
 	d.adminConfigMtx.Lock()
 	for _, cfg := range cfgs {
@@ -93,6 +98,11 @@ func (d *AlertsRouter) SyncAndApplyConfigFromDatabase() error {
 			continue
 		}
 
+		if disableExternal && cfg.SendAlertsTo != models.InternalAlertmanager {
+			d.logger.Warn("Alertmanager choice in configuration will be ignored due to feature flags", "org", cfg.OrgID, "choice", cfg.SendAlertsTo)
+			cfg.SendAlertsTo = models.InternalAlertmanager
+		}
+
 		// Update the Alertmanagers choice for the organization.
 		d.sendAlertsTo[cfg.OrgID] = cfg.SendAlertsTo
 
@@ -363,7 +373,7 @@ func (d *AlertsRouter) Run(ctx context.Context) error {
 	for {
 		select {
 		case <-time.After(d.adminConfigPollInterval):
-			if err := d.SyncAndApplyConfigFromDatabase(); err != nil {
+			if err := d.SyncAndApplyConfigFromDatabase(ctx); err != nil {
 				d.logger.Error("Unable to sync admin configuration", "error", err)
 			}
 		case <-ctx.Done():

pkg/services/ngalert/sender/router_test.go

@@ -63,14 +63,14 @@ func TestIntegrationSendingToExternalAlertmanager(t *testing.T) {
 		}),
 	}
 	alertsRouter := NewAlertsRouter(moa, fakeAdminConfigStore, mockedClock, appUrl, map[int64]struct{}{}, 10*time.Minute,
-		&fake_ds.FakeDataSourceService{DataSources: []*datasources.DataSource{&ds1}}, fake_secrets.NewFakeSecretsService())
+		&fake_ds.FakeDataSourceService{DataSources: []*datasources.DataSource{&ds1}}, fake_secrets.NewFakeSecretsService(), featuremgmt.WithFeatures())
 
 	mockedGetAdminConfigurations.Return([]*models.AdminConfiguration{
 		{OrgID: ruleKey.OrgID, SendAlertsTo: models.AllAlertmanagers},
 	}, nil)
 	// Make sure we sync the configuration at least once before the evaluation happens to guarantee the sender is running
 	// when the first alert triggers.
-	require.NoError(t, alertsRouter.SyncAndApplyConfigFromDatabase())
+	require.NoError(t, alertsRouter.SyncAndApplyConfigFromDatabase(context.Background()))
 	require.Equal(t, 1, len(alertsRouter.externalAlertmanagers))
 	require.Equal(t, 1, len(alertsRouter.externalAlertmanagersCfgHash))
 
@@ -93,7 +93,7 @@ func TestIntegrationSendingToExternalAlertmanager(t *testing.T) {
 	// Now, let's remove the Alertmanager from the admin configuration.
 	mockedGetAdminConfigurations.Return(nil, nil)
 	// Again, make sure we sync and verify the externalAlertmanagers.
-	require.NoError(t, alertsRouter.SyncAndApplyConfigFromDatabase())
+	require.NoError(t, alertsRouter.SyncAndApplyConfigFromDatabase(context.Background()))
 	require.Equal(t, 0, len(alertsRouter.externalAlertmanagers))
 	require.Equal(t, 0, len(alertsRouter.externalAlertmanagersCfgHash))
 
@@ -134,7 +134,7 @@ func TestIntegrationSendingToExternalAlertmanager_WithMultipleOrgs(t *testing.T)
 	}
 	fakeDs := &fake_ds.FakeDataSourceService{DataSources: []*datasources.DataSource{&ds1}}
 	alertsRouter := NewAlertsRouter(moa, fakeAdminConfigStore, mockedClock, appUrl, map[int64]struct{}{}, 10*time.Minute,
-		fakeDs, fake_secrets.NewFakeSecretsService())
+		fakeDs, fake_secrets.NewFakeSecretsService(), featuremgmt.WithFeatures())
 
 	mockedGetAdminConfigurations.Return([]*models.AdminConfiguration{
 		{OrgID: ruleKey1.OrgID, SendAlertsTo: models.AllAlertmanagers},
@@ -142,7 +142,7 @@ func TestIntegrationSendingToExternalAlertmanager_WithMultipleOrgs(t *testing.T)
 
 	// Make sure we sync the configuration at least once before the evaluation happens to guarantee the sender is running
 	// when the first alert triggers.
-	require.NoError(t, alertsRouter.SyncAndApplyConfigFromDatabase())
+	require.NoError(t, alertsRouter.SyncAndApplyConfigFromDatabase(context.Background()))
 	require.Equal(t, 1, len(alertsRouter.externalAlertmanagers))
 	require.Equal(t, 1, len(alertsRouter.externalAlertmanagersCfgHash))
 
@@ -167,7 +167,7 @@ func TestIntegrationSendingToExternalAlertmanager_WithMultipleOrgs(t *testing.T)
 	}, nil)
 
 	// If we sync again, new externalAlertmanagers must have spawned.
-	require.NoError(t, alertsRouter.SyncAndApplyConfigFromDatabase())
+	require.NoError(t, alertsRouter.SyncAndApplyConfigFromDatabase(context.Background()))
 	require.Equal(t, 2, len(alertsRouter.externalAlertmanagers))
 	require.Equal(t, 2, len(alertsRouter.externalAlertmanagersCfgHash))
 
@@ -216,7 +216,7 @@ func TestIntegrationSendingToExternalAlertmanager_WithMultipleOrgs(t *testing.T)
 	currentHash := alertsRouter.externalAlertmanagersCfgHash[ruleKey2.OrgID]
 
 	// Now, sync again.
-	require.NoError(t, alertsRouter.SyncAndApplyConfigFromDatabase())
+	require.NoError(t, alertsRouter.SyncAndApplyConfigFromDatabase(context.Background()))
 
 	// The hash for org two should not be the same and we should still have two externalAlertmanagers.
 	require.NotEqual(t, alertsRouter.externalAlertmanagersCfgHash[ruleKey2.OrgID], currentHash)
@@ -236,7 +236,7 @@ func TestIntegrationSendingToExternalAlertmanager_WithMultipleOrgs(t *testing.T)
 	currentHash = alertsRouter.externalAlertmanagersCfgHash[ruleKey1.OrgID]
 
 	// Now, sync again.
-	require.NoError(t, alertsRouter.SyncAndApplyConfigFromDatabase())
+	require.NoError(t, alertsRouter.SyncAndApplyConfigFromDatabase(context.Background()))
 
 	// The old configuration should not be running.
 	require.NotEqual(t, alertsRouter.externalAlertmanagersCfgHash[ruleKey1.OrgID], currentHash)
@@ -249,13 +249,13 @@ func TestIntegrationSendingToExternalAlertmanager_WithMultipleOrgs(t *testing.T)
 		{OrgID: ruleKey2.OrgID},
 	}, nil)
 
-	require.NoError(t, alertsRouter.SyncAndApplyConfigFromDatabase())
+	require.NoError(t, alertsRouter.SyncAndApplyConfigFromDatabase(context.Background()))
 	require.NotEqual(t, alertsRouter.externalAlertmanagersCfgHash[ruleKey1.OrgID], currentHash)
 
 	// Finally, remove everything.
 	mockedGetAdminConfigurations.Return([]*models.AdminConfiguration{}, nil)
 
-	require.NoError(t, alertsRouter.SyncAndApplyConfigFromDatabase())
+	require.NoError(t, alertsRouter.SyncAndApplyConfigFromDatabase(context.Background()))
 
 	require.Equal(t, 0, len(alertsRouter.externalAlertmanagers))
 	require.Equal(t, 0, len(alertsRouter.externalAlertmanagersCfgHash))
@@ -293,14 +293,14 @@ func TestChangingAlertmanagersChoice(t *testing.T) {
 		}),
 	}
 	alertsRouter := NewAlertsRouter(moa, fakeAdminConfigStore, mockedClock, appUrl, map[int64]struct{}{},
-		10*time.Minute, &fake_ds.FakeDataSourceService{DataSources: []*datasources.DataSource{&ds}}, fake_secrets.NewFakeSecretsService())
+		10*time.Minute, &fake_ds.FakeDataSourceService{DataSources: []*datasources.DataSource{&ds}}, fake_secrets.NewFakeSecretsService(), featuremgmt.WithFeatures())
 
 	mockedGetAdminConfigurations.Return([]*models.AdminConfiguration{
 		{OrgID: ruleKey.OrgID, SendAlertsTo: models.AllAlertmanagers},
 	}, nil)
 	// Make sure we sync the configuration at least once before the evaluation happens to guarantee the sender is running
 	// when the first alert triggers.
-	require.NoError(t, alertsRouter.SyncAndApplyConfigFromDatabase())
+	require.NoError(t, alertsRouter.SyncAndApplyConfigFromDatabase(context.Background()))
 	require.Equal(t, 1, len(alertsRouter.externalAlertmanagers))
 	require.Equal(t, 1, len(alertsRouter.externalAlertmanagersCfgHash))
 	require.Equal(t, models.AllAlertmanagers, alertsRouter.sendAlertsTo[ruleKey.OrgID])
@@ -325,7 +325,7 @@ func TestChangingAlertmanagersChoice(t *testing.T) {
 		{OrgID: ruleKey.OrgID, SendAlertsTo: models.ExternalAlertmanagers},
 	}, nil)
 	// Again, make sure we sync and verify the externalAlertmanagers.
-	require.NoError(t, alertsRouter.SyncAndApplyConfigFromDatabase())
+	require.NoError(t, alertsRouter.SyncAndApplyConfigFromDatabase(context.Background()))
 	require.Equal(t, 1, len(alertsRouter.externalAlertmanagers))
 	require.Equal(t, 1, len(alertsRouter.externalAlertmanagersCfgHash))
 
@@ -339,7 +339,7 @@ func TestChangingAlertmanagersChoice(t *testing.T) {
 
 	// Again, make sure we sync and verify the externalAlertmanagers.
 	// externalAlertmanagers should be running even though alerts are being handled externally.
-	require.NoError(t, alertsRouter.SyncAndApplyConfigFromDatabase())
+	require.NoError(t, alertsRouter.SyncAndApplyConfigFromDatabase(context.Background()))
 	require.Equal(t, 1, len(alertsRouter.externalAlertmanagers))
 	require.Equal(t, 1, len(alertsRouter.externalAlertmanagersCfgHash))
 
@@ -356,6 +356,86 @@ func TestChangingAlertmanagersChoice(t *testing.T) {
 	require.Len(t, actualAlerts, len(expected))
 }
 
+func TestAlertmanagersChoiceWithDisableExternalFeatureToggle(t *testing.T) {
+	ruleKey := models.GenerateRuleKey(1)
+
+	fakeAM := NewFakeExternalAlertmanager(t)
+	defer fakeAM.Close()
+
+	fakeAdminConfigStore := &store.AdminConfigurationStoreMock{}
+	mockedGetAdminConfigurations := fakeAdminConfigStore.EXPECT().GetAdminConfigurations()
+
+	mockedClock := clock.NewMock()
+	mockedClock.Set(time.Now())
+
+	moa := createMultiOrgAlertmanager(t, []int64{1})
+
+	appUrl := &url.URL{
+		Scheme: "http",
+		Host:   "localhost",
+	}
+
+	ds := datasources.DataSource{
+		URL:   fakeAM.Server.URL,
+		OrgID: ruleKey.OrgID,
+		Type:  datasources.DS_ALERTMANAGER,
+		JsonData: simplejson.NewFromAny(map[string]any{
+			"handleGrafanaManagedAlerts": true,
+			"implementation":             "prometheus",
+		}),
+	}
+
+	var expected []*models2.PostableAlert
+	alerts := definitions.PostableAlerts{}
+	for i := 0; i < rand.Intn(5)+1; i++ {
+		alert := generatePostableAlert(t, mockedClock)
+		expected = append(expected, &alert)
+		alerts.PostableAlerts = append(alerts.PostableAlerts, alert)
+	}
+
+	alertsRouter := NewAlertsRouter(moa, fakeAdminConfigStore, mockedClock, appUrl, map[int64]struct{}{},
+		10*time.Minute, &fake_ds.FakeDataSourceService{DataSources: []*datasources.DataSource{&ds}},
+		fake_secrets.NewFakeSecretsService(), featuremgmt.WithFeatures(featuremgmt.FlagAlertingDisableSendAlertsExternal))
+
+	// Test that we only send to the internal Alertmanager even though the configuration specifies AllAlertmanagers.
+
+	mockedGetAdminConfigurations.Return([]*models.AdminConfiguration{
+		{OrgID: ruleKey.OrgID, SendAlertsTo: models.AllAlertmanagers},
+	}, nil)
+
+	require.NoError(t, alertsRouter.SyncAndApplyConfigFromDatabase(context.Background()))
+	require.Equal(t, 0, len(alertsRouter.externalAlertmanagers))
+	require.Equal(t, 0, len(alertsRouter.externalAlertmanagersCfgHash))
+	require.Equal(t, models.InternalAlertmanager, alertsRouter.sendAlertsTo[ruleKey.OrgID])
+
+	alertsRouter.Send(context.Background(), ruleKey, alerts)
+
+	am, err := moa.AlertmanagerFor(ruleKey.OrgID)
+	require.NoError(t, err)
+	actualAlerts, err := am.GetAlerts(context.Background(), true, true, true, nil, "")
+	require.NoError(t, err)
+	require.Len(t, actualAlerts, len(expected))
+
+	// Test that we still only send to the internal alertmanager even though the configuration specifies ExternalAlertmanagers.
+
+	mockedGetAdminConfigurations.Return([]*models.AdminConfiguration{
+		{OrgID: ruleKey.OrgID, SendAlertsTo: models.ExternalAlertmanagers},
+	}, nil)
+
+	require.NoError(t, alertsRouter.SyncAndApplyConfigFromDatabase(context.Background()))
+	require.Equal(t, 0, len(alertsRouter.externalAlertmanagers))
+	require.Equal(t, 0, len(alertsRouter.externalAlertmanagersCfgHash))
+	require.Equal(t, models.InternalAlertmanager, alertsRouter.sendAlertsTo[ruleKey.OrgID])
+
+	alertsRouter.Send(context.Background(), ruleKey, alerts)
+
+	am, err = moa.AlertmanagerFor(ruleKey.OrgID)
+	require.NoError(t, err)
+	actualAlerts, err = am.GetAlerts(context.Background(), true, true, true, nil, "")
+	require.NoError(t, err)
+	require.Len(t, actualAlerts, len(expected))
+}
+
 func assertAlertmanagersStatusForOrg(t *testing.T, alertsRouter *AlertsRouter, orgID int64, active, dropped int) {
 	t.Helper()
 	require.Eventuallyf(t, func() bool {