fix: newServer uses the stored config hash for mismatch

pkg/alertmanager/service.go

@@ -72,7 +72,7 @@ func (service *Service) SyncServers(ctx context.Context) error {
 
 	service.serversMtx.Lock()
 	for _, org := range orgs {
-		config, err := service.getConfig(ctx, org.ID.StringValue())
+		config, _, err := service.getConfig(ctx, org.ID.StringValue())
 		if err != nil {
 			service.settings.Logger().ErrorContext(ctx, "failed to get alertmanager config for org", "org_id", org.ID.StringValue(), "error", err)
 			continue
@@ -171,7 +171,7 @@ func (service *Service) Stop(ctx context.Context) error {
 }
 
 func (service *Service) newServer(ctx context.Context, orgID string) (*alertmanagerserver.Server, error) {
-	config, err := service.getConfig(ctx, orgID)
+	config, storedHash, err := service.getConfig(ctx, orgID)
 	if err != nil {
 		return nil, err
 	}
@@ -181,13 +181,16 @@ func (service *Service) newServer(ctx context.Context, orgID string) (*alertmana
 		return nil, err
 	}
 
-	beforeCompareAndSelectHash := config.StoreableConfig().Hash
 	config, err = service.compareAndSelectConfig(ctx, config)
 	if err != nil {
 		return nil, err
 	}
 
-	if beforeCompareAndSelectHash == config.StoreableConfig().Hash {
+	// compare against the hash of the config stored in the DB (before overlays
+	// were applied by getConfig). This ensures that overlay changes (e.g. new
+	// defaults from an upstream upgrade or something similar) trigger a DB update
+	// so that other code paths reading directly from the store see the up-to-date config.
+	if storedHash == config.StoreableConfig().Hash {
 		service.settings.Logger().DebugContext(ctx, "skipping config store update for org", "org_id", orgID, "hash", config.StoreableConfig().Hash)
 		return server, nil
 	}
@@ -200,27 +203,33 @@ func (service *Service) newServer(ctx context.Context, orgID string) (*alertmana
 	return server, nil
 }
 
-func (service *Service) getConfig(ctx context.Context, orgID string) (*alertmanagertypes.Config, error) {
+// getConfig returns the config for the given orgID with overlays applied, along
+// with the hash that was stored in the DB before overlays. When no config exists
+// in the store yet the stored hash is empty.
+func (service *Service) getConfig(ctx context.Context, orgID string) (*alertmanagertypes.Config, string, error) {
 	config, err := service.configStore.Get(ctx, orgID)
+	var storedHash string
 	if err != nil {
 		if !errors.Ast(err, errors.TypeNotFound) {
-			return nil, err
+			return nil, "", err
 		}
 
 		config, err = alertmanagertypes.NewDefaultConfig(service.config.Global, service.config.Route, orgID)
 		if err != nil {
-			return nil, err
+			return nil, "", err
 		}
+	} else {
+		storedHash = config.StoreableConfig().Hash
 	}
 
 	if err := config.SetGlobalConfig(service.config.Global); err != nil {
-		return nil, err
+		return nil, "", err
 	}
 	if err := config.SetRouteConfig(service.config.Route); err != nil {
-		return nil, err
+		return nil, "", err
 	}
 
-	return config, nil
+	return config, storedHash, nil
 }
 
 func (service *Service) compareAndSelectConfig(ctx context.Context, incomingConfig *alertmanagertypes.Config) (*alertmanagertypes.Config, error) {