feat: support namespace_selector to limit synced namespaces (#487)

Author: AlinsRan

cmd/root/root.go

@@ -35,6 +35,7 @@ import (
 
 	// +kubebuilder:scaffold:imports
 
+	"github.com/apache/apisix-ingress-controller/internal/controller"
 	"github.com/apache/apisix-ingress-controller/internal/controller/config"
 	"github.com/apache/apisix-ingress-controller/internal/manager"
 	"github.com/apache/apisix-ingress-controller/internal/version"
@@ -108,6 +109,9 @@ func newAPISIXIngressController() *cobra.Command {
 			if err := cfg.Validate(); err != nil {
 				return err
 			}
+			if err := controller.SetNamespaceSelector(cfg.NamespaceSelector); err != nil {
+				return err
+			}
 
 			logLevel, err := zapcore.ParseLevel(cfg.LogLevel)
 			if err != nil {

config/samples/config.yaml

@@ -47,6 +47,23 @@ listener_port_match_mode: "off"         # Mode for injecting server_port route v
                                         # "auto"/"explicit" when those two coincide, otherwise routes bound to a
                                         # listener via sectionName/port will never match.
 
+namespace_selector: []                  # Label selectors of the namespaces whose resources are handled by the controller.
+                                        # A namespace is selected when its labels match all entries. Equality and "in"
+                                        # requirements on the same key are merged, so the example below selects namespaces
+                                        # labeled team=a or team=b that are also labeled env=prod:
+                                        #   namespace_selector:
+                                        #   - "team=a"
+                                        #   - "team=b"
+                                        #   - "env=prod"
+                                        # Only separate entries are merged. Within one entry, comma-separated requirements
+                                        # follow the Kubernetes label selector syntax, so "team=a,team=b" matches nothing.
+                                        # It applies to Ingress and apisix.apache.org/v2 resources. Gateway API resources are
+                                        # not filtered, use the allowedRoutes of the Gateway listeners instead. Resources they
+                                        # reference, such as Services, Secrets and GatewayProxies, are read from any namespace.
+                                        # When a namespace stops matching, the configuration of its resources is removed
+                                        # from the data plane.
+                                        # The default value is empty, which selects all namespaces. Empty entries are ignored.
+
 provider:
   type: "api7ee"
 

docs/en/latest/reference/configuration-file.md

@@ -65,6 +65,23 @@ secure_metrics: false                   # The secure metrics configuration.
 exec_adc_timeout: 15s                   # The timeout for the ADC to execute.
                                         # The default value is 15 seconds.
 
+namespace_selector: []                  # Label selectors of the namespaces whose resources are handled by the controller.
+                                        # A namespace is selected when its labels match all entries. Equality and "in"
+                                        # requirements on the same key are merged, so the example below selects namespaces
+                                        # labeled team=a or team=b that are also labeled env=prod:
+                                        #   namespace_selector:
+                                        #   - "team=a"
+                                        #   - "team=b"
+                                        #   - "env=prod"
+                                        # Only separate entries are merged. Within one entry, comma-separated requirements
+                                        # follow the Kubernetes label selector syntax, so "team=a,team=b" matches nothing.
+                                        # It applies to Ingress and apisix.apache.org/v2 resources. Gateway API resources are
+                                        # not filtered, use the allowedRoutes of the Gateway listeners instead. Resources they
+                                        # reference, such as Services, Secrets and GatewayProxies, are read from any namespace.
+                                        # When a namespace stops matching, the configuration of its resources is removed
+                                        # from the data plane.
+                                        # The default value is empty, which selects all namespaces. Empty entries are ignored.
+
 provider:
   type: "api7ee"                        # Provider type.
 

docs/en/latest/upgrade-guide.md

@@ -90,6 +90,15 @@ Because the Admin API fills in default values, the submitted content may differ
 | `apisix.*`           | Static Admin API configuration           |
 | `etcdserver.*`       | Configuration for mock-etcd (deprecated) |
 
+#### Namespace Selector
+
+`kubernetes.namespace_selector` is replaced by the top-level `namespace_selector`. Entries written for 1.x keep their meaning: every entry must match, and the values given for the same key are ORed. Each entry also accepts the full Kubernetes label selector syntax, such as `env in (prod,staging)` or `!legacy`. The command line flag `--namespace-selector` is not available, set the option in the configuration file.
+
+It behaves differently from 1.x in the following ways:
+
+- When a namespace stops matching, 2.x removes the configuration of its resources from the data plane, while 1.x left the synced routes in place. Before upgrading, check for namespaces that were unlabeled in 1.x but still have routes in service, since those routes disappear after the upgrade.
+- Only Ingress and `apisix.apache.org/v2` resources are filtered. Gateway API resources, which 1.x also filtered, are not; use the `allowedRoutes` of the Gateway listeners to limit their namespaces.
+
 #### Example: Legacy Configuration Removed in 2.0.0
 
 ```yaml

internal/controller/apisixconsumer_controller.go

@@ -132,7 +132,7 @@ func (r *ApisixConsumerReconciler) SetupWithManager(mgr ctrl.Manager) error {
 		icWatch = &networkingv1.IngressClass{}
 	}
 
-	return ctrl.NewControllerManagedBy(mgr).
+	bdr := ctrl.NewControllerManagedBy(mgr).
 		For(&apiv2.ApisixConsumer{},
 			builder.WithPredicates(
 				MatchesIngressClassPredicate(r.Client, r.Log, r.ICGV.String()),
@@ -142,6 +142,7 @@ func (r *ApisixConsumerReconciler) SetupWithManager(mgr ctrl.Manager) error {
 				predicate.GenerationChangedPredicate{},
 				predicate.AnnotationChangedPredicate{},
 				predicate.NewPredicateFuncs(TypePredicate[*corev1.Secret]()),
+				predicate.NewPredicateFuncs(TypePredicate[*corev1.Namespace]()),
 			),
 		).
 		Watches(
@@ -156,7 +157,8 @@ func (r *ApisixConsumerReconciler) SetupWithManager(mgr ctrl.Manager) error {
 		).
 		Watches(&corev1.Secret{},
 			handler.EnqueueRequestsFromMapFunc(r.listApisixConsumerForSecret),
-		).
+		)
+	return watchNamespaceSelector(bdr, r.Client, r.Log, func() client.ObjectList { return &apiv2.ApisixConsumerList{} }).
 		Named("apisixconsumer").
 		Complete(r)
 }

internal/controller/apisixglobalrule_controller.go

@@ -95,6 +95,9 @@ func (r *ApisixGlobalRuleReconciler) Reconcile(ctx context.Context, req ctrl.Req
 		r.Log.V(1).Info("no matching IngressClass available",
 			"ingressClassName", globalRule.Spec.IngressClassName,
 			"error", err.Error())
+		if !isIngressClassSelectionAbsent(err) {
+			return ctrl.Result{}, err
+		}
 		if err := r.Provider.Delete(ctx, &globalRule); err != nil {
 			r.Log.Error(err, "failed to delete global rule from provider")
 			return ctrl.Result{}, err
@@ -159,7 +162,7 @@ func (r *ApisixGlobalRuleReconciler) SetupWithManager(mgr ctrl.Manager) error {
 	} else {
 		icWatch = &networkingv1.IngressClass{}
 	}
-	return ctrl.NewControllerManagedBy(mgr).
+	bdr := ctrl.NewControllerManagedBy(mgr).
 		For(&apiv2.ApisixGlobalRule{},
 			builder.WithPredicates(
 				MatchesIngressClassPredicate(r.Client, r.Log, r.ICGV.String()),
@@ -170,6 +173,7 @@ func (r *ApisixGlobalRuleReconciler) SetupWithManager(mgr ctrl.Manager) error {
 				predicate.GenerationChangedPredicate{},
 				predicate.AnnotationChangedPredicate{},
 				predicate.NewPredicateFuncs(TypePredicate[*corev1.Secret]()),
+				predicate.NewPredicateFuncs(TypePredicate[*corev1.Namespace]()),
 			),
 		).
 		Watches(
@@ -184,7 +188,8 @@ func (r *ApisixGlobalRuleReconciler) SetupWithManager(mgr ctrl.Manager) error {
 		).
 		Watches(&corev1.Secret{},
 			handler.EnqueueRequestsFromMapFunc(r.listGlobalRulesForSecret),
-		).
+		)
+	return watchNamespaceSelector(bdr, r.Client, r.Log, func() client.ObjectList { return &apiv2.ApisixGlobalRuleList{} }).
 		Named("apisixglobalrule").
 		Complete(r)
 }

internal/controller/apisixroute_controller.go

@@ -88,6 +88,7 @@ func (r *ApisixRouteReconciler) SetupWithManager(mgr ctrl.Manager) error {
 		predicate.GenerationChangedPredicate{},
 		predicate.AnnotationChangedPredicate{},
 		predicate.NewPredicateFuncs(TypePredicate[*corev1.Secret]()),
+		predicate.NewPredicateFuncs(TypePredicate[*corev1.Namespace]()),
 	}
 
 	if !r.supportsEndpointSlice {
@@ -117,6 +118,7 @@ func (r *ApisixRouteReconciler) SetupWithManager(mgr ctrl.Manager) error {
 		r.listApisixRoutesForService,
 		r.listApisixRoutesForEndpoints,
 		r.Log)
+	bdr = watchNamespaceSelector(bdr, r.Client, r.Log, func() client.ObjectList { return &apiv2.ApisixRouteList{} })
 
 	return bdr.
 		Watches(&corev1.Secret{},
@@ -163,6 +165,9 @@ func (r *ApisixRouteReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 		r.Log.V(1).Info("no matching IngressClass available",
 			"ingressClassName", ar.Spec.IngressClassName,
 			"error", err.Error())
+		if !isIngressClassSelectionAbsent(err) {
+			return ctrl.Result{}, err
+		}
 		if err := r.Provider.Delete(ctx, &ar); err != nil {
 			r.Log.Error(err, "failed to delete apisixroute", "apisixroute", utils.NamespacedName(&ar))
 			return ctrl.Result{}, err

internal/controller/apisixtls_controller.go

@@ -19,6 +19,7 @@ package controller
 
 import (
 	"context"
+	"errors"
 	"fmt"
 
 	"github.com/go-logr/logr"
@@ -67,7 +68,7 @@ func (r *ApisixTlsReconciler) SetupWithManager(mgr ctrl.Manager) error {
 	default:
 		icWatch = &networkingv1.IngressClass{}
 	}
-	return ctrl.NewControllerManagedBy(mgr).
+	bdr := ctrl.NewControllerManagedBy(mgr).
 		For(&apiv2.ApisixTls{},
 			builder.WithPredicates(
 				MatchesIngressClassPredicate(r.Client, r.Log, r.ICGV.String()),
@@ -78,6 +79,7 @@ func (r *ApisixTlsReconciler) SetupWithManager(mgr ctrl.Manager) error {
 				predicate.GenerationChangedPredicate{},
 				predicate.AnnotationChangedPredicate{},
 				predicate.NewPredicateFuncs(TypePredicate[*corev1.Secret]()),
+				predicate.NewPredicateFuncs(TypePredicate[*corev1.Namespace]()),
 			),
 		).
 		Watches(
@@ -92,7 +94,8 @@ func (r *ApisixTlsReconciler) SetupWithManager(mgr ctrl.Manager) error {
 		).
 		Watches(&corev1.Secret{},
 			handler.EnqueueRequestsFromMapFunc(r.listApisixTlsForSecret),
-		).
+		)
+	return watchNamespaceSelector(bdr, r.Client, r.Log, func() client.ObjectList { return &apiv2.ApisixTlsList{} }).
 		Complete(r)
 }
 
@@ -131,6 +134,13 @@ func (r *ApisixTlsReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 		r.Log.V(1).Info("no matching IngressClass available, skip processing",
 			"ingressClassName", tls.Spec.IngressClassName,
 			"error", err.Error())
+		// Retract what was synced before the namespace stopped being watched.
+		if errors.Is(err, ErrNamespaceNotWatched) {
+			if err := r.Provider.Delete(ctx, &tls); err != nil {
+				r.Log.Error(err, "failed to delete TLS from provider")
+				return ctrl.Result{}, err
+			}
+		}
 		return ctrl.Result{}, nil
 	}
 

internal/controller/config/config.go

@@ -27,6 +27,8 @@ import (
 	"time"
 
 	"gopkg.in/yaml.v3"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/selection"
 
 	"github.com/apache/apisix-ingress-controller/internal/types"
 )
@@ -132,6 +134,10 @@ func (c *Config) Validate() error {
 		}
 	}
 
+	if _, err := ParseNamespaceSelector(c.NamespaceSelector); err != nil {
+		return err
+	}
+
 	if err := validateProvider(c.ProviderConfig); err != nil {
 		return err
 	}
@@ -155,6 +161,52 @@ func validateProvider(config ProviderConfig) error {
 	}
 }
 
+// ParseNamespaceSelector combines the namespace_selector entries into one
+// selector, keeping the semantics of 1.x: every entry must match, and entries
+// holding a single equality or "in" requirement on the same key are merged, so
+// ["team=a", "team=b"] selects "team in (a,b)". An entry with several
+// requirements keeps the standard label selector semantics, so "team=a,team=b"
+// matches nothing. Empty entries are ignored, as 1.x used [""] to disable the
+// selector. It returns nil when no entry is left.
+func ParseNamespaceSelector(entries []string) (labels.Selector, error) {
+	var (
+		selector labels.Selector
+		keys     []string
+		values   = map[string][]string{}
+	)
+	for _, entry := range entries {
+		if strings.TrimSpace(entry) == "" {
+			continue
+		}
+		reqs, err := labels.ParseToRequirements(entry)
+		if err != nil {
+			return nil, fmt.Errorf("invalid namespace_selector %q: %w", entry, err)
+		}
+		if selector == nil {
+			selector = labels.NewSelector()
+		}
+		if len(reqs) == 1 {
+			switch req := reqs[0]; req.Operator() {
+			case selection.Equals, selection.DoubleEquals, selection.In:
+				if _, ok := values[req.Key()]; !ok {
+					keys = append(keys, req.Key())
+				}
+				values[req.Key()] = append(values[req.Key()], req.ValuesUnsorted()...)
+				continue
+			}
+		}
+		selector = selector.Add(reqs...)
+	}
+	for _, key := range keys {
+		req, err := labels.NewRequirement(key, selection.In, values[key])
+		if err != nil {
+			return nil, fmt.Errorf("invalid namespace_selector on key %q: %w", key, err)
+		}
+		selector = selector.Add(*req)
+	}
+	return selector, nil
+}
+
 func GetControllerName() string {
 	return ControllerConfig.ControllerName
 }

internal/controller/config/config_test.go

@@ -5,6 +5,9 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/utils/ptr"
 )
 
 func TestNewDefaultConfig(t *testing.T) {
@@ -65,12 +68,85 @@ func TestConfigValidateListenerPortMatchMode(t *testing.T) {
 	}
 }
 
+func TestConfigValidateNamespaceSelector(t *testing.T) {
+	tests := []struct {
+		name      string
+		selector  []string
+		expectErr bool
+	}{
+		{name: "unset", selector: nil},
+		{name: "1.x default", selector: []string{""}},
+		{name: "equality", selector: []string{"team=a"}},
+		{name: "set based", selector: []string{"env in (prod,staging),!legacy", "team=a"}},
+		{name: "invalid", selector: []string{"team in a"}, expectErr: true},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			cfg := NewDefaultConfig()
+			cfg.NamespaceSelector = tt.selector
+
+			err := cfg.Validate()
+			if tt.expectErr {
+				assert.ErrorContains(t, err, "invalid namespace_selector")
+			} else {
+				assert.NoError(t, err)
+			}
+		})
+	}
+}
+
+func TestParseNamespaceSelector(t *testing.T) {
+	nsLabels := labels.Set{"version": "v1", "env": "prod"}
+
+	tests := []struct {
+		name    string
+		entries []string
+		// nil means the selector is disabled.
+		matches *bool
+	}{
+		// Cases ported from TestMultiValueLabelsIsSubsetOf of 1.x.
+		{name: "no entry", entries: nil},
+		{name: "1.x default", entries: []string{""}},
+		{name: "single value", entries: []string{"env=prod"}, matches: ptr.To(true)},
+		{name: "values on one key are ORed", entries: []string{"env=qa", "env=prod"}, matches: ptr.To(true)},
+		{name: "value mismatch", entries: []string{"env=qa"}, matches: ptr.To(false)},
+		{name: "missing key", entries: []string{"env3=not"}, matches: ptr.To(false)},
+		// Entries on different keys are ANDed.
+		{name: "all keys match", entries: []string{"env=prod", "version=v1"}, matches: ptr.To(true)},
+		{name: "one key mismatches", entries: []string{"env=prod", "version=v2"}, matches: ptr.To(false)},
+		{name: "empty entry is ignored", entries: []string{"env=qa", ""}, matches: ptr.To(false)},
+		// Full selector syntax on top of 1.x.
+		{name: "in merges with equality", entries: []string{"env in (qa)", "env==prod"}, matches: ptr.To(true)},
+		{name: "not equal", entries: []string{"env=prod", "version!=v1"}, matches: ptr.To(false)},
+		{name: "does not exist", entries: []string{"!legacy"}, matches: ptr.To(true)},
+		// Only separate entries are merged, one entry keeps the standard semantics.
+		{name: "one entry is not merged", entries: []string{"env=qa,env=prod"}, matches: ptr.To(false)},
+		{name: "one entry with several keys", entries: []string{"env=prod,version=v1"}, matches: ptr.To(true)},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			selector, err := ParseNamespaceSelector(tt.entries)
+			require.NoError(t, err)
+			if tt.matches == nil {
+				assert.Nil(t, selector)
+				return
+			}
+			require.NotNil(t, selector)
+			assert.Equal(t, *tt.matches, selector.Matches(nsLabels), selector.String())
+		})
+	}
+}
+
 func TestNewConfigFromFile(t *testing.T) {
 	// Create a temporary config file
 	fileContent := `
 log_level: debug
 controller_name: test-controller
 disable_gateway_api: true
+namespace_selector:
+- "team=a"
 `
 	tempFile, err := os.CreateTemp("", "config-*.yaml")
 	assert.NoError(t, err)
@@ -87,4 +163,5 @@ disable_gateway_api: true
 	assert.Equal(t, "debug", cfg.LogLevel)
 	assert.Equal(t, "test-controller", cfg.ControllerName)
 	assert.Equal(t, true, cfg.DisableGatewayAPI)
+	assert.Equal(t, []string{"team=a"}, cfg.NamespaceSelector)
 }