feat(multigrescluster): track sticky InitializedAt on first Healthy phase

api/v1alpha1/multigrescluster_types.go

@@ -770,6 +770,13 @@ type MultigresClusterStatus struct {
 	// +optional
 	Phase Phase `json:"phase,omitempty"`
 
+	// InitializedAt records when the cluster first reached PhaseHealthy. It is
+	// set once and never cleared, even if the cluster later becomes Degraded,
+	// so it can distinguish first-time bootstrap from an established cluster
+	// having a real incident.
+	// +optional
+	InitializedAt *metav1.Time `json:"initializedAt,omitempty"`
+
 	// Message provides details about the current phase (e.g. error messages).
 	// +optional
 	Message string `json:"message,omitempty"`

config/crd/bases/multigres.com_multigresclusters.yaml

@@ -12520,6 +12520,14 @@ spec:
                 - effective
                 - source
                 type: object
+              initializedAt:
+                description: |-
+                  InitializedAt records when the cluster first reached PhaseHealthy. It is
+                  set once and never cleared, even if the cluster later becomes Degraded,
+                  so it can distinguish first-time bootstrap from an established cluster
+                  having a real incident.
+                format: date-time
+                type: string
               message:
                 description: Message provides details about the current phase (e.g.
                   error messages).

pkg/cluster-handler/controller/multigrescluster/multigrescluster_controller.go

@@ -342,7 +342,12 @@ func (r *MultigresClusterReconciler) Reconcile(
 	}
 
 	// Emit cluster-level metrics
-	monitoring.SetClusterInfo(cluster.Name, cluster.Namespace, string(cluster.Status.Phase))
+	monitoring.SetClusterInfo(
+		cluster.Name,
+		cluster.Namespace,
+		string(cluster.Status.Phase),
+		cluster.Status.InitializedAt != nil,
+	)
 	var totalShards int
 	for _, db := range cluster.Status.Databases {
 		totalShards += int(db.TotalShards)

pkg/cluster-handler/controller/multigrescluster/status.go

@@ -13,8 +13,21 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	multigresv1alpha1 "github.com/multigres/multigres-operator/api/v1alpha1"
+	"github.com/multigres/multigres-operator/pkg/resolver"
 )
 
+type tableGroupStatusIdentity struct {
+	database   multigresv1alpha1.DatabaseName
+	tableGroup multigresv1alpha1.TableGroupName
+}
+
+func (r *MultigresClusterReconciler) childStatusReader() client.Reader {
+	if r.APIReader != nil {
+		return r.APIReader
+	}
+	return r.Client
+}
+
 // extractExternalEndpoint resolves the externally reachable endpoint for the
 // global multigateway Service. It prefers explicitly assigned Service
 // ExternalIPs, then falls back to load balancer ingress (hostname over IP).
@@ -114,6 +127,13 @@ func (r *MultigresClusterReconciler) updateStatus(
 	ctx context.Context,
 	cluster *multigresv1alpha1.MultigresCluster,
 ) error {
+	globalTopoSpec, err := resolver.NewResolver(r.Client, cluster.Namespace).
+		ResolveGlobalTopo(ctx, cluster)
+	if err != nil {
+		return fmt.Errorf("failed to resolve global topo for status: %w", err)
+	}
+	childReader := r.childStatusReader()
+
 	oldPhase := cluster.Status.Phase
 	cluster.Status.ObservedGeneration = cluster.Generation
 	cluster.Status.Cells = make(map[multigresv1alpha1.CellName]multigresv1alpha1.CellStatusSummary)
@@ -122,7 +142,7 @@ func (r *MultigresClusterReconciler) updateStatus(
 	)
 
 	cells := &multigresv1alpha1.CellList{}
-	if err := r.List(
+	if err := childReader.List(
 		ctx,
 		cells,
 		client.InNamespace(cluster.Namespace),
@@ -147,7 +167,7 @@ func (r *MultigresClusterReconciler) updateStatus(
 	}
 
 	tgs := &multigresv1alpha1.TableGroupList{}
-	if err := r.List(
+	if err := childReader.List(
 		ctx,
 		tgs,
 		client.InNamespace(cluster.Namespace),
@@ -187,8 +207,23 @@ func (r *MultigresClusterReconciler) updateStatus(
 		anyDegraded bool
 		allHealthy  = true
 	)
+	expectedCells := make(map[multigresv1alpha1.CellName]struct{}, len(cluster.Spec.Cells))
+	for _, cell := range cluster.Spec.Cells {
+		expectedCells[cell.Name] = struct{}{}
+	}
+	expectedTableGroups := make(map[tableGroupStatusIdentity]struct{})
+	for _, database := range cluster.Spec.Databases {
+		for _, tableGroup := range database.TableGroups {
+			expectedTableGroups[tableGroupStatusIdentity{
+				database:   database.Name,
+				tableGroup: tableGroup.Name,
+			}] = struct{}{}
+		}
+	}
+	expectsGlobalTopoServer := globalTopoSpec.Etcd != nil
 
 	for _, c := range cells.Items {
+		delete(expectedCells, c.Spec.Name)
 		switch {
 		case c.Status.ObservedGeneration != c.Generation:
 			allHealthy = false
@@ -203,6 +238,10 @@ func (r *MultigresClusterReconciler) updateStatus(
 	}
 
 	for _, tg := range tgs.Items {
+		delete(expectedTableGroups, tableGroupStatusIdentity{
+			database:   tg.Spec.DatabaseName,
+			tableGroup: tg.Spec.TableGroupName,
+		})
 		switch {
 		case tg.Status.ObservedGeneration != tg.Generation:
 			allHealthy = false
@@ -217,7 +256,7 @@ func (r *MultigresClusterReconciler) updateStatus(
 	}
 
 	topoServers := &multigresv1alpha1.TopoServerList{}
-	if err := r.List(
+	if err := childReader.List(
 		ctx,
 		topoServers,
 		client.InNamespace(cluster.Namespace),
@@ -227,6 +266,9 @@ func (r *MultigresClusterReconciler) updateStatus(
 	}
 
 	for _, ts := range topoServers.Items {
+		if expectsGlobalTopoServer && ts.Name == cluster.Name+"-global-topo" {
+			expectsGlobalTopoServer = false
+		}
 		switch {
 		case ts.Status.ObservedGeneration != ts.Generation:
 			allHealthy = false
@@ -240,6 +282,10 @@ func (r *MultigresClusterReconciler) updateStatus(
 		}
 	}
 
+	if len(expectedCells) > 0 || len(expectedTableGroups) > 0 || expectsGlobalTopoServer {
+		allHealthy = false
+	}
+
 	switch {
 	case anyDegraded:
 		cluster.Status.Phase = multigresv1alpha1.PhaseDegraded
@@ -252,6 +298,12 @@ func (r *MultigresClusterReconciler) updateStatus(
 		cluster.Status.Message = "Cluster is progressing"
 	}
 
+	if cluster.Status.Phase == multigresv1alpha1.PhaseHealthy &&
+		cluster.Status.InitializedAt == nil {
+		now := metav1.Now()
+		cluster.Status.InitializedAt = &now
+	}
+
 	allCellsReady := true
 	for _, c := range cluster.Status.Cells {
 		if !c.Ready {