diff --git a/api/v1alpha1/multigrescluster_types.go b/api/v1alpha1/multigrescluster_types.go index b2497375..4cf725f6 100644 --- a/api/v1alpha1/multigrescluster_types.go +++ b/api/v1alpha1/multigrescluster_types.go @@ -770,6 +770,13 @@ type MultigresClusterStatus struct { // +optional Phase Phase `json:"phase,omitempty"` + // InitializedAt records when the cluster first reached PhaseHealthy. It is + // set once and never cleared, even if the cluster later becomes Degraded, + // so it can distinguish first-time bootstrap from an established cluster + // having a real incident. + // +optional + InitializedAt *metav1.Time `json:"initializedAt,omitempty"` + // Message provides details about the current phase (e.g. error messages). // +optional Message string `json:"message,omitempty"` diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 80a0ff7e..3b348198 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -1154,6 +1154,10 @@ func (in *MultigresClusterStatus) DeepCopyInto(out *MultigresClusterStatus) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.InitializedAt != nil { + in, out := &in.InitializedAt, &out.InitializedAt + *out = (*in).DeepCopy() + } if in.Cells != nil { in, out := &in.Cells, &out.Cells *out = make(map[CellName]CellStatusSummary, len(*in)) diff --git a/config/crd/bases/multigres.com_multigresclusters.yaml b/config/crd/bases/multigres.com_multigresclusters.yaml index c0793953..1120e468 100644 --- a/config/crd/bases/multigres.com_multigresclusters.yaml +++ b/config/crd/bases/multigres.com_multigresclusters.yaml @@ -12520,6 +12520,14 @@ spec: - effective - source type: object + initializedAt: + description: |- + InitializedAt records when the cluster first reached PhaseHealthy. It is + set once and never cleared, even if the cluster later becomes Degraded, + so it can distinguish first-time bootstrap from an established cluster + having a real incident. + format: date-time + type: string message: description: Message provides details about the current phase (e.g. error messages). diff --git a/pkg/cluster-handler/controller/multigrescluster/multigrescluster_controller.go b/pkg/cluster-handler/controller/multigrescluster/multigrescluster_controller.go index 6d0c890b..35070f95 100644 --- a/pkg/cluster-handler/controller/multigrescluster/multigrescluster_controller.go +++ b/pkg/cluster-handler/controller/multigrescluster/multigrescluster_controller.go @@ -342,7 +342,12 @@ func (r *MultigresClusterReconciler) Reconcile( } // Emit cluster-level metrics - monitoring.SetClusterInfo(cluster.Name, cluster.Namespace, string(cluster.Status.Phase)) + monitoring.SetClusterInfo( + cluster.Name, + cluster.Namespace, + string(cluster.Status.Phase), + cluster.Status.InitializedAt != nil, + ) var totalShards int for _, db := range cluster.Status.Databases { totalShards += int(db.TotalShards) diff --git a/pkg/cluster-handler/controller/multigrescluster/status.go b/pkg/cluster-handler/controller/multigrescluster/status.go index 4e3def0e..a20c0b83 100644 --- a/pkg/cluster-handler/controller/multigrescluster/status.go +++ b/pkg/cluster-handler/controller/multigrescluster/status.go @@ -13,8 +13,21 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" multigresv1alpha1 "github.com/multigres/multigres-operator/api/v1alpha1" + "github.com/multigres/multigres-operator/pkg/resolver" ) +type tableGroupStatusIdentity struct { + database multigresv1alpha1.DatabaseName + tableGroup multigresv1alpha1.TableGroupName +} + +func (r *MultigresClusterReconciler) childStatusReader() client.Reader { + if r.APIReader != nil { + return r.APIReader + } + return r.Client +} + // extractExternalEndpoint resolves the externally reachable endpoint for the // global multigateway Service. It prefers explicitly assigned Service // ExternalIPs, then falls back to load balancer ingress (hostname over IP). @@ -114,6 +127,13 @@ func (r *MultigresClusterReconciler) updateStatus( ctx context.Context, cluster *multigresv1alpha1.MultigresCluster, ) error { + globalTopoSpec, err := resolver.NewResolver(r.Client, cluster.Namespace). + ResolveGlobalTopo(ctx, cluster) + if err != nil { + return fmt.Errorf("failed to resolve global topo for status: %w", err) + } + childReader := r.childStatusReader() + oldPhase := cluster.Status.Phase cluster.Status.ObservedGeneration = cluster.Generation cluster.Status.Cells = make(map[multigresv1alpha1.CellName]multigresv1alpha1.CellStatusSummary) @@ -122,7 +142,7 @@ func (r *MultigresClusterReconciler) updateStatus( ) cells := &multigresv1alpha1.CellList{} - if err := r.List( + if err := childReader.List( ctx, cells, client.InNamespace(cluster.Namespace), @@ -147,7 +167,7 @@ func (r *MultigresClusterReconciler) updateStatus( } tgs := &multigresv1alpha1.TableGroupList{} - if err := r.List( + if err := childReader.List( ctx, tgs, client.InNamespace(cluster.Namespace), @@ -187,8 +207,23 @@ func (r *MultigresClusterReconciler) updateStatus( anyDegraded bool allHealthy = true ) + expectedCells := make(map[multigresv1alpha1.CellName]struct{}, len(cluster.Spec.Cells)) + for _, cell := range cluster.Spec.Cells { + expectedCells[cell.Name] = struct{}{} + } + expectedTableGroups := make(map[tableGroupStatusIdentity]struct{}) + for _, database := range cluster.Spec.Databases { + for _, tableGroup := range database.TableGroups { + expectedTableGroups[tableGroupStatusIdentity{ + database: database.Name, + tableGroup: tableGroup.Name, + }] = struct{}{} + } + } + expectsGlobalTopoServer := globalTopoSpec.Etcd != nil for _, c := range cells.Items { + delete(expectedCells, c.Spec.Name) switch { case c.Status.ObservedGeneration != c.Generation: allHealthy = false @@ -203,6 +238,10 @@ func (r *MultigresClusterReconciler) updateStatus( } for _, tg := range tgs.Items { + delete(expectedTableGroups, tableGroupStatusIdentity{ + database: tg.Spec.DatabaseName, + tableGroup: tg.Spec.TableGroupName, + }) switch { case tg.Status.ObservedGeneration != tg.Generation: allHealthy = false @@ -217,7 +256,7 @@ func (r *MultigresClusterReconciler) updateStatus( } topoServers := &multigresv1alpha1.TopoServerList{} - if err := r.List( + if err := childReader.List( ctx, topoServers, client.InNamespace(cluster.Namespace), @@ -227,6 +266,9 @@ func (r *MultigresClusterReconciler) updateStatus( } for _, ts := range topoServers.Items { + if expectsGlobalTopoServer && ts.Name == cluster.Name+"-global-topo" { + expectsGlobalTopoServer = false + } switch { case ts.Status.ObservedGeneration != ts.Generation: allHealthy = false @@ -240,6 +282,10 @@ func (r *MultigresClusterReconciler) updateStatus( } } + if len(expectedCells) > 0 || len(expectedTableGroups) > 0 || expectsGlobalTopoServer { + allHealthy = false + } + switch { case anyDegraded: cluster.Status.Phase = multigresv1alpha1.PhaseDegraded @@ -252,6 +298,12 @@ func (r *MultigresClusterReconciler) updateStatus( cluster.Status.Message = "Cluster is progressing" } + if cluster.Status.Phase == multigresv1alpha1.PhaseHealthy && + cluster.Status.InitializedAt == nil { + now := metav1.Now() + cluster.Status.InitializedAt = &now + } + allCellsReady := true for _, c := range cluster.Status.Cells { if !c.Ready { diff --git a/pkg/cluster-handler/controller/multigrescluster/status_test.go b/pkg/cluster-handler/controller/multigrescluster/status_test.go index 2d9d3dee..997e15f3 100644 --- a/pkg/cluster-handler/controller/multigrescluster/status_test.go +++ b/pkg/cluster-handler/controller/multigrescluster/status_test.go @@ -348,6 +348,22 @@ func TestUpdateStatus_ZeroResources(t *testing.T) { Name: "test-cluster", Namespace: "default", }, + Spec: multigresv1alpha1.MultigresClusterSpec{ + GlobalTopoServer: &multigresv1alpha1.GlobalTopoServerSpec{ + External: &multigresv1alpha1.ExternalTopoServerSpec{ + Endpoints: []multigresv1alpha1.EndpointUrl{"http://topo:2379"}, + }, + }, + Cells: []multigresv1alpha1.CellConfig{{Name: "cell-1", Region: "us-east-1"}}, + Databases: []multigresv1alpha1.DatabaseConfig{{ + Name: "postgres", + Default: true, + TableGroups: []multigresv1alpha1.TableGroupConfig{{ + Name: "default", + Default: true, + }}, + }}, + }, } fakeClient := fake.NewClientBuilder(). @@ -374,9 +390,8 @@ func TestUpdateStatus_ZeroResources(t *testing.T) { t.Fatalf("Failed to refresh cluster: %v", err) } - if cluster.Status.Phase != multigresv1alpha1.PhaseHealthy { - t.Errorf("Expected PhaseHealthy (vacuously true), got %s", cluster.Status.Phase) - } + assert.Equal(t, multigresv1alpha1.PhaseProgressing, cluster.Status.Phase) + assert.Nil(t, cluster.Status.InitializedAt) cond := meta.FindStatusCondition(cluster.Status.Conditions, "Available") if cond == nil { @@ -387,6 +402,254 @@ func TestUpdateStatus_ZeroResources(t *testing.T) { } } +func TestUpdateStatus_ExpectedChildren(t *testing.T) { + scheme := runtime.NewScheme() + require.NoError(t, multigresv1alpha1.AddToScheme(scheme)) + + externalTopo := &multigresv1alpha1.GlobalTopoServerSpec{ + External: &multigresv1alpha1.ExternalTopoServerSpec{ + Endpoints: []multigresv1alpha1.EndpointUrl{"http://topo:2379"}, + }, + } + clusterWithExpectedChildren := func(globalTopo *multigresv1alpha1.GlobalTopoServerSpec) *multigresv1alpha1.MultigresCluster { + return &multigresv1alpha1.MultigresCluster{ + ObjectMeta: metav1.ObjectMeta{Name: "test-cluster", Namespace: "default"}, + Spec: multigresv1alpha1.MultigresClusterSpec{ + GlobalTopoServer: globalTopo, + Cells: []multigresv1alpha1.CellConfig{ + {Name: "cell-1", Region: "us-east-1"}, + }, + Databases: []multigresv1alpha1.DatabaseConfig{{ + Name: "postgres", + Default: true, + TableGroups: []multigresv1alpha1.TableGroupConfig{{ + Name: "default", + Default: true, + }}, + }}, + }, + } + } + healthyCell := func(name multigresv1alpha1.CellName) *multigresv1alpha1.Cell { + return &multigresv1alpha1.Cell{ + ObjectMeta: metav1.ObjectMeta{ + Name: string(name), + Namespace: "default", + Labels: map[string]string{"multigres.com/cluster": "test-cluster"}, + }, + Spec: multigresv1alpha1.CellSpec{Name: name}, + Status: multigresv1alpha1.CellStatus{ + Phase: multigresv1alpha1.PhaseHealthy, + }, + } + } + healthyTableGroup := func( + database multigresv1alpha1.DatabaseName, + tableGroup multigresv1alpha1.TableGroupName, + ) *multigresv1alpha1.TableGroup { + return &multigresv1alpha1.TableGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: string(tableGroup), + Namespace: "default", + Labels: map[string]string{"multigres.com/cluster": "test-cluster"}, + }, + Spec: multigresv1alpha1.TableGroupSpec{ + DatabaseName: database, + TableGroupName: tableGroup, + }, + Status: multigresv1alpha1.TableGroupStatus{ + Phase: multigresv1alpha1.PhaseHealthy, + }, + } + } + healthyGlobalTopo := func(name string) *multigresv1alpha1.TopoServer { + return &multigresv1alpha1.TopoServer{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: "default", + Labels: map[string]string{"multigres.com/cluster": "test-cluster"}, + }, + Status: multigresv1alpha1.TopoServerStatus{ + Phase: multigresv1alpha1.PhaseHealthy, + }, + } + } + + tests := map[string]struct { + cluster *multigresv1alpha1.MultigresCluster + children []client.Object + wantPhase multigresv1alpha1.Phase + initialized bool + }{ + "missing expected cell is progressing": { + cluster: clusterWithExpectedChildren(externalTopo), + children: []client.Object{ + healthyCell("other-cell"), + healthyTableGroup("postgres", "default"), + }, + wantPhase: multigresv1alpha1.PhaseProgressing, + }, + "missing expected table group is progressing": { + cluster: clusterWithExpectedChildren(externalTopo), + children: []client.Object{ + healthyCell("cell-1"), + healthyTableGroup("postgres", "other-tg"), + }, + wantPhase: multigresv1alpha1.PhaseProgressing, + }, + "missing managed global topo is progressing": { + cluster: clusterWithExpectedChildren(nil), + children: []client.Object{ + healthyCell("cell-1"), + healthyTableGroup("postgres", "default"), + healthyGlobalTopo("other-topo"), + }, + wantPhase: multigresv1alpha1.PhaseProgressing, + }, + "external global topo does not require a topo server": { + cluster: clusterWithExpectedChildren(externalTopo), + children: []client.Object{ + healthyCell("cell-1"), + healthyTableGroup("postgres", "default"), + }, + wantPhase: multigresv1alpha1.PhaseHealthy, + initialized: true, + }, + "all managed children healthy initializes cluster": { + cluster: clusterWithExpectedChildren(nil), + children: []client.Object{ + healthyCell("cell-1"), + healthyTableGroup("postgres", "default"), + healthyGlobalTopo("test-cluster-global-topo"), + }, + wantPhase: multigresv1alpha1.PhaseHealthy, + initialized: true, + }, + } + + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + objects := append([]client.Object{tt.cluster}, tt.children...) + fakeClient := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(objects...). + WithStatusSubresource(objects...). + Build() + r := &MultigresClusterReconciler{ + Client: fakeClient, + Scheme: scheme, + Recorder: record.NewFakeRecorder(10), + } + + require.NoError(t, r.updateStatus(t.Context(), tt.cluster)) + require.NoError( + t, + fakeClient.Get(t.Context(), client.ObjectKeyFromObject(tt.cluster), tt.cluster), + ) + assert.Equal(t, tt.wantPhase, tt.cluster.Status.Phase) + assert.Equal(t, tt.initialized, tt.cluster.Status.InitializedAt != nil) + }) + } +} + +func TestUpdateStatus_InitializedAtSticky(t *testing.T) { + scheme := runtime.NewScheme() + _ = multigresv1alpha1.AddToScheme(scheme) + + cluster := &multigresv1alpha1.MultigresCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + Namespace: "default", + }, + Spec: multigresv1alpha1.MultigresClusterSpec{ + GlobalTopoServer: &multigresv1alpha1.GlobalTopoServerSpec{ + External: &multigresv1alpha1.ExternalTopoServerSpec{ + Endpoints: []multigresv1alpha1.EndpointUrl{"http://topo:2379"}, + }, + }, + Cells: []multigresv1alpha1.CellConfig{{Name: "cell-1", Region: "us-east-1"}}, + Databases: []multigresv1alpha1.DatabaseConfig{{ + Name: "postgres", + Default: true, + TableGroups: []multigresv1alpha1.TableGroupConfig{{ + Name: "default", + Default: true, + }}, + }}, + }, + } + cell := &multigresv1alpha1.Cell{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cell-1", + Namespace: "default", + Labels: map[string]string{"multigres.com/cluster": "test-cluster"}, + }, + Spec: multigresv1alpha1.CellSpec{Name: "cell-1"}, + Status: multigresv1alpha1.CellStatus{ + Phase: multigresv1alpha1.PhaseHealthy, + }, + } + tableGroup := &multigresv1alpha1.TableGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster-postgres-default", + Namespace: "default", + Labels: map[string]string{"multigres.com/cluster": "test-cluster"}, + }, + Spec: multigresv1alpha1.TableGroupSpec{ + DatabaseName: "postgres", + TableGroupName: "default", + }, + Status: multigresv1alpha1.TableGroupStatus{ + Phase: multigresv1alpha1.PhaseHealthy, + }, + } + + fakeClient := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(cluster, cell, tableGroup). + WithStatusSubresource(cluster, cell, tableGroup). + Build() + + r := &MultigresClusterReconciler{ + Client: fakeClient, + Scheme: scheme, + Recorder: record.NewFakeRecorder(10), + } + + require.NoError(t, r.updateStatus(t.Context(), cluster)) + require.NoError(t, fakeClient.Get(t.Context(), client.ObjectKeyFromObject(cluster), cluster)) + + require.Equal(t, multigresv1alpha1.PhaseHealthy, cluster.Status.Phase) + require.NotNil(t, cluster.Status.InitializedAt) + initializedAt := *cluster.Status.InitializedAt + + degradedCell := &multigresv1alpha1.Cell{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cell-1", + Namespace: "default", + Labels: map[string]string{"multigres.com/cluster": "test-cluster"}, + }, + Spec: multigresv1alpha1.CellSpec{Name: "cell-1"}, + Status: multigresv1alpha1.CellStatus{ + Phase: multigresv1alpha1.PhaseDegraded, + }, + } + + fakeClient = fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(cluster, degradedCell, tableGroup). + WithStatusSubresource(cluster, degradedCell, tableGroup). + Build() + r.Client = fakeClient + + require.NoError(t, r.updateStatus(t.Context(), cluster)) + require.NoError(t, fakeClient.Get(t.Context(), client.ObjectKeyFromObject(cluster), cluster)) + + assert.Equal(t, multigresv1alpha1.PhaseDegraded, cluster.Status.Phase) + require.NotNil(t, cluster.Status.InitializedAt) + assert.Equal(t, initializedAt, *cluster.Status.InitializedAt) +} + func TestUpdateStatus_GenerationMismatch(t *testing.T) { scheme := runtime.NewScheme() _ = multigresv1alpha1.AddToScheme(scheme) @@ -397,6 +660,20 @@ func TestUpdateStatus_GenerationMismatch(t *testing.T) { Namespace: "default", Generation: 2, }, + Spec: multigresv1alpha1.MultigresClusterSpec{ + GlobalTopoServer: &multigresv1alpha1.GlobalTopoServerSpec{ + External: &multigresv1alpha1.ExternalTopoServerSpec{ + Endpoints: []multigresv1alpha1.EndpointUrl{"http://topo:2379"}, + }, + }, + Cells: []multigresv1alpha1.CellConfig{{Name: "cell-1", Region: "us-east-1"}}, + Databases: []multigresv1alpha1.DatabaseConfig{{ + Name: "db1", + TableGroups: []multigresv1alpha1.TableGroupConfig{{ + Name: "tg-1", + }}, + }}, + }, } cell := &multigresv1alpha1.Cell{ @@ -420,7 +697,10 @@ func TestUpdateStatus_GenerationMismatch(t *testing.T) { Labels: map[string]string{"multigres.com/cluster": "test-cluster"}, Generation: 2, }, - Spec: multigresv1alpha1.TableGroupSpec{DatabaseName: "db1"}, + Spec: multigresv1alpha1.TableGroupSpec{ + DatabaseName: "db1", + TableGroupName: "tg-1", + }, Status: multigresv1alpha1.TableGroupStatus{ ObservedGeneration: 1, Phase: multigresv1alpha1.PhaseHealthy, @@ -459,6 +739,84 @@ func TestUpdateStatus_GenerationMismatch(t *testing.T) { } } +func TestUpdateStatus_UsesAPIReaderForChildHealth(t *testing.T) { + scheme := runtime.NewScheme() + require.NoError(t, multigresv1alpha1.AddToScheme(scheme)) + + cluster := &multigresv1alpha1.MultigresCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + Namespace: "default", + }, + Spec: multigresv1alpha1.MultigresClusterSpec{ + GlobalTopoServer: &multigresv1alpha1.GlobalTopoServerSpec{ + External: &multigresv1alpha1.ExternalTopoServerSpec{ + Endpoints: []multigresv1alpha1.EndpointUrl{"http://topo:2379"}, + }, + }, + Cells: []multigresv1alpha1.CellConfig{{Name: "cell-1", Region: "us-east-1"}}, + Databases: []multigresv1alpha1.DatabaseConfig{{ + Name: "db1", + TableGroups: []multigresv1alpha1.TableGroupConfig{{ + Name: "tg-1", + }}, + }}, + }, + } + staleCell := &multigresv1alpha1.Cell{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cell-1", + Namespace: "default", + Labels: map[string]string{"multigres.com/cluster": "test-cluster"}, + Generation: 1, + }, + Spec: multigresv1alpha1.CellSpec{Name: "cell-1"}, + Status: multigresv1alpha1.CellStatus{ + ObservedGeneration: 1, + Phase: multigresv1alpha1.PhaseHealthy, + }, + } + liveCell := staleCell.DeepCopy() + liveCell.Generation = 2 + tableGroup := &multigresv1alpha1.TableGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "tg-1", + Namespace: "default", + Labels: map[string]string{"multigres.com/cluster": "test-cluster"}, + Generation: 1, + }, + Spec: multigresv1alpha1.TableGroupSpec{ + DatabaseName: "db1", + TableGroupName: "tg-1", + }, + Status: multigresv1alpha1.TableGroupStatus{ + ObservedGeneration: 1, + Phase: multigresv1alpha1.PhaseHealthy, + }, + } + + cachedClient := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(cluster, staleCell, tableGroup). + WithStatusSubresource(cluster, staleCell, tableGroup). + Build() + liveReader := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(liveCell, tableGroup.DeepCopy()). + Build() + r := &MultigresClusterReconciler{ + Client: cachedClient, + APIReader: liveReader, + Scheme: scheme, + Recorder: record.NewFakeRecorder(10), + } + + require.NoError(t, r.updateStatus(t.Context(), cluster)) + require.NoError(t, cachedClient.Get(t.Context(), client.ObjectKeyFromObject(cluster), cluster)) + assert.Equal(t, multigresv1alpha1.PhaseProgressing, cluster.Status.Phase) + assert.Nil(t, cluster.Status.InitializedAt) +} + func TestExtractExternalEndpoint(t *testing.T) { tests := []struct { name string diff --git a/pkg/monitoring/doc.go b/pkg/monitoring/doc.go index 13df3c2d..f518129b 100644 --- a/pkg/monitoring/doc.go +++ b/pkg/monitoring/doc.go @@ -9,7 +9,7 @@ // // Usage in controllers: // -// monitoring.SetClusterInfo(cluster.Name, cluster.Namespace, string(cluster.Status.Phase)) +// monitoring.SetClusterInfo(cluster.Name, cluster.Namespace, string(cluster.Status.Phase), initialized) // monitoring.SetCellGatewayReplicas(cell.Name, cell.Namespace, desired, ready) // // Usage in webhooks: diff --git a/pkg/monitoring/metrics.go b/pkg/monitoring/metrics.go index 08ac9698..798b21c9 100644 --- a/pkg/monitoring/metrics.go +++ b/pkg/monitoring/metrics.go @@ -17,7 +17,7 @@ var ( Name: "multigres_operator_cluster_info", Help: "Info-style metric for MultigresCluster discovery and phase tracking. Always 1.", }, - []string{"name", "namespace", "phase"}, + []string{"name", "namespace", "phase", "initialized"}, ) clusterCellsTotal = prometheus.NewGaugeVec( diff --git a/pkg/monitoring/recorder.go b/pkg/monitoring/recorder.go index 7d31c57c..2be0bc0e 100644 --- a/pkg/monitoring/recorder.go +++ b/pkg/monitoring/recorder.go @@ -1,15 +1,18 @@ package monitoring -import "time" +import ( + "strconv" + "time" +) // SetClusterInfo sets the info-style gauge for a MultigresCluster. -// Old phase labels are automatically cleaned up via DeletePartialMatch. -func SetClusterInfo(name, namespace, phase string) { +// Old phase/initialized labels are automatically cleaned up via DeletePartialMatch. +func SetClusterInfo(name, namespace, phase string, initialized bool) { clusterInfo.DeletePartialMatch(map[string]string{ "name": name, "namespace": namespace, }) - clusterInfo.WithLabelValues(name, namespace, phase).Set(1) + clusterInfo.WithLabelValues(name, namespace, phase, strconv.FormatBool(initialized)).Set(1) } // SetClusterTopology sets the cell and shard count gauges for a cluster. diff --git a/pkg/monitoring/recorder_test.go b/pkg/monitoring/recorder_test.go index 47c84991..36c33a10 100644 --- a/pkg/monitoring/recorder_test.go +++ b/pkg/monitoring/recorder_test.go @@ -12,23 +12,23 @@ import ( func TestSetClusterInfo(t *testing.T) { t.Cleanup(func() { clusterInfo.Reset() }) - SetClusterInfo("test-cluster", "default", "Healthy") + SetClusterInfo("test-cluster", "default", "Healthy", true) - val := gaugeValue(t, clusterInfo, "test-cluster", "default", "Healthy") + val := gaugeValue(t, clusterInfo, "test-cluster", "default", "Healthy", "true") if val != 1 { t.Errorf("expected clusterInfo gauge to be 1, got %f", val) } - // Phase change should clean up old label set - SetClusterInfo("test-cluster", "default", "Degraded") + // Phase change should clean up old label set, initialized stays true + SetClusterInfo("test-cluster", "default", "Degraded", true) - val = gaugeValue(t, clusterInfo, "test-cluster", "default", "Degraded") + val = gaugeValue(t, clusterInfo, "test-cluster", "default", "Degraded", "true") if val != 1 { t.Errorf("expected clusterInfo gauge for Degraded to be 1, got %f", val) } // Old phase must have been cleaned up (value 0) - oldVal := gaugeValue(t, clusterInfo, "test-cluster", "default", "Healthy") + oldVal := gaugeValue(t, clusterInfo, "test-cluster", "default", "Healthy", "true") if oldVal != 0 { t.Error("old phase label set should have been cleaned up") }