From 291b93bab4f5209f424765307eed234e709297fc Mon Sep 17 00:00:00 2001 From: Gianluca Mardente Date: Sun, 21 Jun 2026 15:18:15 +0200 Subject: [PATCH] (feat) Add workload identity support - Add `WorkloadIdentity` field to `SveltosClusterSpec` with provider-specific for AWS (IRSA/EKS Pod Identity), GCP (Workload Identity Federation), and Azure (Azure Workload Identity). - Add `lib/clusterproxy/workload_identity.go`: when `WorkloadIdentity` is set on a SveltosCluster, `GetSveltosKubernetesRestConfig` obtains short-lived credentials from the cloud provider instead of reading a kubeconfig Secret. Results are cached in-process with proactive refresh 5 minutes before expiry. - Add `EvictWorkloadIdentityCache(namespace, name string)` for components to call from their SveltosCluster delete handlers. - No behaviour change when `WorkloadIdentity` is not set. --- api/v1beta1/sveltoscluster_type.go | 119 ++++++ api/v1beta1/zz_generated.deepcopy.go | 87 ++++- ...lib.projectsveltos.io_sveltosclusters.yaml | 135 +++++++ go.mod | 30 +- go.sum | 57 ++- lib/clusterproxy/clusterproxy.go | 16 +- lib/clusterproxy/export_test.go | 27 ++ lib/clusterproxy/workload_identity.go | 340 ++++++++++++++++++ lib/clusterproxy/workload_identity_test.go | 136 +++++++ lib/crd/sveltosclusters.go | 135 +++++++ ...sveltosclusters.lib.projectsveltos.io.yaml | 135 +++++++ 11 files changed, 1205 insertions(+), 12 deletions(-) create mode 100644 lib/clusterproxy/workload_identity.go create mode 100644 lib/clusterproxy/workload_identity_test.go diff --git a/api/v1beta1/sveltoscluster_type.go b/api/v1beta1/sveltoscluster_type.go index 15730dd9..7a2bde28 100644 --- a/api/v1beta1/sveltoscluster_type.go +++ b/api/v1beta1/sveltoscluster_type.go @@ -17,6 +17,7 @@ limitations under the License. package v1beta1 import ( + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) @@ -105,7 +106,117 @@ type TokenRequestRenewalOption struct { KubeconfigKeyName *string `json:"kubeconfigKeyName,omitempty"` } +// WorkloadIdentityProvider identifies the cloud provider for workload identity. +// +kubebuilder:validation:Enum=AWS;GCP;Azure +type WorkloadIdentityProvider string + +const ( + WorkloadIdentityProviderAWS WorkloadIdentityProvider = "AWS" + WorkloadIdentityProviderGCP WorkloadIdentityProvider = "GCP" + WorkloadIdentityProviderAzure WorkloadIdentityProvider = "Azure" +) + +// AWSWorkloadIdentityConfig holds AWS-specific workload identity configuration. +type AWSWorkloadIdentityConfig struct { + // ClusterName is the name of the EKS cluster. Required: it is embedded in + // the bearer token header (x-k8s-aws-id) sent to the EKS API server. + // +kubebuilder:validation:MinLength=1 + ClusterName string `json:"clusterName"` + + // RoleARN is the ARN of the IAM role Sveltos should assume before generating + // the EKS bearer token. If empty, the pod's own IRSA credentials are used directly. + // +optional + RoleARN string `json:"roleARN,omitempty"` + + // Region is the AWS region of the EKS cluster. If empty, the AWS_REGION + // environment variable injected by IRSA is used. + // +optional + Region string `json:"region,omitempty"` +} + +// GCPWorkloadIdentityConfig holds GCP-specific workload identity configuration. +// Fields are kept for future auto-discovery support; the current implementation +// obtains tokens via Application Default Credentials without calling the GKE API. +type GCPWorkloadIdentityConfig struct { + // ProjectID is the GCP project ID that contains the GKE cluster. + // +kubebuilder:validation:MinLength=1 + ProjectID string `json:"projectID"` + + // ClusterName is the name of the GKE cluster. + // +kubebuilder:validation:MinLength=1 + ClusterName string `json:"clusterName"` + + // Location is the GCP region or zone of the GKE cluster (e.g. "us-central1"). + // +kubebuilder:validation:MinLength=1 + Location string `json:"location"` +} + +// AzureWorkloadIdentityConfig holds Azure-specific workload identity configuration. +type AzureWorkloadIdentityConfig struct { + // TenantID is the Azure Active Directory tenant ID. + // +kubebuilder:validation:MinLength=1 + TenantID string `json:"tenantID"` + + // ClientID is the client ID of the managed identity or app registration + // federated with the management cluster service account. + // +kubebuilder:validation:MinLength=1 + ClientID string `json:"clientID"` + + // SubscriptionID is the Azure subscription containing the AKS cluster. + // Kept for future auto-discovery support. + // +optional + SubscriptionID string `json:"subscriptionID,omitempty"` + + // ResourceGroup is the Azure resource group containing the AKS cluster. + // Kept for future auto-discovery support. + // +optional + ResourceGroup string `json:"resourceGroup,omitempty"` + + // ClusterName is the name of the AKS cluster. + // Kept for future auto-discovery support. + // +optional + ClusterName string `json:"clusterName,omitempty"` +} + +// WorkloadIdentityConfig specifies how Sveltos authenticates to the managed +// cluster using the cloud provider's workload identity mechanism instead of a +// static kubeconfig Secret. +// +kubebuilder:validation:XValidation:rule="(self.provider == 'AWS') == has(self.aws)",message="aws must be set if and only if provider is AWS" +// +kubebuilder:validation:XValidation:rule="(self.provider == 'GCP') == has(self.gcp)",message="gcp must be set if and only if provider is GCP" +// +kubebuilder:validation:XValidation:rule="(self.provider == 'Azure') == has(self.azure)",message="azure must be set if and only if provider is Azure" +type WorkloadIdentityConfig struct { + // Provider is the cloud provider implementing the workload identity mechanism. + // +kubebuilder:validation:Required + Provider WorkloadIdentityProvider `json:"provider"` + + // Endpoint is the API server endpoint of the managed cluster (e.g. https://…). + // +kubebuilder:validation:MinLength=1 + Endpoint string `json:"endpoint"` + + // CASecretRef references a Secret in the management cluster containing the + // CA certificate of the managed cluster's API server under the key "ca.crt". + // If not set, the system certificate pool is used. + // +optional + CASecretRef *corev1.LocalObjectReference `json:"caSecretRef,omitempty"` + + // AWS contains configuration specific to AWS IRSA / EKS Pod Identity. + // Required when Provider is AWS. + // +optional + AWS *AWSWorkloadIdentityConfig `json:"aws,omitempty"` + + // GCP contains configuration specific to GCP Workload Identity Federation. + // Required when Provider is GCP. + // +optional + GCP *GCPWorkloadIdentityConfig `json:"gcp,omitempty"` + + // Azure contains configuration specific to Azure Workload Identity. + // Required when Provider is Azure. + // +optional + Azure *AzureWorkloadIdentityConfig `json:"azure,omitempty"` +} + // SveltosClusterSpec defines the desired state of SveltosCluster +// +kubebuilder:validation:XValidation:rule="!has(self.workloadIdentity) || !has(self.kubeconfigName)",message="workloadIdentity, kubeconfigName: conflict" type SveltosClusterSpec struct { // KubeconfigName allows overriding the default Sveltos convention which expected a valid kubeconfig // to be hosted in a secret with the pattern ${sveltosClusterName}-sveltos-kubeconfig. @@ -161,6 +272,14 @@ type SveltosClusterSpec struct { // +kubebuilder:default:=false // +optional PullMode bool `json:"pullMode,omitempty"` + + // WorkloadIdentity configures authentication to the managed cluster via the + // cloud provider's workload identity mechanism. When set, Sveltos does not + // read a kubeconfig Secret; instead it obtains short-lived credentials from + // the cloud provider at runtime. + // Mutually exclusive with KubeconfigName/KubeconfigKeyName. + // +optional + WorkloadIdentity *WorkloadIdentityConfig `json:"workloadIdentity,omitempty"` } // SveltosClusterStatus defines the status of SveltosCluster diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index 111a2cdb..7a6c652f 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -21,11 +21,26 @@ limitations under the License. package v1beta1 import ( - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AWSWorkloadIdentityConfig) DeepCopyInto(out *AWSWorkloadIdentityConfig) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AWSWorkloadIdentityConfig. +func (in *AWSWorkloadIdentityConfig) DeepCopy() *AWSWorkloadIdentityConfig { + if in == nil { + return nil + } + out := new(AWSWorkloadIdentityConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *AccessRequest) DeepCopyInto(out *AccessRequest) { *out = *in @@ -141,6 +156,21 @@ func (in *ActiveWindow) DeepCopy() *ActiveWindow { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AzureWorkloadIdentityConfig) DeepCopyInto(out *AzureWorkloadIdentityConfig) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureWorkloadIdentityConfig. +func (in *AzureWorkloadIdentityConfig) DeepCopy() *AzureWorkloadIdentityConfig { + if in == nil { + return nil + } + out := new(AzureWorkloadIdentityConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CELRule) DeepCopyInto(out *CELRule) { *out = *in @@ -1452,6 +1482,21 @@ func (in *FromManagement) DeepCopy() *FromManagement { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GCPWorkloadIdentityConfig) DeepCopyInto(out *GCPWorkloadIdentityConfig) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GCPWorkloadIdentityConfig. +func (in *GCPWorkloadIdentityConfig) DeepCopy() *GCPWorkloadIdentityConfig { + if in == nil { + return nil + } + out := new(GCPWorkloadIdentityConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *HealthCheck) DeepCopyInto(out *HealthCheck) { *out = *in @@ -2694,6 +2739,11 @@ func (in *SveltosClusterSpec) DeepCopyInto(out *SveltosClusterSpec) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.WorkloadIdentity != nil { + in, out := &in.WorkloadIdentity, &out.WorkloadIdentity + *out = new(WorkloadIdentityConfig) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SveltosClusterSpec. @@ -3032,3 +3082,38 @@ func (in *ValidateHealth) DeepCopy() *ValidateHealth { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *WorkloadIdentityConfig) DeepCopyInto(out *WorkloadIdentityConfig) { + *out = *in + if in.CASecretRef != nil { + in, out := &in.CASecretRef, &out.CASecretRef + *out = new(v1.LocalObjectReference) + **out = **in + } + if in.AWS != nil { + in, out := &in.AWS, &out.AWS + *out = new(AWSWorkloadIdentityConfig) + **out = **in + } + if in.GCP != nil { + in, out := &in.GCP, &out.GCP + *out = new(GCPWorkloadIdentityConfig) + **out = **in + } + if in.Azure != nil { + in, out := &in.Azure, &out.Azure + *out = new(AzureWorkloadIdentityConfig) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WorkloadIdentityConfig. +func (in *WorkloadIdentityConfig) DeepCopy() *WorkloadIdentityConfig { + if in == nil { + return nil + } + out := new(WorkloadIdentityConfig) + in.DeepCopyInto(out) + return out +} diff --git a/config/crd/bases/lib.projectsveltos.io_sveltosclusters.yaml b/config/crd/bases/lib.projectsveltos.io_sveltosclusters.yaml index 3e529ee4..20ef51d5 100644 --- a/config/crd/bases/lib.projectsveltos.io_sveltosclusters.yaml +++ b/config/crd/bases/lib.projectsveltos.io_sveltosclusters.yaml @@ -494,7 +494,142 @@ spec: required: - renewTokenRequestInterval type: object + workloadIdentity: + description: |- + WorkloadIdentity configures authentication to the managed cluster via the + cloud provider's workload identity mechanism. When set, Sveltos does not + read a kubeconfig Secret; instead it obtains short-lived credentials from + the cloud provider at runtime. + Mutually exclusive with KubeconfigName/KubeconfigKeyName. + properties: + aws: + description: |- + AWS contains configuration specific to AWS IRSA / EKS Pod Identity. + Required when Provider is AWS. + properties: + clusterName: + description: |- + ClusterName is the name of the EKS cluster. Required: it is embedded in + the bearer token header (x-k8s-aws-id) sent to the EKS API server. + minLength: 1 + type: string + region: + description: |- + Region is the AWS region of the EKS cluster. If empty, the AWS_REGION + environment variable injected by IRSA is used. + type: string + roleARN: + description: |- + RoleARN is the ARN of the IAM role Sveltos should assume before generating + the EKS bearer token. If empty, the pod's own IRSA credentials are used directly. + type: string + required: + - clusterName + type: object + azure: + description: |- + Azure contains configuration specific to Azure Workload Identity. + Required when Provider is Azure. + properties: + clientID: + description: |- + ClientID is the client ID of the managed identity or app registration + federated with the management cluster service account. + minLength: 1 + type: string + clusterName: + description: |- + ClusterName is the name of the AKS cluster. + Kept for future auto-discovery support. + type: string + resourceGroup: + description: |- + ResourceGroup is the Azure resource group containing the AKS cluster. + Kept for future auto-discovery support. + type: string + subscriptionID: + description: |- + SubscriptionID is the Azure subscription containing the AKS cluster. + Kept for future auto-discovery support. + type: string + tenantID: + description: TenantID is the Azure Active Directory tenant + ID. + minLength: 1 + type: string + required: + - clientID + - tenantID + type: object + caSecretRef: + description: |- + CASecretRef references a Secret in the management cluster containing the + CA certificate of the managed cluster's API server under the key "ca.crt". + If not set, the system certificate pool is used. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + endpoint: + description: Endpoint is the API server endpoint of the managed + cluster (e.g. https://…). + minLength: 1 + type: string + gcp: + description: |- + GCP contains configuration specific to GCP Workload Identity Federation. + Required when Provider is GCP. + properties: + clusterName: + description: ClusterName is the name of the GKE cluster. + minLength: 1 + type: string + location: + description: Location is the GCP region or zone of the GKE + cluster (e.g. "us-central1"). + minLength: 1 + type: string + projectID: + description: ProjectID is the GCP project ID that contains + the GKE cluster. + minLength: 1 + type: string + required: + - clusterName + - location + - projectID + type: object + provider: + description: Provider is the cloud provider implementing the workload + identity mechanism. + enum: + - AWS + - GCP + - Azure + type: string + required: + - endpoint + - provider + type: object + x-kubernetes-validations: + - message: aws must be set if and only if provider is AWS + rule: (self.provider == 'AWS') == has(self.aws) + - message: gcp must be set if and only if provider is GCP + rule: (self.provider == 'GCP') == has(self.gcp) + - message: azure must be set if and only if provider is Azure + rule: (self.provider == 'Azure') == has(self.azure) type: object + x-kubernetes-validations: + - message: 'workloadIdentity, kubeconfigName: conflict' + rule: '!has(self.workloadIdentity) || !has(self.kubeconfigName)' status: description: SveltosClusterStatus defines the status of SveltosCluster properties: diff --git a/go.mod b/go.mod index f11bc018..46e49dee 100644 --- a/go.mod +++ b/go.mod @@ -3,8 +3,14 @@ module github.com/projectsveltos/libsveltos go 1.26.4 require ( + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.22.0 + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.0 github.com/BurntSushi/toml v1.6.0 github.com/Masterminds/sprig/v3 v3.3.0 + github.com/aws/aws-sdk-go-v2 v1.42.0 + github.com/aws/aws-sdk-go-v2/config v1.32.25 + github.com/aws/aws-sdk-go-v2/credentials v1.19.24 + github.com/aws/aws-sdk-go-v2/service/sts v1.43.3 github.com/go-logr/logr v1.4.3 github.com/google/cel-go v0.28.1 github.com/hexops/gotextdiff v1.0.3 @@ -17,6 +23,8 @@ require ( github.com/projectsveltos/lua-utils/glua-sprig v0.0.0-20251212200258-2b3cdcb7c0f5 github.com/projectsveltos/lua-utils/glua-strings v0.0.0-20251212200258-2b3cdcb7c0f5 github.com/yuin/gopher-lua v1.1.2 + golang.org/x/oauth2 v0.36.0 + golang.org/x/sync v0.21.0 golang.org/x/text v0.38.0 k8s.io/api v0.36.2 k8s.io/apiextensions-apiserver v0.36.2 @@ -33,10 +41,23 @@ require ( require ( cel.dev/expr v0.25.1 // indirect + cloud.google.com/go/compute/metadata v0.3.0 // indirect dario.cat/mergo v1.0.1 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 // indirect + github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2 // indirect github.com/Masterminds/goutils v1.1.1 // indirect github.com/Masterminds/semver/v3 v3.4.0 // indirect github.com/antlr4-go/antlr/v4 v4.13.1 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.29 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.29 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.29 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.30 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.12 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.29 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.2.0 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.31.3 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.36.6 // indirect + github.com/aws/smithy-go v1.27.1 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver/v4 v4.0.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect @@ -52,6 +73,7 @@ require ( github.com/go-openapi/swag v0.23.0 // indirect github.com/go-task/slim-sprig/v3 v3.0.0 // indirect github.com/gobuffalo/flect v1.0.3 // indirect + github.com/golang-jwt/jwt/v5 v5.3.1 // indirect github.com/google/gnostic-models v0.7.0 // indirect github.com/google/go-cmp v0.7.0 // indirect github.com/google/pprof v0.0.0-20260402051712-545e8a4df936 // indirect @@ -59,6 +81,7 @@ require ( github.com/huandu/xstrings v1.5.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect @@ -66,6 +89,7 @@ require ( github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_golang v1.23.2 // indirect github.com/prometheus/client_model v0.6.2 // indirect @@ -83,10 +107,8 @@ require ( golang.org/x/crypto v0.51.0 // indirect golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93 // indirect golang.org/x/mod v0.36.0 // indirect - golang.org/x/net v0.54.0 // indirect - golang.org/x/oauth2 v0.36.0 // indirect - golang.org/x/sync v0.21.0 // indirect - golang.org/x/sys v0.44.0 // indirect + golang.org/x/net v0.55.0 // indirect + golang.org/x/sys v0.45.0 // indirect golang.org/x/term v0.43.0 // indirect golang.org/x/time v0.14.0 // indirect golang.org/x/tools v0.45.0 // indirect diff --git a/go.sum b/go.sum index 3fcd7468..23000321 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,21 @@ cel.dev/expr v0.25.1 h1:1KrZg61W6TWSxuNZ37Xy49ps13NUovb66QLprthtwi4= cel.dev/expr v0.25.1/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= +cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2QxYC4trgAKZc= +cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= dario.cat/mergo v1.0.1 h1:Ra4+bf83h2ztPIQYNP99R6m+Y7KfnARDfID+a+vLl4s= dario.cat/mergo v1.0.1/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.22.0 h1:aokoqcHvaGjiM3VpjKDfMMnF/8epJ+Q1HLJ7CudztqE= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.22.0/go.mod h1:/WYEx9pcM9Y+Dd/APJaNlSvVSvzl54rrMdZT5+Oi2LM= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.0 h1:CU4+EJeJi3TKYWEcYuSdWsjzw0nVsK/H0MSQOiPcymU= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.0/go.mod h1:q0+UTSRvShwUCrR/s5HtyInYphN7Wvxb7snFM3u+SLA= +github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.4.0 h1:xFaZZ+IubdftrDHnGGwZ6QvQ3KHTtWl2MCK+GMt2vxs= +github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.4.0/go.mod h1:mCBhUhlMjLLJKr5aqw2TNS/VqJOie8MzWq3DAMJeKso= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 h1:fhqpLE3UEXi9lPaBRpQ6XuRW0nU7hgg4zlmZZa+a9q4= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0/go.mod h1:7dCRMLwisfRH3dBupKeNCioWYUZ4SS09Z14H+7i8ZoY= +github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM= +github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE= +github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2 h1:RHK7bS+HQMslb1sZpAokUt+zTVmue0hKSs2C791hhzU= +github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk= github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk= github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= @@ -12,6 +26,34 @@ github.com/Masterminds/sprig/v3 v3.3.0 h1:mQh0Yrg1XPo6vjYXgtf5OtijNAKJRNcTdOOGZe github.com/Masterminds/sprig/v3 v3.3.0/go.mod h1:Zy1iXRYNqNLUolqCpL4uhk6SHUMAOSCzdgBfDb35Lz0= github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ= github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw= +github.com/aws/aws-sdk-go-v2 v1.42.0 h1:XvXMJTkFQtpBKIWZnmr9ZEOc2InWM2yldjXEJ/bymhA= +github.com/aws/aws-sdk-go-v2 v1.42.0/go.mod h1:27+ACypSLljLAEKsCYOmrjKh83vuTRkuAe9Uv/3A4bg= +github.com/aws/aws-sdk-go-v2/config v1.32.25 h1:ACCejvStYoilgwrfegSt5ZntCbPrk52qfwyNcnl3omM= +github.com/aws/aws-sdk-go-v2/config v1.32.25/go.mod h1:LJyU8sDRbXUxFn8xMJIGP+v9QYYwveNLI8a/giAOiAs= +github.com/aws/aws-sdk-go-v2/credentials v1.19.24 h1:2hQqYCV9yqyePQ9o6dCrZc/zO8U3TwPr9mIKlZnPu/I= +github.com/aws/aws-sdk-go-v2/credentials v1.19.24/go.mod h1:IDwpACtwqHLISdzfwUUNq4P9DsB/h5BLg4FwJPNfqFY= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.29 h1:r6qZHbT+wxgWO/e9vYNUEtg7lv5+UN3pRqKhLXvnArg= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.29/go.mod h1:QRnaRcTVGKPGRy8w78HMQtKUGRYcnMZAANATkeVA6Mo= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.29 h1:f3vKqSo13fhTYb+JEcXwXefZQE26I1FB5eTSniU67ko= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.29/go.mod h1:MzoLFUArKGpGD+ukmPiTPG1X5x4o6M2kq4v2dr1FiEc= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.29 h1:RdwIf/CuUsvJX3RgJagbOyotl/cxoLY4xviKuE7p2GY= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.29/go.mod h1:71wt8W2EgswdZy9Mf9KNnzxZ3TiZlv4caKghPktDOkA= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.30 h1:VTGy885W5DKBxWRUJbym9hytNaYzsyaPkCHGRRMAOhU= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.30/go.mod h1:AS0HycUvJRFvTt613AYDOgO2jzw+00cVSMny8XB3yMY= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.12 h1:ZD2+BSw9vFsNlKYIasSNt3uDbjqqXIBcM13UJv/Lx2k= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.12/go.mod h1:Ms4zlcVBbXbiP7EVLhl+lgjvA/a7YphqQ3Ih3174EmI= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.29 h1:DRebniUGZ2MqiiIVmQJ04vIXr918hubdHMnarSLEWyU= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.29/go.mod h1:LfRkPCD8YHDM2E5eTkos2UpwYeZnBcVarTa8L59bJHA= +github.com/aws/aws-sdk-go-v2/service/signin v1.2.0 h1:3nXpRcFwRCW8n7HgO2QGy0Dc20eQNfBuUemGQhpF8m8= +github.com/aws/aws-sdk-go-v2/service/signin v1.2.0/go.mod h1:LxYujSTLPRlp2vTtcUO/+1ilrew8ytt6SvQyOgejzFQ= +github.com/aws/aws-sdk-go-v2/service/sso v1.31.3 h1:ey1XLTYXb9PcLt4535632o5kCGXNXEhNb620Dqwuylo= +github.com/aws/aws-sdk-go-v2/service/sso v1.31.3/go.mod h1:Lk7PlmoTYryQmyBG0EXqj5BcUbj3whXdU2s3yGI3EAc= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.36.6 h1:yLr03zQE/5Eu5l3QU0Si+xMbLMbSDF2YXsigqXngs6g= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.36.6/go.mod h1:Q5N6icH+KJZDLh+ESNwzdv6cZ6vLFF/egy3IOxWhmz4= +github.com/aws/aws-sdk-go-v2/service/sts v1.43.3 h1:VrIhKRCSK1umelSgB9RghvA9RTUYeQffyAS5ApXehNI= +github.com/aws/aws-sdk-go-v2/service/sts v1.43.3/go.mod h1:r8wkDOuLaaMFqFiYAb8dGY2A3gJCOujMc6CFOVC4Zhc= +github.com/aws/smithy-go v1.27.1 h1:4T340VFndXtADGF52gYa1POyL7s9E4Z1OeZ1hCscIw8= +github.com/aws/smithy-go v1.27.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= @@ -58,6 +100,8 @@ github.com/gobuffalo/flect v1.0.3 h1:xeWBM2nui+qnVvNM4S3foBhCAL2XgPU+a7FdpelbTq4 github.com/gobuffalo/flect v1.0.3/go.mod h1:A5msMlrHtLqh9umBSnvabjsMrCcCpAyzglnDvkbYKHs= github.com/goccy/go-yaml v1.18.0 h1:8W7wMFS12Pcas7KU+VVkaiCng+kG8QiFeFwzFb+rwuw= github.com/goccy/go-yaml v1.18.0/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= +github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= +github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/google/cel-go v0.28.1 h1:YWIwi77J4xIsYUwAF/iIuS6haffzIHS8yWI8glSbLWM= github.com/google/cel-go v0.28.1/go.mod h1:X0bD6iVNR8pkROSOoHVdgTkzmRcosof7WQqCD6wcMc8= github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo= @@ -81,6 +125,8 @@ github.com/joshdk/go-junit v1.0.0 h1:S86cUKIdwBHWwA6xCmFlf3RTLfVXYQfvanM5Uh+K6GE github.com/joshdk/go-junit v1.0.0/go.mod h1:TiiV0PqkaNfFXjEiyjWM3XXrhVyCa1K4Zfga6W52ung= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/keybase/go-keychain v0.0.1 h1:way+bWYa6lDppZoZcgMbYsvC7GxljxrskdNInRtuthU= +github.com/keybase/go-keychain v0.0.1/go.mod h1:PdEILRW3i9D8JcdM+FmY6RwkHGnhHxXwkPPMeUgOK1k= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= @@ -115,6 +161,8 @@ github.com/onsi/ginkgo/v2 v2.31.0 h1:GtuJos5DFUV9EerYJo8RhYxosYNGvOdDE5haKq6Grfs github.com/onsi/ginkgo/v2 v2.31.0/go.mod h1:+aXOY+vzZ5mu2iI2HpTZUPmM//oQfsNFX6gU9kNcA44= github.com/onsi/gomega v1.42.0 h1:CJby8u36xb7v34W78F8WKvqTQP7PCMIPB78IVDB73l4= github.com/onsi/gomega v1.42.0/go.mod h1:M/Uqpu/8qTjtzCLUA2zJHX9Iilrau25x1PdoSRbWh5A= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -192,14 +240,15 @@ golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93 h1:fQsdNF2N+/YewlRZiricy4P1i golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93/go.mod h1:EPRbTFwzwjXj9NpYyyrvenVh9Y+GFeEvMNh7Xuz7xgU= golang.org/x/mod v0.36.0 h1:JJjpVx6myfUsUdAzZuOSTTmRE0PfZeNWzzvKrP7amb4= golang.org/x/mod v0.36.0/go.mod h1:moc6ELqsWcOw5Ef3xVprK5ul/MvtVvkIXLziUOICjUQ= -golang.org/x/net v0.54.0 h1:2zJIZAxAHV/OHCDTCOHAYehQzLfSXuf/5SoL/Dv6w/w= -golang.org/x/net v0.54.0/go.mod h1:Sj4oj8jK6XmHpBZU/zWHw3BV3abl4Kvi+Ut7cQcY+cQ= +golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= +golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= -golang.org/x/sys v0.44.0 h1:ildZl3J4uzeKP07r2F++Op7E9B29JRUy+a27EibtBTQ= -golang.org/x/sys v0.44.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= +golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4= golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk= golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE= diff --git a/lib/clusterproxy/clusterproxy.go b/lib/clusterproxy/clusterproxy.go index 0a878a01..f40754d0 100644 --- a/lib/clusterproxy/clusterproxy.go +++ b/lib/clusterproxy/clusterproxy.go @@ -117,12 +117,22 @@ func GetCAPISecretData(ctx context.Context, logger logr.Logger, c client.Client, func GetSveltosKubernetesRestConfig(ctx context.Context, logger logr.Logger, c client.Client, clusterNamespace, clusterName string) (*rest.Config, error) { - pullMode, err := isSveltosClusterInPullMode(ctx, c, clusterNamespace, clusterName, logger) - if err != nil { + cluster := &libsveltosv1beta1.SveltosCluster{} + if err := c.Get(ctx, types.NamespacedName{Namespace: clusterNamespace, Name: clusterName}, cluster); err != nil { + if apierrors.IsNotFound(err) { + logger.Info("SveltosCluster does not exist") + return nil, errors.Wrap(err, + fmt.Sprintf("SveltosCluster %s/%s does not exist", clusterNamespace, clusterName)) + } return nil, err } - if pullMode { + if cluster.Spec.WorkloadIdentity != nil { + return getWorkloadIdentityRestConfig(ctx, c, clusterNamespace, clusterName, + cluster.Spec.WorkloadIdentity, logger) + } + + if cluster.Spec.PullMode { return nil, nil } diff --git a/lib/clusterproxy/export_test.go b/lib/clusterproxy/export_test.go index b085e3c4..984311a2 100644 --- a/lib/clusterproxy/export_test.go +++ b/lib/clusterproxy/export_test.go @@ -16,6 +16,12 @@ limitations under the License. package clusterproxy +import ( + "time" + + "k8s.io/client-go/rest" +) + const ( CapiKubeconfigSecretNamePostfix = capiKubeconfigSecretNamePostfix @@ -25,3 +31,24 @@ const ( var ( IsSveltosClusterInPullMode = isSveltosClusterInPullMode ) + +// StoreTestWiCache inserts a pre-built entry into the workload identity cache. +// For use in tests only. +func StoreTestWiCache(namespace, name string, cfg *rest.Config, expiresAt time.Time) { + wiCache.Store(wiCacheKey(namespace, name), cachedRestConfig{config: cfg, expiresAt: expiresAt}) +} + +// LoadTestWiCache returns the cached rest.Config and expiry for the given cluster. +// For use in tests only. +func LoadTestWiCache(namespace, name string) (*rest.Config, time.Time, bool) { + v, ok := wiCache.Load(wiCacheKey(namespace, name)) + if !ok { + return nil, time.Time{}, false + } + entry := v.(cachedRestConfig) + return entry.config, entry.expiresAt, true +} + +// GetCADataForTest calls the internal getCAData function. +// For use in tests only. +var GetCADataForTest = getCAData diff --git a/lib/clusterproxy/workload_identity.go b/lib/clusterproxy/workload_identity.go new file mode 100644 index 00000000..67dba1b2 --- /dev/null +++ b/lib/clusterproxy/workload_identity.go @@ -0,0 +1,340 @@ +/* +Copyright 2026. projectsveltos.io. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package clusterproxy + +import ( + "context" + "encoding/base64" + "fmt" + "net/http" + "os" + "sync" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/aws/aws-sdk-go-v2/aws" + signerv4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials/stscreds" + "github.com/aws/aws-sdk-go-v2/service/sts" + "github.com/go-logr/logr" + "github.com/pkg/errors" + "golang.org/x/oauth2/google" + "golang.org/x/sync/singleflight" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + + libsveltosv1beta1 "github.com/projectsveltos/libsveltos/api/v1beta1" + logs "github.com/projectsveltos/libsveltos/lib/logsettings" +) + +const ( + // wiRefreshThreshold is how much time before expiry we proactively refresh. + wiRefreshThreshold = 5 * time.Minute + + //nolint:gosec // this is a token format prefix, not a credential + eksBearerTokenPrefix = "k8s-aws-v1." + + // eksTokenTTL is the fixed TTL of EKS pre-signed STS tokens. + eksTokenTTL = 15 * time.Minute + + // aksScope is the AAD scope required to authenticate against an AKS API server. + aksScope = "6dae42f8-4368-4678-94ff-3960e28e3630/.default" + + gcpCloudPlatformScope = "https://www.googleapis.com/auth/cloud-platform" + + caSecretKey = "ca.crt" +) + +type cachedRestConfig struct { + config *rest.Config + expiresAt time.Time +} + +var ( + wiCache sync.Map // map[string]cachedRestConfig + wiGroup singleflight.Group // deduplicates concurrent refreshes for the same cluster +) + +// EvictWorkloadIdentityCache removes the cached rest.Config for the given +// SveltosCluster. Call this from the SveltosCluster delete handler in any +// component that uses workload identity. +func EvictWorkloadIdentityCache(namespace, name string) { + wiCache.Delete(wiCacheKey(namespace, name)) +} + +func wiCacheKey(namespace, name string) string { + return namespace + "/" + name +} + +// getWorkloadIdentityRestConfig returns a rest.Config for a SveltosCluster that +// uses cloud provider workload identity. Results are cached and proactively +// refreshed when they approach expiry; concurrent calls for the same cluster are +// collapsed into a single cloud API call via singleflight. +func getWorkloadIdentityRestConfig( + ctx context.Context, + c client.Client, + clusterNamespace, clusterName string, + wi *libsveltosv1beta1.WorkloadIdentityConfig, + logger logr.Logger, +) (*rest.Config, error) { + + key := wiCacheKey(clusterNamespace, clusterName) + + // Fast path: valid cached entry. + if v, ok := wiCache.Load(key); ok { + entry := v.(cachedRestConfig) + if time.Until(entry.expiresAt) > wiRefreshThreshold { + return entry.config, nil + } + } + + // Slow path: fetch from cloud provider, deduplicated across concurrent callers. + type result struct { + cfg *rest.Config + } + val, err, _ := wiGroup.Do(key, func() (interface{}, error) { + caData, err := getCAData(ctx, c, clusterNamespace, wi.CASecretRef, logger) + if err != nil { + return nil, err + } + + var cfg *rest.Config + var expiresAt time.Time + + switch wi.Provider { + case libsveltosv1beta1.WorkloadIdentityProviderAWS: + cfg, expiresAt, err = getAWSRestConfig(ctx, wi, caData, logger) + case libsveltosv1beta1.WorkloadIdentityProviderGCP: + cfg, expiresAt, err = getGCPRestConfig(ctx, wi, caData, logger) + case libsveltosv1beta1.WorkloadIdentityProviderAzure: + cfg, expiresAt, err = getAzureRestConfig(ctx, wi, caData, logger) + default: + err = fmt.Errorf("unknown workload identity provider %q", wi.Provider) + } + if err != nil { + return nil, err + } + + wiCache.Store(key, cachedRestConfig{config: cfg, expiresAt: expiresAt}) + return result{cfg: cfg}, nil + }) + if err != nil { + return nil, err + } + + return val.(result).cfg, nil +} + +// getCAData fetches the CA certificate bytes from the referenced Secret. +// If caSecretRef is nil, nil is returned and the system certificate pool is used. +func getCAData( + ctx context.Context, + c client.Client, + namespace string, + caSecretRef *corev1.LocalObjectReference, + logger logr.Logger, +) ([]byte, error) { + + if caSecretRef == nil { + return nil, nil + } + + secret := &corev1.Secret{} + key := client.ObjectKey{Namespace: namespace, Name: caSecretRef.Name} + if err := c.Get(ctx, key, secret); err != nil { + if apierrors.IsNotFound(err) { + return nil, errors.Wrap(err, + fmt.Sprintf("CA secret %s/%s not found", namespace, caSecretRef.Name)) + } + return nil, errors.Wrap(err, + fmt.Sprintf("failed to get CA secret %s/%s", namespace, caSecretRef.Name)) + } + + ca, ok := secret.Data[caSecretKey] + if !ok { + return nil, fmt.Errorf("CA secret %s/%s has no %q key", namespace, caSecretRef.Name, caSecretKey) + } + logger.V(logs.LogDebug).Info("loaded CA from secret", "secret", caSecretRef.Name) + return ca, nil +} + +// buildRestConfig assembles a rest.Config from the common fields. +func buildRestConfig(endpoint, bearerToken string, caData []byte) *rest.Config { + cfg := &rest.Config{ + Host: endpoint, + BearerToken: bearerToken, + } + cfg.CAData = caData + return cfg +} + +// ── AWS ────────────────────────────────────────────────────────────────────── + +// eksHTTPPresigner is a standalone SigV4 presigner that adds the x-k8s-aws-id +// header required by the EKS authentication flow before signing. +// It does not wrap the default STS presigner — instead it uses signerv4.Signer +// directly so it never holds a nil interface (the default presigner is only +// initialized later in the STS presign middleware stack). +type eksHTTPPresigner struct { + signer *signerv4.Signer + clusterName string +} + +//nolint:gocritic // creds is passed by value to satisfy the sts.HTTPPresignerV4 interface +func (p *eksHTTPPresigner) PresignHTTP( + ctx context.Context, + creds aws.Credentials, + r *http.Request, + payloadHash, service, region string, + signingTime time.Time, + optFns ...func(*signerv4.SignerOptions), +) (string, http.Header, error) { + + // X-Amz-Expires must be in the signed URL; the STS middleware does not add + // it when a custom presigner is used. EKS rejects tokens without it. + q := r.URL.Query() + q.Set("X-Amz-Expires", "900") + r.URL.RawQuery = q.Encode() + + r.Header.Set("x-k8s-aws-id", p.clusterName) + return p.signer.PresignHTTP(ctx, creds, r, payloadHash, service, region, signingTime, optFns...) +} + +func getAWSRestConfig( + ctx context.Context, + wi *libsveltosv1beta1.WorkloadIdentityConfig, + caData []byte, + logger logr.Logger, +) (*rest.Config, time.Time, error) { + + awsCfg := wi.AWS + region := awsCfg.Region + if region == "" { + region = os.Getenv("AWS_REGION") + if region == "" { + return nil, time.Time{}, errors.New("AWS region not specified and AWS_REGION env var is not set") + } + } + + cfg, err := config.LoadDefaultConfig(ctx, config.WithRegion(region)) + if err != nil { + return nil, time.Time{}, errors.Wrap(err, "failed to load AWS config") + } + + // If a role ARN is provided, assume that role before generating the EKS token. + if awsCfg.RoleARN != "" { + stsClient := sts.NewFromConfig(cfg) + provider := stscreds.NewAssumeRoleProvider(stsClient, awsCfg.RoleARN) + cfg.Credentials = aws.NewCredentialsCache(provider) + logger.V(logs.LogDebug).Info("assuming AWS role", "roleARN", awsCfg.RoleARN) + } + + stsOpts := sts.Options{ + Region: region, + Credentials: cfg.Credentials, + } + presignClient := sts.NewPresignClient(sts.New(stsOpts), func(po *sts.PresignOptions) { + po.Presigner = &eksHTTPPresigner{ + signer: signerv4.NewSigner(), + clusterName: awsCfg.ClusterName, + } + }) + + presigned, err := presignClient.PresignGetCallerIdentity(ctx, &sts.GetCallerIdentityInput{}) + if err != nil { + return nil, time.Time{}, errors.Wrap(err, "failed to presign EKS bearer token") + } + + token := eksBearerTokenPrefix + base64.RawURLEncoding.EncodeToString([]byte(presigned.URL)) + expiresAt := time.Now().Add(eksTokenTTL) + + logger.V(logs.LogDebug).Info("obtained AWS EKS bearer token", + "cluster", awsCfg.ClusterName, "expiresAt", expiresAt) + return buildRestConfig(wi.Endpoint, token, caData), expiresAt, nil +} + +// ── GCP ────────────────────────────────────────────────────────────────────── + +func getGCPRestConfig( + ctx context.Context, + wi *libsveltosv1beta1.WorkloadIdentityConfig, + caData []byte, + logger logr.Logger, +) (*rest.Config, time.Time, error) { + + src, err := google.DefaultTokenSource(ctx, gcpCloudPlatformScope) + if err != nil { + return nil, time.Time{}, errors.Wrap(err, "failed to obtain GCP token source") + } + + token, err := src.Token() + if err != nil { + return nil, time.Time{}, errors.Wrap(err, "failed to obtain GCP access token") + } + + logger.V(logs.LogDebug).Info("obtained GCP access token", "expiresAt", token.Expiry) + return buildRestConfig(wi.Endpoint, token.AccessToken, caData), token.Expiry, nil +} + +// ── Azure ───────────────────────────────────────────────────────────────────── + +func getAzureRestConfig( + ctx context.Context, + wi *libsveltosv1beta1.WorkloadIdentityConfig, + caData []byte, + logger logr.Logger, +) (*rest.Config, time.Time, error) { + + azureCfg := wi.Azure + + // The Azure workload identity webhook injects AZURE_FEDERATED_TOKEN_FILE. + tokenFile := os.Getenv("AZURE_FEDERATED_TOKEN_FILE") + if tokenFile == "" { + return nil, time.Time{}, errors.New( + "AZURE_FEDERATED_TOKEN_FILE env var is not set; " + + "ensure the pod is configured with Azure Workload Identity") + } + + getAssertion := func(_ context.Context) (string, error) { + //nolint:gosec // path comes from the Azure workload identity webhook, not user input + b, err := os.ReadFile(tokenFile) + if err != nil { + return "", fmt.Errorf("failed to read federated token file %s: %w", tokenFile, err) + } + return string(b), nil + } + + cred, err := azidentity.NewClientAssertionCredential( + azureCfg.TenantID, azureCfg.ClientID, getAssertion, nil) + if err != nil { + return nil, time.Time{}, errors.Wrap(err, "failed to create Azure client assertion credential") + } + + tokenResp, err := cred.GetToken(ctx, policy.TokenRequestOptions{ + Scopes: []string{aksScope}, + }) + if err != nil { + return nil, time.Time{}, errors.Wrap(err, "failed to obtain Azure AAD token for AKS") + } + + logger.V(logs.LogDebug).Info("obtained Azure AAD token", "expiresAt", tokenResp.ExpiresOn) + return buildRestConfig(wi.Endpoint, tokenResp.Token, caData), tokenResp.ExpiresOn, nil +} diff --git a/lib/clusterproxy/workload_identity_test.go b/lib/clusterproxy/workload_identity_test.go new file mode 100644 index 00000000..3e5d4a95 --- /dev/null +++ b/lib/clusterproxy/workload_identity_test.go @@ -0,0 +1,136 @@ +/* +Copyright 2026. projectsveltos.io. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package clusterproxy_test + +import ( + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + libsveltosv1beta1 "github.com/projectsveltos/libsveltos/api/v1beta1" + "github.com/projectsveltos/libsveltos/lib/clusterproxy" +) + +const ( + wiTestCachedEndpoint = "https://cached.example.com" + wiTestCASecretName = "my-ca" +) + +var _ = Describe("WorkloadIdentity cache", func() { + const ( + ns = "default" + name = "test-cluster" + ) + + AfterEach(func() { + clusterproxy.EvictWorkloadIdentityCache(ns, name) + }) + + It("EvictWorkloadIdentityCache removes the entry", func() { + cfg := &rest.Config{Host: "https://example.com"} + clusterproxy.StoreTestWiCache(ns, name, cfg, time.Now().Add(time.Hour)) + + _, _, ok := clusterproxy.LoadTestWiCache(ns, name) + Expect(ok).To(BeTrue()) + + clusterproxy.EvictWorkloadIdentityCache(ns, name) + + _, _, ok = clusterproxy.LoadTestWiCache(ns, name) + Expect(ok).To(BeFalse()) + }) + + It("GetSveltosKubernetesRestConfig returns cached config when not near expiry", func() { + cached := &rest.Config{Host: wiTestCachedEndpoint} + clusterproxy.StoreTestWiCache(ns, name, cached, time.Now().Add(time.Hour)) + + wi := &libsveltosv1beta1.WorkloadIdentityConfig{ + Provider: libsveltosv1beta1.WorkloadIdentityProviderGCP, + Endpoint: wiTestCachedEndpoint, + GCP: &libsveltosv1beta1.GCPWorkloadIdentityConfig{ + ProjectID: "proj", + ClusterName: "cluster", + Location: "us-central1", + }, + } + sveltosCluster := &libsveltosv1beta1.SveltosCluster{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: ns, + Name: name, + }, + Spec: libsveltosv1beta1.SveltosClusterSpec{ + WorkloadIdentity: wi, + }, + } + + c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(sveltosCluster).Build() + logger := logr.Discard() + + got, err := clusterproxy.GetSveltosKubernetesRestConfig(ctx, logger, c, ns, name) + Expect(err).To(BeNil()) + Expect(got).To(Equal(cached)) + }) +}) + +var _ = Describe("WorkloadIdentity CA secret", func() { + It("returns CA bytes from a Secret via getCAData", func() { + caData := []byte("-----BEGIN CERTIFICATE-----\nfake\n-----END CERTIFICATE-----") + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: wiTestCASecretName, + }, + Data: map[string][]byte{ + "ca.crt": caData, + }, + } + + c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(secret).Build() + logger := logr.Discard() + + ref := &corev1.LocalObjectReference{Name: wiTestCASecretName} + got, err := clusterproxy.GetCADataForTest(ctx, c, "default", ref, logger) + Expect(err).To(BeNil()) + Expect(got).To(Equal(caData)) + }) + + It("returns nil when caSecretRef is nil", func() { + c := fake.NewClientBuilder().WithScheme(scheme).Build() + logger := logr.Discard() + + got, err := clusterproxy.GetCADataForTest(ctx, c, "default", nil, logger) + Expect(err).To(BeNil()) + Expect(got).To(BeNil()) + }) + + It("returns error when CA Secret does not exist", func() { + c := fake.NewClientBuilder().WithScheme(scheme).Build() + logger := logr.Discard() + + ref := &corev1.LocalObjectReference{Name: "missing-ca"} + _, err := clusterproxy.GetCADataForTest(ctx, c, "default", ref, logger) + Expect(err).ToNot(BeNil()) + Expect(err.Error()).To(ContainSubstring("CA secret")) + Expect(err.Error()).To(ContainSubstring("missing-ca")) + }) +}) diff --git a/lib/crd/sveltosclusters.go b/lib/crd/sveltosclusters.go index 8c466023..9ebce424 100644 --- a/lib/crd/sveltosclusters.go +++ b/lib/crd/sveltosclusters.go @@ -512,7 +512,142 @@ spec: required: - renewTokenRequestInterval type: object + workloadIdentity: + description: |- + WorkloadIdentity configures authentication to the managed cluster via the + cloud provider's workload identity mechanism. When set, Sveltos does not + read a kubeconfig Secret; instead it obtains short-lived credentials from + the cloud provider at runtime. + Mutually exclusive with KubeconfigName/KubeconfigKeyName. + properties: + aws: + description: |- + AWS contains configuration specific to AWS IRSA / EKS Pod Identity. + Required when Provider is AWS. + properties: + clusterName: + description: |- + ClusterName is the name of the EKS cluster. Required: it is embedded in + the bearer token header (x-k8s-aws-id) sent to the EKS API server. + minLength: 1 + type: string + region: + description: |- + Region is the AWS region of the EKS cluster. If empty, the AWS_REGION + environment variable injected by IRSA is used. + type: string + roleARN: + description: |- + RoleARN is the ARN of the IAM role Sveltos should assume before generating + the EKS bearer token. If empty, the pod's own IRSA credentials are used directly. + type: string + required: + - clusterName + type: object + azure: + description: |- + Azure contains configuration specific to Azure Workload Identity. + Required when Provider is Azure. + properties: + clientID: + description: |- + ClientID is the client ID of the managed identity or app registration + federated with the management cluster service account. + minLength: 1 + type: string + clusterName: + description: |- + ClusterName is the name of the AKS cluster. + Kept for future auto-discovery support. + type: string + resourceGroup: + description: |- + ResourceGroup is the Azure resource group containing the AKS cluster. + Kept for future auto-discovery support. + type: string + subscriptionID: + description: |- + SubscriptionID is the Azure subscription containing the AKS cluster. + Kept for future auto-discovery support. + type: string + tenantID: + description: TenantID is the Azure Active Directory tenant + ID. + minLength: 1 + type: string + required: + - clientID + - tenantID + type: object + caSecretRef: + description: |- + CASecretRef references a Secret in the management cluster containing the + CA certificate of the managed cluster's API server under the key "ca.crt". + If not set, the system certificate pool is used. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + endpoint: + description: Endpoint is the API server endpoint of the managed + cluster (e.g. https://…). + minLength: 1 + type: string + gcp: + description: |- + GCP contains configuration specific to GCP Workload Identity Federation. + Required when Provider is GCP. + properties: + clusterName: + description: ClusterName is the name of the GKE cluster. + minLength: 1 + type: string + location: + description: Location is the GCP region or zone of the GKE + cluster (e.g. "us-central1"). + minLength: 1 + type: string + projectID: + description: ProjectID is the GCP project ID that contains + the GKE cluster. + minLength: 1 + type: string + required: + - clusterName + - location + - projectID + type: object + provider: + description: Provider is the cloud provider implementing the workload + identity mechanism. + enum: + - AWS + - GCP + - Azure + type: string + required: + - endpoint + - provider + type: object + x-kubernetes-validations: + - message: aws must be set if and only if provider is AWS + rule: (self.provider == 'AWS') == has(self.aws) + - message: gcp must be set if and only if provider is GCP + rule: (self.provider == 'GCP') == has(self.gcp) + - message: azure must be set if and only if provider is Azure + rule: (self.provider == 'Azure') == has(self.azure) type: object + x-kubernetes-validations: + - message: 'workloadIdentity, kubeconfigName: conflict' + rule: '!has(self.workloadIdentity) || !has(self.kubeconfigName)' status: description: SveltosClusterStatus defines the status of SveltosCluster properties: diff --git a/manifests/apiextensions.k8s.io_v1_customresourcedefinition_sveltosclusters.lib.projectsveltos.io.yaml b/manifests/apiextensions.k8s.io_v1_customresourcedefinition_sveltosclusters.lib.projectsveltos.io.yaml index 657f89f9..2310332d 100644 --- a/manifests/apiextensions.k8s.io_v1_customresourcedefinition_sveltosclusters.lib.projectsveltos.io.yaml +++ b/manifests/apiextensions.k8s.io_v1_customresourcedefinition_sveltosclusters.lib.projectsveltos.io.yaml @@ -493,7 +493,142 @@ spec: required: - renewTokenRequestInterval type: object + workloadIdentity: + description: |- + WorkloadIdentity configures authentication to the managed cluster via the + cloud provider's workload identity mechanism. When set, Sveltos does not + read a kubeconfig Secret; instead it obtains short-lived credentials from + the cloud provider at runtime. + Mutually exclusive with KubeconfigName/KubeconfigKeyName. + properties: + aws: + description: |- + AWS contains configuration specific to AWS IRSA / EKS Pod Identity. + Required when Provider is AWS. + properties: + clusterName: + description: |- + ClusterName is the name of the EKS cluster. Required: it is embedded in + the bearer token header (x-k8s-aws-id) sent to the EKS API server. + minLength: 1 + type: string + region: + description: |- + Region is the AWS region of the EKS cluster. If empty, the AWS_REGION + environment variable injected by IRSA is used. + type: string + roleARN: + description: |- + RoleARN is the ARN of the IAM role Sveltos should assume before generating + the EKS bearer token. If empty, the pod's own IRSA credentials are used directly. + type: string + required: + - clusterName + type: object + azure: + description: |- + Azure contains configuration specific to Azure Workload Identity. + Required when Provider is Azure. + properties: + clientID: + description: |- + ClientID is the client ID of the managed identity or app registration + federated with the management cluster service account. + minLength: 1 + type: string + clusterName: + description: |- + ClusterName is the name of the AKS cluster. + Kept for future auto-discovery support. + type: string + resourceGroup: + description: |- + ResourceGroup is the Azure resource group containing the AKS cluster. + Kept for future auto-discovery support. + type: string + subscriptionID: + description: |- + SubscriptionID is the Azure subscription containing the AKS cluster. + Kept for future auto-discovery support. + type: string + tenantID: + description: TenantID is the Azure Active Directory tenant + ID. + minLength: 1 + type: string + required: + - clientID + - tenantID + type: object + caSecretRef: + description: |- + CASecretRef references a Secret in the management cluster containing the + CA certificate of the managed cluster's API server under the key "ca.crt". + If not set, the system certificate pool is used. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + endpoint: + description: Endpoint is the API server endpoint of the managed + cluster (e.g. https://…). + minLength: 1 + type: string + gcp: + description: |- + GCP contains configuration specific to GCP Workload Identity Federation. + Required when Provider is GCP. + properties: + clusterName: + description: ClusterName is the name of the GKE cluster. + minLength: 1 + type: string + location: + description: Location is the GCP region or zone of the GKE + cluster (e.g. "us-central1"). + minLength: 1 + type: string + projectID: + description: ProjectID is the GCP project ID that contains + the GKE cluster. + minLength: 1 + type: string + required: + - clusterName + - location + - projectID + type: object + provider: + description: Provider is the cloud provider implementing the workload + identity mechanism. + enum: + - AWS + - GCP + - Azure + type: string + required: + - endpoint + - provider + type: object + x-kubernetes-validations: + - message: aws must be set if and only if provider is AWS + rule: (self.provider == 'AWS') == has(self.aws) + - message: gcp must be set if and only if provider is GCP + rule: (self.provider == 'GCP') == has(self.gcp) + - message: azure must be set if and only if provider is Azure + rule: (self.provider == 'Azure') == has(self.azure) type: object + x-kubernetes-validations: + - message: 'workloadIdentity, kubeconfigName: conflict' + rule: '!has(self.workloadIdentity) || !has(self.kubeconfigName)' status: description: SveltosClusterStatus defines the status of SveltosCluster properties: