diff --git a/apis/v1alpha2/nginxproxy_types.go b/apis/v1alpha2/nginxproxy_types.go index d1845411ab..c27b9a7e2f 100644 --- a/apis/v1alpha2/nginxproxy_types.go +++ b/apis/v1alpha2/nginxproxy_types.go @@ -390,6 +390,11 @@ type KubernetesSpec struct { // Deployment is the configuration for the NGINX Deployment. type DeploymentSpec struct { + // Container defines container fields for the NGINX container. + // + // +optional + Container ContainerSpec `json:"container"` + // Number of desired Pods. // // +optional @@ -399,24 +404,19 @@ type DeploymentSpec struct { // // +optional Pod PodSpec `json:"pod"` +} +// DaemonSet is the configuration for the NGINX DaemonSet. +type DaemonSetSpec struct { // Container defines container fields for the NGINX container. // // +optional Container ContainerSpec `json:"container"` -} -// DaemonSet is the configuration for the NGINX DaemonSet. -type DaemonSetSpec struct { // Pod defines Pod-specific fields. // // +optional Pod PodSpec `json:"pod"` - - // Container defines container fields for the NGINX container. - // - // +optional - Container ContainerSpec `json:"container"` } // PodSpec defines Pod-specific fields. @@ -486,6 +486,11 @@ type ContainerSpec struct { // +optional Lifecycle *corev1.Lifecycle `json:"lifecycle,omitempty"` + // ReadinessProbe defines the readiness probe for the NGINX container. + // + // +optional + ReadinessProbe *ReadinessProbeSpec `json:"readinessProbe,omitempty"` + // HostPorts are the list of ports to expose on the host. // // +optional @@ -497,6 +502,26 @@ type ContainerSpec struct { VolumeMounts []corev1.VolumeMount `json:"volumeMounts,omitempty"` } +// ReadinessProbeSpec defines the configuration for the NGINX readiness probe. +type ReadinessProbeSpec struct { + // Port is the port on which the readiness endpoint is exposed. + // If not specified, the default port is 8081. + // + // +optional + // +kubebuilder:validation:Minimum=1 + // +kubebuilder:validation:Maximum=65535 + Port *int32 `json:"port,omitempty"` + + // InitialDelaySeconds is the number of seconds after the container has + // started before the readiness probe is initiated. + // If not specified, the default is 3 seconds. + // + // +optional + // +kubebuilder:validation:Minimum=0 + // +kubebuilder:validation:Maximum=3600 + InitialDelaySeconds *int32 `json:"initialDelaySeconds,omitempty"` +} + // Image is the NGINX image to use. type Image struct { // Repository is the image path. diff --git a/apis/v1alpha2/zz_generated.deepcopy.go b/apis/v1alpha2/zz_generated.deepcopy.go index 61eea4445a..cd5239296b 100644 --- a/apis/v1alpha2/zz_generated.deepcopy.go +++ b/apis/v1alpha2/zz_generated.deepcopy.go @@ -34,6 +34,11 @@ func (in *ContainerSpec) DeepCopyInto(out *ContainerSpec) { *out = new(v1.Lifecycle) (*in).DeepCopyInto(*out) } + if in.ReadinessProbe != nil { + in, out := &in.ReadinessProbe, &out.ReadinessProbe + *out = new(ReadinessProbeSpec) + (*in).DeepCopyInto(*out) + } if in.HostPorts != nil { in, out := &in.HostPorts, &out.HostPorts *out = make([]HostPort, len(*in)) @@ -61,8 +66,8 @@ func (in *ContainerSpec) DeepCopy() *ContainerSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DaemonSetSpec) DeepCopyInto(out *DaemonSetSpec) { *out = *in - in.Pod.DeepCopyInto(&out.Pod) in.Container.DeepCopyInto(&out.Container) + in.Pod.DeepCopyInto(&out.Pod) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DaemonSetSpec. @@ -78,13 +83,13 @@ func (in *DaemonSetSpec) DeepCopy() *DaemonSetSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DeploymentSpec) DeepCopyInto(out *DeploymentSpec) { *out = *in + in.Container.DeepCopyInto(&out.Container) if in.Replicas != nil { in, out := &in.Replicas, &out.Replicas *out = new(int32) **out = **in } in.Pod.DeepCopyInto(&out.Pod) - in.Container.DeepCopyInto(&out.Container) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DeploymentSpec. @@ -529,6 +534,31 @@ func (in *PodSpec) DeepCopy() *PodSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ReadinessProbeSpec) DeepCopyInto(out *ReadinessProbeSpec) { + *out = *in + if in.Port != nil { + in, out := &in.Port, &out.Port + *out = new(int32) + **out = **in + } + if in.InitialDelaySeconds != nil { + in, out := &in.InitialDelaySeconds, &out.InitialDelaySeconds + *out = new(int32) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ReadinessProbeSpec. +func (in *ReadinessProbeSpec) DeepCopy() *ReadinessProbeSpec { + if in == nil { + return nil + } + out := new(ReadinessProbeSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *RewriteClientIP) DeepCopyInto(out *RewriteClientIP) { *out = *in diff --git a/charts/nginx-gateway-fabric/README.md b/charts/nginx-gateway-fabric/README.md index 627335ec19..3bcafee80e 100644 --- a/charts/nginx-gateway-fabric/README.md +++ b/charts/nginx-gateway-fabric/README.md @@ -264,9 +264,9 @@ The following table lists the configurable parameters of the NGINX Gateway Fabri | `certGenerator.ttlSecondsAfterFinished` | How long to wait after the cert generator job has finished before it is removed by the job controller. | int | `30` | | `clusterDomain` | The DNS cluster domain of your Kubernetes cluster. | string | `"cluster.local"` | | `gateways` | A list of Gateway objects. View https://gateway-api.sigs.k8s.io/reference/spec/#gateway for full Gateway reference. | list | `[]` | -| `nginx` | The nginx section contains the configuration for all NGINX data plane deployments installed by the NGINX Gateway Fabric control plane. | object | `{"config":{},"container":{"hostPorts":[],"lifecycle":{},"resources":{},"volumeMounts":[]},"debug":false,"image":{"pullPolicy":"Always","repository":"ghcr.io/nginx/nginx-gateway-fabric/nginx","tag":"edge"},"imagePullSecret":"","imagePullSecrets":[],"kind":"deployment","plus":false,"pod":{},"replicas":1,"service":{"externalTrafficPolicy":"Local","loadBalancerClass":"","loadBalancerIP":"","loadBalancerSourceRanges":[],"nodePorts":[],"type":"LoadBalancer"},"usage":{"caSecretName":"","clientSSLSecretName":"","endpoint":"","resolver":"","secretName":"nplus-license","skipVerify":false}}` | +| `nginx` | The nginx section contains the configuration for all NGINX data plane deployments installed by the NGINX Gateway Fabric control plane. | object | `{"config":{},"container":{"hostPorts":[],"lifecycle":{},"readinessProbe":{},"resources":{},"volumeMounts":[]},"debug":false,"image":{"pullPolicy":"Always","repository":"ghcr.io/nginx/nginx-gateway-fabric/nginx","tag":"edge"},"imagePullSecret":"","imagePullSecrets":[],"kind":"deployment","plus":false,"pod":{},"replicas":1,"service":{"externalTrafficPolicy":"Local","loadBalancerClass":"","loadBalancerIP":"","loadBalancerSourceRanges":[],"nodePorts":[],"type":"LoadBalancer"},"usage":{"caSecretName":"","clientSSLSecretName":"","endpoint":"","resolver":"","secretName":"nplus-license","skipVerify":false}}` | | `nginx.config` | The configuration for the data plane that is contained in the NginxProxy resource. This is applied globally to all Gateways managed by this instance of NGINX Gateway Fabric. | object | `{}` | -| `nginx.container` | The container configuration for the NGINX container. This is applied globally to all Gateways managed by this instance of NGINX Gateway Fabric. | object | `{"hostPorts":[],"lifecycle":{},"resources":{},"volumeMounts":[]}` | +| `nginx.container` | The container configuration for the NGINX container. This is applied globally to all Gateways managed by this instance of NGINX Gateway Fabric. | object | `{"hostPorts":[],"lifecycle":{},"readinessProbe":{},"resources":{},"volumeMounts":[]}` | | `nginx.container.hostPorts` | A list of HostPorts to expose on the host. This configuration allows containers to bind to a specific port on the host node, enabling external network traffic to reach the container directly through the host's IP address and port. Use this option when you need to expose container ports on the host for direct access, such as for debugging, legacy integrations, or when NodePort/LoadBalancer services are not suitable. Note: Using hostPort may have security and scheduling implications, as it ties pods to specific nodes and ports. | list | `[]` | | `nginx.container.lifecycle` | The lifecycle of the NGINX container. | object | `{}` | | `nginx.container.resources` | The resource requirements of the NGINX container. | object | `{}` | diff --git a/charts/nginx-gateway-fabric/values.schema.json b/charts/nginx-gateway-fabric/values.schema.json index 9748af66b0..d750c0a42a 100644 --- a/charts/nginx-gateway-fabric/values.schema.json +++ b/charts/nginx-gateway-fabric/values.schema.json @@ -351,6 +351,12 @@ "title": "lifecycle", "type": "object" }, + "readinessProbe": { + "description": "# -- Defines the settings for the data plane readiness probe. This probe returns Ready when the NGINX data plane is ready to serve traffic.", + "required": [], + "title": "readinessProbe", + "type": "object" + }, "resources": { "description": "The resource requirements of the NGINX container.", "required": [], diff --git a/charts/nginx-gateway-fabric/values.yaml b/charts/nginx-gateway-fabric/values.yaml index cd55ef6eda..e9765ea511 100644 --- a/charts/nginx-gateway-fabric/values.yaml +++ b/charts/nginx-gateway-fabric/values.yaml @@ -441,6 +441,19 @@ nginx: # -- volumeMounts are the additional volume mounts for the NGINX container. volumeMounts: [] + ## -- Defines the settings for the data plane readiness probe. This probe returns Ready when the NGINX data plane is ready to serve traffic. + readinessProbe: {} + # @schema + # type: integer + # minimum: 1 + # maximum: 65535 + # @schema + # -- Port in which the readiness endpoint is exposed. + # port: 8081 + + # -- The number of seconds after the Pod has started before the readiness probes are initiated. + # initialDelaySeconds: 3 + # -- The service configuration for the NGINX data plane. This is applied globally to all Gateways managed by this # instance of NGINX Gateway Fabric. service: diff --git a/config/crd/bases/gateway.nginx.org_nginxproxies.yaml b/config/crd/bases/gateway.nginx.org_nginxproxies.yaml index 8afed8b803..3bcaf309db 100644 --- a/config/crd/bases/gateway.nginx.org_nginxproxies.yaml +++ b/config/crd/bases/gateway.nginx.org_nginxproxies.yaml @@ -362,6 +362,28 @@ spec: StopSignal can only be set for Pods with a non-empty .spec.os.name type: string type: object + readinessProbe: + description: ReadinessProbe defines the readiness probe + for the NGINX container. + properties: + initialDelaySeconds: + description: |- + InitialDelaySeconds is the number of seconds after the container has + started before the readiness probe is initiated. + If not specified, the default is 3 seconds. + format: int32 + maximum: 3600 + minimum: 0 + type: integer + port: + description: |- + Port is the port on which the readiness endpoint is exposed. + If not specified, the default port is 8081. + format: int32 + maximum: 65535 + minimum: 1 + type: integer + type: object resources: description: Resources describes the compute resource requirements. @@ -3773,6 +3795,28 @@ spec: StopSignal can only be set for Pods with a non-empty .spec.os.name type: string type: object + readinessProbe: + description: ReadinessProbe defines the readiness probe + for the NGINX container. + properties: + initialDelaySeconds: + description: |- + InitialDelaySeconds is the number of seconds after the container has + started before the readiness probe is initiated. + If not specified, the default is 3 seconds. + format: int32 + maximum: 3600 + minimum: 0 + type: integer + port: + description: |- + Port is the port on which the readiness endpoint is exposed. + If not specified, the default port is 8081. + format: int32 + maximum: 65535 + minimum: 1 + type: integer + type: object resources: description: Resources describes the compute resource requirements. diff --git a/deploy/crds.yaml b/deploy/crds.yaml index 016c3074d3..555934b682 100644 --- a/deploy/crds.yaml +++ b/deploy/crds.yaml @@ -947,6 +947,28 @@ spec: StopSignal can only be set for Pods with a non-empty .spec.os.name type: string type: object + readinessProbe: + description: ReadinessProbe defines the readiness probe + for the NGINX container. + properties: + initialDelaySeconds: + description: |- + InitialDelaySeconds is the number of seconds after the container has + started before the readiness probe is initiated. + If not specified, the default is 3 seconds. + format: int32 + maximum: 3600 + minimum: 0 + type: integer + port: + description: |- + Port is the port on which the readiness endpoint is exposed. + If not specified, the default port is 8081. + format: int32 + maximum: 65535 + minimum: 1 + type: integer + type: object resources: description: Resources describes the compute resource requirements. @@ -4358,6 +4380,28 @@ spec: StopSignal can only be set for Pods with a non-empty .spec.os.name type: string type: object + readinessProbe: + description: ReadinessProbe defines the readiness probe + for the NGINX container. + properties: + initialDelaySeconds: + description: |- + InitialDelaySeconds is the number of seconds after the container has + started before the readiness probe is initiated. + If not specified, the default is 3 seconds. + format: int32 + maximum: 3600 + minimum: 0 + type: integer + port: + description: |- + Port is the port on which the readiness endpoint is exposed. + If not specified, the default port is 8081. + format: int32 + maximum: 65535 + minimum: 1 + type: integer + type: object resources: description: Resources describes the compute resource requirements. diff --git a/internal/controller/nginx/config/base_http_config.go b/internal/controller/nginx/config/base_http_config.go index f808e86b3e..e4722c1b55 100644 --- a/internal/controller/nginx/config/base_http_config.go +++ b/internal/controller/nginx/config/base_http_config.go @@ -11,16 +11,18 @@ import ( var baseHTTPTemplate = gotemplate.Must(gotemplate.New("baseHttp").Parse(baseHTTPTemplateText)) type httpConfig struct { - Includes []shared.Include - HTTP2 bool + Includes []shared.Include + HTTP2 bool + NginxReadinessProbePort int32 } func executeBaseHTTPConfig(conf dataplane.Configuration) []executeResult { includes := createIncludesFromSnippets(conf.BaseHTTPConfig.Snippets) hc := httpConfig{ - HTTP2: conf.BaseHTTPConfig.HTTP2, - Includes: includes, + HTTP2: conf.BaseHTTPConfig.HTTP2, + Includes: includes, + NginxReadinessProbePort: conf.BaseHTTPConfig.NginxReadinessProbePort, } results := make([]executeResult, 0, len(includes)+1) diff --git a/internal/controller/nginx/config/base_http_config_template.go b/internal/controller/nginx/config/base_http_config_template.go index 5163904e26..169b876a30 100644 --- a/internal/controller/nginx/config/base_http_config_template.go +++ b/internal/controller/nginx/config/base_http_config_template.go @@ -24,6 +24,16 @@ map $request_uri $request_uri_path { "~^(?P[^?]*)(\?.*)?$" $path; } +# NGINX health check server block. +server { + listen {{ .NginxReadinessProbePort }}; + + location = /readyz { + access_log off; + return 200; + } +} + {{ range $i := .Includes -}} include {{ $i.Name }}; {{ end -}} diff --git a/internal/controller/nginx/config/base_http_config_test.go b/internal/controller/nginx/config/base_http_config_test.go index 31cc7aff52..4932fdad89 100644 --- a/internal/controller/nginx/config/base_http_config_test.go +++ b/internal/controller/nginx/config/base_http_config_test.go @@ -107,3 +107,60 @@ func TestExecuteBaseHttp_Snippets(t *testing.T) { snippet2IncludeRes := string(res[2].data) g.Expect(snippet2IncludeRes).To(ContainSubstring("contents2")) } + +func TestExecuteBaseHttp_NginxReadinessProbePort(t *testing.T) { + t.Parallel() + + defaultConfig := dataplane.Configuration{ + BaseHTTPConfig: dataplane.BaseHTTPConfig{ + NginxReadinessProbePort: dataplane.DefaultNginxReadinessProbePort, + }, + } + + customConfig := dataplane.Configuration{ + BaseHTTPConfig: dataplane.BaseHTTPConfig{ + NginxReadinessProbePort: 9090, + }, + } + + tests := []struct { + name string + expectedPort string + expectedListen string + conf dataplane.Configuration + }{ + { + name: "default nginx readiness probe port", + conf: defaultConfig, + expectedPort: "8081", + expectedListen: "listen 8081;", + }, + { + name: "custom nginx readiness probe 9090", + conf: customConfig, + expectedPort: "9090", + expectedListen: "listen 9090;", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + res := executeBaseHTTPConfig(test.conf) + g.Expect(res).To(HaveLen(1)) + + httpConfig := string(res[0].data) + + // check that the listen directive contains the expected port + g.Expect(httpConfig).To(ContainSubstring(test.expectedListen)) + + // check that the health check server block is present + g.Expect(httpConfig).To(ContainSubstring("server {")) + g.Expect(httpConfig).To(ContainSubstring("access_log off;")) + g.Expect(httpConfig).To(ContainSubstring("location = /readyz {")) + g.Expect(httpConfig).To(ContainSubstring("return 200;")) + }) + } +} diff --git a/internal/controller/provisioner/objects.go b/internal/controller/provisioner/objects.go index 84cdefa02f..d24a8971b8 100644 --- a/internal/controller/provisioner/objects.go +++ b/internal/controller/provisioner/objects.go @@ -34,9 +34,10 @@ const ( defaultServiceType = corev1.ServiceTypeLoadBalancer defaultServicePolicy = corev1.ServiceExternalTrafficPolicyLocal - defaultNginxImagePath = "ghcr.io/nginx/nginx-gateway-fabric/nginx" - defaultNginxPlusImagePath = "private-registry.nginx.com/nginx-gateway-fabric/nginx-plus" - defaultImagePullPolicy = corev1.PullIfNotPresent + defaultNginxImagePath = "ghcr.io/nginx/nginx-gateway-fabric/nginx" + defaultNginxPlusImagePath = "private-registry.nginx.com/nginx-gateway-fabric/nginx-plus" + defaultImagePullPolicy = corev1.PullIfNotPresent + defaultInitialDelaySeconds = int32(3) ) var emptyDirVolumeSource = corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}} @@ -623,6 +624,7 @@ func (p *NginxProvisioner) buildNginxPodTemplateSpec( Image: image, ImagePullPolicy: pullPolicy, Ports: containerPorts, + ReadinessProbe: p.buildReadinessProbe(nProxyCfg), SecurityContext: &corev1.SecurityContext{ AllowPrivilegeEscalation: helpers.GetPointer(false), Capabilities: &corev1.Capabilities{ @@ -1037,3 +1039,39 @@ func (p *NginxProvisioner) buildNginxResourceObjectsForDeletion(deploymentNSName return objects } + +// buildReadinessProbe creates a readiness probe configuration for the NGINX container. +func (p *NginxProvisioner) buildReadinessProbe(nProxyCfg *graph.EffectiveNginxProxy) *corev1.Probe { + probe := &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/readyz", + Port: intstr.FromInt32(dataplane.DefaultNginxReadinessProbePort), + }, + }, + InitialDelaySeconds: defaultInitialDelaySeconds, + } + + var containerSpec *ngfAPIv1alpha2.ContainerSpec + if nProxyCfg != nil && nProxyCfg.Kubernetes != nil { + if nProxyCfg.Kubernetes.Deployment != nil { + containerSpec = &nProxyCfg.Kubernetes.Deployment.Container + } else if nProxyCfg.Kubernetes.DaemonSet != nil { + containerSpec = &nProxyCfg.Kubernetes.DaemonSet.Container + } + } + + if containerSpec == nil || containerSpec.ReadinessProbe == nil { + return probe + } + + if containerSpec.ReadinessProbe.Port != nil { + probe.HTTPGet.Port = intstr.FromInt32(*containerSpec.ReadinessProbe.Port) + } + + if containerSpec.ReadinessProbe.InitialDelaySeconds != nil { + probe.InitialDelaySeconds = *containerSpec.ReadinessProbe.InitialDelaySeconds + } + + return probe +} diff --git a/internal/controller/provisioner/objects_test.go b/internal/controller/provisioner/objects_test.go index 608edf8b4d..6486e61bac 100644 --- a/internal/controller/provisioner/objects_test.go +++ b/internal/controller/provisioner/objects_test.go @@ -18,6 +18,7 @@ import ( ngfAPIv1alpha2 "github.com/nginx/nginx-gateway-fabric/apis/v1alpha2" "github.com/nginx/nginx-gateway-fabric/internal/controller/config" + "github.com/nginx/nginx-gateway-fabric/internal/controller/state/dataplane" "github.com/nginx/nginx-gateway-fabric/internal/controller/state/graph" "github.com/nginx/nginx-gateway-fabric/internal/framework/controller" "github.com/nginx/nginx-gateway-fabric/internal/framework/helpers" @@ -298,6 +299,10 @@ func TestBuildNginxResourceObjects_NginxProxyConfig(t *testing.T) { corev1.ResourceCPU: resource.Quantity{Format: "100m"}, }, }, + ReadinessProbe: &ngfAPIv1alpha2.ReadinessProbeSpec{ + Port: helpers.GetPointer[int32](9091), + InitialDelaySeconds: helpers.GetPointer[int32](5), + }, HostPorts: []ngfAPIv1alpha2.HostPort{{ContainerPort: int32(8443), Port: int32(8443)}}, }, }, @@ -356,6 +361,11 @@ func TestBuildNginxResourceObjects_NginxProxyConfig(t *testing.T) { Name: "port-8443", HostPort: 8443, })) + + g.Expect(container.ReadinessProbe).ToNot(BeNil()) + g.Expect(container.ReadinessProbe.HTTPGet.Path).To(Equal("/readyz")) + g.Expect(container.ReadinessProbe.HTTPGet.Port).To(Equal(intstr.FromInt(9091))) + g.Expect(container.ReadinessProbe.InitialDelaySeconds).To(Equal(int32(5))) } func TestBuildNginxResourceObjects_Plus(t *testing.T) { @@ -1058,3 +1068,97 @@ func TestBuildNginxConfigMaps_WorkerConnections(t *testing.T) { g.Expect(ok).To(BeTrue()) g.Expect(bootstrapCM.Data["main.conf"]).To(ContainSubstring("worker_connections 2048;")) } + +func TestBuildReadinessProbe(t *testing.T) { + t.Parallel() + + defaultProbe := &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/readyz", + Port: intstr.FromInt32(dataplane.DefaultNginxReadinessProbePort), + }, + }, + InitialDelaySeconds: 3, + } + + provisioner := &NginxProvisioner{} + + tests := []struct { + nProxyCfg *graph.EffectiveNginxProxy + expected *corev1.Probe + name string + }{ + { + name: "nginx proxy config is nil, default probe is returned", + nProxyCfg: nil, + expected: defaultProbe, + }, + { + name: "deployment is nil, default probe is returned", + nProxyCfg: &graph.EffectiveNginxProxy{ + Kubernetes: &ngfAPIv1alpha2.KubernetesSpec{ + Deployment: nil, + }, + }, + expected: defaultProbe, + }, + { + name: "container is nil, default probe is returned", + nProxyCfg: &graph.EffectiveNginxProxy{ + Kubernetes: &ngfAPIv1alpha2.KubernetesSpec{ + Deployment: &ngfAPIv1alpha2.DeploymentSpec{ + Container: ngfAPIv1alpha2.ContainerSpec{}, + }, + }, + }, + expected: defaultProbe, + }, + { + name: "readinessProbe is nil, default probe is returned", + nProxyCfg: &graph.EffectiveNginxProxy{ + Kubernetes: &ngfAPIv1alpha2.KubernetesSpec{ + Deployment: &ngfAPIv1alpha2.DeploymentSpec{ + Container: ngfAPIv1alpha2.ContainerSpec{ + ReadinessProbe: nil, + }, + }, + }, + }, + expected: defaultProbe, + }, + { + name: "port & initialDelaySeconds is set in readinessProbe, custom probe is returned", + nProxyCfg: &graph.EffectiveNginxProxy{ + Kubernetes: &ngfAPIv1alpha2.KubernetesSpec{ + Deployment: &ngfAPIv1alpha2.DeploymentSpec{ + Container: ngfAPIv1alpha2.ContainerSpec{ + ReadinessProbe: &ngfAPIv1alpha2.ReadinessProbeSpec{ + Port: helpers.GetPointer[int32](9091), + InitialDelaySeconds: helpers.GetPointer[int32](10), + }, + }, + }, + }, + }, + expected: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/readyz", + Port: intstr.FromInt32(9091), + }, + }, + InitialDelaySeconds: 10, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + g := NewWithT(t) + probe := provisioner.buildReadinessProbe(tt.nProxyCfg) + g.Expect(probe).To(Equal(tt.expected)) + }) + } +} diff --git a/internal/controller/state/dataplane/configuration.go b/internal/controller/state/dataplane/configuration.go index ecdba8d3e8..24ff197e55 100644 --- a/internal/controller/state/dataplane/configuration.go +++ b/internal/controller/state/dataplane/configuration.go @@ -22,10 +22,11 @@ import ( ) const ( - wildcardHostname = "~^" - alpineSSLRootCAPath = "/etc/ssl/cert.pem" - defaultErrorLogLevel = "info" - DefaultWorkerConnections = int32(1024) + wildcardHostname = "~^" + alpineSSLRootCAPath = "/etc/ssl/cert.pem" + defaultErrorLogLevel = "info" + DefaultWorkerConnections = int32(1024) + DefaultNginxReadinessProbePort = int32(8081) ) // BuildConfiguration builds the Configuration from the Graph. @@ -974,9 +975,10 @@ func buildBaseHTTPConfig( ) BaseHTTPConfig { baseConfig := BaseHTTPConfig{ // HTTP2 should be enabled by default - HTTP2: true, - IPFamily: Dual, - Snippets: buildSnippetsForContext(gatewaySnippetsFilters, ngfAPIv1alpha1.NginxContextHTTP), + HTTP2: true, + IPFamily: Dual, + Snippets: buildSnippetsForContext(gatewaySnippetsFilters, ngfAPIv1alpha1.NginxContextHTTP), + NginxReadinessProbePort: DefaultNginxReadinessProbePort, } // safe to access EffectiveNginxProxy since we only call this function when the Gateway is not nil. @@ -998,28 +1000,47 @@ func buildBaseHTTPConfig( } } - if np.RewriteClientIP != nil { - if np.RewriteClientIP.Mode != nil { - switch *np.RewriteClientIP.Mode { + baseConfig.RewriteClientIPSettings = buildRewriteClientIPConfig(np.RewriteClientIP) + + if np.Kubernetes != nil { + var containerSpec *ngfAPIv1alpha2.ContainerSpec + if np.Kubernetes.Deployment != nil { + containerSpec = &np.Kubernetes.Deployment.Container + } else if np.Kubernetes.DaemonSet != nil { + containerSpec = &np.Kubernetes.DaemonSet.Container + } + if containerSpec != nil && containerSpec.ReadinessProbe != nil && containerSpec.ReadinessProbe.Port != nil { + baseConfig.NginxReadinessProbePort = *containerSpec.ReadinessProbe.Port + } + } + + return baseConfig +} + +func buildRewriteClientIPConfig(rewriteClientIPConfig *ngfAPIv1alpha2.RewriteClientIP) RewriteClientIPSettings { + var rewriteClientIPSettings RewriteClientIPSettings + if rewriteClientIPConfig != nil { + if rewriteClientIPConfig.Mode != nil { + switch *rewriteClientIPConfig.Mode { case ngfAPIv1alpha2.RewriteClientIPModeProxyProtocol: - baseConfig.RewriteClientIPSettings.Mode = RewriteIPModeProxyProtocol + rewriteClientIPSettings.Mode = RewriteIPModeProxyProtocol case ngfAPIv1alpha2.RewriteClientIPModeXForwardedFor: - baseConfig.RewriteClientIPSettings.Mode = RewriteIPModeXForwardedFor + rewriteClientIPSettings.Mode = RewriteIPModeXForwardedFor } } - if len(np.RewriteClientIP.TrustedAddresses) > 0 { - baseConfig.RewriteClientIPSettings.TrustedAddresses = convertAddresses( - np.RewriteClientIP.TrustedAddresses, + if len(rewriteClientIPConfig.TrustedAddresses) > 0 { + rewriteClientIPSettings.TrustedAddresses = convertAddresses( + rewriteClientIPConfig.TrustedAddresses, ) } - if np.RewriteClientIP.SetIPRecursively != nil { - baseConfig.RewriteClientIPSettings.IPRecursive = *np.RewriteClientIP.SetIPRecursively + if rewriteClientIPConfig.SetIPRecursively != nil { + rewriteClientIPSettings.IPRecursive = *rewriteClientIPConfig.SetIPRecursively } } - return baseConfig + return rewriteClientIPSettings } func createSnippetName(nc ngfAPIv1alpha1.NginxContext, nsname types.NamespacedName) string { diff --git a/internal/controller/state/dataplane/configuration_test.go b/internal/controller/state/dataplane/configuration_test.go index 07b7874937..b374493710 100644 --- a/internal/controller/state/dataplane/configuration_test.go +++ b/internal/controller/state/dataplane/configuration_test.go @@ -31,6 +31,12 @@ import ( "github.com/nginx/nginx-gateway-fabric/internal/framework/kinds" ) +var defaultBaseHTTPConfig = BaseHTTPConfig{ + NginxReadinessProbePort: DefaultNginxReadinessProbePort, + HTTP2: true, + IPFamily: Dual, +} + func getNormalBackendRef() graph.BackendRef { return graph.BackendRef{ SvcNsName: types.NamespacedName{Name: "foo", Namespace: "test"}, @@ -42,7 +48,7 @@ func getNormalBackendRef() graph.BackendRef { func getExpectedConfiguration() Configuration { return Configuration{ - BaseHTTPConfig: BaseHTTPConfig{HTTP2: true, IPFamily: Dual}, + BaseHTTPConfig: defaultBaseHTTPConfig, HTTPServers: []VirtualServer{ { IsDefault: true, @@ -2227,7 +2233,11 @@ func TestBuildConfiguration(t *testing.T) { Ratios: []Ratio{}, SpanAttributes: []SpanAttribute{}, } - conf.BaseHTTPConfig = BaseHTTPConfig{HTTP2: false, IPFamily: Dual} + conf.BaseHTTPConfig = BaseHTTPConfig{ + HTTP2: false, + IPFamily: Dual, + NginxReadinessProbePort: DefaultNginxReadinessProbePort, + } return conf }), msg: "EffectiveNginxProxy with tracing config and http2 disabled", @@ -2350,7 +2360,11 @@ func TestBuildConfiguration(t *testing.T) { expConf: getModifiedExpectedConfiguration(func(conf Configuration) Configuration { conf.SSLServers = []VirtualServer{} conf.SSLKeyPairs = map[SSLKeyPairID]SSLKeyPair{} - conf.BaseHTTPConfig = BaseHTTPConfig{HTTP2: true, IPFamily: IPv4} + conf.BaseHTTPConfig = BaseHTTPConfig{ + HTTP2: true, + IPFamily: IPv4, + NginxReadinessProbePort: DefaultNginxReadinessProbePort, + } return conf }), msg: "GatewayClass has NginxProxy with IPv4 IPFamily and no routes", @@ -2375,7 +2389,11 @@ func TestBuildConfiguration(t *testing.T) { expConf: getModifiedExpectedConfiguration(func(conf Configuration) Configuration { conf.SSLServers = []VirtualServer{} conf.SSLKeyPairs = map[SSLKeyPairID]SSLKeyPair{} - conf.BaseHTTPConfig = BaseHTTPConfig{HTTP2: true, IPFamily: IPv6} + conf.BaseHTTPConfig = BaseHTTPConfig{ + HTTP2: true, + IPFamily: IPv6, + NginxReadinessProbePort: DefaultNginxReadinessProbePort, + } return conf }), msg: "GatewayClass has NginxProxy with IPv6 IPFamily and no routes", @@ -2419,6 +2437,7 @@ func TestBuildConfiguration(t *testing.T) { TrustedAddresses: []string{"1.1.1.1/32"}, Mode: RewriteIPModeProxyProtocol, }, + NginxReadinessProbePort: DefaultNginxReadinessProbePort, } return conf }), @@ -4941,3 +4960,110 @@ func TestBuildWorkerConnections(t *testing.T) { }) } } + +func TestBuildBaseHTTPConfig_ReadinessProbe(t *testing.T) { + t.Parallel() + test := []struct { + msg string + gateway *graph.Gateway + expected BaseHTTPConfig + }{ + { + msg: "nginx proxy config is nil", + gateway: &graph.Gateway{ + EffectiveNginxProxy: &graph.EffectiveNginxProxy{}, + }, + expected: defaultBaseHTTPConfig, + }, + { + msg: "kubernetes spec is nil", + gateway: &graph.Gateway{ + EffectiveNginxProxy: &graph.EffectiveNginxProxy{ + Kubernetes: &ngfAPIv1alpha2.KubernetesSpec{}, + }, + }, + expected: defaultBaseHTTPConfig, + }, + { + msg: "readiness probe spec is nil", + gateway: &graph.Gateway{ + EffectiveNginxProxy: &graph.EffectiveNginxProxy{ + Kubernetes: &ngfAPIv1alpha2.KubernetesSpec{ + Deployment: &ngfAPIv1alpha2.DeploymentSpec{ + Container: ngfAPIv1alpha2.ContainerSpec{ + ReadinessProbe: nil, + }, + }, + }, + }, + }, + expected: defaultBaseHTTPConfig, + }, + { + msg: "readiness probe spec is empty", + gateway: &graph.Gateway{ + EffectiveNginxProxy: &graph.EffectiveNginxProxy{ + Kubernetes: &ngfAPIv1alpha2.KubernetesSpec{ + Deployment: &ngfAPIv1alpha2.DeploymentSpec{ + Container: ngfAPIv1alpha2.ContainerSpec{ + ReadinessProbe: &ngfAPIv1alpha2.ReadinessProbeSpec{}, + }, + }, + }, + }, + }, + expected: defaultBaseHTTPConfig, + }, + { + msg: "readiness probe is configured for deployment kind", + gateway: &graph.Gateway{ + EffectiveNginxProxy: &graph.EffectiveNginxProxy{ + Kubernetes: &ngfAPIv1alpha2.KubernetesSpec{ + Deployment: &ngfAPIv1alpha2.DeploymentSpec{ + Container: ngfAPIv1alpha2.ContainerSpec{ + ReadinessProbe: &ngfAPIv1alpha2.ReadinessProbeSpec{ + Port: helpers.GetPointer(int32(7020)), + }, + }, + }, + }, + }, + }, + expected: BaseHTTPConfig{ + NginxReadinessProbePort: int32(7020), + IPFamily: Dual, + HTTP2: true, + }, + }, + { + msg: "readiness probe is configured for daemonset kind", + gateway: &graph.Gateway{ + EffectiveNginxProxy: &graph.EffectiveNginxProxy{ + Kubernetes: &ngfAPIv1alpha2.KubernetesSpec{ + DaemonSet: &ngfAPIv1alpha2.DaemonSetSpec{ + Container: ngfAPIv1alpha2.ContainerSpec{ + ReadinessProbe: &ngfAPIv1alpha2.ReadinessProbeSpec{ + Port: helpers.GetPointer(int32(8881)), + }, + }, + }, + }, + }, + }, + expected: BaseHTTPConfig{ + NginxReadinessProbePort: int32(8881), + IPFamily: Dual, + HTTP2: true, + }, + }, + } + + for _, tc := range test { + t.Run(tc.msg, func(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + g.Expect(buildBaseHTTPConfig(tc.gateway, nil)).To(Equal(tc.expected)) + }) + } +} diff --git a/internal/controller/state/dataplane/types.go b/internal/controller/state/dataplane/types.go index 0cfe48beb0..646eaa924c 100644 --- a/internal/controller/state/dataplane/types.go +++ b/internal/controller/state/dataplane/types.go @@ -374,6 +374,8 @@ type BaseHTTPConfig struct { RewriteClientIPSettings RewriteClientIPSettings // HTTP2 specifies whether http2 should be enabled for all servers. HTTP2 bool + // NginxReadinessProbePort is the port on which the health check endpoint for NGINX is exposed. + NginxReadinessProbePort int32 } // Snippet is a snippet of configuration.