From 744fe00f9a353eab1a41c0edc8f8f173a3c97167 Mon Sep 17 00:00:00 2001 From: Woojin Na Date: Wed, 11 Jun 2025 18:18:28 +0900 Subject: [PATCH 01/13] Add control/data plane HPA --- apis/v1alpha2/nginxproxy_types.go | 41 ++ apis/v1alpha2/zz_generated.deepcopy.go | 48 ++ charts/nginx-gateway-fabric/README.md | 6 +- .../templates/clusterrole.yaml | 2 + .../nginx-gateway-fabric/templates/hpa.yaml | 46 ++ .../templates/nginxproxy.yaml | 20 + .../nginx-gateway-fabric/values.schema.json | 115 ++- charts/nginx-gateway-fabric/values.yaml | 66 +- .../bases/gateway.nginx.org_nginxproxies.yaml | 661 ++++++++++++++++++ deploy/azure/deploy.yaml | 8 + deploy/crds.yaml | 661 ++++++++++++++++++ deploy/default/deploy.yaml | 8 + deploy/experimental-nginx-plus/deploy.yaml | 8 + deploy/experimental/deploy.yaml | 8 + deploy/nginx-plus/deploy.yaml | 8 + deploy/nodeport/deploy.yaml | 8 + deploy/openshift/deploy.yaml | 8 + .../snippets-filters-nginx-plus/deploy.yaml | 8 + deploy/snippets-filters/deploy.yaml | 8 + internal/controller/manager.go | 2 + internal/controller/provisioner/handler.go | 3 +- internal/controller/provisioner/objects.go | 184 ++++- .../controller/provisioner/objects_test.go | 46 +- .../controller/provisioner/provisioner.go | 1 + .../provisioner/provisioner_test.go | 2 + 25 files changed, 1954 insertions(+), 22 deletions(-) create mode 100644 charts/nginx-gateway-fabric/templates/hpa.yaml diff --git a/apis/v1alpha2/nginxproxy_types.go b/apis/v1alpha2/nginxproxy_types.go index 9ebffa7fac..09a91f6bca 100644 --- a/apis/v1alpha2/nginxproxy_types.go +++ b/apis/v1alpha2/nginxproxy_types.go @@ -1,6 +1,7 @@ package v1alpha2 import ( + autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -431,6 +432,11 @@ type DeploymentSpec struct { // +optional Replicas *int32 `json:"replicas,omitempty"` + // Horizontal Pod Autoscaling. + // + // +optional + Autoscaling HPASpec `json:"autoscaling"` + // Pod defines Pod-specific fields. // // +optional @@ -460,6 +466,41 @@ type DaemonSetSpec struct { Patches []Patch `json:"patches,omitempty"` } +type HPASpec struct { + // Behavior configures the scaling behavior of the target + // in both Up and Down directions (scaleUp and scaleDown fields respectively). + // If not set, the default HPAScalingRules for scale up and scale down are used. + // + // +optional + Behavior *autoscalingv2.HorizontalPodAutoscalerBehavior `json:"behavior,omitempty"` + + // AutoscalingTemplate configures the additional scaling option. + // + // +optional + AutoscalingTemplate *[]autoscalingv2.MetricSpec `json:"autoscalingTemplate,omitempty"` + + // Target cpu utilization percentage of HPA. + // + // +optional + TargetCPUUtilizationPercentage *int32 `json:"targetCPUUtilizationPercentage,omitempty"` + + // Target memory utilization percentage of HPA. + // + // +optional + TargetMemoryUtilizationPercentage *int32 `json:"targetMemoryUtilizationPercentage,omitempty"` + + // Minimum number of replicas. + // + // +optional + MinReplicas *int32 `json:"minReplicas,omitempty"` + + // Maximum number of replicas. + MaxReplicas int32 `json:"maxReplicas"` + + // Enable or disable Horizontal Pod Autoscaler + Enabled bool `json:"enabled"` +} + // PodSpec defines Pod-specific fields. type PodSpec struct { // TerminationGracePeriodSeconds is the optional duration in seconds the pod needs to terminate gracefully. diff --git a/apis/v1alpha2/zz_generated.deepcopy.go b/apis/v1alpha2/zz_generated.deepcopy.go index 1e4f5955c7..3fb3fa78fb 100644 --- a/apis/v1alpha2/zz_generated.deepcopy.go +++ b/apis/v1alpha2/zz_generated.deepcopy.go @@ -6,6 +6,7 @@ package v1alpha2 import ( "github.com/nginx/nginx-gateway-fabric/apis/v1alpha1" + "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/runtime" @@ -97,6 +98,7 @@ func (in *DeploymentSpec) DeepCopyInto(out *DeploymentSpec) { *out = new(int32) **out = **in } + in.Autoscaling.DeepCopyInto(&out.Autoscaling) in.Pod.DeepCopyInto(&out.Pod) if in.Patches != nil { in, out := &in.Patches, &out.Patches @@ -117,6 +119,52 @@ func (in *DeploymentSpec) DeepCopy() *DeploymentSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *HPASpec) DeepCopyInto(out *HPASpec) { + *out = *in + if in.Behavior != nil { + in, out := &in.Behavior, &out.Behavior + *out = new(v2.HorizontalPodAutoscalerBehavior) + (*in).DeepCopyInto(*out) + } + if in.AutoscalingTemplate != nil { + in, out := &in.AutoscalingTemplate, &out.AutoscalingTemplate + *out = new([]v2.MetricSpec) + if **in != nil { + in, out := *in, *out + *out = make([]v2.MetricSpec, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + } + if in.TargetCPUUtilizationPercentage != nil { + in, out := &in.TargetCPUUtilizationPercentage, &out.TargetCPUUtilizationPercentage + *out = new(int32) + **out = **in + } + if in.TargetMemoryUtilizationPercentage != nil { + in, out := &in.TargetMemoryUtilizationPercentage, &out.TargetMemoryUtilizationPercentage + *out = new(int32) + **out = **in + } + if in.MinReplicas != nil { + in, out := &in.MinReplicas, &out.MinReplicas + *out = new(int32) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HPASpec. +func (in *HPASpec) DeepCopy() *HPASpec { + if in == nil { + return nil + } + out := new(HPASpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *HostPort) DeepCopyInto(out *HostPort) { *out = *in diff --git a/charts/nginx-gateway-fabric/README.md b/charts/nginx-gateway-fabric/README.md index 3bcafee80e..fc4e5eccd6 100644 --- a/charts/nginx-gateway-fabric/README.md +++ b/charts/nginx-gateway-fabric/README.md @@ -264,12 +264,12 @@ The following table lists the configurable parameters of the NGINX Gateway Fabri | `certGenerator.ttlSecondsAfterFinished` | How long to wait after the cert generator job has finished before it is removed by the job controller. | int | `30` | | `clusterDomain` | The DNS cluster domain of your Kubernetes cluster. | string | `"cluster.local"` | | `gateways` | A list of Gateway objects. View https://gateway-api.sigs.k8s.io/reference/spec/#gateway for full Gateway reference. | list | `[]` | -| `nginx` | The nginx section contains the configuration for all NGINX data plane deployments installed by the NGINX Gateway Fabric control plane. | object | `{"config":{},"container":{"hostPorts":[],"lifecycle":{},"readinessProbe":{},"resources":{},"volumeMounts":[]},"debug":false,"image":{"pullPolicy":"Always","repository":"ghcr.io/nginx/nginx-gateway-fabric/nginx","tag":"edge"},"imagePullSecret":"","imagePullSecrets":[],"kind":"deployment","plus":false,"pod":{},"replicas":1,"service":{"externalTrafficPolicy":"Local","loadBalancerClass":"","loadBalancerIP":"","loadBalancerSourceRanges":[],"nodePorts":[],"type":"LoadBalancer"},"usage":{"caSecretName":"","clientSSLSecretName":"","endpoint":"","resolver":"","secretName":"nplus-license","skipVerify":false}}` | +| `nginx` | The nginx section contains the configuration for all NGINX data plane deployments installed by the NGINX Gateway Fabric control plane. | object | `{"autoscaling":{"behavior":{},"enabled":true,"hpaAnnotations":{},"maxReplicas":11,"minReplicas":1,"targetCPUUtilizationPercentage":50,"targetMemoryUtilizationPercentage":50},"autoscalingTemplate":[],"config":{},"container":{"hostPorts":[],"lifecycle":{},"readinessProbe":{},"resources":{},"volumeMounts":[]},"debug":false,"image":{"pullPolicy":"Always","repository":"ghcr.io/nginx/nginx-gateway-fabric/nginx","tag":"edge"},"imagePullSecret":"","imagePullSecrets":[],"kind":"deployment","plus":false,"pod":{},"replicas":1,"service":{"externalTrafficPolicy":"Local","loadBalancerClass":"","loadBalancerIP":"","loadBalancerSourceRanges":[],"nodePorts":[],"type":"LoadBalancer"},"usage":{"caSecretName":"","clientSSLSecretName":"","endpoint":"","resolver":"","secretName":"nplus-license","skipVerify":false}}` | | `nginx.config` | The configuration for the data plane that is contained in the NginxProxy resource. This is applied globally to all Gateways managed by this instance of NGINX Gateway Fabric. | object | `{}` | | `nginx.container` | The container configuration for the NGINX container. This is applied globally to all Gateways managed by this instance of NGINX Gateway Fabric. | object | `{"hostPorts":[],"lifecycle":{},"readinessProbe":{},"resources":{},"volumeMounts":[]}` | | `nginx.container.hostPorts` | A list of HostPorts to expose on the host. This configuration allows containers to bind to a specific port on the host node, enabling external network traffic to reach the container directly through the host's IP address and port. Use this option when you need to expose container ports on the host for direct access, such as for debugging, legacy integrations, or when NodePort/LoadBalancer services are not suitable. Note: Using hostPort may have security and scheduling implications, as it ties pods to specific nodes and ports. | list | `[]` | | `nginx.container.lifecycle` | The lifecycle of the NGINX container. | object | `{}` | -| `nginx.container.resources` | The resource requirements of the NGINX container. | object | `{}` | +| `nginx.container.resources` | The resource requirements of the NGINX container. You should be set this value, If you want to use dataplane Autoscaling(HPA). | object | `{}` | | `nginx.container.volumeMounts` | volumeMounts are the additional volume mounts for the NGINX container. | list | `[]` | | `nginx.debug` | Enable debugging for NGINX. Uses the nginx-debug binary. The NGINX error log level should be set to debug in the NginxProxy resource. | bool | `false` | | `nginx.image.repository` | The NGINX image to use. | string | `"ghcr.io/nginx/nginx-gateway-fabric/nginx"` | @@ -292,7 +292,7 @@ The following table lists the configurable parameters of the NGINX Gateway Fabri | `nginx.usage.resolver` | The nameserver used to resolve the NGINX Plus usage reporting endpoint. Used with NGINX Instance Manager. | string | `""` | | `nginx.usage.secretName` | The name of the Secret containing the JWT for NGINX Plus usage reporting. Must exist in the same namespace that the NGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway). | string | `"nplus-license"` | | `nginx.usage.skipVerify` | Disable client verification of the NGINX Plus usage reporting server certificate. | bool | `false` | -| `nginxGateway` | The nginxGateway section contains configuration for the NGINX Gateway Fabric control plane deployment. | object | `{"affinity":{},"config":{"logging":{"level":"info"}},"configAnnotations":{},"extraVolumeMounts":[],"extraVolumes":[],"gatewayClassAnnotations":{},"gatewayClassName":"nginx","gatewayControllerName":"gateway.nginx.org/nginx-gateway-controller","gwAPIExperimentalFeatures":{"enable":false},"image":{"pullPolicy":"Always","repository":"ghcr.io/nginx/nginx-gateway-fabric","tag":"edge"},"kind":"deployment","labels":{},"leaderElection":{"enable":true,"lockName":""},"lifecycle":{},"metrics":{"enable":true,"port":9113,"secure":false},"name":"","nodeSelector":{},"podAnnotations":{},"productTelemetry":{"enable":true},"readinessProbe":{"enable":true,"initialDelaySeconds":3,"port":8081},"replicas":1,"resources":{},"service":{"annotations":{},"labels":{}},"serviceAccount":{"annotations":{},"imagePullSecret":"","imagePullSecrets":[],"name":""},"snippetsFilters":{"enable":false},"terminationGracePeriodSeconds":30,"tolerations":[],"topologySpreadConstraints":[]}` | +| `nginxGateway` | The nginxGateway section contains configuration for the NGINX Gateway Fabric control plane deployment. | object | `{"affinity":{},"autoscaling":{"annotations":{},"behavior":{},"enabled":false,"maxReplicas":11,"minReplicas":1,"targetCPUUtilizationPercentage":50,"targetMemoryUtilizationPercentage":50},"autoscalingTemplate":[],"config":{"logging":{"level":"info"}},"configAnnotations":{},"extraVolumeMounts":[],"extraVolumes":[],"gatewayClassAnnotations":{},"gatewayClassName":"nginx","gatewayControllerName":"gateway.nginx.org/nginx-gateway-controller","gwAPIExperimentalFeatures":{"enable":false},"image":{"pullPolicy":"Always","repository":"ghcr.io/nginx/nginx-gateway-fabric","tag":"edge"},"kind":"deployment","labels":{},"leaderElection":{"enable":true,"lockName":""},"lifecycle":{},"metrics":{"enable":true,"port":9113,"secure":false},"name":"","nodeSelector":{},"podAnnotations":{},"productTelemetry":{"enable":true},"readinessProbe":{"enable":true,"initialDelaySeconds":3,"port":8081},"replicas":1,"resources":{},"service":{"annotations":{},"labels":{}},"serviceAccount":{"annotations":{},"imagePullSecret":"","imagePullSecrets":[],"name":""},"snippetsFilters":{"enable":false},"terminationGracePeriodSeconds":30,"tolerations":[],"topologySpreadConstraints":[]}` | | `nginxGateway.affinity` | The affinity of the NGINX Gateway Fabric control plane pod. | object | `{}` | | `nginxGateway.config.logging.level` | Log level. | string | `"info"` | | `nginxGateway.configAnnotations` | Set of custom annotations for NginxGateway objects. | object | `{}` | diff --git a/charts/nginx-gateway-fabric/templates/clusterrole.yaml b/charts/nginx-gateway-fabric/templates/clusterrole.yaml index 1205570535..8fc4da400e 100644 --- a/charts/nginx-gateway-fabric/templates/clusterrole.yaml +++ b/charts/nginx-gateway-fabric/templates/clusterrole.yaml @@ -8,6 +8,7 @@ rules: - apiGroups: - "" - apps + - autoscaling resources: - secrets - configmaps @@ -15,6 +16,7 @@ rules: - services - deployments - daemonsets + - horizontalpodautoscalers verbs: - create - update diff --git a/charts/nginx-gateway-fabric/templates/hpa.yaml b/charts/nginx-gateway-fabric/templates/hpa.yaml new file mode 100644 index 0000000000..2835eeff9f --- /dev/null +++ b/charts/nginx-gateway-fabric/templates/hpa.yaml @@ -0,0 +1,46 @@ +{{- if and (eq .Values.nginxGateway.kind "deployment") .Values.nginxGateway.autoscaling.enabled -}} +apiVersion: {{ ternary "autoscaling/v2" "autoscaling/v2beta2" (.Capabilities.APIVersions.Has "autoscaling/v2") }} +kind: HorizontalPodAutoscaler +metadata: + {{- with .Values.nginxGateway.autoscaling.annotations }} + annotations: {{ toYaml . | nindent 4 }} + {{- end }} + labels: + {{- include "nginx-gateway.labels" . | nindent 4 }} + {{- with .Values.nginxGateway.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + name: {{ include "nginx-gateway.fullname" . }} + namespace: {{ .Release.Namespace }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "nginx-gateway.fullname" . }} + minReplicas: {{ .Values.nginxGateway.autoscaling.minReplicas }} + maxReplicas: {{ .Values.nginxGateway.autoscaling.maxReplicas }} + metrics: + {{- with .Values.nginxGateway.autoscaling.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: {{ . }} + {{- end }} + {{- with .Values.nginxGateway.autoscaling.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: {{ . }} + {{- end }} + {{- with .Values.autoscalingTemplate }} + {{- toYaml . | nindent 2 }} + {{- end }} + {{- with .Values.nginxGateway.autoscaling.behavior }} + behavior: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/nginx-gateway-fabric/templates/nginxproxy.yaml b/charts/nginx-gateway-fabric/templates/nginxproxy.yaml index 0c4640c5b9..94b96a8f34 100644 --- a/charts/nginx-gateway-fabric/templates/nginxproxy.yaml +++ b/charts/nginx-gateway-fabric/templates/nginxproxy.yaml @@ -12,7 +12,27 @@ spec: kubernetes: {{- if eq .Values.nginx.kind "deployment" }} deployment: + {{- if .Values.nginx.replicas }} replicas: {{ .Values.nginx.replicas }} + {{- end }} + autoscaling: + enabled: {{ .Values.nginx.autoscaling.enabled }} + {{- if .Values.nginx.autoscaling.hpaAnnotations }} + hpaAnnotations: + {{- toYaml .Values.nginx.autoscaling.hpaAnnotations | nindent 10 }} + {{- end }} + minReplicas: {{ .Values.nginx.autoscaling.minReplicas }} + maxReplicas: {{ .Values.nginx.autoscaling.maxReplicas }} + targetCPUUtilizationPercentage: {{ .Values.nginx.autoscaling.targetCPUUtilizationPercentage }} + targetMemoryUtilizationPercentage: {{ .Values.nginx.autoscaling.targetMemoryUtilizationPercentage }} + {{- if .Values.nginx.autoscaling.behavior }} + behavior: + {{- toYaml .Values.nginx.autoscaling.behavior | nindent 10 }} + {{- end }} + {{- if .Values.nginx.autoscalingTemplate }} + autoscalingTemplate: + {{- toYaml .Values.nginx.autoscalingTemplate | nindent 8 }} + {{- end }} {{- if .Values.nginx.pod }} pod: {{- toYaml .Values.nginx.pod | nindent 8 }} diff --git a/charts/nginx-gateway-fabric/values.schema.json b/charts/nginx-gateway-fabric/values.schema.json index d750c0a42a..40b1052053 100644 --- a/charts/nginx-gateway-fabric/values.schema.json +++ b/charts/nginx-gateway-fabric/values.schema.json @@ -98,6 +98,62 @@ "nginx": { "description": "The nginx section contains the configuration for all NGINX data plane deployments\ninstalled by the NGINX Gateway Fabric control plane.", "properties": { + "autoscaling": { + "properties": { + "behavior": { + "required": [], + "title": "behavior", + "type": "object" + }, + "enabled": { + "default": true, + "description": "Enable or disable Horizontal Pod Autoscaler", + "required": [], + "title": "enabled", + "type": "boolean" + }, + "hpaAnnotations": { + "required": [], + "title": "hpaAnnotations", + "type": "object" + }, + "maxReplicas": { + "default": 11, + "required": [], + "title": "maxReplicas", + "type": "integer" + }, + "minReplicas": { + "default": 1, + "required": [], + "title": "minReplicas", + "type": "integer" + }, + "targetCPUUtilizationPercentage": { + "default": 50, + "required": [], + "title": "targetCPUUtilizationPercentage", + "type": "integer" + }, + "targetMemoryUtilizationPercentage": { + "default": 50, + "required": [], + "title": "targetMemoryUtilizationPercentage", + "type": "integer" + } + }, + "required": [], + "title": "autoscaling", + "type": "object" + }, + "autoscalingTemplate": { + "items": { + "required": [] + }, + "required": [], + "title": "autoscalingTemplate", + "type": "array" + }, "config": { "description": "The configuration for the data plane that is contained in the NginxProxy resource. This is applied globally to all Gateways\nmanaged by this instance of NGINX Gateway Fabric.", "properties": { @@ -358,7 +414,7 @@ "type": "object" }, "resources": { - "description": "The resource requirements of the NGINX container.", + "description": "The resource requirements of the NGINX container. You should be set this value, If you want to use dataplane Autoscaling(HPA).", "required": [], "title": "resources", "type": "object" @@ -385,6 +441,7 @@ "type": "boolean" }, "image": { + "description": "Custom or additional autoscaling metrics\nref: https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#support-for-custom-metrics\n- type: Pods\n pods:\n metric:\n name: nginx_gateway_fabric_nginx_process_requests_total\n target:\n type: AverageValue\n averageValue: 10000m", "properties": { "pullPolicy": { "default": "Always", @@ -600,6 +657,62 @@ "title": "affinity", "type": "object" }, + "autoscaling": { + "properties": { + "annotations": { + "required": [], + "title": "annotations", + "type": "object" + }, + "behavior": { + "required": [], + "title": "behavior", + "type": "object" + }, + "enabled": { + "default": false, + "description": "Enable or disable Horizontal Pod Autoscaler", + "required": [], + "title": "enabled", + "type": "boolean" + }, + "maxReplicas": { + "default": 11, + "required": [], + "title": "maxReplicas", + "type": "integer" + }, + "minReplicas": { + "default": 1, + "required": [], + "title": "minReplicas", + "type": "integer" + }, + "targetCPUUtilizationPercentage": { + "default": 50, + "required": [], + "title": "targetCPUUtilizationPercentage", + "type": "integer" + }, + "targetMemoryUtilizationPercentage": { + "default": 50, + "required": [], + "title": "targetMemoryUtilizationPercentage", + "type": "integer" + } + }, + "required": [], + "title": "autoscaling", + "type": "object" + }, + "autoscalingTemplate": { + "items": { + "required": [] + }, + "required": [], + "title": "autoscalingTemplate", + "type": "array" + }, "config": { "description": "The dynamic configuration for the control plane that is contained in the NginxGateway resource.", "properties": { diff --git a/charts/nginx-gateway-fabric/values.yaml b/charts/nginx-gateway-fabric/values.yaml index e9765ea511..2999730897 100644 --- a/charts/nginx-gateway-fabric/values.yaml +++ b/charts/nginx-gateway-fabric/values.yaml @@ -14,6 +14,7 @@ nginxGateway: # -- The name of the NGINX Gateway Fabric deployment - if not present, then by default uses release name given during installation. name: "" + # @schema # required: true # type: string @@ -157,6 +158,38 @@ nginxGateway: # -- The topology spread constraints for the NGINX Gateway Fabric control plane pod. topologySpreadConstraints: [] + autoscaling: + # Enable or disable Horizontal Pod Autoscaler + enabled: false + annotations: {} + minReplicas: 1 + maxReplicas: 11 + targetCPUUtilizationPercentage: 50 + targetMemoryUtilizationPercentage: 50 + behavior: {} + # scaleDown: + # stabilizationWindowSeconds: 300 + # policies: + # - type: Pods + # value: 1 + # periodSeconds: 180 + # scaleUp: + # stabilizationWindowSeconds: 300 + # policies: + # - type: Pods + # value: 2 + # periodSeconds: 60 + autoscalingTemplate: [] + # Custom or additional autoscaling metrics + # ref: https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#support-for-custom-metrics + # - type: Pods + # pods: + # metric: + # name: nginx_gateway_fabric_nginx_process_requests_total + # target: + # type: AverageValue + # averageValue: 10000m + metrics: # -- Enable exposing metrics in the Prometheus format. enable: true @@ -197,6 +230,37 @@ nginx: # -- The number of replicas of the NGINX Deployment. replicas: 1 + autoscaling: + # Enable or disable Horizontal Pod Autoscaler + enabled: true + hpaAnnotations: {} + minReplicas: 1 + maxReplicas: 11 + targetCPUUtilizationPercentage: 50 + targetMemoryUtilizationPercentage: 50 + behavior: {} + # scaleDown: + # stabilizationWindowSeconds: 300 + # policies: + # - type: Pods + # value: 1 + # periodSeconds: 180 + # scaleUp: + # stabilizationWindowSeconds: 300 + # policies: + # - type: Pods + # value: 2 + # periodSeconds: 60 + autoscalingTemplate: [] + # Custom or additional autoscaling metrics + # ref: https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#support-for-custom-metrics + # - type: Pods + # pods: + # metric: + # name: nginx_gateway_fabric_nginx_process_requests_total + # target: + # type: AverageValue + # averageValue: 10000m image: # -- The NGINX image to use. repository: ghcr.io/nginx/nginx-gateway-fabric/nginx @@ -432,7 +496,7 @@ nginx: # - port: 80 # containerPort: 80 - # -- The resource requirements of the NGINX container. + # -- The resource requirements of the NGINX container. You should be set this value, If you want to use dataplane Autoscaling(HPA). resources: {} # -- The lifecycle of the NGINX container. diff --git a/config/crd/bases/gateway.nginx.org_nginxproxies.yaml b/config/crd/bases/gateway.nginx.org_nginxproxies.yaml index 14def1da6d..bbd2acc238 100644 --- a/config/crd/bases/gateway.nginx.org_nginxproxies.yaml +++ b/config/crd/bases/gateway.nginx.org_nginxproxies.yaml @@ -3530,6 +3530,667 @@ spec: Deployment is the configuration for the NGINX Deployment. This is the default deployment option. properties: + autoscaling: + description: Horizontal Pod Autoscaling. + properties: + autoscalingTemplate: + description: AutoscalingTemplate configures the additional + scaling option. + items: + description: |- + MetricSpec specifies how to scale based on a single metric + (only `type` and one other matching field should be set at once). + properties: + containerResource: + description: |- + containerResource refers to a resource metric (such as those specified in + requests and limits) known to Kubernetes describing a single container in + each pod of the current scale target (e.g. CPU or memory). Such metrics are + built in to Kubernetes, and have special scaling options on top of those + available to normal per-pod metrics using the "pods" source. + properties: + container: + description: container is the name of the container + in the pods of the scaling target + type: string + name: + description: name is the name of the resource + in question. + type: string + target: + description: target specifies the target value + for the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the + metric type is Utilization, Value, or + AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of + the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - container + - name + - target + type: object + external: + description: |- + external refers to a global metric that is not associated + with any Kubernetes object. It allows autoscaling based on information + coming from components running outside of cluster + (for example length of queue in cloud messaging service, or + QPS from loadbalancer running outside of cluster). + properties: + metric: + description: metric identifies the target metric + by name and selector + properties: + name: + description: name is the name of the given + metric + type: string + selector: + description: |- + selector is the string-encoded form of a standard kubernetes label selector for the given metric + When set, it is passed as an additional parameter to the metrics server for more specific metrics scoping. + When unset, just the metricName will be used to gather metrics. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target value + for the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the + metric type is Utilization, Value, or + AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of + the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - metric + - target + type: object + object: + description: |- + object refers to a metric describing a single kubernetes object + (for example, hits-per-second on an Ingress object). + properties: + describedObject: + description: describedObject specifies the descriptions + of a object,such as kind,name apiVersion + properties: + apiVersion: + description: apiVersion is the API version + of the referent + type: string + kind: + description: 'kind is the kind of the referent; + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'name is the name of the referent; + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + required: + - kind + - name + type: object + metric: + description: metric identifies the target metric + by name and selector + properties: + name: + description: name is the name of the given + metric + type: string + selector: + description: |- + selector is the string-encoded form of a standard kubernetes label selector for the given metric + When set, it is passed as an additional parameter to the metrics server for more specific metrics scoping. + When unset, just the metricName will be used to gather metrics. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target value + for the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the + metric type is Utilization, Value, or + AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of + the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - describedObject + - metric + - target + type: object + pods: + description: |- + pods refers to a metric describing each pod in the current scale target + (for example, transactions-processed-per-second). The values will be + averaged together before being compared to the target value. + properties: + metric: + description: metric identifies the target metric + by name and selector + properties: + name: + description: name is the name of the given + metric + type: string + selector: + description: |- + selector is the string-encoded form of a standard kubernetes label selector for the given metric + When set, it is passed as an additional parameter to the metrics server for more specific metrics scoping. + When unset, just the metricName will be used to gather metrics. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target value + for the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the + metric type is Utilization, Value, or + AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of + the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - metric + - target + type: object + resource: + description: |- + resource refers to a resource metric (such as those specified in + requests and limits) known to Kubernetes describing each pod in the + current scale target (e.g. CPU or memory). Such metrics are built in to + Kubernetes, and have special scaling options on top of those available + to normal per-pod metrics using the "pods" source. + properties: + name: + description: name is the name of the resource + in question. + type: string + target: + description: target specifies the target value + for the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the + metric type is Utilization, Value, or + AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of + the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - name + - target + type: object + type: + description: |- + type is the type of metric source. It should be one of "ContainerResource", "External", + "Object", "Pods" or "Resource", each mapping to a matching field in the object. + type: string + required: + - type + type: object + type: array + behavior: + description: |- + Behavior configures the scaling behavior of the target + in both Up and Down directions (scaleUp and scaleDown fields respectively). + If not set, the default HPAScalingRules for scale up and scale down are used. + properties: + scaleDown: + description: |- + scaleDown is scaling policy for scaling Down. + If not set, the default value is to allow to scale down to minReplicas pods, with a + 300 second stabilization window (i.e., the highest recommendation for + the last 300sec is used). + properties: + policies: + description: |- + policies is a list of potential scaling polices which can be used during scaling. + If not set, use the default values: + - For scale up: allow doubling the number of pods, or an absolute change of 4 pods in a 15s window. + - For scale down: allow all pods to be removed in a 15s window. + items: + description: HPAScalingPolicy is a single policy + which must hold true for a specified past + interval. + properties: + periodSeconds: + description: |- + periodSeconds specifies the window of time for which the policy should hold true. + PeriodSeconds must be greater than zero and less than or equal to 1800 (30 min). + format: int32 + type: integer + type: + description: type is used to specify the + scaling policy. + type: string + value: + description: |- + value contains the amount of change which is permitted by the policy. + It must be greater than zero + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + x-kubernetes-list-type: atomic + selectPolicy: + description: |- + selectPolicy is used to specify which policy should be used. + If not set, the default value Max is used. + type: string + stabilizationWindowSeconds: + description: |- + stabilizationWindowSeconds is the number of seconds for which past recommendations should be + considered while scaling up or scaling down. + StabilizationWindowSeconds must be greater than or equal to zero and less than or equal to 3600 (one hour). + If not set, use the default values: + - For scale up: 0 (i.e. no stabilization is done). + - For scale down: 300 (i.e. the stabilization window is 300 seconds long). + format: int32 + type: integer + tolerance: + anyOf: + - type: integer + - type: string + description: |- + tolerance is the tolerance on the ratio between the current and desired + metric value under which no updates are made to the desired number of + replicas (e.g. 0.01 for 1%). Must be greater than or equal to zero. If not + set, the default cluster-wide tolerance is applied (by default 10%). + + For example, if autoscaling is configured with a memory consumption target of 100Mi, + and scale-down and scale-up tolerances of 5% and 1% respectively, scaling will be + triggered when the actual consumption falls below 95Mi or exceeds 101Mi. + + This is an alpha field and requires enabling the HPAConfigurableTolerance + feature gate. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + scaleUp: + description: |- + scaleUp is scaling policy for scaling Up. + If not set, the default value is the higher of: + * increase no more than 4 pods per 60 seconds + * double the number of pods per 60 seconds + No stabilization is used. + properties: + policies: + description: |- + policies is a list of potential scaling polices which can be used during scaling. + If not set, use the default values: + - For scale up: allow doubling the number of pods, or an absolute change of 4 pods in a 15s window. + - For scale down: allow all pods to be removed in a 15s window. + items: + description: HPAScalingPolicy is a single policy + which must hold true for a specified past + interval. + properties: + periodSeconds: + description: |- + periodSeconds specifies the window of time for which the policy should hold true. + PeriodSeconds must be greater than zero and less than or equal to 1800 (30 min). + format: int32 + type: integer + type: + description: type is used to specify the + scaling policy. + type: string + value: + description: |- + value contains the amount of change which is permitted by the policy. + It must be greater than zero + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + x-kubernetes-list-type: atomic + selectPolicy: + description: |- + selectPolicy is used to specify which policy should be used. + If not set, the default value Max is used. + type: string + stabilizationWindowSeconds: + description: |- + stabilizationWindowSeconds is the number of seconds for which past recommendations should be + considered while scaling up or scaling down. + StabilizationWindowSeconds must be greater than or equal to zero and less than or equal to 3600 (one hour). + If not set, use the default values: + - For scale up: 0 (i.e. no stabilization is done). + - For scale down: 300 (i.e. the stabilization window is 300 seconds long). + format: int32 + type: integer + tolerance: + anyOf: + - type: integer + - type: string + description: |- + tolerance is the tolerance on the ratio between the current and desired + metric value under which no updates are made to the desired number of + replicas (e.g. 0.01 for 1%). Must be greater than or equal to zero. If not + set, the default cluster-wide tolerance is applied (by default 10%). + + For example, if autoscaling is configured with a memory consumption target of 100Mi, + and scale-down and scale-up tolerances of 5% and 1% respectively, scaling will be + triggered when the actual consumption falls below 95Mi or exceeds 101Mi. + + This is an alpha field and requires enabling the HPAConfigurableTolerance + feature gate. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + enabled: + description: Enable or disable Horizontal Pod Autoscaler + type: boolean + maxReplicas: + description: Maximum number of replicas. + format: int32 + type: integer + minReplicas: + description: Minimum number of replicas. + format: int32 + type: integer + targetCPUUtilizationPercentage: + description: Target cpu utilization percentage of HPA. + format: int32 + type: integer + targetMemoryUtilizationPercentage: + description: Target memory utilization percentage of HPA. + format: int32 + type: integer + required: + - enabled + - maxReplicas + type: object container: description: Container defines container fields for the NGINX container. diff --git a/deploy/azure/deploy.yaml b/deploy/azure/deploy.yaml index 0a7e457685..93fa2c8a69 100644 --- a/deploy/azure/deploy.yaml +++ b/deploy/azure/deploy.yaml @@ -55,6 +55,7 @@ rules: - apiGroups: - "" - apps + - autoscaling resources: - secrets - configmaps @@ -62,6 +63,7 @@ rules: - services - deployments - daemonsets + - horizontalpodautoscalers verbs: - create - update @@ -412,6 +414,12 @@ metadata: spec: kubernetes: deployment: + autoscaling: + enabled: true + maxReplicas: 11 + minReplicas: 1 + targetCPUUtilizationPercentage: 50 + targetMemoryUtilizationPercentage: 50 container: image: pullPolicy: Always diff --git a/deploy/crds.yaml b/deploy/crds.yaml index db7e84980a..6977926c20 100644 --- a/deploy/crds.yaml +++ b/deploy/crds.yaml @@ -4115,6 +4115,667 @@ spec: Deployment is the configuration for the NGINX Deployment. This is the default deployment option. properties: + autoscaling: + description: Horizontal Pod Autoscaling. + properties: + autoscalingTemplate: + description: AutoscalingTemplate configures the additional + scaling option. + items: + description: |- + MetricSpec specifies how to scale based on a single metric + (only `type` and one other matching field should be set at once). + properties: + containerResource: + description: |- + containerResource refers to a resource metric (such as those specified in + requests and limits) known to Kubernetes describing a single container in + each pod of the current scale target (e.g. CPU or memory). Such metrics are + built in to Kubernetes, and have special scaling options on top of those + available to normal per-pod metrics using the "pods" source. + properties: + container: + description: container is the name of the container + in the pods of the scaling target + type: string + name: + description: name is the name of the resource + in question. + type: string + target: + description: target specifies the target value + for the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the + metric type is Utilization, Value, or + AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of + the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - container + - name + - target + type: object + external: + description: |- + external refers to a global metric that is not associated + with any Kubernetes object. It allows autoscaling based on information + coming from components running outside of cluster + (for example length of queue in cloud messaging service, or + QPS from loadbalancer running outside of cluster). + properties: + metric: + description: metric identifies the target metric + by name and selector + properties: + name: + description: name is the name of the given + metric + type: string + selector: + description: |- + selector is the string-encoded form of a standard kubernetes label selector for the given metric + When set, it is passed as an additional parameter to the metrics server for more specific metrics scoping. + When unset, just the metricName will be used to gather metrics. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target value + for the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the + metric type is Utilization, Value, or + AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of + the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - metric + - target + type: object + object: + description: |- + object refers to a metric describing a single kubernetes object + (for example, hits-per-second on an Ingress object). + properties: + describedObject: + description: describedObject specifies the descriptions + of a object,such as kind,name apiVersion + properties: + apiVersion: + description: apiVersion is the API version + of the referent + type: string + kind: + description: 'kind is the kind of the referent; + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'name is the name of the referent; + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + required: + - kind + - name + type: object + metric: + description: metric identifies the target metric + by name and selector + properties: + name: + description: name is the name of the given + metric + type: string + selector: + description: |- + selector is the string-encoded form of a standard kubernetes label selector for the given metric + When set, it is passed as an additional parameter to the metrics server for more specific metrics scoping. + When unset, just the metricName will be used to gather metrics. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target value + for the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the + metric type is Utilization, Value, or + AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of + the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - describedObject + - metric + - target + type: object + pods: + description: |- + pods refers to a metric describing each pod in the current scale target + (for example, transactions-processed-per-second). The values will be + averaged together before being compared to the target value. + properties: + metric: + description: metric identifies the target metric + by name and selector + properties: + name: + description: name is the name of the given + metric + type: string + selector: + description: |- + selector is the string-encoded form of a standard kubernetes label selector for the given metric + When set, it is passed as an additional parameter to the metrics server for more specific metrics scoping. + When unset, just the metricName will be used to gather metrics. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target value + for the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the + metric type is Utilization, Value, or + AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of + the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - metric + - target + type: object + resource: + description: |- + resource refers to a resource metric (such as those specified in + requests and limits) known to Kubernetes describing each pod in the + current scale target (e.g. CPU or memory). Such metrics are built in to + Kubernetes, and have special scaling options on top of those available + to normal per-pod metrics using the "pods" source. + properties: + name: + description: name is the name of the resource + in question. + type: string + target: + description: target specifies the target value + for the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the + metric type is Utilization, Value, or + AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of + the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - name + - target + type: object + type: + description: |- + type is the type of metric source. It should be one of "ContainerResource", "External", + "Object", "Pods" or "Resource", each mapping to a matching field in the object. + type: string + required: + - type + type: object + type: array + behavior: + description: |- + Behavior configures the scaling behavior of the target + in both Up and Down directions (scaleUp and scaleDown fields respectively). + If not set, the default HPAScalingRules for scale up and scale down are used. + properties: + scaleDown: + description: |- + scaleDown is scaling policy for scaling Down. + If not set, the default value is to allow to scale down to minReplicas pods, with a + 300 second stabilization window (i.e., the highest recommendation for + the last 300sec is used). + properties: + policies: + description: |- + policies is a list of potential scaling polices which can be used during scaling. + If not set, use the default values: + - For scale up: allow doubling the number of pods, or an absolute change of 4 pods in a 15s window. + - For scale down: allow all pods to be removed in a 15s window. + items: + description: HPAScalingPolicy is a single policy + which must hold true for a specified past + interval. + properties: + periodSeconds: + description: |- + periodSeconds specifies the window of time for which the policy should hold true. + PeriodSeconds must be greater than zero and less than or equal to 1800 (30 min). + format: int32 + type: integer + type: + description: type is used to specify the + scaling policy. + type: string + value: + description: |- + value contains the amount of change which is permitted by the policy. + It must be greater than zero + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + x-kubernetes-list-type: atomic + selectPolicy: + description: |- + selectPolicy is used to specify which policy should be used. + If not set, the default value Max is used. + type: string + stabilizationWindowSeconds: + description: |- + stabilizationWindowSeconds is the number of seconds for which past recommendations should be + considered while scaling up or scaling down. + StabilizationWindowSeconds must be greater than or equal to zero and less than or equal to 3600 (one hour). + If not set, use the default values: + - For scale up: 0 (i.e. no stabilization is done). + - For scale down: 300 (i.e. the stabilization window is 300 seconds long). + format: int32 + type: integer + tolerance: + anyOf: + - type: integer + - type: string + description: |- + tolerance is the tolerance on the ratio between the current and desired + metric value under which no updates are made to the desired number of + replicas (e.g. 0.01 for 1%). Must be greater than or equal to zero. If not + set, the default cluster-wide tolerance is applied (by default 10%). + + For example, if autoscaling is configured with a memory consumption target of 100Mi, + and scale-down and scale-up tolerances of 5% and 1% respectively, scaling will be + triggered when the actual consumption falls below 95Mi or exceeds 101Mi. + + This is an alpha field and requires enabling the HPAConfigurableTolerance + feature gate. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + scaleUp: + description: |- + scaleUp is scaling policy for scaling Up. + If not set, the default value is the higher of: + * increase no more than 4 pods per 60 seconds + * double the number of pods per 60 seconds + No stabilization is used. + properties: + policies: + description: |- + policies is a list of potential scaling polices which can be used during scaling. + If not set, use the default values: + - For scale up: allow doubling the number of pods, or an absolute change of 4 pods in a 15s window. + - For scale down: allow all pods to be removed in a 15s window. + items: + description: HPAScalingPolicy is a single policy + which must hold true for a specified past + interval. + properties: + periodSeconds: + description: |- + periodSeconds specifies the window of time for which the policy should hold true. + PeriodSeconds must be greater than zero and less than or equal to 1800 (30 min). + format: int32 + type: integer + type: + description: type is used to specify the + scaling policy. + type: string + value: + description: |- + value contains the amount of change which is permitted by the policy. + It must be greater than zero + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + x-kubernetes-list-type: atomic + selectPolicy: + description: |- + selectPolicy is used to specify which policy should be used. + If not set, the default value Max is used. + type: string + stabilizationWindowSeconds: + description: |- + stabilizationWindowSeconds is the number of seconds for which past recommendations should be + considered while scaling up or scaling down. + StabilizationWindowSeconds must be greater than or equal to zero and less than or equal to 3600 (one hour). + If not set, use the default values: + - For scale up: 0 (i.e. no stabilization is done). + - For scale down: 300 (i.e. the stabilization window is 300 seconds long). + format: int32 + type: integer + tolerance: + anyOf: + - type: integer + - type: string + description: |- + tolerance is the tolerance on the ratio between the current and desired + metric value under which no updates are made to the desired number of + replicas (e.g. 0.01 for 1%). Must be greater than or equal to zero. If not + set, the default cluster-wide tolerance is applied (by default 10%). + + For example, if autoscaling is configured with a memory consumption target of 100Mi, + and scale-down and scale-up tolerances of 5% and 1% respectively, scaling will be + triggered when the actual consumption falls below 95Mi or exceeds 101Mi. + + This is an alpha field and requires enabling the HPAConfigurableTolerance + feature gate. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + enabled: + description: Enable or disable Horizontal Pod Autoscaler + type: boolean + maxReplicas: + description: Maximum number of replicas. + format: int32 + type: integer + minReplicas: + description: Minimum number of replicas. + format: int32 + type: integer + targetCPUUtilizationPercentage: + description: Target cpu utilization percentage of HPA. + format: int32 + type: integer + targetMemoryUtilizationPercentage: + description: Target memory utilization percentage of HPA. + format: int32 + type: integer + required: + - enabled + - maxReplicas + type: object container: description: Container defines container fields for the NGINX container. diff --git a/deploy/default/deploy.yaml b/deploy/default/deploy.yaml index 4324fc92f7..ba0b1be9f4 100644 --- a/deploy/default/deploy.yaml +++ b/deploy/default/deploy.yaml @@ -55,6 +55,7 @@ rules: - apiGroups: - "" - apps + - autoscaling resources: - secrets - configmaps @@ -62,6 +63,7 @@ rules: - services - deployments - daemonsets + - horizontalpodautoscalers verbs: - create - update @@ -410,6 +412,12 @@ metadata: spec: kubernetes: deployment: + autoscaling: + enabled: true + maxReplicas: 11 + minReplicas: 1 + targetCPUUtilizationPercentage: 50 + targetMemoryUtilizationPercentage: 50 container: image: pullPolicy: Always diff --git a/deploy/experimental-nginx-plus/deploy.yaml b/deploy/experimental-nginx-plus/deploy.yaml index f0ac53ba0d..4287e65caa 100644 --- a/deploy/experimental-nginx-plus/deploy.yaml +++ b/deploy/experimental-nginx-plus/deploy.yaml @@ -55,6 +55,7 @@ rules: - apiGroups: - "" - apps + - autoscaling resources: - secrets - configmaps @@ -62,6 +63,7 @@ rules: - services - deployments - daemonsets + - horizontalpodautoscalers verbs: - create - update @@ -418,6 +420,12 @@ metadata: spec: kubernetes: deployment: + autoscaling: + enabled: true + maxReplicas: 11 + minReplicas: 1 + targetCPUUtilizationPercentage: 50 + targetMemoryUtilizationPercentage: 50 container: image: pullPolicy: Always diff --git a/deploy/experimental/deploy.yaml b/deploy/experimental/deploy.yaml index ad3cf361a6..11465b3d06 100644 --- a/deploy/experimental/deploy.yaml +++ b/deploy/experimental/deploy.yaml @@ -55,6 +55,7 @@ rules: - apiGroups: - "" - apps + - autoscaling resources: - secrets - configmaps @@ -62,6 +63,7 @@ rules: - services - deployments - daemonsets + - horizontalpodautoscalers verbs: - create - update @@ -415,6 +417,12 @@ metadata: spec: kubernetes: deployment: + autoscaling: + enabled: true + maxReplicas: 11 + minReplicas: 1 + targetCPUUtilizationPercentage: 50 + targetMemoryUtilizationPercentage: 50 container: image: pullPolicy: Always diff --git a/deploy/nginx-plus/deploy.yaml b/deploy/nginx-plus/deploy.yaml index a966b9e325..62a44b9628 100644 --- a/deploy/nginx-plus/deploy.yaml +++ b/deploy/nginx-plus/deploy.yaml @@ -55,6 +55,7 @@ rules: - apiGroups: - "" - apps + - autoscaling resources: - secrets - configmaps @@ -62,6 +63,7 @@ rules: - services - deployments - daemonsets + - horizontalpodautoscalers verbs: - create - update @@ -413,6 +415,12 @@ metadata: spec: kubernetes: deployment: + autoscaling: + enabled: true + maxReplicas: 11 + minReplicas: 1 + targetCPUUtilizationPercentage: 50 + targetMemoryUtilizationPercentage: 50 container: image: pullPolicy: Always diff --git a/deploy/nodeport/deploy.yaml b/deploy/nodeport/deploy.yaml index d151c82319..8b58ba7f0b 100644 --- a/deploy/nodeport/deploy.yaml +++ b/deploy/nodeport/deploy.yaml @@ -55,6 +55,7 @@ rules: - apiGroups: - "" - apps + - autoscaling resources: - secrets - configmaps @@ -62,6 +63,7 @@ rules: - services - deployments - daemonsets + - horizontalpodautoscalers verbs: - create - update @@ -410,6 +412,12 @@ metadata: spec: kubernetes: deployment: + autoscaling: + enabled: true + maxReplicas: 11 + minReplicas: 1 + targetCPUUtilizationPercentage: 50 + targetMemoryUtilizationPercentage: 50 container: image: pullPolicy: Always diff --git a/deploy/openshift/deploy.yaml b/deploy/openshift/deploy.yaml index e6ecf6b3e0..e121698c9c 100644 --- a/deploy/openshift/deploy.yaml +++ b/deploy/openshift/deploy.yaml @@ -55,6 +55,7 @@ rules: - apiGroups: - "" - apps + - autoscaling resources: - secrets - configmaps @@ -62,6 +63,7 @@ rules: - services - deployments - daemonsets + - horizontalpodautoscalers verbs: - create - update @@ -432,6 +434,12 @@ metadata: spec: kubernetes: deployment: + autoscaling: + enabled: true + maxReplicas: 11 + minReplicas: 1 + targetCPUUtilizationPercentage: 50 + targetMemoryUtilizationPercentage: 50 container: image: pullPolicy: Always diff --git a/deploy/snippets-filters-nginx-plus/deploy.yaml b/deploy/snippets-filters-nginx-plus/deploy.yaml index 7461912539..8150952bbb 100644 --- a/deploy/snippets-filters-nginx-plus/deploy.yaml +++ b/deploy/snippets-filters-nginx-plus/deploy.yaml @@ -55,6 +55,7 @@ rules: - apiGroups: - "" - apps + - autoscaling resources: - secrets - configmaps @@ -62,6 +63,7 @@ rules: - services - deployments - daemonsets + - horizontalpodautoscalers verbs: - create - update @@ -416,6 +418,12 @@ metadata: spec: kubernetes: deployment: + autoscaling: + enabled: true + maxReplicas: 11 + minReplicas: 1 + targetCPUUtilizationPercentage: 50 + targetMemoryUtilizationPercentage: 50 container: image: pullPolicy: Always diff --git a/deploy/snippets-filters/deploy.yaml b/deploy/snippets-filters/deploy.yaml index d23d775600..c28df71c3b 100644 --- a/deploy/snippets-filters/deploy.yaml +++ b/deploy/snippets-filters/deploy.yaml @@ -55,6 +55,7 @@ rules: - apiGroups: - "" - apps + - autoscaling resources: - secrets - configmaps @@ -62,6 +63,7 @@ rules: - services - deployments - daemonsets + - horizontalpodautoscalers verbs: - create - update @@ -413,6 +415,12 @@ metadata: spec: kubernetes: deployment: + autoscaling: + enabled: true + maxReplicas: 11 + minReplicas: 1 + targetCPUUtilizationPercentage: 50 + targetMemoryUtilizationPercentage: 50 container: image: pullPolicy: Always diff --git a/internal/controller/manager.go b/internal/controller/manager.go index a8a1afe99b..3b27a1ede8 100644 --- a/internal/controller/manager.go +++ b/internal/controller/manager.go @@ -12,6 +12,7 @@ import ( "google.golang.org/grpc" appsv1 "k8s.io/api/apps/v1" authv1 "k8s.io/api/authentication/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" apiv1 "k8s.io/api/core/v1" discoveryV1 "k8s.io/api/discovery/v1" rbacv1 "k8s.io/api/rbac/v1" @@ -91,6 +92,7 @@ func init() { utilruntime.Must(ngfAPIv1alpha2.AddToScheme(scheme)) utilruntime.Must(apiext.AddToScheme(scheme)) utilruntime.Must(appsv1.AddToScheme(scheme)) + utilruntime.Must(autoscalingv2.AddToScheme(scheme)) utilruntime.Must(authv1.AddToScheme(scheme)) utilruntime.Must(rbacv1.AddToScheme(scheme)) } diff --git a/internal/controller/provisioner/handler.go b/internal/controller/provisioner/handler.go index c5f3f2f4a9..42c6e88130 100644 --- a/internal/controller/provisioner/handler.go +++ b/internal/controller/provisioner/handler.go @@ -9,6 +9,7 @@ import ( "github.com/go-logr/logr" appsv1 "k8s.io/api/apps/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -62,7 +63,7 @@ func (h *eventHandler) HandleEventBatch(ctx context.Context, logger logr.Logger, case *gatewayv1.Gateway: h.store.updateGateway(obj) case *appsv1.Deployment, *appsv1.DaemonSet, *corev1.ServiceAccount, - *corev1.ConfigMap, *rbacv1.Role, *rbacv1.RoleBinding: + *corev1.ConfigMap, *rbacv1.Role, *rbacv1.RoleBinding, *autoscalingv2.HorizontalPodAutoscaler: objLabels := labels.Set(obj.GetLabels()) if h.labelSelector.Matches(objLabels) { gatewayName := objLabels.Get(controller.GatewayLabel) diff --git a/internal/controller/provisioner/objects.go b/internal/controller/provisioner/objects.go index 8a9d39c079..d8440d3197 100644 --- a/internal/controller/provisioner/objects.go +++ b/internal/controller/provisioner/objects.go @@ -12,6 +12,7 @@ import ( jsonpatch "gopkg.in/evanphx/json-patch.v4" appsv1 "k8s.io/api/apps/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -170,6 +171,7 @@ func (p *NginxProvisioner) buildNginxResourceObjects( // role/binding (if openshift) // service // deployment/daemonset + // hpa objects := make([]client.Object, 0, len(configmaps)+len(secrets)+len(openshiftObjs)+3) objects = append(objects, secrets...) @@ -178,11 +180,31 @@ func (p *NginxProvisioner) buildNginxResourceObjects( if p.isOpenshift { objects = append(objects, openshiftObjs...) } + objects = append(objects, service, deployment) + if hpa := p.buildHPA(objectMeta, nProxyCfg); hpa != nil { + objects = append(objects, hpa) + } + return objects, errors.Join(errs...) } +func isAutoscalingEnabled(dep *ngfAPIv1alpha2.DeploymentSpec) bool { + return dep != nil && dep.Autoscaling.Enabled +} + +func (p *NginxProvisioner) buildHPA( + objectMeta metav1.ObjectMeta, + nProxyCfg *graph.EffectiveNginxProxy, +) client.Object { + if nProxyCfg == nil || nProxyCfg.Kubernetes == nil || !isAutoscalingEnabled(nProxyCfg.Kubernetes.Deployment) { + return nil + } + + return buildNginxDeploymentHPA(objectMeta, nProxyCfg.Kubernetes.Deployment.Autoscaling) +} + func (p *NginxProvisioner) buildNginxSecrets( objectMeta metav1.ObjectMeta, agentTLSSecretName string, @@ -1018,6 +1040,156 @@ func (p *NginxProvisioner) buildImage(nProxyCfg *graph.EffectiveNginxProxy) (str return fmt.Sprintf("%s:%s", image, tag), pullPolicy } +func getMetricTargetByType( + target autoscalingv2.MetricTarget, +) autoscalingv2.MetricTarget { + switch target.Type { + case autoscalingv2.UtilizationMetricType: + return autoscalingv2.MetricTarget{ + Type: autoscalingv2.UtilizationMetricType, + AverageUtilization: target.AverageUtilization, + } + + case autoscalingv2.AverageValueMetricType: + return autoscalingv2.MetricTarget{ + Type: autoscalingv2.AverageValueMetricType, + AverageValue: target.AverageValue, + } + + case autoscalingv2.ValueMetricType: + return autoscalingv2.MetricTarget{ + Type: autoscalingv2.ValueMetricType, + Value: target.Value, + } + default: + return autoscalingv2.MetricTarget{} + } +} + +func buildNginxDeploymentHPA( + objectMeta metav1.ObjectMeta, + autoScaling ngfAPIv1alpha2.HPASpec, +) *autoscalingv2.HorizontalPodAutoscaler { + if !autoScaling.Enabled { + return nil + } + var metrics []autoscalingv2.MetricSpec + + cpuUtil := autoScaling.TargetCPUUtilizationPercentage + memUtil := autoScaling.TargetMemoryUtilizationPercentage + autoscalingTemplate := autoScaling.AutoscalingTemplate + + if cpuUtil != nil { + metrics = append(metrics, autoscalingv2.MetricSpec{ + Type: autoscalingv2.ResourceMetricSourceType, + Resource: &autoscalingv2.ResourceMetricSource{ + Name: "cpu", + Target: autoscalingv2.MetricTarget{ + Type: autoscalingv2.UtilizationMetricType, + AverageUtilization: cpuUtil, + }, + }, + }) + } + + if memUtil != nil { + metrics = append(metrics, autoscalingv2.MetricSpec{ + Type: autoscalingv2.ResourceMetricSourceType, + Resource: &autoscalingv2.ResourceMetricSource{ + Name: "memory", + Target: autoscalingv2.MetricTarget{ + Type: autoscalingv2.UtilizationMetricType, + AverageUtilization: memUtil, + }, + }, + }) + } + + if autoscalingTemplate != nil { + for _, additionalAutoscaling := range *autoscalingTemplate { + metric := buildAdditionalMetric(additionalAutoscaling) + if metric != nil { + metrics = append(metrics, *metric) + } + } + } + + if len(metrics) == 0 { + // No metrics configured, skip HPA creation + return nil + } + + return &autoscalingv2.HorizontalPodAutoscaler{ + ObjectMeta: objectMeta, + Spec: autoscalingv2.HorizontalPodAutoscalerSpec{ + ScaleTargetRef: autoscalingv2.CrossVersionObjectReference{ + APIVersion: "apps/v1", + Kind: "Deployment", + Name: objectMeta.Name, + }, + MinReplicas: autoScaling.MinReplicas, + MaxReplicas: autoScaling.MaxReplicas, + Metrics: metrics, + Behavior: autoScaling.Behavior, + }, + } +} + +func buildAdditionalMetric(additionalAutoscaling autoscalingv2.MetricSpec) *autoscalingv2.MetricSpec { + switch additionalAutoscaling.Type { + case autoscalingv2.ResourceMetricSourceType: + return &autoscalingv2.MetricSpec{ + Type: additionalAutoscaling.Type, + Resource: &autoscalingv2.ResourceMetricSource{ + Name: additionalAutoscaling.Resource.Name, + Target: getMetricTargetByType(additionalAutoscaling.Resource.Target), + }, + } + case autoscalingv2.PodsMetricSourceType: + return &autoscalingv2.MetricSpec{ + Type: additionalAutoscaling.Type, + Pods: &autoscalingv2.PodsMetricSource{ + Metric: additionalAutoscaling.Pods.Metric, + Target: getMetricTargetByType(additionalAutoscaling.Pods.Target), + }, + } + case autoscalingv2.ContainerResourceMetricSourceType: + return &autoscalingv2.MetricSpec{ + Type: additionalAutoscaling.Type, + ContainerResource: &autoscalingv2.ContainerResourceMetricSource{ + Name: additionalAutoscaling.ContainerResource.Name, + Target: getMetricTargetByType(additionalAutoscaling.ContainerResource.Target), + Container: additionalAutoscaling.ContainerResource.Container, + }, + } + case autoscalingv2.ObjectMetricSourceType: + return &autoscalingv2.MetricSpec{ + Type: additionalAutoscaling.Type, + Object: &autoscalingv2.ObjectMetricSource{ + DescribedObject: additionalAutoscaling.Object.DescribedObject, + Target: getMetricTargetByType(additionalAutoscaling.Object.Target), + Metric: autoscalingv2.MetricIdentifier{ + Name: additionalAutoscaling.Object.Metric.Name, + Selector: additionalAutoscaling.Object.Metric.Selector, + }, + }, + } + case autoscalingv2.ExternalMetricSourceType: + return &autoscalingv2.MetricSpec{ + Type: additionalAutoscaling.Type, + External: &autoscalingv2.ExternalMetricSource{ + Metric: autoscalingv2.MetricIdentifier{ + Name: additionalAutoscaling.External.Metric.Name, + Selector: additionalAutoscaling.External.Metric.Selector, + }, + Target: getMetricTargetByType(additionalAutoscaling.External.Target), + }, + } + default: + return nil + } +} + // TODO(sberman): see about how this can be made more elegant. Maybe create some sort of Object factory // that can better store/build all the objects we need, to reduce the amount of duplicate object lists that we // have everywhere. @@ -1029,6 +1201,7 @@ func (p *NginxProvisioner) buildNginxResourceObjectsForDeletion(deploymentNSName // serviceaccount // configmaps // secrets + // hpa objectMeta := metav1.ObjectMeta{ Name: deploymentNSName.Name, @@ -1044,8 +1217,17 @@ func (p *NginxProvisioner) buildNginxResourceObjectsForDeletion(deploymentNSName service := &corev1.Service{ ObjectMeta: objectMeta, } + hpa := &autoscalingv2.HorizontalPodAutoscaler{ + ObjectMeta: objectMeta, + } + + objects := []client.Object{deployment, daemonSet, service, hpa} + + // objects := []client.Object{deployment, daemonSet, service} - objects := []client.Object{deployment, daemonSet, service} + // // if hpa := p.buildHPA(objectMeta, nProxyCfg); hpa != nil { + // // objects = append(objects, hpa) + // // } if p.isOpenshift { role := &rbacv1.Role{ diff --git a/internal/controller/provisioner/objects_test.go b/internal/controller/provisioner/objects_test.go index c32310b474..9f2ea3f8cf 100644 --- a/internal/controller/provisioner/objects_test.go +++ b/internal/controller/provisioner/objects_test.go @@ -6,6 +6,7 @@ import ( . "github.com/onsi/gomega" appsv1 "k8s.io/api/apps/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" @@ -826,7 +827,7 @@ func TestBuildNginxResourceObjectsForDeletion(t *testing.T) { objects := provisioner.buildNginxResourceObjectsForDeletion(deploymentNSName) - g.Expect(objects).To(HaveLen(7)) + g.Expect(objects).To(HaveLen(8)) validateMeta := func(obj client.Object, name string) { g.Expect(obj.GetName()).To(Equal(name)) @@ -848,17 +849,22 @@ func TestBuildNginxResourceObjectsForDeletion(t *testing.T) { g.Expect(ok).To(BeTrue()) validateMeta(svc, deploymentNSName.Name) - svcAcctObj := objects[3] + hpaObj := objects[3] + hpa, ok := hpaObj.(*autoscalingv2.HorizontalPodAutoscaler) + g.Expect(ok).To(BeTrue()) + validateMeta(hpa, deploymentNSName.Name) + + svcAcctObj := objects[4] svcAcct, ok := svcAcctObj.(*corev1.ServiceAccount) g.Expect(ok).To(BeTrue()) validateMeta(svcAcct, deploymentNSName.Name) - cmObj := objects[4] + cmObj := objects[5] cm, ok := cmObj.(*corev1.ConfigMap) g.Expect(ok).To(BeTrue()) validateMeta(cm, controller.CreateNginxResourceName(deploymentNSName.Name, nginxIncludesConfigMapNameSuffix)) - cmObj = objects[5] + cmObj = objects[6] cm, ok = cmObj.(*corev1.ConfigMap) g.Expect(ok).To(BeTrue()) validateMeta(cm, controller.CreateNginxResourceName(deploymentNSName.Name, nginxAgentConfigMapNameSuffix)) @@ -888,7 +894,7 @@ func TestBuildNginxResourceObjectsForDeletion_Plus(t *testing.T) { objects := provisioner.buildNginxResourceObjectsForDeletion(deploymentNSName) - g.Expect(objects).To(HaveLen(11)) + g.Expect(objects).To(HaveLen(12)) validateMeta := func(obj client.Object, name string) { g.Expect(obj.GetName()).To(Equal(name)) @@ -910,22 +916,27 @@ func TestBuildNginxResourceObjectsForDeletion_Plus(t *testing.T) { g.Expect(ok).To(BeTrue()) validateMeta(svc, deploymentNSName.Name) - svcAcctObj := objects[3] + hpaObj := objects[3] + hpa, ok := hpaObj.(*autoscalingv2.HorizontalPodAutoscaler) + g.Expect(ok).To(BeTrue()) + validateMeta(hpa, deploymentNSName.Name) + + svcAcctObj := objects[4] svcAcct, ok := svcAcctObj.(*corev1.ServiceAccount) g.Expect(ok).To(BeTrue()) validateMeta(svcAcct, deploymentNSName.Name) - cmObj := objects[4] + cmObj := objects[5] cm, ok := cmObj.(*corev1.ConfigMap) g.Expect(ok).To(BeTrue()) validateMeta(cm, controller.CreateNginxResourceName(deploymentNSName.Name, nginxIncludesConfigMapNameSuffix)) - cmObj = objects[5] + cmObj = objects[6] cm, ok = cmObj.(*corev1.ConfigMap) g.Expect(ok).To(BeTrue()) validateMeta(cm, controller.CreateNginxResourceName(deploymentNSName.Name, nginxAgentConfigMapNameSuffix)) - secretObj := objects[6] + secretObj := objects[7] secret, ok := secretObj.(*corev1.Secret) g.Expect(ok).To(BeTrue()) validateMeta(secret, controller.CreateNginxResourceName( @@ -933,7 +944,7 @@ func TestBuildNginxResourceObjectsForDeletion_Plus(t *testing.T) { provisioner.cfg.AgentTLSSecretName, )) - secretObj = objects[7] + secretObj = objects[8] secret, ok = secretObj.(*corev1.Secret) g.Expect(ok).To(BeTrue()) validateMeta(secret, controller.CreateNginxResourceName( @@ -941,7 +952,7 @@ func TestBuildNginxResourceObjectsForDeletion_Plus(t *testing.T) { provisioner.cfg.NginxDockerSecretNames[0], )) - secretObj = objects[8] + secretObj = objects[9] secret, ok = secretObj.(*corev1.Secret) g.Expect(ok).To(BeTrue()) validateMeta(secret, controller.CreateNginxResourceName( @@ -949,7 +960,7 @@ func TestBuildNginxResourceObjectsForDeletion_Plus(t *testing.T) { provisioner.cfg.PlusUsageConfig.CASecretName, )) - secretObj = objects[9] + secretObj = objects[10] secret, ok = secretObj.(*corev1.Secret) g.Expect(ok).To(BeTrue()) validateMeta(secret, controller.CreateNginxResourceName( @@ -971,19 +982,24 @@ func TestBuildNginxResourceObjectsForDeletion_OpenShift(t *testing.T) { objects := provisioner.buildNginxResourceObjectsForDeletion(deploymentNSName) - g.Expect(objects).To(HaveLen(9)) + g.Expect(objects).To(HaveLen(10)) validateMeta := func(obj client.Object, name string) { g.Expect(obj.GetName()).To(Equal(name)) g.Expect(obj.GetNamespace()).To(Equal(deploymentNSName.Namespace)) } - roleObj := objects[3] + hpaObj := objects[3] + hpa, ok := hpaObj.(*autoscalingv2.HorizontalPodAutoscaler) + g.Expect(ok).To(BeTrue()) + validateMeta(hpa, deploymentNSName.Name) + + roleObj := objects[4] role, ok := roleObj.(*rbacv1.Role) g.Expect(ok).To(BeTrue()) validateMeta(role, deploymentNSName.Name) - roleBindingObj := objects[4] + roleBindingObj := objects[5] roleBinding, ok := roleBindingObj.(*rbacv1.RoleBinding) g.Expect(ok).To(BeTrue()) validateMeta(roleBinding, deploymentNSName.Name) diff --git a/internal/controller/provisioner/provisioner.go b/internal/controller/provisioner/provisioner.go index 6c60cf1384..d840d1b665 100644 --- a/internal/controller/provisioner/provisioner.go +++ b/internal/controller/provisioner/provisioner.go @@ -200,6 +200,7 @@ func (p *NginxProvisioner) provisionNginx( var agentConfigMapUpdated, deploymentCreated bool var deploymentObj *appsv1.Deployment var daemonSetObj *appsv1.DaemonSet + for _, obj := range objects { createCtx, cancel := context.WithTimeout(ctx, 30*time.Second) diff --git a/internal/controller/provisioner/provisioner_test.go b/internal/controller/provisioner/provisioner_test.go index 9102a8193f..76edf13d5d 100644 --- a/internal/controller/provisioner/provisioner_test.go +++ b/internal/controller/provisioner/provisioner_test.go @@ -7,6 +7,7 @@ import ( "github.com/go-logr/logr" . "github.com/onsi/gomega" appsv1 "k8s.io/api/apps/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -43,6 +44,7 @@ func createScheme() *runtime.Scheme { utilruntime.Must(gatewayv1.Install(scheme)) utilruntime.Must(corev1.AddToScheme(scheme)) utilruntime.Must(appsv1.AddToScheme(scheme)) + utilruntime.Must(autoscalingv2.AddToScheme(scheme)) return scheme } From e351ae33239f780c416df0ec73497d7994dad240 Mon Sep 17 00:00:00 2001 From: nowjean Date: Wed, 25 Jun 2025 16:18:29 +0900 Subject: [PATCH 02/13] Fix Pipeline err --- charts/nginx-gateway-fabric/values.schema.json | 1 + charts/nginx-gateway-fabric/values.yaml | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/nginx-gateway-fabric/values.schema.json b/charts/nginx-gateway-fabric/values.schema.json index 40b1052053..5f82d063a0 100644 --- a/charts/nginx-gateway-fabric/values.schema.json +++ b/charts/nginx-gateway-fabric/values.schema.json @@ -870,6 +870,7 @@ "type": "object" }, "metrics": { + "description": "Custom or additional autoscaling metrics\nref: https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#support-for-custom-metrics\n- type: Pods\n pods:\n metric:\n name: nginx_gateway_fabric_nginx_process_requests_total\n target:\n type: AverageValue\n averageValue: 10000m", "properties": { "enable": { "default": true, diff --git a/charts/nginx-gateway-fabric/values.yaml b/charts/nginx-gateway-fabric/values.yaml index 2999730897..d073dd024d 100644 --- a/charts/nginx-gateway-fabric/values.yaml +++ b/charts/nginx-gateway-fabric/values.yaml @@ -608,7 +608,6 @@ certGenerator: # -- A list of Gateway objects. View https://gateway-api.sigs.k8s.io/reference/spec/#gateway for full Gateway reference. gateways: [] - # Example gateway object: # name: nginx-gateway # namespace: default From 59510a14d71dd0e5cc8f15b341439f4f854f75d1 Mon Sep 17 00:00:00 2001 From: nowjean Date: Thu, 26 Jun 2025 08:34:30 +0900 Subject: [PATCH 03/13] Update generated files --- apis/v1alpha2/zz_generated.deepcopy.go | 4 ++-- charts/nginx-gateway-fabric/values.schema.json | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/apis/v1alpha2/zz_generated.deepcopy.go b/apis/v1alpha2/zz_generated.deepcopy.go index 3fb3fa78fb..836ffda2d5 100644 --- a/apis/v1alpha2/zz_generated.deepcopy.go +++ b/apis/v1alpha2/zz_generated.deepcopy.go @@ -6,9 +6,9 @@ package v1alpha2 import ( "github.com/nginx/nginx-gateway-fabric/apis/v1alpha1" - "k8s.io/api/autoscaling/v2" + v2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" - "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/runtime" apisv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" ) diff --git a/charts/nginx-gateway-fabric/values.schema.json b/charts/nginx-gateway-fabric/values.schema.json index 5f82d063a0..40b1052053 100644 --- a/charts/nginx-gateway-fabric/values.schema.json +++ b/charts/nginx-gateway-fabric/values.schema.json @@ -870,7 +870,6 @@ "type": "object" }, "metrics": { - "description": "Custom or additional autoscaling metrics\nref: https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#support-for-custom-metrics\n- type: Pods\n pods:\n metric:\n name: nginx_gateway_fabric_nginx_process_requests_total\n target:\n type: AverageValue\n averageValue: 10000m", "properties": { "enable": { "default": true, From 853188abfc230a0f959b95c4a298cccebe352c38 Mon Sep 17 00:00:00 2001 From: nowjean Date: Thu, 26 Jun 2025 13:12:10 +0900 Subject: [PATCH 04/13] Add Test case --- internal/controller/provisioner/objects.go | 15 +++++++++++++++ internal/controller/provisioner/objects_test.go | 8 +++++++- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/internal/controller/provisioner/objects.go b/internal/controller/provisioner/objects.go index d8440d3197..1d1c3fe516 100644 --- a/internal/controller/provisioner/objects.go +++ b/internal/controller/provisioner/objects.go @@ -46,6 +46,21 @@ const ( var emptyDirVolumeSource = corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}} +func autoscalingEnabled(dep *ngfAPIv1alpha2.DeploymentSpec) bool { + return dep != nil && dep.Autoscaling.Enabled +} + +func cloneHPAAnnotationMap(src map[string]string) map[string]string { + if src == nil { + return nil + } + annotations := make(map[string]string, len(src)) + for k, v := range src { + annotations[k] = v + } + return annotations +} + func (p *NginxProvisioner) buildNginxResourceObjects( resourceName string, gateway *gatewayv1.Gateway, diff --git a/internal/controller/provisioner/objects_test.go b/internal/controller/provisioner/objects_test.go index 9f2ea3f8cf..3ed7b0ac0b 100644 --- a/internal/controller/provisioner/objects_test.go +++ b/internal/controller/provisioner/objects_test.go @@ -287,6 +287,12 @@ func TestBuildNginxResourceObjects_NginxProxyConfig(t *testing.T) { }, Deployment: &ngfAPIv1alpha2.DeploymentSpec{ Replicas: helpers.GetPointer[int32](3), + Autoscaling: ngfAPIv1alpha2.HPASpec{ + Enabled: true, + MinReplicas: helpers.GetPointer[int32](1), + MaxReplicas: 5, + TargetMemoryUtilizationPercentage: helpers.GetPointer[int32](60), + }, Pod: ngfAPIv1alpha2.PodSpec{ TerminationGracePeriodSeconds: helpers.GetPointer[int64](25), }, @@ -314,7 +320,7 @@ func TestBuildNginxResourceObjects_NginxProxyConfig(t *testing.T) { objects, err := provisioner.buildNginxResourceObjects(resourceName, gateway, nProxyCfg) g.Expect(err).ToNot(HaveOccurred()) - g.Expect(objects).To(HaveLen(6)) + g.Expect(objects).To(HaveLen(7)) cmObj := objects[1] cm, ok := cmObj.(*corev1.ConfigMap) From 81f5d6ab5f190e21f753ccf9cb52ac88cca29e32 Mon Sep 17 00:00:00 2001 From: nowjean Date: Thu, 26 Jun 2025 16:36:11 +0900 Subject: [PATCH 05/13] Refactoring --- internal/controller/provisioner/objects.go | 21 +-------------------- 1 file changed, 1 insertion(+), 20 deletions(-) diff --git a/internal/controller/provisioner/objects.go b/internal/controller/provisioner/objects.go index 1d1c3fe516..658e0f0df9 100644 --- a/internal/controller/provisioner/objects.go +++ b/internal/controller/provisioner/objects.go @@ -46,21 +46,6 @@ const ( var emptyDirVolumeSource = corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}} -func autoscalingEnabled(dep *ngfAPIv1alpha2.DeploymentSpec) bool { - return dep != nil && dep.Autoscaling.Enabled -} - -func cloneHPAAnnotationMap(src map[string]string) map[string]string { - if src == nil { - return nil - } - annotations := make(map[string]string, len(src)) - for k, v := range src { - annotations[k] = v - } - return annotations -} - func (p *NginxProvisioner) buildNginxResourceObjects( resourceName string, gateway *gatewayv1.Gateway, @@ -1055,22 +1040,18 @@ func (p *NginxProvisioner) buildImage(nProxyCfg *graph.EffectiveNginxProxy) (str return fmt.Sprintf("%s:%s", image, tag), pullPolicy } -func getMetricTargetByType( - target autoscalingv2.MetricTarget, -) autoscalingv2.MetricTarget { +func getMetricTargetByType(target autoscalingv2.MetricTarget) autoscalingv2.MetricTarget { switch target.Type { case autoscalingv2.UtilizationMetricType: return autoscalingv2.MetricTarget{ Type: autoscalingv2.UtilizationMetricType, AverageUtilization: target.AverageUtilization, } - case autoscalingv2.AverageValueMetricType: return autoscalingv2.MetricTarget{ Type: autoscalingv2.AverageValueMetricType, AverageValue: target.AverageValue, } - case autoscalingv2.ValueMetricType: return autoscalingv2.MetricTarget{ Type: autoscalingv2.ValueMetricType, From f686a2d4fa3422a6c88cfcc70993f9e16c104b4d Mon Sep 17 00:00:00 2001 From: nowjean Date: Mon, 30 Jun 2025 13:14:24 +0900 Subject: [PATCH 06/13] Code Review --- apis/v1alpha2/nginxproxy_types.go | 6 ++ .../templates/nginxproxy.yaml | 6 ++ .../nginx-gateway-fabric/values.schema.json | 73 +------------------ charts/nginx-gateway-fabric/values.yaml | 27 ++++--- .../bases/gateway.nginx.org_nginxproxies.yaml | 45 ++++++++++++ deploy/azure/deploy.yaml | 6 -- deploy/crds.yaml | 14 ++++ deploy/default/deploy.yaml | 6 -- deploy/experimental-nginx-plus/deploy.yaml | 6 -- deploy/experimental/deploy.yaml | 6 -- deploy/nginx-plus/deploy.yaml | 6 -- deploy/nodeport/deploy.yaml | 6 -- deploy/openshift/deploy.yaml | 6 -- .../snippets-filters-nginx-plus/deploy.yaml | 6 -- deploy/snippets-filters/deploy.yaml | 6 -- 15 files changed, 88 insertions(+), 137 deletions(-) diff --git a/apis/v1alpha2/nginxproxy_types.go b/apis/v1alpha2/nginxproxy_types.go index 09a91f6bca..ffa0ef7e32 100644 --- a/apis/v1alpha2/nginxproxy_types.go +++ b/apis/v1alpha2/nginxproxy_types.go @@ -466,6 +466,12 @@ type DaemonSetSpec struct { Patches []Patch `json:"patches,omitempty"` } +// +kubebuilder:validation:XValidation:message="at least one metric must be specified when autoscaling is enabled",rule="!self.enabled || (has(self.targetCPUUtilizationPercentage) || has(self.targetMemoryUtilizationPercentage) || (has(self.autoscalingTemplate) && size(self.autoscalingTemplate) > 0))" +// +kubebuilder:validation:XValidation:message="minReplicas must be less than or equal to maxReplicas",rule="self.minReplicas <= self.maxReplicas" +// +kubebuilder:validation:XValidation:message="CPU utilization must be between 1 and 100",rule="!has(self.targetCPUUtilizationPercentage) || (self.targetCPUUtilizationPercentage >= 1 && self.targetCPUUtilizationPercentage <= 100)" +// +kubebuilder:validation:XValidation:message="memory utilization must be between 1 and 100",rule="!has(self.targetMemoryUtilizationPercentage) || (self.targetMemoryUtilizationPercentage >= 1 && self.targetMemoryUtilizationPercentage <= 100)" +// +//nolint:lll type HPASpec struct { // Behavior configures the scaling behavior of the target // in both Up and Down directions (scaleUp and scaleDown fields respectively). diff --git a/charts/nginx-gateway-fabric/templates/nginxproxy.yaml b/charts/nginx-gateway-fabric/templates/nginxproxy.yaml index 94b96a8f34..d83f0f602d 100644 --- a/charts/nginx-gateway-fabric/templates/nginxproxy.yaml +++ b/charts/nginx-gateway-fabric/templates/nginxproxy.yaml @@ -15,6 +15,7 @@ spec: {{- if .Values.nginx.replicas }} replicas: {{ .Values.nginx.replicas }} {{- end }} + {{- if .Values.nginx.autoscaling.enabled }} autoscaling: enabled: {{ .Values.nginx.autoscaling.enabled }} {{- if .Values.nginx.autoscaling.hpaAnnotations }} @@ -23,8 +24,12 @@ spec: {{- end }} minReplicas: {{ .Values.nginx.autoscaling.minReplicas }} maxReplicas: {{ .Values.nginx.autoscaling.maxReplicas }} + {{- if .Values.nginx.autoscaling.targetCPUUtilizationPercentage }} targetCPUUtilizationPercentage: {{ .Values.nginx.autoscaling.targetCPUUtilizationPercentage }} + {{- end }} + {{- if .Values.nginx.autoscaling.targetMemoryUtilizationPercentage }} targetMemoryUtilizationPercentage: {{ .Values.nginx.autoscaling.targetMemoryUtilizationPercentage }} + {{- end }} {{- if .Values.nginx.autoscaling.behavior }} behavior: {{- toYaml .Values.nginx.autoscaling.behavior | nindent 10 }} @@ -33,6 +38,7 @@ spec: autoscalingTemplate: {{- toYaml .Values.nginx.autoscalingTemplate | nindent 8 }} {{- end }} + {{- end }} {{- if .Values.nginx.pod }} pod: {{- toYaml .Values.nginx.pod | nindent 8 }} diff --git a/charts/nginx-gateway-fabric/values.schema.json b/charts/nginx-gateway-fabric/values.schema.json index 40b1052053..d7309e8d99 100644 --- a/charts/nginx-gateway-fabric/values.schema.json +++ b/charts/nginx-gateway-fabric/values.schema.json @@ -100,46 +100,12 @@ "properties": { "autoscaling": { "properties": { - "behavior": { - "required": [], - "title": "behavior", - "type": "object" - }, "enabled": { - "default": true, + "default": false, "description": "Enable or disable Horizontal Pod Autoscaler", "required": [], "title": "enabled", "type": "boolean" - }, - "hpaAnnotations": { - "required": [], - "title": "hpaAnnotations", - "type": "object" - }, - "maxReplicas": { - "default": 11, - "required": [], - "title": "maxReplicas", - "type": "integer" - }, - "minReplicas": { - "default": 1, - "required": [], - "title": "minReplicas", - "type": "integer" - }, - "targetCPUUtilizationPercentage": { - "default": 50, - "required": [], - "title": "targetCPUUtilizationPercentage", - "type": "integer" - }, - "targetMemoryUtilizationPercentage": { - "default": 50, - "required": [], - "title": "targetMemoryUtilizationPercentage", - "type": "integer" } }, "required": [], @@ -376,6 +342,7 @@ }, "container": { "description": "The container configuration for the NGINX container. This is applied globally to all Gateways managed by this\ninstance of NGINX Gateway Fabric.", +<<<<<<< HEAD "properties": { "hostPorts": { "description": "A list of HostPorts to expose on the host.\nThis configuration allows containers to bind to a specific port on the host node,\nenabling external network traffic to reach the container directly through the host's IP address and port.\nUse this option when you need to expose container ports on the host for direct access,\nsuch as for debugging, legacy integrations, or when NodePort/LoadBalancer services are not suitable.\nNote: Using hostPort may have security and scheduling implications, as it ties pods to specific nodes and ports.", @@ -429,6 +396,8 @@ "type": "array" } }, +======= +>>>>>>> Code Review "required": [], "title": "container", "type": "object" @@ -659,46 +628,12 @@ }, "autoscaling": { "properties": { - "annotations": { - "required": [], - "title": "annotations", - "type": "object" - }, - "behavior": { - "required": [], - "title": "behavior", - "type": "object" - }, "enabled": { "default": false, "description": "Enable or disable Horizontal Pod Autoscaler", "required": [], "title": "enabled", "type": "boolean" - }, - "maxReplicas": { - "default": 11, - "required": [], - "title": "maxReplicas", - "type": "integer" - }, - "minReplicas": { - "default": 1, - "required": [], - "title": "minReplicas", - "type": "integer" - }, - "targetCPUUtilizationPercentage": { - "default": 50, - "required": [], - "title": "targetCPUUtilizationPercentage", - "type": "integer" - }, - "targetMemoryUtilizationPercentage": { - "default": 50, - "required": [], - "title": "targetMemoryUtilizationPercentage", - "type": "integer" } }, "required": [], diff --git a/charts/nginx-gateway-fabric/values.yaml b/charts/nginx-gateway-fabric/values.yaml index d073dd024d..9e2c395714 100644 --- a/charts/nginx-gateway-fabric/values.yaml +++ b/charts/nginx-gateway-fabric/values.yaml @@ -14,7 +14,6 @@ nginxGateway: # -- The name of the NGINX Gateway Fabric deployment - if not present, then by default uses release name given during installation. name: "" - # @schema # required: true # type: string @@ -161,12 +160,12 @@ nginxGateway: autoscaling: # Enable or disable Horizontal Pod Autoscaler enabled: false - annotations: {} - minReplicas: 1 - maxReplicas: 11 - targetCPUUtilizationPercentage: 50 - targetMemoryUtilizationPercentage: 50 - behavior: {} + # annotations: {} + # minReplicas: 1 + # maxReplicas: 11 + # targetCPUUtilizationPercentage: 50 + # targetMemoryUtilizationPercentage: 50 + # behavior: {} # scaleDown: # stabilizationWindowSeconds: 300 # policies: @@ -232,13 +231,13 @@ nginx: autoscaling: # Enable or disable Horizontal Pod Autoscaler - enabled: true - hpaAnnotations: {} - minReplicas: 1 - maxReplicas: 11 - targetCPUUtilizationPercentage: 50 - targetMemoryUtilizationPercentage: 50 - behavior: {} + enabled: false + # hpaAnnotations: {} + # minReplicas: 1 + # maxReplicas: 11 + # targetCPUUtilizationPercentage: 50 + # targetMemoryUtilizationPercentage: 50 + # behavior: {} # scaleDown: # stabilizationWindowSeconds: 300 # policies: diff --git a/config/crd/bases/gateway.nginx.org_nginxproxies.yaml b/config/crd/bases/gateway.nginx.org_nginxproxies.yaml index bbd2acc238..85232b0dfb 100644 --- a/config/crd/bases/gateway.nginx.org_nginxproxies.yaml +++ b/config/crd/bases/gateway.nginx.org_nginxproxies.yaml @@ -4012,7 +4012,11 @@ spec: type: array behavior: description: |- +<<<<<<< HEAD Behavior configures the scaling behavior of the target +======= + behavior configures the scaling behavior of the target +>>>>>>> Code Review in both Up and Down directions (scaleUp and scaleDown fields respectively). If not set, the default HPAScalingRules for scale up and scale down are used. properties: @@ -4171,6 +4175,19 @@ spec: enabled: description: Enable or disable Horizontal Pod Autoscaler type: boolean +<<<<<<< HEAD +======= + hpaAnnotations: + additionalProperties: + type: string + description: |- + Annotation for Horizontal Pod Autoscaler + Annotations is an unstructured key value map stored with a resource that may be + set by external tools to store and retrieve arbitrary metadata. They are not + queryable and should be preserved when modifying objects. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations + type: object +>>>>>>> Code Review maxReplicas: description: Maximum number of replicas. format: int32 @@ -4191,6 +4208,20 @@ spec: - enabled - maxReplicas type: object + x-kubernetes-validations: + - message: at least one metric must be specified when autoscaling + is enabled + rule: '!self.enabled || (has(self.targetCPUUtilizationPercentage) + || has(self.targetMemoryUtilizationPercentage) || (has(self.autoscalingTemplate) + && size(self.autoscalingTemplate) > 0))' + - message: minReplicas must be less than or equal to maxReplicas + rule: self.minReplicas <= self.maxReplicas + - message: CPU utilization must be between 1 and 100 + rule: '!has(self.targetCPUUtilizationPercentage) || (self.targetCPUUtilizationPercentage + >= 1 && self.targetCPUUtilizationPercentage <= 100)' + - message: memory utilization must be between 1 and 100 + rule: '!has(self.targetMemoryUtilizationPercentage) || (self.targetMemoryUtilizationPercentage + >= 1 && self.targetMemoryUtilizationPercentage <= 100)' container: description: Container defines container fields for the NGINX container. @@ -7688,14 +7719,28 @@ spec: automatically if required. The default NodePort range enforced by Kubernetes is 30000-32767. properties: listenerPort: +<<<<<<< HEAD description: ListenerPort is the Gateway listener port that this NodePort maps to. +======= + description: |- + ListenerPort is the Gateway listener port that this NodePort maps to. + kubebuilder:validation:Minimum=1 + kubebuilder:validation:Maximum=65535 +>>>>>>> Code Review format: int32 maximum: 65535 minimum: 1 type: integer port: +<<<<<<< HEAD description: Port is the NodePort to expose. +======= + description: |- + Port is the NodePort to expose. + kubebuilder:validation:Minimum=1 + kubebuilder:validation:Maximum=65535 +>>>>>>> Code Review format: int32 maximum: 65535 minimum: 1 diff --git a/deploy/azure/deploy.yaml b/deploy/azure/deploy.yaml index 93fa2c8a69..12ddab6742 100644 --- a/deploy/azure/deploy.yaml +++ b/deploy/azure/deploy.yaml @@ -414,12 +414,6 @@ metadata: spec: kubernetes: deployment: - autoscaling: - enabled: true - maxReplicas: 11 - minReplicas: 1 - targetCPUUtilizationPercentage: 50 - targetMemoryUtilizationPercentage: 50 container: image: pullPolicy: Always diff --git a/deploy/crds.yaml b/deploy/crds.yaml index 6977926c20..f11ce167f1 100644 --- a/deploy/crds.yaml +++ b/deploy/crds.yaml @@ -4776,6 +4776,20 @@ spec: - enabled - maxReplicas type: object + x-kubernetes-validations: + - message: at least one metric must be specified when autoscaling + is enabled + rule: '!self.enabled || (has(self.targetCPUUtilizationPercentage) + || has(self.targetMemoryUtilizationPercentage) || (has(self.autoscalingTemplate) + && size(self.autoscalingTemplate) > 0))' + - message: minReplicas must be less than or equal to maxReplicas + rule: self.minReplicas <= self.maxReplicas + - message: CPU utilization must be between 1 and 100 + rule: '!has(self.targetCPUUtilizationPercentage) || (self.targetCPUUtilizationPercentage + >= 1 && self.targetCPUUtilizationPercentage <= 100)' + - message: memory utilization must be between 1 and 100 + rule: '!has(self.targetMemoryUtilizationPercentage) || (self.targetMemoryUtilizationPercentage + >= 1 && self.targetMemoryUtilizationPercentage <= 100)' container: description: Container defines container fields for the NGINX container. diff --git a/deploy/default/deploy.yaml b/deploy/default/deploy.yaml index ba0b1be9f4..3da8598bfd 100644 --- a/deploy/default/deploy.yaml +++ b/deploy/default/deploy.yaml @@ -412,12 +412,6 @@ metadata: spec: kubernetes: deployment: - autoscaling: - enabled: true - maxReplicas: 11 - minReplicas: 1 - targetCPUUtilizationPercentage: 50 - targetMemoryUtilizationPercentage: 50 container: image: pullPolicy: Always diff --git a/deploy/experimental-nginx-plus/deploy.yaml b/deploy/experimental-nginx-plus/deploy.yaml index 4287e65caa..5b478afd22 100644 --- a/deploy/experimental-nginx-plus/deploy.yaml +++ b/deploy/experimental-nginx-plus/deploy.yaml @@ -420,12 +420,6 @@ metadata: spec: kubernetes: deployment: - autoscaling: - enabled: true - maxReplicas: 11 - minReplicas: 1 - targetCPUUtilizationPercentage: 50 - targetMemoryUtilizationPercentage: 50 container: image: pullPolicy: Always diff --git a/deploy/experimental/deploy.yaml b/deploy/experimental/deploy.yaml index 11465b3d06..cf774bc07b 100644 --- a/deploy/experimental/deploy.yaml +++ b/deploy/experimental/deploy.yaml @@ -417,12 +417,6 @@ metadata: spec: kubernetes: deployment: - autoscaling: - enabled: true - maxReplicas: 11 - minReplicas: 1 - targetCPUUtilizationPercentage: 50 - targetMemoryUtilizationPercentage: 50 container: image: pullPolicy: Always diff --git a/deploy/nginx-plus/deploy.yaml b/deploy/nginx-plus/deploy.yaml index 62a44b9628..c82a24e0e9 100644 --- a/deploy/nginx-plus/deploy.yaml +++ b/deploy/nginx-plus/deploy.yaml @@ -415,12 +415,6 @@ metadata: spec: kubernetes: deployment: - autoscaling: - enabled: true - maxReplicas: 11 - minReplicas: 1 - targetCPUUtilizationPercentage: 50 - targetMemoryUtilizationPercentage: 50 container: image: pullPolicy: Always diff --git a/deploy/nodeport/deploy.yaml b/deploy/nodeport/deploy.yaml index 8b58ba7f0b..438e6f975d 100644 --- a/deploy/nodeport/deploy.yaml +++ b/deploy/nodeport/deploy.yaml @@ -412,12 +412,6 @@ metadata: spec: kubernetes: deployment: - autoscaling: - enabled: true - maxReplicas: 11 - minReplicas: 1 - targetCPUUtilizationPercentage: 50 - targetMemoryUtilizationPercentage: 50 container: image: pullPolicy: Always diff --git a/deploy/openshift/deploy.yaml b/deploy/openshift/deploy.yaml index e121698c9c..cc05b461b0 100644 --- a/deploy/openshift/deploy.yaml +++ b/deploy/openshift/deploy.yaml @@ -434,12 +434,6 @@ metadata: spec: kubernetes: deployment: - autoscaling: - enabled: true - maxReplicas: 11 - minReplicas: 1 - targetCPUUtilizationPercentage: 50 - targetMemoryUtilizationPercentage: 50 container: image: pullPolicy: Always diff --git a/deploy/snippets-filters-nginx-plus/deploy.yaml b/deploy/snippets-filters-nginx-plus/deploy.yaml index 8150952bbb..79b8a2bf0f 100644 --- a/deploy/snippets-filters-nginx-plus/deploy.yaml +++ b/deploy/snippets-filters-nginx-plus/deploy.yaml @@ -418,12 +418,6 @@ metadata: spec: kubernetes: deployment: - autoscaling: - enabled: true - maxReplicas: 11 - minReplicas: 1 - targetCPUUtilizationPercentage: 50 - targetMemoryUtilizationPercentage: 50 container: image: pullPolicy: Always diff --git a/deploy/snippets-filters/deploy.yaml b/deploy/snippets-filters/deploy.yaml index c28df71c3b..dc4cf56dd7 100644 --- a/deploy/snippets-filters/deploy.yaml +++ b/deploy/snippets-filters/deploy.yaml @@ -415,12 +415,6 @@ metadata: spec: kubernetes: deployment: - autoscaling: - enabled: true - maxReplicas: 11 - minReplicas: 1 - targetCPUUtilizationPercentage: 50 - targetMemoryUtilizationPercentage: 50 container: image: pullPolicy: Always From 1e1a296cc3f13dd9744e15d3bba43534500a26b2 Mon Sep 17 00:00:00 2001 From: nowjean Date: Tue, 1 Jul 2025 19:07:37 +0900 Subject: [PATCH 07/13] Code Review & HPA event handler --- apis/v1alpha2/nginxproxy_types.go | 4 +- .../templates/clusterrole.yaml | 4 + .../nginx-gateway-fabric/templates/hpa.yaml | 4 +- charts/nginx-gateway-fabric/values-aks.yaml | 657 ++++++++++++++++++ charts/nginx-gateway-fabric/values.yaml | 1 + .../bases/gateway.nginx.org_nginxproxies.yaml | 7 +- deploy/azure/deploy.yaml | 2 - deploy/crds.yaml | 3 +- deploy/default/deploy.yaml | 2 - deploy/experimental-nginx-plus/deploy.yaml | 2 - deploy/experimental/deploy.yaml | 2 - deploy/nginx-plus/deploy.yaml | 2 - deploy/nodeport/deploy.yaml | 2 - deploy/openshift/deploy.yaml | 2 - .../snippets-filters-nginx-plus/deploy.yaml | 2 - deploy/snippets-filters/deploy.yaml | 2 - internal/controller/provisioner/eventloop.go | 14 + internal/controller/provisioner/handler.go | 2 +- .../controller/provisioner/provisioner.go | 1 - internal/controller/provisioner/setter.go | 16 + internal/controller/provisioner/store.go | 18 + 21 files changed, 724 insertions(+), 25 deletions(-) create mode 100644 charts/nginx-gateway-fabric/values-aks.yaml diff --git a/apis/v1alpha2/nginxproxy_types.go b/apis/v1alpha2/nginxproxy_types.go index ffa0ef7e32..0a17861b28 100644 --- a/apis/v1alpha2/nginxproxy_types.go +++ b/apis/v1alpha2/nginxproxy_types.go @@ -432,7 +432,7 @@ type DeploymentSpec struct { // +optional Replicas *int32 `json:"replicas,omitempty"` - // Horizontal Pod Autoscaling. + // Autoscaling defines the configuration for Horizontal Pod Autoscaling. // // +optional Autoscaling HPASpec `json:"autoscaling"` @@ -471,6 +471,8 @@ type DaemonSetSpec struct { // +kubebuilder:validation:XValidation:message="CPU utilization must be between 1 and 100",rule="!has(self.targetCPUUtilizationPercentage) || (self.targetCPUUtilizationPercentage >= 1 && self.targetCPUUtilizationPercentage <= 100)" // +kubebuilder:validation:XValidation:message="memory utilization must be between 1 and 100",rule="!has(self.targetMemoryUtilizationPercentage) || (self.targetMemoryUtilizationPercentage >= 1 && self.targetMemoryUtilizationPercentage <= 100)" // +// HPASpec is the configuration for the Horizontal Pod Autoscaling. +// //nolint:lll type HPASpec struct { // Behavior configures the scaling behavior of the target diff --git a/charts/nginx-gateway-fabric/templates/clusterrole.yaml b/charts/nginx-gateway-fabric/templates/clusterrole.yaml index 8fc4da400e..84d32a1fc8 100644 --- a/charts/nginx-gateway-fabric/templates/clusterrole.yaml +++ b/charts/nginx-gateway-fabric/templates/clusterrole.yaml @@ -8,7 +8,9 @@ rules: - apiGroups: - "" - apps + {{- if or .Values.nginx.autoscaling.enabled .Values.nginxGateway.autoscaling.enabled }} - autoscaling + {{- end }} resources: - secrets - configmaps @@ -16,7 +18,9 @@ rules: - services - deployments - daemonsets + {{- if or .Values.nginx.autoscaling.enabled .Values.nginxGateway.autoscaling.enabled }} - horizontalpodautoscalers + {{- end }} verbs: - create - update diff --git a/charts/nginx-gateway-fabric/templates/hpa.yaml b/charts/nginx-gateway-fabric/templates/hpa.yaml index 2835eeff9f..86730bcfa5 100644 --- a/charts/nginx-gateway-fabric/templates/hpa.yaml +++ b/charts/nginx-gateway-fabric/templates/hpa.yaml @@ -1,5 +1,5 @@ -{{- if and (eq .Values.nginxGateway.kind "deployment") .Values.nginxGateway.autoscaling.enabled -}} -apiVersion: {{ ternary "autoscaling/v2" "autoscaling/v2beta2" (.Capabilities.APIVersions.Has "autoscaling/v2") }} +{{- if and (eq .Values.nginxGateway.kind "deployment") .Values.nginxGateway.autoscaling.enabled (.Capabilities.APIVersions.Has "autoscaling/v2") -}} +apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler metadata: {{- with .Values.nginxGateway.autoscaling.annotations }} diff --git a/charts/nginx-gateway-fabric/values-aks.yaml b/charts/nginx-gateway-fabric/values-aks.yaml new file mode 100644 index 0000000000..31cd16b6f6 --- /dev/null +++ b/charts/nginx-gateway-fabric/values-aks.yaml @@ -0,0 +1,657 @@ +# yaml-language-server: $schema=values.schema.json + +# -- The DNS cluster domain of your Kubernetes cluster. +clusterDomain: cluster.local + +# -- The nginxGateway section contains configuration for the NGINX Gateway Fabric control plane deployment. +nginxGateway: + # FIXME(lucacome): https://github.com/nginx/nginx-gateway-fabric/issues/2490 + + # @schema + # const: deployment + # @schema + # -- The kind of the NGINX Gateway Fabric installation - currently, only deployment is supported. + kind: deployment + + # @schema + # required: true + # type: string + # @schema + # -- The name of the GatewayClass that will be created as part of this release. Every NGINX Gateway + # Fabric must have a unique corresponding GatewayClass resource. NGINX Gateway Fabric only processes resources that + # belong to its class - i.e. have the "gatewayClassName" field resource equal to the class. + gatewayClassName: nginx-test + + # -- Set of custom annotations for the NGINX Gateway Fabric pods. + podAnnotations: {} + + # -- Set of labels to be added for NGINX Gateway Fabric deployment. + labels: {} + + # -- Set of custom annotations for GatewayClass objects. + gatewayClassAnnotations: {} + + # @schema + # pattern: ^gateway.nginx.org/.* + # required: true + # @schema + # -- The name of the Gateway controller. The controller name must be of the form: DOMAIN/PATH. The controller's domain + # is gateway.nginx.org. + gatewayControllerName: gateway.nginx.org/nginx-gateway-controller + + # The dynamic configuration for the control plane that is contained in the NginxGateway resource. + config: + logging: + # @schema + # enum: + # - info + # - debug + # - error + # @schema + # -- Log level. + level: info + + # -- Set of custom annotations for NginxGateway objects. + configAnnotations: {} + + # -- The service configuration for the NGINX Gateway Fabric control plane. + service: + # -- The annotations of the NGINX Gateway Fabric control plane service. + annotations: {} + + # -- The labels of the NGINX Gateway Fabric control plane service. + labels: {} + + # -- The serviceaccount configuration for the NGINX Gateway Fabric control plane. + serviceAccount: + # -- Set of custom annotations for the NGINX Gateway Fabric control plane service account. + annotations: {} + + # -- The name of the service account of the NGINX Gateway Fabric control plane pods. Used for RBAC. + # @default -- Autogenerated if not set or set to "" + name: "" + + # -- The name of the secret containing docker registry credentials for the control plane. + # Secret must exist in the same namespace as the helm release. + imagePullSecret: "" + + # -- A list of secret names containing docker registry credentials for the control plane. + # Secrets must exist in the same namespace as the helm release. + imagePullSecrets: [] + + # -- The number of replicas of the NGINX Gateway Fabric Deployment. + replicas: 1 + + # The configuration for leader election. + leaderElection: + # -- Enable leader election. Leader election is used to avoid multiple replicas of the NGINX Gateway Fabric + # reporting the status of the Gateway API resources. If not enabled, all replicas of NGINX Gateway Fabric + # will update the statuses of the Gateway API resources. + enable: true + + # -- The name of the leader election lock. A Lease object with this name will be created in the same Namespace as + # the controller. + # @default -- Autogenerated if not set or set to "". + lockName: "" + + ## Defines the settings for the control plane readiness probe. This probe returns Ready when the controller + ## has started and is ready to configure NGINX. + readinessProbe: + # -- Enable the /readyz endpoint on the control plane. + enable: true + + # @schema + # type: integer + # minimum: 1 + # maximum: 65535 + # @schema + # -- Port in which the readiness endpoint is exposed. + port: 8081 + + # -- The number of seconds after the Pod has started before the readiness probes are initiated. + initialDelaySeconds: 3 + + # -- The image configuration for the NGINX Gateway Fabric control plane. + image: + # -- The NGINX Gateway Fabric image to use + #repository: ghcr.io/nginx/nginx-gateway-fabric + repository: prodonlinelabcontainerreg.azurecr.io/nginx-gateway-fabric + tag: edge + # @schema + # enum: + # - Always + # - IfNotPresent + # - Never + # @schema + pullPolicy: Always + + productTelemetry: + # -- Enable the collection of product telemetry. + enable: true + + # -- The lifecycle of the nginx-gateway container. + lifecycle: {} + + # -- The resource requests and/or limits of the nginx-gateway container. + #resources: {} + resources: + requests: + cpu: 1000m + memory: 1Gi + + # -- extraVolumes for the NGINX Gateway Fabric control plane pod. Use in conjunction with + # nginxGateway.extraVolumeMounts mount additional volumes to the container. + extraVolumes: [] + + # -- extraVolumeMounts are the additional volume mounts for the nginx-gateway container. + extraVolumeMounts: [] + + # -- The termination grace period of the NGINX Gateway Fabric control plane pod. + terminationGracePeriodSeconds: 30 + + # -- Tolerations for the NGINX Gateway Fabric control plane pod. + tolerations: + - key: "kubernetes.azure.com/scalesetpriority" + operator: "Equal" + value: "spot" + effect: "NoSchedule" + + # -- The nodeSelector of the NGINX Gateway Fabric control plane pod. + nodeSelector: {} + + # -- The affinity of the NGINX Gateway Fabric control plane pod. + affinity: {} + + # -- The topology spread constraints for the NGINX Gateway Fabric control plane pod. + topologySpreadConstraints: [] + + autoscaling: + # Enable or disable Horizontal Pod Autoscaler + enabled: true + annotations: {} + minReplicas: 1 + maxReplicas: 11 + targetCPUUtilizationPercentage: 50 + targetMemoryUtilizationPercentage: 50 + behavior: {} + # scaleDown: + # stabilizationWindowSeconds: 300 + # policies: + # - type: Pods + # value: 1 + # periodSeconds: 180 + # scaleUp: + # stabilizationWindowSeconds: 300 + # policies: + # - type: Pods + # value: 2 + # periodSeconds: 60 + autoscalingTemplate: [] + # Custom or additional autoscaling metrics + # ref: https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#support-for-custom-metrics + # - type: Pods + # pods: + # metric: + # name: nginx_gateway_fabric_nginx_process_requests_total + # target: + # type: AverageValue + # averageValue: 10000m + + metrics: + # -- Enable exposing metrics in the Prometheus format. + enable: true + + # @schema + # type: integer + # minimum: 1 + # maximum: 65535 + # @schema + # -- Set the port where the Prometheus metrics are exposed. + port: 9113 + + # -- Enable serving metrics via https. By default metrics are served via http. + # Please note that this endpoint will be secured with a self-signed certificate. + secure: false + + gwAPIExperimentalFeatures: + # -- Enable the experimental features of Gateway API which are supported by NGINX Gateway Fabric. Requires the Gateway + # APIs installed from the experimental channel. + enable: false + + snippetsFilters: + # -- Enable SnippetsFilters feature. SnippetsFilters allow inserting NGINX configuration into the generated NGINX + # config for HTTPRoute and GRPCRoute resources. + enable: true + +# -- The nginx section contains the configuration for all NGINX data plane deployments +# installed by the NGINX Gateway Fabric control plane. +nginx: + # @schema + # enum: + # - deployment + # - daemonSet + # @schema + # -- The kind of NGINX deployment. + kind: deployment + + # -- The number of replicas of the NGINX Deployment. + replicas: 1 + + autoscaling: + # Enable or disable Horizontal Pod Autoscaler + enabled: true + hpaAnnotations: + test/owner: "woojin" + minReplicas: 1 + maxReplicas: 11 + targetCPUUtilizationPercentage: 50 + targetMemoryUtilizationPercentage: 50 + behavior: + {} + # scaleDown: + # stabilizationWindowSeconds: 300 + # policies: + # - type: Pods + # value: 1 + # periodSeconds: 180 + # scaleUp: + # stabilizationWindowSeconds: 300 + # policies: + # - type: Pods + # value: 2 + # periodSeconds: 60 + autoscalingTemplate: + [] + # - type: Pods + # pods: + # metric: + # name: nginx_gateway_fabric_nginx_process_requests_total + # target: + # type: AverageValue + # averageValue: 10000m + # Custom or additional autoscaling metrics + # ref: https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#support-for-custom-metrics + + image: + # -- The NGINX image to use. + #repository: ghcr.io/nginx/nginx-gateway-fabric/nginx + repository: prodonlinelabcontainerreg.azurecr.io/nginx-gateway-fabric/nginx + tag: edge + # @schema + # enum: + # - Always + # - IfNotPresent + # - Never + # @schema + pullPolicy: Always + + # -- Is NGINX Plus image being used. + plus: false + + # -- The name of the secret containing docker registry credentials. + # Secret must exist in the same namespace as the helm release. The control + # plane will copy this secret into any namespace where NGINX is deployed. + imagePullSecret: "" + + # -- A list of secret names containing docker registry credentials. + # Secrets must exist in the same namespace as the helm release. The control + # plane will copy these secrets into any namespace where NGINX is deployed. + imagePullSecrets: [] + + # Configuration for NGINX Plus usage reporting. + usage: + # -- The name of the Secret containing the JWT for NGINX Plus usage reporting. Must exist in the same namespace + # that the NGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway). + secretName: "nplus-license" + + # -- The endpoint of the NGINX Plus usage reporting server. Default: product.connect.nginx.com + endpoint: "" + + # -- The nameserver used to resolve the NGINX Plus usage reporting endpoint. Used with NGINX Instance Manager. + resolver: "" + + # -- Disable client verification of the NGINX Plus usage reporting server certificate. + skipVerify: false + + # -- The name of the Secret containing the NGINX Instance Manager CA certificate. + # Must exist in the same namespace that the NGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway). + caSecretName: "" + + # -- The name of the Secret containing the client certificate and key for authenticating with NGINX Instance Manager. + # Must exist in the same namespace that the NGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway). + clientSSLSecretName: "" + + # @schema + # type: object + # properties: + # disableHTTP2: + # description: DisableHTTP2 defines if http2 should be disabled for all servers. + # type: boolean + # ipFamily: + # description: IPFamily specifies the IP family to be used by the NGINX. + # type: string + # enum: + # - ipv4 + # - ipv6 + # - dual + # rewriteClientIP: + # type: object + # description: RewriteClientIP defines configuration for rewriting the client IP to the original client's IP. + # properties: + # mode: + # type: string + # enum: + # - ProxyProtocol + # - XForwardedFor + # setIPRecursively: + # type: boolean + # trustedAddresses: + # type: array + # items: + # properties: + # type: + # type: string + # enum: + # - CIDR + # - IPAddress + # - Hostname + # value: + # type: string + # telemetry: + # type: object + # description: Telemetry specifies the OpenTelemetry configuration. + # properties: + # exporter: + # type: object + # properties: + # endpoint: + # type: string + # pattern: ^(?:http?:\/\/)?[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?(?:\.[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?)*(?::\d{1,5})?$ + # interval: + # type: string + # pattern: ^\d{1,4}(ms|s)?$ + # batchSize: + # type: integer + # minimum: 0 + # batchCount: + # type: integer + # minimum: 0 + # serviceName: + # type: string + # pattern: ^[a-zA-Z0-9_-]+$ + # spanAttributes: + # type: array + # items: + # properties: + # key: + # type: string + # pattern: ^([^"$\\]|\\[^$])*$ + # minLength: 1 + # maxLength: 255 + # value: + # type: string + # pattern: ^([^"$\\]|\\[^$])*$ + # minLength: 1 + # maxLength: 255 + # disabledFeatures: + # type: array + # items: + # type: string + # enum: + # - DisableTracing + # metrics: + # type: object + # description: Metrics defines the configuration for Prometheus scraping metrics. + # properties: + # disable: + # type: boolean + # port: + # type: integer + # minimum: 1 + # maximum: 65535 + # logging: + # type: object + # description: Logging defines logging related settings for NGINX. + # properties: + # errorLevel: + # type: string + # enum: + # - debug + # - info + # - notice + # - warn + # - error + # - crit + # - alert + # - emerg + # agentLevel: + # type: string + # enum: + # - debug + # - info + # - error + # - panic + # - fatal + # nginxPlus: + # type: object + # description: NginxPlus specifies NGINX Plus additional settings. + # properties: + # allowedAddresses: + # type: array + # items: + # properties: + # type: + # type: string + # enum: + # - CIDR + # - IPAddress + # value: + # type: string + # @schema + # -- The configuration for the data plane that is contained in the NginxProxy resource. This is applied globally to all Gateways + # managed by this instance of NGINX Gateway Fabric. + config: {} + + # -- The pod configuration for the NGINX data plane pod. This is applied globally to all Gateways managed by this + # instance of NGINX Gateway Fabric. + pod: + # -- The termination grace period of the NGINX data plane pod. + # terminationGracePeriodSeconds: 30 + + # -- Tolerations for the NGINX data plane pod. + tolerations: + - key: "kubernetes.azure.com/scalesetpriority" + operator: "Equal" + value: "spot" + effect: "NoSchedule" + + # -- The nodeSelector of the NGINX data plane pod. + # nodeSelector: {} + + # -- The affinity of the NGINX data plane pod. + # affinity: {} + + # -- The topology spread constraints for the NGINX data plane pod. + # topologySpreadConstraints: [] + + # -- extraVolumes for the NGINX data plane pod. Use in conjunction with + # nginx.container.extraVolumeMounts mount additional volumes to the container. + # extraVolumes: [] + + # -- The container configuration for the NGINX container. This is applied globally to all Gateways managed by this + # instance of NGINX Gateway Fabric. + container: + # -- The resource requirements of the NGINX container. You should be set this value, If you want to use dataplane Autoscaling(HPA). + # resources: {} + resources: + requests: + cpu: 1000m + memory: 1Gi + + # -- The lifecycle of the NGINX container. + # lifecycle: {} + + # -- extraVolumeMounts are the additional volume mounts for the NGINX container. + # extraVolumeMounts: [] + + # -- The service configuration for the NGINX data plane. This is applied globally to all Gateways managed by this + # instance of NGINX Gateway Fabric. + service: + # @schema + # enum: + # - ClusterIP + # - NodePort + # - LoadBalancer + # @schema + # -- The type of service to create for the NGINX data plane. + type: LoadBalancer + + # @schema + # enum: + # - Cluster + # - Local + # @schema + # -- The externalTrafficPolicy of the service. The value Local preserves the client source IP. + externalTrafficPolicy: Local + + # -- The static IP address for the load balancer. Requires nginx.service.type set to LoadBalancer. + loadBalancerIP: "" + + # -- LoadBalancerClass is the class of the load balancer implementation this Service belongs to. + # Requires nginx.service.type set to LoadBalancer. + loadBalancerClass: "" + + # -- The IP ranges (CIDR) that are allowed to access the load balancer. Requires nginx.service.type set to LoadBalancer. + loadBalancerSourceRanges: [] + + # @schema + # type: array + # items: + # type: object + # properties: + # port: + # type: integer + # required: true + # minimum: 1 + # maximum: 65535 + # listenerPort: + # type: integer + # required: true + # minimum: 1 + # maximum: 65535 + # @schema + # -- A list of NodePorts to expose on the NGINX data plane service. Each NodePort MUST map to a Gateway listener port, + # otherwise it will be ignored. The default NodePort range enforced by Kubernetes is 30000-32767. + nodePorts: [] + # - port: 30025 + # listenerPort: 80 + + # -- Enable debugging for NGINX. Uses the nginx-debug binary. The NGINX error log level should be set to debug in the NginxProxy resource. + debug: false + +# -- The certGenerator section contains the configuration for the cert-generator Job. +certGenerator: + # -- The annotations of the cert-generator Job. + annotations: {} + + # -- The name of the Secret containing TLS CA, certificate, and key for the NGINX Gateway Fabric control plane + # to securely communicate with the NGINX Agent. Must exist in the same namespace that the NGINX Gateway Fabric + # control plane is running in (default namespace: nginx-gateway). + serverTLSSecretName: server-tls + + # -- The name of the base Secret containing TLS CA, certificate, and key for the NGINX Agent to securely + # communicate with the NGINX Gateway Fabric control plane. Must exist in the same namespace that the + # NGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway). + agentTLSSecretName: agent-tls + + # -- Overwrite existing TLS Secrets on startup. + overwrite: false + + # -- How long to wait after the cert generator job has finished before it is removed by the job controller. + ttlSecondsAfterFinished: 30 + + # -- Tolerations for the cert-generator pod. + tolerations: [] + + # -- The nodeSelector of the cert-generator pod. + nodeSelector: {} + + # -- The affinity of the cert-generator pod. + affinity: {} + + # -- The topology spread constraints for the cert-generator pod. + topologySpreadConstraints: [] + +# -- A list of Gateway objects. View https://gateway-api.sigs.k8s.io/reference/spec/#gateway for full Gateway reference. +gateways: + - name: practice-gateway-test + namespace: practice + annotations: + #cert-manager.io/cluster-issuer: letsencrypt-practice + spec: + gatewayClassName: nginx-test + infrastructure: + # annotations: + # service.beta.kubernetes.io/azure-load-balancer-ipv4: "40.82.159.101" + listeners: + - name: http + port: 80 + protocol: HTTP + hostname: "*.ssafy-practicestg2.com" + - name: https + port: 443 + protocol: HTTPS + hostname: "*.ssafy-practicestg2.com" + # tls: + # mode: Terminate + # certificateRefs: + # - kind: Secret + # name: practice-tls + # - name: env-gateway + # namespace: env + # annotations: + # cert-manager.io/cluster-issuer: letsencrypt-env + # spec: + # gatewayClassName: nginx + # infrastructure: + # annotations: + # service.beta.kubernetes.io/azure-load-balancer-ipv4: "52.141.28.70" + # listeners: + # - name: http-env + # port: 80 + # protocol: HTTP + # hostname: "*.ssafy-execstg.com" + # - name: https-env + # port: 443 + # protocol: HTTPS + # hostname: "*.ssafy-execstg.com" + # tls: + # mode: Terminate + # certificateRefs: + # - kind: Secret + # name: env-tls +# Example gateway object: +# name: nginx-gateway +# namespace: default +# labels: +# key: value +# annotations: +# annotationKey: annotationValue +# spec: +# gatewayClassName: nginx +# infrastructure: +# annotations: +# networking.gke.io/load-balancer-type: Internal +# listeners: +# - name: https +# port: 80 +# protocol: HTTPS +# tls: +# mode: Terminate +# certificateRefs: +# - kind: Secret +# name: my-secret +# namespace: certificate +# allowedRoutes: +# namespaces: +# from: Same diff --git a/charts/nginx-gateway-fabric/values.yaml b/charts/nginx-gateway-fabric/values.yaml index 9e2c395714..8ca94fa0ba 100644 --- a/charts/nginx-gateway-fabric/values.yaml +++ b/charts/nginx-gateway-fabric/values.yaml @@ -607,6 +607,7 @@ certGenerator: # -- A list of Gateway objects. View https://gateway-api.sigs.k8s.io/reference/spec/#gateway for full Gateway reference. gateways: [] + # Example gateway object: # name: nginx-gateway # namespace: default diff --git a/config/crd/bases/gateway.nginx.org_nginxproxies.yaml b/config/crd/bases/gateway.nginx.org_nginxproxies.yaml index 85232b0dfb..843e222069 100644 --- a/config/crd/bases/gateway.nginx.org_nginxproxies.yaml +++ b/config/crd/bases/gateway.nginx.org_nginxproxies.yaml @@ -3531,7 +3531,8 @@ spec: This is the default deployment option. properties: autoscaling: - description: Horizontal Pod Autoscaling. + description: Autoscaling defines the configuration for Horizontal + Pod Autoscaling. properties: autoscalingTemplate: description: AutoscalingTemplate configures the additional @@ -4012,11 +4013,15 @@ spec: type: array behavior: description: |- +<<<<<<< HEAD <<<<<<< HEAD Behavior configures the scaling behavior of the target ======= behavior configures the scaling behavior of the target >>>>>>> Code Review +======= + Behavior configures the scaling behavior of the target +>>>>>>> Code Review & HPA event handler in both Up and Down directions (scaleUp and scaleDown fields respectively). If not set, the default HPAScalingRules for scale up and scale down are used. properties: diff --git a/deploy/azure/deploy.yaml b/deploy/azure/deploy.yaml index 12ddab6742..0a7e457685 100644 --- a/deploy/azure/deploy.yaml +++ b/deploy/azure/deploy.yaml @@ -55,7 +55,6 @@ rules: - apiGroups: - "" - apps - - autoscaling resources: - secrets - configmaps @@ -63,7 +62,6 @@ rules: - services - deployments - daemonsets - - horizontalpodautoscalers verbs: - create - update diff --git a/deploy/crds.yaml b/deploy/crds.yaml index f11ce167f1..a03165f354 100644 --- a/deploy/crds.yaml +++ b/deploy/crds.yaml @@ -4116,7 +4116,8 @@ spec: This is the default deployment option. properties: autoscaling: - description: Horizontal Pod Autoscaling. + description: Autoscaling defines the configuration for Horizontal + Pod Autoscaling. properties: autoscalingTemplate: description: AutoscalingTemplate configures the additional diff --git a/deploy/default/deploy.yaml b/deploy/default/deploy.yaml index 3da8598bfd..4324fc92f7 100644 --- a/deploy/default/deploy.yaml +++ b/deploy/default/deploy.yaml @@ -55,7 +55,6 @@ rules: - apiGroups: - "" - apps - - autoscaling resources: - secrets - configmaps @@ -63,7 +62,6 @@ rules: - services - deployments - daemonsets - - horizontalpodautoscalers verbs: - create - update diff --git a/deploy/experimental-nginx-plus/deploy.yaml b/deploy/experimental-nginx-plus/deploy.yaml index 5b478afd22..f0ac53ba0d 100644 --- a/deploy/experimental-nginx-plus/deploy.yaml +++ b/deploy/experimental-nginx-plus/deploy.yaml @@ -55,7 +55,6 @@ rules: - apiGroups: - "" - apps - - autoscaling resources: - secrets - configmaps @@ -63,7 +62,6 @@ rules: - services - deployments - daemonsets - - horizontalpodautoscalers verbs: - create - update diff --git a/deploy/experimental/deploy.yaml b/deploy/experimental/deploy.yaml index cf774bc07b..ad3cf361a6 100644 --- a/deploy/experimental/deploy.yaml +++ b/deploy/experimental/deploy.yaml @@ -55,7 +55,6 @@ rules: - apiGroups: - "" - apps - - autoscaling resources: - secrets - configmaps @@ -63,7 +62,6 @@ rules: - services - deployments - daemonsets - - horizontalpodautoscalers verbs: - create - update diff --git a/deploy/nginx-plus/deploy.yaml b/deploy/nginx-plus/deploy.yaml index c82a24e0e9..a966b9e325 100644 --- a/deploy/nginx-plus/deploy.yaml +++ b/deploy/nginx-plus/deploy.yaml @@ -55,7 +55,6 @@ rules: - apiGroups: - "" - apps - - autoscaling resources: - secrets - configmaps @@ -63,7 +62,6 @@ rules: - services - deployments - daemonsets - - horizontalpodautoscalers verbs: - create - update diff --git a/deploy/nodeport/deploy.yaml b/deploy/nodeport/deploy.yaml index 438e6f975d..d151c82319 100644 --- a/deploy/nodeport/deploy.yaml +++ b/deploy/nodeport/deploy.yaml @@ -55,7 +55,6 @@ rules: - apiGroups: - "" - apps - - autoscaling resources: - secrets - configmaps @@ -63,7 +62,6 @@ rules: - services - deployments - daemonsets - - horizontalpodautoscalers verbs: - create - update diff --git a/deploy/openshift/deploy.yaml b/deploy/openshift/deploy.yaml index cc05b461b0..e6ecf6b3e0 100644 --- a/deploy/openshift/deploy.yaml +++ b/deploy/openshift/deploy.yaml @@ -55,7 +55,6 @@ rules: - apiGroups: - "" - apps - - autoscaling resources: - secrets - configmaps @@ -63,7 +62,6 @@ rules: - services - deployments - daemonsets - - horizontalpodautoscalers verbs: - create - update diff --git a/deploy/snippets-filters-nginx-plus/deploy.yaml b/deploy/snippets-filters-nginx-plus/deploy.yaml index 79b8a2bf0f..7461912539 100644 --- a/deploy/snippets-filters-nginx-plus/deploy.yaml +++ b/deploy/snippets-filters-nginx-plus/deploy.yaml @@ -55,7 +55,6 @@ rules: - apiGroups: - "" - apps - - autoscaling resources: - secrets - configmaps @@ -63,7 +62,6 @@ rules: - services - deployments - daemonsets - - horizontalpodautoscalers verbs: - create - update diff --git a/deploy/snippets-filters/deploy.yaml b/deploy/snippets-filters/deploy.yaml index dc4cf56dd7..d23d775600 100644 --- a/deploy/snippets-filters/deploy.yaml +++ b/deploy/snippets-filters/deploy.yaml @@ -55,7 +55,6 @@ rules: - apiGroups: - "" - apps - - autoscaling resources: - secrets - configmaps @@ -63,7 +62,6 @@ rules: - services - deployments - daemonsets - - horizontalpodautoscalers verbs: - create - update diff --git a/internal/controller/provisioner/eventloop.go b/internal/controller/provisioner/eventloop.go index 7b85a47780..6f38d75c7c 100644 --- a/internal/controller/provisioner/eventloop.go +++ b/internal/controller/provisioner/eventloop.go @@ -6,6 +6,7 @@ import ( "github.com/go-logr/logr" appsv1 "k8s.io/api/apps/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -73,6 +74,18 @@ func newEventLoop( ), }, }, + { + objectType: &autoscalingv2.HorizontalPodAutoscaler{}, + options: []controller.Option{ + controller.WithK8sPredicate( + k8spredicate.And( + k8spredicate.GenerationChangedPredicate{}, + nginxResourceLabelPredicate, + predicate.RestartDeploymentAnnotationPredicate{}, + ), + ), + }, + }, { objectType: &appsv1.DaemonSet{}, options: []controller.Option{ @@ -184,6 +197,7 @@ func newEventLoop( // to provision or deprovision any nginx resources. &gatewayv1.GatewayList{}, &appsv1.DeploymentList{}, + &autoscalingv2.HorizontalPodAutoscalerList{}, &corev1.ServiceList{}, &corev1.ServiceAccountList{}, &corev1.ConfigMapList{}, diff --git a/internal/controller/provisioner/handler.go b/internal/controller/provisioner/handler.go index 42c6e88130..f84160ca09 100644 --- a/internal/controller/provisioner/handler.go +++ b/internal/controller/provisioner/handler.go @@ -119,7 +119,7 @@ func (h *eventHandler) HandleEventBatch(ctx context.Context, logger logr.Logger, } h.store.deleteGateway(e.NamespacedName) case *appsv1.Deployment, *appsv1.DaemonSet, *corev1.Service, *corev1.ServiceAccount, - *corev1.ConfigMap, *rbacv1.Role, *rbacv1.RoleBinding: + *corev1.ConfigMap, *rbacv1.Role, *rbacv1.RoleBinding, *autoscalingv2.HorizontalPodAutoscaler: if err := h.reprovisionResources(ctx, e); err != nil { logger.Error(err, "error re-provisioning nginx resources") } diff --git a/internal/controller/provisioner/provisioner.go b/internal/controller/provisioner/provisioner.go index d840d1b665..6c60cf1384 100644 --- a/internal/controller/provisioner/provisioner.go +++ b/internal/controller/provisioner/provisioner.go @@ -200,7 +200,6 @@ func (p *NginxProvisioner) provisionNginx( var agentConfigMapUpdated, deploymentCreated bool var deploymentObj *appsv1.Deployment var daemonSetObj *appsv1.DaemonSet - for _, obj := range objects { createCtx, cancel := context.WithTimeout(ctx, 30*time.Second) diff --git a/internal/controller/provisioner/setter.go b/internal/controller/provisioner/setter.go index 59718440f4..0ffbf53d49 100644 --- a/internal/controller/provisioner/setter.go +++ b/internal/controller/provisioner/setter.go @@ -4,6 +4,7 @@ import ( "maps" appsv1 "k8s.io/api/apps/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -16,6 +17,8 @@ func objectSpecSetter(object client.Object) controllerutil.MutateFn { switch obj := object.(type) { case *appsv1.Deployment: return deploymentSpecSetter(obj, obj.Spec, obj.ObjectMeta) + case *autoscalingv2.HorizontalPodAutoscaler: + return hpaSpecSetter(obj, obj.Spec, obj.ObjectMeta) case *appsv1.DaemonSet: return daemonSetSpecSetter(obj, obj.Spec, obj.ObjectMeta) case *corev1.Service: @@ -48,6 +51,19 @@ func deploymentSpecSetter( } } +func hpaSpecSetter( + hpa *autoscalingv2.HorizontalPodAutoscaler, + spec autoscalingv2.HorizontalPodAutoscalerSpec, + objectMeta metav1.ObjectMeta, +) controllerutil.MutateFn { + return func() error { + hpa.Labels = objectMeta.Labels + hpa.Annotations = objectMeta.Annotations + hpa.Spec = spec + return nil + } +} + func daemonSetSpecSetter( daemonSet *appsv1.DaemonSet, spec appsv1.DaemonSetSpec, diff --git a/internal/controller/provisioner/store.go b/internal/controller/provisioner/store.go index d089a38f9b..fd095eae5a 100644 --- a/internal/controller/provisioner/store.go +++ b/internal/controller/provisioner/store.go @@ -6,6 +6,7 @@ import ( "sync" appsv1 "k8s.io/api/apps/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -20,6 +21,7 @@ import ( type NginxResources struct { Gateway *graph.Gateway Deployment metav1.ObjectMeta + HPA metav1.ObjectMeta DaemonSet metav1.ObjectMeta Service metav1.ObjectMeta ServiceAccount metav1.ObjectMeta @@ -133,6 +135,14 @@ func (s *store) registerResourceInGatewayConfig(gatewayNSName types.NamespacedNa } else { cfg.Deployment = obj.ObjectMeta } + case *autoscalingv2.HorizontalPodAutoscaler: + if cfg, ok := s.nginxResources[gatewayNSName]; !ok { + s.nginxResources[gatewayNSName] = &NginxResources{ + HPA: obj.ObjectMeta, + } + } else { + cfg.HPA = obj.ObjectMeta + } case *appsv1.DaemonSet: if cfg, ok := s.nginxResources[gatewayNSName]; !ok { s.nginxResources[gatewayNSName] = &NginxResources{ @@ -301,6 +311,10 @@ func (s *store) gatewayExistsForResource(object client.Object, nsName types.Name if resourceMatches(resources.Deployment, nsName) { return resources.Gateway } + case *autoscalingv2.HorizontalPodAutoscaler: + if resourceMatches(resources.HPA, nsName) { + return resources.Gateway + } case *appsv1.DaemonSet: if resourceMatches(resources.DaemonSet, nsName) { return resources.Gateway @@ -379,6 +393,10 @@ func (s *store) getResourceVersionForObject(gatewayNSName types.NamespacedName, if resources.Deployment.GetName() == obj.GetName() { return resources.Deployment.GetResourceVersion() } + case *autoscalingv2.HorizontalPodAutoscaler: + if resources.HPA.GetName() == obj.GetName() { + return resources.HPA.GetResourceVersion() + } case *appsv1.DaemonSet: if resources.DaemonSet.GetName() == obj.GetName() { return resources.DaemonSet.GetResourceVersion() From 1ad0f63ca3865c3e50ac4ffcb89e00d9887c82d0 Mon Sep 17 00:00:00 2001 From: nowjean Date: Wed, 2 Jul 2025 09:41:33 +0900 Subject: [PATCH 08/13] Code Review & HPA event handler --- charts/nginx-gateway-fabric/values-aks.yaml | 657 -------------------- 1 file changed, 657 deletions(-) delete mode 100644 charts/nginx-gateway-fabric/values-aks.yaml diff --git a/charts/nginx-gateway-fabric/values-aks.yaml b/charts/nginx-gateway-fabric/values-aks.yaml deleted file mode 100644 index 31cd16b6f6..0000000000 --- a/charts/nginx-gateway-fabric/values-aks.yaml +++ /dev/null @@ -1,657 +0,0 @@ -# yaml-language-server: $schema=values.schema.json - -# -- The DNS cluster domain of your Kubernetes cluster. -clusterDomain: cluster.local - -# -- The nginxGateway section contains configuration for the NGINX Gateway Fabric control plane deployment. -nginxGateway: - # FIXME(lucacome): https://github.com/nginx/nginx-gateway-fabric/issues/2490 - - # @schema - # const: deployment - # @schema - # -- The kind of the NGINX Gateway Fabric installation - currently, only deployment is supported. - kind: deployment - - # @schema - # required: true - # type: string - # @schema - # -- The name of the GatewayClass that will be created as part of this release. Every NGINX Gateway - # Fabric must have a unique corresponding GatewayClass resource. NGINX Gateway Fabric only processes resources that - # belong to its class - i.e. have the "gatewayClassName" field resource equal to the class. - gatewayClassName: nginx-test - - # -- Set of custom annotations for the NGINX Gateway Fabric pods. - podAnnotations: {} - - # -- Set of labels to be added for NGINX Gateway Fabric deployment. - labels: {} - - # -- Set of custom annotations for GatewayClass objects. - gatewayClassAnnotations: {} - - # @schema - # pattern: ^gateway.nginx.org/.* - # required: true - # @schema - # -- The name of the Gateway controller. The controller name must be of the form: DOMAIN/PATH. The controller's domain - # is gateway.nginx.org. - gatewayControllerName: gateway.nginx.org/nginx-gateway-controller - - # The dynamic configuration for the control plane that is contained in the NginxGateway resource. - config: - logging: - # @schema - # enum: - # - info - # - debug - # - error - # @schema - # -- Log level. - level: info - - # -- Set of custom annotations for NginxGateway objects. - configAnnotations: {} - - # -- The service configuration for the NGINX Gateway Fabric control plane. - service: - # -- The annotations of the NGINX Gateway Fabric control plane service. - annotations: {} - - # -- The labels of the NGINX Gateway Fabric control plane service. - labels: {} - - # -- The serviceaccount configuration for the NGINX Gateway Fabric control plane. - serviceAccount: - # -- Set of custom annotations for the NGINX Gateway Fabric control plane service account. - annotations: {} - - # -- The name of the service account of the NGINX Gateway Fabric control plane pods. Used for RBAC. - # @default -- Autogenerated if not set or set to "" - name: "" - - # -- The name of the secret containing docker registry credentials for the control plane. - # Secret must exist in the same namespace as the helm release. - imagePullSecret: "" - - # -- A list of secret names containing docker registry credentials for the control plane. - # Secrets must exist in the same namespace as the helm release. - imagePullSecrets: [] - - # -- The number of replicas of the NGINX Gateway Fabric Deployment. - replicas: 1 - - # The configuration for leader election. - leaderElection: - # -- Enable leader election. Leader election is used to avoid multiple replicas of the NGINX Gateway Fabric - # reporting the status of the Gateway API resources. If not enabled, all replicas of NGINX Gateway Fabric - # will update the statuses of the Gateway API resources. - enable: true - - # -- The name of the leader election lock. A Lease object with this name will be created in the same Namespace as - # the controller. - # @default -- Autogenerated if not set or set to "". - lockName: "" - - ## Defines the settings for the control plane readiness probe. This probe returns Ready when the controller - ## has started and is ready to configure NGINX. - readinessProbe: - # -- Enable the /readyz endpoint on the control plane. - enable: true - - # @schema - # type: integer - # minimum: 1 - # maximum: 65535 - # @schema - # -- Port in which the readiness endpoint is exposed. - port: 8081 - - # -- The number of seconds after the Pod has started before the readiness probes are initiated. - initialDelaySeconds: 3 - - # -- The image configuration for the NGINX Gateway Fabric control plane. - image: - # -- The NGINX Gateway Fabric image to use - #repository: ghcr.io/nginx/nginx-gateway-fabric - repository: prodonlinelabcontainerreg.azurecr.io/nginx-gateway-fabric - tag: edge - # @schema - # enum: - # - Always - # - IfNotPresent - # - Never - # @schema - pullPolicy: Always - - productTelemetry: - # -- Enable the collection of product telemetry. - enable: true - - # -- The lifecycle of the nginx-gateway container. - lifecycle: {} - - # -- The resource requests and/or limits of the nginx-gateway container. - #resources: {} - resources: - requests: - cpu: 1000m - memory: 1Gi - - # -- extraVolumes for the NGINX Gateway Fabric control plane pod. Use in conjunction with - # nginxGateway.extraVolumeMounts mount additional volumes to the container. - extraVolumes: [] - - # -- extraVolumeMounts are the additional volume mounts for the nginx-gateway container. - extraVolumeMounts: [] - - # -- The termination grace period of the NGINX Gateway Fabric control plane pod. - terminationGracePeriodSeconds: 30 - - # -- Tolerations for the NGINX Gateway Fabric control plane pod. - tolerations: - - key: "kubernetes.azure.com/scalesetpriority" - operator: "Equal" - value: "spot" - effect: "NoSchedule" - - # -- The nodeSelector of the NGINX Gateway Fabric control plane pod. - nodeSelector: {} - - # -- The affinity of the NGINX Gateway Fabric control plane pod. - affinity: {} - - # -- The topology spread constraints for the NGINX Gateway Fabric control plane pod. - topologySpreadConstraints: [] - - autoscaling: - # Enable or disable Horizontal Pod Autoscaler - enabled: true - annotations: {} - minReplicas: 1 - maxReplicas: 11 - targetCPUUtilizationPercentage: 50 - targetMemoryUtilizationPercentage: 50 - behavior: {} - # scaleDown: - # stabilizationWindowSeconds: 300 - # policies: - # - type: Pods - # value: 1 - # periodSeconds: 180 - # scaleUp: - # stabilizationWindowSeconds: 300 - # policies: - # - type: Pods - # value: 2 - # periodSeconds: 60 - autoscalingTemplate: [] - # Custom or additional autoscaling metrics - # ref: https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#support-for-custom-metrics - # - type: Pods - # pods: - # metric: - # name: nginx_gateway_fabric_nginx_process_requests_total - # target: - # type: AverageValue - # averageValue: 10000m - - metrics: - # -- Enable exposing metrics in the Prometheus format. - enable: true - - # @schema - # type: integer - # minimum: 1 - # maximum: 65535 - # @schema - # -- Set the port where the Prometheus metrics are exposed. - port: 9113 - - # -- Enable serving metrics via https. By default metrics are served via http. - # Please note that this endpoint will be secured with a self-signed certificate. - secure: false - - gwAPIExperimentalFeatures: - # -- Enable the experimental features of Gateway API which are supported by NGINX Gateway Fabric. Requires the Gateway - # APIs installed from the experimental channel. - enable: false - - snippetsFilters: - # -- Enable SnippetsFilters feature. SnippetsFilters allow inserting NGINX configuration into the generated NGINX - # config for HTTPRoute and GRPCRoute resources. - enable: true - -# -- The nginx section contains the configuration for all NGINX data plane deployments -# installed by the NGINX Gateway Fabric control plane. -nginx: - # @schema - # enum: - # - deployment - # - daemonSet - # @schema - # -- The kind of NGINX deployment. - kind: deployment - - # -- The number of replicas of the NGINX Deployment. - replicas: 1 - - autoscaling: - # Enable or disable Horizontal Pod Autoscaler - enabled: true - hpaAnnotations: - test/owner: "woojin" - minReplicas: 1 - maxReplicas: 11 - targetCPUUtilizationPercentage: 50 - targetMemoryUtilizationPercentage: 50 - behavior: - {} - # scaleDown: - # stabilizationWindowSeconds: 300 - # policies: - # - type: Pods - # value: 1 - # periodSeconds: 180 - # scaleUp: - # stabilizationWindowSeconds: 300 - # policies: - # - type: Pods - # value: 2 - # periodSeconds: 60 - autoscalingTemplate: - [] - # - type: Pods - # pods: - # metric: - # name: nginx_gateway_fabric_nginx_process_requests_total - # target: - # type: AverageValue - # averageValue: 10000m - # Custom or additional autoscaling metrics - # ref: https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#support-for-custom-metrics - - image: - # -- The NGINX image to use. - #repository: ghcr.io/nginx/nginx-gateway-fabric/nginx - repository: prodonlinelabcontainerreg.azurecr.io/nginx-gateway-fabric/nginx - tag: edge - # @schema - # enum: - # - Always - # - IfNotPresent - # - Never - # @schema - pullPolicy: Always - - # -- Is NGINX Plus image being used. - plus: false - - # -- The name of the secret containing docker registry credentials. - # Secret must exist in the same namespace as the helm release. The control - # plane will copy this secret into any namespace where NGINX is deployed. - imagePullSecret: "" - - # -- A list of secret names containing docker registry credentials. - # Secrets must exist in the same namespace as the helm release. The control - # plane will copy these secrets into any namespace where NGINX is deployed. - imagePullSecrets: [] - - # Configuration for NGINX Plus usage reporting. - usage: - # -- The name of the Secret containing the JWT for NGINX Plus usage reporting. Must exist in the same namespace - # that the NGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway). - secretName: "nplus-license" - - # -- The endpoint of the NGINX Plus usage reporting server. Default: product.connect.nginx.com - endpoint: "" - - # -- The nameserver used to resolve the NGINX Plus usage reporting endpoint. Used with NGINX Instance Manager. - resolver: "" - - # -- Disable client verification of the NGINX Plus usage reporting server certificate. - skipVerify: false - - # -- The name of the Secret containing the NGINX Instance Manager CA certificate. - # Must exist in the same namespace that the NGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway). - caSecretName: "" - - # -- The name of the Secret containing the client certificate and key for authenticating with NGINX Instance Manager. - # Must exist in the same namespace that the NGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway). - clientSSLSecretName: "" - - # @schema - # type: object - # properties: - # disableHTTP2: - # description: DisableHTTP2 defines if http2 should be disabled for all servers. - # type: boolean - # ipFamily: - # description: IPFamily specifies the IP family to be used by the NGINX. - # type: string - # enum: - # - ipv4 - # - ipv6 - # - dual - # rewriteClientIP: - # type: object - # description: RewriteClientIP defines configuration for rewriting the client IP to the original client's IP. - # properties: - # mode: - # type: string - # enum: - # - ProxyProtocol - # - XForwardedFor - # setIPRecursively: - # type: boolean - # trustedAddresses: - # type: array - # items: - # properties: - # type: - # type: string - # enum: - # - CIDR - # - IPAddress - # - Hostname - # value: - # type: string - # telemetry: - # type: object - # description: Telemetry specifies the OpenTelemetry configuration. - # properties: - # exporter: - # type: object - # properties: - # endpoint: - # type: string - # pattern: ^(?:http?:\/\/)?[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?(?:\.[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?)*(?::\d{1,5})?$ - # interval: - # type: string - # pattern: ^\d{1,4}(ms|s)?$ - # batchSize: - # type: integer - # minimum: 0 - # batchCount: - # type: integer - # minimum: 0 - # serviceName: - # type: string - # pattern: ^[a-zA-Z0-9_-]+$ - # spanAttributes: - # type: array - # items: - # properties: - # key: - # type: string - # pattern: ^([^"$\\]|\\[^$])*$ - # minLength: 1 - # maxLength: 255 - # value: - # type: string - # pattern: ^([^"$\\]|\\[^$])*$ - # minLength: 1 - # maxLength: 255 - # disabledFeatures: - # type: array - # items: - # type: string - # enum: - # - DisableTracing - # metrics: - # type: object - # description: Metrics defines the configuration for Prometheus scraping metrics. - # properties: - # disable: - # type: boolean - # port: - # type: integer - # minimum: 1 - # maximum: 65535 - # logging: - # type: object - # description: Logging defines logging related settings for NGINX. - # properties: - # errorLevel: - # type: string - # enum: - # - debug - # - info - # - notice - # - warn - # - error - # - crit - # - alert - # - emerg - # agentLevel: - # type: string - # enum: - # - debug - # - info - # - error - # - panic - # - fatal - # nginxPlus: - # type: object - # description: NginxPlus specifies NGINX Plus additional settings. - # properties: - # allowedAddresses: - # type: array - # items: - # properties: - # type: - # type: string - # enum: - # - CIDR - # - IPAddress - # value: - # type: string - # @schema - # -- The configuration for the data plane that is contained in the NginxProxy resource. This is applied globally to all Gateways - # managed by this instance of NGINX Gateway Fabric. - config: {} - - # -- The pod configuration for the NGINX data plane pod. This is applied globally to all Gateways managed by this - # instance of NGINX Gateway Fabric. - pod: - # -- The termination grace period of the NGINX data plane pod. - # terminationGracePeriodSeconds: 30 - - # -- Tolerations for the NGINX data plane pod. - tolerations: - - key: "kubernetes.azure.com/scalesetpriority" - operator: "Equal" - value: "spot" - effect: "NoSchedule" - - # -- The nodeSelector of the NGINX data plane pod. - # nodeSelector: {} - - # -- The affinity of the NGINX data plane pod. - # affinity: {} - - # -- The topology spread constraints for the NGINX data plane pod. - # topologySpreadConstraints: [] - - # -- extraVolumes for the NGINX data plane pod. Use in conjunction with - # nginx.container.extraVolumeMounts mount additional volumes to the container. - # extraVolumes: [] - - # -- The container configuration for the NGINX container. This is applied globally to all Gateways managed by this - # instance of NGINX Gateway Fabric. - container: - # -- The resource requirements of the NGINX container. You should be set this value, If you want to use dataplane Autoscaling(HPA). - # resources: {} - resources: - requests: - cpu: 1000m - memory: 1Gi - - # -- The lifecycle of the NGINX container. - # lifecycle: {} - - # -- extraVolumeMounts are the additional volume mounts for the NGINX container. - # extraVolumeMounts: [] - - # -- The service configuration for the NGINX data plane. This is applied globally to all Gateways managed by this - # instance of NGINX Gateway Fabric. - service: - # @schema - # enum: - # - ClusterIP - # - NodePort - # - LoadBalancer - # @schema - # -- The type of service to create for the NGINX data plane. - type: LoadBalancer - - # @schema - # enum: - # - Cluster - # - Local - # @schema - # -- The externalTrafficPolicy of the service. The value Local preserves the client source IP. - externalTrafficPolicy: Local - - # -- The static IP address for the load balancer. Requires nginx.service.type set to LoadBalancer. - loadBalancerIP: "" - - # -- LoadBalancerClass is the class of the load balancer implementation this Service belongs to. - # Requires nginx.service.type set to LoadBalancer. - loadBalancerClass: "" - - # -- The IP ranges (CIDR) that are allowed to access the load balancer. Requires nginx.service.type set to LoadBalancer. - loadBalancerSourceRanges: [] - - # @schema - # type: array - # items: - # type: object - # properties: - # port: - # type: integer - # required: true - # minimum: 1 - # maximum: 65535 - # listenerPort: - # type: integer - # required: true - # minimum: 1 - # maximum: 65535 - # @schema - # -- A list of NodePorts to expose on the NGINX data plane service. Each NodePort MUST map to a Gateway listener port, - # otherwise it will be ignored. The default NodePort range enforced by Kubernetes is 30000-32767. - nodePorts: [] - # - port: 30025 - # listenerPort: 80 - - # -- Enable debugging for NGINX. Uses the nginx-debug binary. The NGINX error log level should be set to debug in the NginxProxy resource. - debug: false - -# -- The certGenerator section contains the configuration for the cert-generator Job. -certGenerator: - # -- The annotations of the cert-generator Job. - annotations: {} - - # -- The name of the Secret containing TLS CA, certificate, and key for the NGINX Gateway Fabric control plane - # to securely communicate with the NGINX Agent. Must exist in the same namespace that the NGINX Gateway Fabric - # control plane is running in (default namespace: nginx-gateway). - serverTLSSecretName: server-tls - - # -- The name of the base Secret containing TLS CA, certificate, and key for the NGINX Agent to securely - # communicate with the NGINX Gateway Fabric control plane. Must exist in the same namespace that the - # NGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway). - agentTLSSecretName: agent-tls - - # -- Overwrite existing TLS Secrets on startup. - overwrite: false - - # -- How long to wait after the cert generator job has finished before it is removed by the job controller. - ttlSecondsAfterFinished: 30 - - # -- Tolerations for the cert-generator pod. - tolerations: [] - - # -- The nodeSelector of the cert-generator pod. - nodeSelector: {} - - # -- The affinity of the cert-generator pod. - affinity: {} - - # -- The topology spread constraints for the cert-generator pod. - topologySpreadConstraints: [] - -# -- A list of Gateway objects. View https://gateway-api.sigs.k8s.io/reference/spec/#gateway for full Gateway reference. -gateways: - - name: practice-gateway-test - namespace: practice - annotations: - #cert-manager.io/cluster-issuer: letsencrypt-practice - spec: - gatewayClassName: nginx-test - infrastructure: - # annotations: - # service.beta.kubernetes.io/azure-load-balancer-ipv4: "40.82.159.101" - listeners: - - name: http - port: 80 - protocol: HTTP - hostname: "*.ssafy-practicestg2.com" - - name: https - port: 443 - protocol: HTTPS - hostname: "*.ssafy-practicestg2.com" - # tls: - # mode: Terminate - # certificateRefs: - # - kind: Secret - # name: practice-tls - # - name: env-gateway - # namespace: env - # annotations: - # cert-manager.io/cluster-issuer: letsencrypt-env - # spec: - # gatewayClassName: nginx - # infrastructure: - # annotations: - # service.beta.kubernetes.io/azure-load-balancer-ipv4: "52.141.28.70" - # listeners: - # - name: http-env - # port: 80 - # protocol: HTTP - # hostname: "*.ssafy-execstg.com" - # - name: https-env - # port: 443 - # protocol: HTTPS - # hostname: "*.ssafy-execstg.com" - # tls: - # mode: Terminate - # certificateRefs: - # - kind: Secret - # name: env-tls -# Example gateway object: -# name: nginx-gateway -# namespace: default -# labels: -# key: value -# annotations: -# annotationKey: annotationValue -# spec: -# gatewayClassName: nginx -# infrastructure: -# annotations: -# networking.gke.io/load-balancer-type: Internal -# listeners: -# - name: https -# port: 80 -# protocol: HTTPS -# tls: -# mode: Terminate -# certificateRefs: -# - kind: Secret -# name: my-secret -# namespace: certificate -# allowedRoutes: -# namespaces: -# from: Same From be4952c5cfc274538ae687eddc6b3aa80c49e719 Mon Sep 17 00:00:00 2001 From: nowjean Date: Wed, 23 Jul 2025 17:59:01 +0900 Subject: [PATCH 09/13] Code Review --- charts/nginx-gateway-fabric/templates/nginxproxy.yaml | 4 ---- charts/nginx-gateway-fabric/values.yaml | 1 - config/crd/bases/gateway.nginx.org_nginxproxies.yaml | 3 +++ internal/controller/provisioner/eventloop.go | 1 - 4 files changed, 3 insertions(+), 6 deletions(-) diff --git a/charts/nginx-gateway-fabric/templates/nginxproxy.yaml b/charts/nginx-gateway-fabric/templates/nginxproxy.yaml index d83f0f602d..e2b5f5c002 100644 --- a/charts/nginx-gateway-fabric/templates/nginxproxy.yaml +++ b/charts/nginx-gateway-fabric/templates/nginxproxy.yaml @@ -18,10 +18,6 @@ spec: {{- if .Values.nginx.autoscaling.enabled }} autoscaling: enabled: {{ .Values.nginx.autoscaling.enabled }} - {{- if .Values.nginx.autoscaling.hpaAnnotations }} - hpaAnnotations: - {{- toYaml .Values.nginx.autoscaling.hpaAnnotations | nindent 10 }} - {{- end }} minReplicas: {{ .Values.nginx.autoscaling.minReplicas }} maxReplicas: {{ .Values.nginx.autoscaling.maxReplicas }} {{- if .Values.nginx.autoscaling.targetCPUUtilizationPercentage }} diff --git a/charts/nginx-gateway-fabric/values.yaml b/charts/nginx-gateway-fabric/values.yaml index 8ca94fa0ba..78b1c307c1 100644 --- a/charts/nginx-gateway-fabric/values.yaml +++ b/charts/nginx-gateway-fabric/values.yaml @@ -232,7 +232,6 @@ nginx: autoscaling: # Enable or disable Horizontal Pod Autoscaler enabled: false - # hpaAnnotations: {} # minReplicas: 1 # maxReplicas: 11 # targetCPUUtilizationPercentage: 50 diff --git a/config/crd/bases/gateway.nginx.org_nginxproxies.yaml b/config/crd/bases/gateway.nginx.org_nginxproxies.yaml index 843e222069..ad14c2d1c2 100644 --- a/config/crd/bases/gateway.nginx.org_nginxproxies.yaml +++ b/config/crd/bases/gateway.nginx.org_nginxproxies.yaml @@ -4181,6 +4181,7 @@ spec: description: Enable or disable Horizontal Pod Autoscaler type: boolean <<<<<<< HEAD +<<<<<<< HEAD ======= hpaAnnotations: additionalProperties: @@ -4192,6 +4193,8 @@ spec: queryable and should be preserved when modifying objects. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations type: object +>>>>>>> Code Review +======= >>>>>>> Code Review maxReplicas: description: Maximum number of replicas. diff --git a/internal/controller/provisioner/eventloop.go b/internal/controller/provisioner/eventloop.go index 6f38d75c7c..a3a8730620 100644 --- a/internal/controller/provisioner/eventloop.go +++ b/internal/controller/provisioner/eventloop.go @@ -81,7 +81,6 @@ func newEventLoop( k8spredicate.And( k8spredicate.GenerationChangedPredicate{}, nginxResourceLabelPredicate, - predicate.RestartDeploymentAnnotationPredicate{}, ), ), }, From 444740c3c833aa9c6a5272ad44ed9fa010ecd65a Mon Sep 17 00:00:00 2001 From: nowjean Date: Thu, 24 Jul 2025 17:49:45 +0900 Subject: [PATCH 10/13] Resolve Conflict files & rebase & make generate-all --- apis/v1alpha2/zz_generated.deepcopy.go | 4 +- charts/nginx-gateway-fabric/README.md | 4 +- .../nginx-gateway-fabric/values.schema.json | 3 -- .../bases/gateway.nginx.org_nginxproxies.yaml | 38 ------------------- 4 files changed, 4 insertions(+), 45 deletions(-) diff --git a/apis/v1alpha2/zz_generated.deepcopy.go b/apis/v1alpha2/zz_generated.deepcopy.go index 836ffda2d5..3fb3fa78fb 100644 --- a/apis/v1alpha2/zz_generated.deepcopy.go +++ b/apis/v1alpha2/zz_generated.deepcopy.go @@ -6,9 +6,9 @@ package v1alpha2 import ( "github.com/nginx/nginx-gateway-fabric/apis/v1alpha1" - v2 "k8s.io/api/autoscaling/v2" + "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" - v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/runtime" apisv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" ) diff --git a/charts/nginx-gateway-fabric/README.md b/charts/nginx-gateway-fabric/README.md index fc4e5eccd6..e4365fea42 100644 --- a/charts/nginx-gateway-fabric/README.md +++ b/charts/nginx-gateway-fabric/README.md @@ -264,7 +264,7 @@ The following table lists the configurable parameters of the NGINX Gateway Fabri | `certGenerator.ttlSecondsAfterFinished` | How long to wait after the cert generator job has finished before it is removed by the job controller. | int | `30` | | `clusterDomain` | The DNS cluster domain of your Kubernetes cluster. | string | `"cluster.local"` | | `gateways` | A list of Gateway objects. View https://gateway-api.sigs.k8s.io/reference/spec/#gateway for full Gateway reference. | list | `[]` | -| `nginx` | The nginx section contains the configuration for all NGINX data plane deployments installed by the NGINX Gateway Fabric control plane. | object | `{"autoscaling":{"behavior":{},"enabled":true,"hpaAnnotations":{},"maxReplicas":11,"minReplicas":1,"targetCPUUtilizationPercentage":50,"targetMemoryUtilizationPercentage":50},"autoscalingTemplate":[],"config":{},"container":{"hostPorts":[],"lifecycle":{},"readinessProbe":{},"resources":{},"volumeMounts":[]},"debug":false,"image":{"pullPolicy":"Always","repository":"ghcr.io/nginx/nginx-gateway-fabric/nginx","tag":"edge"},"imagePullSecret":"","imagePullSecrets":[],"kind":"deployment","plus":false,"pod":{},"replicas":1,"service":{"externalTrafficPolicy":"Local","loadBalancerClass":"","loadBalancerIP":"","loadBalancerSourceRanges":[],"nodePorts":[],"type":"LoadBalancer"},"usage":{"caSecretName":"","clientSSLSecretName":"","endpoint":"","resolver":"","secretName":"nplus-license","skipVerify":false}}` | +| `nginx` | The nginx section contains the configuration for all NGINX data plane deployments installed by the NGINX Gateway Fabric control plane. | object | `{"autoscaling":{"enabled":false},"autoscalingTemplate":[],"config":{},"container":{"hostPorts":[],"lifecycle":{},"readinessProbe":{},"resources":{},"volumeMounts":[]},"debug":false,"image":{"pullPolicy":"Always","repository":"ghcr.io/nginx/nginx-gateway-fabric/nginx","tag":"edge"},"imagePullSecret":"","imagePullSecrets":[],"kind":"deployment","plus":false,"pod":{},"replicas":1,"service":{"externalTrafficPolicy":"Local","loadBalancerClass":"","loadBalancerIP":"","loadBalancerSourceRanges":[],"nodePorts":[],"type":"LoadBalancer"},"usage":{"caSecretName":"","clientSSLSecretName":"","endpoint":"","resolver":"","secretName":"nplus-license","skipVerify":false}}` | | `nginx.config` | The configuration for the data plane that is contained in the NginxProxy resource. This is applied globally to all Gateways managed by this instance of NGINX Gateway Fabric. | object | `{}` | | `nginx.container` | The container configuration for the NGINX container. This is applied globally to all Gateways managed by this instance of NGINX Gateway Fabric. | object | `{"hostPorts":[],"lifecycle":{},"readinessProbe":{},"resources":{},"volumeMounts":[]}` | | `nginx.container.hostPorts` | A list of HostPorts to expose on the host. This configuration allows containers to bind to a specific port on the host node, enabling external network traffic to reach the container directly through the host's IP address and port. Use this option when you need to expose container ports on the host for direct access, such as for debugging, legacy integrations, or when NodePort/LoadBalancer services are not suitable. Note: Using hostPort may have security and scheduling implications, as it ties pods to specific nodes and ports. | list | `[]` | @@ -292,7 +292,7 @@ The following table lists the configurable parameters of the NGINX Gateway Fabri | `nginx.usage.resolver` | The nameserver used to resolve the NGINX Plus usage reporting endpoint. Used with NGINX Instance Manager. | string | `""` | | `nginx.usage.secretName` | The name of the Secret containing the JWT for NGINX Plus usage reporting. Must exist in the same namespace that the NGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway). | string | `"nplus-license"` | | `nginx.usage.skipVerify` | Disable client verification of the NGINX Plus usage reporting server certificate. | bool | `false` | -| `nginxGateway` | The nginxGateway section contains configuration for the NGINX Gateway Fabric control plane deployment. | object | `{"affinity":{},"autoscaling":{"annotations":{},"behavior":{},"enabled":false,"maxReplicas":11,"minReplicas":1,"targetCPUUtilizationPercentage":50,"targetMemoryUtilizationPercentage":50},"autoscalingTemplate":[],"config":{"logging":{"level":"info"}},"configAnnotations":{},"extraVolumeMounts":[],"extraVolumes":[],"gatewayClassAnnotations":{},"gatewayClassName":"nginx","gatewayControllerName":"gateway.nginx.org/nginx-gateway-controller","gwAPIExperimentalFeatures":{"enable":false},"image":{"pullPolicy":"Always","repository":"ghcr.io/nginx/nginx-gateway-fabric","tag":"edge"},"kind":"deployment","labels":{},"leaderElection":{"enable":true,"lockName":""},"lifecycle":{},"metrics":{"enable":true,"port":9113,"secure":false},"name":"","nodeSelector":{},"podAnnotations":{},"productTelemetry":{"enable":true},"readinessProbe":{"enable":true,"initialDelaySeconds":3,"port":8081},"replicas":1,"resources":{},"service":{"annotations":{},"labels":{}},"serviceAccount":{"annotations":{},"imagePullSecret":"","imagePullSecrets":[],"name":""},"snippetsFilters":{"enable":false},"terminationGracePeriodSeconds":30,"tolerations":[],"topologySpreadConstraints":[]}` | +| `nginxGateway` | The nginxGateway section contains configuration for the NGINX Gateway Fabric control plane deployment. | object | `{"affinity":{},"autoscaling":{"enabled":false},"autoscalingTemplate":[],"config":{"logging":{"level":"info"}},"configAnnotations":{},"extraVolumeMounts":[],"extraVolumes":[],"gatewayClassAnnotations":{},"gatewayClassName":"nginx","gatewayControllerName":"gateway.nginx.org/nginx-gateway-controller","gwAPIExperimentalFeatures":{"enable":false},"image":{"pullPolicy":"Always","repository":"ghcr.io/nginx/nginx-gateway-fabric","tag":"edge"},"kind":"deployment","labels":{},"leaderElection":{"enable":true,"lockName":""},"lifecycle":{},"metrics":{"enable":true,"port":9113,"secure":false},"name":"","nodeSelector":{},"podAnnotations":{},"productTelemetry":{"enable":true},"readinessProbe":{"enable":true,"initialDelaySeconds":3,"port":8081},"replicas":1,"resources":{},"service":{"annotations":{},"labels":{}},"serviceAccount":{"annotations":{},"imagePullSecret":"","imagePullSecrets":[],"name":""},"snippetsFilters":{"enable":false},"terminationGracePeriodSeconds":30,"tolerations":[],"topologySpreadConstraints":[]}` | | `nginxGateway.affinity` | The affinity of the NGINX Gateway Fabric control plane pod. | object | `{}` | | `nginxGateway.config.logging.level` | Log level. | string | `"info"` | | `nginxGateway.configAnnotations` | Set of custom annotations for NginxGateway objects. | object | `{}` | diff --git a/charts/nginx-gateway-fabric/values.schema.json b/charts/nginx-gateway-fabric/values.schema.json index d7309e8d99..f188636a0a 100644 --- a/charts/nginx-gateway-fabric/values.schema.json +++ b/charts/nginx-gateway-fabric/values.schema.json @@ -342,7 +342,6 @@ }, "container": { "description": "The container configuration for the NGINX container. This is applied globally to all Gateways managed by this\ninstance of NGINX Gateway Fabric.", -<<<<<<< HEAD "properties": { "hostPorts": { "description": "A list of HostPorts to expose on the host.\nThis configuration allows containers to bind to a specific port on the host node,\nenabling external network traffic to reach the container directly through the host's IP address and port.\nUse this option when you need to expose container ports on the host for direct access,\nsuch as for debugging, legacy integrations, or when NodePort/LoadBalancer services are not suitable.\nNote: Using hostPort may have security and scheduling implications, as it ties pods to specific nodes and ports.", @@ -396,8 +395,6 @@ "type": "array" } }, -======= ->>>>>>> Code Review "required": [], "title": "container", "type": "object" diff --git a/config/crd/bases/gateway.nginx.org_nginxproxies.yaml b/config/crd/bases/gateway.nginx.org_nginxproxies.yaml index ad14c2d1c2..1f7c54373b 100644 --- a/config/crd/bases/gateway.nginx.org_nginxproxies.yaml +++ b/config/crd/bases/gateway.nginx.org_nginxproxies.yaml @@ -4013,15 +4013,7 @@ spec: type: array behavior: description: |- -<<<<<<< HEAD -<<<<<<< HEAD Behavior configures the scaling behavior of the target -======= - behavior configures the scaling behavior of the target ->>>>>>> Code Review -======= - Behavior configures the scaling behavior of the target ->>>>>>> Code Review & HPA event handler in both Up and Down directions (scaleUp and scaleDown fields respectively). If not set, the default HPAScalingRules for scale up and scale down are used. properties: @@ -4180,22 +4172,6 @@ spec: enabled: description: Enable or disable Horizontal Pod Autoscaler type: boolean -<<<<<<< HEAD -<<<<<<< HEAD -======= - hpaAnnotations: - additionalProperties: - type: string - description: |- - Annotation for Horizontal Pod Autoscaler - Annotations is an unstructured key value map stored with a resource that may be - set by external tools to store and retrieve arbitrary metadata. They are not - queryable and should be preserved when modifying objects. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations - type: object ->>>>>>> Code Review -======= ->>>>>>> Code Review maxReplicas: description: Maximum number of replicas. format: int32 @@ -7727,28 +7703,14 @@ spec: automatically if required. The default NodePort range enforced by Kubernetes is 30000-32767. properties: listenerPort: -<<<<<<< HEAD description: ListenerPort is the Gateway listener port that this NodePort maps to. -======= - description: |- - ListenerPort is the Gateway listener port that this NodePort maps to. - kubebuilder:validation:Minimum=1 - kubebuilder:validation:Maximum=65535 ->>>>>>> Code Review format: int32 maximum: 65535 minimum: 1 type: integer port: -<<<<<<< HEAD description: Port is the NodePort to expose. -======= - description: |- - Port is the NodePort to expose. - kubebuilder:validation:Minimum=1 - kubebuilder:validation:Maximum=65535 ->>>>>>> Code Review format: int32 maximum: 65535 minimum: 1 From 29d48e3191d996b6f1ba143b129bb550a940d9c6 Mon Sep 17 00:00:00 2001 From: nowjean Date: Mon, 28 Jul 2025 14:02:40 +0900 Subject: [PATCH 11/13] Code Review --- apis/v1alpha2/nginxproxy_types.go | 12 ++++++------ apis/v1alpha2/zz_generated.deepcopy.go | 10 +++------- charts/nginx-gateway-fabric/templates/hpa.yaml | 2 ++ .../templates/nginxproxy.yaml | 2 ++ .../bases/gateway.nginx.org_nginxproxies.yaml | 14 -------------- deploy/crds.yaml | 14 -------------- internal/controller/provisioner/objects.go | 18 +++++------------- 7 files changed, 18 insertions(+), 54 deletions(-) diff --git a/apis/v1alpha2/nginxproxy_types.go b/apis/v1alpha2/nginxproxy_types.go index 0a17861b28..0b80747659 100644 --- a/apis/v1alpha2/nginxproxy_types.go +++ b/apis/v1alpha2/nginxproxy_types.go @@ -466,13 +466,13 @@ type DaemonSetSpec struct { Patches []Patch `json:"patches,omitempty"` } -// +kubebuilder:validation:XValidation:message="at least one metric must be specified when autoscaling is enabled",rule="!self.enabled || (has(self.targetCPUUtilizationPercentage) || has(self.targetMemoryUtilizationPercentage) || (has(self.autoscalingTemplate) && size(self.autoscalingTemplate) > 0))" -// +kubebuilder:validation:XValidation:message="minReplicas must be less than or equal to maxReplicas",rule="self.minReplicas <= self.maxReplicas" -// +kubebuilder:validation:XValidation:message="CPU utilization must be between 1 and 100",rule="!has(self.targetCPUUtilizationPercentage) || (self.targetCPUUtilizationPercentage >= 1 && self.targetCPUUtilizationPercentage <= 100)" -// +kubebuilder:validation:XValidation:message="memory utilization must be between 1 and 100",rule="!has(self.targetMemoryUtilizationPercentage) || (self.targetMemoryUtilizationPercentage >= 1 && self.targetMemoryUtilizationPercentage <= 100)" -// // HPASpec is the configuration for the Horizontal Pod Autoscaling. // +// +kubebuilder:message="at least one metric must be specified when autoscaling is enabled",rule="!self.enabled || (has(self.targetCPUUtilizationPercentage) || has(self.targetMemoryUtilizationPercentage) || (has(self.autoscalingTemplate) && size(self.autoscalingTemplate) > 0))" +// +kubebuilder:message="minReplicas must be less than or equal to maxReplicas",rule="self.minReplicas <= self.maxReplicas" +// +kubebuilder:message="CPU utilization must be between 1 and 100",rule="!has(self.targetCPUUtilizationPercentage) || (self.targetCPUUtilizationPercentage >= 1 && self.targetCPUUtilizationPercentage <= 100)" +// +kubebuilder:message="memory utilization must be between 1 and 100",rule="!has(self.targetMemoryUtilizationPercentage) || (self.targetMemoryUtilizationPercentage >= 1 && self.targetMemoryUtilizationPercentage <= 100)" +// //nolint:lll type HPASpec struct { // Behavior configures the scaling behavior of the target @@ -485,7 +485,7 @@ type HPASpec struct { // AutoscalingTemplate configures the additional scaling option. // // +optional - AutoscalingTemplate *[]autoscalingv2.MetricSpec `json:"autoscalingTemplate,omitempty"` + AutoscalingTemplate []autoscalingv2.MetricSpec `json:"autoscalingTemplate,omitempty"` // Target cpu utilization percentage of HPA. // diff --git a/apis/v1alpha2/zz_generated.deepcopy.go b/apis/v1alpha2/zz_generated.deepcopy.go index 3fb3fa78fb..7769e1bdda 100644 --- a/apis/v1alpha2/zz_generated.deepcopy.go +++ b/apis/v1alpha2/zz_generated.deepcopy.go @@ -129,13 +129,9 @@ func (in *HPASpec) DeepCopyInto(out *HPASpec) { } if in.AutoscalingTemplate != nil { in, out := &in.AutoscalingTemplate, &out.AutoscalingTemplate - *out = new([]v2.MetricSpec) - if **in != nil { - in, out := *in, *out - *out = make([]v2.MetricSpec, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } + *out = make([]v2.MetricSpec, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) } } if in.TargetCPUUtilizationPercentage != nil { diff --git a/charts/nginx-gateway-fabric/templates/hpa.yaml b/charts/nginx-gateway-fabric/templates/hpa.yaml index 86730bcfa5..01f30eca57 100644 --- a/charts/nginx-gateway-fabric/templates/hpa.yaml +++ b/charts/nginx-gateway-fabric/templates/hpa.yaml @@ -17,7 +17,9 @@ spec: apiVersion: apps/v1 kind: Deployment name: {{ include "nginx-gateway.fullname" . }} + {{- if .Values.nginxGateway.autoscaling.minReplicas }} minReplicas: {{ .Values.nginxGateway.autoscaling.minReplicas }} + {{- end }} maxReplicas: {{ .Values.nginxGateway.autoscaling.maxReplicas }} metrics: {{- with .Values.nginxGateway.autoscaling.targetMemoryUtilizationPercentage }} diff --git a/charts/nginx-gateway-fabric/templates/nginxproxy.yaml b/charts/nginx-gateway-fabric/templates/nginxproxy.yaml index e2b5f5c002..2d864df72a 100644 --- a/charts/nginx-gateway-fabric/templates/nginxproxy.yaml +++ b/charts/nginx-gateway-fabric/templates/nginxproxy.yaml @@ -18,7 +18,9 @@ spec: {{- if .Values.nginx.autoscaling.enabled }} autoscaling: enabled: {{ .Values.nginx.autoscaling.enabled }} + {{- if .Values.nginx.autoscaling.minReplicas }} minReplicas: {{ .Values.nginx.autoscaling.minReplicas }} + {{- end }} maxReplicas: {{ .Values.nginx.autoscaling.maxReplicas }} {{- if .Values.nginx.autoscaling.targetCPUUtilizationPercentage }} targetCPUUtilizationPercentage: {{ .Values.nginx.autoscaling.targetCPUUtilizationPercentage }} diff --git a/config/crd/bases/gateway.nginx.org_nginxproxies.yaml b/config/crd/bases/gateway.nginx.org_nginxproxies.yaml index 1f7c54373b..d10e3a9851 100644 --- a/config/crd/bases/gateway.nginx.org_nginxproxies.yaml +++ b/config/crd/bases/gateway.nginx.org_nginxproxies.yaml @@ -4192,20 +4192,6 @@ spec: - enabled - maxReplicas type: object - x-kubernetes-validations: - - message: at least one metric must be specified when autoscaling - is enabled - rule: '!self.enabled || (has(self.targetCPUUtilizationPercentage) - || has(self.targetMemoryUtilizationPercentage) || (has(self.autoscalingTemplate) - && size(self.autoscalingTemplate) > 0))' - - message: minReplicas must be less than or equal to maxReplicas - rule: self.minReplicas <= self.maxReplicas - - message: CPU utilization must be between 1 and 100 - rule: '!has(self.targetCPUUtilizationPercentage) || (self.targetCPUUtilizationPercentage - >= 1 && self.targetCPUUtilizationPercentage <= 100)' - - message: memory utilization must be between 1 and 100 - rule: '!has(self.targetMemoryUtilizationPercentage) || (self.targetMemoryUtilizationPercentage - >= 1 && self.targetMemoryUtilizationPercentage <= 100)' container: description: Container defines container fields for the NGINX container. diff --git a/deploy/crds.yaml b/deploy/crds.yaml index a03165f354..297cfee89a 100644 --- a/deploy/crds.yaml +++ b/deploy/crds.yaml @@ -4777,20 +4777,6 @@ spec: - enabled - maxReplicas type: object - x-kubernetes-validations: - - message: at least one metric must be specified when autoscaling - is enabled - rule: '!self.enabled || (has(self.targetCPUUtilizationPercentage) - || has(self.targetMemoryUtilizationPercentage) || (has(self.autoscalingTemplate) - && size(self.autoscalingTemplate) > 0))' - - message: minReplicas must be less than or equal to maxReplicas - rule: self.minReplicas <= self.maxReplicas - - message: CPU utilization must be between 1 and 100 - rule: '!has(self.targetCPUUtilizationPercentage) || (self.targetCPUUtilizationPercentage - >= 1 && self.targetCPUUtilizationPercentage <= 100)' - - message: memory utilization must be between 1 and 100 - rule: '!has(self.targetMemoryUtilizationPercentage) || (self.targetMemoryUtilizationPercentage - >= 1 && self.targetMemoryUtilizationPercentage <= 100)' container: description: Container defines container fields for the NGINX container. diff --git a/internal/controller/provisioner/objects.go b/internal/controller/provisioner/objects.go index 658e0f0df9..3a8e4db742 100644 --- a/internal/controller/provisioner/objects.go +++ b/internal/controller/provisioner/objects.go @@ -1101,12 +1101,10 @@ func buildNginxDeploymentHPA( }) } - if autoscalingTemplate != nil { - for _, additionalAutoscaling := range *autoscalingTemplate { - metric := buildAdditionalMetric(additionalAutoscaling) - if metric != nil { - metrics = append(metrics, *metric) - } + for _, additionalAutoscaling := range autoscalingTemplate { + metric := buildAdditionalMetric(additionalAutoscaling) + if metric != nil { + metrics = append(metrics, *metric) } } @@ -1193,11 +1191,11 @@ func (p *NginxProvisioner) buildNginxResourceObjectsForDeletion(deploymentNSName // order to delete: // deployment/daemonset // service + // hpa // role/binding (if openshift) // serviceaccount // configmaps // secrets - // hpa objectMeta := metav1.ObjectMeta{ Name: deploymentNSName.Name, @@ -1219,12 +1217,6 @@ func (p *NginxProvisioner) buildNginxResourceObjectsForDeletion(deploymentNSName objects := []client.Object{deployment, daemonSet, service, hpa} - // objects := []client.Object{deployment, daemonSet, service} - - // // if hpa := p.buildHPA(objectMeta, nProxyCfg); hpa != nil { - // // objects = append(objects, hpa) - // // } - if p.isOpenshift { role := &rbacv1.Role{ ObjectMeta: objectMeta, From aff80d8689d45873cda33d6fee1a977ba8028790 Mon Sep 17 00:00:00 2001 From: nowjean Date: Wed, 30 Jul 2025 17:00:50 +0900 Subject: [PATCH 12/13] Code Review --- apis/v1alpha2/nginxproxy_types.go | 8 ++++---- charts/nginx-gateway-fabric/README.md | 2 +- charts/nginx-gateway-fabric/values.schema.json | 2 +- charts/nginx-gateway-fabric/values.yaml | 6 +++--- .../crd/bases/gateway.nginx.org_nginxproxies.yaml | 14 ++++++++++++++ deploy/crds.yaml | 14 ++++++++++++++ 6 files changed, 37 insertions(+), 9 deletions(-) diff --git a/apis/v1alpha2/nginxproxy_types.go b/apis/v1alpha2/nginxproxy_types.go index 0b80747659..6b6f164d4a 100644 --- a/apis/v1alpha2/nginxproxy_types.go +++ b/apis/v1alpha2/nginxproxy_types.go @@ -468,10 +468,10 @@ type DaemonSetSpec struct { // HPASpec is the configuration for the Horizontal Pod Autoscaling. // -// +kubebuilder:message="at least one metric must be specified when autoscaling is enabled",rule="!self.enabled || (has(self.targetCPUUtilizationPercentage) || has(self.targetMemoryUtilizationPercentage) || (has(self.autoscalingTemplate) && size(self.autoscalingTemplate) > 0))" -// +kubebuilder:message="minReplicas must be less than or equal to maxReplicas",rule="self.minReplicas <= self.maxReplicas" -// +kubebuilder:message="CPU utilization must be between 1 and 100",rule="!has(self.targetCPUUtilizationPercentage) || (self.targetCPUUtilizationPercentage >= 1 && self.targetCPUUtilizationPercentage <= 100)" -// +kubebuilder:message="memory utilization must be between 1 and 100",rule="!has(self.targetMemoryUtilizationPercentage) || (self.targetMemoryUtilizationPercentage >= 1 && self.targetMemoryUtilizationPercentage <= 100)" +// +kubebuilder:validation:XValidation:message="at least one metric must be specified when autoscaling is enabled",rule="!self.enabled || (has(self.targetCPUUtilizationPercentage) || has(self.targetMemoryUtilizationPercentage) || (has(self.autoscalingTemplate) && size(self.autoscalingTemplate) > 0))" +// +kubebuilder:validation:XValidation:message="minReplicas must be less than or equal to maxReplicas",rule="self.minReplicas <= self.maxReplicas" +// +kubebuilder:validation:XValidation:message="CPU utilization must be between 1 and 100",rule="!has(self.targetCPUUtilizationPercentage) || (self.targetCPUUtilizationPercentage >= 1 && self.targetCPUUtilizationPercentage <= 100)" +// +kubebuilder:validation:XValidation:message="memory utilization must be between 1 and 100",rule="!has(self.targetMemoryUtilizationPercentage) || (self.targetMemoryUtilizationPercentage >= 1 && self.targetMemoryUtilizationPercentage <= 100)" // //nolint:lll type HPASpec struct { diff --git a/charts/nginx-gateway-fabric/README.md b/charts/nginx-gateway-fabric/README.md index e4365fea42..7f4ccd7137 100644 --- a/charts/nginx-gateway-fabric/README.md +++ b/charts/nginx-gateway-fabric/README.md @@ -269,7 +269,7 @@ The following table lists the configurable parameters of the NGINX Gateway Fabri | `nginx.container` | The container configuration for the NGINX container. This is applied globally to all Gateways managed by this instance of NGINX Gateway Fabric. | object | `{"hostPorts":[],"lifecycle":{},"readinessProbe":{},"resources":{},"volumeMounts":[]}` | | `nginx.container.hostPorts` | A list of HostPorts to expose on the host. This configuration allows containers to bind to a specific port on the host node, enabling external network traffic to reach the container directly through the host's IP address and port. Use this option when you need to expose container ports on the host for direct access, such as for debugging, legacy integrations, or when NodePort/LoadBalancer services are not suitable. Note: Using hostPort may have security and scheduling implications, as it ties pods to specific nodes and ports. | list | `[]` | | `nginx.container.lifecycle` | The lifecycle of the NGINX container. | object | `{}` | -| `nginx.container.resources` | The resource requirements of the NGINX container. You should be set this value, If you want to use dataplane Autoscaling(HPA). | object | `{}` | +| `nginx.container.resources` | The resource requirements of the NGINX container. You should set this value if you want to use dataplane Autoscaling(HPA). | object | `{}` | | `nginx.container.volumeMounts` | volumeMounts are the additional volume mounts for the NGINX container. | list | `[]` | | `nginx.debug` | Enable debugging for NGINX. Uses the nginx-debug binary. The NGINX error log level should be set to debug in the NginxProxy resource. | bool | `false` | | `nginx.image.repository` | The NGINX image to use. | string | `"ghcr.io/nginx/nginx-gateway-fabric/nginx"` | diff --git a/charts/nginx-gateway-fabric/values.schema.json b/charts/nginx-gateway-fabric/values.schema.json index f188636a0a..48f6ac3321 100644 --- a/charts/nginx-gateway-fabric/values.schema.json +++ b/charts/nginx-gateway-fabric/values.schema.json @@ -380,7 +380,7 @@ "type": "object" }, "resources": { - "description": "The resource requirements of the NGINX container. You should be set this value, If you want to use dataplane Autoscaling(HPA).", + "description": "The resource requirements of the NGINX container. You should set this value if you want to use dataplane Autoscaling(HPA).", "required": [], "title": "resources", "type": "object" diff --git a/charts/nginx-gateway-fabric/values.yaml b/charts/nginx-gateway-fabric/values.yaml index 78b1c307c1..320068d19a 100644 --- a/charts/nginx-gateway-fabric/values.yaml +++ b/charts/nginx-gateway-fabric/values.yaml @@ -184,10 +184,10 @@ nginxGateway: # - type: Pods # pods: # metric: - # name: nginx_gateway_fabric_nginx_process_requests_total + # name: http_requests # target: # type: AverageValue - # averageValue: 10000m + # averageValue: "100" metrics: # -- Enable exposing metrics in the Prometheus format. @@ -494,7 +494,7 @@ nginx: # - port: 80 # containerPort: 80 - # -- The resource requirements of the NGINX container. You should be set this value, If you want to use dataplane Autoscaling(HPA). + # -- The resource requirements of the NGINX container. You should set this value if you want to use dataplane Autoscaling(HPA). resources: {} # -- The lifecycle of the NGINX container. diff --git a/config/crd/bases/gateway.nginx.org_nginxproxies.yaml b/config/crd/bases/gateway.nginx.org_nginxproxies.yaml index d10e3a9851..1f7c54373b 100644 --- a/config/crd/bases/gateway.nginx.org_nginxproxies.yaml +++ b/config/crd/bases/gateway.nginx.org_nginxproxies.yaml @@ -4192,6 +4192,20 @@ spec: - enabled - maxReplicas type: object + x-kubernetes-validations: + - message: at least one metric must be specified when autoscaling + is enabled + rule: '!self.enabled || (has(self.targetCPUUtilizationPercentage) + || has(self.targetMemoryUtilizationPercentage) || (has(self.autoscalingTemplate) + && size(self.autoscalingTemplate) > 0))' + - message: minReplicas must be less than or equal to maxReplicas + rule: self.minReplicas <= self.maxReplicas + - message: CPU utilization must be between 1 and 100 + rule: '!has(self.targetCPUUtilizationPercentage) || (self.targetCPUUtilizationPercentage + >= 1 && self.targetCPUUtilizationPercentage <= 100)' + - message: memory utilization must be between 1 and 100 + rule: '!has(self.targetMemoryUtilizationPercentage) || (self.targetMemoryUtilizationPercentage + >= 1 && self.targetMemoryUtilizationPercentage <= 100)' container: description: Container defines container fields for the NGINX container. diff --git a/deploy/crds.yaml b/deploy/crds.yaml index 297cfee89a..a03165f354 100644 --- a/deploy/crds.yaml +++ b/deploy/crds.yaml @@ -4777,6 +4777,20 @@ spec: - enabled - maxReplicas type: object + x-kubernetes-validations: + - message: at least one metric must be specified when autoscaling + is enabled + rule: '!self.enabled || (has(self.targetCPUUtilizationPercentage) + || has(self.targetMemoryUtilizationPercentage) || (has(self.autoscalingTemplate) + && size(self.autoscalingTemplate) > 0))' + - message: minReplicas must be less than or equal to maxReplicas + rule: self.minReplicas <= self.maxReplicas + - message: CPU utilization must be between 1 and 100 + rule: '!has(self.targetCPUUtilizationPercentage) || (self.targetCPUUtilizationPercentage + >= 1 && self.targetCPUUtilizationPercentage <= 100)' + - message: memory utilization must be between 1 and 100 + rule: '!has(self.targetMemoryUtilizationPercentage) || (self.targetMemoryUtilizationPercentage + >= 1 && self.targetMemoryUtilizationPercentage <= 100)' container: description: Container defines container fields for the NGINX container. From ca0383c3695266d9c8ca7e04720e64c54efa9ffa Mon Sep 17 00:00:00 2001 From: nowjean Date: Thu, 31 Jul 2025 10:18:37 +0900 Subject: [PATCH 13/13] Code Review --- charts/nginx-gateway-fabric/values.schema.json | 2 +- charts/nginx-gateway-fabric/values.yaml | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/charts/nginx-gateway-fabric/values.schema.json b/charts/nginx-gateway-fabric/values.schema.json index 48f6ac3321..12444f0912 100644 --- a/charts/nginx-gateway-fabric/values.schema.json +++ b/charts/nginx-gateway-fabric/values.schema.json @@ -407,7 +407,7 @@ "type": "boolean" }, "image": { - "description": "Custom or additional autoscaling metrics\nref: https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#support-for-custom-metrics\n- type: Pods\n pods:\n metric:\n name: nginx_gateway_fabric_nginx_process_requests_total\n target:\n type: AverageValue\n averageValue: 10000m", + "description": "Custom or additional autoscaling metrics\nref: https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#support-for-custom-metrics\n- type: Pods\n pods:\n metric:\n name: nginx_gateway_fabric_nginx_process_requests_total\n target:\n type: nginx_http_requests_total\n averageValue: 400", "properties": { "pullPolicy": { "default": "Always", diff --git a/charts/nginx-gateway-fabric/values.yaml b/charts/nginx-gateway-fabric/values.yaml index 320068d19a..3928ebde96 100644 --- a/charts/nginx-gateway-fabric/values.yaml +++ b/charts/nginx-gateway-fabric/values.yaml @@ -184,10 +184,10 @@ nginxGateway: # - type: Pods # pods: # metric: - # name: http_requests + # name: container_memory_usage_bytes # target: # type: AverageValue - # averageValue: "100" + # averageValue: "400Mi" metrics: # -- Enable exposing metrics in the Prometheus format. @@ -257,8 +257,8 @@ nginx: # metric: # name: nginx_gateway_fabric_nginx_process_requests_total # target: - # type: AverageValue - # averageValue: 10000m + # type: nginx_http_requests_total + # averageValue: 400 image: # -- The NGINX image to use. repository: ghcr.io/nginx/nginx-gateway-fabric/nginx