diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index 7b1f5e4447..486c208841 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -30,6 +30,7 @@ jobs: matrix: k8s-version: ["1.23.17", "latest"] nginx-image: [nginx, nginx-plus] + enable-experimental: [true, false] permissions: contents: write # needed for uploading release artifacts steps: @@ -148,6 +149,7 @@ jobs: ngf_tag=${{ steps.ngf-meta.outputs.version }} if [ ${{ github.event_name }} == "schedule" ]; then export GW_API_VERSION=main; fi if [ ${{ startsWith(matrix.k8s-version, '1.23') || startsWith(matrix.k8s-version, '1.24') }} == "true" ]; then export INSTALL_WEBHOOK=true; fi + if [ ${{ matrix.enable-experimental }} == "true" ]; then export ENABLE_EXPERIMENTAL=true; fi make install-ngf-local-no-build${{ matrix.nginx-image == 'nginx-plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag} working-directory: ./conformance diff --git a/.gitleaksignore b/.gitleaksignore index 9cc154248f..5b2f473b18 100644 --- a/.gitleaksignore +++ b/.gitleaksignore @@ -3,3 +3,4 @@ 68d1f6eb80d23c8650c11629459dd6a06c986ca1:internal/state/graph/graph_test.go:private-key:44 890fddb787ff3560b9b743647a36b649d498ae51:internal/state/graph/secret_test.go:private-key:35 890fddb787ff3560b9b743647a36b649d498ae51:internal/state/change_processor_test.go:private-key:211 +internal/mode/static/state/graph/config_maps_test.go:private-key:35 diff --git a/Makefile b/Makefile index 1f717fe4eb..cac2bf24e0 100644 --- a/Makefile +++ b/Makefile @@ -81,18 +81,17 @@ generate-crds: ## Generate CRDs and Go types using kubebuilder go run sigs.k8s.io/controller-tools/cmd/controller-gen object paths=./apis/... .PHONY: generate-manifests -generate-manifests: generate-manifests-plus ## Generate manifests using Helm. +generate-manifests: ## Generate manifests using Helm. cp $(CHART_DIR)/crds/* $(MANIFEST_DIR)/crds/ helm template nginx-gateway $(CHART_DIR) $(HELM_TEMPLATE_COMMON_ARGS) $(HELM_TEMPLATE_EXTRA_ARGS_FOR_ALL_MANIFESTS_FILE) -n nginx-gateway | cat $(strip $(MANIFEST_DIR))/namespace.yaml - > $(strip $(MANIFEST_DIR))/nginx-gateway.yaml + helm template nginx-gateway $(CHART_DIR) $(HELM_TEMPLATE_COMMON_ARGS) $(HELM_TEMPLATE_EXTRA_ARGS_FOR_ALL_MANIFESTS_FILE) --set nginx.plus=true --set nginx.image.repository=$(NGINX_PLUS_PREFIX) -n nginx-gateway | cat $(strip $(MANIFEST_DIR))/namespace.yaml - > $(strip $(MANIFEST_DIR))/nginx-plus-gateway.yaml + helm template nginx-gateway $(CHART_DIR) $(HELM_TEMPLATE_COMMON_ARGS) $(HELM_TEMPLATE_EXTRA_ARGS_FOR_ALL_MANIFESTS_FILE) --set nginxGateway.gwAPIExperimentalFeatures.enable=true -n nginx-gateway | cat $(strip $(MANIFEST_DIR))/namespace.yaml - > $(strip $(MANIFEST_DIR))/nginx-gateway-experimental.yaml + helm template nginx-gateway $(CHART_DIR) $(HELM_TEMPLATE_COMMON_ARGS) $(HELM_TEMPLATE_EXTRA_ARGS_FOR_ALL_MANIFESTS_FILE) --set nginxGateway.gwAPIExperimentalFeatures.enable=true --set nginx.plus=true --set nginx.image.repository=$(NGINX_PLUS_PREFIX) -n nginx-gateway | cat $(strip $(MANIFEST_DIR))/namespace.yaml - > $(strip $(MANIFEST_DIR))/nginx-plus-gateway-experimental.yaml helm template nginx-gateway $(CHART_DIR) $(HELM_TEMPLATE_COMMON_ARGS) --set metrics.enable=false -n nginx-gateway -s templates/deployment.yaml > conformance/provisioner/static-deployment.yaml helm template nginx-gateway $(CHART_DIR) $(HELM_TEMPLATE_COMMON_ARGS) -n nginx-gateway -s templates/service.yaml > $(strip $(MANIFEST_DIR))/service/loadbalancer.yaml helm template nginx-gateway $(CHART_DIR) $(HELM_TEMPLATE_COMMON_ARGS) --set service.annotations.'service\.beta\.kubernetes\.io\/aws-load-balancer-type'="nlb" -n nginx-gateway -s templates/service.yaml > $(strip $(MANIFEST_DIR))/service/loadbalancer-aws-nlb.yaml helm template nginx-gateway $(CHART_DIR) $(HELM_TEMPLATE_COMMON_ARGS) --set service.type=NodePort --set service.externalTrafficPolicy="" -n nginx-gateway -s templates/service.yaml > $(strip $(MANIFEST_DIR))/service/nodeport.yaml -.PHONY: generate-manifests-plus -generate-manifests-plus: ## Generate manifests using Helm for NGINX Plus. - helm template nginx-gateway $(CHART_DIR) $(HELM_TEMPLATE_COMMON_ARGS) $(HELM_TEMPLATE_EXTRA_ARGS_FOR_ALL_MANIFESTS_FILE) --set nginx.plus=true --set nginx.image.repository=$(NGINX_PLUS_PREFIX) -n nginx-gateway | cat $(strip $(MANIFEST_DIR))/namespace.yaml - > $(strip $(MANIFEST_DIR))/nginx-plus-gateway.yaml - .PHONY: crds-release-file crds-release-file: ## Generate combined crds file for releases scripts/combine-crds.sh diff --git a/cmd/gateway/commands.go b/cmd/gateway/commands.go index ac97185fd1..7265f5c4a7 100644 --- a/cmd/gateway/commands.go +++ b/cmd/gateway/commands.go @@ -56,6 +56,7 @@ func createStaticModeCommand() *cobra.Command { leaderElectionDisableFlag = "leader-election-disable" leaderElectionLockNameFlag = "leader-election-lock-name" plusFlag = "nginx-plus" + gwAPIExperimentalFlag = "gateway-api-experimental-features" ) // flag values @@ -95,6 +96,8 @@ func createStaticModeCommand() *cobra.Command { } plus bool + + gwExperimentalFeatures bool ) cmd := &cobra.Command{ @@ -172,6 +175,7 @@ func createStaticModeCommand() *cobra.Command { Plus: plus, TelemetryReportPeriod: period, Version: version, + ExperimentalFeatures: gwExperimentalFeatures, } if err := static.StartManager(conf); err != nil { @@ -285,6 +289,14 @@ func createStaticModeCommand() *cobra.Command { "Use NGINX Plus", ) + cmd.Flags().BoolVar( + &gwExperimentalFeatures, + gwAPIExperimentalFlag, + false, + "Enable the experimental features of Gateway API which are supported by NGINX Gateway Fabric. "+ + "Requires the Gateway APIs installed from the experimental channel.", + ) + return cmd } diff --git a/conformance/Makefile b/conformance/Makefile index 795348b914..30f772ceb4 100644 --- a/conformance/Makefile +++ b/conformance/Makefile @@ -15,6 +15,7 @@ CRDS=../deploy/manifests/crds/ STATIC_MANIFEST=provisioner/static-deployment.yaml PROVISIONER_MANIFEST=provisioner/provisioner.yaml INSTALL_WEBHOOK ?= false +ENABLE_EXPERIMENTAL ?= false .DEFAULT_GOAL := help .PHONY: help @@ -37,7 +38,7 @@ create-kind-cluster: ## Create a kind cluster .PHONY: update-ngf-manifest update-ngf-manifest: ## Update the NGF deployment manifest image names and imagePullPolicies - cd .. && make generate-manifests HELM_TEMPLATE_EXTRA_ARGS_FOR_ALL_MANIFESTS_FILE="--set nginxGateway.kind=skip" HELM_TEMPLATE_COMMON_ARGS="--set nginxGateway.image.repository=$(PREFIX) --set nginxGateway.image.tag=$(TAG) --set nginxGateway.image.pullPolicy=Never --set nginx.image.repository=$(NGINX_PREFIX) --set nginx.image.tag=$(TAG) --set nginx.image.pullPolicy=Never" && cd - + cd .. && make generate-manifests HELM_TEMPLATE_EXTRA_ARGS_FOR_ALL_MANIFESTS_FILE="--set nginxGateway.kind=skip" HELM_TEMPLATE_COMMON_ARGS="--set nginxGateway.image.repository=$(PREFIX) --set nginxGateway.image.tag=$(TAG) --set nginxGateway.image.pullPolicy=Never --set nginx.image.repository=$(NGINX_PREFIX) --set nginx.image.tag=$(TAG) --set nginx.image.pullPolicy=Never --set nginxGateway.experimentalFeatures.enable=$(ENABLE_EXPERIMENTAL)" && cd - .PHONY: update-ngf-manifest-with-plus update-ngf-manifest-with-plus: ## Update the NGF deployment manifest image names and imagePullPolicies including nginx-plus @@ -61,7 +62,7 @@ load-images-with-plus: ## Load NGF and NGINX Plus images on configured kind clus .PHONY: prepare-ngf-dependencies prepare-ngf-dependencies: update-ngf-manifest ## Install NGF dependencies on configured kind cluster - ./scripts/install-gateway.sh $(GW_API_VERSION) $(INSTALL_WEBHOOK) + ./scripts/install-gateway.sh $(GW_API_VERSION) $(INSTALL_WEBHOOK) $(ENABLE_EXPERIMENTAL) kubectl apply -f $(CRDS) kubectl apply -f $(NGF_MANIFEST) @@ -118,7 +119,7 @@ uninstall-ngf: uninstall-k8s-components undo-manifests-update ## Uninstall NGF o .PHONY: uninstall-k8s-components uninstall-k8s-components: ## Uninstall installed components on configured kind cluster -kubectl delete -f $(NGF_MANIFEST) - ./scripts/uninstall-gateway.sh $(GW_API_VERSION) $(INSTALL_WEBHOOK) + ./scripts/uninstall-gateway.sh $(GW_API_VERSION) $(INSTALL_WEBHOOK) $(ENABLE_EXPERIMENTAL) kubectl delete clusterrole nginx-gateway-provisioner kubectl delete clusterrolebinding nginx-gateway-provisioner diff --git a/conformance/README.md b/conformance/README.md index aba66e6933..87c06d4383 100644 --- a/conformance/README.md +++ b/conformance/README.md @@ -44,23 +44,24 @@ update-ngf-manifest Update the NGF deployment manifest image na **Note:** The following variables are configurable when running the below `make` commands: -| Variable | Default | Description | -|----------------------|---------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------| -| CONFORMANCE_TAG | latest | The tag for the conformance test image | -| CONFORMANCE_PREFIX | conformance-test-runner | The prefix for the conformance test image | -| TAG | edge | The tag for the locally built NGF image | -| PREFIX | nginx-gateway-fabric | The prefix for the locally built NGF image | -| GW_API_VERSION | 1.0.0 | Tag for the Gateway API version to check out. Set to `main` to get the latest version | -| KIND_IMAGE | Latest kind image, as defined in the tests/Dockerfile | The kind image to use | -| KIND_KUBE_CONFIG | ~/.kube/kind/config | The location of the kubeconfig | -| GATEWAY_CLASS | nginx | The gateway class that should be used for the tests | -| SUPPORTED_FEATURES | HTTPRoute,HTTPRouteQueryParamMatching, HTTPRouteMethodMatching,HTTPRoutePortRedirect, HTTPRouteSchemeRedirect | The supported features that should be tested by the conformance tests. Ensure the list is comma separated with no spaces. | -| EXEMPT_FEATURES | ReferenceGrant | The features that should not be tested by the conformance tests | -| NGF_MANIFEST | ../deploy/manifests/nginx-gateway.yaml | The location of the NGF manifest | -| SERVICE_MANIFEST | ../deploy/manifests/service/nodeport.yaml | The location of the NGF Service manifest | -| STATIC_MANIFEST | provisioner/static-deployment.yaml | The location of the NGF static deployment manifest | -| PROVISIONER_MANIFEST | provisioner/provisioner.yaml | The location of the NGF provisioner manifest | -| INSTALL_WEBHOOK | false | Install the Gateway API Validating Webhook. Necessary for Kubernetes versions < 1.25. | +| Variable | Default | Description | +| -------------------- | ------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------- | +| CONFORMANCE_TAG | latest | The tag for the conformance test image | +| CONFORMANCE_PREFIX | conformance-test-runner | The prefix for the conformance test image | +| TAG | edge | The tag for the locally built NGF image | +| PREFIX | nginx-gateway-fabric | The prefix for the locally built NGF image | +| GW_API_VERSION | 1.0.0 | Tag for the Gateway API version to check out. Set to `main` to get the latest version | +| KIND_IMAGE | Latest kind image, as defined in the tests/Dockerfile | The kind image to use | +| KIND_KUBE_CONFIG | ~/.kube/kind/config | The location of the kubeconfig | +| GATEWAY_CLASS | nginx | The gateway class that should be used for the tests | +| SUPPORTED_FEATURES | HTTPRoute,HTTPRouteQueryParamMatching, HTTPRouteMethodMatching,HTTPRoutePortRedirect, HTTPRouteSchemeRedirect | The supported features that should be tested by the conformance tests. Ensure the list is comma separated with no spaces. | +| EXEMPT_FEATURES | ReferenceGrant | The features that should not be tested by the conformance tests | +| NGF_MANIFEST | ../deploy/manifests/nginx-gateway.yaml | The location of the NGF manifest | +| SERVICE_MANIFEST | ../deploy/manifests/service/nodeport.yaml | The location of the NGF Service manifest | +| STATIC_MANIFEST | provisioner/static-deployment.yaml | The location of the NGF static deployment manifest | +| PROVISIONER_MANIFEST | provisioner/provisioner.yaml | The location of the NGF provisioner manifest | +| INSTALL_WEBHOOK | false | Install the Gateway API Validating Webhook. Necessary for Kubernetes versions < 1.25. | +| ENABLE_EXPERIMENTAL | false | Enable experimental features. Installs the Gateway APIs from the experimental channel and enables any supported experimental features in NGF. | ### Step 1 - Create a kind Cluster @@ -85,6 +86,12 @@ make create-kind-cluster KIND_IMAGE=kindest/node:v1.27.3 ``` > Otherwise, the latest stable version will be used by default. +> Additionally, if you want to run conformance tests with experimental features enabled, set the following +> environment variable before deploying NGF: + +```bash + export ENABLE_EXPERIMENTAL=true +``` #### *Option 1* Build and install NGINX Gateway Fabric from local to configured kind cluster diff --git a/conformance/scripts/install-gateway.sh b/conformance/scripts/install-gateway.sh index 485e77353e..641567d987 100755 --- a/conformance/scripts/install-gateway.sh +++ b/conformance/scripts/install-gateway.sh @@ -14,14 +14,22 @@ if [ $1 == "main" ]; then temp_dir=$(mktemp -d) cd ${temp_dir} curl -s https://codeload.github.com/kubernetes-sigs/gateway-api/tar.gz/main | tar -xz --strip=2 gateway-api-main/config - kubectl apply -f crd/standard + if [ $3 == "true" ]; then + kubectl apply -f crd/experimental + else + kubectl apply -f crd/standard + fi if [ $2 == "true" ]; then kubectl apply -f webhook kubectl wait --for=condition=available --timeout=60s deployment gateway-api-admission-server -n gateway-system fi rm -rf ${temp_dir} else - kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v$1/standard-install.yaml + if [ $3 == "true" ]; then + kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v$1/experimental-install.yaml + else + kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v$1/standard-install.yaml + fi if [ $2 == "true" ]; then kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v$1/webhook-install.yaml kubectl wait --for=condition=available --timeout=60s deployment gateway-api-admission-server -n gateway-system diff --git a/conformance/scripts/uninstall-gateway.sh b/conformance/scripts/uninstall-gateway.sh index 333085661d..f6f26c2c00 100755 --- a/conformance/scripts/uninstall-gateway.sh +++ b/conformance/scripts/uninstall-gateway.sh @@ -5,17 +5,30 @@ if [ -z $1 ]; then exit 1 fi +if [ -z $2 ]; then + echo "install webhook argument not set; exiting" + exit 1 +fi + if [ $1 == "main" ]; then temp_dir=$(mktemp -d) cd ${temp_dir} curl -s https://codeload.github.com/kubernetes-sigs/gateway-api/tar.gz/main | tar -xz --strip=2 gateway-api-main/config - kubectl delete -f crd/standard + if [ $3 == "true" ]; then + kubectl delete -f crd/experimental + else + kubectl delete -f crd/standard + fi if [ $2 == "true" ]; then kubectl delete -f webhook fi rm -rf ${temp_dir} else - kubectl delete -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v$1/standard-install.yaml + if [ $3 == "true" ]; then + kubectl delete -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v$1/experimental-install.yaml + else + kubectl delete -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v$1/standard-install.yaml + fi if [ $2 == "true" ]; then kubectl delete -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v$1/webhook-install.yaml fi diff --git a/deploy/helm-chart/templates/deployment.yaml b/deploy/helm-chart/templates/deployment.yaml index bb5a64c4f0..79b0e242cb 100644 --- a/deploy/helm-chart/templates/deployment.yaml +++ b/deploy/helm-chart/templates/deployment.yaml @@ -52,6 +52,9 @@ spec: {{- else }} - --leader-election-disable {{- end }} + {{- if .Values.nginxGateway.gwAPIExperimentalFeatures.enable }} + - --gateway-api-experimental-features + {{- end }} env: - name: POD_IP valueFrom: diff --git a/deploy/helm-chart/templates/rbac.yaml b/deploy/helm-chart/templates/rbac.yaml index cbcb66565e..16939ef6c9 100644 --- a/deploy/helm-chart/templates/rbac.yaml +++ b/deploy/helm-chart/templates/rbac.yaml @@ -32,6 +32,9 @@ rules: - namespaces - services - secrets +{{- if .Values.nginxGateway.gwAPIExperimentalFeatures.enable }} + - configmaps +{{- end }} verbs: - list - watch @@ -76,6 +79,9 @@ rules: - gateways - httproutes - referencegrants +{{- if .Values.nginxGateway.gwAPIExperimentalFeatures.enable }} + - backendtlspolicies +{{- end }} verbs: - list - watch @@ -85,6 +91,9 @@ rules: - httproutes/status - gateways/status - gatewayclasses/status +{{- if .Values.nginxGateway.gwAPIExperimentalFeatures.enable }} + - backendtlspolicies/status +{{- end }} verbs: - update - apiGroups: diff --git a/deploy/helm-chart/values.yaml b/deploy/helm-chart/values.yaml index 961f327603..8154343043 100644 --- a/deploy/helm-chart/values.yaml +++ b/deploy/helm-chart/values.yaml @@ -51,6 +51,11 @@ nginxGateway: ## extraVolumeMounts are the additional volume mounts for the nginx-gateway container. extraVolumeMounts: [] + gwAPIExperimentalFeatures: + ## Enable the experimental features of Gateway API which are supported by NGINX Gateway Fabric. Requires the Gateway + ## APIs installed from the experimental channel. + enable: false + nginx: ## The NGINX image to use image: diff --git a/deploy/manifests/nginx-gateway-experimental.yaml b/deploy/manifests/nginx-gateway-experimental.yaml new file mode 100644 index 0000000000..0625b4b5e2 --- /dev/null +++ b/deploy/manifests/nginx-gateway-experimental.yaml @@ -0,0 +1,291 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: nginx-gateway +--- +# Source: nginx-gateway-fabric/templates/rbac.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: nginx-gateway + namespace: nginx-gateway + labels: + app.kubernetes.io/name: nginx-gateway + app.kubernetes.io/instance: nginx-gateway + app.kubernetes.io/version: "edge" + annotations: + {} +--- +# Source: nginx-gateway-fabric/templates/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: nginx-gateway + labels: + app.kubernetes.io/name: nginx-gateway + app.kubernetes.io/instance: nginx-gateway + app.kubernetes.io/version: "edge" +rules: +- apiGroups: + - "" + resources: + - namespaces + - services + - secrets + - configmaps + verbs: + - list + - watch +# FIXME(bjee19): make nodes, pods, replicasets permission dependent on telemetry being enabled. +# https://github.com/nginxinc/nginx-gateway-fabric/issues/1317. +- apiGroups: + - "" + resources: + - pods + verbs: + - get +- apiGroups: + - "" + resources: + - nodes + verbs: + - list +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch +- apiGroups: + - apps + resources: + - replicasets + verbs: + - get +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - list + - watch +- apiGroups: + - gateway.networking.k8s.io + resources: + - gatewayclasses + - gateways + - httproutes + - referencegrants + - backendtlspolicies + verbs: + - list + - watch +- apiGroups: + - gateway.networking.k8s.io + resources: + - httproutes/status + - gateways/status + - gatewayclasses/status + - backendtlspolicies/status + verbs: + - update +- apiGroups: + - gateway.nginx.org + resources: + - nginxgateways + verbs: + - get + - list + - watch +- apiGroups: + - gateway.nginx.org + resources: + - nginxgateways/status + verbs: + - update +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - get + - update +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - list + - watch +--- +# Source: nginx-gateway-fabric/templates/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: nginx-gateway + labels: + app.kubernetes.io/name: nginx-gateway + app.kubernetes.io/instance: nginx-gateway + app.kubernetes.io/version: "edge" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: nginx-gateway +subjects: +- kind: ServiceAccount + name: nginx-gateway + namespace: nginx-gateway +--- +# Source: nginx-gateway-fabric/templates/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx-gateway + namespace: nginx-gateway + labels: + app.kubernetes.io/name: nginx-gateway + app.kubernetes.io/instance: nginx-gateway + app.kubernetes.io/version: "edge" +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: nginx-gateway + app.kubernetes.io/instance: nginx-gateway + template: + metadata: + labels: + app.kubernetes.io/name: nginx-gateway + app.kubernetes.io/instance: nginx-gateway + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9113" + spec: + containers: + - args: + - static-mode + - --gateway-ctlr-name=gateway.nginx.org/nginx-gateway-controller + - --gatewayclass=nginx + - --config=nginx-gateway-config + - --service=nginx-gateway + - --metrics-port=9113 + - --health-port=8081 + - --leader-election-lock-name=nginx-gateway-leader-election + - --gateway-api-experimental-features + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + image: ghcr.io/nginxinc/nginx-gateway-fabric:edge + imagePullPolicy: Always + name: nginx-gateway + ports: + - name: metrics + containerPort: 9113 + - name: health + containerPort: 8081 + readinessProbe: + httpGet: + path: /readyz + port: health + initialDelaySeconds: 3 + periodSeconds: 1 + securityContext: + allowPrivilegeEscalation: false + capabilities: + add: + - KILL + drop: + - ALL + readOnlyRootFilesystem: true + runAsUser: 102 + runAsGroup: 1001 + volumeMounts: + - name: nginx-conf + mountPath: /etc/nginx/conf.d + - name: nginx-secrets + mountPath: /etc/nginx/secrets + - name: nginx-run + mountPath: /var/run/nginx + - image: ghcr.io/nginxinc/nginx-gateway-fabric/nginx:edge + imagePullPolicy: Always + name: nginx + ports: + - containerPort: 80 + name: http + - containerPort: 443 + name: https + securityContext: + capabilities: + add: + - NET_BIND_SERVICE + drop: + - ALL + readOnlyRootFilesystem: true + runAsUser: 101 + runAsGroup: 1001 + volumeMounts: + - name: nginx-conf + mountPath: /etc/nginx/conf.d + - name: nginx-secrets + mountPath: /etc/nginx/secrets + - name: nginx-run + mountPath: /var/run/nginx + - name: nginx-cache + mountPath: /var/cache/nginx + - name: nginx-lib + mountPath: /var/lib/nginx + terminationGracePeriodSeconds: 30 + serviceAccountName: nginx-gateway + shareProcessNamespace: true + securityContext: + fsGroup: 1001 + runAsNonRoot: true + volumes: + - name: nginx-conf + emptyDir: {} + - name: nginx-secrets + emptyDir: {} + - name: nginx-run + emptyDir: {} + - name: nginx-cache + emptyDir: {} + - name: nginx-lib + emptyDir: {} +--- +# Source: nginx-gateway-fabric/templates/gatewayclass.yaml +apiVersion: gateway.networking.k8s.io/v1 +kind: GatewayClass +metadata: + name: nginx + labels: + app.kubernetes.io/name: nginx-gateway + app.kubernetes.io/instance: nginx-gateway + app.kubernetes.io/version: "edge" +spec: + controllerName: gateway.nginx.org/nginx-gateway-controller +--- +# Source: nginx-gateway-fabric/templates/nginxgateway.yaml +apiVersion: gateway.nginx.org/v1alpha1 +kind: NginxGateway +metadata: + name: nginx-gateway-config + namespace: nginx-gateway + labels: + app.kubernetes.io/name: nginx-gateway + app.kubernetes.io/instance: nginx-gateway + app.kubernetes.io/version: "edge" +spec: + logging: + level: info diff --git a/deploy/manifests/nginx-plus-gateway-experimental.yaml b/deploy/manifests/nginx-plus-gateway-experimental.yaml new file mode 100644 index 0000000000..49d099b894 --- /dev/null +++ b/deploy/manifests/nginx-plus-gateway-experimental.yaml @@ -0,0 +1,292 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: nginx-gateway +--- +# Source: nginx-gateway-fabric/templates/rbac.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: nginx-gateway + namespace: nginx-gateway + labels: + app.kubernetes.io/name: nginx-gateway + app.kubernetes.io/instance: nginx-gateway + app.kubernetes.io/version: "edge" + annotations: + {} +--- +# Source: nginx-gateway-fabric/templates/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: nginx-gateway + labels: + app.kubernetes.io/name: nginx-gateway + app.kubernetes.io/instance: nginx-gateway + app.kubernetes.io/version: "edge" +rules: +- apiGroups: + - "" + resources: + - namespaces + - services + - secrets + - configmaps + verbs: + - list + - watch +# FIXME(bjee19): make nodes, pods, replicasets permission dependent on telemetry being enabled. +# https://github.com/nginxinc/nginx-gateway-fabric/issues/1317. +- apiGroups: + - "" + resources: + - pods + verbs: + - get +- apiGroups: + - "" + resources: + - nodes + verbs: + - list +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch +- apiGroups: + - apps + resources: + - replicasets + verbs: + - get +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - list + - watch +- apiGroups: + - gateway.networking.k8s.io + resources: + - gatewayclasses + - gateways + - httproutes + - referencegrants + - backendtlspolicies + verbs: + - list + - watch +- apiGroups: + - gateway.networking.k8s.io + resources: + - httproutes/status + - gateways/status + - gatewayclasses/status + - backendtlspolicies/status + verbs: + - update +- apiGroups: + - gateway.nginx.org + resources: + - nginxgateways + verbs: + - get + - list + - watch +- apiGroups: + - gateway.nginx.org + resources: + - nginxgateways/status + verbs: + - update +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - get + - update +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - list + - watch +--- +# Source: nginx-gateway-fabric/templates/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: nginx-gateway + labels: + app.kubernetes.io/name: nginx-gateway + app.kubernetes.io/instance: nginx-gateway + app.kubernetes.io/version: "edge" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: nginx-gateway +subjects: +- kind: ServiceAccount + name: nginx-gateway + namespace: nginx-gateway +--- +# Source: nginx-gateway-fabric/templates/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx-gateway + namespace: nginx-gateway + labels: + app.kubernetes.io/name: nginx-gateway + app.kubernetes.io/instance: nginx-gateway + app.kubernetes.io/version: "edge" +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: nginx-gateway + app.kubernetes.io/instance: nginx-gateway + template: + metadata: + labels: + app.kubernetes.io/name: nginx-gateway + app.kubernetes.io/instance: nginx-gateway + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9113" + spec: + containers: + - args: + - static-mode + - --gateway-ctlr-name=gateway.nginx.org/nginx-gateway-controller + - --gatewayclass=nginx + - --config=nginx-gateway-config + - --service=nginx-gateway + - --nginx-plus + - --metrics-port=9113 + - --health-port=8081 + - --leader-election-lock-name=nginx-gateway-leader-election + - --gateway-api-experimental-features + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + image: ghcr.io/nginxinc/nginx-gateway-fabric:edge + imagePullPolicy: Always + name: nginx-gateway + ports: + - name: metrics + containerPort: 9113 + - name: health + containerPort: 8081 + readinessProbe: + httpGet: + path: /readyz + port: health + initialDelaySeconds: 3 + periodSeconds: 1 + securityContext: + allowPrivilegeEscalation: false + capabilities: + add: + - KILL + drop: + - ALL + readOnlyRootFilesystem: true + runAsUser: 102 + runAsGroup: 1001 + volumeMounts: + - name: nginx-conf + mountPath: /etc/nginx/conf.d + - name: nginx-secrets + mountPath: /etc/nginx/secrets + - name: nginx-run + mountPath: /var/run/nginx + - image: nginx-gateway-fabric/nginx-plus:edge + imagePullPolicy: Always + name: nginx + ports: + - containerPort: 80 + name: http + - containerPort: 443 + name: https + securityContext: + capabilities: + add: + - NET_BIND_SERVICE + drop: + - ALL + readOnlyRootFilesystem: true + runAsUser: 101 + runAsGroup: 1001 + volumeMounts: + - name: nginx-conf + mountPath: /etc/nginx/conf.d + - name: nginx-secrets + mountPath: /etc/nginx/secrets + - name: nginx-run + mountPath: /var/run/nginx + - name: nginx-cache + mountPath: /var/cache/nginx + - name: nginx-lib + mountPath: /var/lib/nginx + terminationGracePeriodSeconds: 30 + serviceAccountName: nginx-gateway + shareProcessNamespace: true + securityContext: + fsGroup: 1001 + runAsNonRoot: true + volumes: + - name: nginx-conf + emptyDir: {} + - name: nginx-secrets + emptyDir: {} + - name: nginx-run + emptyDir: {} + - name: nginx-cache + emptyDir: {} + - name: nginx-lib + emptyDir: {} +--- +# Source: nginx-gateway-fabric/templates/gatewayclass.yaml +apiVersion: gateway.networking.k8s.io/v1 +kind: GatewayClass +metadata: + name: nginx + labels: + app.kubernetes.io/name: nginx-gateway + app.kubernetes.io/instance: nginx-gateway + app.kubernetes.io/version: "edge" +spec: + controllerName: gateway.nginx.org/nginx-gateway-controller +--- +# Source: nginx-gateway-fabric/templates/nginxgateway.yaml +apiVersion: gateway.nginx.org/v1alpha1 +kind: NginxGateway +metadata: + name: nginx-gateway-config + namespace: nginx-gateway + labels: + app.kubernetes.io/name: nginx-gateway + app.kubernetes.io/instance: nginx-gateway + app.kubernetes.io/version: "edge" +spec: + logging: + level: info diff --git a/docs/developer/quickstart.md b/docs/developer/quickstart.md index b339484ea8..41de3a4b6d 100644 --- a/docs/developer/quickstart.md +++ b/docs/developer/quickstart.md @@ -128,6 +128,12 @@ This will build the docker images `nginx-gateway-fabric:` and `nginx- kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.0.0/standard-install.yaml ``` + If you're implementing experimental Gateway API features, install Gateway API CRDs from the experimental channel: + + ```shell + kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.0.0/experimental-install.yaml + ``` + 4. Install NGF using your custom image and expose NGF with a NodePort Service: - To install with Helm (where your release name is `my-release`): @@ -162,6 +168,15 @@ This will build the docker images `nginx-gateway-fabric:` and `nginx- kubectl apply -f deploy/manifests/service/nodeport.yaml ``` + - To install with experimental manifests: + + ```shell + make generate-manifests HELM_TEMPLATE_COMMON_ARGS="--set nginxGateway.image.repository=nginx-gateway-fabric --set nginxGateway.image.tag=$(whoami) --set nginxGateway.image.pullPolicy=Never --set nginx.image.repository=nginx-gateway-fabric/nginx --set nginx.image.tag=$(whoami) --set nginx.image.pullPolicy=Never" + kubectl apply -f deploy/manifests/crds + kubectl apply -f deploy/manifests/nginx-gateway-experimental.yaml + kubectl apply -f deploy/manifests/service/nodeport.yaml + ``` + ### Run Examples To make sure NGF is running properly, try out the [examples](/examples). diff --git a/internal/framework/gatewayclass/validate.go b/internal/framework/gatewayclass/validate.go index 266e6ca5bc..828f8eb06e 100644 --- a/internal/framework/gatewayclass/validate.go +++ b/internal/framework/gatewayclass/validate.go @@ -17,10 +17,11 @@ const ( ) var gatewayCRDs = map[string]apiVersion{ - "gatewayclasses.gateway.networking.k8s.io": {}, - "gateways.gateway.networking.k8s.io": {}, - "httproutes.gateway.networking.k8s.io": {}, - "referencegrants.gateway.networking.k8s.io": {}, + "gatewayclasses.gateway.networking.k8s.io": {}, + "gateways.gateway.networking.k8s.io": {}, + "httproutes.gateway.networking.k8s.io": {}, + "referencegrants.gateway.networking.k8s.io": {}, + "backendtlspolicies.gateway.networking.k8s.io": {}, } type apiVersion struct { diff --git a/internal/framework/status/backend_tls.go b/internal/framework/status/backend_tls.go new file mode 100644 index 0000000000..7dae216846 --- /dev/null +++ b/internal/framework/status/backend_tls.go @@ -0,0 +1,45 @@ +package status + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + v1 "sigs.k8s.io/gateway-api/apis/v1" + "sigs.k8s.io/gateway-api/apis/v1alpha2" +) + +// prepareBackendTLSPolicyStatus prepares the status for a BackendTLSPolicy resource. +func prepareBackendTLSPolicyStatus( + oldStatus v1alpha2.PolicyStatus, + status BackendTLSPolicyStatus, + gatewayCtlrName string, + transitionTime metav1.Time, +) v1alpha2.PolicyStatus { + // maxAncestors is the max number of ancestor statuses which is the sum of all new ancestor statuses and all old + // ancestor statuses. + maxAncestors := len(status.AncestorStatuses) + len(oldStatus.Ancestors) + ancestors := make([]v1alpha2.PolicyAncestorStatus, 0, maxAncestors) + + // keep all the ancestor statuses that belong to other controllers + for _, os := range oldStatus.Ancestors { + if string(os.ControllerName) != gatewayCtlrName { + ancestors = append(ancestors, os) + } + } + + for _, as := range status.AncestorStatuses { + // reassign the iteration variable inside the loop to fix implicit memory aliasing + as := as + a := v1alpha2.PolicyAncestorStatus{ + AncestorRef: v1.ParentReference{ + Namespace: (*v1.Namespace)(&as.GatewayNsName.Namespace), + Name: v1alpha2.ObjectName(as.GatewayNsName.Name), + }, + ControllerName: v1alpha2.GatewayController(gatewayCtlrName), + Conditions: convertConditions(as.Conditions, status.ObservedGeneration, transitionTime), + } + ancestors = append(ancestors, a) + } + + return v1alpha2.PolicyStatus{ + Ancestors: ancestors, + } +} diff --git a/internal/framework/status/backend_tls_test.go b/internal/framework/status/backend_tls_test.go new file mode 100644 index 0000000000..8a9c967ff4 --- /dev/null +++ b/internal/framework/status/backend_tls_test.go @@ -0,0 +1,51 @@ +package status + +import ( + "testing" + + . "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + v1 "sigs.k8s.io/gateway-api/apis/v1" + "sigs.k8s.io/gateway-api/apis/v1alpha2" + + "github.com/nginxinc/nginx-gateway-fabric/internal/framework/conditions" + "github.com/nginxinc/nginx-gateway-fabric/internal/framework/helpers" +) + +func TestPrepareBackendTLSPolicyStatus(t *testing.T) { + oldStatus := v1alpha2.PolicyStatus{ + Ancestors: []v1alpha2.PolicyAncestorStatus{ + { + AncestorRef: v1.ParentReference{ + Namespace: helpers.GetPointer((v1.Namespace)("ns1")), + Name: v1alpha2.ObjectName("other-gw"), + }, + ControllerName: v1alpha2.GatewayController("otherCtlr"), + Conditions: []metav1.Condition{{Type: "otherType", Status: "otherStatus"}}, + }, + }, + } + + newStatus := BackendTLSPolicyStatus{ + AncestorStatuses: []AncestorStatus{ + { + GatewayNsName: types.NamespacedName{ + Namespace: "ns1", + Name: "gw1", + }, + Conditions: []conditions.Condition{{Type: "type1", Status: "status1"}}, + }, + }, + ObservedGeneration: 1, + } + + transistionTime := metav1.Now() + ctlrName := "nginx-gateway" + + policyStatus := prepareBackendTLSPolicyStatus(oldStatus, newStatus, ctlrName, transistionTime) + + g := NewWithT(t) + + g.Expect(policyStatus.Ancestors).To(HaveLen(2)) +} diff --git a/internal/framework/status/setters.go b/internal/framework/status/setters.go index fc67136bd0..0ca2be80aa 100644 --- a/internal/framework/status/setters.go +++ b/internal/framework/status/setters.go @@ -6,6 +6,7 @@ import ( v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" + gatewayv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" ngfAPI "github.com/nginxinc/nginx-gateway-fabric/apis/v1alpha1" ) @@ -85,6 +86,25 @@ func newHTTPRouteStatusSetter(gatewayCtlrName string, clock Clock, rs HTTPRouteS } } +func newBackendTLSPolicyStatusSetter( + gatewayCtlrName string, + clock Clock, + bs BackendTLSPolicyStatus, +) func(client.Object) bool { + return func(object client.Object) bool { + btp := object.(*gatewayv1alpha2.BackendTLSPolicy) + status := prepareBackendTLSPolicyStatus(btp.Status, bs, gatewayCtlrName, clock.Now()) + + if btpStatusEqual(gatewayCtlrName, btp.Status, status) { + return false + } + + btp.Status = status + + return true + } +} + func gwStatusEqual(prev, cur gatewayv1.GatewayStatus) bool { addressesEqual := slices.EqualFunc(prev.Addresses, cur.Addresses, func(a1, a2 gatewayv1.GatewayStatusAddress) bool { if !equalPointers[gatewayv1.AddressType](a1.Type, a2.Type) { @@ -181,6 +201,58 @@ func routeParentStatusEqual(p1, p2 gatewayv1.RouteParentStatus) bool { return conditionsEqual(p1.Conditions, p2.Conditions) } +func btpStatusEqual(gatewayCtlrName string, prev, cur gatewayv1alpha2.PolicyStatus) bool { + // Since other controllers may update BackendTLSPolicy status we can't assume anything about the order of the + // statuses, and we have to ignore statuses written by other controllers when checking for equality. + // Therefore, we can't use slices.EqualFunc here because it cares about the order. + + // First, we check if the prev status has any PolicyAncestorStatuses that are no longer present in the cur status. + for _, prevAncestor := range prev.Ancestors { + if prevAncestor.ControllerName != gatewayv1.GatewayController(gatewayCtlrName) { + continue + } + + exists := slices.ContainsFunc(cur.Ancestors, func(curAncestor gatewayv1alpha2.PolicyAncestorStatus) bool { + return btpAncestorStatusEqual(prevAncestor, curAncestor) + }) + + if !exists { + return false + } + } + + // Then, we check if the cur status has any PolicyAncestorStatuses that are no longer present in the prev status. + for _, curParent := range cur.Ancestors { + exists := slices.ContainsFunc(prev.Ancestors, func(prevAncestor gatewayv1alpha2.PolicyAncestorStatus) bool { + return btpAncestorStatusEqual(curParent, prevAncestor) + }) + + if !exists { + return false + } + } + + return true +} + +func btpAncestorStatusEqual(p1, p2 gatewayv1alpha2.PolicyAncestorStatus) bool { + if p1.ControllerName != p2.ControllerName { + return false + } + + if p1.AncestorRef.Name != p2.AncestorRef.Name { + return false + } + + if !equalPointers(p1.AncestorRef.Namespace, p2.AncestorRef.Namespace) { + return false + } + + // we ignore the rest of the AncestorRef fields because we do not set them + + return conditionsEqual(p1.Conditions, p2.Conditions) +} + func conditionsEqual(prev, cur []v1.Condition) bool { return slices.EqualFunc(prev, cur, func(c1, c2 v1.Condition) bool { if c1.ObservedGeneration != c2.ObservedGeneration { diff --git a/internal/framework/status/setters_test.go b/internal/framework/status/setters_test.go index c00fb2b447..b86f14a133 100644 --- a/internal/framework/status/setters_test.go +++ b/internal/framework/status/setters_test.go @@ -7,6 +7,7 @@ import ( . "github.com/onsi/gomega" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" + gatewayv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" ngfAPI "github.com/nginxinc/nginx-gateway-fabric/apis/v1alpha1" "github.com/nginxinc/nginx-gateway-fabric/internal/framework/conditions" @@ -848,3 +849,84 @@ func TestEqualPointers(t *testing.T) { }) } } + +func TestBtpStatusEqual(t *testing.T) { + getPolicyStatus := func(ancestorName, ancestorNs, ctlrName string) gatewayv1alpha2.PolicyStatus { + return gatewayv1alpha2.PolicyStatus{ + Ancestors: []gatewayv1alpha2.PolicyAncestorStatus{ + { + AncestorRef: gatewayv1.ParentReference{ + Namespace: helpers.GetPointer[gatewayv1.Namespace]((gatewayv1.Namespace)(ancestorNs)), + Name: gatewayv1alpha2.ObjectName(ancestorName), + }, + ControllerName: gatewayv1alpha2.GatewayController(ctlrName), + Conditions: []v1.Condition{{Type: "otherType", Status: "otherStatus"}}, + }, + }, + } + } + prevMultiple := getPolicyStatus("ancestor1", "ns1", "ctlr1") + prevMultiple.Ancestors = append(prevMultiple.Ancestors, getPolicyStatus("ancestor2", "ns2", "ctlr2").Ancestors...) + + currMultiple := getPolicyStatus("ancestor1", "ns1", "ctlr1") + currMultiple.Ancestors = append(currMultiple.Ancestors, getPolicyStatus("ancestor3", "ns3", "ctlr2").Ancestors...) + + tests := []struct { + name string + controllerName string + previous gatewayv1alpha2.PolicyStatus + current gatewayv1alpha2.PolicyStatus + expEqual bool + }{ + { + name: "status equal", + previous: getPolicyStatus("ancestor1", "ns1", "ctlr1"), + current: getPolicyStatus("ancestor1", "ns1", "ctlr1"), + controllerName: "ctlr1", + expEqual: true, + }, + { + name: "status not equal, different ancestor name", + previous: getPolicyStatus("ancestor1", "ns1", "ctlr1"), + current: getPolicyStatus("ancestor2", "ns1", "ctlr1"), + controllerName: "ctlr1", + expEqual: false, + }, + { + name: "status not equal, different ancestor namespace", + previous: getPolicyStatus("ancestor1", "ns1", "ctlr1"), + current: getPolicyStatus("ancestor1", "ns2", "ctlr1"), + controllerName: "ctlr1", + expEqual: false, + }, + { + name: "status not equal, different controller name on current", + previous: getPolicyStatus("ancestor1", "ns1", "ctlr1"), + current: getPolicyStatus("ancestor1", "ns1", "ctlr2"), + controllerName: "ctlr1", + expEqual: false, + }, + { + name: "status not equal, different controller name on previous", + previous: getPolicyStatus("ancestor1", "ns1", "ctlr2"), + current: getPolicyStatus("ancestor1", "ns1", "ctlr1"), + controllerName: "ctlr1", + expEqual: false, + }, + { + name: "status not equal, different controller ancestor changed", + previous: prevMultiple, + current: currMultiple, + controllerName: "ctlr1", + expEqual: false, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + g := NewWithT(t) + equal := btpStatusEqual(test.controllerName, test.previous, test.current) + g.Expect(equal).To(Equal(test.expEqual)) + }) + } +} diff --git a/internal/framework/status/statuses.go b/internal/framework/status/statuses.go index 92102d6aea..1de369768b 100644 --- a/internal/framework/status/statuses.go +++ b/internal/framework/status/statuses.go @@ -16,9 +16,10 @@ type Status interface { // GatewayAPIStatuses holds the status-related information about Gateway API resources. type GatewayAPIStatuses struct { - GatewayClassStatuses GatewayClassStatuses - GatewayStatuses GatewayStatuses - HTTPRouteStatuses HTTPRouteStatuses + GatewayClassStatuses GatewayClassStatuses + GatewayStatuses GatewayStatuses + HTTPRouteStatuses HTTPRouteStatuses + BackendTLSPolicyStatuses BackendTLSPolicyStatuses } func (g GatewayAPIStatuses) APIGroup() string { @@ -51,6 +52,10 @@ type GatewayStatuses map[types.NamespacedName]GatewayStatus // GatewayClassStatuses holds the statuses of GatewayClasses where the key is the namespaced name of a GatewayClass. type GatewayClassStatuses map[types.NamespacedName]GatewayClassStatus +// BackendTLSPolicyStatuses holds the statuses of BackendTLSPolicies where the key is the namespaced name of a +// BackendTLSPolicy. +type BackendTLSPolicyStatuses map[types.NamespacedName]BackendTLSPolicyStatus + // GatewayStatus holds the status of the winning Gateway resource. type GatewayStatus struct { // ListenerStatuses holds the statuses of listeners defined on the Gateway. @@ -85,6 +90,14 @@ type HTTPRouteStatus struct { ObservedGeneration int64 } +// BackendTLSPolicyStatus holds the status-related information about a BackendTLSPolicy resource. +type BackendTLSPolicyStatus struct { + // AncestorStatuses holds the statuses for parentRefs of the BackendTLSPolicy. + AncestorStatuses []AncestorStatus + // ObservedGeneration is the generation of the resource that was processed. + ObservedGeneration int64 +} + // ParentStatus holds status-related information related to how the HTTPRoute binds to a specific parentRef. type ParentStatus struct { // GatewayNsName is the Namespaced name of the Gateway, which the parentRef references. @@ -102,3 +115,11 @@ type GatewayClassStatus struct { // ObservedGeneration is the generation of the resource that was processed. ObservedGeneration int64 } + +// AncestorStatus holds status-related information related to how the BackendTLSPolicy binds to a specific ancestorRef. +type AncestorStatus struct { + // GatewayNsName is the Namespaced name of the Gateway, which the ancestorRef references. + GatewayNsName types.NamespacedName + // Conditions is the list of conditions that are relevant to the ancestor. + Conditions []conditions.Condition +} diff --git a/internal/framework/status/updater.go b/internal/framework/status/updater.go index c2ff173ba0..8b8e451915 100644 --- a/internal/framework/status/updater.go +++ b/internal/framework/status/updater.go @@ -13,6 +13,7 @@ import ( "k8s.io/apimachinery/pkg/util/wait" "sigs.k8s.io/controller-runtime/pkg/client" v1 "sigs.k8s.io/gateway-api/apis/v1" + "sigs.k8s.io/gateway-api/apis/v1alpha2" ngfAPI "github.com/nginxinc/nginx-gateway-fabric/apis/v1alpha1" "github.com/nginxinc/nginx-gateway-fabric/internal/framework/controller" @@ -204,6 +205,21 @@ func (upd *UpdaterImpl) updateGatewayAPI(ctx context.Context, statuses GatewayAP newHTTPRouteStatusSetter(upd.cfg.GatewayCtlrName, upd.cfg.Clock, rs), ) } + + for nsname, bs := range statuses.BackendTLSPolicyStatuses { + select { + case <-ctx.Done(): + return + default: + } + + upd.writeStatuses( + ctx, + nsname, + &v1alpha2.BackendTLSPolicy{}, + newBackendTLSPolicyStatusSetter(upd.cfg.GatewayCtlrName, upd.cfg.Clock, bs), + ) + } } func (upd *UpdaterImpl) writeStatuses( diff --git a/internal/framework/status/updater_test.go b/internal/framework/status/updater_test.go index d743097c07..faed9195a4 100644 --- a/internal/framework/status/updater_test.go +++ b/internal/framework/status/updater_test.go @@ -14,6 +14,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/log/zap" v1 "sigs.k8s.io/gateway-api/apis/v1" + v1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" ngfAPI "github.com/nginxinc/nginx-gateway-fabric/apis/v1alpha1" "github.com/nginxinc/nginx-gateway-fabric/internal/framework/helpers" @@ -42,6 +43,7 @@ var _ = Describe("Updater", func() { scheme := runtime.NewScheme() Expect(v1.AddToScheme(scheme)).Should(Succeed()) + Expect(v1alpha2.AddToScheme(scheme)).Should(Succeed()) Expect(ngfAPI.AddToScheme(scheme)).Should(Succeed()) client = fake.NewClientBuilder(). @@ -51,6 +53,7 @@ var _ = Describe("Updater", func() { &v1.Gateway{}, &v1.HTTPRoute{}, &ngfAPI.NginxGateway{}, + &v1alpha2.BackendTLSPolicy{}, ). Build() @@ -63,8 +66,9 @@ var _ = Describe("Updater", func() { Describe("Process status updates", Ordered, func() { type generations struct { - gatewayClass int64 - gateways int64 + gatewayClass int64 + gateways int64 + backendTLSPolicies int64 } var ( @@ -73,6 +77,7 @@ var _ = Describe("Updater", func() { gw, ignoredGw *v1.Gateway hr *v1.HTTPRoute ng *ngfAPI.NginxGateway + btls *v1alpha2.BackendTLSPolicy addr = v1.GatewayStatusAddress{ Type: helpers.GetPointer(v1.IPAddressType), Value: "1.2.3.4", @@ -118,6 +123,17 @@ var _ = Describe("Updater", func() { }, }, }, + BackendTLSPolicyStatuses: status.BackendTLSPolicyStatuses{ + {Namespace: "test", Name: "backend-tls-policy"}: { + ObservedGeneration: gens.backendTLSPolicies, + AncestorStatuses: []status.AncestorStatus{ + { + GatewayNsName: types.NamespacedName{Namespace: "test", Name: "gateway"}, + Conditions: status.CreateTestConditions("Test"), + }, + }, + }, + }, } } @@ -233,6 +249,31 @@ var _ = Describe("Updater", func() { } } + createExpectedBtlsWithGeneration = func(gen int64) *v1alpha2.BackendTLSPolicy { + return &v1alpha2.BackendTLSPolicy{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "test", + Name: "backend-tls-policy", + }, + TypeMeta: metav1.TypeMeta{ + Kind: "BackendTLSPolicy", + APIVersion: "gateway.networking.k8s.io/v1alpha2", + }, + Status: v1alpha2.PolicyStatus{ + Ancestors: []v1alpha2.PolicyAncestorStatus{ + { + AncestorRef: v1.ParentReference{ + Namespace: (*v1.Namespace)(helpers.GetPointer("test")), + Name: "gateway", + }, + ControllerName: v1alpha2.GatewayController(gatewayCtrlName), + Conditions: status.CreateExpectedAPIConditions("Test", gen, fakeClockTime), + }, + }, + }, + } + } + createExpectedNGWithGeneration = func(gen int64) *ngfAPI.NginxGateway { return &ngfAPI.NginxGateway{ ObjectMeta: metav1.ObjectMeta{ @@ -299,6 +340,16 @@ var _ = Describe("Updater", func() { APIVersion: "gateway.networking.k8s.io/v1", }, } + btls = &v1alpha2.BackendTLSPolicy{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "test", + Name: "backend-tls-policy", + }, + TypeMeta: metav1.TypeMeta{ + Kind: "BackendTLSPolicy", + APIVersion: "gateway.networking.k8s.io/v1alpha2", + }, + } ng = &ngfAPI.NginxGateway{ ObjectMeta: metav1.ObjectMeta{ Namespace: "nginx-gateway", @@ -317,12 +368,14 @@ var _ = Describe("Updater", func() { Expect(client.Create(context.Background(), ignoredGw)).Should(Succeed()) Expect(client.Create(context.Background(), hr)).Should(Succeed()) Expect(client.Create(context.Background(), ng)).Should(Succeed()) + Expect(client.Create(context.Background(), btls)).Should(Succeed()) }) It("should update gateway API statuses", func() { updater.Update(context.Background(), createGwAPIStatuses(generations{ - gatewayClass: 1, - gateways: 1, + gatewayClass: 1, + gateways: 1, + backendTLSPolicies: 1, })) }) @@ -378,6 +431,22 @@ var _ = Describe("Updater", func() { Expect(helpers.Diff(expectedHR, latestHR)).To(BeEmpty()) }) + It("should have the updated status of BackendTLSPolicy in the API server", func() { + latestBtls := &v1alpha2.BackendTLSPolicy{} + expectedBtls := createExpectedBtlsWithGeneration(1) + + err := client.Get( + context.Background(), + types.NamespacedName{Namespace: "test", Name: "backend-tls-policy"}, + latestBtls, + ) + Expect(err).ToNot(HaveOccurred()) + + expectedBtls.ResourceVersion = latestBtls.ResourceVersion + + Expect(helpers.Diff(expectedBtls, latestBtls)).To(BeEmpty()) + }) + It("should update nginx gateway status", func() { updater.Update(context.Background(), createNGStatus(1)) }) diff --git a/internal/mode/static/build_statuses.go b/internal/mode/static/build_statuses.go index 9527bf145b..6055d9fdd1 100644 --- a/internal/mode/static/build_statuses.go +++ b/internal/mode/static/build_statuses.go @@ -29,6 +29,8 @@ func buildGatewayAPIStatuses( statuses.GatewayStatuses = buildGatewayStatuses(graph.Gateway, graph.IgnoredGateways, gwAddresses, nginxReloadRes) + statuses.BackendTLSPolicyStatuses = buildBackendTLSPolicyStatuses(graph.BackendTLSPolicies) + for nsname, r := range graph.Routes { parentStatuses := make([]status.ParentStatus, 0, len(r.ParentRefs)) @@ -190,3 +192,24 @@ func buildGatewayStatus( ObservedGeneration: gateway.Source.Generation, } } + +func buildBackendTLSPolicyStatuses(backendTLSPolicies map[types.NamespacedName]*graph.BackendTLSPolicy, +) status.BackendTLSPolicyStatuses { + statuses := make(status.BackendTLSPolicyStatuses, len(backendTLSPolicies)) + + for nsname, backendTLSPolicy := range backendTLSPolicies { + if backendTLSPolicy.IsReferenced { + if !backendTLSPolicy.Ignored { + statuses[nsname] = status.BackendTLSPolicyStatus{ + AncestorStatuses: []status.AncestorStatus{ + { + GatewayNsName: backendTLSPolicy.Gateway, + Conditions: conditions.DeduplicateConditions(backendTLSPolicy.Conditions), + }, + }, + } + } + } + } + return statuses +} diff --git a/internal/mode/static/build_statuses_test.go b/internal/mode/static/build_statuses_test.go index c3ee37cfbc..146b34409b 100644 --- a/internal/mode/static/build_statuses_test.go +++ b/internal/mode/static/build_statuses_test.go @@ -9,6 +9,7 @@ import ( "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" v1 "sigs.k8s.io/gateway-api/apis/v1" + v1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" "github.com/nginxinc/nginx-gateway-fabric/internal/framework/conditions" "github.com/nginxinc/nginx-gateway-fabric/internal/framework/helpers" @@ -202,6 +203,7 @@ func TestBuildStatuses(t *testing.T) { }, }, }, + BackendTLSPolicyStatuses: status.BackendTLSPolicyStatuses{}, } g := NewWithT(t) @@ -304,6 +306,7 @@ func TestBuildStatusesNginxErr(t *testing.T) { }, }, }, + BackendTLSPolicyStatuses: status.BackendTLSPolicyStatuses{}, } g := NewWithT(t) @@ -611,3 +614,133 @@ func TestBuildGatewayStatuses(t *testing.T) { }) } } + +func TestBuildBackendTLSPolicyStatuses(t *testing.T) { + type policyCfg struct { + Name string + Conditions []conditions.Condition + Valid bool + Ignored bool + IsReferenced bool + } + + getBackendTLSPolicy := func(policyCfg policyCfg) *graph.BackendTLSPolicy { + return &graph.BackendTLSPolicy{ + Source: &v1alpha2.BackendTLSPolicy{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "test", + Name: policyCfg.Name, + Generation: 1, + }, + }, + Valid: policyCfg.Valid, + Ignored: policyCfg.Ignored, + IsReferenced: policyCfg.IsReferenced, + Conditions: policyCfg.Conditions, + Gateway: types.NamespacedName{Name: "gateway", Namespace: "test"}, + } + } + + attachedConds := []conditions.Condition{staticConds.NewBackendTLSPolicyAccepted()} + invalidConds := []conditions.Condition{staticConds.NewBackendTLSPolicyInvalid("invalid backendTLSPolicy")} + + validPolicyCfg := policyCfg{ + Name: "valid-bt", + Valid: true, + IsReferenced: true, + Conditions: attachedConds, + } + + invalidPolicyCfg := policyCfg{ + Name: "invalid-bt", + IsReferenced: true, + Conditions: invalidConds, + } + + ignoredPolicyCfg := policyCfg{ + Name: "ignored-bt", + Ignored: true, + IsReferenced: true, + } + + notReferencedPolicyCfg := policyCfg{ + Name: "not-referenced", + Valid: true, + } + + tests := []struct { + backendTLSPolicies map[types.NamespacedName]*graph.BackendTLSPolicy + expected status.BackendTLSPolicyStatuses + name string + }{ + { + name: "nil backendTLSPolicies", + expected: status.BackendTLSPolicyStatuses{}, + }, + { + name: "valid backendTLSPolicy", + backendTLSPolicies: map[types.NamespacedName]*graph.BackendTLSPolicy{ + {Namespace: "test", Name: "valid-bt"}: getBackendTLSPolicy(validPolicyCfg), + }, + expected: status.BackendTLSPolicyStatuses{ + {Namespace: "test", Name: "valid-bt"}: { + AncestorStatuses: []status.AncestorStatus{ + { + Conditions: attachedConds, + GatewayNsName: types.NamespacedName{Name: "gateway", Namespace: "test"}, + }, + }, + }, + }, + }, + { + name: "invalid backendTLSPolicy", + backendTLSPolicies: map[types.NamespacedName]*graph.BackendTLSPolicy{ + {Namespace: "test", Name: "invalid-bt"}: getBackendTLSPolicy(invalidPolicyCfg), + }, + expected: status.BackendTLSPolicyStatuses{ + {Namespace: "test", Name: "invalid-bt"}: { + AncestorStatuses: []status.AncestorStatus{ + { + Conditions: invalidConds, + GatewayNsName: types.NamespacedName{Name: "gateway", Namespace: "test"}, + }, + }, + }, + }, + }, + { + name: "ignored or not referenced backendTLSPolicies", + backendTLSPolicies: map[types.NamespacedName]*graph.BackendTLSPolicy{ + {Namespace: "test", Name: "ignored-bt"}: getBackendTLSPolicy(ignoredPolicyCfg), + {Namespace: "test", Name: "not-referenced"}: getBackendTLSPolicy(notReferencedPolicyCfg), + }, + expected: status.BackendTLSPolicyStatuses{}, + }, + { + name: "mix valid and ignored backendTLSPolicies", + backendTLSPolicies: map[types.NamespacedName]*graph.BackendTLSPolicy{ + {Namespace: "test", Name: "ignored-bt"}: getBackendTLSPolicy(ignoredPolicyCfg), + {Namespace: "test", Name: "valid-bt"}: getBackendTLSPolicy(validPolicyCfg), + }, + expected: status.BackendTLSPolicyStatuses{ + {Namespace: "test", Name: "valid-bt"}: { + AncestorStatuses: []status.AncestorStatus{ + { + Conditions: attachedConds, + GatewayNsName: types.NamespacedName{Name: "gateway", Namespace: "test"}, + }, + }, + }, + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + g := NewWithT(t) + + result := buildBackendTLSPolicyStatuses(test.backendTLSPolicies) + g.Expect(helpers.Diff(test.expected, result)).To(BeEmpty()) + }) + } +} diff --git a/internal/mode/static/config/config.go b/internal/mode/static/config/config.go index cb1ee1efb2..95c2f1377d 100644 --- a/internal/mode/static/config/config.go +++ b/internal/mode/static/config/config.go @@ -28,16 +28,18 @@ type Config struct { GatewayClassName string // LeaderElection contains the configuration for leader election. LeaderElection LeaderElection - // UpdateGatewayClassStatus enables updating the status of the GatewayClass resource. - UpdateGatewayClassStatus bool - // Plus indicates whether NGINX Plus is being used. - Plus bool // MetricsConfig specifies the metrics config. MetricsConfig MetricsConfig // HealthConfig specifies the health probe config. HealthConfig HealthConfig // TelemetryReportPeriod is the period at which telemetry reports are sent. TelemetryReportPeriod time.Duration + // UpdateGatewayClassStatus enables updating the status of the GatewayClass resource. + UpdateGatewayClassStatus bool + // Plus indicates whether NGINX Plus is being used. + Plus bool + // ExperimentalFeatures indicates if experimental features are enabled. + ExperimentalFeatures bool } // GatewayPodConfig contains information about this Pod. diff --git a/internal/mode/static/manager.go b/internal/mode/static/manager.go index 5967190cb3..6d63d61626 100644 --- a/internal/mode/static/manager.go +++ b/internal/mode/static/manager.go @@ -28,6 +28,7 @@ import ( metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" k8spredicate "sigs.k8s.io/controller-runtime/pkg/predicate" gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" + gatewayv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" gatewayv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" ngfAPI "github.com/nginxinc/nginx-gateway-fabric/apis/v1alpha1" @@ -62,6 +63,7 @@ var scheme = runtime.NewScheme() func init() { utilruntime.Must(gatewayv1beta1.AddToScheme(scheme)) utilruntime.Must(gatewayv1.AddToScheme(scheme)) + utilruntime.Must(gatewayv1alpha2.AddToScheme(scheme)) utilruntime.Must(apiv1.AddToScheme(scheme)) utilruntime.Must(discoveryV1.AddToScheme(scheme)) utilruntime.Must(ngfAPI.AddToScheme(scheme)) @@ -198,7 +200,11 @@ func StartManager(cfg config.Config) error { metricsCollector: handlerCollector, }) - objects, objectLists := prepareFirstEventBatchPreparerArgs(cfg.GatewayClassName, cfg.GatewayNsName) + objects, objectLists := prepareFirstEventBatchPreparerArgs( + cfg.GatewayClassName, + cfg.GatewayNsName, + cfg.ExperimentalFeatures, + ) firstBatchPreparer := events.NewFirstEventBatchPreparerImpl(mgr.GetCache(), objects, objectLists) eventLoop := events.NewEventLoop( eventCh, @@ -378,6 +384,23 @@ func registerControllers( }, } + if cfg.ExperimentalFeatures { + backendTLSObjs := []ctlrCfg{ + { + objectType: &gatewayv1alpha2.BackendTLSPolicy{}, + options: []controller.Option{ + controller.WithK8sPredicate(k8spredicate.GenerationChangedPredicate{}), + }, + }, + { + // FIXME(ciarams87): If possible, use only metadata predicate + // https://github.com/nginxinc/nginx-gateway-fabric/issues/1545 + objectType: &apiv1.ConfigMap{}, + }, + } + controllerRegCfgs = append(controllerRegCfgs, backendTLSObjs...) + } + if cfg.ConfigName != "" { controllerRegCfgs = append(controllerRegCfgs, ctlrCfg{ @@ -441,6 +464,7 @@ func createTelemetryJob( func prepareFirstEventBatchPreparerArgs( gcName string, gwNsName *types.NamespacedName, + enableExperimentalFeatures bool, ) ([]client.Object, []client.ObjectList) { objects := []client.Object{ &gatewayv1.GatewayClass{ObjectMeta: metav1.ObjectMeta{Name: gcName}}, @@ -465,6 +489,10 @@ func prepareFirstEventBatchPreparerArgs( partialObjectMetadataList, } + if enableExperimentalFeatures { + objectLists = append(objectLists, &gatewayv1alpha2.BackendTLSPolicyList{}, &apiv1.ConfigMapList{}) + } + if gwNsName == nil { objectLists = append(objectLists, &gatewayv1.GatewayList{}) } else { diff --git a/internal/mode/static/manager_test.go b/internal/mode/static/manager_test.go index 3191a2e01d..532807d374 100644 --- a/internal/mode/static/manager_test.go +++ b/internal/mode/static/manager_test.go @@ -13,6 +13,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" + gatewayv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" gatewayv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/config" @@ -35,6 +36,7 @@ func TestPrepareFirstEventBatchPreparerArgs(t *testing.T) { gwNsName *types.NamespacedName expectedObjects []client.Object expectedObjectLists []client.ObjectList + experimentalEnabled bool }{ { name: "gwNsName is nil", @@ -73,13 +75,36 @@ func TestPrepareFirstEventBatchPreparerArgs(t *testing.T) { partialObjectMetadataList, }, }, + { + name: "gwNsName is not nil and experimental enabled", + gwNsName: &types.NamespacedName{ + Namespace: "test", + Name: "my-gateway", + }, + expectedObjects: []client.Object{ + &gatewayv1.GatewayClass{ObjectMeta: metav1.ObjectMeta{Name: "nginx"}}, + &gatewayv1.Gateway{ObjectMeta: metav1.ObjectMeta{Name: "my-gateway", Namespace: "test"}}, + }, + expectedObjectLists: []client.ObjectList{ + &apiv1.ServiceList{}, + &apiv1.SecretList{}, + &apiv1.NamespaceList{}, + &apiv1.ConfigMapList{}, + &discoveryV1.EndpointSliceList{}, + &gatewayv1.HTTPRouteList{}, + &gatewayv1beta1.ReferenceGrantList{}, + partialObjectMetadataList, + &gatewayv1alpha2.BackendTLSPolicyList{}, + }, + experimentalEnabled: true, + }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { g := NewWithT(t) - objects, objectLists := prepareFirstEventBatchPreparerArgs(gcName, test.gwNsName) + objects, objectLists := prepareFirstEventBatchPreparerArgs(gcName, test.gwNsName, test.experimentalEnabled) g.Expect(objects).To(ConsistOf(test.expectedObjects)) g.Expect(objectLists).To(ConsistOf(test.expectedObjectLists)) diff --git a/internal/mode/static/nginx/config/generator.go b/internal/mode/static/nginx/config/generator.go index ecbb5cece2..4917a80cac 100644 --- a/internal/mode/static/nginx/config/generator.go +++ b/internal/mode/static/nginx/config/generator.go @@ -70,6 +70,10 @@ func (g GeneratorImpl) Generate(conf dataplane.Configuration) []file.File { files = append(files, generateConfigVersion(conf.Version)) + for id, bundle := range conf.CertBundles { + files = append(files, generateCertBundle(id, bundle)) + } + return files } @@ -90,6 +94,18 @@ func generatePEMFileName(id dataplane.SSLKeyPairID) string { return filepath.Join(secretsFolder, string(id)+".pem") } +func generateCertBundle(id dataplane.CertBundleID, cert []byte) file.File { + return file.File{ + Content: cert, + Path: generateCertBundleFileName(id), + Type: file.TypeRegular, + } +} + +func generateCertBundleFileName(id dataplane.CertBundleID) string { + return filepath.Join(secretsFolder, string(id)+".crt") +} + func (g GeneratorImpl) generateHTTPConfig(conf dataplane.Configuration) file.File { var c []byte for _, execute := range g.getExecuteFuncs() { diff --git a/internal/mode/static/nginx/config/generator_test.go b/internal/mode/static/nginx/config/generator_test.go index c42d44cb49..30fd4befc8 100644 --- a/internal/mode/static/nginx/config/generator_test.go +++ b/internal/mode/static/nginx/config/generator_test.go @@ -59,6 +59,9 @@ func TestGenerate(t *testing.T) { Key: []byte("test-key"), }, }, + CertBundles: map[dataplane.CertBundleID]dataplane.CertBundle{ + "test-certbundle": []byte("test-cert"), + }, } g := NewWithT(t) @@ -67,7 +70,7 @@ func TestGenerate(t *testing.T) { files := generator.Generate(conf) - g.Expect(files).To(HaveLen(3)) + g.Expect(files).To(HaveLen(4)) g.Expect(files[0]).To(Equal(file.File{ Type: file.TypeSecret, @@ -89,4 +92,8 @@ func TestGenerate(t *testing.T) { g.Expect(files[2].Path).To(Equal("/etc/nginx/conf.d/config-version.conf")) configVersion := string(files[2].Content) g.Expect(configVersion).To(ContainSubstring(fmt.Sprintf("return 200 %d", conf.Version))) + + g.Expect(files[3].Path).To(Equal("/etc/nginx/secrets/test-certbundle.crt")) + certBundle := string(files[3].Content) + g.Expect(certBundle).To(Equal("test-cert")) } diff --git a/internal/mode/static/nginx/config/http/config.go b/internal/mode/static/nginx/config/http/config.go index ae0d53546e..8486b7d464 100644 --- a/internal/mode/static/nginx/config/http/config.go +++ b/internal/mode/static/nginx/config/http/config.go @@ -13,10 +13,11 @@ type Server struct { // Location holds all configuration for an HTTP location. type Location struct { Return *Return - Rewrites []string + ProxySSLVerify *ProxySSLVerify Path string ProxyPass string HTTPMatchVar string + Rewrites []string ProxySetHeaders []Header } @@ -86,3 +87,9 @@ type MapParameter struct { Value string Result string } + +// ProxySSLVerify holds the proxied HTTPS server verification configuration. +type ProxySSLVerify struct { + TrustedCertificate string + Name string +} diff --git a/internal/mode/static/nginx/config/servers.go b/internal/mode/static/nginx/config/servers.go index b4cdeb4779..a80de123f7 100644 --- a/internal/mode/static/nginx/config/servers.go +++ b/internal/mode/static/nginx/config/servers.go @@ -247,7 +247,6 @@ func updateLocationsForFilters( rewrites := createRewritesValForRewriteFilter(filters.RequestURLRewrite, path) proxySetHeaders := generateProxySetHeaders(&matchRule.Filters) - proxyPass := createProxyPass(matchRule.BackendGroup, matchRule.Filters.RequestURLRewrite) for i := range buildLocations { if rewrites != nil { if rewrites.Rewrite != "" { @@ -255,12 +254,57 @@ func updateLocationsForFilters( } } buildLocations[i].ProxySetHeaders = proxySetHeaders + buildLocations[i].ProxySSLVerify = createProxyTLSFromBackends(matchRule.BackendGroup.Backends) + proxyPass := createProxyPass( + matchRule.BackendGroup, + matchRule.Filters.RequestURLRewrite, + generateProtocolString(buildLocations[i].ProxySSLVerify), + ) buildLocations[i].ProxyPass = proxyPass } return buildLocations } +func generateProtocolString(ssl *http.ProxySSLVerify) string { + if ssl != nil { + return "https" + } + return "http" +} + +func createProxyTLSFromBackends(backends []dataplane.Backend) *http.ProxySSLVerify { + if len(backends) == 0 { + return nil + } + for _, b := range backends { + proxyVerify := createProxySSLVerify(b.VerifyTLS) + if proxyVerify != nil { + // If any backend has a backend TLS policy defined, then we use that for the proxy SSL verification. + // We require that all backends in a group have the same backend TLS policy. + // Verification that all backends in a group have the same backend TLS policy is done in the graph package. + return proxyVerify + } + } + return nil +} + +func createProxySSLVerify(v *dataplane.VerifyTLS) *http.ProxySSLVerify { + if v == nil { + return nil + } + var trustedCert string + if v.CertBundleID != "" { + trustedCert = generateCertBundleFileName(v.CertBundleID) + } else { + trustedCert = v.RootCAPath + } + return &http.ProxySSLVerify{ + TrustedCertificate: trustedCert, + Name: v.Hostname, + } +} + func createReturnValForRedirectFilter(filter *dataplane.HTTPRequestRedirectFilter, listenerPort int32) *http.Return { if filter == nil { return nil @@ -427,7 +471,11 @@ func isPathOnlyMatch(match dataplane.Match) bool { return match.Method == nil && len(match.Headers) == 0 && len(match.QueryParams) == 0 } -func createProxyPass(backendGroup dataplane.BackendGroup, filter *dataplane.HTTPURLRewriteFilter) string { +func createProxyPass( + backendGroup dataplane.BackendGroup, + filter *dataplane.HTTPURLRewriteFilter, + protocol string, +) string { var requestURI string if filter == nil || filter.Path == nil { requestURI = "$request_uri" @@ -435,10 +483,10 @@ func createProxyPass(backendGroup dataplane.BackendGroup, filter *dataplane.HTTP backendName := backendGroupName(backendGroup) if backendGroupNeedsSplit(backendGroup) { - return "http://$" + convertStringToSafeVariableName(backendName) + requestURI + return protocol + "://$" + convertStringToSafeVariableName(backendName) + requestURI } - return "http://" + backendName + requestURI + return protocol + "://" + backendName + requestURI } func createMatchLocation(path string) http.Location { diff --git a/internal/mode/static/nginx/config/servers_template.go b/internal/mode/static/nginx/config/servers_template.go index a56c55820e..dbf37575ae 100644 --- a/internal/mode/static/nginx/config/servers_template.go +++ b/internal/mode/static/nginx/config/servers_template.go @@ -52,6 +52,11 @@ server { {{- end }} proxy_http_version 1.1; proxy_pass {{ $l.ProxyPass }}; + {{- if $l.ProxySSLVerify }} + proxy_ssl_verify on; + proxy_ssl_name {{ $l.ProxySSLVerify.Name }}; + proxy_ssl_trusted_certificate {{ $l.ProxySSLVerify.TrustedCertificate }}; + {{- end }} {{- end }} } {{ end }} diff --git a/internal/mode/static/nginx/config/servers_test.go b/internal/mode/static/nginx/config/servers_test.go index 2040bcfef1..218e59d9d3 100644 --- a/internal/mode/static/nginx/config/servers_test.go +++ b/internal/mode/static/nginx/config/servers_test.go @@ -203,6 +203,22 @@ func TestCreateServers(t *testing.T) { }, } + btpGroup := dataplane.BackendGroup{ + Source: hrNsName, + RuleIdx: 3, + Backends: []dataplane.Backend{ + { + UpstreamName: "test_btp_80", + Valid: true, + Weight: 1, + VerifyTLS: &dataplane.VerifyTLS{ + CertBundleID: "test-btp", + Hostname: "test-btp.example.com", + }, + }, + }, + } + filterGroup1 := dataplane.BackendGroup{Source: hrNsName, RuleIdx: 3} filterGroup2 := dataplane.BackendGroup{Source: hrNsName, RuleIdx: 4} @@ -281,6 +297,16 @@ func TestCreateServers(t *testing.T) { }, }, }, + { + Path: "/backend-tls-policy", + PathType: dataplane.PathTypePrefix, + MatchRules: []dataplane.MatchRule{ + { + Match: dataplane.Match{}, + BackendGroup: btpGroup, + }, + }, + }, { Path: "/redirect-implicit-port", PathType: dataplane.PathTypePrefix, @@ -505,19 +531,19 @@ func TestCreateServers(t *testing.T) { exactMatches := []httpMatch{ { Method: "GET", - RedirectPath: "@rule11-route0", + RedirectPath: "@rule12-route0", }, } redirectHeaderMatches := []httpMatch{ { Headers: []string{"redirect:this"}, - RedirectPath: "@rule5-route0", + RedirectPath: "@rule6-route0", }, } rewriteHeaderMatches := []httpMatch{ { Headers: []string{"rewrite:this"}, - RedirectPath: "@rule7-route0", + RedirectPath: "@rule8-route0", }, } rewriteProxySetHeaders := []http.Header{ @@ -541,7 +567,7 @@ func TestCreateServers(t *testing.T) { invalidFilterHeaderMatches := []httpMatch{ { Headers: []string{"filter:this"}, - RedirectPath: "@rule9-route0", + RedirectPath: "@rule10-route0", }, } @@ -590,6 +616,24 @@ func TestCreateServers(t *testing.T) { ProxyPass: "http://invalid-backend-ref$request_uri", ProxySetHeaders: baseHeaders, }, + { + Path: "/backend-tls-policy/", + ProxyPass: "https://test_btp_80$request_uri", + ProxySetHeaders: baseHeaders, + ProxySSLVerify: &http.ProxySSLVerify{ + Name: "test-btp.example.com", + TrustedCertificate: "/etc/nginx/secrets/test-btp.crt", + }, + }, + { + Path: "= /backend-tls-policy", + ProxyPass: "https://test_btp_80$request_uri", + ProxySetHeaders: baseHeaders, + ProxySSLVerify: &http.ProxySSLVerify{ + Name: "test-btp.example.com", + TrustedCertificate: "/etc/nginx/secrets/test-btp.crt", + }, + }, { Path: "/redirect-implicit-port/", Return: &http.Return{ @@ -619,7 +663,7 @@ func TestCreateServers(t *testing.T) { }, }, { - Path: "@rule5-route0", + Path: "@rule6-route0", Return: &http.Return{ Body: "$scheme://foo.example.com:8080$request_uri", Code: 302, @@ -646,7 +690,7 @@ func TestCreateServers(t *testing.T) { ProxySetHeaders: rewriteProxySetHeaders, }, { - Path: "@rule7-route0", + Path: "@rule8-route0", Rewrites: []string{"^/rewrite-with-headers(.*)$ /prefix-replacement$1 break"}, ProxyPass: "http://test_foo_80", ProxySetHeaders: rewriteProxySetHeaders, @@ -672,7 +716,7 @@ func TestCreateServers(t *testing.T) { }, }, { - Path: "@rule9-route0", + Path: "@rule10-route0", Return: &http.Return{ Code: http.StatusInternalServerError, }, @@ -691,7 +735,7 @@ func TestCreateServers(t *testing.T) { ProxySetHeaders: baseHeaders, }, { - Path: "@rule11-route0", + Path: "@rule12-route0", ProxyPass: "http://test_foo_80$request_uri", ProxySetHeaders: baseHeaders, }, @@ -1670,7 +1714,7 @@ func TestCreateProxyPass(t *testing.T) { } for _, tc := range tests { - result := createProxyPass(tc.grp, tc.rewrite) + result := createProxyPass(tc.grp, tc.rewrite, generateProtocolString(nil)) g.Expect(result).To(Equal(tc.expected)) } } @@ -1791,3 +1835,92 @@ func TestGenerateProxySetHeaders(t *testing.T) { }) } } + +func TestConvertBackendTLSFromGroup(t *testing.T) { + g := NewWithT(t) + + tests := []struct { + expected *http.ProxySSLVerify + msg string + grp []dataplane.Backend + }{ + { + msg: "tls enabled, one backend", + grp: []dataplane.Backend{ + { + UpstreamName: "my-upstream", + Valid: true, + Weight: 1, + VerifyTLS: &dataplane.VerifyTLS{ + CertBundleID: "default-my-cert", + Hostname: "my-hostname", + }, + }, + }, + expected: &http.ProxySSLVerify{ + TrustedCertificate: "/etc/nginx/secrets/default-my-cert.crt", + Name: "my-hostname", + }, + }, + { + msg: "tls disabled", + grp: []dataplane.Backend{ + { + UpstreamName: "my-upstream", + Valid: true, + Weight: 1, + VerifyTLS: nil, + }, + }, + expected: nil, + }, + { + msg: "tls enabled, multiple backends", + grp: []dataplane.Backend{ + { + UpstreamName: "my-upstream", + Valid: true, + Weight: 1, + VerifyTLS: &dataplane.VerifyTLS{ + CertBundleID: "default-my-cert", + Hostname: "my-hostname", + }, + }, + { + UpstreamName: "my-upstream", + Valid: true, + Weight: 2, + }, + }, + expected: &http.ProxySSLVerify{ + TrustedCertificate: "/etc/nginx/secrets/default-my-cert.crt", + Name: "my-hostname", + }, + }, + { + msg: "tls enabled, system certs enabled", + grp: []dataplane.Backend{ + { + UpstreamName: "my-upstream", + Valid: true, + Weight: 1, + VerifyTLS: &dataplane.VerifyTLS{ + Hostname: "my-hostname", + RootCAPath: "/etc/ssl/certs/ca-certificates.crt", + }, + }, + }, + expected: &http.ProxySSLVerify{ + TrustedCertificate: "/etc/ssl/certs/ca-certificates.crt", + Name: "my-hostname", + }, + }, + } + + for _, tc := range tests { + t.Run(tc.msg, func(_ *testing.T) { + result := createProxyTLSFromBackends(tc.grp) + g.Expect(result).To(Equal(tc.expected)) + }) + } +} diff --git a/internal/mode/static/state/change_processor.go b/internal/mode/static/state/change_processor.go index 385ae0e938..210238f0fe 100644 --- a/internal/mode/static/state/change_processor.go +++ b/internal/mode/static/state/change_processor.go @@ -17,6 +17,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/apiutil" v1 "sigs.k8s.io/gateway-api/apis/v1" gwapivalidation "sigs.k8s.io/gateway-api/apis/v1/validation" + "sigs.k8s.io/gateway-api/apis/v1alpha2" "sigs.k8s.io/gateway-api/apis/v1beta1" "github.com/nginxinc/nginx-gateway-fabric/internal/framework/gatewayclass" @@ -102,14 +103,16 @@ type ChangeProcessorImpl struct { // NewChangeProcessorImpl creates a new ChangeProcessorImpl for the Gateway resource with the configured namespace name. func NewChangeProcessorImpl(cfg ChangeProcessorConfig) *ChangeProcessorImpl { clusterStore := graph.ClusterState{ - GatewayClasses: make(map[types.NamespacedName]*v1.GatewayClass), - Gateways: make(map[types.NamespacedName]*v1.Gateway), - HTTPRoutes: make(map[types.NamespacedName]*v1.HTTPRoute), - Services: make(map[types.NamespacedName]*apiv1.Service), - Namespaces: make(map[types.NamespacedName]*apiv1.Namespace), - ReferenceGrants: make(map[types.NamespacedName]*v1beta1.ReferenceGrant), - Secrets: make(map[types.NamespacedName]*apiv1.Secret), - CRDMetadata: make(map[types.NamespacedName]*metav1.PartialObjectMetadata), + GatewayClasses: make(map[types.NamespacedName]*v1.GatewayClass), + Gateways: make(map[types.NamespacedName]*v1.Gateway), + HTTPRoutes: make(map[types.NamespacedName]*v1.HTTPRoute), + Services: make(map[types.NamespacedName]*apiv1.Service), + Namespaces: make(map[types.NamespacedName]*apiv1.Namespace), + ReferenceGrants: make(map[types.NamespacedName]*v1beta1.ReferenceGrant), + Secrets: make(map[types.NamespacedName]*apiv1.Secret), + CRDMetadata: make(map[types.NamespacedName]*metav1.PartialObjectMetadata), + BackendTLSPolicies: make(map[types.NamespacedName]*v1alpha2.BackendTLSPolicy), + ConfigMaps: make(map[types.NamespacedName]*apiv1.ConfigMap), } extractGVK := func(obj client.Object) schema.GroupVersionKind { @@ -152,6 +155,11 @@ func NewChangeProcessorImpl(cfg ChangeProcessorConfig) *ChangeProcessorImpl { store: newObjectStoreMapAdapter(clusterStore.ReferenceGrants), predicate: nil, }, + { + gvk: extractGVK(&v1alpha2.BackendTLSPolicy{}), + store: newObjectStoreMapAdapter(clusterStore.BackendTLSPolicies), + predicate: nil, + }, { gvk: extractGVK(&apiv1.Namespace{}), store: newObjectStoreMapAdapter(clusterStore.Namespaces), @@ -172,6 +180,11 @@ func NewChangeProcessorImpl(cfg ChangeProcessorConfig) *ChangeProcessorImpl { store: newObjectStoreMapAdapter(clusterStore.Secrets), predicate: funcPredicate{stateChanged: isReferenced}, }, + { + gvk: extractGVK(&apiv1.ConfigMap{}), + store: newObjectStoreMapAdapter(clusterStore.ConfigMaps), + predicate: funcPredicate{stateChanged: isReferenced}, + }, { gvk: extractGVK(&apiext.CustomResourceDefinition{}), store: newObjectStoreMapAdapter(clusterStore.CRDMetadata), @@ -218,6 +231,9 @@ func NewChangeProcessorImpl(cfg ChangeProcessorConfig) *ChangeProcessorImpl { // belong to the NGINX Gateway Fabric or an HTTPRoute that doesn't belong to any of the Gateways of the // NGINX Gateway Fabric. Find a way to ignore changes that don't affect the configuration and/or statuses of // the resources. +// Tracking issues: https://github.com/nginxinc/nginx-gateway-fabric/issues/1123, +// https://github.com/nginxinc/nginx-gateway-fabric/issues/1124, +// https://github.com/nginxinc/nginx-gateway-fabric/issues/1577 // FIXME(pleshakov) // Remove CaptureUpsertChange() and CaptureDeleteChange() from ChangeProcessor and pass all changes directly to diff --git a/internal/mode/static/state/change_processor_test.go b/internal/mode/static/state/change_processor_test.go index dba175af51..790f42e49e 100644 --- a/internal/mode/static/state/change_processor_test.go +++ b/internal/mode/static/state/change_processor_test.go @@ -184,6 +184,7 @@ func createScheme() *runtime.Scheme { utilruntime.Must(v1.AddToScheme(scheme)) utilruntime.Must(v1beta1.AddToScheme(scheme)) + utilruntime.Must(v1alpha2.AddToScheme(scheme)) utilruntime.Must(apiv1.AddToScheme(scheme)) utilruntime.Must(discoveryV1.AddToScheme(scheme)) utilruntime.Must(apiext.AddToScheme(scheme)) @@ -573,11 +574,11 @@ var _ = Describe("ChangeProcessor", func() { } expGraph.Routes[hr1Name].ParentRefs[0].Attachment = &graph.ParentRefAttachmentStatus{ AcceptedHostnames: map[string][]string{}, - FailedCondition: staticConds.NewRouteInvalidGateway(), + FailedCondition: staticConds.NewRouteNoMatchingParent(), } expGraph.Routes[hr1Name].ParentRefs[1].Attachment = &graph.ParentRefAttachmentStatus{ AcceptedHostnames: map[string][]string{}, - FailedCondition: staticConds.NewRouteInvalidGateway(), + FailedCondition: staticConds.NewRouteNoMatchingParent(), } expGraph.ReferencedSecrets = nil @@ -1001,6 +1002,7 @@ var _ = Describe("ChangeProcessor", func() { hr1svc, sharedSvc, bazSvc1, bazSvc2, bazSvc3, invalidSvc, notRefSvc *apiv1.Service hr1slice1, hr1slice2, noRefSlice, missingSvcNameSlice *discoveryV1.EndpointSlice gw *v1.Gateway + btls *v1alpha2.BackendTLSPolicy ) createSvc := func(name string) *apiv1.Service { @@ -1022,6 +1024,24 @@ var _ = Describe("ChangeProcessor", func() { } } + createBackendTLSPolicy := func(name string, svcName string) *v1alpha2.BackendTLSPolicy { + return &v1alpha2.BackendTLSPolicy{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "test", + Name: name, + }, + Spec: v1alpha2.BackendTLSPolicySpec{ + TargetRef: v1alpha2.PolicyTargetReferenceWithSectionName{ + PolicyTargetReference: v1alpha2.PolicyTargetReference{ + Kind: v1.Kind("Service"), + Name: v1.ObjectName(svcName), + Namespace: helpers.GetPointer(v1.Namespace("test")), + }, + }, + }, + } + } + BeforeAll(func() { testNamespace := v1.Namespace("test") kindService := v1.Kind("Service") @@ -1067,6 +1087,9 @@ var _ = Describe("ChangeProcessor", func() { noRefSlice = createEndpointSlice("no-ref", "no-ref") missingSvcNameSlice = createEndpointSlice("missing-svc-name", "") + // backendTLSPolicy + btls = createBackendTLSPolicy("btls", "foo-svc") + gw = createGateway("gw") processor.CaptureUpsertChange(gc) processor.CaptureUpsertChange(gw) @@ -1099,6 +1122,11 @@ var _ = Describe("ChangeProcessor", func() { testUpsertTriggersChange(hr1svc, state.ClusterStateChange) }) }) + When("a backendTLSPolicy is added for referenced service", func() { + It("should trigger a change", func() { + testUpsertTriggersChange(btls, state.ClusterStateChange) + }) + }) When("an hr1 endpoint slice is added", func() { It("should trigger a change", func() { testUpsertTriggersChange(hr1slice1, state.EndpointsOnlyChange) @@ -1502,17 +1530,20 @@ var _ = Describe("ChangeProcessor", func() { // Note: in these tests, we deliberately don't fully inspect the returned configuration and statuses // -- this is done in 'Normal cases of processing changes' + //nolint:lll var ( - processor *state.ChangeProcessorImpl - gcNsName, gwNsName, hrNsName, hr2NsName, rgNsName, svcNsName, sliceNsName, secretNsName types.NamespacedName - gc, gcUpdated *v1.GatewayClass - gw1, gw1Updated, gw2 *v1.Gateway - hr1, hr1Updated, hr2 *v1.HTTPRoute - rg1, rg1Updated, rg2 *v1beta1.ReferenceGrant - svc, barSvc, unrelatedSvc *apiv1.Service - slice, barSlice, unrelatedSlice *discoveryV1.EndpointSlice - ns, unrelatedNS, testNs, barNs *apiv1.Namespace - secret, secretUpdated, unrelatedSecret, barSecret, barSecretUpdated *apiv1.Secret + processor *state.ChangeProcessorImpl + gcNsName, gwNsName, hrNsName, hr2NsName, rgNsName, svcNsName, sliceNsName, secretNsName, cmNsName, btlsNsName types.NamespacedName + gc, gcUpdated *v1.GatewayClass + gw1, gw1Updated, gw2 *v1.Gateway + hr1, hr1Updated, hr2 *v1.HTTPRoute + rg1, rg1Updated, rg2 *v1beta1.ReferenceGrant + svc, barSvc, unrelatedSvc *apiv1.Service + slice, barSlice, unrelatedSlice *discoveryV1.EndpointSlice + ns, unrelatedNS, testNs, barNs *apiv1.Namespace + secret, secretUpdated, unrelatedSecret, barSecret, barSecretUpdated *apiv1.Secret + cm, cmUpdated, unrelatedCM *apiv1.ConfigMap + btls, btlsUpdated *v1alpha2.BackendTLSPolicy ) BeforeEach(OncePerOrdered, func() { @@ -1755,6 +1786,55 @@ var _ = Describe("ChangeProcessor", func() { rg2 = rg1.DeepCopy() rg2.Name = "rg-2" + + cmNsName = types.NamespacedName{Namespace: "test", Name: "cm-1"} + cm = &apiv1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: cmNsName.Name, + Namespace: cmNsName.Namespace, + }, + Data: map[string]string{ + "ca.crt": "value", + }, + } + cmUpdated = cm.DeepCopy() + cmUpdated.Data["ca.crt"] = "updated-value" + + unrelatedCM = &apiv1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "unrelated-cm", + Namespace: "unrelated-ns", + }, + Data: map[string]string{ + "ca.crt": "value", + }, + } + + btlsNsName = types.NamespacedName{Namespace: "test", Name: "btls-1"} + btls = &v1alpha2.BackendTLSPolicy{ + ObjectMeta: metav1.ObjectMeta{ + Name: btlsNsName.Name, + Namespace: btlsNsName.Namespace, + Generation: 1, + }, + Spec: v1alpha2.BackendTLSPolicySpec{ + TargetRef: v1alpha2.PolicyTargetReferenceWithSectionName{ + PolicyTargetReference: v1alpha2.PolicyTargetReference{ + Kind: "Service", + Name: v1.ObjectName(svc.Name), + Namespace: helpers.GetPointer(v1.Namespace(svc.Namespace)), + }, + }, + TLS: v1alpha2.BackendTLSPolicyConfig{ + CACertRefs: []v1.LocalObjectReference{ + { + Name: v1.ObjectName(cm.Name), + }, + }, + }, + }, + } + btlsUpdated = btls.DeepCopy() }) // Changing change - a change that makes processor.Process() report changed // Non-changing change - a change that doesn't do that @@ -1770,6 +1850,8 @@ var _ = Describe("ChangeProcessor", func() { processor.CaptureUpsertChange(testNs) processor.CaptureUpsertChange(hr1) processor.CaptureUpsertChange(rg1) + processor.CaptureUpsertChange(btls) + processor.CaptureUpsertChange(cm) changed, _ := processor.Process() Expect(changed).To(Equal(state.ClusterStateChange)) @@ -1781,12 +1863,16 @@ var _ = Describe("ChangeProcessor", func() { processor.CaptureUpsertChange(gw1Updated) processor.CaptureUpsertChange(hr1Updated) processor.CaptureUpsertChange(rg1Updated) + processor.CaptureUpsertChange(btlsUpdated) + processor.CaptureUpsertChange(cmUpdated) // there are non-changing changes processor.CaptureUpsertChange(gcUpdated) processor.CaptureUpsertChange(gw1Updated) processor.CaptureUpsertChange(hr1Updated) processor.CaptureUpsertChange(rg1Updated) + processor.CaptureUpsertChange(btlsUpdated) + processor.CaptureUpsertChange(cmUpdated) changed, _ := processor.Process() Expect(changed).To(Equal(state.ClusterStateChange)) @@ -1808,6 +1894,8 @@ var _ = Describe("ChangeProcessor", func() { processor.CaptureDeleteChange(&v1.Gateway{}, gwNsName) processor.CaptureDeleteChange(&v1.HTTPRoute{}, hrNsName) processor.CaptureDeleteChange(&v1beta1.ReferenceGrant{}, rgNsName) + processor.CaptureDeleteChange(&v1alpha2.BackendTLSPolicy{}, btlsNsName) + processor.CaptureDeleteChange(&apiv1.ConfigMap{}, cmNsName) // these are non-changing changes processor.CaptureUpsertChange(gw2) @@ -1846,6 +1934,7 @@ var _ = Describe("ChangeProcessor", func() { processor.CaptureUpsertChange(hr1) processor.CaptureUpsertChange(secret) processor.CaptureUpsertChange(barSecret) + processor.CaptureUpsertChange(cm) changed, _ := processor.Process() Expect(changed).To(Equal(state.ClusterStateChange)) }) @@ -1855,6 +1944,7 @@ var _ = Describe("ChangeProcessor", func() { processor.CaptureUpsertChange(slice) processor.CaptureUpsertChange(ns) processor.CaptureUpsertChange(secretUpdated) + processor.CaptureUpsertChange(cmUpdated) changed, _ := processor.Process() Expect(changed).To(Equal(state.ClusterStateChange)) }) @@ -1863,6 +1953,7 @@ var _ = Describe("ChangeProcessor", func() { processor.CaptureUpsertChange(unrelatedSlice) processor.CaptureUpsertChange(unrelatedNS) processor.CaptureUpsertChange(unrelatedSecret) + processor.CaptureUpsertChange(unrelatedCM) changed, _ := processor.Process() Expect(changed).To(Equal(state.NoChange)) @@ -1874,12 +1965,14 @@ var _ = Describe("ChangeProcessor", func() { processor.CaptureUpsertChange(barSlice) processor.CaptureUpsertChange(barNs) processor.CaptureUpsertChange(barSecretUpdated) + processor.CaptureUpsertChange(cmUpdated) // there are non-changing changes processor.CaptureUpsertChange(unrelatedSvc) processor.CaptureUpsertChange(unrelatedSlice) processor.CaptureUpsertChange(unrelatedNS) processor.CaptureUpsertChange(unrelatedSecret) + processor.CaptureUpsertChange(unrelatedCM) changed, _ := processor.Process() Expect(changed).To(Equal(state.ClusterStateChange)) @@ -1892,12 +1985,14 @@ var _ = Describe("ChangeProcessor", func() { processor.CaptureDeleteChange(&discoveryV1.EndpointSlice{}, sliceNsName) processor.CaptureDeleteChange(&apiv1.Namespace{}, types.NamespacedName{Name: "ns"}) processor.CaptureDeleteChange(&apiv1.Secret{}, secretNsName) + processor.CaptureDeleteChange(&apiv1.ConfigMap{}, cmNsName) // these are non-changing changes processor.CaptureUpsertChange(unrelatedSvc) processor.CaptureUpsertChange(unrelatedSlice) processor.CaptureUpsertChange(unrelatedNS) processor.CaptureUpsertChange(unrelatedSecret) + processor.CaptureUpsertChange(unrelatedCM) changed, _ := processor.Process() Expect(changed).To(Equal(state.ClusterStateChange)) @@ -1912,12 +2007,14 @@ var _ = Describe("ChangeProcessor", func() { processor.CaptureUpsertChange(testNs) processor.CaptureUpsertChange(hr1) processor.CaptureUpsertChange(rg1) + processor.CaptureUpsertChange(btls) // related Kubernetes API resources processor.CaptureUpsertChange(svc) processor.CaptureUpsertChange(slice) processor.CaptureUpsertChange(ns) processor.CaptureUpsertChange(secret) + processor.CaptureUpsertChange(cm) changed, _ := processor.Process() Expect(changed).To(Equal(state.ClusterStateChange)) @@ -1928,6 +2025,7 @@ var _ = Describe("ChangeProcessor", func() { processor.CaptureUpsertChange(unrelatedSlice) processor.CaptureUpsertChange(unrelatedNS) processor.CaptureUpsertChange(unrelatedSecret) + processor.CaptureUpsertChange(unrelatedCM) changed, _ := processor.Process() Expect(changed).To(Equal(state.NoChange)) @@ -1939,12 +2037,14 @@ var _ = Describe("ChangeProcessor", func() { processor.CaptureUpsertChange(gw1Updated) processor.CaptureUpsertChange(hr1Updated) processor.CaptureUpsertChange(rg1Updated) + processor.CaptureUpsertChange(btlsUpdated) // these are non-changing changes processor.CaptureUpsertChange(unrelatedSvc) processor.CaptureUpsertChange(unrelatedSlice) processor.CaptureUpsertChange(unrelatedNS) processor.CaptureUpsertChange(unrelatedSecret) + processor.CaptureUpsertChange(unrelatedCM) changed, _ := processor.Process() Expect(changed).To(Equal(state.ClusterStateChange)) diff --git a/internal/mode/static/state/conditions/conditions.go b/internal/mode/static/state/conditions/conditions.go index 25ade0b204..894fb3596c 100644 --- a/internal/mode/static/state/conditions/conditions.go +++ b/internal/mode/static/state/conditions/conditions.go @@ -5,11 +5,20 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" v1 "sigs.k8s.io/gateway-api/apis/v1" + "sigs.k8s.io/gateway-api/apis/v1alpha2" ngfAPI "github.com/nginxinc/nginx-gateway-fabric/apis/v1alpha1" "github.com/nginxinc/nginx-gateway-fabric/internal/framework/conditions" ) +// BackendTLSPolicyConditionType is a type of condition associated with a BackendTLSPolicy. +// This type should be used with the BackendTLSPolicyStatus.Conditions field. +type BackendTLSPolicyConditionType string + +// BackendTLSPolicyConditionReason defines the set of reasons that explain why a particular BackendTLSPolicy condition +// type has been raised. +type BackendTLSPolicyConditionReason string + const ( // ListenerReasonUnsupportedValue is used with the "Accepted" condition when a value of a field in a Listener // is invalid or not supported. @@ -545,3 +554,24 @@ func NewNginxGatewayInvalid(msg string) conditions.Condition { Message: msg, } } + +// NewBackendTLSPolicyAccepted returns a Condition that indicates that the BackendTLSPolicy config is valid and accepted +// by the Gateway. +func NewBackendTLSPolicyAccepted() conditions.Condition { + return conditions.Condition{ + Type: string(v1alpha2.PolicyConditionAccepted), + Status: metav1.ConditionTrue, + Reason: string(v1alpha2.PolicyReasonAccepted), + Message: "BackendTLSPolicy is accepted by the Gateway", + } +} + +// NewBackendTLSPolicyInvalid returns a Condition that indicates that the BackendTLSPolicy config is invalid. +func NewBackendTLSPolicyInvalid(msg string) conditions.Condition { + return conditions.Condition{ + Type: string(v1alpha2.PolicyConditionAccepted), + Status: metav1.ConditionFalse, + Reason: string(v1alpha2.PolicyReasonInvalid), + Message: msg, + } +} diff --git a/internal/mode/static/state/dataplane/configuration.go b/internal/mode/static/state/dataplane/configuration.go index d48a304b32..25ed7e217a 100644 --- a/internal/mode/static/state/dataplane/configuration.go +++ b/internal/mode/static/state/dataplane/configuration.go @@ -2,6 +2,7 @@ package dataplane import ( "context" + "encoding/base64" "fmt" "sort" @@ -14,7 +15,10 @@ import ( "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/state/resolver" ) -const wildcardHostname = "~^" +const ( + wildcardHostname = "~^" + alpineSSLRootCAPath = "/etc/ssl/cert.pem" +) // BuildConfiguration builds the Configuration from the Graph. func BuildConfiguration( @@ -35,6 +39,7 @@ func BuildConfiguration( httpServers, sslServers := buildServers(g.Gateway.Listeners) backendGroups := buildBackendGroups(append(httpServers, sslServers...)) keyPairs := buildSSLKeyPairs(g.ReferencedSecrets, g.Gateway.Listeners) + certBundles := buildCertBundles(g.ReferencedCaCertConfigMaps, backendGroups) config := Configuration{ HTTPServers: httpServers, @@ -43,6 +48,7 @@ func BuildConfiguration( BackendGroups: backendGroups, SSLKeyPairs: keyPairs, Version: configVersion, + CertBundles: certBundles, } return config @@ -72,6 +78,47 @@ func buildSSLKeyPairs( return keyPairs } +func buildCertBundles( + caCertConfigMaps map[types.NamespacedName]*graph.CaCertConfigMap, + backendGroups []BackendGroup, +) map[CertBundleID]CertBundle { + bundles := make(map[CertBundleID]CertBundle) + refByBG := make(map[CertBundleID]struct{}) + + // We only need to build the cert bundles if there are valid backend groups that reference them. + if len(backendGroups) == 0 { + return bundles + } + for _, bg := range backendGroups { + if bg.Backends == nil { + continue + } + for _, b := range bg.Backends { + if !b.Valid || b.VerifyTLS == nil { + continue + } + refByBG[b.VerifyTLS.CertBundleID] = struct{}{} + } + } + + for cmName, cm := range caCertConfigMaps { + id := generateCertBundleID(cmName) + if _, exists := refByBG[id]; exists { + if cm.CACert != nil || len(cm.CACert) > 0 { + // the cert could be base64 encoded or plaintext + data := make([]byte, base64.StdEncoding.DecodedLen(len(cm.CACert))) + _, err := base64.StdEncoding.Decode(data, cm.CACert) + if err != nil { + data = cm.CACert + } + bundles[id] = CertBundle(data) + } + } + } + + return bundles +} + func buildBackendGroups(servers []VirtualServer) []BackendGroup { type key struct { nsname types.NamespacedName @@ -122,6 +169,7 @@ func newBackendGroup(refs []graph.BackendRef, sourceNsName types.NamespacedName, UpstreamName: ref.ServicePortReference(), Weight: ref.Weight, Valid: ref.Valid, + VerifyTLS: convertBackendTLS(ref.BackendTLSPolicy), }) } @@ -132,6 +180,20 @@ func newBackendGroup(refs []graph.BackendRef, sourceNsName types.NamespacedName, } } +func convertBackendTLS(btp *graph.BackendTLSPolicy) *VerifyTLS { + if btp == nil || !btp.Valid { + return nil + } + verify := &VerifyTLS{} + if btp.CaCertRef.Name != "" { + verify.CertBundleID = generateCertBundleID(btp.CaCertRef) + } else { + verify.RootCAPath = alpineSSLRootCAPath + } + verify.Hostname = string(btp.Source.Spec.TLS.Hostname) + return verify +} + func buildServers(listeners []*graph.Listener) (http, ssl []VirtualServer) { rulesForProtocol := map[v1.ProtocolType]portPathRules{ v1.HTTPProtocolType: make(portPathRules), @@ -490,3 +552,10 @@ func listenerHostnameMoreSpecific(host1, host2 *v1.Hostname) bool { func generateSSLKeyPairID(secret types.NamespacedName) SSLKeyPairID { return SSLKeyPairID(fmt.Sprintf("ssl_keypair_%s_%s", secret.Namespace, secret.Name)) } + +// generateCertBundleID generates an ID for the certificate bundle based on the ConfigMap namespaced name. +// It is guaranteed to be unique per unique namespaced name. +// The ID is safe to use as a file name. +func generateCertBundleID(configMap types.NamespacedName) CertBundleID { + return CertBundleID(fmt.Sprintf("cert_bundle_%s_%s", configMap.Namespace, configMap.Name)) +} diff --git a/internal/mode/static/state/dataplane/configuration_test.go b/internal/mode/static/state/dataplane/configuration_test.go index f797dd9a66..4a44b5c5b7 100644 --- a/internal/mode/static/state/dataplane/configuration_test.go +++ b/internal/mode/static/state/dataplane/configuration_test.go @@ -12,6 +12,7 @@ import ( "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" v1 "sigs.k8s.io/gateway-api/apis/v1" + v1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" "github.com/nginxinc/nginx-gateway-fabric/internal/framework/helpers" "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/state/graph" @@ -313,6 +314,92 @@ func TestBuildConfiguration(t *testing.T) { pathAndType{path: "/", pathType: prefix}, pathAndType{path: "/third", pathType: prefix}, ) + httpsHR8, expHTTPSHR8Groups, httpsRouteHR8 := createTestResources( + "https-hr-8", + "foo.example.com", + "listener-443-1", + pathAndType{path: "/", pathType: prefix}, pathAndType{path: "/", pathType: prefix}, + ) + + httpsRouteHR8.Rules[0].BackendRefs[0].BackendTLSPolicy = &graph.BackendTLSPolicy{ + Source: &v1alpha2.BackendTLSPolicy{ + ObjectMeta: metav1.ObjectMeta{ + Name: "btp", + Namespace: "test", + }, + Spec: v1alpha2.BackendTLSPolicySpec{ + TargetRef: v1alpha2.PolicyTargetReferenceWithSectionName{ + PolicyTargetReference: v1alpha2.PolicyTargetReference{ + Group: "", + Kind: "Service", + Name: "foo", + Namespace: (*v1.Namespace)(helpers.GetPointer("test")), + }, + }, + TLS: v1alpha2.BackendTLSPolicyConfig{ + Hostname: "foo.example.com", + CACertRefs: []v1.LocalObjectReference{ + { + Kind: "ConfigMap", + Name: "configmap-1", + Group: "", + }, + }, + }, + }, + }, + CaCertRef: types.NamespacedName{Namespace: "test", Name: "configmap-1"}, + Valid: true, + } + + expHTTPSHR8Groups[0].Backends[0].VerifyTLS = &VerifyTLS{ + CertBundleID: generateCertBundleID(types.NamespacedName{Namespace: "test", Name: "configmap-1"}), + Hostname: "foo.example.com", + } + + httpsHR9, expHTTPSHR9Groups, httpsRouteHR9 := createTestResources( + "https-hr-9", + "foo.example.com", + "listener-443-1", + pathAndType{path: "/", pathType: prefix}, pathAndType{path: "/", pathType: prefix}, + ) + + httpsRouteHR9.Rules[0].BackendRefs[0].BackendTLSPolicy = &graph.BackendTLSPolicy{ + Source: &v1alpha2.BackendTLSPolicy{ + ObjectMeta: metav1.ObjectMeta{ + Name: "btp2", + Namespace: "test", + }, + Spec: v1alpha2.BackendTLSPolicySpec{ + TargetRef: v1alpha2.PolicyTargetReferenceWithSectionName{ + PolicyTargetReference: v1alpha2.PolicyTargetReference{ + Group: "", + Kind: "Service", + Name: "foo", + Namespace: (*v1.Namespace)(helpers.GetPointer("test")), + }, + }, + TLS: v1alpha2.BackendTLSPolicyConfig{ + Hostname: "foo.example.com", + CACertRefs: []v1.LocalObjectReference{ + { + Kind: "ConfigMap", + Name: "configmap-2", + Group: "", + }, + }, + }, + }, + }, + CaCertRef: types.NamespacedName{Namespace: "test", Name: "configmap-2"}, + Valid: true, + } + + expHTTPSHR9Groups[0].Backends[0].VerifyTLS = &VerifyTLS{ + CertBundleID: generateCertBundleID(types.NamespacedName{Namespace: "test", Name: "configmap-2"}), + Hostname: "foo.example.com", + } + secret1NsName := types.NamespacedName{Namespace: "test", Name: "secret-1"} secret1 := &graph.Secret{ Source: &apiv1.Secret{ @@ -425,6 +512,33 @@ func TestBuildConfiguration(t *testing.T) { }, } + referencedConfigMaps := map[types.NamespacedName]*graph.CaCertConfigMap{ + {Namespace: "test", Name: "configmap-1"}: { + Source: &apiv1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "configmap-1", + Namespace: "test", + }, + Data: map[string]string{ + "ca.crt": "cert-1", + }, + }, + CACert: []byte("cert-1"), + }, + {Namespace: "test", Name: "configmap-2"}: { + Source: &apiv1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "configmap-2", + Namespace: "test", + }, + BinaryData: map[string][]byte{ + "ca.crt": []byte("cert-2"), + }, + }, + CACert: []byte("cert-2"), + }, + } + tests := []struct { graph *graph.Graph msg string @@ -446,6 +560,7 @@ func TestBuildConfiguration(t *testing.T) { HTTPServers: []VirtualServer{}, SSLServers: []VirtualServer{}, SSLKeyPairs: map[SSLKeyPairID]SSLKeyPair{}, + CertBundles: map[CertBundleID]CertBundle{}, }, msg: "no listeners and routes", }, @@ -477,6 +592,7 @@ func TestBuildConfiguration(t *testing.T) { }, SSLServers: []VirtualServer{}, SSLKeyPairs: map[SSLKeyPairID]SSLKeyPair{}, + CertBundles: map[CertBundleID]CertBundle{}, }, msg: "http listener with no routes", }, @@ -539,6 +655,7 @@ func TestBuildConfiguration(t *testing.T) { Key: []byte("privateKey-1"), }, }, + CertBundles: map[CertBundleID]CertBundle{}, }, msg: "http and https listeners with no valid routes", }, @@ -601,6 +718,7 @@ func TestBuildConfiguration(t *testing.T) { Key: []byte("privateKey-2"), }, }, + CertBundles: map[CertBundleID]CertBundle{}, }, msg: "https listeners with no routes", }, @@ -633,6 +751,7 @@ func TestBuildConfiguration(t *testing.T) { HTTPServers: []VirtualServer{}, SSLServers: []VirtualServer{}, SSLKeyPairs: map[SSLKeyPairID]SSLKeyPair{}, + CertBundles: map[CertBundleID]CertBundle{}, }, msg: "invalid https listener with resolved secret", }, @@ -704,6 +823,7 @@ func TestBuildConfiguration(t *testing.T) { Upstreams: []Upstream{fooUpstream}, BackendGroups: []BackendGroup{expHR1Groups[0], expHR2Groups[0]}, SSLKeyPairs: map[SSLKeyPairID]SSLKeyPair{}, + CertBundles: map[CertBundleID]CertBundle{}, }, msg: "one http listener with two routes for different hostnames", }, @@ -823,6 +943,7 @@ func TestBuildConfiguration(t *testing.T) { Key: []byte("privateKey-2"), }, }, + CertBundles: map[CertBundleID]CertBundle{}, }, msg: "two https listeners each with routes for different hostnames", }, @@ -982,6 +1103,7 @@ func TestBuildConfiguration(t *testing.T) { Key: []byte("privateKey-1"), }, }, + CertBundles: map[CertBundleID]CertBundle{}, }, msg: "one http and one https listener with two routes with the same hostname with and without collisions", }, @@ -1194,6 +1316,7 @@ func TestBuildConfiguration(t *testing.T) { Key: []byte("privateKey-1"), }, }, + CertBundles: map[CertBundleID]CertBundle{}, }, msg: "multiple http and https listener; different ports", @@ -1325,6 +1448,7 @@ func TestBuildConfiguration(t *testing.T) { Upstreams: []Upstream{fooUpstream}, BackendGroups: []BackendGroup{expHR5Groups[0], expHR5Groups[1]}, SSLKeyPairs: map[SSLKeyPairID]SSLKeyPair{}, + CertBundles: map[CertBundleID]CertBundle{}, }, msg: "one http listener with one route with filters", }, @@ -1426,6 +1550,7 @@ func TestBuildConfiguration(t *testing.T) { Key: []byte("privateKey-1"), }, }, + CertBundles: map[CertBundleID]CertBundle{}, }, msg: "one http and one https listener with routes with valid and invalid rules", }, @@ -1489,6 +1614,7 @@ func TestBuildConfiguration(t *testing.T) { Upstreams: []Upstream{fooUpstream}, BackendGroups: []BackendGroup{expHR7Groups[0], expHR7Groups[1]}, SSLKeyPairs: map[SSLKeyPairID]SSLKeyPair{}, + CertBundles: map[CertBundleID]CertBundle{}, }, msg: "duplicate paths with different types", }, @@ -1576,9 +1702,162 @@ func TestBuildConfiguration(t *testing.T) { Key: []byte("privateKey-2"), }, }, + CertBundles: map[CertBundleID]CertBundle{}, }, msg: "two https listeners with different hostnames but same route; chooses listener with more specific hostname", }, + { + graph: &graph.Graph{ + GatewayClass: &graph.GatewayClass{ + Source: &v1.GatewayClass{}, + Valid: true, + }, + Gateway: &graph.Gateway{ + Source: &v1.Gateway{}, + Listeners: []*graph.Listener{ + { + Name: "listener-443", + Source: listener443, + Valid: true, + Routes: map[types.NamespacedName]*graph.Route{ + {Namespace: "test", Name: "https-hr-8"}: httpsRouteHR8, + }, + ResolvedSecret: &secret1NsName, + }, + }, + }, + Routes: map[types.NamespacedName]*graph.Route{ + {Namespace: "test", Name: "https-hr-8"}: httpsRouteHR8, + }, + ReferencedSecrets: map[types.NamespacedName]*graph.Secret{ + secret1NsName: secret1, + }, + ReferencedCaCertConfigMaps: referencedConfigMaps, + }, + expConf: Configuration{ + HTTPServers: []VirtualServer{}, + SSLServers: []VirtualServer{ + { + IsDefault: true, + Port: 443, + }, + { + Hostname: "foo.example.com", + PathRules: []PathRule{ + { + Path: "/", + PathType: PathTypePrefix, + MatchRules: []MatchRule{ + { + BackendGroup: expHTTPSHR8Groups[0], + Source: &httpsHR8.ObjectMeta, + }, + { + BackendGroup: expHTTPSHR8Groups[1], + Source: &httpsHR8.ObjectMeta, + }, + }, + }, + }, + SSL: &SSL{KeyPairID: "ssl_keypair_test_secret-1"}, + Port: 443, + }, + { + Hostname: wildcardHostname, + SSL: &SSL{KeyPairID: "ssl_keypair_test_secret-1"}, + Port: 443, + }, + }, + Upstreams: []Upstream{fooUpstream}, + BackendGroups: []BackendGroup{expHTTPSHR8Groups[0], expHTTPSHR8Groups[1]}, + SSLKeyPairs: map[SSLKeyPairID]SSLKeyPair{ + "ssl_keypair_test_secret-1": { + Cert: []byte("cert-1"), + Key: []byte("privateKey-1"), + }, + }, + CertBundles: map[CertBundleID]CertBundle{ + "cert_bundle_test_configmap-1": []byte("cert-1"), + }, + }, + msg: "https listener with httproute with backend that has a backend TLS policy attached", + }, + { + graph: &graph.Graph{ + GatewayClass: &graph.GatewayClass{ + Source: &v1.GatewayClass{}, + Valid: true, + }, + Gateway: &graph.Gateway{ + Source: &v1.Gateway{}, + Listeners: []*graph.Listener{ + { + Name: "listener-443", + Source: listener443, + Valid: true, + Routes: map[types.NamespacedName]*graph.Route{ + {Namespace: "test", Name: "https-hr-9"}: httpsRouteHR9, + }, + ResolvedSecret: &secret1NsName, + }, + }, + }, + Routes: map[types.NamespacedName]*graph.Route{ + {Namespace: "test", Name: "https-hr-9"}: httpsRouteHR9, + }, + ReferencedSecrets: map[types.NamespacedName]*graph.Secret{ + secret1NsName: secret1, + }, + ReferencedCaCertConfigMaps: referencedConfigMaps, + }, + expConf: Configuration{ + HTTPServers: []VirtualServer{}, + SSLServers: []VirtualServer{ + { + IsDefault: true, + Port: 443, + }, + { + Hostname: "foo.example.com", + PathRules: []PathRule{ + { + Path: "/", + PathType: PathTypePrefix, + MatchRules: []MatchRule{ + { + BackendGroup: expHTTPSHR9Groups[0], + Source: &httpsHR9.ObjectMeta, + }, + { + BackendGroup: expHTTPSHR9Groups[1], + Source: &httpsHR9.ObjectMeta, + }, + }, + }, + }, + SSL: &SSL{KeyPairID: "ssl_keypair_test_secret-1"}, + Port: 443, + }, + { + Hostname: wildcardHostname, + SSL: &SSL{KeyPairID: "ssl_keypair_test_secret-1"}, + Port: 443, + }, + }, + Upstreams: []Upstream{fooUpstream}, + BackendGroups: []BackendGroup{expHTTPSHR9Groups[0], expHTTPSHR9Groups[1]}, + SSLKeyPairs: map[SSLKeyPairID]SSLKeyPair{ + "ssl_keypair_test_secret-1": { + Cert: []byte("cert-1"), + Key: []byte("privateKey-1"), + }, + }, + CertBundles: map[CertBundleID]CertBundle{ + "cert_bundle_test_configmap-2": []byte("cert-2"), + }, + }, + msg: "https listener with httproute with backend that has a backend TLS policy with binaryData attached", + }, } for _, test := range tests { @@ -1593,6 +1872,7 @@ func TestBuildConfiguration(t *testing.T) { g.Expect(result.SSLServers).To(ConsistOf(test.expConf.SSLServers)) g.Expect(result.SSLKeyPairs).To(Equal(test.expConf.SSLKeyPairs)) g.Expect(result.Version).To(Equal(1)) + g.Expect(result.CertBundles).To(Equal(test.expConf.CertBundles)) }) } } @@ -2155,3 +2435,79 @@ func TestHostnameMoreSpecific(t *testing.T) { }) } } + +func TestConvertBackendTLS(t *testing.T) { + btpCaCertRefs := &graph.BackendTLSPolicy{ + Source: &v1alpha2.BackendTLSPolicy{ + ObjectMeta: metav1.ObjectMeta{ + Name: "btp", + Namespace: "test", + }, + Spec: v1alpha2.BackendTLSPolicySpec{ + TLS: v1alpha2.BackendTLSPolicyConfig{ + CACertRefs: []v1.LocalObjectReference{ + { + Name: "ca-cert", + }, + }, + Hostname: "example.com", + }, + }, + }, + Valid: true, + CaCertRef: types.NamespacedName{Namespace: "test", Name: "ca-cert"}, + } + + btpWellKnownCerts := &graph.BackendTLSPolicy{ + Source: &v1alpha2.BackendTLSPolicy{ + Spec: v1alpha2.BackendTLSPolicySpec{ + TLS: v1alpha2.BackendTLSPolicyConfig{ + Hostname: "example.com", + }, + }, + }, + Valid: true, + } + + expectedWithCertPath := &VerifyTLS{ + CertBundleID: generateCertBundleID( + types.NamespacedName{Namespace: "test", Name: "ca-cert"}, + ), + Hostname: "example.com", + } + + expectedWithWellKnownCerts := &VerifyTLS{ + Hostname: "example.com", + RootCAPath: alpineSSLRootCAPath, + } + + tests := []struct { + btp *graph.BackendTLSPolicy + expected *VerifyTLS + msg string + }{ + { + btp: nil, + expected: nil, + msg: "nil backend tls policy", + }, + { + btp: btpCaCertRefs, + expected: expectedWithCertPath, + msg: "normal case with cert path", + }, + { + btp: btpWellKnownCerts, + expected: expectedWithWellKnownCerts, + msg: "normal case no cert path", + }, + } + + for _, tc := range tests { + t.Run(tc.msg, func(t *testing.T) { + g := NewWithT(t) + + g.Expect(convertBackendTLS(tc.btp)).To(Equal(tc.expected)) + }) + } +} diff --git a/internal/mode/static/state/dataplane/types.go b/internal/mode/static/state/dataplane/types.go index 465e421ea3..da1b00afaa 100644 --- a/internal/mode/static/state/dataplane/types.go +++ b/internal/mode/static/state/dataplane/types.go @@ -23,6 +23,8 @@ const ( type Configuration struct { // SSLKeyPairs holds all unique SSLKeyPairs. SSLKeyPairs map[SSLKeyPairID]SSLKeyPair + // CertBundles holds all unique Certificate Bundles. + CertBundles map[CertBundleID]CertBundle // HTTPServers holds all HTTPServers. HTTPServers []VirtualServer // SSLServers holds all SSLServers. @@ -39,6 +41,13 @@ type Configuration struct { // The ID is safe to use as a file name. type SSLKeyPairID string +// CertBundleID is a unique identifier for a Certificate bundle. +// The ID is safe to use as a file name. +type CertBundleID string + +// CertBundle is a Certificate bundle. +type CertBundle []byte + // SSLKeyPair is an SSL private/public key pair. type SSLKeyPair struct { // Cert is the certificate. @@ -222,6 +231,8 @@ func (bg *BackendGroup) Name() string { // Backend represents a Backend for a routing rule. type Backend struct { + // VerifyTLS holds the backend TLS verification configuration. + VerifyTLS *VerifyTLS // UpstreamName is the name of the upstream for this backend. UpstreamName string // Weight is the weight of the BackendRef. @@ -231,3 +242,10 @@ type Backend struct { // Valid indicates whether the Backend is valid. Valid bool } + +// VerifyTLS holds the backend TLS verification configuration. +type VerifyTLS struct { + CertBundleID CertBundleID + Hostname string + RootCAPath string +} diff --git a/internal/mode/static/state/graph/backend_refs.go b/internal/mode/static/state/graph/backend_refs.go index f8eab82d80..8d66d9b3a2 100644 --- a/internal/mode/static/state/graph/backend_refs.go +++ b/internal/mode/static/state/graph/backend_refs.go @@ -2,18 +2,24 @@ package graph import ( "fmt" + "slices" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/validation/field" gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" + v1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" "github.com/nginxinc/nginx-gateway-fabric/internal/framework/conditions" + "github.com/nginxinc/nginx-gateway-fabric/internal/framework/helpers" + "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/sort" staticConds "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/state/conditions" ) // BackendRef is an internal representation of a backendRef in an HTTPRoute. type BackendRef struct { + // BackendTLSPolicy is the BackendTLSPolicy of the Service which is referenced by the backendRef. + BackendTLSPolicy *BackendTLSPolicy // SvcNsName is the NamespacedName of the Service referenced by the backendRef. SvcNsName types.NamespacedName // ServicePort is the ServicePort of the Service which is referenced by the backendRef. @@ -37,9 +43,10 @@ func addBackendRefsToRouteRules( routes map[types.NamespacedName]*Route, refGrantResolver *referenceGrantResolver, services map[types.NamespacedName]*v1.Service, + backendTLSPolicies map[types.NamespacedName]*BackendTLSPolicy, ) { for _, r := range routes { - addBackendRefsToRules(r, refGrantResolver, services) + addBackendRefsToRules(r, refGrantResolver, services, backendTLSPolicies) } } @@ -50,6 +57,7 @@ func addBackendRefsToRules( route *Route, refGrantResolver *referenceGrantResolver, services map[types.NamespacedName]*v1.Service, + backendTLSPolicies map[types.NamespacedName]*BackendTLSPolicy, ) { if !route.Valid { return @@ -73,7 +81,14 @@ func addBackendRefsToRules( for refIdx, ref := range rule.BackendRefs { refPath := field.NewPath("spec").Child("rules").Index(idx).Child("backendRefs").Index(refIdx) - ref, cond := createBackendRef(ref, route.Source.Namespace, refGrantResolver, services, refPath) + ref, cond := createBackendRef( + ref, + route.Source.Namespace, + refGrantResolver, + services, + refPath, + backendTLSPolicies, + ) backendRefs = append(backendRefs, ref) if cond != nil { @@ -81,6 +96,17 @@ func addBackendRefsToRules( } } + if len(backendRefs) > 1 { + cond := validateBackendTLSPolicyMatchingAllBackends(backendRefs) + if cond != nil { + route.Conditions = append(route.Conditions, *cond) + // mark all backendRefs as invalid + for i := range backendRefs { + backendRefs[i].Valid = false + } + } + } + route.Rules[idx].BackendRefs = backendRefs } } @@ -91,6 +117,7 @@ func createBackendRef( refGrantResolver *referenceGrantResolver, services map[types.NamespacedName]*v1.Service, refPath *field.Path, + backendTLSPolicies map[types.NamespacedName]*BackendTLSPolicy, ) (BackendRef, *conditions.Condition) { // Data plane will handle invalid ref by responding with 500. // Because of that, we always need to add a BackendRef to group.Backends, even if the ref is invalid. @@ -130,16 +157,119 @@ func createBackendRef( return backendRef, &cond } + backendTLSPolicy, err := findBackendTLSPolicyForService( + backendTLSPolicies, + ref, + sourceNamespace, + ) + if err != nil { + backendRef = BackendRef{ + SvcNsName: svcNsName, + ServicePort: svcPort, + Weight: weight, + Valid: false, + } + + cond := staticConds.NewRouteBackendRefUnsupportedValue(err.Error()) + return backendRef, &cond + } + backendRef = BackendRef{ - SvcNsName: svcNsName, - ServicePort: svcPort, - Valid: true, - Weight: weight, + SvcNsName: svcNsName, + BackendTLSPolicy: backendTLSPolicy, + ServicePort: svcPort, + Valid: true, + Weight: weight, } return backendRef, nil } +// validateBackendTLSPolicyMatchingAllBackends validates that all backends in a rule reference the same +// BackendTLSPolicy. We require that all backends in a group have the same backend TLS policy configuration. +// The backend TLS policy configuration is considered matching if: 1. CACertRefs reference the same ConfigMap, or +// 2. WellKnownCACerts are the same, and 3. Hostname is the same. +// FIXME (ciarams87): This is a temporary solution until we can support multiple backend TLS policies per group. +// https://github.com/nginxinc/nginx-gateway-fabric/issues/1546 +func validateBackendTLSPolicyMatchingAllBackends(backendRefs []BackendRef) *conditions.Condition { + var mismatch bool + var referencePolicy *BackendTLSPolicy + + checkPoliciesEqual := func(p1, p2 *v1alpha2.BackendTLSPolicy) bool { + return !slices.Equal(p1.Spec.TLS.CACertRefs, p2.Spec.TLS.CACertRefs) || + p1.Spec.TLS.WellKnownCACerts != p2.Spec.TLS.WellKnownCACerts || + p1.Spec.TLS.Hostname != p2.Spec.TLS.Hostname + } + + for _, backendRef := range backendRefs { + if backendRef.BackendTLSPolicy == nil { + if referencePolicy != nil { + // There was a reference before, so they do not all match + mismatch = true + break + } + continue + } + + if referencePolicy == nil { + // First reference, store the policy as reference + referencePolicy = backendRef.BackendTLSPolicy + } else { + // Check if the policies match + if checkPoliciesEqual(backendRef.BackendTLSPolicy.Source, referencePolicy.Source) { + mismatch = true + break + } + } + } + if mismatch { + msg := "Backend TLS policies do not match for all backends" + return helpers.GetPointer(staticConds.NewRouteBackendRefUnsupportedValue(msg)) + } + return nil +} + +func findBackendTLSPolicyForService( + backendTLSPolicies map[types.NamespacedName]*BackendTLSPolicy, + ref gatewayv1.HTTPBackendRef, + routeNamespace string, +) (*BackendTLSPolicy, error) { + var beTLSPolicy *BackendTLSPolicy + var err error + + refNs := routeNamespace + if ref.Namespace != nil { + refNs = string(*ref.Namespace) + } + + for _, btp := range backendTLSPolicies { + btpNs := btp.Source.Namespace + if btp.Source.Spec.TargetRef.Namespace != nil { + btpNs = string(*btp.Source.Spec.TargetRef.Namespace) + } + if btp.Source.Spec.TargetRef.Name == ref.Name && btpNs == refNs { + if beTLSPolicy != nil { + if sort.LessObjectMeta(&btp.Source.ObjectMeta, &beTLSPolicy.Source.ObjectMeta) { + beTLSPolicy = btp + } + } else { + beTLSPolicy = btp + } + } + } + + if beTLSPolicy != nil { + beTLSPolicy.IsReferenced = true + if !beTLSPolicy.Valid { + err = fmt.Errorf("The backend TLS policy is invalid: %s", beTLSPolicy.Conditions[0].Message) + } else { + beTLSPolicy.Conditions = append(beTLSPolicy.Conditions, staticConds.NewBackendTLSPolicyAccepted()) + } + } + + return beTLSPolicy, err +} + // getServiceAndPortFromRef extracts the NamespacedName of the Service and the port from a BackendRef. // It can return an error and an empty v1.ServicePort in two cases: // 1. The Service referenced from the BackendRef does not exist in the cluster/state. diff --git a/internal/mode/static/state/graph/backend_refs_test.go b/internal/mode/static/state/graph/backend_refs_test.go index d2205c752f..32d963ed5c 100644 --- a/internal/mode/static/state/graph/backend_refs_test.go +++ b/internal/mode/static/state/graph/backend_refs_test.go @@ -10,6 +10,7 @@ import ( "k8s.io/apimachinery/pkg/util/validation/field" "sigs.k8s.io/controller-runtime/pkg/client" gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" + "sigs.k8s.io/gateway-api/apis/v1alpha2" "sigs.k8s.io/gateway-api/apis/v1beta1" "github.com/nginxinc/nginx-gateway-fabric/internal/framework/conditions" @@ -393,38 +394,142 @@ func TestAddBackendRefsToRulesTest(t *testing.T) { hrWithOneBackend := createRoute("hr1", "Service", 1, "svc1") hrWithTwoBackends := createRoute("hr2", "Service", 2, "svc1") + hrWithTwoDiffBackends := createRoute("hr2", "Service", 2, "svc1") hrWithInvalidRule := createRoute("hr3", "NotService", 1, "svc1") hrWithZeroBackendRefs := createRoute("hr4", "Service", 1, "svc1") hrWithZeroBackendRefs.Spec.Rules[0].BackendRefs = nil + hrWithTwoDiffBackends.Spec.Rules[0].BackendRefs[1].Name = "svc2" - svc1 := &v1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: "test", - Name: "svc1", - }, - Spec: v1.ServiceSpec{ - Ports: []v1.ServicePort{ - { - Port: 80, - }, - { - Port: 81, + getSvc := func(name string) *v1.Service { + return &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "test", + Name: name, + }, + Spec: v1.ServiceSpec{ + Ports: []v1.ServicePort{ + { + Port: 80, + }, + { + Port: 81, + }, }, }, - }, + } } + svc1 := getSvc("svc1") svc1NsName := types.NamespacedName{ Namespace: "test", Name: "svc1", } + svc2 := getSvc("svc2") + svc2NsName := types.NamespacedName{ + Namespace: "test", + Name: "svc2", + } + services := map[types.NamespacedName]*v1.Service{ {Namespace: "test", Name: "svc1"}: svc1, + {Namespace: "test", Name: "svc2"}: svc2, + } + emptyPolicies := map[types.NamespacedName]*BackendTLSPolicy{} + + getPolicy := func(name, svcName, cmName string) *BackendTLSPolicy { + return &BackendTLSPolicy{ + Valid: true, + Source: &v1alpha2.BackendTLSPolicy{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: "test", + }, + Spec: v1alpha2.BackendTLSPolicySpec{ + TargetRef: v1alpha2.PolicyTargetReferenceWithSectionName{ + PolicyTargetReference: v1alpha2.PolicyTargetReference{ + Group: "", + Kind: "Service", + Name: gatewayv1.ObjectName(svcName), + Namespace: (*gatewayv1.Namespace)(helpers.GetPointer("test")), + }, + }, + TLS: v1alpha2.BackendTLSPolicyConfig{ + Hostname: "foo.example.com", + CACertRefs: []gatewayv1.LocalObjectReference{ + { + Group: "", + Kind: "ConfigMap", + Name: gatewayv1.ObjectName(cmName), + }, + }, + }, + }, + }, + } } + policiesMatching := map[types.NamespacedName]*BackendTLSPolicy{ + {Namespace: "test", Name: "btp1"}: getPolicy("btp1", "svc1", "test"), + {Namespace: "test", Name: "btp2"}: getPolicy("btp2", "svc2", "test"), + } + policiesNotMatching := map[types.NamespacedName]*BackendTLSPolicy{ + {Namespace: "test", Name: "btp1"}: getPolicy("btp1", "svc1", "test1"), + {Namespace: "test", Name: "btp2"}: getPolicy("btp2", "svc2", "test2"), + } + + getBtp := func(name string, svcName string, cmName string) *BackendTLSPolicy { + return &BackendTLSPolicy{ + Source: &v1alpha2.BackendTLSPolicy{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "test"}, + Spec: v1alpha2.BackendTLSPolicySpec{ + TargetRef: v1alpha2.PolicyTargetReferenceWithSectionName{ + PolicyTargetReference: v1alpha2.PolicyTargetReference{ + Group: "", + Kind: "Service", + Name: gatewayv1.ObjectName(svcName), + Namespace: (*gatewayv1.Namespace)(helpers.GetPointer("test")), + }, + }, + TLS: v1alpha2.BackendTLSPolicyConfig{ + Hostname: "foo.example.com", + CACertRefs: []gatewayv1.LocalObjectReference{ + { + Group: "", + Kind: "ConfigMap", + Name: gatewayv1.ObjectName(cmName), + }, + }, + }, + }, + }, + Conditions: []conditions.Condition{ + { + Type: "Accepted", + Status: "True", + Reason: "Accepted", + Message: "BackendTLSPolicy is accepted by the Gateway", + }, + }, + Valid: true, + IsReferenced: true, + } + } + + btp1 := getBtp("btp1", "svc1", "test1") + btp2 := getBtp("btp2", "svc2", "test2") + btp3 := getBtp("btp1", "svc1", "test") + btp3.Conditions = append(btp3.Conditions, conditions.Condition{ + Type: "Accepted", + Status: "True", + Reason: "Accepted", + Message: "BackendTLSPolicy is accepted by the Gateway", + }, + ) + tests := []struct { - name string route *Route + policies map[types.NamespacedName]*BackendTLSPolicy + name string expectedBackendRefs []BackendRef expectedConditions []conditions.Condition }{ @@ -444,6 +549,7 @@ func TestAddBackendRefsToRulesTest(t *testing.T) { }, }, expectedConditions: nil, + policies: emptyPolicies, name: "normal case with one rule with one backend", }, { @@ -468,8 +574,36 @@ func TestAddBackendRefsToRulesTest(t *testing.T) { }, }, expectedConditions: nil, + policies: emptyPolicies, name: "normal case with one rule with two backends", }, + { + route: &Route{ + Source: hrWithTwoBackends, + ParentRefs: sectionNameRefs, + Valid: true, + Rules: createRules(hrWithTwoBackends, allValid, allValid), + }, + expectedBackendRefs: []BackendRef{ + { + SvcNsName: svc1NsName, + ServicePort: svc1.Spec.Ports[0], + Valid: true, + Weight: 1, + BackendTLSPolicy: btp3, + }, + { + SvcNsName: svc1NsName, + ServicePort: svc1.Spec.Ports[1], + Valid: true, + Weight: 5, + BackendTLSPolicy: btp3, + }, + }, + expectedConditions: nil, + policies: policiesMatching, + name: "normal case with one rule with two backends and matching policies", + }, { route: &Route{ Source: hrWithOneBackend, @@ -478,6 +612,7 @@ func TestAddBackendRefsToRulesTest(t *testing.T) { }, expectedBackendRefs: nil, expectedConditions: nil, + policies: emptyPolicies, name: "invalid route", }, { @@ -489,6 +624,7 @@ func TestAddBackendRefsToRulesTest(t *testing.T) { }, expectedBackendRefs: nil, expectedConditions: nil, + policies: emptyPolicies, name: "invalid matches", }, { @@ -500,6 +636,7 @@ func TestAddBackendRefsToRulesTest(t *testing.T) { }, expectedBackendRefs: nil, expectedConditions: nil, + policies: emptyPolicies, name: "invalid filters", }, { @@ -519,7 +656,39 @@ func TestAddBackendRefsToRulesTest(t *testing.T) { `spec.rules[0].backendRefs[0].kind: Unsupported value: "NotService": supported values: "Service"`, ), }, - name: "invalid backendRef", + policies: emptyPolicies, + name: "invalid backendRef", + }, + { + route: &Route{ + Source: hrWithTwoDiffBackends, + ParentRefs: sectionNameRefs, + Valid: true, + Rules: createRules(hrWithTwoDiffBackends, allValid, allValid), + }, + expectedBackendRefs: []BackendRef{ + { + SvcNsName: svc1NsName, + ServicePort: svc1.Spec.Ports[0], + Valid: false, + Weight: 1, + BackendTLSPolicy: btp1, + }, + { + SvcNsName: svc2NsName, + ServicePort: svc2.Spec.Ports[1], + Valid: false, + Weight: 5, + BackendTLSPolicy: btp2, + }, + }, + expectedConditions: []conditions.Condition{ + staticConds.NewRouteBackendRefUnsupportedValue( + `Backend TLS policies do not match for all backends`, + ), + }, + policies: policiesNotMatching, + name: "invalid backendRef - backend TLS policies do not match for all backends", }, { route: &Route{ @@ -538,7 +707,7 @@ func TestAddBackendRefsToRulesTest(t *testing.T) { t.Run(test.name, func(t *testing.T) { g := NewWithT(t) resolver := newReferenceGrantResolver(nil) - addBackendRefsToRules(test.route, resolver, services) + addBackendRefsToRules(test.route, resolver, services, test.policies) var actual []BackendRef if test.route.Rules != nil { @@ -552,22 +721,77 @@ func TestAddBackendRefsToRulesTest(t *testing.T) { } func TestCreateBackend(t *testing.T) { - svc1 := &v1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: "test", - Name: "service1", - }, - Spec: v1.ServiceSpec{ - Ports: []v1.ServicePort{ - { - Port: 80, + createService := func(name string) *v1.Service { + return &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: "test", + }, + Spec: v1.ServiceSpec{ + Ports: []v1.ServicePort{ + { + Port: 80, + }, + }, + }, + } + } + svc1 := createService("service1") + svc2 := createService("service2") + svc3 := createService("service3") + svc1NamespacedName := types.NamespacedName{Namespace: "test", Name: "service1"} + svc2NamespacedName := types.NamespacedName{Namespace: "test", Name: "service2"} + svc3NamespacedName := types.NamespacedName{Namespace: "test", Name: "service3"} + + btp := BackendTLSPolicy{ + Source: &v1alpha2.BackendTLSPolicy{ + ObjectMeta: metav1.ObjectMeta{ + Name: "btp", + Namespace: "test", + }, + Spec: v1alpha2.BackendTLSPolicySpec{ + TargetRef: v1alpha2.PolicyTargetReferenceWithSectionName{ + PolicyTargetReference: v1alpha2.PolicyTargetReference{ + Group: "", + Kind: "Service", + Name: "service2", + Namespace: (*gatewayv1.Namespace)(helpers.GetPointer("test")), + }, + }, + TLS: v1alpha2.BackendTLSPolicyConfig{ + Hostname: "foo.example.com", + WellKnownCACerts: (helpers.GetPointer(v1alpha2.WellKnownCACertSystem)), }, }, }, + Valid: true, } - svc1NamespacedName := types.NamespacedName{ - Namespace: "test", - Name: "service1", + + btp2 := BackendTLSPolicy{ + Source: &v1alpha2.BackendTLSPolicy{ + ObjectMeta: metav1.ObjectMeta{ + Name: "btp2", + Namespace: "test", + }, + Spec: v1alpha2.BackendTLSPolicySpec{ + TargetRef: v1alpha2.PolicyTargetReferenceWithSectionName{ + PolicyTargetReference: v1alpha2.PolicyTargetReference{ + Group: "", + Kind: "Service", + Name: "service3", + Namespace: (*gatewayv1.Namespace)(helpers.GetPointer("test")), + }, + }, + TLS: v1alpha2.BackendTLSPolicyConfig{ + Hostname: "foo.example.com", + WellKnownCACerts: (helpers.GetPointer(v1alpha2.WellKnownCACertType("unknown"))), + }, + }, + }, + Valid: false, + Conditions: []conditions.Condition{ + staticConds.NewBackendTLSPolicyInvalid("unsupported value"), + }, } tests := []struct { @@ -669,11 +893,57 @@ func TestCreateBackend(t *testing.T) { ), name: "service doesn't exist", }, + { + ref: gatewayv1.HTTPBackendRef{ + BackendRef: getModifiedRef(func(backend gatewayv1.BackendRef) gatewayv1.BackendRef { + backend.Name = "service2" + return backend + }), + }, + expectedBackend: BackendRef{ + SvcNsName: svc2NamespacedName, + ServicePort: svc1.Spec.Ports[0], + Weight: 5, + Valid: true, + BackendTLSPolicy: &btp, + }, + expectedServicePortReference: "test_service2_80", + expectedCondition: nil, + name: "normal case with policy", + }, + { + ref: gatewayv1.HTTPBackendRef{ + BackendRef: getModifiedRef(func(backend gatewayv1.BackendRef) gatewayv1.BackendRef { + backend.Name = "service3" + return backend + }), + }, + expectedBackend: BackendRef{ + SvcNsName: svc3NamespacedName, + ServicePort: svc1.Spec.Ports[0], + Weight: 5, + Valid: false, + }, + expectedServicePortReference: "", + expectedCondition: helpers.GetPointer( + staticConds.NewRouteBackendRefUnsupportedValue( + "The backend TLS policy is invalid: unsupported value", + ), + ), + name: "invalid policy", + }, } services := map[types.NamespacedName]*v1.Service{ client.ObjectKeyFromObject(svc1): svc1, + client.ObjectKeyFromObject(svc2): svc2, + client.ObjectKeyFromObject(svc3): svc3, + } + policies := map[types.NamespacedName]*BackendTLSPolicy{ + client.ObjectKeyFromObject(btp.Source): &btp, + client.ObjectKeyFromObject(btp2.Source): &btp2, } + sourceNamespace := "test" refPath := field.NewPath("test") @@ -683,7 +953,14 @@ func TestCreateBackend(t *testing.T) { g := NewWithT(t) resolver := newReferenceGrantResolver(nil) - backend, cond := createBackendRef(test.ref, sourceNamespace, resolver, services, refPath) + backend, cond := createBackendRef( + test.ref, + sourceNamespace, + resolver, + services, + refPath, + policies, + ) g.Expect(helpers.Diff(test.expectedBackend, backend)).To(BeEmpty()) g.Expect(cond).To(Equal(test.expectedCondition)) @@ -710,7 +987,6 @@ func TestGetServicePort(t *testing.T) { }, }, } - g := NewWithT(t) // ports exist for _, p := range []int32{80, 81, 82} { @@ -724,3 +1000,179 @@ func TestGetServicePort(t *testing.T) { g.Expect(err).Should(HaveOccurred()) g.Expect(port.Port).To(Equal(int32(0))) } + +func TestValidateBackendTLSPolicyMatchingAllBackends(t *testing.T) { + getBtp := func(name, caCertName string) *BackendTLSPolicy { + return &BackendTLSPolicy{ + Source: &v1alpha2.BackendTLSPolicy{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: "test", + }, + Spec: v1alpha2.BackendTLSPolicySpec{ + TLS: v1alpha2.BackendTLSPolicyConfig{ + Hostname: "foo.example.com", + CACertRefs: []gatewayv1.LocalObjectReference{ + { + Group: "", + Kind: "ConfigMap", + Name: gatewayv1.ObjectName(caCertName), + }, + }, + }, + }, + }, + } + } + + backendRefsNoPolicies := []BackendRef{ + { + SvcNsName: types.NamespacedName{Namespace: "test", Name: "svc1"}, + }, + { + SvcNsName: types.NamespacedName{Namespace: "test", Name: "svc2"}, + }, + } + + backendRefsWithMatchingPolicies := []BackendRef{ + { + SvcNsName: types.NamespacedName{Namespace: "test", Name: "svc1"}, + BackendTLSPolicy: getBtp("btp1", "ca1"), + }, + { + SvcNsName: types.NamespacedName{Namespace: "test", Name: "svc2"}, + BackendTLSPolicy: getBtp("btp2", "ca1"), + }, + } + backendRefsWithNotMatchingPolicies := []BackendRef{ + { + SvcNsName: types.NamespacedName{Namespace: "test", Name: "svc1"}, + BackendTLSPolicy: getBtp("btp1", "ca1"), + }, + { + SvcNsName: types.NamespacedName{Namespace: "test", Name: "svc2"}, + BackendTLSPolicy: getBtp("btp2", "ca2"), + }, + } + backendRefsOnePolicy := []BackendRef{ + { + SvcNsName: types.NamespacedName{Namespace: "test", Name: "svc1"}, + BackendTLSPolicy: getBtp("btp1", "ca1"), + }, + { + SvcNsName: types.NamespacedName{Namespace: "test", Name: "svc2"}, + }, + } + msg := "Backend TLS policies do not match for all backends" + tests := []struct { + expectedCondition *conditions.Condition + name string + backendRefs []BackendRef + }{ + { + name: "no policies", + backendRefs: backendRefsNoPolicies, + expectedCondition: nil, + }, + { + name: "matching policies", + backendRefs: backendRefsWithMatchingPolicies, + expectedCondition: nil, + }, + { + name: "not matching policies", + backendRefs: backendRefsWithNotMatchingPolicies, + expectedCondition: helpers.GetPointer(staticConds.NewRouteBackendRefUnsupportedValue(msg)), + }, + { + name: "only one policy", + backendRefs: backendRefsOnePolicy, + expectedCondition: helpers.GetPointer(staticConds.NewRouteBackendRefUnsupportedValue(msg)), + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + g := NewWithT(t) + + cond := validateBackendTLSPolicyMatchingAllBackends(test.backendRefs) + + g.Expect(cond).To(Equal(test.expectedCondition)) + }) + } +} + +func TestFindBackendTLSPolicyForService(t *testing.T) { + oldCreationTimestamp := metav1.Now() + newCreationTimestamp := metav1.Now() + getBtp := func(name string, timestamp metav1.Time) *BackendTLSPolicy { + return &BackendTLSPolicy{ + Valid: true, + Source: &v1alpha2.BackendTLSPolicy{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: "test", + CreationTimestamp: timestamp, + }, + Spec: v1alpha2.BackendTLSPolicySpec{ + TargetRef: v1alpha2.PolicyTargetReferenceWithSectionName{ + PolicyTargetReference: v1alpha2.PolicyTargetReference{ + Group: "", + Kind: "Service", + Name: "svc1", + Namespace: (*gatewayv1.Namespace)(helpers.GetPointer("test")), + }, + }, + }, + }, + } + } + oldestBtp := getBtp("oldest", oldCreationTimestamp) + newestBtp := getBtp("newest", newCreationTimestamp) + alphaFirstBtp := getBtp("alphabeticallyfirst", oldCreationTimestamp) + + ref := gatewayv1.HTTPBackendRef{ + BackendRef: gatewayv1.BackendRef{ + BackendObjectReference: gatewayv1.BackendObjectReference{ + Kind: helpers.GetPointer[gatewayv1.Kind]("Service"), + Name: "svc1", + Namespace: helpers.GetPointer[gatewayv1.Namespace]("test"), + }, + }, + } + + tests := []struct { + name string + backendTLSPolicies map[types.NamespacedName]*BackendTLSPolicy + expectedBtpName string + }{ + { + name: "oldest wins", + backendTLSPolicies: map[types.NamespacedName]*BackendTLSPolicy{ + client.ObjectKeyFromObject(newestBtp.Source): newestBtp, + client.ObjectKeyFromObject(oldestBtp.Source): oldestBtp, + }, + expectedBtpName: "oldest", + }, + { + name: "alphabetically first wins", + backendTLSPolicies: map[types.NamespacedName]*BackendTLSPolicy{ + client.ObjectKeyFromObject(oldestBtp.Source): oldestBtp, + client.ObjectKeyFromObject(alphaFirstBtp.Source): alphaFirstBtp, + client.ObjectKeyFromObject(newestBtp.Source): newestBtp, + }, + expectedBtpName: "alphabeticallyfirst", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + g := NewWithT(t) + + btp, err := findBackendTLSPolicyForService(test.backendTLSPolicies, ref, "test") + + g.Expect(btp.Source.Name).To(Equal(test.expectedBtpName)) + g.Expect(err).ToNot(HaveOccurred()) + }) + } +} diff --git a/internal/mode/static/state/graph/backend_tls_policy.go b/internal/mode/static/state/graph/backend_tls_policy.go new file mode 100644 index 0000000000..ee6ed9a366 --- /dev/null +++ b/internal/mode/static/state/graph/backend_tls_policy.go @@ -0,0 +1,178 @@ +package graph + +import ( + "errors" + "fmt" + + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/validation/field" + v1 "sigs.k8s.io/gateway-api/apis/v1" + "sigs.k8s.io/gateway-api/apis/v1alpha2" + + "github.com/nginxinc/nginx-gateway-fabric/internal/framework/conditions" + "github.com/nginxinc/nginx-gateway-fabric/internal/framework/helpers" + staticConds "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/state/conditions" +) + +type BackendTLSPolicy struct { + // Source is the source resource. + Source *v1alpha2.BackendTLSPolicy + // CaCertRef is the name of the ConfigMap that contains the CA certificate. + CaCertRef types.NamespacedName + // Gateway is the name of the Gateway that is being checked for this BackendTLSPolicy. + Gateway types.NamespacedName + // Conditions include Conditions for the BackendTLSPolicy. + Conditions []conditions.Condition + // Valid shows whether the BackendTLSPolicy is valid. + Valid bool + // IsReferenced shows whether the BackendTLSPolicy is referenced by a BackendRef. + IsReferenced bool + // Ignored shows whether the BackendTLSPolicy is ignored. + Ignored bool +} + +func processBackendTLSPolicies( + backendTLSPolicies map[types.NamespacedName]*v1alpha2.BackendTLSPolicy, + configMapResolver *configMapResolver, + ctlrName string, + gateway *Gateway, +) map[types.NamespacedName]*BackendTLSPolicy { + if len(backendTLSPolicies) == 0 || gateway == nil { + return nil + } + + processedBackendTLSPolicies := make(map[types.NamespacedName]*BackendTLSPolicy, len(backendTLSPolicies)) + for nsname, backendTLSPolicy := range backendTLSPolicies { + var caCertRef types.NamespacedName + valid, ignored, conds := validateBackendTLSPolicy( + backendTLSPolicy, + configMapResolver, + ctlrName, + gateway, + ) + + if valid && !ignored && backendTLSPolicy.Spec.TLS.CACertRefs != nil { + caCertRef = types.NamespacedName{ + Namespace: backendTLSPolicy.Namespace, Name: string(backendTLSPolicy.Spec.TLS.CACertRefs[0].Name), + } + } + + processedBackendTLSPolicies[nsname] = &BackendTLSPolicy{ + Source: backendTLSPolicy, + Valid: valid, + Conditions: conds, + Gateway: types.NamespacedName{ + Namespace: gateway.Source.Namespace, + Name: gateway.Source.Name, + }, + CaCertRef: caCertRef, + Ignored: ignored, + } + } + return processedBackendTLSPolicies +} + +func validateBackendTLSPolicy( + backendTLSPolicy *v1alpha2.BackendTLSPolicy, + configMapResolver *configMapResolver, + ctlrName string, + gateway *Gateway, +) (valid, ignored bool, conds []conditions.Condition) { + valid = true + ignored = false + if err := validateAncestorMaxCount(backendTLSPolicy, ctlrName, gateway); err != nil { + valid = false + ignored = true + } + if err := validateBackendTLSHostname(backendTLSPolicy); err != nil { + valid = false + conds = append(conds, staticConds.NewBackendTLSPolicyInvalid(fmt.Sprintf("invalid hostname: %s", err.Error()))) + } + if backendTLSPolicy.Spec.TLS.CACertRefs != nil && backendTLSPolicy.Spec.TLS.WellKnownCACerts != nil { + valid = false + msg := "CACertRefs and WellKnownCACerts are mutually exclusive" + conds = append(conds, staticConds.NewBackendTLSPolicyInvalid(msg)) + } else if backendTLSPolicy.Spec.TLS.CACertRefs != nil && len(backendTLSPolicy.Spec.TLS.CACertRefs) > 0 { + if err := validateBackendTLSCACertRef(backendTLSPolicy, configMapResolver); err != nil { + valid = false + conds = append(conds, staticConds.NewBackendTLSPolicyInvalid( + fmt.Sprintf("invalid CACertRef: %s", err.Error()))) + } + } else if backendTLSPolicy.Spec.TLS.WellKnownCACerts != nil { + if err := validateBackendTLSWellKnownCACerts(backendTLSPolicy); err != nil { + valid = false + conds = append(conds, staticConds.NewBackendTLSPolicyInvalid( + fmt.Sprintf("invalid WellKnownCACerts: %s", err.Error()))) + } + } else { + valid = false + conds = append(conds, staticConds.NewBackendTLSPolicyInvalid("CACertRefs and WellKnownCACerts are both nil")) + } + return valid, ignored, conds +} + +func validateAncestorMaxCount(backendTLSPolicy *v1alpha2.BackendTLSPolicy, ctlrName string, gateway *Gateway) error { + var err error + if len(backendTLSPolicy.Status.Ancestors) >= 16 { + // check if we already are an ancestor on this policy. If we are, we are safe to continue. + ancestorRef := v1.ParentReference{ + Namespace: helpers.GetPointer((v1.Namespace)(gateway.Source.Namespace)), + Name: v1.ObjectName(gateway.Source.Name), + } + var alreadyAncestor bool + for _, ancestor := range backendTLSPolicy.Status.Ancestors { + if string(ancestor.ControllerName) == ctlrName && ancestor.AncestorRef.Name == ancestorRef.Name && + ancestor.AncestorRef.Namespace != nil && *ancestor.AncestorRef.Namespace == *ancestorRef.Namespace { + alreadyAncestor = true + break + } + } + if !alreadyAncestor { + err = errors.New("too many ancestors, cannot attach a new Gateway") + } + } + return err +} + +func validateBackendTLSHostname(btp *v1alpha2.BackendTLSPolicy) error { + h := string(btp.Spec.TLS.Hostname) + + if err := validateHostname(h); err != nil { + path := field.NewPath("tls.hostname") + valErr := field.Invalid(path, btp.Spec.TLS.Hostname, err.Error()) + return valErr + } + return nil +} + +func validateBackendTLSCACertRef(btp *v1alpha2.BackendTLSPolicy, configMapResolver *configMapResolver) error { + if len(btp.Spec.TLS.CACertRefs) != 1 { + path := field.NewPath("tls.cacertrefs") + valErr := field.TooMany(path, len(btp.Spec.TLS.CACertRefs), 1) + return valErr + } + if btp.Spec.TLS.CACertRefs[0].Kind != "ConfigMap" { + path := field.NewPath("tls.cacertrefs[0].kind") + valErr := field.NotSupported(path, btp.Spec.TLS.CACertRefs[0].Kind, []string{"ConfigMap"}) + return valErr + } + if btp.Spec.TLS.CACertRefs[0].Group != "" && btp.Spec.TLS.CACertRefs[0].Group != "core" { + path := field.NewPath("tls.cacertrefs[0].group") + valErr := field.NotSupported(path, btp.Spec.TLS.CACertRefs[0].Group, []string{"", "core"}) + return valErr + } + nsName := types.NamespacedName{Namespace: btp.Namespace, Name: string(btp.Spec.TLS.CACertRefs[0].Name)} + if err := configMapResolver.resolve(nsName); err != nil { + path := field.NewPath("tls.cacertrefs[0]") + return field.Invalid(path, btp.Spec.TLS.CACertRefs[0], err.Error()) + } + return nil +} + +func validateBackendTLSWellKnownCACerts(btp *v1alpha2.BackendTLSPolicy) error { + if *btp.Spec.TLS.WellKnownCACerts != v1alpha2.WellKnownCACertSystem { + path := field.NewPath("tls.wellknowncacerts") + return field.NotSupported(path, btp.Spec.TLS.WellKnownCACerts, []string{string(v1alpha2.WellKnownCACertSystem)}) + } + return nil +} diff --git a/internal/mode/static/state/graph/backend_tls_policy_test.go b/internal/mode/static/state/graph/backend_tls_policy_test.go new file mode 100644 index 0000000000..0cedce2222 --- /dev/null +++ b/internal/mode/static/state/graph/backend_tls_policy_test.go @@ -0,0 +1,412 @@ +package graph + +import ( + "testing" + + . "github.com/onsi/gomega" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" + "sigs.k8s.io/gateway-api/apis/v1alpha2" + + "github.com/nginxinc/nginx-gateway-fabric/internal/framework/helpers" +) + +func TestProcessBackendTLSPoliciesEmpty(t *testing.T) { + backendTLSPolicies := map[types.NamespacedName]*v1alpha2.BackendTLSPolicy{ + {Namespace: "test", Name: "tls-policy"}: { + ObjectMeta: metav1.ObjectMeta{ + Name: "tls-policy", + Namespace: "test", + }, + Spec: v1alpha2.BackendTLSPolicySpec{ + TargetRef: v1alpha2.PolicyTargetReferenceWithSectionName{ + PolicyTargetReference: v1alpha2.PolicyTargetReference{ + Kind: "Service", + Name: "service1", + Namespace: (*gatewayv1.Namespace)(helpers.GetPointer("test")), + }, + }, + TLS: v1alpha2.BackendTLSPolicyConfig{ + CACertRefs: []gatewayv1.LocalObjectReference{ + { + Kind: "ConfigMap", + Name: "configmap", + Group: "", + }, + }, + Hostname: "foo.test.com", + }, + }, + }, + } + + gateway := &Gateway{ + Source: &gatewayv1.Gateway{ObjectMeta: metav1.ObjectMeta{Name: "gateway", Namespace: "test"}}, + } + + tests := []struct { + expected map[types.NamespacedName]*BackendTLSPolicy + gateway *Gateway + backendTLSPolicies map[types.NamespacedName]*v1alpha2.BackendTLSPolicy + name string + }{ + { + name: "no policies", + expected: nil, + gateway: gateway, + backendTLSPolicies: nil, + }, + { + name: "nil gateway", + expected: nil, + backendTLSPolicies: backendTLSPolicies, + gateway: nil, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + g := NewWithT(t) + + processed := processBackendTLSPolicies(test.backendTLSPolicies, nil, "test", test.gateway) + + g.Expect(processed).To(Equal(test.expected)) + }) + } +} + +func TestValidateBackendTLSPolicy(t *testing.T) { + targetRefNormalCase := &v1alpha2.PolicyTargetReferenceWithSectionName{ + PolicyTargetReference: v1alpha2.PolicyTargetReference{ + Kind: "Service", + Name: "service1", + Namespace: (*gatewayv1.Namespace)(helpers.GetPointer("test")), + }, + } + + localObjectRefNormalCase := []gatewayv1.LocalObjectReference{ + { + Kind: "ConfigMap", + Name: "configmap", + Group: "", + }, + } + + localObjectRefInvalidName := []gatewayv1.LocalObjectReference{ + { + Kind: "ConfigMap", + Name: "invalid", + Group: "", + }, + } + + localObjectRefInvalidKind := []gatewayv1.LocalObjectReference{ + { + Kind: "Secret", + Name: "secret", + Group: "", + }, + } + + localObjectRefInvalidGroup := []gatewayv1.LocalObjectReference{ + { + Kind: "ConfigMap", + Name: "configmap", + Group: "bhu", + }, + } + + localObjectRefTooManyCerts := append(localObjectRefNormalCase, localObjectRefInvalidName...) + + getAncestorRef := func(ctlrName, parentName string) v1alpha2.PolicyAncestorStatus { + return v1alpha2.PolicyAncestorStatus{ + ControllerName: gatewayv1.GatewayController(ctlrName), + AncestorRef: gatewayv1.ParentReference{ + Name: gatewayv1.ObjectName(parentName), + Namespace: helpers.GetPointer(gatewayv1.Namespace("test")), + }, + } + } + + ancestors := []v1alpha2.PolicyAncestorStatus{ + getAncestorRef("not-us", "not-us"), + getAncestorRef("not-us", "not-us"), + getAncestorRef("not-us", "not-us"), + getAncestorRef("not-us", "not-us"), + getAncestorRef("not-us", "not-us"), + getAncestorRef("not-us", "not-us"), + getAncestorRef("not-us", "not-us"), + getAncestorRef("not-us", "not-us"), + getAncestorRef("not-us", "not-us"), + getAncestorRef("not-us", "not-us"), + getAncestorRef("not-us", "not-us"), + getAncestorRef("not-us", "not-us"), + getAncestorRef("not-us", "not-us"), + getAncestorRef("not-us", "not-us"), + getAncestorRef("not-us", "not-us"), + getAncestorRef("not-us", "not-us"), + } + + ancestorsWithUs := append(ancestors, getAncestorRef("test", "gateway")) + + tests := []struct { + tlsPolicy *v1alpha2.BackendTLSPolicy + gateway *Gateway + name string + isValid bool + ignored bool + }{ + { + name: "normal case with ca cert refs", + tlsPolicy: &v1alpha2.BackendTLSPolicy{ + ObjectMeta: metav1.ObjectMeta{ + Name: "tls-policy", + Namespace: "test", + }, + Spec: v1alpha2.BackendTLSPolicySpec{ + TargetRef: *targetRefNormalCase, + TLS: v1alpha2.BackendTLSPolicyConfig{ + CACertRefs: localObjectRefNormalCase, + Hostname: "foo.test.com", + }, + }, + }, + isValid: true, + }, + { + name: "normal case with ca cert refs and 16 ancestors including us", + tlsPolicy: &v1alpha2.BackendTLSPolicy{ + ObjectMeta: metav1.ObjectMeta{ + Name: "tls-policy", + Namespace: "test", + }, + Spec: v1alpha2.BackendTLSPolicySpec{ + TargetRef: *targetRefNormalCase, + TLS: v1alpha2.BackendTLSPolicyConfig{ + CACertRefs: localObjectRefNormalCase, + Hostname: "foo.test.com", + }, + }, + Status: v1alpha2.PolicyStatus{ + Ancestors: ancestorsWithUs, + }, + }, + isValid: true, + }, + { + name: "normal case with well known certs", + tlsPolicy: &v1alpha2.BackendTLSPolicy{ + ObjectMeta: metav1.ObjectMeta{ + Name: "tls-policy", + Namespace: "test", + }, + Spec: v1alpha2.BackendTLSPolicySpec{ + TargetRef: *targetRefNormalCase, + TLS: v1alpha2.BackendTLSPolicyConfig{ + WellKnownCACerts: (helpers.GetPointer(v1alpha2.WellKnownCACertSystem)), + Hostname: "foo.test.com", + }, + }, + }, + isValid: true, + }, + { + name: "no hostname invalid case", + tlsPolicy: &v1alpha2.BackendTLSPolicy{ + ObjectMeta: metav1.ObjectMeta{ + Name: "tls-policy", + Namespace: "test", + }, + Spec: v1alpha2.BackendTLSPolicySpec{ + TargetRef: *targetRefNormalCase, + TLS: v1alpha2.BackendTLSPolicyConfig{ + CACertRefs: localObjectRefNormalCase, + Hostname: "", + }, + }, + }, + }, + { + name: "invalid ca cert ref name", + tlsPolicy: &v1alpha2.BackendTLSPolicy{ + ObjectMeta: metav1.ObjectMeta{ + Name: "tls-policy", + Namespace: "test", + }, + Spec: v1alpha2.BackendTLSPolicySpec{ + TargetRef: *targetRefNormalCase, + TLS: v1alpha2.BackendTLSPolicyConfig{ + CACertRefs: localObjectRefInvalidName, + Hostname: "foo.test.com", + }, + }, + }, + }, + { + name: "invalid ca cert ref kind", + tlsPolicy: &v1alpha2.BackendTLSPolicy{ + ObjectMeta: metav1.ObjectMeta{ + Name: "tls-policy", + Namespace: "test", + }, + Spec: v1alpha2.BackendTLSPolicySpec{ + TargetRef: *targetRefNormalCase, + TLS: v1alpha2.BackendTLSPolicyConfig{ + CACertRefs: localObjectRefInvalidKind, + Hostname: "foo.test.com", + }, + }, + }, + }, + { + name: "invalid ca cert ref group", + tlsPolicy: &v1alpha2.BackendTLSPolicy{ + ObjectMeta: metav1.ObjectMeta{ + Name: "tls-policy", + Namespace: "test", + }, + Spec: v1alpha2.BackendTLSPolicySpec{ + TargetRef: *targetRefNormalCase, + TLS: v1alpha2.BackendTLSPolicyConfig{ + CACertRefs: localObjectRefInvalidGroup, + Hostname: "foo.test.com", + }, + }, + }, + }, + { + name: "invalid case with well known certs", + tlsPolicy: &v1alpha2.BackendTLSPolicy{ + ObjectMeta: metav1.ObjectMeta{ + Name: "tls-policy", + Namespace: "test", + }, + Spec: v1alpha2.BackendTLSPolicySpec{ + TargetRef: *targetRefNormalCase, + TLS: v1alpha2.BackendTLSPolicyConfig{ + WellKnownCACerts: (helpers.GetPointer(v1alpha2.WellKnownCACertType("unknown"))), + Hostname: "foo.test.com", + }, + }, + }, + }, + { + name: "invalid case neither TLS config option chosen", + tlsPolicy: &v1alpha2.BackendTLSPolicy{ + ObjectMeta: metav1.ObjectMeta{ + Name: "tls-policy", + Namespace: "test", + }, + Spec: v1alpha2.BackendTLSPolicySpec{ + TargetRef: *targetRefNormalCase, + TLS: v1alpha2.BackendTLSPolicyConfig{ + Hostname: "foo.test.com", + }, + }, + }, + }, + { + name: "invalid case with too many ca cert refs", + tlsPolicy: &v1alpha2.BackendTLSPolicy{ + ObjectMeta: metav1.ObjectMeta{ + Name: "tls-policy", + Namespace: "test", + }, + Spec: v1alpha2.BackendTLSPolicySpec{ + TargetRef: *targetRefNormalCase, + TLS: v1alpha2.BackendTLSPolicyConfig{ + CACertRefs: localObjectRefTooManyCerts, + Hostname: "foo.test.com", + }, + }, + }, + }, + { + name: "invalid case with too both ca cert refs and wellknowncerts", + tlsPolicy: &v1alpha2.BackendTLSPolicy{ + ObjectMeta: metav1.ObjectMeta{ + Name: "tls-policy", + Namespace: "test", + }, + Spec: v1alpha2.BackendTLSPolicySpec{ + TargetRef: *targetRefNormalCase, + TLS: v1alpha2.BackendTLSPolicyConfig{ + CACertRefs: localObjectRefNormalCase, + Hostname: "foo.test.com", + WellKnownCACerts: (helpers.GetPointer(v1alpha2.WellKnownCACertSystem)), + }, + }, + }, + }, + { + name: "invalid case with too many ancestors", + tlsPolicy: &v1alpha2.BackendTLSPolicy{ + ObjectMeta: metav1.ObjectMeta{ + Name: "tls-policy", + Namespace: "test", + }, + Spec: v1alpha2.BackendTLSPolicySpec{ + TargetRef: *targetRefNormalCase, + TLS: v1alpha2.BackendTLSPolicyConfig{ + CACertRefs: localObjectRefNormalCase, + Hostname: "foo.test.com", + }, + }, + Status: v1alpha2.PolicyStatus{ + Ancestors: ancestors, + }, + }, + ignored: true, + }, + } + + configMaps := map[types.NamespacedName]*v1.ConfigMap{ + {Namespace: "test", Name: "configmap"}: { + ObjectMeta: metav1.ObjectMeta{ + Name: "configmap", + Namespace: "test", + }, + Data: map[string]string{ + "ca.crt": caBlock, + }, + }, + {Namespace: "test", Name: "invalid"}: { + ObjectMeta: metav1.ObjectMeta{ + Name: "invalid", + Namespace: "test", + }, + Data: map[string]string{ + "ca.crt": "invalid", + }, + }, + } + + configMapResolver := newConfigMapResolver(configMaps) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + g := NewWithT(t) + + gateway := &Gateway{ + Source: &gatewayv1.Gateway{ObjectMeta: metav1.ObjectMeta{Name: "gateway", Namespace: "test"}}, + } + + valid, ignored, conds := validateBackendTLSPolicy( + test.tlsPolicy, + configMapResolver, + "test", + gateway, + ) + + g.Expect(valid).To(Equal(test.isValid)) + g.Expect(ignored).To(Equal(test.ignored)) + if !test.isValid && !test.ignored { + g.Expect(conds).To(HaveLen(1)) + } else { + g.Expect(conds).To(HaveLen(0)) + } + }) + } +} diff --git a/internal/mode/static/state/graph/configmaps.go b/internal/mode/static/state/graph/configmaps.go new file mode 100644 index 0000000000..23bec08c63 --- /dev/null +++ b/internal/mode/static/state/graph/configmaps.go @@ -0,0 +1,122 @@ +package graph + +import ( + "crypto/x509" + "encoding/base64" + "encoding/pem" + "errors" + "fmt" + + apiv1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" +) + +const CAKey = "ca.crt" + +// CaCertConfigMap represents a ConfigMap resource that holds CA Cert data. +type CaCertConfigMap struct { + // Source holds the actual ConfigMap resource. Can be nil if the ConfigMap does not exist. + Source *apiv1.ConfigMap + // CACert holds the actual CA Cert data. + CACert []byte +} + +type caCertConfigMapEntry struct { + // err holds the corresponding error if the ConfigMap is invalid or does not exist. + err error + caCertConfigMap CaCertConfigMap +} + +// configMapResolver wraps the cluster ConfigMaps so that they can be resolved (includes validation). All resolved +// ConfigMaps are saved to be used later. +type configMapResolver struct { + clusterConfigMaps map[types.NamespacedName]*apiv1.ConfigMap + resolvedCaCertConfigMaps map[types.NamespacedName]*caCertConfigMapEntry +} + +func newConfigMapResolver(configMaps map[types.NamespacedName]*apiv1.ConfigMap) *configMapResolver { + return &configMapResolver{ + clusterConfigMaps: configMaps, + resolvedCaCertConfigMaps: make(map[types.NamespacedName]*caCertConfigMapEntry), + } +} + +func (r *configMapResolver) resolve(nsname types.NamespacedName) error { + if s, resolved := r.resolvedCaCertConfigMaps[nsname]; resolved { + return s.err + } + + cm, exist := r.clusterConfigMaps[nsname] + + var validationErr error + var caCert []byte + + if !exist { + validationErr = errors.New("ConfigMap does not exist") + } else { + if cm.Data != nil { + if _, exists := cm.Data[CAKey]; exists { + validationErr = validateCA([]byte(cm.Data[CAKey])) + caCert = []byte(cm.Data[CAKey]) + } + } + if cm.BinaryData != nil { + if _, exists := cm.BinaryData[CAKey]; exists { + validationErr = validateCA(cm.BinaryData[CAKey]) + caCert = cm.BinaryData[CAKey] + } + } + if len(caCert) == 0 { + validationErr = fmt.Errorf("ConfigMap does not have the data or binaryData field %v", CAKey) + } + } + + r.resolvedCaCertConfigMaps[nsname] = &caCertConfigMapEntry{ + caCertConfigMap: CaCertConfigMap{ + Source: cm, + CACert: caCert, + }, + err: validationErr, + } + + return validationErr +} + +func (r *configMapResolver) getResolvedConfigMaps() map[types.NamespacedName]*CaCertConfigMap { + if len(r.resolvedCaCertConfigMaps) == 0 { + return nil + } + + resolved := make(map[types.NamespacedName]*CaCertConfigMap) + + for nsname, entry := range r.resolvedCaCertConfigMaps { + // create iteration variable inside the loop to fix implicit memory aliasing + caCertConfigMap := entry.caCertConfigMap + resolved[nsname] = &caCertConfigMap + } + + return resolved +} + +// validateCA validates the ca.crt entry in the ConfigMap. If it is valid, the function returns nil. +func validateCA(caData []byte) error { + data := make([]byte, base64.StdEncoding.DecodedLen(len(caData))) + _, err := base64.StdEncoding.Decode(data, caData) + if err != nil { + data = caData + } + block, _ := pem.Decode(data) + if block == nil { + return fmt.Errorf("the data field %s must hold a valid CERTIFICATE PEM block", CAKey) + } + if block.Type != "CERTIFICATE" { + return fmt.Errorf("the data field %s must hold a valid CERTIFICATE PEM block, but got '%s'", CAKey, block.Type) + } + + _, err = x509.ParseCertificate(block.Bytes) + if err != nil { + return fmt.Errorf("failed to validate certificate: %w", err) + } + + return nil +} diff --git a/internal/mode/static/state/graph/configmaps_test.go b/internal/mode/static/state/graph/configmaps_test.go new file mode 100644 index 0000000000..4e9430c510 --- /dev/null +++ b/internal/mode/static/state/graph/configmaps_test.go @@ -0,0 +1,221 @@ +package graph + +import ( + "encoding/base64" + "testing" + + . "github.com/onsi/gomega" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" +) + +const ( + caBlock = `-----BEGIN CERTIFICATE----- +MIIDSDCCAjACCQDKWvrpwiIyCDANBgkqhkiG9w0BAQsFADBmMQswCQYDVQQGEwJV +UzELMAkGA1UECAwCQ0ExFjAUBgNVBAcMDVNhbiBGcmFuc2lzY28xDjAMBgNVBAoM +BU5HSU5YMQwwCgYDVQQLDANLSUMxFDASBgNVBAMMC2V4YW1wbGUuY29tMB4XDTIw +MTExMjIxMjg0MloXDTMwMTExMDIxMjg0MlowZjELMAkGA1UEBhMCVVMxCzAJBgNV +BAgMAkNBMRYwFAYDVQQHDA1TYW4gRnJhbnNpc2NvMQ4wDAYDVQQKDAVOR0lOWDEM +MAoGA1UECwwDS0lDMRQwEgYDVQQDDAtleGFtcGxlLmNvbTCCASIwDQYJKoZIhvcN +AQEBBQADggEPADCCAQoCggEBAMrlKMqrHfMR4mgaL2zZG2DYYfKCFVmINjlYuOeC +FDTcRgQKtu2YcCxZYBADwHZxEf6NIKtVsMWLhSNS/Nc0BmtiQM/IExhlCiDC6Sl8 +ONrI3w7qJzN6IUERB6tVlQt07rgM0V26UTYu0Ikv1Y8trfLYPZckzBkorQjpcium +qoP2BJf4yyc9LqpxtlWKxelkunVL5ijMEzpj9gEE26TEHbsdEbhoR8g0OeHZqH7e +mXCnSIBR0A/o/s6noGNX+F19lY7Tgw77jOuQQ5Ysi+7nhN2lKvcC819RX7oMpgvt +V5B3nI0mF6BaznjeTs4yQcr1Sm3UTVBwX9ZuvL7RbIXkUm8CAwEAATANBgkqhkiG +9w0BAQsFAAOCAQEAgm04w6OIWGj6tka9ccccnblF0oZzeEAIywjvR5sDcPdvLIeM +eesJy6rFH4DBmMygpcIxJGrSOzZlF3LMvw7zK4stqNtm1HiprF8bzxfTffVYncg6 +hVKErHtZ2FZRj/2TMJ01aRDZSuVbL6UJiokpU6xxT7yy0dFZkKrjUR349gKxRqJw +Am2as0bhi51EqK1GEx3m4c0un2vNh5qP2hv6e/Qze6P96vefNaSk9QMFfuB1kSAk +fGpkiL7bjmjnhKwAmf8jDWDZltB6S56Qy2QjPR8JoOusbYxar4c6EcIwVHv6mdgP +yZxWqQsgtSfFx+Pwon9IPKuq0jQYgeZPSxRMLA== +-----END CERTIFICATE----- +` + caBlockInvalidType = `-----BEGIN PRIVATE KEY----- +MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQCas08k/NzwAGNC +RgTPPF/gKd2K2gP13jvPmpPf1BMFyn+bGEyHRP81cqSHKoatigrR/+rvwTnzbt/X +pyjSelom3OhIOje64Kqi7uaFmGESxjz1C02IbVNLfNyNi1WCaX5U3Wf7u3F+K6Lf +tCSnvg75lkXje9ZiYib6o5/X/ZzZDQ0ryqg9+7CjufDmDfRFs47rp1Lj+VS3+PDP +kGn6f/jD1Q/o0tn44KIjU/gv1F+NnYIpZDixBZwtWQqeVqv5ngiYmhXFTfYCDzFL +34iEcZqWoN99X8zW8itUMVaS2DKcYp29/Gpj9q+Ub9VOnGX1Y2MJ9hUKZJBv++n9 +M3trwJrXkh5XDz7ya4TyP+8sSuIyJ4VQsv1/d0ZSshFw2/6p9NDUABOcBa9RZmrS +shp4sxtiY3xQBOZoAEajFFEwZeILsI7cz9UrISXbXbLOOoIr3aEbPbHfSPPP5oJn +106srUJVnGdYIUY1dGbzMgNttHd+5SlvxPUPM/WlucMSb4CXpJEIIAcYqNlZFznA +ojMYwKVaHFWvY0QUVNg6iMgNBTNnSAK/p23OrzvOVdKIinomXMyAF2ctvJ4Q5qPl +RNakP+W8pNZ+T+sNJ1HAZ4WZ53sAbTioi6c1LIcr99pvo9v7oEWkV5fPGjhLp0Rw +sK7wCos2u+C0E1tMK3KlnwmQ740J0QIDAQABAoICAATSkCYMB+snZ/C59A5tyGNZ +isF4WGVCv0SSggeZOdqVXHL+R+xzly0YXM6l4brpMbsoKi+9K0xOaYX0fQ5KqCLM +AiW2QuR9enRH1EHX5TbLnTzaVFlrZwxUYR+8dzbwiPKmUEaFql0PiS1GFVpxT1Ay +gg08YAuDGcn4bdQy4L/Xa1CxKZt9DB2ef0b8ql+94DeyaKAYtq5hgUhHLTaU5LFe +I/fTEt5ySjuls3fyO+RTQ6p8qFPEZAD55J3Y/9VxOr1fGEylSIT56kR+PGg8jmAh +tbXX1a/hrr4aJ6O+P52maVpx0vM4znJnJhQkRf1nUsANvswrJGGJTdsJztAmGe2Q +BMwMi78B9veg7bB85Orn/ZaumiOkgaK2Qsv8wQXCIbQ1yBypzKDggjHJDL999LsI +rvNDErraz/1CyFaenp+mXLMikODq4j1ArHrF+J9YbkGGZYejPwrxXN6i6w3HngJP +C8MxxBRKs9Oi6q746hjQrepnYO5HgFA2CclS1bvC9B7UPgy6kRNjD6XSxO7Wcjyr +eI34xj9UuotTtw9Gf0CjY2s2ggjkHipRryQVyPNB4yP7+4P/y8DTyHjfTkSJRV7G +CDHLpcvECd2d5oLTxlzOGM9fCTalMyN7c84Y6VqsViOoLU1Lvph/+B2rT++ZKNqS +qyYgYZJs8/59O+1i6Yz9AoIBAQDKQCdYbLVnZ22ozJA4N/N+5aehpSZkpC/DiZ10 +mwi8RVqaOIoxbvsw80ZwoBn5fcv2H6pCAqzUav9jT23NOqNCKpVzLWjgNHtO7aiU +KT5cCHCcpHvnWgBLrM9EsrSdra2HuiwDIrzxlnpkOITdzI/oXUer1dPOt3yc0Bz+ +lAKw/54bu3qNYWH1gteSdAWYt/AK5bbBD9Q3bogAt8zN9XOfDEx+GPqClCa2L9yC +tMuVcPyk078mS+7iEyJzWC4PIZtMikVMMOXi344DnNWh8bolWnrpfB6hr9R4nqzw +P7Mn4VyZDZApNzkBIvsoyFvkEh7uOrOaz9DYmp3OrNtVN06jAoIBAQDD0CTFAajw +0kKRNLoSvVD3ANBDvZCAnflX2V5sppqvhwuxwDLLsadj0juHNOH1G6WJjsbW0HFs +aPmuDLyWPh4AVE13+GUuYMFOVXGHWONZjGRgQyPhE7W9sWH3RMs+GHzX5OdCMT9G +Bq/YZ04i2FQDGLVH8cnwgjzeC7lXetrJOrrLK8sj43vQQuQ/ZKc4VUdFCQoinX6F +LovHi42VyCWzu1r8kOz3RHuo+cncyVvtRnpo/XFuIO9TuKbHE3hg5TSXdLfYC+0l +apirUU5Sq2kO5uQZIruEum+bZCpdd/8Ua8ynfSeg8oG5edhX9UAu7+qgss8IrzfX +3b06ca7bQFD7AoIBAQCdTWBMqeA9WHg1vUS+NOYxYDUMyAIgbIKptrK8KoiUxew9 +3pO89vBvlgbHOf55yZmFCAPH64S4ga+4ceKYqG6p26z5M+xJ1QfCz505/wn9UqMj +cdrciWeJdBKQ/9zydk5tLiNlHPOPgtYWdM8CI0QaGdLQlzJxqMxGuqaSalPdjjJO +p3Yd2Av0g5te0NY5fXY5Q4jsh38qzdEBnfKwjaMrpMkpmgvc25VwRbFgB3X/+SzG +ldop0w0s0G0PARpxslWzJifXpoBmADHYJXcSyYtZ2hGW326DmtnKJr+i7ChPcDww +3hettsGjXK2zfoHZ1S4xY36lfdSVY0wxnsfIc4e5AoIBAG/NKSFe6EHQG3fi/hbz +BwZw7XiwBJCbIiHZl4M7wPhViATOc3JAFg31nE1/kUAsr+CRp9BBJXG7okuRNCAo +iWKwv6avKb5IOjbqrC6WPwEDGtCnpRW+9ja/z+qp2c2zl5yBMtVlXvYxnTdXDJLy +p005T1ArqpxrECvLz+A14jOhF8QnVg5AtZHcj4vugVe1wUKWfbXz7KhIQkEF2ipa +I8SyRaoNaW9pJ538ORiZ06XvZrcJdjlmDp/jvz3NTR8t31BWsR1m+dkyOsceXjTv +b8W1aSk83opTFKRJlbLWb8sOHcTHvde0fwMSocbe3e2uyG1GitUvjhfvoDp9bFP9 +Lf8CggEAGFhWv/+Iur0Sq5DZskChe9BEJp7P/I8VmvI8bT/0LRepkvFt6iAQjyAP +07EQ2ujeQ6BrGeGwNoA3ha49KarBX6OE26pRxUWFLU8Ab74yfycZVAIeUwG2e6p7 +uQy9GGkjWWQ+0eL5UwTjj8D/bors+6rgfUH1iarZ/HxP2boxdJJrj59+R5/DRg7M +zIpoWIuspSbo6AVK8H778qfb6f95oAxRgbahq3jpR0O1ZpDJxja7PC1Bs/hsabjH +atIGfDRw+YXfJBgy43hfbJXTLZJ2cLaKA6xc3HbGEuLwtx9MktjY/4xuUS5aOY35 +UdxohGqleWFMQ3UNLOvc9Fk+q72ryg== +-----END PRIVATE KEY----- +` +) + +func TestValidateCA(t *testing.T) { + base64Data := make([]byte, base64.StdEncoding.EncodedLen(len(caBlock))) + base64.StdEncoding.Encode(base64Data, []byte(caBlock)) + + tests := []struct { + name string + data []byte + errorExpected bool + }{ + { + name: "valid base64", + data: base64Data, + errorExpected: false, + }, + { + name: "valid plain text", + data: []byte(caBlock), + errorExpected: false, + }, + { + name: "invalid pem", + data: []byte("invalid"), + errorExpected: true, + }, + { + name: "invalid type", + data: []byte(caBlockInvalidType), + errorExpected: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + g := NewWithT(t) + + err := validateCA(test.data) + if test.errorExpected { + g.Expect(err).To(HaveOccurred()) + } else { + g.Expect(err).ToNot(HaveOccurred()) + } + }) + } +} + +func TestResolve(t *testing.T) { + configMaps := map[types.NamespacedName]*v1.ConfigMap{ + {Namespace: "test", Name: "configmap1"}: { + ObjectMeta: metav1.ObjectMeta{ + Name: "configmap1", + Namespace: "test", + }, + Data: map[string]string{ + "ca.crt": caBlock, + }, + }, + {Namespace: "test", Name: "configmap2"}: { + ObjectMeta: metav1.ObjectMeta{ + Name: "configmap2", + Namespace: "test", + }, + BinaryData: map[string][]byte{ + "ca.crt": []byte(caBlock), + }, + }, + {Namespace: "test", Name: "invalid"}: { + ObjectMeta: metav1.ObjectMeta{ + Name: "invalid", + Namespace: "test", + }, + Data: map[string]string{ + "ca.crt": "invalid", + }, + }, + {Namespace: "test", Name: "nocaentry"}: { + ObjectMeta: metav1.ObjectMeta{ + Name: "nocaentry", + Namespace: "test", + }, + Data: map[string]string{ + "noca.crt": "something else", + }, + }, + } + + configMapResolver := newConfigMapResolver(configMaps) + + tests := []struct { + name string + nsname types.NamespacedName + errorExpected bool + }{ + { + name: "valid configmap1", + nsname: types.NamespacedName{Namespace: "test", Name: "configmap1"}, + errorExpected: false, + }, + { + name: "valid configmap2", + nsname: types.NamespacedName{Namespace: "test", Name: "configmap2"}, + errorExpected: false, + }, + { + name: "invalid configmap", + nsname: types.NamespacedName{Namespace: "test", Name: "invalid"}, + errorExpected: true, + }, + { + name: "non-existent configmap", + nsname: types.NamespacedName{Namespace: "test", Name: "non-existent"}, + errorExpected: true, + }, + { + name: "configmap missing ca entry", + nsname: types.NamespacedName{Namespace: "test", Name: "nocaentry"}, + errorExpected: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + g := NewWithT(t) + + err := configMapResolver.resolve(test.nsname) + if test.errorExpected { + g.Expect(err).To(HaveOccurred()) + } else { + g.Expect(err).ToNot(HaveOccurred()) + } + }) + } +} diff --git a/internal/mode/static/state/graph/graph.go b/internal/mode/static/state/graph/graph.go index 556c38496b..f6ee5598b9 100644 --- a/internal/mode/static/state/graph/graph.go +++ b/internal/mode/static/state/graph/graph.go @@ -7,6 +7,7 @@ import ( "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" + "sigs.k8s.io/gateway-api/apis/v1alpha2" "sigs.k8s.io/gateway-api/apis/v1beta1" "github.com/nginxinc/nginx-gateway-fabric/internal/framework/controller/index" @@ -15,14 +16,16 @@ import ( // ClusterState includes cluster resources necessary to build the Graph. type ClusterState struct { - GatewayClasses map[types.NamespacedName]*gatewayv1.GatewayClass - Gateways map[types.NamespacedName]*gatewayv1.Gateway - HTTPRoutes map[types.NamespacedName]*gatewayv1.HTTPRoute - Services map[types.NamespacedName]*v1.Service - Namespaces map[types.NamespacedName]*v1.Namespace - ReferenceGrants map[types.NamespacedName]*v1beta1.ReferenceGrant - Secrets map[types.NamespacedName]*v1.Secret - CRDMetadata map[types.NamespacedName]*metav1.PartialObjectMetadata + GatewayClasses map[types.NamespacedName]*gatewayv1.GatewayClass + Gateways map[types.NamespacedName]*gatewayv1.Gateway + HTTPRoutes map[types.NamespacedName]*gatewayv1.HTTPRoute + Services map[types.NamespacedName]*v1.Service + Namespaces map[types.NamespacedName]*v1.Namespace + ReferenceGrants map[types.NamespacedName]*v1beta1.ReferenceGrant + Secrets map[types.NamespacedName]*v1.Secret + CRDMetadata map[types.NamespacedName]*metav1.PartialObjectMetadata + BackendTLSPolicies map[types.NamespacedName]*v1alpha2.BackendTLSPolicy + ConfigMaps map[types.NamespacedName]*v1.ConfigMap } // Graph is a Graph-like representation of Gateway API resources. @@ -51,6 +54,10 @@ type Graph struct { // ReferencedServices includes the NamespacedNames of all the Services that are referenced by at least one HTTPRoute. // Storing the whole resource is not necessary, compared to the similar maps above. ReferencedServices map[types.NamespacedName]struct{} + // ReferencedCaCertConfigMaps includes ConfigMaps that have been referenced by any BackendTLSPolicies. + ReferencedCaCertConfigMaps map[types.NamespacedName]*CaCertConfigMap + // BackendTLSPolicies holds BackendTLSPolicy resources. + BackendTLSPolicies map[types.NamespacedName]*BackendTLSPolicy } // ProtectedPorts are the ports that may not be configured by a listener with a descriptive name of each port. @@ -62,6 +69,9 @@ func (g *Graph) IsReferenced(resourceType client.Object, nsname types.Namespaced case *v1.Secret: _, exists := g.ReferencedSecrets[nsname] return exists + case *v1.ConfigMap: + _, exists := g.ReferencedCaCertConfigMaps[nsname] + return exists case *v1.Namespace: // `existed` is needed as it checks the graph's ReferencedNamespaces which stores all the namespaces that // match the Gateway listener's label selector when the graph was created. This covers the case when @@ -111,29 +121,39 @@ func BuildGraph( gc := buildGatewayClass(processedGwClasses.Winner, state.CRDMetadata) secretResolver := newSecretResolver(state.Secrets) + configMapResolver := newConfigMapResolver(state.ConfigMaps) processedGws := processGateways(state.Gateways, gcName) refGrantResolver := newReferenceGrantResolver(state.ReferenceGrants) gw := buildGateway(processedGws.Winner, secretResolver, gc, refGrantResolver, protectedPorts) + processedBackendTLSPolicies := processBackendTLSPolicies( + state.BackendTLSPolicies, + configMapResolver, + controllerName, + gw, + ) + routes := buildRoutesForGateways(validators.HTTPFieldsValidator, state.HTTPRoutes, processedGws.GetAllNsNames()) bindRoutesToListeners(routes, gw, state.Namespaces) - addBackendRefsToRouteRules(routes, refGrantResolver, state.Services) + addBackendRefsToRouteRules(routes, refGrantResolver, state.Services, processedBackendTLSPolicies) referencedNamespaces := buildReferencedNamespaces(state.Namespaces, gw) referencedServices := buildReferencedServices(routes) g := &Graph{ - GatewayClass: gc, - Gateway: gw, - Routes: routes, - IgnoredGatewayClasses: processedGwClasses.Ignored, - IgnoredGateways: processedGws.Ignored, - ReferencedSecrets: secretResolver.getResolvedSecrets(), - ReferencedNamespaces: referencedNamespaces, - ReferencedServices: referencedServices, + GatewayClass: gc, + Gateway: gw, + Routes: routes, + IgnoredGatewayClasses: processedGwClasses.Ignored, + IgnoredGateways: processedGws.Ignored, + ReferencedSecrets: secretResolver.getResolvedSecrets(), + ReferencedNamespaces: referencedNamespaces, + ReferencedServices: referencedServices, + ReferencedCaCertConfigMaps: configMapResolver.getResolvedConfigMaps(), + BackendTLSPolicies: processedBackendTLSPolicies, } return g diff --git a/internal/mode/static/state/graph/graph_test.go b/internal/mode/static/state/graph/graph_test.go index ed10c01d51..8a278d673a 100644 --- a/internal/mode/static/state/graph/graph_test.go +++ b/internal/mode/static/state/graph/graph_test.go @@ -12,10 +12,13 @@ import ( "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" + "sigs.k8s.io/gateway-api/apis/v1alpha2" "sigs.k8s.io/gateway-api/apis/v1beta1" + "github.com/nginxinc/nginx-gateway-fabric/internal/framework/conditions" "github.com/nginxinc/nginx-gateway-fabric/internal/framework/controller/index" "github.com/nginxinc/nginx-gateway-fabric/internal/framework/helpers" + staticConds "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/state/conditions" "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/state/validation" "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/state/validation/validationfakes" ) @@ -90,21 +93,72 @@ func TestBuildGraph(t *testing.T) { hr2 := createRoute("hr-2", "wrong-gateway", "listener-80-1") hr3 := createRoute("hr-3", "gateway-1", "listener-443-1") // https listener; should not conflict with hr1 + cm := &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "configmap", + Namespace: "service", + }, + Data: map[string]string{ + "ca.crt": caBlock, + }, + } + + btpAcceptedConds := []conditions.Condition{ + staticConds.NewBackendTLSPolicyAccepted(), + staticConds.NewBackendTLSPolicyAccepted(), + } + + btp := BackendTLSPolicy{ + Source: &v1alpha2.BackendTLSPolicy{ + ObjectMeta: metav1.ObjectMeta{ + Name: "btp", + Namespace: "service", + }, + Spec: v1alpha2.BackendTLSPolicySpec{ + TargetRef: v1alpha2.PolicyTargetReferenceWithSectionName{ + PolicyTargetReference: v1alpha2.PolicyTargetReference{ + Group: "", + Kind: "Service", + Name: "foo", + Namespace: (*gatewayv1.Namespace)(helpers.GetPointer("service")), + }, + }, + TLS: v1alpha2.BackendTLSPolicyConfig{ + Hostname: "foo.example.com", + CACertRefs: []v1alpha2.LocalObjectReference{ + { + Kind: "ConfigMap", + Name: "configmap", + Group: "", + }, + }, + }, + }, + }, + Valid: true, + IsReferenced: true, + Gateway: types.NamespacedName{Namespace: "test", Name: "gateway-1"}, + Conditions: btpAcceptedConds, + CaCertRef: types.NamespacedName{Namespace: "service", Name: "configmap"}, + } + hr1Refs := []BackendRef{ { - SvcNsName: types.NamespacedName{Namespace: "service", Name: "foo"}, - ServicePort: v1.ServicePort{Port: 80}, - Valid: true, - Weight: 1, + SvcNsName: types.NamespacedName{Namespace: "service", Name: "foo"}, + ServicePort: v1.ServicePort{Port: 80}, + Valid: true, + Weight: 1, + BackendTLSPolicy: &btp, }, } hr3Refs := []BackendRef{ { - SvcNsName: types.NamespacedName{Namespace: "service", Name: "foo"}, - ServicePort: v1.ServicePort{Port: 80}, - Valid: true, - Weight: 1, + SvcNsName: types.NamespacedName{Namespace: "service", Name: "foo"}, + ServicePort: v1.ServicePort{Port: 80}, + Valid: true, + Weight: 1, + BackendTLSPolicy: &btp, }, } @@ -261,6 +315,12 @@ func TestBuildGraph(t *testing.T) { Secrets: map[types.NamespacedName]*v1.Secret{ client.ObjectKeyFromObject(secret): secret, }, + BackendTLSPolicies: map[types.NamespacedName]*v1alpha2.BackendTLSPolicy{ + client.ObjectKeyFromObject(btp.Source): btp.Source, + }, + ConfigMaps: map[types.NamespacedName]*v1.ConfigMap{ + client.ObjectKeyFromObject(cm): cm, + }, } } @@ -350,6 +410,15 @@ func TestBuildGraph(t *testing.T) { ReferencedServices: map[types.NamespacedName]struct{}{ client.ObjectKeyFromObject(svc): {}, }, + ReferencedCaCertConfigMaps: map[types.NamespacedName]*CaCertConfigMap{ + client.ObjectKeyFromObject(cm): { + Source: cm, + CACert: []byte(caBlock), + }, + }, + BackendTLSPolicies: map[types.NamespacedName]*BackendTLSPolicy{ + client.ObjectKeyFromObject(btp.Source): &btp, + }, } } @@ -494,6 +563,25 @@ func TestIsReferenced(t *testing.T) { }, } + baseConfigMap := &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "test", + Name: "configmap", + }, + } + sameNamespaceDifferentNameConfigMap := &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "test", + Name: "configmap-different-name", + }, + } + differentNamespaceSameNameConfigMap := &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "test-different-namespace", + Name: "configmap", + }, + } + graph := &Graph{ Gateway: gw, ReferencedSecrets: map[types.NamespacedName]*Secret{ @@ -507,6 +595,12 @@ func TestIsReferenced(t *testing.T) { ReferencedServices: map[types.NamespacedName]struct{}{ client.ObjectKeyFromObject(serviceInGraph): {}, }, + ReferencedCaCertConfigMaps: map[types.NamespacedName]*CaCertConfigMap{ + client.ObjectKeyFromObject(baseConfigMap): { + Source: baseConfigMap, + CACert: []byte(caBlock), + }, + }, } tests := []struct { @@ -602,6 +696,26 @@ func TestIsReferenced(t *testing.T) { expected: false, }, + // ConfigMap cases + { + name: "ConfigMap in graph's ReferencedConfigMaps is referenced", + resource: baseConfigMap, + graph: graph, + expected: true, + }, + { + name: "ConfigMap not in ReferencedConfigMaps with same Namespace and different Name is not referenced", + resource: sameNamespaceDifferentNameConfigMap, + graph: graph, + expected: false, + }, + { + name: "ConfigMap not in ReferencedConfigMaps with different Namespace and same Name is not referenced", + resource: differentNamespaceSameNameConfigMap, + graph: graph, + expected: false, + }, + // Edge cases { name: "Resource is not supported by IsReferenced", diff --git a/internal/mode/static/state/graph/httproute.go b/internal/mode/static/state/graph/httproute.go index c8f42997e2..d9135237d3 100644 --- a/internal/mode/static/state/graph/httproute.go +++ b/internal/mode/static/state/graph/httproute.go @@ -279,7 +279,19 @@ func bindRouteToListeners(r *Route, gw *Gateway, namespaces map[types.Namespaced path := field.NewPath("spec").Child("parentRefs").Index(ref.Idx) - // Case 1: Attachment is not possible due to unsupported configuration + attachableListeners, listenerExists := findAttachableListeners( + getSectionName(routeRef.SectionName), + gw.Listeners, + ) + + // Case 1: Attachment is not possible because the specified SectionName does not match any Listeners in the + // Gateway. + if !listenerExists { + attachment.FailedCondition = staticConds.NewRouteNoMatchingParent() + continue + } + + // Case 2: Attachment is not possible due to unsupported configuration if routeRef.Port != nil { valErr := field.Forbidden(path.Child("port"), "cannot be set") @@ -287,7 +299,7 @@ func bindRouteToListeners(r *Route, gw *Gateway, namespaces map[types.Namespaced continue } - // Case 2: the parentRef references an ignored Gateway resource. + // Case 3: the parentRef references an ignored Gateway resource. referencesWinningGw := ref.Gateway.Namespace == gw.Source.Namespace && ref.Gateway.Name == gw.Source.Name @@ -296,18 +308,18 @@ func bindRouteToListeners(r *Route, gw *Gateway, namespaces map[types.Namespaced continue } - // Case 3: Attachment is not possible because Gateway is invalid + // Case 4: Attachment is not possible because Gateway is invalid if !gw.Valid { attachment.FailedCondition = staticConds.NewRouteInvalidGateway() continue } - // Case 4 - winning Gateway + // Case 5 - winning Gateway // Try to attach Route to all matching listeners - cond, attached := tryToAttachRouteToListeners(ref.Attachment, routeRef.SectionName, r, gw, namespaces) + cond, attached := tryToAttachRouteToListeners(ref.Attachment, attachableListeners, r, gw, namespaces) if !attached { attachment.FailedCondition = cond continue @@ -327,17 +339,11 @@ func bindRouteToListeners(r *Route, gw *Gateway, namespaces map[types.Namespaced // (2) If it fails to attach the route, it will return false and the failure condition. func tryToAttachRouteToListeners( refStatus *ParentRefAttachmentStatus, - sectionName *v1.SectionName, + attachableListeners []*Listener, route *Route, gw *Gateway, namespaces map[types.NamespacedName]*apiv1.Namespace, ) (conditions.Condition, bool) { - attachableListeners, listenerExists := findAttachableListeners(getSectionName(sectionName), gw.Listeners) - - if !listenerExists { - return staticConds.NewRouteNoMatchingParent(), false - } - if len(attachableListeners) == 0 { return staticConds.NewRouteInvalidListener(), false } diff --git a/site/content/how-to/traffic-management/securing-backend-traffic.md b/site/content/how-to/traffic-management/securing-backend-traffic.md new file mode 100644 index 0000000000..64b2db51c9 --- /dev/null +++ b/site/content/how-to/traffic-management/securing-backend-traffic.md @@ -0,0 +1,315 @@ +--- +title: "Securing Traffic to Backends" +description: "Learn how to encrypt HTTP traffic between NGINX Gateway Fabric and your backend pods." +weight: 600 +toc: true +docs: "DOCS-000" +--- + +In this guide, we will show how to specify the TLS configuration of the connection from the Gateway to a backend pod/s via the Service API object using a [BackendTLSPolicy](https://gateway-api.sigs.k8s.io/api-types/backendtlspolicy/). This covers the use-case where the service or backend owner is doing their own TLS and NGINX Gateway Fabric needs to know how to connect to this backend pod that has its own certificate over HTTPS. + +## Prerequisites + +- [Install]({{< relref "installation/" >}}) NGINX Gateway Fabric. Please note that the Gateway APIs from the experimental channel are required, and NGF must be deployed with the `--gateway-api-experimental-features` flag. +- [Expose NGINX Gateway Fabric]({{< relref "installation/expose-nginx-gateway-fabric.md" >}}) and save the public IP address and port of NGINX Gateway Fabric into shell variables: + + ```text + GW_IP=XXX.YYY.ZZZ.III + GW_PORT= + ``` + +{{< note >}}In a production environment, you should have a DNS record for the external IP address that is exposed, and it should refer to the hostname that the gateway will forward for.{{< /note >}} + +## Set up + +Create the **secure-app** application in Kubernetes by copying and pasting the following block into your terminal: + +```yaml +kubectl apply -f - < 8443/TCP 9s +``` + +## Configure Routing rules + +First, we will create the Gateway resource with an HTTP listener: + +```yaml +kubectl apply -f - <}}If you have a DNS record allocated for `secure-app.example.com`, you can send the request directly to that hostname, without needing to resolve.{{< /note >}} + +```shell +curl --resolve secure-app.example.com:$GW_PORT:$GW_IP http://secure-app.example.com:$GW_PORT/ +``` + +```text + +400 The plain HTTP request was sent to HTTPS port + +

400 Bad Request

+
The plain HTTP request was sent to HTTPS port
+
nginx/1.25.3
+ + +``` + +We can see we a status 400 Bad Request message from NGINX. + +## Create the Backend TLS configuration + +To configure the backend TLS terminationm, first we will create the ConfigMap that holds the `ca.crt` entry for verifying our self-signed certificates: + +```yaml +kubectl apply -f - < +Annotations: +API Version: gateway.networking.k8s.io/v1alpha2 +Kind: BackendTLSPolicy +Metadata: + Creation Timestamp: 2024-02-01T12:02:38Z + Generation: 1 + Resource Version: 19380 + UID: b3983a6e-92f1-4a98-b2af-64b317d74528 +Spec: + Target Ref: + Group: + Kind: Service + Name: secure-app + Namespace: default + Tls: + Ca Cert Refs: + Group: + Kind: ConfigMap + Name: backend-cert + Hostname: secure-app.example.com +Status: + Ancestors: + Ancestor Ref: + Group: gateway.networking.k8s.io + Kind: Gateway + Name: gateway + Namespace: default + Conditions: + Last Transition Time: 2024-02-01T12:02:38Z + Message: BackendTLSPolicy is accepted by the Gateway + Reason: Accepted + Status: True + Type: Accepted + Controller Name: gateway.nginx.org/nginx-gateway-controller +Events: +``` + +## Send Traffic with backend TLS configuration + +Now let's try sending traffic again: + +```shell +curl --resolve secure-app.example.com:$GW_PORT:$GW_IP http://secure-app.example.com:$GW_PORT/ +``` + +```text +hello from pod secure-app +``` + +## Further Reading + +To learn more about configuring backend TLS termination using the Gateway API, see the following resources: + +- [Backend TLS Policy](https://gateway-api.sigs.k8s.io/api-types/backendtlspolicy/) +- [Backend TLS Policy GEP](https://gateway-api.sigs.k8s.io/geps/gep-1897/) diff --git a/site/content/includes/installation/install-gateway-api-resources.md b/site/content/includes/installation/install-gateway-api-resources.md index 3636897b2f..55c669069d 100644 --- a/site/content/includes/installation/install-gateway-api-resources.md +++ b/site/content/includes/installation/install-gateway-api-resources.md @@ -10,6 +10,13 @@ To install the Gateway API resources, run the following: kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.0.0/standard-install.yaml ``` +Alternatively, you can install the Gateway API resources from the experimental channel. We support a subset of the +additional features provided by the experimental channel. To install from the experimental channel, run the following: + +```shell +kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.0.0/experimental-install.yaml +``` + If you are running on Kubernetes 1.23 or 1.24, you also need to install the validating webhook. To do so, run: ```shell diff --git a/site/content/includes/installation/uninstall-gateway-api-resources.md b/site/content/includes/installation/uninstall-gateway-api-resources.md index 87b648d572..a402fadf4f 100644 --- a/site/content/includes/installation/uninstall-gateway-api-resources.md +++ b/site/content/includes/installation/uninstall-gateway-api-resources.md @@ -10,6 +10,12 @@ docs: kubectl delete -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.0.0/standard-install.yaml ``` + Alternatively, if you installed the Gateway APIs from the experimental channel, run the following: + + ```shell + kubectl delete -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.0.0/experimental-install.yaml + ``` + If you are running on Kubernetes 1.23 or 1.24, you also need to delete the validating webhook. To do so, run: ```shell diff --git a/site/content/installation/installing-ngf/helm.md b/site/content/installation/installing-ngf/helm.md index 3e80c5a77b..a7ffd02300 100644 --- a/site/content/installation/installing-ngf/helm.md +++ b/site/content/installation/installing-ngf/helm.md @@ -109,6 +109,17 @@ To disable the creation of a Service: ``` +#### Experimental features + +We support a subset of the additional features provided by the Gateway API experimental channel. To enable the +experimental features of Gateway API which are supported by NGINX Gateway Fabric: + +```shell +helm install ngf oci://ghcr.io/nginxinc/charts/nginx-gateway-fabric --create-namespace -n nginx-gateway --set nginxGateway.experimentalFeatures.enable=true +``` + +{{}}Requires the Gateway APIs installed from the experimental channel.{{}} + ## Upgrade NGINX Gateway Fabric {{}}For guidance on zero downtime upgrades, see the [Delay Pod Termination](#configure-delayed-pod-termination-for-zero-downtime-upgrades) section below.{{}} @@ -127,6 +138,12 @@ To upgrade your Gateway API resources, take the following steps: kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.0.0/standard-install.yaml ``` + or, if you installed the from the experimental channel: + + ```shell + kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.0.0/experimental-install.yaml + ``` + ### Upgrade NGINX Gateway Fabric CRDs Helm's upgrade process does not automatically upgrade the NGINX Gateway Fabric CRDs (Custom Resource Definitions). diff --git a/site/content/installation/installing-ngf/manifests.md b/site/content/installation/installing-ngf/manifests.md index bd2361dae1..94ca6ab1aa 100644 --- a/site/content/installation/installing-ngf/manifests.md +++ b/site/content/installation/installing-ngf/manifests.md @@ -73,6 +73,25 @@ Deploying NGINX Gateway Fabric with Kubernetes manifests takes only a few steps. Update the nginx-plus-gateway.yaml file to include your chosen image from the F5 Container registry or your custom container image. +#### Enable experimental features + +We support a subset of the additional features provided by the Gateway API experimental channel. To enable the experimental features of Gateway API which are supported by NGINX Gateway Fabric: + +- For NGINX: + + ```shell + kubectl apply -f deploy/manifests/nginx-gateway-experimental.yaml + ``` + +- For NGINX Plus + + ```shell + kubectl apply -f deploy/manifests/nginx-plus-gateway-experimental.yaml + ``` + + Update the nginx-plus-gateway-experimental.yaml file to include your chosen image from the F5 Container registry or your custom container image. + +{{}}Requires the Gateway APIs installed from the experimental channel.{{}} ### 4. Verify the Deployment @@ -106,6 +125,12 @@ To upgrade NGINX Gateway Fabric and get the latest features and improvements, ta kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.0.0/standard-install.yaml ``` + or, if you installed the from the experimental channel: + + ```shell + kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.0.0/experimental-install.yaml + ``` + - If you are running on Kubernetes 1.23 or 1.24, you also need to update the validating webhook: ```shell diff --git a/site/content/overview/gateway-api-compatibility.md b/site/content/overview/gateway-api-compatibility.md index 1bd314749b..5a8a4e7b6f 100644 --- a/site/content/overview/gateway-api-compatibility.md +++ b/site/content/overview/gateway-api-compatibility.md @@ -9,16 +9,17 @@ docs: "DOCS-000" ## Summary {{< bootstrap-table "table table-striped table-bordered" >}} -| Resource | Core Support Level | Extended Support Level | Implementation-Specific Support Level | API Version | -|-------------------------------------|--------------------|------------------------|---------------------------------------|-------------| -| [GatewayClass](#gatewayclass) | Supported | Not supported | Not supported | v1 | -| [Gateway](#gateway) | Supported | Not supported | Not supported | v1 | -| [HTTPRoute](#httproute) | Supported | Partially supported | Not supported | v1 | -| [ReferenceGrant](#referencegrant) | Supported | N/A | Not supported | v1beta1 | -| [TLSRoute](#tlsroute) | Not supported | Not supported | Not supported | N/A | -| [TCPRoute](#tcproute) | Not supported | Not supported | Not supported | N/A | -| [UDPRoute](#udproute) | Not supported | Not supported | Not supported | N/A | -| [Custom policies](#custom-policies) | Not supported | N/A | Not supported | N/A | +| Resource | Core Support Level | Extended Support Level | Implementation-Specific Support Level | API Version | +| ------------------------------------- | ------------------ | ---------------------- | ------------------------------------- | ----------- | +| [GatewayClass](#gatewayclass) | Supported | Not supported | Not supported | v1 | +| [Gateway](#gateway) | Supported | Not supported | Not supported | v1 | +| [HTTPRoute](#httproute) | Supported | Partially supported | Not supported | v1 | +| [ReferenceGrant](#referencegrant) | Supported | N/A | Not supported | v1beta1 | +| [TLSRoute](#tlsroute) | Not supported | Not supported | Not supported | N/A | +| [TCPRoute](#tcproute) | Not supported | Not supported | Not supported | N/A | +| [UDPRoute](#udproute) | Not supported | Not supported | Not supported | N/A | +| [BackendTLSPolicy](#backendtlspolicy) | Supported | Supported | Not supported | v1alpha2 | +| [Custom policies](#custom-policies) | Not supported | N/A | Not supported | N/A | {{< /bootstrap-table >}} --- @@ -46,9 +47,9 @@ For a description of each field, visit the [Gateway API documentation](https://g ### GatewayClass {{< bootstrap-table "table table-striped table-bordered" >}} -| Resource | Core Support Level | Extended Support Level | Implementation-Specific Support Level | API Version | -|-------------------------------------|--------------------|------------------------|---------------------------------------|-------------| -| GatewayClass | Supported | Not supported | Not supported | v1 | +| Resource | Core Support Level | Extended Support Level | Implementation-Specific Support Level | API Version | +| ------------ | ------------------ | ---------------------- | ------------------------------------- | ----------- | +| GatewayClass | Supported | Not supported | Not supported | v1 | {{< /bootstrap-table >}} NGINX Gateway Fabric supports a single GatewayClass resource configured with the `--gatewayclass` flag of the [static-mode]({{< relref "/reference/cli-help.md#static-mode">}}) command. @@ -74,9 +75,9 @@ NGINX Gateway Fabric supports a single GatewayClass resource configured with the ### Gateway {{< bootstrap-table "table table-striped table-bordered" >}} -| Resource | Core Support Level | Extended Support Level | Implementation-Specific Support Level | API Version | -|-------------------------------------|--------------------|------------------------|---------------------------------------|-------------| -| Gateway | Supported | Not supported | Not supported | v1 | +| Resource | Core Support Level | Extended Support Level | Implementation-Specific Support Level | API Version | +| -------- | ------------------ | ---------------------- | ------------------------------------- | ----------- | +| Gateway | Supported | Not supported | Not supported | v1 | {{< /bootstrap-table >}} NGINX Gateway Fabric supports a single Gateway resource. The Gateway resource must reference NGINX Gateway Fabric's corresponding GatewayClass. @@ -135,9 +136,9 @@ See the [static-mode]({{< relref "/reference/cli-help.md#static-mode">}}) comman ### HTTPRoute {{< bootstrap-table "table table-striped table-bordered" >}} -| Resource | Core Support Level | Extended Support Level | Implementation-Specific Support Level | API Version | -|-------------------------------------|--------------------|------------------------|---------------------------------------|-------------| -| HTTPRoute | Supported | Partially supported | Not supported | v1 | +| Resource | Core Support Level | Extended Support Level | Implementation-Specific Support Level | API Version | +| --------- | ------------------ | ---------------------- | ------------------------------------- | ----------- | +| HTTPRoute | Supported | Partially supported | Not supported | v1 | {{< /bootstrap-table >}} **Fields**: @@ -182,9 +183,9 @@ See the [static-mode]({{< relref "/reference/cli-help.md#static-mode">}}) comman ### ReferenceGrant {{< bootstrap-table "table table-striped table-bordered" >}} -| Resource | Core Support Level | Extended Support Level | Implementation-Specific Support Level | API Version | -|-------------------------------------|--------------------|------------------------|---------------------------------------|-------------| -| ReferenceGrant | Supported | N/A | Not supported | v1beta1 | +| Resource | Core Support Level | Extended Support Level | Implementation-Specific Support Level | API Version | +| -------------- | ------------------ | ---------------------- | ------------------------------------- | ----------- | +| ReferenceGrant | Supported | N/A | Not supported | v1beta1 | {{< /bootstrap-table >}} Fields: @@ -204,9 +205,9 @@ Fields: ### TLSRoute {{< bootstrap-table "table table-striped table-bordered" >}} -| Resource | Core Support Level | Extended Support Level | Implementation-Specific Support Level | API Version | -|-------------------------------------|--------------------|------------------------|---------------------------------------|-------------| -| TLSRoute | Not supported | Not supported | Not supported | N/A | +| Resource | Core Support Level | Extended Support Level | Implementation-Specific Support Level | API Version | +| -------- | ------------------ | ---------------------- | ------------------------------------- | ----------- | +| TLSRoute | Not supported | Not supported | Not supported | N/A | {{< /bootstrap-table >}} --- @@ -214,9 +215,9 @@ Fields: ### TCPRoute {{< bootstrap-table "table table-striped table-bordered" >}} -| Resource | Core Support Level | Extended Support Level | Implementation-Specific Support Level | API Version | -|-------------------------------------|--------------------|------------------------|---------------------------------------|-------------| -| TCPRoute | Not supported | Not supported | Not supported | N/A | +| Resource | Core Support Level | Extended Support Level | Implementation-Specific Support Level | API Version | +| -------- | ------------------ | ---------------------- | ------------------------------------- | ----------- | +| TCPRoute | Not supported | Not supported | Not supported | N/A | {{< /bootstrap-table >}} --- @@ -224,18 +225,51 @@ Fields: ### UDPRoute {{< bootstrap-table "table table-striped table-bordered" >}} -| Resource | Core Support Level | Extended Support Level | Implementation-Specific Support Level | API Version | -|-------------------------------------|--------------------|------------------------|---------------------------------------|-------------| -| UDPRoute | Not supported | Not supported | Not supported | N/A | +| Resource | Core Support Level | Extended Support Level | Implementation-Specific Support Level | API Version | +| -------- | ------------------ | ---------------------- | ------------------------------------- | ----------- | +| UDPRoute | Not supported | Not supported | Not supported | N/A | {{< /bootstrap-table >}} --- +### BackendTLSPolicy + +{{< bootstrap-table "table table-striped table-bordered" >}} +| Resource | Core Support Level | Extended Support Level | Implementation-Specific Support Level | API Version | +| ---------------- | ------------------ | ---------------------- | ------------------------------------- | ----------- | +| BackendTLSPolicy | Supported | Supported | Not supported | v1alpha2 | +{{< /bootstrap-table >}} + +Fields: + +- `spec` + - `targetRef` + - `group` - supported. + - `kind` - supports `Service`. + - `name` - supported. + - `namespace` - supported. + - `tls` + - `caCertRefs` - supports single reference to a `ConfigMap`, with the CA certificate in a key named `ca.crt`. + - `name`- supported. + - `group` - supported. + - `kind` - supports `ConfigMap`. + - `hostname` - supported. + - `wellKnownCerts` - supports `System`. This will set the CA certificate to the Alpine system root CA path `/etc/ssl/cert.pem`. NB: This option will only work if the NGINX image used is Alpine based. The NGF NGINX images are Alpine based by default. +- `status` + - `ancestors` + - `ancestorRef` - supported. + - `controllerName`: supported. + - `conditions`: Partially supported. Supported (Condition/Status/Reason): + - `Accepted/True/PolicyReasonAccepted` + - `Accepted/False/PolicyReasonInvalid` + +{{}}If multiple `backendRefs` are defined for a HTTPRoute rule, all the referenced Services *must* have matching BackendTLSPolicy configuration. BackendTLSPolicy configuration is considered to be matching if 1. CACertRefs reference the same ConfigMap, or 2. WellKnownCACerts are the same, and 3. Hostname is the same.{{}} + ### Custom Policies {{< bootstrap-table "table table-striped table-bordered" >}} -| Resource | Core Support Level | Extended Support Level | Implementation-Specific Support Level | API Version | -|-------------------------------------|--------------------|------------------------|---------------------------------------|-------------| +| Resource | Core Support Level | Extended Support Level | Implementation-Specific Support Level | API Version | +| --------------- | ------------------ | ---------------------- | ------------------------------------- | ----------- | | Custom policies | Not supported | N/A | Not supported | N/A | {{< /bootstrap-table >}} diff --git a/site/content/reference/cli-help.md b/site/content/reference/cli-help.md index a018b50ca8..1c7c661e55 100644 --- a/site/content/reference/cli-help.md +++ b/site/content/reference/cli-help.md @@ -19,22 +19,23 @@ _Usage_: ### Flags {{< bootstrap-table "table table-bordered table-striped table-responsive" >}} -| Name | Type | Description | -|------------------------------|----------|-------------| -| _gateway-ctlr-name_ | _string_ | The name of the Gateway controller. The controller name must be in the form: `DOMAIN/PATH`. The controller's domain is `gateway.nginx.org`. | -| _gatewayclass_ | _string_ | The name of the GatewayClass resource. Every NGINX Gateway Fabric must have a unique corresponding GatewayClass resource. | -| _gateway_ | _string_ | The namespaced name of the Gateway resource to use. Must be of the form: `NAMESPACE/NAME`. If not specified, the control plane will process all Gateways for the configured GatewayClass. Among them, it will choose the oldest resource by creation timestamp. If the timestamps are equal, it will choose the resource that appears first in alphabetical order by {namespace}/{name}. | -| _nginx-plus_ | _bool_ | Enable support for NGINX Plus. | -| _config_ | _string_ | The name of the NginxGateway resource to be used for this controller's dynamic configuration. Lives in the same namespace as the controller. | -| _service_ | _string_ | The name of the service that fronts this NGINX Gateway Fabric pod. Lives in the same namespace as the controller. | -| _metrics-disable_ | _bool_ | Disable exposing metrics in the Prometheus format (Default: `false`). | -| _metrics-listen-port_ | _int_ | Sets the port where the Prometheus metrics are exposed. An integer between 1024 - 65535 (Default: `9113`) | -| _metrics-secure-serving_ | _bool_ | Configures if the metrics endpoint should be secured using https. Note that this endpoint will be secured with a self-signed certificate (Default `false`). | -| _update-gatewayclass-status_ | _bool_ | Update the status of the GatewayClass resource (Default: `true`). | -| _health-disable_ | _bool_ | Disable running the health probe server (Default: `false`). | -| _health-port_ | _int_ | Set the port where the health probe server is exposed. An integer between 1024 - 65535 (Default: `8081`). | -| _leader-election-disable_ | _bool_ | Disable leader election, which is used to avoid multiple replicas of the NGINX Gateway Fabric reporting the status of the Gateway API resources. If disabled, all replicas of NGINX Gateway Fabric will update the statuses of the Gateway API resources (Default: `false`). | -| _leader-election-lock-name_ | _string_ | The name of the leader election lock. A lease object with this name will be created in the same namespace as the controller (Default: `"nginx-gateway-leader-election-lock"`). | +| Name | Type | Description | +| ----------------------------------- | -------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| _gateway-ctlr-name_ | _string_ | The name of the Gateway controller. The controller name must be in the form: `DOMAIN/PATH`. The controller's domain is `gateway.nginx.org`. | +| _gatewayclass_ | _string_ | The name of the GatewayClass resource. Every NGINX Gateway Fabric must have a unique corresponding GatewayClass resource. | +| _gateway_ | _string_ | The namespaced name of the Gateway resource to use. Must be of the form: `NAMESPACE/NAME`. If not specified, the control plane will process all Gateways for the configured GatewayClass. Among them, it will choose the oldest resource by creation timestamp. If the timestamps are equal, it will choose the resource that appears first in alphabetical order by {namespace}/{name}. | +| _nginx-plus_ | _bool_ | Enable support for NGINX Plus. | +| _gateway-api-experimental-features_ | _bool_ | Enable the experimental features of Gateway API which are supported by NGINX Gateway Fabric. Requires the Gateway APIs installed from the experimental channel. | +| _config_ | _string_ | The name of the NginxGateway resource to be used for this controller's dynamic configuration. Lives in the same namespace as the controller. | +| _service_ | _string_ | The name of the service that fronts this NGINX Gateway Fabric pod. Lives in the same namespace as the controller. | +| _metrics-disable_ | _bool_ | Disable exposing metrics in the Prometheus format (Default: `false`). | +| _metrics-listen-port_ | _int_ | Sets the port where the Prometheus metrics are exposed. An integer between 1024 - 65535 (Default: `9113`) | +| _metrics-secure-serving_ | _bool_ | Configures if the metrics endpoint should be secured using https. Note that this endpoint will be secured with a self-signed certificate (Default `false`). | +| _update-gatewayclass-status_ | _bool_ | Update the status of the GatewayClass resource (Default: `true`). | +| _health-disable_ | _bool_ | Disable running the health probe server (Default: `false`). | +| _health-port_ | _int_ | Set the port where the health probe server is exposed. An integer between 1024 - 65535 (Default: `8081`). | +| _leader-election-disable_ | _bool_ | Disable leader election, which is used to avoid multiple replicas of the NGINX Gateway Fabric reporting the status of the Gateway API resources. If disabled, all replicas of NGINX Gateway Fabric will update the statuses of the Gateway API resources (Default: `false`). | +| _leader-election-lock-name_ | _string_ | The name of the leader election lock. A lease object with this name will be created in the same namespace as the controller (Default: `"nginx-gateway-leader-election-lock"`). | {{% /bootstrap-table %}} ## Sleep @@ -48,7 +49,7 @@ _Usage_: ``` {{< bootstrap-table "table table-bordered table-striped table-responsive" >}} -| Name | Type | Description | -|----------|-----------------|-------------------------------------------------------------------------------------------------------| +| Name | Type | Description | +| -------- | --------------- | ----------------------------------------------------------------------------------------------------------------------------- | | duration | `time.Duration` | Set the duration of sleep. Must be parsable by [`time.ParseDuration`](https://pkg.go.dev/time#ParseDuration). (default `30s`) | {{% /bootstrap-table %}}