Run functional and NFR tests with --race (#3614)

Problem: The functional and NFR tests do not run with --race enabled, so race conditions within the tests may be present and unknown.

Solution: Enable --race and fix any issues.

Author: sjberman

tests/Makefile

@@ -120,8 +120,8 @@ stop-longevity-test: nfr-test ## Stop the longevity test and collects results
 
 .PHONY: .vm-nfr-test
 .vm-nfr-test: ## Runs the NFR tests on the GCP VM (called by `nfr-test`)
-	go run github.com/onsi/ginkgo/v2/ginkgo --randomize-all --randomize-suites --keep-going --fail-on-pending \
-		--trace -r -v --buildvcs --force-newlines $(GITHUB_OUTPUT) \
+	CGO_ENABLED=1 go run github.com/onsi/ginkgo/v2/ginkgo --race --randomize-all --randomize-suites --keep-going --fail-on-pending \
+		--trace -r -v --buildvcs --force-newlines $(GITHUB_OUTPUT) --flake-attempts=2 \
 		--label-filter "nfr" $(GINKGO_FLAGS) --timeout 5h ./suite --  --gateway-api-version=$(GW_API_VERSION) \
 		--gateway-api-prev-version=$(GW_API_PREV_VERSION) --image-tag=$(TAG) --version-under-test=$(NGF_VERSION) \
 		--ngf-image-repo=$(PREFIX) --nginx-image-repo=$(NGINX_PREFIX) --nginx-plus-image-repo=$(NGINX_PLUS_PREFIX) \
@@ -132,7 +132,7 @@ stop-longevity-test: nfr-test ## Stop the longevity test and collects results
 .PHONY: test
 test: build-crossplane-image ## Runs the functional tests on your kind k8s cluster
 	kind load docker-image nginx-crossplane:latest --name $(CLUSTER_NAME)
-	go run github.com/onsi/ginkgo/v2/ginkgo --randomize-all --randomize-suites --keep-going --fail-on-pending \
+	go run github.com/onsi/ginkgo/v2/ginkgo --race --randomize-all --randomize-suites --keep-going --fail-on-pending \
 		--trace -r -v --buildvcs --force-newlines $(GITHUB_OUTPUT) \
 		--label-filter "functional" $(GINKGO_FLAGS) ./suite -- \
 		--gateway-api-version=$(GW_API_VERSION) --gateway-api-prev-version=$(GW_API_PREV_VERSION) \

tests/framework/portforward.go

@@ -7,6 +7,7 @@ import (
 	"net/http"
 	"net/url"
 	"path"
+	"sync"
 	"time"
 
 	"k8s.io/client-go/rest"
@@ -35,12 +36,10 @@ func PortForward(config *rest.Config, namespace, podName string, ports []string,
 
 	dialer := spdy.NewDialer(upgrader, &http.Client{Transport: roundTripper}, http.MethodPost, serverURL)
 
-	out, errOut := new(bytes.Buffer), new(bytes.Buffer)
-
 	forward := func() error {
 		readyCh := make(chan struct{}, 1)
 
-		forwarder, err := portforward.New(dialer, ports, stopCh, readyCh, out, errOut)
+		forwarder, err := portforward.New(dialer, ports, stopCh, readyCh, newSafeBuffer(), newSafeBuffer())
 		if err != nil {
 			return fmt.Errorf("error creating port forwarder: %w", err)
 		}
@@ -66,3 +65,31 @@ func PortForward(config *rest.Config, namespace, podName string, ports []string,
 
 	return nil
 }
+
+// safeBuffer is a goroutine safe bytes.Buffer.
+type safeBuffer struct {
+	buffer bytes.Buffer
+	mutex  sync.Mutex
+}
+
+func newSafeBuffer() *safeBuffer {
+	return &safeBuffer{}
+}
+
+// Write appends the contents of p to the buffer, growing the buffer as needed. It returns
+// the number of bytes written.
+func (s *safeBuffer) Write(p []byte) (n int, err error) {
+	s.mutex.Lock()
+	defer s.mutex.Unlock()
+
+	return s.buffer.Write(p)
+}
+
+// String returns the contents of the unread portion of the buffer
+// as a string.  If the Buffer is a nil pointer, it returns "<nil>".
+func (s *safeBuffer) String() string {
+	s.mutex.Lock()
+	defer s.mutex.Unlock()
+
+	return s.buffer.String()
+}

tests/framework/request.go

@@ -61,7 +61,8 @@ func makeRequest(
 		return nil, errors.New("transport is not of type *http.Transport")
 	}
 
-	transport.DialContext = func(
+	customTransport := transport.Clone()
+	customTransport.DialContext = func(
 		ctx context.Context,
 		network,
 		addr string,
@@ -93,21 +94,13 @@ func makeRequest(
 
 	var resp *http.Response
 	if strings.HasPrefix(url, "https") {
-		transport, ok := http.DefaultTransport.(*http.Transport)
-		if !ok {
-			return nil, errors.New("transport is not of type *http.Transport")
-		}
-
-		customTransport := transport.Clone()
 		// similar to how in our examples with https requests we run our curl command
 		// we turn off verification of the certificate, we do the same here
 		customTransport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} //nolint:gosec // for https test traffic
-		client := &http.Client{Transport: customTransport}
-		resp, err = client.Do(req)
-	} else {
-		resp, err = http.DefaultClient.Do(req)
 	}
 
+	client := &http.Client{Transport: customTransport}
+	resp, err = client.Do(req)
 	if err != nil {
 		return nil, err
 	}

tests/scripts/create-and-setup-gcp-vm.sh

@@ -48,6 +48,8 @@ done
 
 gcloud compute scp --zone "${GKE_CLUSTER_ZONE}" --project="${GKE_PROJECT}" "${SCRIPT_DIR}"/vars.env username@"${RESOURCE_NAME}":~
 
+gcloud compute ssh --zone "${GKE_CLUSTER_ZONE}" --project="${GKE_PROJECT}" username@"${RESOURCE_NAME}" --command="bash -s" <"${SCRIPT_DIR}"/remote-scripts/install-deps.sh
+
 if [ -n "${NGF_REPO}" ] && [ "${NGF_REPO}" != "nginx" ]; then
     gcloud compute ssh --zone "${GKE_CLUSTER_ZONE}" --project="${GKE_PROJECT}" username@"${RESOURCE_NAME}" \
         --command="bash -i <<EOF

tests/scripts/remote-scripts/install-deps.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+set -e
+
+source ~/vars.env
+
+sudo apt-get -y update && sudo apt-get -y install gcc