Fix benchmark failures and improve results quality

.gitlab/benchmarks.yml

@@ -15,7 +15,7 @@
   script:
     - export ARTIFACTS_DIR="$(pwd)/reports" && mkdir -p "${ARTIFACTS_DIR}"
     - git config --global url."https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/".insteadOf "https://github.com/DataDog/"
-    - git clone --branch dd-trace-java/tracer-benchmarks https://github.com/DataDog/benchmarking-platform.git /platform && cd /platform
+    - git clone --branch dd-trace-java/tracer-benchmarks-parallel https://github.com/DataDog/benchmarking-platform.git /platform && cd /platform
   artifacts:
     name: "reports"
     paths:

benchmark/benchmarks.sh

@@ -33,10 +33,6 @@ if [[ ! -f "${TRACER}" ]]; then
   cd "${SCRIPT_DIR}"
 fi
 
-# Cleanup previous reports
-rm -rf "${REPORTS_DIR}"
-mkdir -p "${REPORTS_DIR}"
-
 if [[ "$#" == '0' ]]; then
   for type in 'startup' 'load' 'dacapo'; do
     run_benchmarks "$type"

benchmark/load/insecure-bank/k6.js

@@ -1,18 +1,62 @@
 import http from 'k6/http';
 import {checkResponse, isOk, isRedirect} from "../../utils/k6.js";
 
-const baseUrl = 'http://localhost:8080';
+const variants = {
+  "no_agent": {
+    "APP_URL": 'http://localhost:8080',
+  },
+  "tracing": {
+    "APP_URL": 'http://localhost:8081',
+  },
+  "profiling": {
+    "APP_URL": 'http://localhost:8082',
+  },
+  "iast": {
+    "APP_URL": 'http://localhost:8083',
+  },
+  "iast_GLOBAL": {
+    "APP_URL": 'http://localhost:8084',
+  },
+  "iast_FULL": {
+    "APP_URL": 'http://localhost:8085',
+  },
+}
+
+export const options = function (variants) {
+  let scenarios = {};
+  for (const variant of Object.keys(variants)) {
+    scenarios[`load--insecure-bank--${variant}--warmup`] = {
+      executor: 'constant-vus',  // https://grafana.com/docs/k6/latest/using-k6/scenarios/executors/#all-executors
+      vus: 5,
+      duration: '20s',
+      gracefulStop: '2s',
+      env: {
+        "APP_URL": variants[variant]["APP_URL"]
+      }
+    };
+
+    scenarios[`load--insecure-bank--${variant}--high_load`] = {
+      executor: 'constant-vus',
+      vus: 5,
+      startTime: '22s',
+      duration: '15s',
+      gracefulStop: '2s',
+      env: {
+        "APP_URL": variants[variant]["APP_URL"]
+      }
+    };
+  }
 
-export const options = {
-  discardResponseBodies: true,
-  vus: 5,
-  iterations: 40000
-};
+  return {
+    discardResponseBodies: true,
+    scenarios,
+  }
+}(variants);
 
 export default function () {
 
   // login form
-  const loginResponse = http.post(`${baseUrl}/login`, {
+  const loginResponse = http.post(`${__ENV.APP_URL}/login`, {
     username: 'john',
     password: 'test'
   }, {
@@ -21,11 +65,11 @@ export default function () {
   checkResponse(loginResponse, isRedirect);
 
   // dashboard
-  const dashboard = http.get(`${baseUrl}/dashboard`);
+  const dashboard = http.get(`${__ENV.APP_URL}/dashboard`);
   checkResponse(dashboard, isOk);
 
   // logout
-  const logout = http.get(`${baseUrl}/j_spring_security_logout`, {
+  const logout = http.get(`${__ENV.APP_URL}/j_spring_security_logout`, {
     redirects: 0
   });
   checkResponse(logout, isRedirect);

benchmark/load/insecure-bank/start-servers.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+
+set -e
+
+start_server() {
+  local VARIANT=$1
+  local JAVA_OPTS=$2
+
+  if [ -n "$CI_JOB_TOKEN" ]; then
+    # Inside BP, so we can assume 24 CPU cores available and set CPU affinity
+    CPU_AFFINITY_APP=$3
+  else
+    CPU_AFFINITY_APP=""
+  fi
+
+  mkdir -p "${LOGS_DIR}/${VARIANT}"
+  ${CPU_AFFINITY_APP}java ${JAVA_OPTS} -Xms3G -Xmx3G -jar ${INSECURE_BANK} &> ${LOGS_DIR}/${VARIANT}/insecure-bank.log &PID=$!
+  echo "${CPU_AFFINITY_APP}java ${JAVA_OPTS} -Xms3G -Xmx3G -jar ${INSECURE_BANK} &> ${LOGS_DIR}/${VARIANT}/insecure-bank.log [PID=$PID]"
+}
+
+start_server "no_agent" "-Dserver.port=8080" "taskset -c 47 " &
+start_server "tracing" "-javaagent:${TRACER} -Dserver.port=8081" "taskset -c 46 " &
+start_server "profiling" "-javaagent:${TRACER} -Ddd.profiling.enabled=true -Dserver.port=8082" "taskset -c 45 " &
+start_server "iast" "-javaagent:${TRACER} -Ddd.iast.enabled=true -Dserver.port=8083" "taskset -c 44 " &
+start_server "iast_GLOBAL" "-javaagent:${TRACER} -Ddd.iast.enabled=true -Ddd.iast.context.mode=GLOBAL -Dserver.port=8084" "taskset -c 43 " &
+start_server "iast_FULL" "-javaagent:${TRACER} -Ddd.iast.enabled=true -Ddd.iast.detection.mode=FULL -Dserver.port=8085" "taskset -c 42 " &
+
+wait

benchmark/load/petclinic/k6.js

@@ -1,17 +1,61 @@
 import http from 'k6/http';
 import {checkResponse, isOk} from "../../utils/k6.js";
 
-const baseUrl = 'http://localhost:8080';
+const variants = {
+  "no_agent": {
+    "APP_URL": 'http://localhost:8080',
+  },
+  "tracing": {
+    "APP_URL": 'http://localhost:8081',
+  },
+  "profiling": {
+    "APP_URL": 'http://localhost:8082',
+  },
+  "appsec": {
+    "APP_URL": 'http://localhost:8083',
+  },
+  "iast": {
+    "APP_URL": 'http://localhost:8084',
+  },
+  "code_origins": {
+    "APP_URL": 'http://localhost:8085',
+  }
+}
+
+export const options = function (variants) {
+  const scenarios = {};
+  for (const variant of Object.keys(variants)) {
+    scenarios[`load--petclinic--${variant}--warmup`] = {
+      executor: 'constant-vus',  // https://grafana.com/docs/k6/latest/using-k6/scenarios/executors/#all-executors
+      vus: 5,
+      duration: '20s',
+      gracefulStop: '2s',
+      env: {
+        "APP_URL": variants[variant]["APP_URL"]
+      }
+    };
+
+    scenarios[`load--petclinic--${variant}--high_load`] = {
+      executor: 'constant-vus',
+      vus: 5,
+      startTime: '22s',
+      duration: '15s',
+      gracefulStop: '2s',
+      env: {
+        "APP_URL": variants[variant]["APP_URL"]
+      }
+    };
+  }
 
-export const options = {
-  discardResponseBodies: true,
-  vus: 5,
-  iterations: 80000
-};
+  return {
+    discardResponseBodies: true,
+    scenarios,
+  }
+}(variants);
 
 export default function () {
 
   // find owner
-  const ownersList = http.get(`${baseUrl}/owners?lastName=`);
+  const ownersList = http.get(`${__ENV.APP_URL}/owners?lastName=`);
   checkResponse(ownersList, isOk);
 }

benchmark/load/petclinic/start-servers.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+
+set -e
+
+start_server() {
+  local VARIANT=$1
+  local JAVA_OPTS=$2
+
+  if [ -n "$CI_JOB_TOKEN" ]; then
+    # Inside BP, so we can assume 24 CPU cores available and set CPU affinity
+    CPU_AFFINITY_APP=$3
+  else
+    CPU_AFFINITY_APP=""
+  fi
+
+  mkdir -p "${LOGS_DIR}/${VARIANT}"
+  ${CPU_AFFINITY_APP}java ${JAVA_OPTS} -Xms2G -Xmx2G -jar ${PETCLINIC} &> ${LOGS_DIR}/${VARIANT}/petclinic.log &PID=$!
+  echo "${CPU_AFFINITY_APP}java ${JAVA_OPTS} -Xms2G -Xmx2G -jar ${PETCLINIC} &> ${LOGS_DIR}/${VARIANT}/petclinic.log [PID=$!]"
+}
+
+start_server "no_agent" "-Dserver.port=8080" "taskset -c 31-32 " &
+start_server "tracing" "-javaagent:${TRACER} -Dserver.port=8081" "taskset -c 33-34 " &
+start_server "profiling" "-javaagent:${TRACER} -Ddd.profiling.enabled=true -Dserver.port=8082" "taskset -c 35-36 " &
+start_server "appsec" "-javaagent:${TRACER} -Ddd.appsec.enabled=true -Dserver.port=8083" "taskset -c 37-38 " &
+start_server "iast" "-javaagent:${TRACER} -Ddd.iast.enabled=true -Dserver.port=8084" "taskset -c 39-40 " &
+start_server "code_origins" "-javaagent:${TRACER} -Ddd.code.origin.for.spans.enabled=true -Dserver.port=8085" "taskset -c 41-42 " &
+
+wait

benchmark/load/run.sh

@@ -1,5 +1,77 @@
 #!/usr/bin/env bash
-set -eu
+
+set -e
+
+function message() {
+  echo "$(date +"%T"): $1"
+}
+
+function healthcheck() {
+  local url=$1
+
+  while true; do
+    if [[ $(curl -fso /dev/null -w "%{http_code}" "${url}") = 200 ]]; then
+      break
+    fi
+  done
+}
+
+type=$1
+
+if [ -n "$CI_JOB_TOKEN" ]; then
+  # Inside BP, so we can assume 24 CPU cores on the second socket available and set CPU affinity
+  export CPU_AFFINITY_K6="taskset -c 24-27 "
+else
+  export CPU_AFFINITY_K6=""
+fi
 
 source "${UTILS_DIR}/update-java-version.sh" 17
-"${UTILS_DIR}/run-sirun-benchmarks.sh" "$@"
+
+for app in *; do
+  if [[ ! -d "${app}" ]]; then
+    continue
+  fi
+
+  message "${type} benchmark: ${app} started"
+
+  export OUTPUT_DIR="${REPORTS_DIR}/${type}/${app}"
+  mkdir -p ${OUTPUT_DIR}
+
+  export LOGS_DIR="${ARTIFACTS_DIR}/${type}/${app}"
+  mkdir -p ${LOGS_DIR}
+
+  # Using profiler variants for healthcheck as they are the slowest
+  if [ "${app}" == "petclinic" ]; then
+    HEALTHCHECK_URL=http://localhost:8082
+    REPETITIONS_COUNT=5
+  elif [ "${app}" == "insecure-bank" ]; then
+    HEALTHCHECK_URL=http://localhost:8082/login
+    REPETITIONS_COUNT=2
+  else
+    echo "Unknown app ${app}"
+    exit 1
+  fi
+
+  for i in $(seq 1 $REPETITIONS_COUNT); do
+    bash -c "${UTILS_DIR}/../${type}/${app}/start-servers.sh" &
+
+    echo "Waiting for serves to start..."
+    if [ "${app}" == "petclinic" ]; then
+      for port in $(seq 8080 8085); do
+        healthcheck http://localhost:$port
+      done
+    elif [ "${app}" == "insecure-bank" ]; then
+      for port in $(seq 8080 8085); do
+        healthcheck http://localhost:$port/login
+      done
+    fi
+    echo "Servers are up!"
+
+    (
+      cd ${app} &&
+      bash -c "${CPU_AFFINITY_K6}${UTILS_DIR}/run-k6-load-test.sh 'pkill java'"
+    )
+  done
+
+  message "${type} benchmark: ${app} finished"
+done