Skip to content

Commit 002d46f

Browse files
ddyurchenkosarahchen6
ddyurchenko
authored and
sarahchen6
committed
Fix benchmark failures and improve results quality (#8962)
Fix benchmark failures and improve results quality
1 parent a6e9abe commit 002d46f

File tree

11 files changed

+242
-138
lines changed

11 files changed

+242
-138
lines changed

‎.gitlab/benchmarks.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
script:
1616
- export ARTIFACTS_DIR="$(pwd)/reports" && mkdir -p "${ARTIFACTS_DIR}"
1717
- git config --global url."https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/".insteadOf "https://github.com/DataDog/"
18-
- git clone --branch dd-trace-java/tracer-benchmarks https://github.com/DataDog/benchmarking-platform.git /platform && cd /platform
18+
- git clone --branch dd-trace-java/tracer-benchmarks-parallel https://github.com/DataDog/benchmarking-platform.git /platform && cd /platform
1919
artifacts:
2020
name: "reports"
2121
paths:

‎benchmark/benchmarks.sh

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -33,10 +33,6 @@ if [[ ! -f "${TRACER}" ]]; then
3333
cd "${SCRIPT_DIR}"
3434
fi
3535

36-
# Cleanup previous reports
37-
rm -rf "${REPORTS_DIR}"
38-
mkdir -p "${REPORTS_DIR}"
39-
4036
if [[ "$#" == '0' ]]; then
4137
for type in 'startup' 'load' 'dacapo'; do
4238
run_benchmarks "$type"

‎benchmark/load/insecure-bank/benchmark.json

Lines changed: 0 additions & 58 deletions
This file was deleted.

‎benchmark/load/insecure-bank/k6.js

Lines changed: 53 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,62 @@
11
import http from 'k6/http';
22
import {checkResponse, isOk, isRedirect} from "../../utils/k6.js";
33

4-
const baseUrl = 'http://localhost:8080';
4+
const variants = {
5+
"no_agent": {
6+
"APP_URL": 'http://localhost:8080',
7+
},
8+
"tracing": {
9+
"APP_URL": 'http://localhost:8081',
10+
},
11+
"profiling": {
12+
"APP_URL": 'http://localhost:8082',
13+
},
14+
"iast": {
15+
"APP_URL": 'http://localhost:8083',
16+
},
17+
"iast_GLOBAL": {
18+
"APP_URL": 'http://localhost:8084',
19+
},
20+
"iast_FULL": {
21+
"APP_URL": 'http://localhost:8085',
22+
},
23+
}
24+
25+
export const options = function (variants) {
26+
let scenarios = {};
27+
for (const variant of Object.keys(variants)) {
28+
scenarios[`load--insecure-bank--${variant}--warmup`] = {
29+
executor: 'constant-vus', // https://grafana.com/docs/k6/latest/using-k6/scenarios/executors/#all-executors
30+
vus: 5,
31+
duration: '20s',
32+
gracefulStop: '2s',
33+
env: {
34+
"APP_URL": variants[variant]["APP_URL"]
35+
}
36+
};
37+
38+
scenarios[`load--insecure-bank--${variant}--high_load`] = {
39+
executor: 'constant-vus',
40+
vus: 5,
41+
startTime: '22s',
42+
duration: '15s',
43+
gracefulStop: '2s',
44+
env: {
45+
"APP_URL": variants[variant]["APP_URL"]
46+
}
47+
};
48+
}
549

6-
export const options = {
7-
discardResponseBodies: true,
8-
vus: 5,
9-
iterations: 40000
10-
};
50+
return {
51+
discardResponseBodies: true,
52+
scenarios,
53+
}
54+
}(variants);
1155

1256
export default function () {
1357

1458
// login form
15-
const loginResponse = http.post(`${baseUrl}/login`, {
59+
const loginResponse = http.post(`${__ENV.APP_URL}/login`, {
1660
username: 'john',
1761
password: 'test'
1862
}, {
@@ -21,11 +65,11 @@ export default function () {
2165
checkResponse(loginResponse, isRedirect);
2266

2367
// dashboard
24-
const dashboard = http.get(`${baseUrl}/dashboard`);
68+
const dashboard = http.get(`${__ENV.APP_URL}/dashboard`);
2569
checkResponse(dashboard, isOk);
2670

2771
// logout
28-
const logout = http.get(`${baseUrl}/j_spring_security_logout`, {
72+
const logout = http.get(`${__ENV.APP_URL}/j_spring_security_logout`, {
2973
redirects: 0
3074
});
3175
checkResponse(logout, isRedirect);

‎benchmark/load/insecure-bank/start-servers.sh

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
#!/usr/bin/env bash
2+
3+
set -e
4+
5+
start_server() {
6+
local VARIANT=$1
7+
local JAVA_OPTS=$2
8+
9+
if [ -n "$CI_JOB_TOKEN" ]; then
10+
# Inside BP, so we can assume 24 CPU cores available and set CPU affinity
11+
CPU_AFFINITY_APP=$3
12+
else
13+
CPU_AFFINITY_APP=""
14+
fi
15+
16+
mkdir -p "${LOGS_DIR}/${VARIANT}"
17+
${CPU_AFFINITY_APP}java ${JAVA_OPTS} -Xms3G -Xmx3G -jar ${INSECURE_BANK} &> ${LOGS_DIR}/${VARIANT}/insecure-bank.log &PID=$!
18+
echo "${CPU_AFFINITY_APP}java ${JAVA_OPTS} -Xms3G -Xmx3G -jar ${INSECURE_BANK} &> ${LOGS_DIR}/${VARIANT}/insecure-bank.log [PID=$PID]"
19+
}
20+
21+
start_server "no_agent" "-Dserver.port=8080" "taskset -c 47 " &
22+
start_server "tracing" "-javaagent:${TRACER} -Dserver.port=8081" "taskset -c 46 " &
23+
start_server "profiling" "-javaagent:${TRACER} -Ddd.profiling.enabled=true -Dserver.port=8082" "taskset -c 45 " &
24+
start_server "iast" "-javaagent:${TRACER} -Ddd.iast.enabled=true -Dserver.port=8083" "taskset -c 44 " &
25+
start_server "iast_GLOBAL" "-javaagent:${TRACER} -Ddd.iast.enabled=true -Ddd.iast.context.mode=GLOBAL -Dserver.port=8084" "taskset -c 43 " &
26+
start_server "iast_FULL" "-javaagent:${TRACER} -Ddd.iast.enabled=true -Ddd.iast.detection.mode=FULL -Dserver.port=8085" "taskset -c 42 " &
27+
28+
wait

‎benchmark/load/petclinic/benchmark.json

Lines changed: 0 additions & 46 deletions
This file was deleted.

‎benchmark/load/petclinic/k6.js

Lines changed: 51 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,61 @@
11
import http from 'k6/http';
22
import {checkResponse, isOk} from "../../utils/k6.js";
33

4-
const baseUrl = 'http://localhost:8080';
4+
const variants = {
5+
"no_agent": {
6+
"APP_URL": 'http://localhost:8080',
7+
},
8+
"tracing": {
9+
"APP_URL": 'http://localhost:8081',
10+
},
11+
"profiling": {
12+
"APP_URL": 'http://localhost:8082',
13+
},
14+
"appsec": {
15+
"APP_URL": 'http://localhost:8083',
16+
},
17+
"iast": {
18+
"APP_URL": 'http://localhost:8084',
19+
},
20+
"code_origins": {
21+
"APP_URL": 'http://localhost:8085',
22+
}
23+
}
24+
25+
export const options = function (variants) {
26+
const scenarios = {};
27+
for (const variant of Object.keys(variants)) {
28+
scenarios[`load--petclinic--${variant}--warmup`] = {
29+
executor: 'constant-vus', // https://grafana.com/docs/k6/latest/using-k6/scenarios/executors/#all-executors
30+
vus: 5,
31+
duration: '20s',
32+
gracefulStop: '2s',
33+
env: {
34+
"APP_URL": variants[variant]["APP_URL"]
35+
}
36+
};
37+
38+
scenarios[`load--petclinic--${variant}--high_load`] = {
39+
executor: 'constant-vus',
40+
vus: 5,
41+
startTime: '22s',
42+
duration: '15s',
43+
gracefulStop: '2s',
44+
env: {
45+
"APP_URL": variants[variant]["APP_URL"]
46+
}
47+
};
48+
}
549

6-
export const options = {
7-
discardResponseBodies: true,
8-
vus: 5,
9-
iterations: 80000
10-
};
50+
return {
51+
discardResponseBodies: true,
52+
scenarios,
53+
}
54+
}(variants);
1155

1256
export default function () {
1357

1458
// find owner
15-
const ownersList = http.get(`${baseUrl}/owners?lastName=`);
59+
const ownersList = http.get(`${__ENV.APP_URL}/owners?lastName=`);
1660
checkResponse(ownersList, isOk);
1761
}

‎benchmark/load/petclinic/start-servers.sh

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
#!/usr/bin/env bash
2+
3+
set -e
4+
5+
start_server() {
6+
local VARIANT=$1
7+
local JAVA_OPTS=$2
8+
9+
if [ -n "$CI_JOB_TOKEN" ]; then
10+
# Inside BP, so we can assume 24 CPU cores available and set CPU affinity
11+
CPU_AFFINITY_APP=$3
12+
else
13+
CPU_AFFINITY_APP=""
14+
fi
15+
16+
mkdir -p "${LOGS_DIR}/${VARIANT}"
17+
${CPU_AFFINITY_APP}java ${JAVA_OPTS} -Xms2G -Xmx2G -jar ${PETCLINIC} &> ${LOGS_DIR}/${VARIANT}/petclinic.log &PID=$!
18+
echo "${CPU_AFFINITY_APP}java ${JAVA_OPTS} -Xms2G -Xmx2G -jar ${PETCLINIC} &> ${LOGS_DIR}/${VARIANT}/petclinic.log [PID=$!]"
19+
}
20+
21+
start_server "no_agent" "-Dserver.port=8080" "taskset -c 31-32 " &
22+
start_server "tracing" "-javaagent:${TRACER} -Dserver.port=8081" "taskset -c 33-34 " &
23+
start_server "profiling" "-javaagent:${TRACER} -Ddd.profiling.enabled=true -Dserver.port=8082" "taskset -c 35-36 " &
24+
start_server "appsec" "-javaagent:${TRACER} -Ddd.appsec.enabled=true -Dserver.port=8083" "taskset -c 37-38 " &
25+
start_server "iast" "-javaagent:${TRACER} -Ddd.iast.enabled=true -Dserver.port=8084" "taskset -c 39-40 " &
26+
start_server "code_origins" "-javaagent:${TRACER} -Ddd.code.origin.for.spans.enabled=true -Dserver.port=8085" "taskset -c 41-42 " &
27+
28+
wait

0 commit comments

Comments
 (0)