Skip to content

Commit 91a0514

Browse files
ruben-ayrapetyanLaszloLango
ruben-ayrapetyan
authored and
LaszloLango
committed
Estimate performance measurement inaccuracy in tools/perf.sh and tools/run-perf-test.sh
JerryScript-DCO-1.0-Signed-off-by: Ruben Ayrapetyan [email protected]
1 parent 0219f37 commit 91a0514

File tree

2 files changed

+350
-26
lines changed

2 files changed

+350
-26
lines changed

‎tools/perf.sh

Lines changed: 159 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
#!/bin/bash
22

3-
# Copyright 2014 Samsung Electronics Co., Ltd.
3+
# Copyright 2014-2016 Samsung Electronics Co., Ltd.
44
#
55
# Licensed under the Apache License, Version 2.0 (the "License");
66
# you may not use this file except in compliance with the License.
@@ -30,17 +30,166 @@ fi
3030
perf_values=$( (( for i in `seq 1 1 $ITERS`; do time $ENGINE "$BENCHMARK"; if [ $? -ne 0 ]; then exit 1; fi; done ) 2>&1 ) | \
3131
grep user | \
3232
sed "$time_regexp" | \
33-
awk 'BEGIN { min_v = -1; } { v = $1 * 60 + $2; if (min_v == -1 || v < min_v) { min_v = v; }; s += v; n += 1; } END { print s / n, min_v; }';
34-
if [ ${PIPESTATUS[0]} -ne 0 ]; then exit 1; fi;);
35-
36-
if [ $? -ne 0 ];
37-
then
38-
exit 1;
39-
fi;
33+
awk '{ print ($1 * 60 + $2); }';
34+
if [ ${PIPESTATUS[0]} -ne 0 ]; then exit 1; fi; );
4035
4136
if [ "$PRINT_MIN" == "-min" ]
4237
then
43-
echo $perf_values | cut -d ' ' -f 2
38+
perf_values=$( echo "$perf_values" | \
39+
awk "BEGIN {
40+
min_v = -1;
41+
}
42+
{
43+
if (min_v == -1 || $1 < min_v) {
44+
min_v = $1;
45+
}
46+
}
47+
END {
48+
print min_v
49+
}" || exit 1;
50+
);
51+
calc_status=$?
4452
else
45-
echo $perf_values | cut -d ' ' -f 1
53+
perf_values=$( echo "$perf_values" | \
54+
awk "BEGIN {
55+
n = 0
56+
}
57+
{
58+
n++
59+
a[n] = \$1
60+
}
61+
END {
62+
#
63+
# Values of 99% quantiles of two-sided t-distribution for given number of degrees of freedom
64+
#
65+
t_gamma_n_m1 [1] = 63.657
66+
t_gamma_n_m1 [2] = 9.9248
67+
t_gamma_n_m1 [3] = 5.8409
68+
t_gamma_n_m1 [4] = 4.6041
69+
t_gamma_n_m1 [5] = 4.0321
70+
t_gamma_n_m1 [6] = 3.7074
71+
t_gamma_n_m1 [7] = 3.4995
72+
t_gamma_n_m1 [8] = 3.3554
73+
t_gamma_n_m1 [9] = 3.2498
74+
t_gamma_n_m1 [10] = 3.1693
75+
t_gamma_n_m1 [11] = 3.1058
76+
t_gamma_n_m1 [12] = 3.0545
77+
t_gamma_n_m1 [13] = 3.0123
78+
t_gamma_n_m1 [14] = 2.9768
79+
t_gamma_n_m1 [15] = 2.9467
80+
t_gamma_n_m1 [16] = 2.9208
81+
t_gamma_n_m1 [17] = 2.8982
82+
t_gamma_n_m1 [18] = 2.8784
83+
t_gamma_n_m1 [19] = 2.8609
84+
t_gamma_n_m1 [20] = 2.8453
85+
t_gamma_n_m1 [21] = 2.8314
86+
t_gamma_n_m1 [22] = 2.8188
87+
t_gamma_n_m1 [23] = 2.8073
88+
t_gamma_n_m1 [24] = 2.7969
89+
t_gamma_n_m1 [25] = 2.7874
90+
t_gamma_n_m1 [26] = 2.7787
91+
t_gamma_n_m1 [27] = 2.7707
92+
t_gamma_n_m1 [28] = 2.7633
93+
t_gamma_n_m1 [29] = 2.7564
94+
t_gamma_n_m1 [30] = 2.75
95+
t_gamma_n_m1 [31] = 2.744
96+
t_gamma_n_m1 [32] = 2.7385
97+
t_gamma_n_m1 [33] = 2.7333
98+
t_gamma_n_m1 [34] = 2.7284
99+
t_gamma_n_m1 [35] = 2.7238
100+
t_gamma_n_m1 [36] = 2.7195
101+
t_gamma_n_m1 [37] = 2.7154
102+
t_gamma_n_m1 [38] = 2.7116
103+
t_gamma_n_m1 [39] = 2.7079
104+
t_gamma_n_m1 [40] = 2.7045
105+
t_gamma_n_m1 [41] = 2.7012
106+
t_gamma_n_m1 [42] = 2.6981
107+
t_gamma_n_m1 [43] = 2.6951
108+
t_gamma_n_m1 [44] = 2.6923
109+
t_gamma_n_m1 [45] = 2.6896
110+
t_gamma_n_m1 [46] = 2.687
111+
t_gamma_n_m1 [47] = 2.6846
112+
t_gamma_n_m1 [48] = 2.6822
113+
t_gamma_n_m1 [49] = 2.68
114+
t_gamma_n_m1 [50] = 2.6778
115+
116+
#
117+
# Sort array of measurements
118+
#
119+
for (i = 2; i <= n; i++) {
120+
j = i
121+
k = a [j]
122+
while (j > 1 && a [j - 1] > k) {
123+
a [j] = a [j - 1]
124+
j--
125+
}
126+
a [j] = k
127+
}
128+
129+
#
130+
# Remove 20% of lowest and 20% of highest values
131+
#
132+
n_20_percent = int (n / 5)
133+
134+
for (i = 1; i <= n_20_percent; i++) {
135+
delete a[n]
136+
n--
137+
}
138+
139+
for (i = 1; i <= n - n_20_percent; i++) {
140+
a[i] = a[i + n_20_percent]
141+
}
142+
143+
n -= n_20_percent
144+
145+
#
146+
# Calculate average
147+
#
148+
sum = 0
149+
for (i = 1; i <= n; i++) {
150+
sum += a[i]
151+
}
152+
153+
avg = sum / n
154+
155+
if (n > 1) {
156+
if (n - 1 <= 50) {
157+
t_coef = t_gamma_n_m1 [n - 1]
158+
} else {
159+
# For greater degrees of freedom, values of corresponding quantiles
160+
# are insignificantly less than the value.
161+
#
162+
# For example, the value for infinite number of freedoms is 2.5758
163+
#
164+
# So, to reduce table size, we take this, greater value,
165+
# overestimating inaccuracy for no more than 4%.
166+
#
167+
t_coef = t_gamma_n_m1 [50]
168+
}
169+
170+
#
171+
# Calculate inaccuracy estimation
172+
#
173+
sum_delta_squares = 0
174+
for (i = 1; i <= n; i++) {
175+
sum_delta_squares += (avg - a[i]) ^ 2
176+
}
177+
178+
delta = t_coef * sqrt (sum_delta_squares / (n * (n - 1)))
179+
180+
print avg, delta
181+
} else {
182+
print avg
183+
}
184+
}
185+
" || exit 1;
186+
);
187+
calc_status=$?
46188
fi
189+
190+
echo "$perf_values"
191+
192+
if [ $? -ne 0 ];
193+
then
194+
exit 1;
195+
fi;

0 commit comments

Comments
 (0)