Skip to content

Add parallel test262 runner. #275

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Jul 1, 2015
Merged

Add parallel test262 runner. #275

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
181 changes: 89 additions & 92 deletions tools/runners/run-test-suite-test262.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
#!/bin/bash

# Copyright 2014-2015 Samsung Electronics Co., Ltd.
# Copyright 2015 Samsung Electronics Co., Ltd.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
Expand All @@ -14,108 +14,105 @@
# See the License for the specific language governing permissions and
# limitations under the License.

TARGET=$1
shift

PARSE_ONLY=""
VERBOSE_REPORT=0
TIMEOUT=5

function usage() {
echo "Usage: $0 {path to engine} [-p] [-v] [-t {timeout in seconds}]" >&2
echo " -p - parser only mode" >&2
echo " -v - verbose report mode" >&2
echo " -t - timeout for each test" >&2
ENGINE="$1"
PATH_TO_TEST262="$2"
REPORT_PREFIX="report"
RUN_PIDS=""
RESULT_OK=1
TIMEOUT="5s"

if [ $# -lt 2 ]
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What is the current usage of this script? We should write a comment at the beginning of this file at least.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'll add like in #206

then
echo "This script performs parallel test262 compliance testing of the specified engine."
echo ""
echo "Usage:"
echo " 1st parameter: JavaScript engine to be tested."
echo " 2nd parameter: path to the directory with official test262 testsuite."
echo " 3rd parameter: (optional) call this script with the '--sub-chapters' flag to print the detailed report."
echo ""
echo "Example:"
echo " ./run-test-suite-test262.sh <engine> <test262_dir> --sub-chapters"
exit 1
}
fi

[ -x "$TARGET" ] || usage

while getopts pvt: OPTION; do
case "$OPTION" in
p)
PARSE_ONLY="--parse-only" ;;
v)
VERBOSE_REPORT=1 ;;
t)
TIMEOUT="$OPTARG"; [ "$TIMEOUT" -eq "$TIMEOUT" ] || usage;;
[?])
usage ;;
esac
done

TMP_DIR=`mktemp -d --tmpdir=./`
rm "${REPORT_PREFIX}".* &> /dev/null

declare -a CHAPTER07_TO_TEST=("7.1" "7.2" "7.3" "7.4" "7.5" "7.6" "7.6.1" "7.7" \
"7.8" "7.8.1" "7.8.2" "7.8.3" "7.8.4" "7.8.5" "7.9" "7.9.2")
declare -a CHAPTER08_TO_TEST=("8.1" "8.2" "8.3" "8.4" "8.5" "8.6" "8.6.1" "8.6.2" "8.7" "8.7.1" "8.7.2" "8.8" "8.12" \
"8.12.1" "8.12.3" "8.12.4" "8.12.5" "8.12.6" "8.12.7" "8.12.8" "8.12.9")
declare -a CHAPTER09_TO_TEST=("9.1" "9.2" "9.3" "9.3.1" "9.4" "9.5" "9.6" "9.7" "9.8" "9.8.1" "9.9")
declare -a CHAPTER10_TO_TEST=("10.1" "10.1.1" "10.2" "10.2.1" "10.2.2" "10.2.3" "10.3" "10.3.1" "10.4" "10.4.1" \
"10.4.2" "10.4.3" "10.5" "10.6")
declare -a CHAPTER11_TO_TEST=("11.1" "11.1.1" "11.1.2" "11.1.3" "11.1.4" "11.1.5" "11.1.6" "11.2" "11.2.1" "11.2.2" \
"11.2.3" "11.2.4" "11.3" "11.3.1" "11.3.2" "11.4" "11.4.1" "11.4.2" "11.4.3" \
"11.4.4" "11.4.5" "11.4.6" "11.4.7" "11.4.8" "11.4.9" "11.5" "11.5.1" "11.5.2" "11.5.3" \
"11.6" "11.6.1" "11.6.2" "11.7" "11.7.1" "11.7.2" "11.7.3" "11.8" "11.8.1" "11.8.2" \
"11.8.3" "11.8.4" "11.8.6" "11.8.7" "11.9" "11.9.1" "11.9.2" "11.9.4" "11.9.5" "11.10" \
"11.11" "11.12" "11.13" "11.13.1" "11.13.2" "11.14")
declare -a CHAPTER12_TO_TEST=("12.1" "12.2" "12.2.1" "12.3" "12.4" "12.5" "12.6" "12.6.1" "12.6.2" "12.6.3" "12.6.4" \
"12.7" "12.8" "12.9" "12.10" "12.10.1" "12.11" "12.12" "12.13" "12.14" "12.14.1")
declare -a CHAPTER13_TO_TEST=("13.1" "13.2" "13.2.1" "13.2.2" "13.2.3")
declare -a CHAPTER14_TO_TEST=("14.1")
declare -a CHAPTER14_TO_TEST=("12.6.4")
declare -a CHAPTER15_TO_TEST=("15.1" "15.1.1" "15.1.2" "15.1.3" "15.1.4" "15.1.5" "15.2" "15.2.1" "15.2.2" "15.2.3" \
"15.2.4" "15.2.5" "15.3" "15.3.1" "15.3.2" "15.3.3" "15.3.4" "15.3.5" "15.4" "15.4.1" \
"15.4.2" "15.4.3" "15.4.4" "15.4.5" "15.5" "15.5.1" "15.5.2" "15.5.3" "15.5.4" "15.5.5" \
"15.6" "15.6.1" "15.6.2" "15.6.3" "15.6.4" "15.6.5" "15.7" "15.7.1" "15.7.2" "15.7.3" \
"15.7.4" "15.7.5" "15.8" "15.8.1" "15.8.2" "15.9" "15.9.1" "15.9.2" "15.9.3" "15.9.4" \
"15.9.5" "15.9.6" "15.10" "15.10.1" "15.10.2" "15.10.3" "15.10.4" "15.10.5" "15.10.6" \
"15.10.7" "15.11" "15.11.1" "15.11.2" "15.11.3" "15.11.4" "15.11.5" "15.11.6" "15.11.7" \
"15.12" "15.12.1" "15.12.2" "15.12.3")
declare -a FULL_CHAPTERS_TO_TEST=("ch06" "ch07" "ch08" "ch09" "ch10" "ch11" "ch12" "ch13" "ch14" "ch15")
declare -a SUB_CHAPTERS_TO_TEST=("${CHAPTER07_TO_TEST[@]}" "${CHAPTER08_TO_TEST[@]}" "${CHAPTER09_TO_TEST[@]}" \
"${CHAPTER10_TO_TEST[@]}" "${CHAPTER11_TO_TEST[@]}" "${CHAPTER12_TO_TEST[@]}" \
"${CHAPTER13_TO_TEST[@]}" "${CHAPTER14_TO_TEST[@]}" "${CHAPTER15_TO_TEST[@]}")

declare -a CHAPTERS_TO_TEST=("${FULL_CHAPTERS_TO_TEST[@]}")

if [[ $* == *--sub-chapters* ]]
then
declare -a CHAPTERS_TO_TEST=("${SUB_CHAPTERS_TO_TEST[@]}")
fi

echo $TMP_DIR | grep -q tmp || exit 1
function run_test262 () {
ARG_ENGINE="$1"
ARG_TEST262_PATH="$2"
ARG_CHAPTER="$3"

trap clean_on_exit INT
"${ARG_TEST262_PATH}"/tools/packaging/test262.py --command "timeout ${TIMEOUT} ${ARG_ENGINE}" \
--tests="${ARG_TEST262_PATH}" --full-summary "${ARG_CHAPTER}" \
> "${REPORT_PREFIX}"."${ARG_CHAPTER}"
}

function clean_on_exit() {
rm -rf $TMP_DIR
function show_report_results () {
ARG_CHAPTER="$1"

[[ $1 == "OK" ]] || exit 1
echo ""
echo "Chapter ${ARG_CHAPTER}:"
grep -A3 "=== Summary ===" "${REPORT_PREFIX}"."${ARG_CHAPTER}"
echo "==============="
}

TEST_SUITE_PATH="./tests/test262/test"
STA_JS="$TEST_SUITE_PATH/harness/sta-jerry.js"
MAX_THREADS=${MAX_CPUS:-32}

[ $VERBOSE_REPORT -eq 0 ] || rm -f jerry.error.*
rm -f test262.report

find $TEST_SUITE_PATH/suite -name *.js -printf "%p %D%i\0" | xargs -0 -n 1 -P $MAX_THREADS bash -c '
target=$0
sta_js_path=$1;
timeout=$2;
tmp_dir=$3;
test=`echo $4 | cut -d " " -f 1`;
test_id=`echo $4 | cut -d " " -f 2`;
chapter=`echo $test | cut -d "/" -f 6`;

grep "\* @negative" $test 2>&1 >/dev/null;
negative=$?;

cmd="$target $test $sta_js_path";
output=`ulimit -t $timeout; $cmd 2>&1`;
status=$?;
output=`echo "$output" | sed -e "s/^/ /"`

if [[ $status -eq 0 && $negative -eq 0 || $status -ne 0 && $negative -ne 0 ]];
then
(echo "====================";
echo "Run \"$cmd\" failed: $status";
echo "---------------------";
echo "$output";
echo;) >> $tmp_dir/jerry.error."$chapter"."$test_id";
else
(echo "====================";
echo "Run \"$cmd\" successful";
echo "---------------------";
echo "$output";
echo;) >> $tmp_dir/jerry.ok."$chapter"."$test_id";
fi;
' "$TARGET $PARSE" $STA_JS $TIMEOUT $TMP_DIR 2>/dev/null;

if [ $VERBOSE_REPORT -eq 1 ]
then
for CHAPTER in `ls $TEST_SUITE_PATH/suite`;
do
cat $TMP_DIR/jerry.error."$CHAPTER".* > jerry.error."$CHAPTER"
done
fi
echo "Starting test262 testing for ${ENGINE}."

cat $TMP_DIR/jerry.ok.* $TMP_DIR/jerry.error.* | grep "^Run" > test262.report

ok_num=`grep "successful$" test262.report | wc -l`
error_num=`grep "Run .* failed: [0-9]*$" test262.report | wc -l`
for TEST_NAME in "${CHAPTERS_TO_TEST[@]}"
do
run_test262 "${ENGINE}" "${PATH_TO_TEST262}" "$TEST_NAME" &
RUN_PIDS="$RUN_PIDS $!";
done

ok_percent=`echo $ok_num $error_num | awk '{print 100.0 * $1 / ($1 + $2);}'`
error_percent=`echo $ok_num $error_num | awk '{print 100.0 * $2 / ($1 + $2);}'`
for RUN_PIDS in $RUN_PIDS
do
wait "$RUN_PIDS" || RESULT_OK=0
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Don't we need a timeout for tests?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm passing timeout via command --command "timeout 3s $ARG_ENGINE".
Btw, it's good idea to make it variable or flag, thanks.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed.

done;
#[ $RESULT_OK -eq 1 ] || exit 1

echo -e "OK:\t$ok_num ($ok_percent %)"
echo -e "FAIL:\t$error_num ($error_percent %)"
echo
echo -e "Report:\t$PWD/test262.report"
echo "Testing is completed."

clean_on_exit OK
for TEST_NAME in "${CHAPTERS_TO_TEST[@]}"
do
echo "$TEST_NAME"
show_report_results "$TEST_NAME"
done