Add parallel test262 runner.

JerryScript-DCO-1.0-Signed-off-by: Evgeny Gavrin [email protected]

Author: egavrin

tools/runners/run-test-suite-test262.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright 2014-2015 Samsung Electronics Co., Ltd.
+# Copyright 2015 Samsung Electronics Co., Ltd.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -14,108 +14,58 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-TARGET=$1
-shift
+ENGINE="$1"
+REPORT_PREFIX="report"
+RUN_PIDS=""
+RESULT_OK=1
 
-PARSE_ONLY=""
-VERBOSE_REPORT=0
-TIMEOUT=5
+#rm "${REPORT_PREFIX}".* &> /dev/null
 
-function usage() {
-  echo "Usage: $0 {path to engine} [-p] [-v] [-t {timeout in seconds}]" >&2
-  echo "  -p - parser only mode" >&2
-  echo "  -v - verbose report mode" >&2
-  echo "  -t - timeout for each test" >&2
+#declare -a CHAPTER07_TO_TEST=("7.1" "7.2" "7.3" "7.4" "7.5" "7.6" "7.6.1" "7.7" "7.8" "7.8.1" "7.8.2" "7.8.3" "7.8.4" "7.8.5" "7.9" "7.9.1" "7.9.2")
+#declare -a CHAPTER08_TO_TEST=("8.1" "8.2" "8.3" "8.4" "8.5" "8.6" "8.6.1" "8.6.2" "8.7" "8.7.1" "8.7.2" "8.8" "8.9" "8.10" "8.10.1" "8.10.2" "8.10.3" "8.10.4" "8.10.5" "8.11" "8.12" "8.12.1" "8.12.2" "8.12.3" "8.12.4" "8.12.5" "8.12.6" "8.12.7" "8.12.8" "8.12.9")
+#declare -a CHAPTER09_TO_TEST=("9.1" "9.2" "9.3" "9.3.1" "9.4" "9.5" "9.6" "9.7" "9.8" "9.8.1" "9.9" "9.10" "9.11" "9.12")
+#declare -a CHAPTER10_TO_TEST=("10.1" "10.1.1" "10.2" "10.2.1" "10.2.2" "10.2.3" "10.3" "10.3.1" "10.4" "10.4.1" "10.4.2" "10.4.3" "10.5" "10.6")
+#declare -a CHAPTER11_TO_TEST=("11.1" "11.1.1" "11.1.2" "11.1.3" "11.1.4" "11.1.5" "11.1.6" "11.2" "11.2.1" "11.2.2" "11.2.3" "11.2.4" "11.2.5" "11.3" "11.3.1" "11.3.2" "11.4" "11.4.1" "11.4.2" "11.4.3" "11.4.4" "11.4.5" "11.4.6" "11.4.7" "11.4.8" "11.4.9" "11.5" "11.5.1" "11.5.2" "11.5.3" "11.6" "11.6.1" "11.6.2" "11.6.3" "11.7" "11.7.1" "11.7.2" "11.7.3" "11.8" "11.8.1" "11.8.2" "11.8.3" "11.8.4" "11.8.5" "11.8.6" "11.8.7" "11.9" "11.9.1" "11.9.2" "11.9.3" "11.9.4" "11.9.5" "11.9.6" "11.10" "11.11" "11.12" "11.13" "11.13.1" "11.13.2" "11.14")
+#declare -a CHAPTER12_TO_TEST=("12.1" "12.2" "12.2.1" "12.3" "12.4" "12.5" "12.6" "12.6.1" "12.6.2" "12.6.3" "12.6.4" "12.7" "12.8" "12.9" "12.10" "12.10.1" "12.11" "12.12" "12.13" "12.14" "12.14.1" "12.15")
+#declare -a CHAPTER13_TO_TEST=("13.1" "13.2" "13.2.1" "13.2.2" "13.2.3")
+#declare -a CHAPTER14_TO_TEST=("14.1")
+#declare -a CHAPTER14_TO_TEST=("12.6.4")
+#declare -a CHAPTER15_TO_TEST=("15.1" "15.1.1" "15.1.2" "15.1.3" "15.1.4" "15.1.5" "15.2" "15.2.1" "15.2.2" "15.2.3" "15.2.4" "15.2.5" "15.3" "15.3.1" "15.3.2" "15.3.3" "15.3.4" "15.3.5" "15.4" "15.4.1" "15.4.2" "15.4.3" "15.4.4" "15.4.5" "15.5" "15.5.1" "15.5.2" "15.5.3" "15.5.4" "15.5.5" "15.6" "15.6.1" "15.6.2" "15.6.3" "15.6.4" "15.6.5" "15.7" "15.7.1" "15.7.2" "15.7.3" "15.7.4" "15.7.5" "15.8" "15.8.1" "15.8.2" "15.9" "15.9.1" "15.9.2" "15.9.3" "15.9.4" "15.9.5" "15.9.6" "15.10" "15.10.1" "15.10.2" "15.10.3" "15.10.4" "15.10.5" "15.10.6" "15.10.7" "15.11" "15.11.1" "15.11.2" "15.11.3" "15.11.4" "15.11.5" "15.11.6" "15.11.7" "15.12" "15.12.1" "15.12.2" "15.12.3")
 
-  exit 1
-}
-
-[ -x "$TARGET" ] || usage
-
-while getopts pvt: OPTION; do
-    case "$OPTION" in
-      p)
-        PARSE_ONLY="--parse-only" ;;
-      v)
-        VERBOSE_REPORT=1 ;;
-      t)
-        TIMEOUT="$OPTARG"; [ "$TIMEOUT" -eq "$TIMEOUT" ] || usage;;
-      [?])
-        usage ;;
-    esac
-done
+declare -a CHAPTERS_TO_TEST=("ch06" "ch07" "ch08" "ch09" "ch10" "ch11" "ch12" "ch13" "ch14")
+declare -a SUBCHAPTERS_TO_TEST=("${CHAPTER07_TO_TEST[@]}" "${CHAPTER08_TO_TEST[@]}" "${CHAPTER09_TO_TEST[@]}" "${CHAPTER10_TO_TEST[@]}" "${CHAPTER11_TO_TEST[@]}" "${CHAPTER12_TO_TEST[@]}" "${CHAPTER13_TO_TEST[@]}" "${CHAPTER14_TO_TEST[@]}")
 
-TMP_DIR=`mktemp -d --tmpdir=./`
+declare -a CHAPTERS_WITH_SUBCHAPTERS=("${CHAPTERS_TO_TEST[@]}" "${SUBCHAPTERS_TO_TEST[@]}")
 
-echo $TMP_DIR | grep -q tmp || exit 1
-
-trap clean_on_exit INT
-
-function clean_on_exit() {
-  rm -rf $TMP_DIR
-
-  [[ $1 == "OK" ]] || exit 1
+function run_test262 () {
+  ./tools/packaging/test262.py --command "timeout 3s $1" --full-summary "$2" > "${REPORT_PREFIX}"."$2"
 }
 
-TEST_SUITE_PATH="./tests/test262/test"
-STA_JS="$TEST_SUITE_PATH/harness/sta-jerry.js"
-MAX_THREADS=${MAX_CPUS:-32}
-
-[ $VERBOSE_REPORT -eq 0 ] || rm -f jerry.error.*
-rm -f test262.report
-
-find $TEST_SUITE_PATH/suite -name *.js -printf "%p %D%i\0" | xargs -0 -n 1 -P $MAX_THREADS bash -c '
-  target=$0
-  sta_js_path=$1;
-  timeout=$2;
-  tmp_dir=$3;
-  test=`echo $4 | cut -d " " -f 1`;
-  test_id=`echo $4 | cut -d " " -f 2`;
-  chapter=`echo $test | cut -d "/" -f 6`;
-
-  grep "\* @negative" $test 2>&1 >/dev/null;
-  negative=$?;
-
-  cmd="$target $test $sta_js_path";
-  output=`ulimit -t $timeout; $cmd 2>&1`;
-  status=$?;
-  output=`echo "$output" | sed -e "s/^/ /"`
-
-  if [[ $status -eq 0 && $negative -eq 0 || $status -ne 0 && $negative -ne 0 ]];
-  then
-    (echo "====================";
-     echo "Run \"$cmd\" failed: $status";
-     echo "---------------------";
-     echo "$output";
-     echo;) >> $tmp_dir/jerry.error."$chapter"."$test_id";
-  else
-    (echo "====================";
-     echo "Run \"$cmd\" successful";
-     echo "---------------------";
-     echo "$output";
-     echo;) >> $tmp_dir/jerry.ok."$chapter"."$test_id";
-  fi;
-' "$TARGET $PARSE" $STA_JS $TIMEOUT $TMP_DIR 2>/dev/null;
-
-if [ $VERBOSE_REPORT -eq 1 ]
-then
-  for CHAPTER in `ls $TEST_SUITE_PATH/suite`;
-  do
-    cat $TMP_DIR/jerry.error."$CHAPTER".* > jerry.error."$CHAPTER"
-  done
-fi
+function show_report_results () {
+  echo ""
+  echo "Chapter $1:"
+  grep -A3 "=== Summary ===" "${REPORT_PREFIX}"."$1"
+  echo "==============="
+}
 
-cat $TMP_DIR/jerry.ok.* $TMP_DIR/jerry.error.* | grep "^Run" > test262.report
+echo "Starting test262 testing for ${ENGINE}."
 
-ok_num=`grep "successful$" test262.report | wc -l`
-error_num=`grep "Run .* failed: [0-9]*$" test262.report | wc -l`
+for TEST_NAME in "${CHAPTERS_WITH_SUBCHAPTERS[@]}"
+do
+  run_test262 "${ENGINE}" "$TEST_NAME" &
+  RUN_PIDS="$RUN_PIDS $!";
+done
 
-ok_percent=`echo $ok_num $error_num | awk '{print 100.0 * $1 / ($1 + $2);}'`
-error_percent=`echo $ok_num $error_num | awk '{print 100.0 * $2 / ($1 + $2);}'`
+for RUN_PIDS in $RUN_PIDS
+do
+  wait "$RUN_PIDS" || RESULT_OK=0
+done;
+#[ $RESULT_OK -eq 1 ] || exit 1
 
-echo -e "OK:\t$ok_num ($ok_percent %)"
-echo -e "FAIL:\t$error_num ($error_percent %)"
-echo
-echo -e "Report:\t$PWD/test262.report"
+echo "Testing is completed."
 
-clean_on_exit OK
+for TEST_NAME in "${CHAPTERS_WITH_SUBCHAPTERS[@]}"
+do
+  echo "$TEST_NAME"
+  show_report_results "$TEST_NAME"
+done