Merged pass and xfail test suite runners

Also applied some renaming and coding style unification to the
scripts.

JerryScript-DCO-1.0-Signed-off-by: Akos Kiss [email protected]

Author: akosthekiss

Makefile

@@ -312,10 +312,10 @@ $(eval $(call BUILD_RULE,unittests,$(NATIVE_SYSTEM),unittests,Build))
 # its non-deterministically vanishing .a files.
 define JSTEST_SUITE_RULE
 test-js.$(1).$(2): build-all.$$(NATIVE_SYSTEM)
-	$$(Q) $$(call SHLOG,./tools/runners/run-precommit-check-for-target.sh \
+	$$(Q) $$(call SHLOG,./tools/runners/run-test-suite.sh \
           $$(OUT_DIR)/$(1)/jerry \
           $$(OUT_DIR)/$(1)/check/$(2) \
-          $(3),$$(OUT_DIR)/$(1)/check/$(2)/jerry_test.log,Testing)
+          $(3),$$(OUT_DIR)/$(1)/check/$(2)/test.log,Testing)
 endef
 
 $(foreach __TARGET,$(JERRY_TEST_TARGETS), \
@@ -356,7 +356,7 @@ build: $(patsubst %,build-all.%,$(NATIVE_SYSTEM) $(foreach __SYSTEM,$(MCU_SYSTEM
 test-unit: unittests
 	$(Q) rm -rf $(OUT_DIR)/unittests/check
 	$(Q) mkdir -p $(OUT_DIR)/unittests/check
-	$(Q) $(call SHLOG,./tools/runners/run-unittests.sh $(OUT_DIR)/unittests,$(OUT_DIR)/unittests/unittests.log,Unit tests)
+	$(Q) $(call SHLOG,./tools/runners/run-unittests.sh $(OUT_DIR)/unittests,$(OUT_DIR)/unittests/check/unittests.log,Unit tests)
 
 .PHONY: test-js
 test-js: $(foreach __TARGET,$(JERRY_TEST_TARGETS),test-js.$(__TARGET))

tools/runners/run-test-pass.sh

@@ -15,8 +15,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-TIMEOUT=${TIMEOUT:=30}
-
 ENGINE=$1
 shift
 
@@ -26,45 +24,58 @@ shift
 ERROR_CODE=$1
 shift
 
-TESTS=$1/fail/$ERROR_CODE
+if [ $ERROR_CODE -eq 0 ]
+then
+    TESTS=$1
+    TIMEOUT=${TIMEOUT:=5}
+    PASS="PASS"
+else
+    TESTS=$1/fail/$ERROR_CODE
+    TIMEOUT=${TIMEOUT:=30}
+    PASS="XFAIL"
+fi
 shift
 
 JERRY_ARGS="$@"
 
 if [ ! -x $ENGINE ]
 then
-  echo "\"$ENGINE\" is not an executable file"
-  exit 1
+    echo "$0: $ENGINE: not an executable"
+    exit 1
 fi
 
 if [ ! -d $OUT_DIR ]
 then
-  mkdir -p $OUT_DIR
+    mkdir -p $OUT_DIR
 fi
 
-JS_FILES=$OUT_DIR/js.fail.files
-JERRY_ERROR=$OUT_DIR/jerry.error
-JERRY_OK=$OUT_DIR/jerry.passed
+JS_FILES=$OUT_DIR/test.$ERROR_CODE.files
+JERRY_ERROR=$OUT_DIR/test.$ERROR_CODE.failed
+JERRY_OK=$OUT_DIR/test.$ERROR_CODE.passed
 
 rm -f $JS_FILES $JERRY_ERROR $JERRY_OK
 
-if [ -d $TESTS ];
+if [ -d $TESTS ]
 then
- find $TESTS -name [^N]*.js -print | sort > $JS_FILES
+    if [ $ERROR_CODE -eq 0 ]
+    then
+        find $TESTS -path $TESTS/fail -prune -o -name "[^N]*.js" -print | sort > $JS_FILES
+    else
+        find $TESTS -name "[^N]*.js" -print | sort > $JS_FILES
+    fi
+elif [ -f $TESTS ]
+then
+    cp $TESTS $JS_FILES
 else
- if [ -f $TESTS ];
- then
-  cp $TESTS $JS_FILES
- else
-  exit 1
- fi;
-fi;
+    echo "$0: $TESTS: not a test suite"
+    exit 1
+fi
 total=$(cat $JS_FILES | wc -l)
 
 if [ "$total" -eq 0 ]
 then
- echo "No test to execute"
- exit 1
+    echo "$0: $TESTS: no test in test suite"
+    exit 1
 fi
 
 tested=1
@@ -75,17 +86,17 @@ JERRY_TEMP=`mktemp`
 
 for test in `cat $JS_FILES`
 do
-    echo -n "[${tested}/${total}] ${ENGINE} ${JERRY_ARGS} ${test}: "
+    echo -n "[$tested/$total] $ENGINE $JERRY_ARGS $test: "
 
-    ( ulimit -t $TIMEOUT; ${ENGINE} ${JERRY_ARGS} ${test} &>$JERRY_TEMP; exit $? );
+    ( ulimit -t $TIMEOUT; $ENGINE $JERRY_ARGS $test &>$JERRY_TEMP; exit $? )
     status_code=$?
 
     if [ $status_code -ne $ERROR_CODE ]
     then
         echo "FAIL ($status_code)"
         cat $JERRY_TEMP
 
-        echo "$status_code: ${test}" >> $JERRY_ERROR
+        echo "$status_code: $test" >> $JERRY_ERROR
         echo "============================================" >> $JERRY_ERROR
         cat $JERRY_TEMP >> $JERRY_ERROR
         echo "============================================" >> $JERRY_ERROR
@@ -94,9 +105,9 @@ do
 
         failed=$((failed+1))
     else
-        echo "XFAIL"
+        echo "$PASS"
 
-        echo "${test}" >> $JERRY_OK
+        echo "$test" >> $JERRY_OK
 
         passed=$((passed+1))
     fi
@@ -108,12 +119,12 @@ rm -f $JERRY_TEMP
 
 ratio=$(echo $passed*100/$total | bc)
 
-echo "[summary] ${ENGINE} ${JERRY_ARGS} ${TESTS}: ${passed} XFAIL, ${failed} FAIL, ${total} total, ${ratio}% success"
+echo "[summary] $ENGINE $JERRY_ARGS $TESTS: $passed $PASS, $failed FAIL, $total total, $ratio% success"
 
-if [ ${failed} -ne 0 ]
+if [ $failed -ne 0 ]
 then
-   echo "See $JERRY_ERROR for details about failures"
-   exit 1;
+    echo "$0: see $JERRY_ERROR for details about failures"
+    exit 1
 fi
 
 exit 0

tools/runners/run-test-xfail.sh renamed to tools/runners/run-test-suite-status.sh

@@ -1,6 +1,7 @@
 #!/bin/bash
 
-# Copyright 2015 Samsung Electronics Co., Ltd.
+# Copyright 2015-2016 Samsung Electronics Co., Ltd.
+# Copyright 2016 University of Szeged
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -14,32 +15,39 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-ENGINE_PATH="$1"
-OUTPUT_PATH="$2"
-TESTS_PATH="$3"
-TESTS_OPTS="$4"
+ENGINE_PATH=$1
+shift
+
+OUTPUT_PATH=$1
+shift
+
+TESTS_PATH=$1
+shift
+
+TESTS_OPTS="$@"
 
 [ -x $ENGINE_PATH ] || exit 1
 [[ -d $TESTS_PATH || -f $TESTS_PATH ]] || exit 1
 mkdir -p $OUTPUT_PATH || exit 1
 
-./tools/runners/run-test-pass.sh $ENGINE_PATH $OUTPUT_PATH $TESTS_PATH $TESTS_OPTS; status_code=$?
+failed=0
+
+./tools/runners/run-test-suite-status.sh $ENGINE_PATH $OUTPUT_PATH 0 $TESTS_PATH $TESTS_OPTS; status_code=$?
 if [ $status_code -ne 0 ]
 then
-  exit $status_code
+    failed=1
 fi
 
 if [ -d $TESTS_PATH/fail ]
 then
-  for error_code in `cd $TESTS_PATH/fail && ls -d [0-9]*`
-  do
-    ./tools/runners/run-test-xfail.sh $ENGINE_PATH $OUTPUT_PATH $error_code $TESTS_PATH $TESTS_OPTS; status_code=$?
-
-    if [ $status_code -ne 0 ]
-    then
-      exit $status_code
-    fi
-  done
+    for error_code in `cd $TESTS_PATH/fail && ls -d [0-9]*`
+    do
+        ./tools/runners/run-test-suite-status.sh $ENGINE_PATH $OUTPUT_PATH $error_code $TESTS_PATH $TESTS_OPTS; status_code=$?
+        if [ $status_code -ne 0 ]
+        then
+            failed=1
+        fi
+    done
 fi
 
-exit 0
+exit $failed