Unified test runner scripts and their output format

Eliminated nothing-to-stdout everything-to-log-file policy: info
is printed to stdout and it is the caller's responsibility to
redirect it to a file if needed.

JerryScript-DCO-1.0-Signed-off-by: Akos Kiss [email protected]

Author: akosthekiss

Makefile

@@ -315,7 +315,7 @@ test-js.$(1).$(2): build-all.$$(NATIVE_SYSTEM)
 	$$(Q) $$(call SHLOG,./tools/runners/run-precommit-check-for-target.sh \
           $$(OUT_DIR)/$(1)/jerry \
           $$(OUT_DIR)/$(1)/check/$(2) \
-          $(3), $$(OUT_DIR)/$(1)/check/$(2)/jerry_test.log,Testing)
+          $(3),$$(OUT_DIR)/$(1)/check/$(2)/jerry_test.log,Testing)
 endef
 
 $(foreach __TARGET,$(JERRY_TEST_TARGETS), \
@@ -356,8 +356,7 @@ build: $(patsubst %,build-all.%,$(NATIVE_SYSTEM) $(foreach __SYSTEM,$(MCU_SYSTEM
 test-unit: unittests
 	$(Q) rm -rf $(OUT_DIR)/unittests/check
 	$(Q) mkdir -p $(OUT_DIR)/unittests/check
-	$(Q) ./tools/runners/run-unittests.sh $(OUT_DIR)/unittests || \
-         (echo "Unit tests run failed. See $(OUT_DIR)/unittests/unit_tests_run.log for details."; exit 1;)
+	$(Q) $(call SHLOG,./tools/runners/run-unittests.sh $(OUT_DIR)/unittests,$(OUT_DIR)/unittests/unittests.log,Unit tests)
 
 .PHONY: test-js
 test-js: $(foreach __TARGET,$(JERRY_TEST_TARGETS),test-js.$(__TARGET))

tools/runners/run-test-pass.sh

@@ -1,6 +1,7 @@
 #!/bin/bash
 
 # Copyright 2014-2016 Samsung Electronics Co., Ltd.
+# Copyright 2016 University of Szeged
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -16,80 +17,71 @@
 
 TIMEOUT=${TIMEOUT:=5}
 
-START_DIR=`pwd`
-
-ENGINE=$START_DIR/$1
+ENGINE=$1
 shift
 
 OUT_DIR=$1
 shift
 
-TESTS=$START_DIR/$1
+TESTS=$1
 shift
 
-ECHO_PROGRESS=1
-JERRY_ARGS=
-while (( "$#" ))
-do
-  if [ "$1" = "--parse-only" ]
-  then
-    JERRY_ARGS="$JERRY_ARGS $1"
-  fi
-  if [ "$1" = "--output-to-log" ]
-  then
-    exec 1>$OUT_DIR/jerry_test.log
-    ECHO_PROGRESS=0
-  fi
-
-  shift
-done
+JERRY_ARGS="$@"
 
 if [ ! -x $ENGINE ]
 then
-  echo \"$ENGINE\" is not an executable file
+  echo "\"$ENGINE\" is not an executable file"
+  exit 1
 fi
 
-cd $OUT_DIR
+if [ ! -d $OUT_DIR ]
+then
+  mkdir -p $OUT_DIR
+fi
 
-JS_FILES=js.files
-JERRY_ERROR=jerry.error
-JERRY_OK=jerry.passed
+JS_FILES=$OUT_DIR/js.files
+JERRY_ERROR=$OUT_DIR/jerry.error
+JERRY_OK=$OUT_DIR/jerry.passed
 
-rm -f $JS_FILES $JERRY_ERROR
+rm -f $JS_FILES $JERRY_ERROR $JERRY_OK
 
 if [ -d $TESTS ];
 then
  find $TESTS -path $TESTS/fail -prune -o -name "[^N]*.js" -print | sort > $JS_FILES
 else
  if [ -f $TESTS ];
  then
-  awk "{print \"$START_DIR/\"\$1;}" $TESTS > $JS_FILES
+  cp $TESTS $JS_FILES
+ else
+  exit 1
  fi;
 fi;
 total=$(cat $JS_FILES | wc -l)
 
-tested=0
+if [ "$total" -eq 0 ]
+then
+ echo "No test to execute"
+ exit 1
+fi
+
+tested=1
 failed=0
 passed=0
 
-JERRY_TEMP=jerry.tmp
-
-echo "  Passed / Failed / Tested / Total / Percent"
+JERRY_TEMP=`mktemp`
 
 for test in `cat $JS_FILES`
 do
-    percent=$(echo $tested*100/$total | bc)
+    echo -n "[${tested}/${total}] ${ENGINE} ${JERRY_ARGS} ${test}: "
 
     ( ulimit -t $TIMEOUT; ${ENGINE} ${JERRY_ARGS} ${test} &>$JERRY_TEMP; exit $? );
     status_code=$?
 
-    if [ $ECHO_PROGRESS -eq 1 ]
-    then
-      printf "\r\e[2K[ %6d / %6d / %6d / %5d / %3d%%    ]" ${passed} ${failed} ${tested} ${total} ${percent}
-    fi
-
     if [ $status_code -ne 0 ]
     then
+        echo "FAIL ($status_code)"
+        cat $JERRY_TEMP
+
         echo "$status_code: ${test}" >> $JERRY_ERROR
         echo "============================================" >> $JERRY_ERROR
         cat $JERRY_TEMP >> $JERRY_ERROR
@@ -99,7 +91,10 @@ do
 
         failed=$((failed+1))
     else
+        echo "PASS"
+
         echo "${test}" >> $JERRY_OK
+
         passed=$((passed+1))
     fi
 
@@ -108,16 +103,9 @@ done
 
 rm -f $JERRY_TEMP
 
-printf "\r\e[2K[ %6d / %6d / %6d / %5d / %3d%%    ]\n" ${passed} ${failed} ${tested} ${total} ${percent}
-
 ratio=$(echo $passed*100/$total | bc)
 
-echo ==========================
-echo "Number of tests passed:   ${passed}"
-echo "Number of tests failed:   ${failed}"
-echo --------------------------
-echo "Total number of tests:    ${total}"
-echo "Passed:                   ${ratio}%"
+echo "[summary] ${ENGINE} ${JERRY_ARGS} ${TESTS}: ${passed} PASS, ${failed} FAIL, ${total} total, ${ratio}% success"
 
 if [ ${failed} -ne 0 ]
 then

tools/runners/run-test-xfail.sh

@@ -1,6 +1,7 @@
 #!/bin/bash
 
-# Copyright 2014-2015 Samsung Electronics Co., Ltd.
+# Copyright 2014-2016 Samsung Electronics Co., Ltd.
+# Copyright 2016 University of Szeged
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -16,9 +17,7 @@
 
 TIMEOUT=${TIMEOUT:=30}
 
-START_DIR=`pwd`
-
-ENGINE=$START_DIR/$1
+ENGINE=$1
 shift
 
 OUT_DIR=$1
@@ -27,41 +26,27 @@ shift
 ERROR_CODE=$1
 shift
 
-TESTS=$START_DIR/$1/fail/$ERROR_CODE
+TESTS=$1/fail/$ERROR_CODE
 shift
 
-ECHO_PROGRESS=1
-JERRY_ARGS=
-while (( "$#" ))
-do
-  if [ "$1" = "--parse-only" ]
-  then
-    JERRY_ARGS="$JERRY_ARGS $1"
-  fi
-  if [ "$1" = "--output-to-log" ]
-  then
-    exec 1>$OUT_DIR/jerry_test.log
-    ECHO_PROGRESS=0
-  fi
-  shift
-done
+JERRY_ARGS="$@"
 
 if [ ! -x $ENGINE ]
 then
-  echo \"$ENGINE\" is not an executable file
+  echo "\"$ENGINE\" is not an executable file"
+  exit 1
 fi
 
 if [ ! -d $OUT_DIR ]
 then
   mkdir -p $OUT_DIR
 fi
-cd $OUT_DIR
 
-JS_FILES=js.fail.files
-JERRY_ERROR=jerry.error
-JERRY_OK=jerry.passed
+JS_FILES=$OUT_DIR/js.fail.files
+JERRY_ERROR=$OUT_DIR/jerry.error
+JERRY_OK=$OUT_DIR/jerry.passed
 
-rm -f $JS_FILES $JERRY_ERROR
+rm -f $JS_FILES $JERRY_ERROR $JERRY_OK
 
 if [ -d $TESTS ];
 then
@@ -76,28 +61,30 @@ else
 fi;
 total=$(cat $JS_FILES | wc -l)
 
-tested=0
+if [ "$total" -eq 0 ]
+then
+ echo "No test to execute"
+ exit 1
+fi
+
+tested=1
 failed=0
 passed=0
 
-JERRY_TEMP=jerry.tmp
-
-echo "  Passed / Failed / Tested / Total / Percent"
+JERRY_TEMP=`mktemp`
 
 for test in `cat $JS_FILES`
 do
-    percent=$(echo $tested*100/$total | bc)
+    echo -n "[${tested}/${total}] ${ENGINE} ${JERRY_ARGS} ${test}: "
 
-    ( ulimit -t $TIMEOUT; ${ENGINE} ${test} ${JERRY_ARGS} >&$JERRY_TEMP; exit $? );
+    ( ulimit -t $TIMEOUT; ${ENGINE} ${JERRY_ARGS} ${test} &>$JERRY_TEMP; exit $? );
     status_code=$?
 
-    if [ $ECHO_PROGRESS -eq 1 ]
-    then
-      printf "\r\e[2K[ %6d / %6d / %6d / %5d / %3d%%    ]" ${passed} ${failed} ${tested} ${total} ${percent}
-    fi
-
     if [ $status_code -ne $ERROR_CODE ]
     then
+        echo "FAIL ($status_code)"
+        cat $JERRY_TEMP
+
         echo "$status_code: ${test}" >> $JERRY_ERROR
         echo "============================================" >> $JERRY_ERROR
         cat $JERRY_TEMP >> $JERRY_ERROR
@@ -107,25 +94,21 @@ do
 
         failed=$((failed+1))
     else
+        echo "XFAIL"
+
         echo "${test}" >> $JERRY_OK
+
         passed=$((passed+1))
     fi
 
     tested=$((tested+1))
 done
 
-rm $JERRY_TEMP
-
-printf "\r\e[2K[ %6d / %6d / %6d / %5d / %3d%%    ]\n" ${passed} ${failed} ${tested} ${total} ${percent}
+rm -f $JERRY_TEMP
 
 ratio=$(echo $passed*100/$total | bc)
 
-echo ==========================
-echo "Number of tests passed:   ${passed}"
-echo "Number of tests failed:   ${failed}"
-echo --------------------------
-echo "Total number of tests:    ${total}"
-echo "Passed:                   ${ratio}%"
+echo "[summary] ${ENGINE} ${JERRY_ARGS} ${TESTS}: ${passed} XFAIL, ${failed} FAIL, ${total} total, ${ratio}% success"
 
 if [ ${failed} -ne 0 ]
 then