Let TokenizerTester/TreeTester take directory name

This change allows TokenizerTester and TreeTester to be given a
directory name rather than a list of files (or pathname with a shell
wildcard). And when given a directory name, they recurse the directory
looking for either *.test files (for TokenizerTester) or *.dat files
(for TreeTester), and then run the tests from those files.

Without this change, we can’t easily run TokenizerTester and TreeTester
from AntRun in Maven — because we can’t use shell wildcards in the “arg”
value for the Ant “java” task, and any list of files we otherwise
construct within Maven ends up getting put into the java arg value as a
single string (single argument) — including the spaces between filenames.

So it’s far easier to just give TokenizerTester and TreeTester a
directory name, and use directory recursion within Java to get all the
filenames (as this change does).

Author: sideshowbarker

test-src/nu/validator/htmlparser/test/TokenizerTester.java

@@ -22,6 +22,8 @@
 
 package nu.validator.htmlparser.test;
 
+import java.io.ByteArrayInputStream;
+import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStream;
@@ -31,6 +33,9 @@
 import java.io.StringReader;
 import java.io.UnsupportedEncodingException;
 import java.io.Writer;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Paths;
 
 import nu.validator.htmlparser.common.XmlViolationPolicy;
 import nu.validator.htmlparser.impl.ErrorReportingTokenizer;
@@ -97,6 +102,7 @@ private static boolean jsonDeepEquals(JSONValue one, JSONValue other) {
 
     private TokenizerTester(InputStream stream) throws TokenStreamException,
             RecognitionException, UnsupportedEncodingException {
+        tests = null;
         tokenHandler = new JSONArrayTokenHandler();
         driver = new Driver(new ErrorReportingTokenizer(tokenHandler));
         driver.setCommentPolicy(XmlViolationPolicy.ALLOW);
@@ -106,6 +112,9 @@ private TokenizerTester(InputStream stream) throws TokenStreamException,
         driver.setXmlnsPolicy(XmlViolationPolicy.ALLOW);
         driver.setErrorHandler(tokenHandler);
         writer = new OutputStreamWriter(System.out, "UTF-8");
+        if (stream == null) {
+            return;
+        }
         JSONParser jsonParser = new JSONParser(new InputStreamReader(stream,
                 "UTF-8"));
         JSONObject obj = (JSONObject) jsonParser.nextValue();
@@ -202,19 +211,48 @@ private void runTestInner(String inputString, JSONArray expectedTokens,
         }
     }
 
+    private void recurseDirectory(File directory) throws Throwable {
+        if (directory.canRead()) {
+            File[] files = directory.listFiles();
+            for (File file : files) {
+                if (file.isDirectory()) {
+                    recurseDirectory(file);
+                } else {
+                    test(file.getPath().toString());
+                }
+            }
+        }
+    }
+
+    private void test(String file)
+            throws IOException, TokenStreamException, RecognitionException,
+            SAXException {
+        if (!file.endsWith(".test")) {
+            return;
+        }
+        byte[] fileBytes = Files.readAllBytes(Paths.get(file));
+        String fileContent = new String(fileBytes, StandardCharsets.UTF_8);
+        String unescapedContent = fileContent.replace("\\\\u", "\\u");
+        byte[] newBytes = unescapedContent.getBytes(StandardCharsets.UTF_8);
+        ByteArrayInputStream bais = new ByteArrayInputStream(newBytes);
+        TokenizerTester tester = new TokenizerTester(bais);
+        tester.runTests();
+    }
+
+
     /**
      * @param args
-     * @throws RecognitionException
-     * @throws TokenStreamException
-     * @throws IOException
-     * @throws SAXException
+     * @throws Throwable
      */
-    public static void main(String[] args) throws TokenStreamException,
-            RecognitionException, SAXException, IOException {
+    public static void main(String[] args) throws Throwable {
         for (int i = 0; i < args.length; i++) {
-            TokenizerTester tester = new TokenizerTester(new FileInputStream(
-                    args[i]));
-            tester.runTests();
+            TokenizerTester tester = new TokenizerTester(null);
+            File file = new File(args[i]);
+            if (file.isDirectory()) {
+                tester.recurseDirectory(file);
+            } else {
+                tester.test(file.getPath().toString());
+            }
         }
     }
 

test-src/nu/validator/htmlparser/test/TreeTester.java

@@ -24,6 +24,7 @@
 
 import java.io.BufferedInputStream;
 import java.io.BufferedReader;
+import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStream;
@@ -39,18 +40,22 @@
 
 public class TreeTester {
 
-    private final BufferedInputStream aggregateStream;
+    private BufferedInputStream aggregateStream;
 
     private boolean streaming = false;
 
     /**
      * @param aggregateStream
      */
-    public TreeTester(InputStream aggregateStream) {
-        this.aggregateStream = new BufferedInputStream(aggregateStream);
+    public TreeTester() {
+        this.aggregateStream = null;
     }
 
-    private void runTests() throws Throwable {
+    private void runTests(String filename) throws Throwable {
+        if (!filename.endsWith(".dat")) {
+            return;
+        }
+        aggregateStream = new BufferedInputStream(new FileInputStream(filename));
         if (aggregateStream.read() != '#') {
             System.err.println("No hash at start!");
             return;
@@ -257,14 +262,35 @@ private boolean skipLabel() throws IOException {
         }
     }
 
+    private void recurseDirectory(File directory) throws Throwable {
+        if ("scripted".equals(directory.getName())) {
+            return;
+        }
+        if (directory.canRead()) {
+            File[] files = directory.listFiles();
+            for (File file : files) {
+                if (file.isDirectory()) {
+                    recurseDirectory(file);
+                } else {
+                    runTests(file.getPath().toString());
+                }
+            }
+        }
+    }
+
     /**
      * @param args
      * @throws Throwable
      */
     public static void main(String[] args) throws Throwable {
         for (int i = 0; i < args.length; i++) {
-            TreeTester tester = new TreeTester(new FileInputStream(args[i]));
-            tester.runTests();
+            TreeTester tester = new TreeTester();
+            File file = new File(args[i]);
+            if (file.isDirectory()) {
+                tester.recurseDirectory(file);
+            } else {
+                tester.runTests(file.getPath().toString());
+            }
         }
     }