Add support for comments

This wires up the comment logic from prettier. This needs some modification inside of prettier itself, so it won't work as is.

- locStart/End is ast-specific, we need a way to set it up for each parser. In this case I reused on that already existed: https://github.com/prettier/prettier/blob/master/src/util.js#L260-L262

- getSortedChildNodes as well, need a way to traverse the ast: https://github.com/prettier/prettier/blob/master/src/comments.js#L38

```js
        (node.kind && node.kind !== "Comment") ||
        node.ast_type && node.ast_type !== "comment") // <-- added
    ) {
```

- printComments as well: https://github.com/prettier/prettier/blob/master/src/comments.js#L928-L929

```js
  switch (comment.type || comment.kind || comment.ast_type) {
    case "comment":
      return comment.value;
    case "Comment":
```

If we tweak those two small things in prettier itself, then it's able to print comments using the heuristic. @azz, do you think you'd be able to extract those out to be parser-specific?

Author: vjeux

src/parser.js

@@ -25,11 +25,8 @@ function parse(text, parsers, opts) {
   const pythonExectuable = `python${opts.pythonVersion == "2" ? "" : "3"}`;
   const executionResult = parseText(text, pythonExectuable);
 
-  const ast = JSON.parse(executionResult.stdout.toString());
-
-  // TODO: add comments
-
-  ast.comments = [];
+  const res = executionResult.stdout.toString();
+  const ast = JSON.parse(res);
   return ast;
 }
 

tests/python_comments/__snapshots__/jsfmt.spec.js.snap

@@ -0,0 +1,31 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`comments.py 1`] = `
+if a:  # a
+    # b
+    c
+
+    # c
+
+    d
+
+    # e
+
+    call(
+        # a
+    )
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+if a: # a
+    # b
+    c
+
+    # c
+
+    d
+
+    # e
+
+    call()
+    # a
+
+`;

tests/python_comments/comments.py

@@ -0,0 +1,13 @@
+if a:  # a
+    # b
+    c
+
+    # c
+
+    d
+
+    # e
+
+    call(
+        # a
+    )

tests/python_comments/jsfmt.spec.js

@@ -0,0 +1 @@
+run_spec(__dirname, ["python"]);

tests/python_expressions/__snapshots__/jsfmt.spec.js.snap

@@ -1039,14 +1039,18 @@ a=1;a&=2
 a=1;a^=2
 a=1;a|=2
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+# https://bitbucket.org/pypy/pypy/src/f9e4e9cb7b1949c548cb16ab1520e7ebced75dda/pypy/interpreter/pyparser/test/expressions.py
+
 0
 
 7
 
 -3
 
+# 053
 24
 
+# 14L
 1
 
 3.9
@@ -1059,6 +1063,8 @@ a=1;a|=2
 
 90000000000000
 
+# 3j
+
 x = a + 1
 
 x = 1 - a
@@ -1083,8 +1089,13 @@ x = a + 1 * b
 
 x = a * b / c
 
+# x = a * (1 + c)
 x, y, z = 1, 2, 3
 
+# x = 'a' 'b' 'c'
+# del foo
+# del foo[bar]
+# del foo.bar
 l[0]
 
 k[v]
@@ -1105,6 +1116,8 @@ f("foo")("bar")("spam").read()[0]
 
 a.b[0][0]
 
+# a.b[0][:]
+# a.b[0][::]
 a.b[0][0].pop()[0].push("bar")("baz").spam
 
 a.b[0].read()[1][2].foo().spam()[0].bar
@@ -1119,8 +1132,12 @@ a.b[0].read()[1][2].foo().spam()[0].bar ** 2
 
 l[start:end] = l2
 
+# l[::] = l2
+# a = \`s\`
+# a = \`1 + 2 + f(3, 4)\`
 [a, b] = c
 
+# (a, b) = c
 [a, (b, c), d] = e
 
 a, (b, c), d = e
@@ -1133,6 +1150,9 @@ l = func()
 
 l = func(10)
 
+# l = func(10, 12, a, b=c, *args)
+# l = func(10, 12, a, b=c, **kwargs)
+# l = func(10, 12, a, b=c, *args, **kwargs)
 l = func(10, 12, a, b=c)
 
 e = l.pop(3)
@@ -1157,8 +1177,21 @@ l = [i for j in range(10) for i in range(j) if j % 2 == 0]
 
 l = [i for j in range(10) for i in range(j) if j % 2 == 0 and i % 2 == 0]
 
+# l = [(a, b) for (a,b,c) in l2]
+# l = [{a:b} for (a,b,c) in l2]
 l = [i for j in k if j % 2 == 0 j * 2 < 20 for i in j if i % 2 == 0]
 
+# l = (i for i in j)
+# l = (i for i in j if i%2 == 0)
+# l = (i for j in k for i in j)
+# l = (i for j in k for i in j if j%2==0)
+# l = (i for j in k if j%2 == 0 if j*2 < 20 for i in j if i%2==0)
+# l = (i for i in [ j*2 for j in range(10) ] )
+# l = [i for i in ( j*2 for j in range(10) ) ]
+# l = (i for i in [ j*2 for j in ( k*3 for k in range(10) ) ] )
+# l = [i for j in ( j*2 for j in [ k*3 for k in range(10) ] ) ]
+# l = f(i for i in j)
+
 l = {a: b,"c": 0}
 
 l = {}
@@ -1195,32 +1228,64 @@ a is b
 
 3 < x < 5
 
+# (3 < x) < 5
 a < b < c < d
 
+# (a < b) < (c < d)
+# a < (b < c) < d
+
 a = b
 
 c = d
 
+# a = b = c = d
+
 a.b = 2
 
 x = a.b
 
+# l[:]
+# l[::]
 l[1:2]
 
+# l[1:]
 l[:2]
 
+# l[1::]
+# l[:1:]
 l[::1]
 
+# l[1:2:]
 l[:1:2]
 
 l[1::2]
 
 l[0:1:2]
 
+# a.b.l[:]
 a.b.l[1:2]
 
+# a.b.l[1:]
+# a.b.l[:2]
 a.b.l[0:1:2]
 
+# a[1:2:3, 100]
+# a[:2:3, 100]
+# a[1::3, 100,]
+# a[1:2:, 100]
+# a[1:2, 100]
+# a[1:, 100,]
+# a[:2, 100]
+# a[:, 100]
+# a[100, 1:2:3,]
+# a[100, :2:3]
+# a[100, 1::3]
+# a[100, 1:2:,]
+# a[100, 1:2]
+# a[100, 1:]
+# a[100, :2,]
+# a[100, :]
+#
 import os
 
 import sys, os
@@ -1247,6 +1312,7 @@ from os import path, system
 
 from os import path as P, system as S
 
+# from os import (path as P, system as S,)
 from os import *
 
 if a == 1:
@@ -1259,6 +1325,10 @@ elif a == 2:
 else:
     a += 4
 
+# if a and not b == c: pass
+# if a and not not not b == c: pass
+# if 0: print 'foo'
+
 assert False
 
 assert a == 1
@@ -1267,12 +1337,33 @@ assert a == 1 and b == 2
 
 assert a == 1 and b == 2, "assertion failed"
 
+# exec a
+# exec "a=b+3"
+# exec a in f()
+# exec a in f(), g()
+
+# print
+# print a
+# print a,
+# print a, b
+# print a, "b", c
+# print >> err
+# print >> err, "error"
+# print >> err, "error",
+# print >> err, "error", a
+
+# global a
+# global a,b,c
+
 raise
 
 raise ValueError
 
 raise ValueError("error")
 
+# raise ValueError, "error"
+# raise ValueError, "error", foo
+
 try:
     a
 
@@ -1287,6 +1378,55 @@ try:
 except NameError:
     pass
 
+# try:
+#     a
+#     b
+# except NameError, err:
+#     pass
+# try:
+#     a
+#     b
+# except (NameError, ValueError):
+#     pass
+# try:
+#     a
+#     b
+# except (NameError, ValueError), err:
+#     pass
+# try:
+#     a
+# except NameError, err:
+#     pass
+# except ValueError, err:
+#     pass
+# def f():
+#     try:
+#         a
+#     except NameError, err:
+#         a = 1
+#         b = 2
+#     except ValueError, err:
+#         a = 2
+#         return a
+# try:
+#     a
+# except NameError, err:
+#     a = 1
+# except ValueError, err:
+#     a = 2
+# else:
+#     a += 3
+# try:
+#     a
+# finally:
+#     b
+# def f():
+#     try:
+#         return a
+#     finally:
+#         a = 3
+#         return 1
+
 def f():
     return 1
 
@@ -1314,6 +1454,12 @@ def f(*args):
 def f(**kwargs):
     return 1
 
+# def f(t=()): pass
+# def f(a, b, (c, d), e): pass
+# def f(a, b, (c, (d, e), f, (g, h))): pass
+# def f(a, b, (c, (d, e), f, (g, h)), i): pass
+# def f((a)): pass
+
 class Pdb(bdb.Bdb,cmd.Cmd):
     pass
 
@@ -1343,6 +1489,12 @@ def foo():
 
     return a
 
+# def foo():
+#     """doc"""; print 1
+#     a=1
+
+# """Docstring""";print 1
+
 def f():
     return
 

vendor/python/astexport.py

@@ -6,8 +6,15 @@
 
 
 def export_json(atok, pretty_print=False):
+    dict = export_dict(atok)
+    dict['comments'] = [{
+            'ast_type': 'comment',
+            'value': token.string,
+            'start': token.startpos,
+            'end': token.endpos,
+        } for token in atok.tokens if token.type == 57]
     return json.dumps(
-        export_dict(atok),
+        dict,
         indent=4 if pretty_print else None,
         sort_keys=True,
         separators=(",", ": ") if pretty_print else (",", ":")
@@ -53,6 +60,10 @@ def default_visit(self, node):
             # Use None as default when lineno/col_offset are not set
             args[attr] = meth(getattr(node, attr, None))
 
+        if hasattr(node, 'first_token'):
+            args['start'] = node.first_token.startpos
+            args['end'] = node.last_token.endpos
+
         args['source'] = self.atok.get_text(node)
 
         return args