Update Python integration

.travis.yml

@@ -4,22 +4,19 @@ node_js:
   - 6
   - 8
   - 9
+env:
+  - PYTHON_VERSION=2.7.14
+  - PYTHON_VERSION=3.6.3
 cache:
   yarn: true
   directories:
   - node_modules
-  - /home/travis/.pyenv_cache
+before_install:
+  - pyenv install -s $PYTHON_VERSION
+  - pyenv global $PYTHON_VERSION
+  - python --version
 install:
   - yarn install
-before_install:
-  - export PYTHON_BUILD_CACHE_PATH="/home/travis/.pyenv_cache"
-  - curl -L https://raw.githubusercontent.com/pyenv/pyenv-installer/master/bin/pyenv-installer | bash
-  - export PATH="/home/travis/.pyenv/bin:$PATH"
-  - eval "$(pyenv init -)"
-  - eval "$(pyenv virtualenv-init -)"
-  - pyenv install -s 2.7.11
-  - pyenv install -s 3.6.3
-  - pyenv global 2.7.11 3.6.3
 script:
   - yarn lint
-  - yarn test -- --runInBand
+  - yarn test --runInBand

jest.test.config.js

@@ -7,7 +7,6 @@ module.exports = {
   setupFiles: ["<rootDir>/tests_config/run_spec.js"],
   snapshotSerializers: ["<rootDir>/tests_config/raw-serializer.js"],
   testRegex: "jsfmt\\.spec\\.js$|__tests__/.*\\.js$",
-  testPathIgnorePatterns: ["tests/new_react", "tests/more_react"],
   collectCoverage: ENABLE_COVERAGE,
   collectCoverageFrom: ["src/**/*.js", "!<rootDir>/node_modules/"],
   transform: {}

package.json

@@ -7,8 +7,8 @@
   "author": "Lucas Azzola <@azz>",
   "license": "MIT",
   "files": [
-    "src",
-    "vendor"
+    "python",
+    "src"
   ],
   "engines": {
     "node": ">=6"
@@ -23,13 +23,16 @@
     "eslint-plugin-jest": "^21.5.0",
     "eslint-plugin-prettier": "^2.4.0",
     "jest": "^21.1.0",
-    "jest-runner-eslint": "^0.3.0"
+    "jest-runner-eslint": "^0.3.0",
+    "semver": "^5.4.1"
   },
   "scripts": {
     "lint": "prettier src/**/*.js --list-different",
     "test": "jest"
   },
   "jest": {
-    "projects": ["<rootDir>/jest.*.config.js"]
+    "projects": [
+      "<rootDir>/jest.*.config.js"
+    ]
   }
 }

vendor/python/astexport.py → python/patched/astexport.py

@@ -1,31 +1,8 @@
 import ast
-import fileinput
-import json
+import tokenize
 
-import asttokens
 
-
-def export_json(atok, pretty_print=False):
-    dict = export_dict(atok)
-    dict['comments'] = [{
-            'ast_type': 'comment',
-            'value': token.string,
-            'start': token.startpos,
-            'end': token.endpos,
-        } for token in atok.tokens if token.type == 57]
-    return json.dumps(
-        dict,
-        indent=4 if pretty_print else None,
-        sort_keys=True,
-        separators=(",", ": ") if pretty_print else (",", ":")
-    )
-
-
-def export_dict(atok):
-    return DictExportVisitor(atok).visit(atok.tree)
-
-
-class DictExportVisitor:
+class DictExportVisitor(object):
     ast_type_field = "ast_type"
 
     def __init__(self, atok):
@@ -102,21 +79,17 @@ def visit_field_Num_n(self, val):
             }
 
 
-def parse(source):
-    assert (isinstance(source, str))
-
-    atok = asttokens.ASTTokens(source, parse=True)
-
-    return atok
-
-
-def main():
-    source = "".join(fileinput.input())
-
-    tree = parse(source)
-    json = export_json(tree, True)
-    print(json)
+def export(atok):
+    exported_ast = DictExportVisitor(atok).visit(atok.tree)
 
+    exported_ast['comments'] = [
+        {
+            'ast_type': 'comment',
+            'value': token.string,
+            'start': token.startpos,
+            'end': token.endpos,
+        }
+        for token in atok.tokens if token.type == tokenize.COMMENT
+    ]
 
-if __name__ == '__main__':
-    main()
+    return exported_ast

python/prettier/__init__.py

@@ -0,0 +1,7 @@
+import os
+import sys
+
+base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+
+sys.path.insert(0, os.path.join(base_dir, 'vendor'))
+sys.path.insert(0, os.path.join(base_dir, 'patched'))

python/prettier/parser.py

@@ -0,0 +1,17 @@
+import json
+import sys
+
+import astexport
+import asttokens
+
+
+def main():
+    source = sys.stdin.read()
+    atok = asttokens.ASTTokens(source, parse=True)
+
+    exported_ast = astexport.export(atok)
+    json.dump(exported_ast, sys.stdout, separators=(',', ':'))
+
+
+if __name__ == '__main__':
+    main()

src/parser.js

@@ -3,31 +3,24 @@
 const spawnSync = require("child_process").spawnSync;
 const path = require("path");
 
-function parseText(text, pythonExecutable) {
-  const executionResult = spawnSync(
-    pythonExecutable,
-    [path.join(__dirname, "../vendor/python/astexport.py")],
-    {
-      input: text
-    }
-  );
-
-  const error = executionResult.stderr.toString();
-
-  if (error) {
-    throw new Error(error);
+function parse(text) {
+  const executionResult = spawnSync("python", ["-m", "prettier.parser"], {
+    env: {
+      PATH: process.env.PATH,
+      PYTHONPATH: [
+        path.join(__dirname, "../python"),
+        process.env.PYTHONPATH
+      ].join(path.delimiter)
+    },
+    input: text
+  });
+
+  if (executionResult.status) {
+    throw new Error(executionResult.stderr.toString());
   }
 
-  return executionResult;
-}
-
-function parse(text, parsers, opts) {
-  const pythonExectuable = `python${opts.pythonVersion == "2" ? "" : "3"}`;
-  const executionResult = parseText(text, pythonExectuable);
-
   const res = executionResult.stdout.toString();
-  const ast = JSON.parse(res);
-  return ast;
+  return JSON.parse(res);
 }
 
 module.exports = parse;

src/printer/index.js

@@ -128,6 +128,24 @@ function printForIn(path, print) {
   ]);
 }
 
+function printWith(path, items, print) {
+  return concat([
+    group(concat(["with", line, join(", ", items), ":"])),
+    indent(concat([hardline, printBody(path, print)]))
+  ]);
+}
+
+function printPython2With(path, items, print) {
+  items.push(printWithItem(path, print));
+
+  const n = path.getValue();
+  if (n.body.length === 1 && n.body[0].ast_type === "With") {
+    return concat(path.map(p => printPython2With(p, items, print), "body"));
+  }
+
+  return printWith(path, items, print);
+}
+
 function printWithItem(path, print) {
   const parts = [path.call(print, "context_expr")];
 
@@ -603,15 +621,9 @@ function genericPrint(path, options, print) {
     }
 
     case "With": {
-      // python 2 and 3
-      const items = n.items
-        ? path.map(print, "items")
-        : [printWithItem(path, print)];
-
-      return concat([
-        group(concat(["with", line, join(",", items), ":"])),
-        indent(concat([hardline, printBody(path, print)]))
-      ]);
+      return n.items
+        ? printWith(path, path.map(print, "items"), print) // Python 3
+        : printPython2With(path, [], print); // Python 2
     }
 
     case "withitem": {

tests/python3_args_mixed/__snapshots__/jsfmt.spec.js.snap

@@ -3,7 +3,7 @@
 exports[`args_mixed.py 1`] = `
 def hello(a, *, example=False, **kwargs):
     print("hello world", example)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 def hello(a, *, example=False, **kwargs):
     print("hello world", example)
 

tests/python3_args_mixed/jsfmt.spec.js

@@ -1,3 +1 @@
-run_spec(__dirname, ["python"], {
-  pythonVersion: "3"
-});
+run_spec(__dirname, ["python"], ">=3");

tests/python3_async_func/__snapshots__/jsfmt.spec.js.snap

@@ -3,7 +3,7 @@
 exports[`async_func.py 1`] = `
 async def hello(x, a=123, b = 456):
     print("hello world", a)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 async def hello(x, a=123, b=456):
     print("hello world", a)
 

tests/python3_async_func/jsfmt.spec.js

@@ -1 +1 @@
-run_spec(__dirname, ["python"], { pythonVersion: "3" });
+run_spec(__dirname, ["python"], ">=3");

tests/python3_aug_assign/__snapshots__/jsfmt.spec.js.snap

@@ -2,7 +2,7 @@
 
 exports[`aug_assign.py 1`] = `
 a @= 1
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 a @= 1
 
 `;

tests/python3_aug_assign/jsfmt.spec.js

@@ -1 +1 @@
-run_spec(__dirname, ["python"], { pythonVersion: "3" });
+run_spec(__dirname, ["python"], ">=3.5");

tests/python3_await/__snapshots__/jsfmt.spec.js.snap

@@ -7,7 +7,7 @@ async def example():
         'image': image,
         'username': data.user.screen_name,
     }))
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 async def example():
     await channel.put(json.dumps({
         "text": data.text,

tests/python3_await/jsfmt.spec.js

@@ -1 +1 @@
-run_spec(__dirname, ["python"], { pythonVersion: "3" });
+run_spec(__dirname, ["python"], ">=3.5");

tests/python3_f_strings/__snapshots__/jsfmt.spec.js.snap

@@ -6,7 +6,7 @@ precision = 4
 value = decimal.Decimal("12.34567")
 f"result: {value:{width}.{precision}}"
 rf"result: {value:{width}.{precision}}"
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 width = 10
 
 precision = 4

tests/python3_f_strings/jsfmt.spec.js

@@ -1 +1 @@
-run_spec(__dirname, ["python"], { pythonVersion: "3" });
+run_spec(__dirname, ["python"], ">=3.6");

tests/python3_kwargs_only/__snapshots__/jsfmt.spec.js.snap

@@ -3,7 +3,7 @@
 exports[`kwargs_only.py 1`] = `
 def hello(a, *, delete=False):
     print("hello world", delete)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 def hello(a, *, delete=False):
     print("hello world", delete)
 

tests/python3_kwargs_only/jsfmt.spec.js

@@ -1 +1 @@
-run_spec(__dirname, ["python"], { pythonVersion: "3" });
+run_spec(__dirname, ["python"], ">=3");

tests/python_args/__snapshots__/jsfmt.spec.js.snap

@@ -3,16 +3,7 @@
 exports[`args.py 1`] = `
 def hello(*args):
     print("hello world", args)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-def hello(*args):
-    print("hello world", args)
-
-`;
-
-exports[`args.py 2`] = `
-def hello(*args):
-    print("hello world", args)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 def hello(*args):
     print("hello world", args)
 

tests/python_args/jsfmt.spec.js

@@ -1,2 +1 @@
-run_spec(__dirname, ["python"], { pythonVersion: "2" });
-run_spec(__dirname, ["python"], { pythonVersion: "3" });
+run_spec(__dirname, ["python"], "*");

tests/python_args_mixed/__snapshots__/jsfmt.spec.js.snap

@@ -3,16 +3,7 @@
 exports[`args_mixed.py 1`] = `
 def hello(a, example=False, **kwargs):
     print("hello world", example)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-def hello(a, example=False, **kwargs):
-    print("hello world", example)
-
-`;
-
-exports[`args_mixed.py 2`] = `
-def hello(a, example=False, **kwargs):
-    print("hello world", example)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 def hello(a, example=False, **kwargs):
     print("hello world", example)
 

tests/python_args_mixed/jsfmt.spec.js

@@ -1,2 +1 @@
-run_spec(__dirname, ["python"], { pythonVersion: "2" });
-run_spec(__dirname, ["python"], { pythonVersion: "3" });
+run_spec(__dirname, ["python"], "*");

tests/python_assert/__snapshots__/jsfmt.spec.js.snap

@@ -3,17 +3,7 @@
 exports[`assert.py 1`] = `
 assert 3 + 3
 assert False, 'message'
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-assert 3 + 3
-
-assert False, "message"
-
-`;
-
-exports[`assert.py 2`] = `
-assert 3 + 3
-assert False, 'message'
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 assert 3 + 3
 
 assert False, "message"

tests/python_assert/jsfmt.spec.js

@@ -1,2 +1 @@
-run_spec(__dirname, ["python"], { pythonVersion: "2" });
-run_spec(__dirname, ["python"], { pythonVersion: "3" });
+run_spec(__dirname, ["python"], "*");