diff --git a/Gulpfile.ts b/Gulpfile.ts index d677b042250e6..2d17325c4d880 100644 --- a/Gulpfile.ts +++ b/Gulpfile.ts @@ -34,7 +34,6 @@ import through2 = require("through2"); import merge2 = require("merge2"); import intoStream = require("into-stream"); import * as os from "os"; -import Linter = require("tslint"); import fold = require("travis-fold"); const gulp = helpMaker(originalGulp); const mochaParallel = require("./scripts/mocha-parallel.js"); @@ -929,26 +928,6 @@ gulp.task("build-rules", "Compiles tslint rules to js", () => { .pipe(gulp.dest(dest)); }); -function getLinterOptions() { - return { - configuration: require("./tslint.json"), - formatter: "prose", - formattersDirectory: undefined, - rulesDirectory: "built/local/tslint" - }; -} - -function lintFileContents(options, path, contents) { - const ll = new Linter(path, contents, options); - console.log("Linting '" + path + "'."); - return ll.lint(); -} - -function lintFile(options, path) { - const contents = fs.readFileSync(path, "utf8"); - return lintFileContents(options, path, contents); -} - const lintTargets = [ "Gulpfile.ts", "src/compiler/**/*.ts", @@ -960,29 +939,72 @@ const lintTargets = [ "tests/*.ts", "tests/webhost/*.ts" // Note: does *not* descend recursively ]; +function sendNextFile(files: {path: string}[], child: cp.ChildProcess, callback: (failures: number) => void, failures: number) { + const file = files.pop(); + if (file) { + console.log(`Linting '${file.path}'.`); + child.send({ kind: "file", name: file.path }); + } + else { + child.send({ kind: "close" }); + callback(failures); + } +} + +function spawnLintWorker(files: {path: string}[], callback: (failures: number) => void) { + const child = cp.fork("./scripts/parallel-lint"); + let failures = 0; + child.on("message", function(data) { + switch (data.kind) { + case "result": + if (data.failures > 0) { + failures += data.failures; + console.log(data.output); + } + sendNextFile(files, child, callback, failures); + break; + case "error": + console.error(data.error); + failures++; + sendNextFile(files, child, callback, failures); + break; + } + }); + sendNextFile(files, child, callback, failures); +} gulp.task("lint", "Runs tslint on the compiler sources. Optional arguments are: --f[iles]=regex", ["build-rules"], () => { const fileMatcher = RegExp(cmdLineOptions["files"]); - const lintOptions = getLinterOptions(); - let failed = 0; if (fold.isTravis()) console.log(fold.start("lint")); - return gulp.src(lintTargets) - .pipe(insert.transform((contents, file) => { - if (!fileMatcher.test(file.path)) return contents; - const result = lintFile(lintOptions, file.path); - if (result.failureCount > 0) { - console.log(result.output); - failed += result.failureCount; + + const files: {stat: fs.Stats, path: string}[] = []; + return gulp.src(lintTargets, { read: false }) + .pipe(through2.obj((chunk, enc, cb) => { + files.push(chunk); + cb(); + }, (cb) => { + files.filter(file => fileMatcher.test(file.path)).sort((filea, fileb) => filea.stat.size - fileb.stat.size); + const workerCount = (process.env.workerCount && +process.env.workerCount) || os.cpus().length; + for (let i = 0; i < workerCount; i++) { + spawnLintWorker(files, finished); } - return contents; // TODO (weswig): Automatically apply fixes? :3 - })) - .on("end", () => { - if (fold.isTravis()) console.log(fold.end("lint")); - if (failed > 0) { - console.error("Linter errors."); - process.exit(1); + + let completed = 0; + let failures = 0; + function finished(fails) { + completed++; + failures += fails; + if (completed === workerCount) { + if (fold.isTravis()) console.log(fold.end("lint")); + if (failures > 0) { + throw new Error(`Linter errors: ${failures}`); + } + else { + cb(); + } + } } - }); + })); }); diff --git a/Jakefile.js b/Jakefile.js index 01aac82a47ad8..e4aaf330dc72f 100644 --- a/Jakefile.js +++ b/Jakefile.js @@ -4,7 +4,6 @@ var fs = require("fs"); var os = require("os"); var path = require("path"); var child_process = require("child_process"); -var Linter = require("tslint"); var fold = require("travis-fold"); var runTestsInParallel = require("./scripts/mocha-parallel").runTestsInParallel; @@ -1054,36 +1053,6 @@ task("build-rules-end", [] , function() { if (fold.isTravis()) console.log(fold.end("build-rules")); }); -function getLinterOptions() { - return { - configuration: require("./tslint.json"), - formatter: "prose", - formattersDirectory: undefined, - rulesDirectory: "built/local/tslint" - }; -} - -function lintFileContents(options, path, contents) { - var ll = new Linter(path, contents, options); - console.log("Linting '" + path + "'."); - return ll.lint(); -} - -function lintFile(options, path) { - var contents = fs.readFileSync(path, "utf8"); - return lintFileContents(options, path, contents); -} - -function lintFileAsync(options, path, cb) { - fs.readFile(path, "utf8", function(err, contents) { - if (err) { - return cb(err); - } - var result = lintFileContents(options, path, contents); - cb(undefined, result); - }); -} - var lintTargets = compilerSources .concat(harnessSources) // Other harness sources @@ -1094,75 +1063,78 @@ var lintTargets = compilerSources .concat(["Gulpfile.ts"]) .concat([nodeServerInFile, perftscPath, "tests/perfsys.ts", webhostPath]); +function sendNextFile(files, child, callback, failures) { + var file = files.pop(); + if (file) { + console.log("Linting '" + file + "'."); + child.send({kind: "file", name: file}); + } + else { + child.send({kind: "close"}); + callback(failures); + } +} + +function spawnLintWorker(files, callback) { + var child = child_process.fork("./scripts/parallel-lint"); + var failures = 0; + child.on("message", function(data) { + switch (data.kind) { + case "result": + if (data.failures > 0) { + failures += data.failures; + console.log(data.output); + } + sendNextFile(files, child, callback, failures); + break; + case "error": + console.error(data.error); + failures++; + sendNextFile(files, child, callback, failures); + break; + } + }); + sendNextFile(files, child, callback, failures); +} desc("Runs tslint on the compiler sources. Optional arguments are: f[iles]=regex"); task("lint", ["build-rules"], function() { if (fold.isTravis()) console.log(fold.start("lint")); var startTime = mark(); - var lintOptions = getLinterOptions(); var failed = 0; var fileMatcher = RegExp(process.env.f || process.env.file || process.env.files || ""); var done = {}; for (var i in lintTargets) { var target = lintTargets[i]; if (!done[target] && fileMatcher.test(target)) { - var result = lintFile(lintOptions, target); - if (result.failureCount > 0) { - console.log(result.output); - failed += result.failureCount; - } - done[target] = true; + done[target] = fs.statSync(target).size; } } - measure(startTime); - if (fold.isTravis()) console.log(fold.end("lint")); - if (failed > 0) { - fail('Linter errors.', failed); - } -}); -/** - * This is required because file watches on Windows get fires _twice_ - * when a file changes on some node/windows version configuations - * (node v4 and win 10, for example). By not running a lint for a file - * which already has a pending lint, we avoid duplicating our work. - * (And avoid printing duplicate results!) - */ -var lintSemaphores = {}; - -function lintWatchFile(filename) { - fs.watch(filename, {persistent: true}, function(event) { - if (event !== "change") { - return; - } + var workerCount = (process.env.workerCount && +process.env.workerCount) || os.cpus().length; - if (!lintSemaphores[filename]) { - lintSemaphores[filename] = true; - lintFileAsync(getLinterOptions(), filename, function(err, result) { - delete lintSemaphores[filename]; - if (err) { - console.log(err); - return; - } - if (result.failureCount > 0) { - console.log("***Lint failure***"); - for (var i = 0; i < result.failures.length; i++) { - var failure = result.failures[i]; - var start = failure.startPosition.lineAndCharacter; - var end = failure.endPosition.lineAndCharacter; - console.log("warning " + filename + " (" + (start.line + 1) + "," + (start.character + 1) + "," + (end.line + 1) + "," + (end.character + 1) + "): " + failure.failure); - } - console.log("*** Total " + result.failureCount + " failures."); - } - }); - } + var names = Object.keys(done).sort(function(namea, nameb) { + return done[namea] - done[nameb]; }); -} -desc("Watches files for changes to rerun a lint pass"); -task("lint-server", ["build-rules"], function() { - console.log("Watching ./src for changes to linted files"); - for (var i = 0; i < lintTargets.length; i++) { - lintWatchFile(lintTargets[i]); + for (var i = 0; i < workerCount; i++) { + spawnLintWorker(names, finished); } -}); + + var completed = 0; + var failures = 0; + function finished(fails) { + completed++; + failures += fails; + if (completed === workerCount) { + measure(startTime); + if (fold.isTravis()) console.log(fold.end("lint")); + if (failures > 0) { + fail('Linter errors.', failed); + } + else { + complete(); + } + } + } +}, {async: true}); diff --git a/scripts/parallel-lint.js b/scripts/parallel-lint.js new file mode 100644 index 0000000000000..a9aec06c2dfa0 --- /dev/null +++ b/scripts/parallel-lint.js @@ -0,0 +1,45 @@ +var Linter = require("tslint"); +var fs = require("fs"); + +function getLinterOptions() { + return { + configuration: require("../tslint.json"), + formatter: "prose", + formattersDirectory: undefined, + rulesDirectory: "built/local/tslint" + }; +} + +function lintFileContents(options, path, contents) { + var ll = new Linter(path, contents, options); + return ll.lint(); +} + +function lintFileAsync(options, path, cb) { + fs.readFile(path, "utf8", function (err, contents) { + if (err) { + return cb(err); + } + var result = lintFileContents(options, path, contents); + cb(undefined, result); + }); +} + +process.on("message", function (data) { + switch (data.kind) { + case "file": + var target = data.name; + var lintOptions = getLinterOptions(); + lintFileAsync(lintOptions, target, function (err, result) { + if (err) { + process.send({ kind: "error", error: err.toString() }); + return; + } + process.send({ kind: "result", failures: result.failureCount, output: result.output }); + }); + break; + case "close": + process.exit(0); + break; + } +}); \ No newline at end of file