From ef10c967d389bd896616ee5dacd7dc690811e345 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89tienne=20T=C3=A9treault-Pinard?= Date: Wed, 29 Mar 2017 11:59:30 -0400 Subject: [PATCH 1/6] improve karma.conf.js argument handling with minimist - better handle test suite arguments (normalize file name) - add --bundleTest=<> argument to make testing bundle test easier to run - add --watch, --verbose, --width, --height and --info flags --- test/jasmine/karma.conf.js | 201 +++++++++++++++++++++++++++---------- 1 file changed, 147 insertions(+), 54 deletions(-) diff --git a/test/jasmine/karma.conf.js b/test/jasmine/karma.conf.js index 9412e30666c..8c7b939c925 100644 --- a/test/jasmine/karma.conf.js +++ b/test/jasmine/karma.conf.js @@ -1,30 +1,102 @@ /* eslint-env node*/ -// Karma configuration - -/* - * Test file globs can be passed with an argument. - * - * Example: - * - * $ npm run test-jasmine -- tests/axes_test.js - * - * will only run the tests in axes_test.js - * - */ - +var path = require('path'); +var minimist = require('minimist'); var constants = require('../../tasks/util/constants'); -var arg = process.argv[4]; - var isCI = !!process.env.CIRCLECI; -var testFileGlob = arg ? arg : 'tests/*_test.js'; -var isSingleSuiteRun = (arg && arg.indexOf('bundle_tests/') === -1); -var isRequireJSTest = (arg && arg.indexOf('bundle_tests/requirejs') !== -1); -var isIE9Test = (arg && arg.indexOf('bundle_tests/ie9') !== -1); +var argv = minimist(process.argv.slice(4), { + string: ['bundleTest', 'width', 'height'], + 'boolean': ['info', 'watch', 'verbose', 'Chrome', 'Firefox'], + alias: { + 'Chrome': 'chrome', + 'Firefox': ['firefox', 'FF'], + 'bundleTest': ['bundletest', 'bundle_test'] + }, + 'default': { + info: false, + watch: false, + verbose: false, + width: '1035', + height: '617' + } +}); + +if(argv.info) { + console.log([ + 'plotly.js karma runner for jasmine tests CLI info', + '', + 'Examples:', + '', + 'Run `axes_test.js`, `bar_test.js` and `scatter_test.js` suites in `autoWatch` / multiple run mode:', + ' $ npm run test-jasmine -- axes bar_test.js scatter --watch', + '', + 'Run all tests with the `noCI` tag on Firefox in a 1500px wide window:', + ' $ npm run test-jasmine -- --tags=noCI --FF --width=1500', + '', + 'Run the `ie9_test.js` bundle test with the verbose reporter:', + ' $ npm run test-jasmine -- --bundleTest=ie9 --verbose', + '', + 'Arguments:', + ' - All non-flagged arguments corresponds to the test suites in `test/jasmine/tests/` to be run.', + ' No need to add the `_test.js` suffix, we expand them correctly here.', + ' - `--bundleTest` set the bundle test suite `test/jasmine/bundle_tests/ to be run.', + ' Note that only one bundle test can be run at a time.', + '', + 'Other options:', + ' - `--info`: show this info message', + ' - `--Chrome` (alias `--chrome`): run test in (our custom) Chrome browser', + ' - `--Firefox` (alias `--FF`, `--firefox`): run test in (our custom) Firefox browser', + ' - `--watch (dflt: `false`)`: run karma in `autoWatch` / multiple run mode', + ' - `--verbose` (dflt: `false`): show test result using verbose reporter', + ' - `--tags`: run only test with given tags (using the `jasmine-spec-tags` framework)', + ' - `--width`(dflt: 1035): set width of the browser window', + ' - `--height` (dflt: 617): set height of the browser window', + '', + 'For info on the karma CLI options run `npm run test-jasmine -- --help`' + ].join('\n')); + process.exit(0); +} + +var SUFFIX = '_test.js'; +var basename = function(s) { return path.basename(s, SUFFIX); }; +var merge = function(_) { + var list = []; + + (Array.isArray(_) ? _ : [_]).forEach(function(p) { + list = list.concat(p.split(',')); + }); + + return list; +}; +var glob = function(_) { + return _.length === 1 ? + _[0] + SUFFIX : + '{' + _.join(',') + '}' + SUFFIX; +}; + +var isBundleTest = !!argv.bundleTest; +var isFullSuite = !isBundleTest && argv._.length === 0; +var testFileGlob; + +if(isFullSuite) { + testFileGlob = path.join('tests', '*' + SUFFIX); +} else if(isBundleTest) { + var _ = merge(argv.bundleTest); + + if(_.length > 1) { + console.warn('Can only run one bundle test suite at a time, ignoring ', _.slice(1)); + } + + testFileGlob = path.join('bundle_tests', glob([basename(_[0])])); +} else { + testFileGlob = path.join('tests', glob(merge(argv._).map(basename))); +} -var pathToMain = '../../lib/index.js'; -var pathToJQuery = 'assets/jquery-1.8.3.min.js'; +var pathToShortcutPath = path.join(__dirname, '..', '..', 'tasks', 'util', 'shortcut_paths.js'); +var pathToMain = path.join(__dirname, '..', '..', 'lib', 'index.js'); +var pathToJQuery = path.join(__dirname, 'assets', 'jquery-1.8.3.min.js'); +var pathToIE9mock = path.join(__dirname, 'assets', 'ie9_mock.js'); function func(config) { @@ -71,13 +143,13 @@ func.defaultConfig = { preprocessors: {}, // test results reporter to use - // possible values: 'dots', 'progress' + // possible values: 'dots', 'progress', 'spec' and 'verbose' // available reporters: https://npmjs.org/browse/keyword/karma-reporter // // See note in CONTRIBUTING.md about more verbose reporting via karma-verbose-reporter: // https://www.npmjs.com/package/karma-verbose-reporter ('verbose') // - reporters: isSingleSuiteRun ? ['progress'] : ['dots', 'spec'], + reporters: (isFullSuite && !argv.tags) ? ['dots', 'spec'] : ['progress'], // web server port port: 9876, @@ -86,17 +158,19 @@ func.defaultConfig = { colors: true, // enable / disable watching file and executing tests whenever any file changes - autoWatch: !isCI, + autoWatch: argv.watch, // if true, Karma captures browsers, runs the tests and exits - singleRun: isCI, + singleRun: !argv.watch, // how long will Karma wait for a message from a browser before disconnecting (30 ms) browserNoActivityTimeout: 30000, // start these browsers // available browser launchers: https://npmjs.org/browse/keyword/karma-launcher - browsers: ['Chrome_WindowSized'], + // + // N.B. this field is filled below + browsers: [], // custom browser options // @@ -104,20 +178,23 @@ func.defaultConfig = { // // '--ignore-gpu-blacklist' allow to test WebGL on CI (!!!) customLaunchers: { - Chrome_WindowSized: { + _Chrome: { base: 'Chrome', - flags: ['--window-size=1035,617', '--ignore-gpu-blacklist'] + flags: [ + '--window-size=' + argv.width + ',' + argv.height, + isCI ? '--ignore-gpu-blacklist' : '' + ] }, - Firefox_WindowSized: { + _Firefox: { base: 'Firefox', - flags: ['--width=1035', '--height=617'] + flags: ['--width=' + argv.width, '--height=' + argv.height] } }, browserify: { - transform: ['../../tasks/util/shortcut_paths.js'], + transform: [pathToShortcutPath], extensions: ['.js'], - watch: !isCI, + watch: argv.watch, debug: true }, @@ -140,10 +217,33 @@ func.defaultConfig = { } }; -// Add lib/index.js to single-suite runs, -// to avoid import conflicts due to plotly.js -// circular dependencies. -if(isSingleSuiteRun) { +if(isFullSuite) { + func.defaultConfig.files.push(pathToJQuery); + func.defaultConfig.preprocessors[testFileGlob] = ['browserify']; +} else if(isBundleTest) { + switch(basename(testFileGlob)) { + case 'requirejs': + func.defaultConfig.files = [ + constants.pathToRequireJS, + constants.pathToRequireJSFixture + ]; + break; + case 'ie9': + // load ie9_mock.js before plotly.js+test bundle + // to catch reference errors that could occur + // when plotly.js is first loaded. + func.defaultConfig.files.push(pathToIE9mock); + func.defaultConfig.preprocessors[testFileGlob] = ['browserify']; + break; + default: + func.defaultConfig.preprocessors[testFileGlob] = ['browserify']; + break; + } +} else { + // Add lib/index.js to non-full-suite runs, + // to avoid import conflicts due to plotly.js + // circular dependencies. + func.defaultConfig.files.push( pathToJQuery, pathToMain @@ -152,26 +252,19 @@ if(isSingleSuiteRun) { func.defaultConfig.preprocessors[pathToMain] = ['browserify']; func.defaultConfig.preprocessors[testFileGlob] = ['browserify']; } -else if(isRequireJSTest) { - func.defaultConfig.files = [ - constants.pathToRequireJS, - constants.pathToRequireJSFixture - ]; -} -else if(isIE9Test) { - // load ie9_mock.js before plotly.js+test bundle - // to catch reference errors that could occur - // when plotly.js is first loaded. - - func.defaultConfig.files.push('./assets/ie9_mock.js'); - func.defaultConfig.preprocessors[testFileGlob] = ['browserify']; -} -else { - func.defaultConfig.files.push(pathToJQuery); - func.defaultConfig.preprocessors[testFileGlob] = ['browserify']; -} // lastly, load test file glob func.defaultConfig.files.push(testFileGlob); +// add browsers +var browsers = func.defaultConfig.browsers; +if(argv.Chrome) browsers.push('_Chrome'); +if(argv.Firefox) browsers.push('_Firefox'); +if(browsers.length === 0) browsers.push('_Chrome'); + +// add verbose reporter if specified +if(argv.verbose) { + func.defaultConfig.reporters.push('verbose'); +} + module.exports = func; From 5cb3c2d23634e90ed6f28f17d8732fc814ceb208 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89tienne=20T=C3=A9treault-Pinard?= Date: Wed, 29 Mar 2017 11:59:48 -0400 Subject: [PATCH 2/6] fixup --- package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/package.json b/package.json index f5477919ca8..f571ffa0bee 100644 --- a/package.json +++ b/package.json @@ -124,6 +124,7 @@ "karma-spec-reporter": "0.0.30", "karma-verbose-reporter": "0.0.6", "madge": "^1.6.0", + "minimist": "^1.2.0", "node-sass": "^4.5.0", "npm-link-check": "^1.2.0", "open": "0.0.5", From 84bf21a48f938a8283849da9e713fa728280ba26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89tienne=20T=C3=A9treault-Pinard?= Date: Wed, 29 Mar 2017 12:00:30 -0400 Subject: [PATCH 3/6] :hocho: citest - use npm run test-jasmine arguments instead! --- package.json | 1 - tasks/noci_test.sh | 2 +- tasks/test_bundle.js | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index f571ffa0bee..7be070445e6 100644 --- a/package.json +++ b/package.json @@ -33,7 +33,6 @@ "docker": "node tasks/docker.js", "pretest": "node tasks/pretest.js", "test-jasmine": "karma start test/jasmine/karma.conf.js", - "citest-jasmine": "CIRCLECI=1 karma start test/jasmine/karma.conf.js", "test-image": "node tasks/test_image.js", "test-image-gl2d": "node tasks/test_image.js gl2d_* --queue", "test-export": "node tasks/test_export.js", diff --git a/tasks/noci_test.sh b/tasks/noci_test.sh index 46d0c779e43..34d048f8ef8 100755 --- a/tasks/noci_test.sh +++ b/tasks/noci_test.sh @@ -5,7 +5,7 @@ EXIT_STATE=0 # tests that aren't run on CI # jasmine specs with @noCI tag -npm run citest-jasmine -- tests/*_test.js --tags noCI || EXIT_STATE=$? +npm run test-jasmine -- --tags=noCI || EXIT_STATE=$? # mapbox image tests take too much resources on CI npm run test-image -- mapbox_* || EXIT_STATE=$? diff --git a/tasks/test_bundle.js b/tasks/test_bundle.js index a8d69c3ab07..9071ce96656 100644 --- a/tasks/test_bundle.js +++ b/tasks/test_bundle.js @@ -9,7 +9,7 @@ var pathToJasmineBundleTests = path.join(constants.pathToJasmineBundleTests); glob(pathToJasmineBundleTests + '/*.js', function(err, files) { files.forEach(function(file) { var baseName = path.basename(file); - var cmd = 'npm run citest-jasmine -- bundle_tests/' + baseName; + var cmd = 'npm run test-jasmine -- --bundleTest=' + baseName; common.execCmd(cmd); }); From e574951846748f671fe1831074d18d132fac77fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89tienne=20T=C3=A9treault-Pinard?= Date: Wed, 29 Mar 2017 12:01:05 -0400 Subject: [PATCH 4/6] add all test commands the `npm test` script --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 7be070445e6..4f83fa1e065 100644 --- a/package.json +++ b/package.json @@ -38,7 +38,7 @@ "test-export": "node tasks/test_export.js", "test-syntax": "node tasks/test_syntax.js", "test-bundle": "node tasks/test_bundle.js", - "test": "npm run citest-jasmine && npm run test-image && npm run test-image-gl2d && npm run test-syntax && npm run test-bundle", + "test": "npm run test-jasmine && npm run test-bundle && npm run test-image && npm run test-image-gl2d && npm run test-syntax && npm run lint", "start-test_dashboard": "node devtools/test_dashboard/server.js", "start-image_viewer": "node devtools/image_viewer/server.js", "start": "npm run start-test_dashboard", From 0ff387db66e334910bdc99377f75d0379c9a56dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89tienne=20T=C3=A9treault-Pinard?= Date: Wed, 29 Mar 2017 12:01:25 -0400 Subject: [PATCH 5/6] update CONTRIBUTING for new `test-jasmine` API --- CONTRIBUTING.md | 37 +++++++++++++++++++++++++++++++++---- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ecfc958bc26..f68b4aa6f37 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -104,6 +104,8 @@ Both jasmine and image tests are run on [CircleCI](https://circleci.com/gh/plotly/plotly.js) on every push to this repo. +### Jasmine tests + Jasmine tests are run in a browser using [karma](https://github.com/karma-runner/karma). To run them locally: @@ -114,17 +116,44 @@ npm run test-jasmine To run a specific suite, use: ``` -npm run test-jasmine -- tests/.js +npm run test-jasmine -- +``` + +where the `` corresponds to the suite's file name as found in +[`test/jasmine/tests/`](https://github.com/plotly/plotly.js/tree/master/test/jasmine/tests). + +You can also test multiple suites at a time, for example: + +``` +npm run test-jasmine -- bar axes scatter +``` + +which will run tests in the `bar_test.js`, `axes_test.js` and `scatter_test.js` +suites. + +To run the tests in an `autoWatch` / auto-bundle / multiple run mode: + +``` +npm run test-jasmine -- --watch ``` -where the `` corresponds to the suite's file name as found in [`test/jasmine/tests/`](https://github.com/plotly/plotly.js/tree/master/test/jasmine/tests). In certain situations, you may find that the default reporting is not verbose enough to pin down the source of the failing test. In this situation, you may wish to use [karma-verbose-reporter](https://www.npmjs.com/package/karma-verbose-reporter). You can use it without adding as a dev dependency by running: +In certain situations, you may find that the default reporting is not verbose +enough to pin down the source of the failing test. In this situation, you may +wish to use +[karma-verbose-reporter](https://www.npmjs.com/package/karma-verbose-reporter): ``` -npm install karma-verbose-reporter +npm run test-jasmine -- --verbose ``` -and adding `reporters: ['verbose']` to the corresponding karma configuration file. (You should disable the `progress` reporter when using `verbose`.) +For more info on the karma / jasmine CLI: + +``` +npm run test-jasmine -- --help +npm run test-jasmine -- --info +``` +### Image pixel comparison tests Image pixel comparison tests are run in a docker container. For more information on how to run them locally, please refer to [image test From 7cf9943ab5717543bfd0b72c14f6b3f167655f5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89tienne=20T=C3=A9treault-Pinard?= Date: Wed, 29 Mar 2017 13:42:42 -0400 Subject: [PATCH 6/6] replace 'watch' flag with 'nowatch' - `nowatch` is false by default locally and true by default on CI --- CONTRIBUTING.md | 4 ++-- tasks/noci_test.sh | 2 +- test/jasmine/karma.conf.js | 19 ++++++++++--------- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f68b4aa6f37..6419b65e1c4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -131,10 +131,10 @@ npm run test-jasmine -- bar axes scatter which will run tests in the `bar_test.js`, `axes_test.js` and `scatter_test.js` suites. -To run the tests in an `autoWatch` / auto-bundle / multiple run mode: +To turn off the `autoWatch` / auto-bundle / multiple run mode: ``` -npm run test-jasmine -- --watch +npm run test-jasmine -- --nowatch ``` In certain situations, you may find that the default reporting is not verbose diff --git a/tasks/noci_test.sh b/tasks/noci_test.sh index 34d048f8ef8..b4a2283b673 100755 --- a/tasks/noci_test.sh +++ b/tasks/noci_test.sh @@ -5,7 +5,7 @@ EXIT_STATE=0 # tests that aren't run on CI # jasmine specs with @noCI tag -npm run test-jasmine -- --tags=noCI || EXIT_STATE=$? +npm run test-jasmine -- --tags=noCI --nowatch || EXIT_STATE=$? # mapbox image tests take too much resources on CI npm run test-image -- mapbox_* || EXIT_STATE=$? diff --git a/test/jasmine/karma.conf.js b/test/jasmine/karma.conf.js index 8c7b939c925..8f8af91fbf7 100644 --- a/test/jasmine/karma.conf.js +++ b/test/jasmine/karma.conf.js @@ -7,15 +7,16 @@ var constants = require('../../tasks/util/constants'); var isCI = !!process.env.CIRCLECI; var argv = minimist(process.argv.slice(4), { string: ['bundleTest', 'width', 'height'], - 'boolean': ['info', 'watch', 'verbose', 'Chrome', 'Firefox'], + 'boolean': ['info', 'nowatch', 'verbose', 'Chrome', 'Firefox'], alias: { 'Chrome': 'chrome', 'Firefox': ['firefox', 'FF'], - 'bundleTest': ['bundletest', 'bundle_test'] + 'bundleTest': ['bundletest', 'bundle_test'], + 'nowatch': 'no-watch' }, 'default': { info: false, - watch: false, + nowatch: isCI, verbose: false, width: '1035', height: '617' @@ -28,8 +29,8 @@ if(argv.info) { '', 'Examples:', '', - 'Run `axes_test.js`, `bar_test.js` and `scatter_test.js` suites in `autoWatch` / multiple run mode:', - ' $ npm run test-jasmine -- axes bar_test.js scatter --watch', + 'Run `axes_test.js`, `bar_test.js` and `scatter_test.js` suites w/o `autoWatch`:', + ' $ npm run test-jasmine -- axes bar_test.js scatter --nowatch', '', 'Run all tests with the `noCI` tag on Firefox in a 1500px wide window:', ' $ npm run test-jasmine -- --tags=noCI --FF --width=1500', @@ -47,7 +48,7 @@ if(argv.info) { ' - `--info`: show this info message', ' - `--Chrome` (alias `--chrome`): run test in (our custom) Chrome browser', ' - `--Firefox` (alias `--FF`, `--firefox`): run test in (our custom) Firefox browser', - ' - `--watch (dflt: `false`)`: run karma in `autoWatch` / multiple run mode', + ' - `--nowatch (dflt: `false`, `true` on CI)`: run karma w/o `autoWatch` / multiple run mode', ' - `--verbose` (dflt: `false`): show test result using verbose reporter', ' - `--tags`: run only test with given tags (using the `jasmine-spec-tags` framework)', ' - `--width`(dflt: 1035): set width of the browser window', @@ -158,10 +159,10 @@ func.defaultConfig = { colors: true, // enable / disable watching file and executing tests whenever any file changes - autoWatch: argv.watch, + autoWatch: !argv.nowatch, // if true, Karma captures browsers, runs the tests and exits - singleRun: !argv.watch, + singleRun: argv.nowatch, // how long will Karma wait for a message from a browser before disconnecting (30 ms) browserNoActivityTimeout: 30000, @@ -194,7 +195,7 @@ func.defaultConfig = { browserify: { transform: [pathToShortcutPath], extensions: ['.js'], - watch: argv.watch, + watch: !argv.nowatch, debug: true },