From 5f5e8456b03e1388c5afc9eb5178583637bc71ab Mon Sep 17 00:00:00 2001 From: nagpalkaran95 Date: Fri, 4 Sep 2020 14:36:42 +0530 Subject: [PATCH 1/4] Support for excluding files & folders from uploads --- bin/commands/runs.js | 4 ++-- bin/helpers/archiver.js | 18 ++++++++++++++---- bin/helpers/capabilityHelper.js | 2 +- bin/helpers/constants.js | 13 ++++++++----- bin/helpers/utils.js | 6 +++++- bin/runner.js | 6 ++++++ 6 files changed, 36 insertions(+), 13 deletions(-) diff --git a/bin/commands/runs.js b/bin/commands/runs.js index b91fdc89..fcd73f10 100644 --- a/bin/commands/runs.js +++ b/bin/commands/runs.js @@ -34,7 +34,7 @@ module.exports = function run(args) { utils.setParallels(bsConfig, args); // Archive the spec files - return archiver.archive(bsConfig.run_settings, config.fileName).then(function (data) { + return archiver.archive(bsConfig.run_settings, config.fileName, args.exclude).then(function (data) { // Uploaded zip file return zipUploader.zipUpload(bsConfig, config.fileName).then(function (zip) { @@ -44,7 +44,7 @@ module.exports = function run(args) { let message = `${data.message}! ${Constants.userMessages.BUILD_CREATED} with build id: ${data.build_id}`; let dashboardLink = `${Constants.userMessages.VISIT_DASHBOARD} ${config.dashboardUrl}${data.build_id}`; utils.exportResults(data.build_id, `${config.dashboardUrl}${data.build_id}`); - if ((utils.isUndefined(bsConfig.run_settings.parallels) && utils.isUndefined(args.parallels)) || (!utils.isUndefined(bsConfig.run_settings.parallels) && bsConfig.run_settings.parallels == Constants.constants.DEFAULT_PARALLEL_MESSAGE)) { + if ((utils.isUndefined(bsConfig.run_settings.parallels) && utils.isUndefined(args.parallels)) || (!utils.isUndefined(bsConfig.run_settings.parallels) && bsConfig.run_settings.parallels == Constants.cliMessages.RUN.DEFAULT_PARALLEL_MESSAGE)) { logger.warn(Constants.userMessages.NO_PARALLELS); } diff --git a/bin/helpers/archiver.js b/bin/helpers/archiver.js index 24e9c854..b4deea00 100644 --- a/bin/helpers/archiver.js +++ b/bin/helpers/archiver.js @@ -2,9 +2,10 @@ const fs = require("fs"); const archiver = require("archiver"), + Constants = require('../helpers/constants'), logger = require("./logger").winstonLogger; -const archiveSpecs = (runSettings, filePath) => { +const archiveSpecs = (runSettings, filePath, excludeFiles) => { return new Promise(function (resolve, reject) { var output = fs.createWriteStream(filePath); @@ -36,9 +37,18 @@ const archiveSpecs = (runSettings, filePath) => { archive.pipe(output); - let allowedFileTypes = [ 'js', 'json', 'txt', 'ts', 'feature', 'features', 'pdf', 'jpg', 'jpeg', 'png', 'zip' ]; - allowedFileTypes.forEach(fileType => { - archive.glob(`**/*.${fileType}`, { cwd: cypressFolderPath, matchBase: true, ignore: ['**/node_modules/**', './node_modules/**', 'package-lock.json', 'package.json', 'browserstack-package.json', 'tests.zip'] }); + let ignoreFiles = Constants.filesToIgnoreWhileUploading; + + // exclude files asked by the user + // args will take precedence over config file + if (!Utils.isUndefined(excludeFiles)) { + ignoreFiles = ignoreFile.concat(Utils.fixCommaSeparatedString(excludeFiles).split(',')); + } else if (!Utils.isUndefined(runSettings.exclude)) { + ignoreFiles = ignoreFile.concat(excludeFiles); + } + + Constants.allowedFileTypes.forEach(fileType => { + archive.glob(`**/*.${fileType}`, { cwd: cypressFolderPath, matchBase: true, ignore: ignoreFiles }); }); let packageJSON = {}; diff --git a/bin/helpers/capabilityHelper.js b/bin/helpers/capabilityHelper.js index 451a42be..3698d878 100644 --- a/bin/helpers/capabilityHelper.js +++ b/bin/helpers/capabilityHelper.js @@ -69,7 +69,7 @@ const caps = (bsConfig, zip) => { obj.parallels = bsConfig.run_settings.parallels; } - if(obj.parallels === Constants.constants.DEFAULT_PARALLEL_MESSAGE) obj.parallels = undefined + if(obj.parallels === Constants.cliMessages.RUN.DEFAULT_PARALLEL_MESSAGE) obj.parallels = undefined if (obj.project) logger.log(`Project name is: ${obj.project}`); diff --git a/bin/helpers/constants.js b/bin/helpers/constants.js index b844cfc0..a2b383bb 100644 --- a/bin/helpers/constants.js +++ b/bin/helpers/constants.js @@ -60,7 +60,9 @@ const cliMessages = { INFO: "Run your tests on BrowserStack.", DESC: "Path to BrowserStack config", CONFIG_DEMAND: "config file is required", - BUILD_NAME: "The build name you want to use to name your test runs" + BUILD_NAME: "The build name you want to use to name your test runs", + EXCLUDE: "Exclude files matching a pattern from zipping and uploading", + DEFAULT_PARALLEL_MESSAGE: "Here goes the number of parallels you want to run" }, COMMON: { DISABLE_USAGE_REPORTING: "Disable usage reporting", @@ -79,14 +81,15 @@ const messageTypes = { NULL: null } -const constants = { - DEFAULT_PARALLEL_MESSAGE: "Here goes the number of parallels you want to run" -} +const allowedFileTypes = ['js', 'json', 'txt', 'ts', 'feature', 'features', 'pdf', 'jpg', 'jpeg', 'png', 'zip']; + +const filesToIgnoreWhileUploading = ['node_modules/**', 'package-lock.json', 'package.json', 'browserstack-package.json', 'tests.zip'] module.exports = Object.freeze({ userMessages, cliMessages, validationMessages, messageTypes, - constants + allowedFileTypes, + filesToIgnoreWhileUploading }); diff --git a/bin/helpers/utils.js b/bin/helpers/utils.js index 05b76805..e45b9651 100644 --- a/bin/helpers/utils.js +++ b/bin/helpers/utils.js @@ -111,12 +111,16 @@ exports.setBuildName = (bsConfig, args) => { } } +exports.fixCommaSeparatedString = (string) => { + return string.split(/\s{0,},\s+/).join(','); +} + exports.isUndefined = value => (value === undefined || value === null); exports.isFloat = value => (Number(value) && Number(value) % 1 !== 0); exports.isParallelValid = (value) => { - return this.isUndefined(value) || !(isNaN(value) || this.isFloat(value) || parseInt(value, 10) === 0 || parseInt(value, 10) < -1 ) || value === Constants.constants.DEFAULT_PARALLEL_MESSAGE; + return this.isUndefined(value) || !(isNaN(value) || this.isFloat(value) || parseInt(value, 10) === 0 || parseInt(value, 10) < -1 ) || value === Constants.cliMessages.RUN.DEFAULT_PARALLEL_MESSAGE; } exports.getUserAgent = () => { diff --git a/bin/runner.js b/bin/runner.js index c84e716c..f3dab434 100755 --- a/bin/runner.js +++ b/bin/runner.js @@ -165,6 +165,12 @@ var argv = yargs type: "string", default: undefined }, + 'e': { + alias: 'exclude', + describe: Constants.cliMessages.RUN.EXCLUDE, + type: "string", + default: undefined + }, 'disable-npm-warning': { default: false, description: Constants.cliMessages.COMMON.NO_NPM_WARNING, From c95523706f297efd351e952c524992346c322e79 Mon Sep 17 00:00:00 2001 From: nagpalkaran95 Date: Fri, 4 Sep 2020 15:30:16 +0530 Subject: [PATCH 2/4] Add tests and syntax fix --- bin/helpers/archiver.js | 29 +++++++++------- test/unit/bin/helpers/archiver.js | 57 +++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+), 12 deletions(-) create mode 100644 test/unit/bin/helpers/archiver.js diff --git a/bin/helpers/archiver.js b/bin/helpers/archiver.js index b4deea00..2ab67c4a 100644 --- a/bin/helpers/archiver.js +++ b/bin/helpers/archiver.js @@ -3,7 +3,8 @@ const fs = require("fs"); const archiver = require("archiver"), Constants = require('../helpers/constants'), - logger = require("./logger").winstonLogger; + logger = require("./logger").winstonLogger, + utils = require('../helpers/utils'); const archiveSpecs = (runSettings, filePath, excludeFiles) => { return new Promise(function (resolve, reject) { @@ -37,18 +38,8 @@ const archiveSpecs = (runSettings, filePath, excludeFiles) => { archive.pipe(output); - let ignoreFiles = Constants.filesToIgnoreWhileUploading; - - // exclude files asked by the user - // args will take precedence over config file - if (!Utils.isUndefined(excludeFiles)) { - ignoreFiles = ignoreFile.concat(Utils.fixCommaSeparatedString(excludeFiles).split(',')); - } else if (!Utils.isUndefined(runSettings.exclude)) { - ignoreFiles = ignoreFile.concat(excludeFiles); - } - Constants.allowedFileTypes.forEach(fileType => { - archive.glob(`**/*.${fileType}`, { cwd: cypressFolderPath, matchBase: true, ignore: ignoreFiles }); + archive.glob(`**/*.${fileType}`, { cwd: cypressFolderPath, matchBase: true, ignore: getFilesToIgnore(runSettings, excludeFiles) }); }); let packageJSON = {}; @@ -70,4 +61,18 @@ const archiveSpecs = (runSettings, filePath, excludeFiles) => { }); } +const getFilesToIgnore = (runSettings, excludeFiles) => { + let ignoreFiles = Constants.filesToIgnoreWhileUploading; + + // exclude files asked by the user + // args will take precedence over config file + if (!utils.isUndefined(excludeFiles)) { + ignoreFiles = ignoreFiles.concat(utils.fixCommaSeparatedString(excludeFiles).split(',')); + } else if (!utils.isUndefined(runSettings.exclude) && runSettings.exclude.length) { + ignoreFiles = ignoreFiles.concat(runSettings.exclude); + } + + return ignoreFiles; +} + exports.archive = archiveSpecs diff --git a/test/unit/bin/helpers/archiver.js b/test/unit/bin/helpers/archiver.js new file mode 100644 index 00000000..07f9e239 --- /dev/null +++ b/test/unit/bin/helpers/archiver.js @@ -0,0 +1,57 @@ +const chai = require("chai"), + rewire = require("rewire"), + chaiAsPromised = require("chai-as-promised"); + +const utils = require("../../../../bin/helpers/utils"), + Constants = require("../../../../bin/helpers/constants"), + logger = require("../../../../bin/helpers/logger").winstonLogger; + +chai.use(chaiAsPromised); +logger.transports["console.info"].silent = true; + +const archiver = rewire("../../../../bin/helpers/archiver"); + +_getFilesToIgnore = archiver.__get__("getFilesToIgnore"); + +describe("archiver.js", () => { + + describe("getFilesToIgnore", () => { + it("no args, no exclude in runSettings", () => { + chai.expect(_getFilesToIgnore({}, undefined)).to.be.eql(Constants.filesToIgnoreWhileUploading); + }); + + it("args passed, no exclude in runSettings", () => { + let excludeFiles = "file1.js, file2.json"; + let argsToArray = utils.fixCommaSeparatedString(excludeFiles).split(','); + chai.expect(_getFilesToIgnore({}, excludeFiles)).to.be.eql(Constants.filesToIgnoreWhileUploading.concat(argsToArray)); + + excludeFiles = "file1.js,file2.json"; + argsToArray = utils.fixCommaSeparatedString(excludeFiles).split(','); + chai.expect(_getFilesToIgnore({}, excludeFiles)).to.be.eql(Constants.filesToIgnoreWhileUploading.concat(argsToArray)); + + excludeFiles = " file1.js , file2.json "; + argsToArray = utils.fixCommaSeparatedString(excludeFiles).split(','); + chai.expect(_getFilesToIgnore({}, excludeFiles)).to.be.eql(Constants.filesToIgnoreWhileUploading.concat(argsToArray)); + }); + + it("args passed, exclude added in runSettings", () => { + // args preceed over config file + let excludeFiles = "file1.js, file2.json "; + let argsToArray = utils.fixCommaSeparatedString(excludeFiles).split(','); + + let runSettings = { exclude: [] }; + chai.expect(_getFilesToIgnore(runSettings, excludeFiles)).to.be.eql(Constants.filesToIgnoreWhileUploading.concat(argsToArray)); + + runSettings = { exclude: ["sample1.js", "sample2.json"] }; + chai.expect(_getFilesToIgnore(runSettings, excludeFiles)).to.be.eql(Constants.filesToIgnoreWhileUploading.concat(argsToArray)); + }); + + it("no args, exclude added in runSettings", () => { + let runSettings = { exclude: [] }; + chai.expect(_getFilesToIgnore(runSettings, undefined)).to.be.eql(Constants.filesToIgnoreWhileUploading); + + runSettings = { exclude: ["sample1.js", "sample2.json"] }; + chai.expect(_getFilesToIgnore(runSettings, undefined)).to.be.eql(Constants.filesToIgnoreWhileUploading.concat(runSettings.exclude)); + }); + }); +}); From b5ab88a4e6930d107a98edee1fb85a52c7bf5371 Mon Sep 17 00:00:00 2001 From: nagpalkaran95 Date: Mon, 7 Sep 2020 13:49:49 +0530 Subject: [PATCH 3/4] Add log line --- bin/helpers/archiver.js | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/bin/helpers/archiver.js b/bin/helpers/archiver.js index 2ab67c4a..d4b5ab44 100644 --- a/bin/helpers/archiver.js +++ b/bin/helpers/archiver.js @@ -38,8 +38,10 @@ const archiveSpecs = (runSettings, filePath, excludeFiles) => { archive.pipe(output); + let ignoreFiles = getFilesToIgnore(runSettings, excludeFiles); + Constants.allowedFileTypes.forEach(fileType => { - archive.glob(`**/*.${fileType}`, { cwd: cypressFolderPath, matchBase: true, ignore: getFilesToIgnore(runSettings, excludeFiles) }); + archive.glob(`**/*.${fileType}`, { cwd: cypressFolderPath, matchBase: true, ignore: ignoreFiles }); }); let packageJSON = {}; @@ -67,9 +69,12 @@ const getFilesToIgnore = (runSettings, excludeFiles) => { // exclude files asked by the user // args will take precedence over config file if (!utils.isUndefined(excludeFiles)) { - ignoreFiles = ignoreFiles.concat(utils.fixCommaSeparatedString(excludeFiles).split(',')); + let excludePatterns = utils.fixCommaSeparatedString(excludeFiles).split(','); + ignoreFiles = ignoreFiles.concat(excludePatterns); + logger.info(`Excluding files matching: ${JSON.stringify(excludePatterns)}`); } else if (!utils.isUndefined(runSettings.exclude) && runSettings.exclude.length) { ignoreFiles = ignoreFiles.concat(runSettings.exclude); + logger.info(`Excluding files matching: ${JSON.stringify(runSettings.exclude)}`); } return ignoreFiles; From 44bfd5d1b6abef7266ab470cc224af8d393f0107 Mon Sep 17 00:00:00 2001 From: nagpalkaran95 Date: Mon, 7 Sep 2020 14:22:20 +0530 Subject: [PATCH 4/4] Add exclude to configTemplate --- bin/templates/configTemplate.js | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/templates/configTemplate.js b/bin/templates/configTemplate.js index 08349830..7b3556be 100644 --- a/bin/templates/configTemplate.js +++ b/bin/templates/configTemplate.js @@ -55,6 +55,7 @@ module.exports = function () { "cypress_proj_dir" : "/path/to/directory-that-contains--file", "project_name": "project-name", "build_name": "build-name", + "exclude": [], "parallels": "Here goes the number of parallels you want to run", "npm_dependencies": { },