Merge branch 'master' of github.com:roshan04/browserstack-cypress-cli

Author: roshan04

bin/commands/runs.js

@@ -19,11 +19,13 @@ const archiver = require("../helpers/archiver"),
   downloadBuildArtifacts = require('../helpers/buildArtifacts').downloadBuildArtifacts,
   downloadBuildStacktrace = require('../helpers/downloadBuildStacktrace').downloadBuildStacktrace,
   updateNotifier = require('update-notifier'),
-  pkg = require('../../package.json');
+  pkg = require('../../package.json'),
+  packageDiff = require('../helpers/package-diff');
 const { getStackTraceUrl } = require('../helpers/sync/syncSpecsLogs');
 
 module.exports = function run(args, rawArgs) {
 
+  markBlockStart('preBuild');
   // set debug mode (--cli-debug)
   utils.setDebugMode(args);
 
@@ -158,13 +160,21 @@ module.exports = function run(args, rawArgs) {
           // Archive the spec files
           logger.debug("Started archiving test suite");
           markBlockStart('zip.archive');
-          return archiver.archive(bsConfig.run_settings, config.fileName, args.exclude, md5data).then(function (data) {
+          return archiver.archive(bsConfig.run_settings, config.fileName, args.exclude, md5data).then(async function (data) {
             logger.debug("Completed archiving test suite");
             markBlockEnd('zip.archive');
 
             let test_zip_size = utils.fetchZipSize(path.join(process.cwd(), config.fileName));
             let npm_zip_size = utils.fetchZipSize(path.join(process.cwd(), config.packageFileName));
-
+            let node_modules_size = await utils.fetchFolderSize(path.join(process.cwd(), "node_modules"))            
+            
+            //Package diff
+            let isPackageDiff = false;
+            if(!md5data.zipUrlPresent){
+              isPackageDiff = packageDiff.run(`package.json`, `${config.packageDirName}/package.json`);
+              logger.debug(`Package difference was ${isPackageDiff ? `found` : `not found`}`);
+            }
+            
             // Uploaded zip file
             logger.debug("Started uploading the test suite zip");
             logger.debug("Started uploading the node_module zip");
@@ -185,6 +195,8 @@ module.exports = function run(args, rawArgs) {
               logger.debug("Started build creation");
               markBlockStart('createBuild');
               return build.createBuild(bsConfig, zip).then(function (data) {
+                markBlockEnd('preBuild');
+                markBlockStart('buildProcessing');
                 logger.debug("Completed build creation");
                 markBlockEnd('createBuild');
                 markBlockEnd('total');
@@ -216,6 +228,8 @@ module.exports = function run(args, rawArgs) {
                 if (args.sync) {
                   logger.debug("Started polling build status from BrowserStack");
                   syncRunner.pollBuildStatus(bsConfig, data, rawArgs, buildReportData).then(async (exitCode) => {
+                    markBlockEnd('buildProcessing');
+                    markBlockStart('postBuild');
                     logger.debug("Completed polling of build status");
 
                     // stop the Local instance
@@ -234,6 +248,7 @@ module.exports = function run(args, rawArgs) {
                       // Generate custom report!
                       reportGenerator(bsConfig, data.build_id, args, rawArgs, buildReportData, function(){
                         utils.sendUsageReport(bsConfig, args, `${message}\n${dashboardLink}`, Constants.messageTypes.SUCCESS, null, buildReportData, rawArgs);
+                        markBlockEnd('postBuild');
                         utils.handleSyncExit(exitCode, data.dashboard_url);
                       });
                     } else {
@@ -267,9 +282,13 @@ module.exports = function run(args, rawArgs) {
                   build_id: data.build_id,
                   test_zip_size: test_zip_size,
                   npm_zip_size: npm_zip_size,
+                  node_modules_size: node_modules_size,
                   test_suite_zip_upload: md5data.zipUrlPresent ? 0 : 1,
                   package_zip_upload: md5data.packageUrlPresent ? 0 : 1
                 };
+                if(dataToSend.test_suite_zip_upload === 1 ){
+                  dataToSend['is_package_diff'] = isPackageDiff;
+                }
 
                 if (!md5data.zipUrlPresent && zip.tests_upload_time) {
                   dataToSend.test_suite_zip_size = parseFloat((test_zip_size / 1024).toFixed(2));

bin/helpers/archiver.js

@@ -1,4 +1,5 @@
 'use strict';
+const config = require('./config.js');
 const fs = require("fs"),
     path = require("path");
 
@@ -76,6 +77,14 @@ const archiveSpecs = (runSettings, filePath, excludeFiles, md5data) => {
       archive.append(packageJSONString, {name: `${cypressAppendFilesZipLocation}browserstack-package.json`});
     }
 
+    //Create copy of package.json
+    if(fs.existsSync('package.json')){
+      let originalPackageJson = JSON.parse(fs.readFileSync('package.json'));
+      let originalPackageJsonString = JSON.stringify(originalPackageJson, null, 4);
+      archive.append(originalPackageJsonString, {name: `${cypressAppendFilesZipLocation}userPackage.json`});
+      logger.debug(`Created copy of package.json in ${config.packageDirName} folder`)
+    }
+
     // do not add cypress.json if arg provided is false
     if (
       runSettings.cypress_config_file &&

bin/helpers/checkUploaded.js

@@ -86,6 +86,7 @@ const checkUploadedMd5 = (bsConfig, args, instrumentBlocks) => {
       zipUrlPresent: false,
       packageUrlPresent: false,
     };
+    utils.setCypressNpmDependency(bsConfig);
     if (args["force-upload"]) {
       logger.debug("force-upload set to true. Uploading tests and npm packages.");
       return resolve(obj);

bin/helpers/config.js

@@ -13,7 +13,9 @@ if(config.env !== "production") {
 }
 
 config.cypress_v1 = `${config.rails_host}/automate/cypress/v1`;
+config.cypress_v2 = `${config.rails_host}/automate/cypress/v2`;
 config.buildUrl = `${config.cypress_v1}/builds/`;
+config.buildUrlV2 = `${config.cypress_v2}/builds/`;
 config.buildStopUrl = `${config.cypress_v1}/builds/stop/`;
 config.checkMd5sum = `${config.cypress_v1}/md5sumcheck/`;
 config.getInitialDetails = `${config.cypress_v1}/get_initial_details/`;

bin/helpers/constants.js

@@ -110,6 +110,8 @@ const userMessages = {
     "Value for the 'spec_timeout' key not in the 1-120 range. Going ahead with 30 mins as the default spec timeout. Read more about how to specify the option in https://browserstack.com/docs/automate/cypress/spec-timeout",
   SPEC_LIMIT_SUCCESS_MESSAGE:
     "Spec timeout specified as <x> minutes. If any of your specs exceed the specified time limit, it would be forcibly killed by BrowserStack",
+  NO_CONNECTION_WHILE_UPDATING_UPLOAD_PROGRESS_BAR:
+    "Unable to determine zip upload progress due to undefined/null connection request"
 };
 
 const validationMessages = {

bin/helpers/package-diff.js

@@ -0,0 +1,93 @@
+"use strict";
+const fs = require("fs");
+const path = require("path");
+const logger = require("./logger").winstonLogger;
+
+exports.run = (basePath, comparePath) => {
+  if (!basePath || !comparePath) {
+    logger.debug("Skipping package difference check.");
+  }
+
+  let base;
+  let compare;
+  let isDiff = false;
+  try {
+    base = readModules(basePath);
+    compare = readModules(comparePath);
+  } catch (error) {
+    logger.debug('Unable to process package difference');
+    return isDiff;
+  }
+  
+  Object.keys(base.deps).forEach((baseKey) => {
+    if (baseKey in compare.deps) {
+      if (base.deps[baseKey] !== compare.deps[baseKey]) {
+        isDiff = true;
+        return;
+      }
+    } else {
+      isDiff = true;
+      return;
+    }
+  });
+  return isDiff;
+};
+const readModules = (location) => {
+  const table = {};
+
+  // Resolve package dependencies
+  if (location.indexOf("package.json") !== -1) {
+    const data = fs.readFileSync(location.replace(":dev", ""), "utf-8");
+    let parsed;
+    try {
+      parsed = JSON.parse(data);
+    } catch (e) {
+      parsed = false;
+    }
+    if (!parsed) {
+      return;
+    }
+
+    const depsKey =
+      location.indexOf(":dev") !== -1 ? "devDependencies" : "dependencies";
+    const deps = parsed[depsKey]
+      ? parsed[depsKey]
+      : parsed.dependencies || parsed.devDependencies;
+
+    Object.keys(deps).forEach((key) => {
+      deps[key] = deps[key].replace(/\^|~/g, "");
+    });
+    return {
+      name: `${location} {${depsKey}}`,
+      deps,
+    };
+  }
+
+  fs.readdirSync(location)
+    .filter((name) => name !== ".bin")
+    .map((name) => {
+      const pkg = path.join(location, name, "package.json");
+      const exists = fs.existsSync(pkg);
+      if (!exists) {
+        return;
+      }
+
+      const data = fs.readFileSync(pkg, "utf-8");
+      let parsed;
+
+      try {
+        parsed = JSON.parse(data);
+      } catch (e) {
+        parsed = false;
+      }
+      if (!parsed) {
+        return;
+      }
+
+      table[name] = parsed.version;
+    });
+  return {
+    name: location,
+    deps: table,
+  };
+};

bin/helpers/sync/specsSummary.js

@@ -33,7 +33,8 @@ let printSpecsRunSummary = (data, machines, customErrorsToPrint) => {
     });
 
     logger.info(`Total tests: ${summary.total}, passed: ${summary.passed}, failed: ${summary.failed}, skipped: ${summary.skipped}, passed_with_skipped: ${summary.passed_with_skipped}, pending: ${summary.pending}`);
-    logger.info(`Done in ${data.duration/1000} seconds using ${machines} machines\n`);
+    logger.info(`Done in ${data.duration} seconds using ${data.parallels} machines\n`);
+    winstonlogger.debug(`CLI calculated duration is ${data.cliDuration/1000}`);
 
     if (customErrorsToPrint && customErrorsToPrint.length > 0) {
       for (const error of customErrorsToPrint) {

bin/helpers/sync/syncSpecsLogs.js

@@ -14,6 +14,8 @@ let specSummary = {
   "buildError": null,
   "specs": [],
   "duration": null,
+  "parallels": null,
+  "cliDuration": null,
   "customErrorsToPrint": []
 }
 
@@ -24,7 +26,7 @@ if (!isNaN(terminalWidth)) lineSeparator = "\n" + "-".repeat(terminalWidth);
 
 let  getOptions = (auth, build_id) => {
   return {
-    url: `${config.buildUrl}${build_id}`,
+    url: `${config.buildUrlV2}${build_id}`,
     auth: {
       user: auth.username,
       password: auth.access_key
@@ -120,7 +122,7 @@ let printSpecsStatus = (bsConfig, buildDetails, rawArgs, buildReportData) => {
           }
         }
         logger.info(lineSeparator);
-        specSummary.duration =  endTime - startTime
+        specSummary.cliDuration =  endTime - startTime
         resolve(specSummary);
       }
     );
@@ -147,15 +149,15 @@ let whileProcess = (whilstCallback) => {
     switch (response.statusCode) {
       case 202: // get data here and print it
         n = 2
-        showSpecsStatus(body);
+        showSpecsStatus(body, 202);
         return setTimeout(whilstCallback, timeout * n, null);
       case 204: // No data available, wait for some time and ask again
         n = 1
         return setTimeout(whilstCallback, timeout * n, null);
       case 200: // Build is completed.
         whileLoop = false;
         endTime = Date.now();
-        showSpecsStatus(body);
+        showSpecsStatus(body, 200);
         return specSummary.exitCode == Constants.BUILD_FAILED_EXIT_CODE ? 
         whilstCallback({ status: 204, message: "No specs ran in the build"} ) : whilstCallback(null, body);
       default:
@@ -169,9 +171,9 @@ let getStackTraceUrl = () => {
   return specSummary.buildError
 }
 
-let showSpecsStatus = (data) => {
+let showSpecsStatus = (data, statusCode) => {
   let specData = JSON.parse(data);
-  specData.forEach(specDetails => {
+  specData["specData"].forEach(specDetails => {
     if (specDetails.type === Constants.CYPRESS_CUSTOM_ERRORS_TO_PRINT_KEY) {
       addCustomErrorToPrint(specDetails);
     } else {
@@ -190,6 +192,17 @@ let showSpecsStatus = (data) => {
       }
     }
   });
+  if ( statusCode != 200 ) return; 
+  // Below block is for printing build details, return if non 200 status code
+  if ("buildData" in specData) {
+    const buildDetails = specData.buildData;
+    const totalDuration = (utils.isUndefined(buildDetails.duration)) ? "-" : buildDetails.duration.total_duration
+    const parallels = (utils.isUndefined(buildDetails.parallels)) ? "-" : buildDetails.parallels
+    specSummary.duration = totalDuration;
+    specSummary.parallels = parallels;
+  } else {
+    logger.debug(`Build details not sent`)
+  }
 }
 
 let printInitialLog = () => {

bin/helpers/utils.js

@@ -8,6 +8,9 @@ const { v4: uuidv4 } = require('uuid');
 const browserstack = require('browserstack-local');
 const crypto = require('crypto');
 const util = require('util');
+const { promisify } = require('util');
+const readdir = promisify(fs.readdir);
+const stat = promisify(fs.stat);
 
 const usageReporting = require("./usageReporting"),
   logger = require("./logger").winstonLogger,
@@ -328,6 +331,23 @@ exports.setCypressTestSuiteType = (bsConfig) => {
   logger.debug(`Setting cypress test suite type as ${bsConfig.run_settings.cypressTestSuiteType}`);
 }
 
+exports.setCypressNpmDependency = (bsConfig) => {
+  const runSettings = bsConfig.run_settings;
+  if (runSettings.npm_dependencies !== undefined && 
+    Object.keys(runSettings.npm_dependencies).length !== 0 &&
+    typeof runSettings.npm_dependencies === 'object') {
+    if (!("cypress" in runSettings.npm_dependencies) && runSettings.cypressTestSuiteType === Constants.CYPRESS_V10_AND_ABOVE_TYPE) {
+      logger.warn("Missing cypress not found in npm_dependencies");        
+      if("cypress_version" in runSettings){
+        runSettings.npm_dependencies.cypress = `^${runSettings.cypress_version.toString().split(".")[0]}`;
+      } else if (runSettings.cypressTestSuiteType === Constants.CYPRESS_V10_AND_ABOVE_TYPE)  {
+        runSettings.npm_dependencies.cypress = "latest";
+      }
+      logger.warn(`Adding cypress version ${runSettings.npm_dependencies.cypress} in npm_dependencies`);
+    }
+  }
+}
+
 exports.verifyGeolocationOption = () => {
   let glOptionsSet = (this.searchForOption('-gl') || this.searchForOption('--gl'));
   let geoHyphenLocationOptionsSet = (this.searchForOption('-geo-location') || this.searchForOption('--geo-location'));
@@ -1278,6 +1298,34 @@ exports.fetchZipSize = (fileName) => {
   }
 }
 
+const getDirectorySize = async function(dir) {
+  try{
+    const subdirs = (await readdir(dir));
+    const files = await Promise.all(subdirs.map(async (subdir) => {
+      const res = path.resolve(dir, subdir);
+      const s = (await stat(res));
+      return s.isDirectory() ? getDirectorySize(res) : (s.size);
+    }));
+    return files.reduce((a, f) => a+f, 0);
+  }catch(e){
+    console.log(`Error ${e}`)
+    logger.debug('Failed to get file or directory.');
+    return 0;
+  }
+};
+
+exports.fetchFolderSize = async (dir) => {
+  try {
+    if(fs.existsSync(dir)){
+      return (await getDirectorySize(dir) / 1024 / 1024);
+    }
+    return 0;
+  } catch (error) {
+    logger.debug(`Failed to get directory size.`);
+    return 0;
+  }
+}
+
 exports.getVideoConfig = (cypressConfig) => {
   let conf = {
     video: true,