Merge pull request #91 from browserstack/sync-cli-retry-polling

[Sync CLI] Retry polling in case of network issue

Author: SagarGaniga

bin/commands/runs.js

@@ -8,8 +8,7 @@ const archiver = require("../helpers/archiver"),
   Constants = require("../helpers/constants"),
   utils = require("../helpers/utils"),
   fileHelpers = require("../helpers/fileHelpers"),
-  syncRunner = require("../helpers/syncRunner"),
-  syncCliLogger = require("../helpers/logger").syncCliLogger;
+  syncRunner = require("../helpers/syncRunner");
 
 module.exports = function run(args) {
   let bsConfigPath = utils.getConfigPath(args.cf);
@@ -73,9 +72,7 @@ module.exports = function run(args) {
             if (args.sync) {
               syncRunner.pollBuildStatus(bsConfig, data).then((exitCode) => {
                 utils.sendUsageReport(bsConfig, args, `${message}\n${dashboardLink}`, Constants.messageTypes.SUCCESS, null);
-                syncCliLogger.info(Constants.userMessages.BUILD_REPORT_MESSAGE);
-                syncCliLogger.info(data.dashboard_url);
-                process.exit(exitCode);
+                utils.handleSyncExit(exitCode, data.dashboard_url)
               });
             }
 

bin/helpers/config.js

@@ -16,5 +16,7 @@ config.cypress_v1 = `${config.rails_host}/automate/cypress/v1`;
 config.buildUrl = `${config.cypress_v1}/builds/`;
 config.buildStopUrl = `${config.cypress_v1}/builds/stop/`;
 config.fileName = "tests.zip";
+config.retries = 5;
+config.networkErrorExitCode = 2;
 
 module.exports = config;

bin/helpers/constants.js

@@ -1,3 +1,5 @@
+let config = require("./config");
+
 const syncCLI = {
   FAILED_SPEC_DETAILS_COL_HEADER: ['Spec', 'Status', 'Browser', 'BrowserStack Session ID'],
   LOGS: {
@@ -30,7 +32,10 @@ const userMessages = {
   UPLOADING_TESTS: "Uploading the tests to BrowserStack",
   LOCAL_TRUE: "you will now be able to test localhost / private URLs",
   LOCAL_FALSE: "you won't be able to test localhost / private URLs",
-  EXIT_SYNC_CLI_MESSAGE: "Exiting the CLI, but your build is still running. You can use the --sync option to keep getting test updates. You can also use the build-info <build-id> command now."
+  EXIT_SYNC_CLI_MESSAGE: "Exiting the CLI, but your build is still running. You can use the --sync option to keep getting test updates. You can also use the build-info <build-id> command now.",
+  FATAL_NETWORK_ERROR: `fatal: unable to access '${config.buildUrl}': Could not resolve host: ${config.rails_host}`,
+  RETRY_LIMIT_EXCEEDED: `Max retries exceeded trying to connect to the host (retries: ${config.retries})`,
+  CHECK_DASHBOARD_AT: "Please check the build status at: "
 };
 
 const validationMessages = {

bin/helpers/sync/failedSpecsDetails.js

@@ -1,7 +1,8 @@
 const tablePrinter = require('table'), // { table, getBorderCharacters }
       chalk = require('chalk'),
       Constants = require("../constants"),
-      logger = require("../logger").syncCliLogger;
+      logger = require("../logger").syncCliLogger,
+      config = require("../config");
 
 /**
  *
@@ -19,7 +20,8 @@ const tablePrinter = require('table'), // { table, getBorderCharacters }
 //
 let failedSpecsDetails = (data) => {
   return new Promise((resolve, reject) => {
-    data.exitCode = 0;
+    if (!data.exitCode) data.exitCode = 0;
+
     if (data.specs.length === 0) resolve(data); // return if no failed/skipped tests.
 
     let failedSpecs = false;
@@ -48,7 +50,7 @@ let failedSpecsDetails = (data) => {
       ]);
     });
 
-    let config = {
+    let tableConfig = {
       border: tablePrinter.getBorderCharacters('ramac'),
       columns: {
         0: { alignment: 'left' },
@@ -67,12 +69,12 @@ let failedSpecsDetails = (data) => {
       }
     }
 
-    let result = tablePrinter.table(specData, config);
+    let result = tablePrinter.table(specData, tableConfig);
 
     logger.info('\nFailed / skipped test report:');
     logger.info(result);
 
-    if (failedSpecs) data.exitCode = 1 ; // specs failed, send exitCode as 1
+    if (failedSpecs && data.exitCode !== config.networkErrorExitCode) data.exitCode = 1 ; // specs failed, send exitCode as 1
     resolve(data); // No Specs failed, maybe skipped, but not failed, send exitCode as 0
   });
 }

bin/helpers/sync/syncSpecsLogs.js

@@ -8,7 +8,7 @@ const request = require("request"),
   tableStream = require('table').createStream,
   chalk = require('chalk');
 
-let whileLoop = true, whileTries = 40, options, timeout = 3000, n = 10, tableConfig, stream, startTime, endTime;
+let whileLoop = true, whileTries = config.retries, options, timeout = 3000, n = 10, tableConfig, stream, endTime, startTime = Date.now();
 let specSummary = {
   "specs": [],
   "duration": null
@@ -22,6 +22,7 @@ let  getOptions = (auth, build_id) => {
       user: auth.username,
       password: auth.access_key
     },
+    timeout: 10000,
     headers: {
       "Content-Type": "application/json",
       "User-Agent": utils.getUserAgent()
@@ -80,6 +81,7 @@ let printSpecsStatus = (bsConfig, buildDetails) => {
         whileProcess(callback)
       },
       function(err, result) { // when loop ends
+        logger.info("\n--------------------------------------------------------------------------------")
         specSummary.duration =  endTime - startTime
         resolve(specSummary)
       }
@@ -90,8 +92,20 @@ let printSpecsStatus = (bsConfig, buildDetails) => {
 let whileProcess = (whilstCallback) => {
   request.post(options, function(error, response, body) {
     if (error) {
-      return whilstCallback(error);
+      whileTries -= 1;
+      if (whileTries === 0) {
+        whileLoop = false;
+        endTime = Date.now();
+        specSummary.exitCode = config.networkErrorExitCode;
+        return whilstCallback({ status: 504, message: "Tries limit reached" }); //Gateway Timeout
+      } else {
+        n = 2
+        return setTimeout(whilstCallback, timeout * n, null);
+      }
     }
+
+    whileTries = config.retries; // reset to default after every successful request
+
     switch (response.statusCode) {
       case 202: // get data here and print it
         n = 2
@@ -104,7 +118,6 @@ let whileProcess = (whilstCallback) => {
         whileLoop = false;
         endTime = Date.now();
         showSpecsStatus(body);
-        logger.info("\n--------------------------------------------------------------------------------")
         return whilstCallback(null, body);
       default:
         whileLoop = false;
@@ -125,7 +138,6 @@ let showSpecsStatus = (data) => {
 }
 
 let printInitialLog = () => {
-  startTime = Date.now();
   logger.info(`\n${Constants.syncCLI.LOGS.INIT_LOG}`)
   logger.info("--------------------------------------------------------------------------------")
   n = Constants.syncCLI.INITIAL_DELAY_MULTIPLIER

bin/helpers/utils.js

@@ -5,7 +5,10 @@ const fs = require("fs");
 
 const usageReporting = require("./usageReporting"),
   logger = require("./logger").winstonLogger,
-  Constants = require("./constants");
+  Constants = require("./constants"),
+  chalk = require('chalk'),
+  syncCliLogger = require("../helpers/logger").syncCliLogger,
+  config = require("../helpers/config");
 
 exports.validateBstackJson = (bsConfigPath) => {
   return new Promise(function (resolve, reject) {
@@ -318,3 +321,20 @@ exports.setLocalIdentifier = (bsConfig) => {
 exports.capitalizeFirstLetter = (stringToCapitalize) => {
   return stringToCapitalize && (stringToCapitalize[0].toUpperCase() + stringToCapitalize.slice(1));
 };
+
+exports.handleSyncExit = (exitCode, dashboard_url) => {
+  if (exitCode === config.networkErrorExitCode) {
+    syncCliLogger.info(this.getNetworkErrorMessage(dashboard_url));
+  } else {
+    syncCliLogger.info(Constants.userMessages.BUILD_REPORT_MESSAGE);
+    syncCliLogger.info(dashboard_url);
+  }
+  process.exit(exitCode);
+}
+
+exports.getNetworkErrorMessage = (dashboard_url) => {
+  let message  =  Constants.userMessages.FATAL_NETWORK_ERROR + '\n'
+                  + Constants.userMessages.RETRY_LIMIT_EXCEEDED + '\n'
+                  + Constants.userMessages.CHECK_DASHBOARD_AT  + dashboard_url
+  return chalk.red(message)
+}

test/unit/bin/helpers/sync/failedSpecDetails.js

@@ -59,4 +59,18 @@ describe("failedSpecsDetails", () => {
       });
     });
   });
+
+  context("failed because of network issue", () => {
+    let data = {
+      specs: [ {specName: 'spec2.name.js', status: 'Failed', combination: 'Win 10 / Chrome 78', sessionId: '3d3rdf3r...'},
+      {specName: 'spec2.name.js', status: 'Passed', combination: 'Win 10 / Chrome 78', sessionId: '3d3rdf3r...'}],
+      exitCode: 2
+    };
+
+    it("returns 2 exit code", () => {
+      return specDetails.failedSpecsDetails(data).then((result) => {
+        expect(result).to.equal(data);
+      });
+    });
+  });
 });

test/unit/bin/helpers/sync/specSummary.js

@@ -18,6 +18,15 @@ describe("printSpecsRunSummary", () => {
     });
   });
 
+  context("request failure", () => {
+    let data = { specs: [], duration: 6000, exitCode: 2}, machines = 2;
+    it('returns passed specs data with proper exit code', () => {
+      return specSummary.printSpecsRunSummary(data, machines).then((specsData) => {
+        expect(data.exitCode).to.equal(specsData);
+      });
+    });
+  });
+
   context("with data", () => {
     let time = 6000,
         machines = 2,

test/unit/bin/helpers/sync/syncSpecsLogs.js

@@ -96,7 +96,7 @@ describe("syncSpecsLogs", () => {
       expect(options.columns[1].alignment).to.equal('center');
       expect(options.columns[2].alignment).to.equal('left');
       expect(options.columns[1].width).to.equal(1);
-      expect(options.columns[2].width).to.equal(30);
+      expect(options.columns[2].width).to.equal(50);
       expect(options.columnCount).to.equal(3);
       expect(getBorderConfigStub.calledOnce).to.be.true;
     });
@@ -259,20 +259,53 @@ describe("syncSpecsLogs", () => {
   context("whileProcess", () => {
     const whileProcess = syncSpecsLogs.__get__("whileProcess");
 
-    it('Should break the loop if request has error', () => {
+    it('Should retry when request fails with error', () => {
+      let delayed_n = 2, timeout = 3000, n = 1;
       let error = new Error("error");
+
       let requestStub = sandbox.stub();
+
+      let postStub = sandbox
+        .stub(request, "post")
+        .yields(error, { statusCode: 502 }, JSON.stringify({}));
+
+      requestStub.post = postStub;
+
+      let setTimeout = sandbox.stub();
+      syncSpecsLogs.__set__('setTimeout', setTimeout);
+      syncSpecsLogs.__set__('n', n);
+      syncSpecsLogs.__set__('timeout', timeout);
+      syncSpecsLogs.__set__('request', requestStub);
+      syncSpecsLogs.__set__('whileTries', 5);
+
+      let whilstCallback = sandbox.stub();
+      whileProcess(whilstCallback);
+
+      sinon.assert.calledWith(setTimeout, whilstCallback, timeout * delayed_n, null);
+      expect(syncSpecsLogs.__get__("whileTries")).to.equal(4);
+    });
+
+    it('Should exit after defined number of retries in case of error', () => {
+      let error = new Error("error"), requestStub = sandbox.stub();
+
       let postStub = sandbox
         .stub(request, "post")
-        .yields(error, { statusCode: 200 }, JSON.stringify({}));
+        .yields(error, { statusCode: 502 }, JSON.stringify({}));
+
       requestStub.post = postStub;
 
       syncSpecsLogs.__set__('request', requestStub);
+      syncSpecsLogs.__set__('whileTries', 1);
+      syncSpecsLogs.__set__('specSummary', {});
+      syncSpecsLogs.__set__('whileLoop', true);
 
       let whilstCallback = sandbox.stub();
       whileProcess(whilstCallback);
 
-      sinon.assert.calledWith(whilstCallback, error);
+      sinon.assert.calledWith(whilstCallback, { status: 504, message: "Tries limit reached" });
+      expect(syncSpecsLogs.__get__("whileTries")).to.equal(0);
+      expect(syncSpecsLogs.__get__("whileLoop")).to.equal(false);
+      expect(syncSpecsLogs.__get__("specSummary.exitCode")).to.equal(2);
     });
 
     it('Should print spec details when data is returned from server', () => {

test/unit/bin/helpers/utils.js

@@ -5,12 +5,14 @@ const chai = require('chai'),
   expect = chai.expect,
   sinon = require('sinon'),
   chaiAsPromised = require('chai-as-promised'),
+  chalk = require('chalk'),
   fs = require('fs');
 
 const utils = require('../../../../bin/helpers/utils'),
   constant = require('../../../../bin/helpers/constants'),
   logger = require('../../../../bin/helpers/logger').winstonLogger,
-  testObjects = require('../../support/fixtures/testObjects');
+  testObjects = require('../../support/fixtures/testObjects'),
+  syncLogger = require("../../../../bin/helpers/logger").syncCliLogger;
 
 chai.use(chaiAsPromised);
 logger.transports['console.info'].silent = true;
@@ -943,4 +945,42 @@ describe('utils', () => {
     });
 
   });
+
+  describe('#handleSyncExit', () => {
+    let processStub;
+    beforeEach(function () {
+      processStub = sinon.stub(process, 'exit');
+    });
+
+    afterEach(function () {
+      processStub.restore();
+    });
+    it('should print network error message when exit code is set to network error code', () => {
+      let dashboard_url = "dashboard_url", exitCode = 2;
+      let getNetworkErrorMessageStub = sinon.stub(utils, 'getNetworkErrorMessage');
+      utils.handleSyncExit(exitCode, dashboard_url);
+      sinon.assert.calledOnce(getNetworkErrorMessageStub);
+      sinon.assert.calledOnceWithExactly(processStub, exitCode);
+      getNetworkErrorMessageStub.restore();
+    });
+
+    it('should print dashboard link when exit code is not network error code', () => {
+      let dashboard_url = "dashboard_url", exitCode = 1;
+      let syncCliLoggerStub = sinon.stub(syncLogger, 'info');
+      utils.handleSyncExit(exitCode, dashboard_url);
+      sinon.assert.calledTwice(syncCliLoggerStub);
+      sinon.assert.calledOnceWithExactly(processStub, exitCode);
+    });
+  });
+
+  describe('#getNetworkErrorMessage', () => {
+    it('should return the error message in red color', () => {
+      let dashboard_url = "dashboard_url";
+      let message  =  constant.userMessages.FATAL_NETWORK_ERROR + '\n'
+                  + constant.userMessages.RETRY_LIMIT_EXCEEDED + '\n'
+                  + constant.userMessages.CHECK_DASHBOARD_AT  + dashboard_url
+      utils.getNetworkErrorMessage(dashboard_url);
+      expect(utils.getNetworkErrorMessage(dashboard_url)).to.eq(chalk.red(message))
+    });
+  });
 });