Skip to content

Add support for checking if zip already uploaded to browserstack. #154

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 4 commits into from
Closed

Add support for checking if zip already uploaded to browserstack. #154

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
9d84e50
Add support for checking if zip aready uploaded to browserstack.
sparsh0204 Jul 20, 2021
7ced873
Merge branch 'master' into check_already_uploaded_md5
sparsh0204 Jul 27, 2021
e76ce5a
improved messaging
sparsh0204 Aug 5, 2021
8c730e8
defined variable
sparsh0204 Aug 9, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
175 changes: 92 additions & 83 deletions bin/commands/runs.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ const archiver = require("../helpers/archiver"),
utils = require("../helpers/utils"),
fileHelpers = require("../helpers/fileHelpers"),
syncRunner = require("../helpers/syncRunner"),
checkUploaded = require("../helpers/checkUploaded"),
reportGenerator = require('../helpers/reporterHTML').reportGenerator,
{initTimeComponents, markBlockStart, markBlockEnd, getTimeComponents} = require('../helpers/timeComponents');

Expand Down Expand Up @@ -78,107 +79,115 @@ module.exports = function run(args) {

// warn if specFiles cross our limit
utils.warnSpecLimit(bsConfig, args, specFiles);

markBlockEnd('preArchiveSteps');
// Archive the spec files
markBlockStart('zip');
markBlockStart('zip.archive');
return archiver.archive(bsConfig.run_settings, config.fileName, args.exclude).then(function (data) {

markBlockEnd('zip.archive');
// Uploaded zip file
markBlockStart('zip.zipUpload');
return zipUploader.zipUpload(bsConfig, config.fileName).then(async function (zip) {

markBlockEnd('zip.zipUpload');
markBlockEnd('zip');
// Create build

//setup Local Testing
markBlockStart('localSetup');
let bs_local = await utils.setupLocalTesting(bsConfig, args);
markBlockEnd('localSetup');
markBlockStart('createBuild');
return build.createBuild(bsConfig, zip).then(function (data) {
markBlockEnd('createBuild');
markBlockEnd('total');
let message = `${data.message}! ${Constants.userMessages.BUILD_CREATED} with build id: ${data.build_id}`;
let dashboardLink = `${Constants.userMessages.VISIT_DASHBOARD} ${data.dashboard_url}`;
utils.exportResults(data.build_id, `${config.dashboardUrl}${data.build_id}`);
if ((utils.isUndefined(bsConfig.run_settings.parallels) && utils.isUndefined(args.parallels)) || (!utils.isUndefined(bsConfig.run_settings.parallels) && bsConfig.run_settings.parallels == Constants.cliMessages.RUN.DEFAULT_PARALLEL_MESSAGE)) {
logger.warn(Constants.userMessages.NO_PARALLELS);
}
markBlockStart('checkAlreadyUploaded');
return checkUploaded.checkUploadedMd5(bsConfig, args).then(function (md5data) {
markBlockEnd('checkAlreadyUploaded');

// Archive the spec files
markBlockStart('zip');
markBlockStart('zip.archive');
return archiver.archive(bsConfig.run_settings, config.fileName, args.exclude, md5data).then(function (data) {
markBlockEnd('zip.archive');

// Uploaded zip file
markBlockStart('zip.zipUpload');
return zipUploader.zipUpload(bsConfig, config.fileName, md5data).then(async function (zip) {
markBlockEnd('zip.zipUpload');
markBlockEnd('zip');
// Create build

//setup Local Testing
markBlockStart('localSetup');
let bs_local = await utils.setupLocalTesting(bsConfig, args);
markBlockEnd('localSetup');
markBlockStart('createBuild');
return build.createBuild(bsConfig, zip).then(function (data) {
markBlockEnd('createBuild');
markBlockEnd('total');
let message = `${data.message}! ${Constants.userMessages.BUILD_CREATED} with build id: ${data.build_id}`;
let dashboardLink = `${Constants.userMessages.VISIT_DASHBOARD} ${data.dashboard_url}`;
utils.exportResults(data.build_id, `${config.dashboardUrl}${data.build_id}`);
if ((utils.isUndefined(bsConfig.run_settings.parallels) && utils.isUndefined(args.parallels)) || (!utils.isUndefined(bsConfig.run_settings.parallels) && bsConfig.run_settings.parallels == Constants.cliMessages.RUN.DEFAULT_PARALLEL_MESSAGE)) {
logger.warn(Constants.userMessages.NO_PARALLELS);
}

if (bsConfig.run_settings.cypress_version && bsConfig.run_settings.cypress_version !== data.cypress_version) {
let versionMessage = utils.versionChangedMessage(bsConfig.run_settings.cypress_version, data.cypress_version)
logger.warn(versionMessage);
}
if (bsConfig.run_settings.cypress_version && bsConfig.run_settings.cypress_version !== data.cypress_version) {
let versionMessage = utils.versionChangedMessage(bsConfig.run_settings.cypress_version, data.cypress_version)
logger.warn(versionMessage);
}

if (!args.disableNpmWarning && bsConfig.run_settings.npm_dependencies && Object.keys(bsConfig.run_settings.npm_dependencies).length <= 0) {
logger.warn(Constants.userMessages.NO_NPM_DEPENDENCIES);
logger.warn(Constants.userMessages.NO_NPM_DEPENDENCIES_READ_MORE);
}
if (!args.disableNpmWarning && bsConfig.run_settings.npm_dependencies && Object.keys(bsConfig.run_settings.npm_dependencies).length <= 0) {
logger.warn(Constants.userMessages.NO_NPM_DEPENDENCIES);
logger.warn(Constants.userMessages.NO_NPM_DEPENDENCIES_READ_MORE);
}

if (args.sync) {
syncRunner.pollBuildStatus(bsConfig, data).then(async (exitCode) => {
if (args.sync) {
syncRunner.pollBuildStatus(bsConfig, data).then(async (exitCode) => {

// stop the Local instance
await utils.stopLocalBinary(bsConfig, bs_local, args);
// stop the Local instance
await utils.stopLocalBinary(bsConfig, bs_local, args);

// Generate custom report!
reportGenerator(bsConfig, data.build_id, args, function(){
utils.sendUsageReport(bsConfig, args, `${message}\n${dashboardLink}`, Constants.messageTypes.SUCCESS, null);
utils.handleSyncExit(exitCode, data.dashboard_url);
// Generate custom report!
reportGenerator(bsConfig, data.build_id, args, function(){
utils.sendUsageReport(bsConfig, args, `${message}\n${dashboardLink}`, Constants.messageTypes.SUCCESS, null);
utils.handleSyncExit(exitCode, data.dashboard_url);
});
});
});
}

logger.info(message);
logger.info(dashboardLink);
if(!args.sync) logger.info(Constants.userMessages.EXIT_SYNC_CLI_MESSAGE.replace("<build-id>",data.build_id));
let dataToSend = {
time_components: getTimeComponents(),
build_id: data.build_id,
};
if (bsConfig && bsConfig.connection_settings) {
if (bsConfig.connection_settings.local_mode) {
dataToSend.local_mode = bsConfig.connection_settings.local_mode;
}
if (bsConfig.connection_settings.usedAutoLocal) {
dataToSend.used_auto_local = bsConfig.connection_settings.usedAutoLocal;

logger.info(message);
logger.info(dashboardLink);
if(!args.sync) logger.info(Constants.userMessages.EXIT_SYNC_CLI_MESSAGE.replace("<build-id>",data.build_id));
let dataToSend = {
time_components: getTimeComponents(),
build_id: data.build_id,
};
if (bsConfig && bsConfig.connection_settings) {
if (bsConfig.connection_settings.local_mode) {
dataToSend.local_mode = bsConfig.connection_settings.local_mode;
}
if (bsConfig.connection_settings.usedAutoLocal) {
dataToSend.used_auto_local = bsConfig.connection_settings.usedAutoLocal;
}
}
}
utils.sendUsageReport(bsConfig, args, `${message}\n${dashboardLink}`, Constants.messageTypes.SUCCESS, null, dataToSend);
return;
}).catch(async function (err) {
// Build creation failed
utils.sendUsageReport(bsConfig, args, `${message}\n${dashboardLink}`, Constants.messageTypes.SUCCESS, null, dataToSend);
return;
}).catch(async function (err) {
// Build creation failed
logger.error(err);
// stop the Local instance
await utils.stopLocalBinary(bsConfig, bs_local, args);

utils.sendUsageReport(bsConfig, args, err, Constants.messageTypes.ERROR, 'build_failed');
});
}).catch(function (err) {
// Zip Upload failed | Local Start failed
logger.error(err);
// stop the Local instance
await utils.stopLocalBinary(bsConfig, bs_local, args);

utils.sendUsageReport(bsConfig, args, err, Constants.messageTypes.ERROR, 'build_failed');
if(err === Constants.userMessages.LOCAL_START_FAILED){
utils.sendUsageReport(bsConfig, args, `${err}\n${Constants.userMessages.LOCAL_START_FAILED}`, Constants.messageTypes.ERROR, 'local_start_failed');
} else {
logger.error(Constants.userMessages.ZIP_UPLOAD_FAILED);
fileHelpers.deleteZip();
utils.sendUsageReport(bsConfig, args, `${err}\n${Constants.userMessages.ZIP_UPLOAD_FAILED}`, Constants.messageTypes.ERROR, 'zip_upload_failed');
}
});
}).catch(function (err) {
// Zip Upload failed | Local Start failed
// Zipping failed
logger.error(err);
if(err === Constants.userMessages.LOCAL_START_FAILED){
utils.sendUsageReport(bsConfig, args, `${err}\n${Constants.userMessages.LOCAL_START_FAILED}`, Constants.messageTypes.ERROR, 'local_start_failed');
} else {
logger.error(Constants.userMessages.ZIP_UPLOAD_FAILED);
logger.error(Constants.userMessages.FAILED_TO_ZIP);
utils.sendUsageReport(bsConfig, args, `${err}\n${Constants.userMessages.FAILED_TO_ZIP}`, Constants.messageTypes.ERROR, 'zip_creation_failed');
try {
fileHelpers.deleteZip();
utils.sendUsageReport(bsConfig, args, `${err}\n${Constants.userMessages.ZIP_UPLOAD_FAILED}`, Constants.messageTypes.ERROR, 'zip_upload_failed');
} catch (err) {
utils.sendUsageReport(bsConfig, args, Constants.userMessages.ZIP_DELETE_FAILED, Constants.messageTypes.ERROR, 'zip_deletion_failed');
}
});
}).catch(function (err) {
// Zipping failed
// md5 check failed
logger.error(err);
logger.error(Constants.userMessages.FAILED_TO_ZIP);
utils.sendUsageReport(bsConfig, args, `${err}\n${Constants.userMessages.FAILED_TO_ZIP}`, Constants.messageTypes.ERROR, 'zip_creation_failed');
try {
fileHelpers.deleteZip();
} catch (err) {
utils.sendUsageReport(bsConfig, args, Constants.userMessages.ZIP_DELETE_FAILED, Constants.messageTypes.ERROR, 'zip_deletion_failed');
}
logger.error(Constants.userMessages.FAILED_MD5_CHECK);
utils.sendUsageReport(bsConfig, args, Constants.userMessages.MD5_CHECK_FAILED, Constants.messageTypes.ERROR, 'zip_already_uploaded_failed');
});
}).catch(function (err) {
// browerstack.json is not valid
Expand Down
25 changes: 5 additions & 20 deletions bin/helpers/archiver.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,11 @@ const archiver = require("archiver"),
utils = require('../helpers/utils'),
path = require('path');

const archiveSpecs = (runSettings, filePath, excludeFiles) => {
const archiveSpecs = (runSettings, filePath, excludeFiles, md5data) => {
return new Promise(function (resolve, reject) {
if (md5data.zipUrlPresent) {
return resolve('Zipping not required');
}
var output = fs.createWriteStream(filePath);

var cypressFolderPath = path.dirname(runSettings.cypressConfigFilePath);
Expand Down Expand Up @@ -41,8 +44,7 @@ const archiveSpecs = (runSettings, filePath, excludeFiles) => {

archive.pipe(output);

let ignoreFiles = getFilesToIgnore(runSettings, excludeFiles);

let ignoreFiles = utils.getFilesToIgnore(runSettings, excludeFiles);
archive.glob(`**/*.+(${Constants.allowedFileTypes.join("|")})`, { cwd: cypressFolderPath, matchBase: true, ignore: ignoreFiles, dot:true });

let packageJSON = {};
Expand Down Expand Up @@ -78,21 +80,4 @@ const archiveSpecs = (runSettings, filePath, excludeFiles) => {
});
}

const getFilesToIgnore = (runSettings, excludeFiles) => {
let ignoreFiles = Constants.filesToIgnoreWhileUploading;

// exclude files asked by the user
// args will take precedence over config file
if (!utils.isUndefined(excludeFiles)) {
let excludePatterns = utils.fixCommaSeparatedString(excludeFiles).split(',');
ignoreFiles = ignoreFiles.concat(excludePatterns);
logger.info(`Excluding files matching: ${JSON.stringify(excludePatterns)}`);
} else if (!utils.isUndefined(runSettings.exclude) && runSettings.exclude.length) {
ignoreFiles = ignoreFiles.concat(runSettings.exclude);
logger.info(`Excluding files matching: ${JSON.stringify(runSettings.exclude)}`);
}

return ignoreFiles;
}

exports.archive = archiveSpecs
106 changes: 106 additions & 0 deletions bin/helpers/checkUploaded.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,106 @@
'use strict';
const request = require('request');

const crypto = require('crypto'),
Constants = require('./constants'),
hashHelper = require('./hashUtil'),
config = require('./config'),
path = require('path'),
fs = require("fs"),
utils = require('./utils');


const checkSpecsMd5 = (runSettings, excludeFiles) => {
return new Promise(function (resolve, reject) {
let cypressFolderPath = path.dirname(runSettings.cypressConfigFilePath);
let ignoreFiles = utils.getFilesToIgnore(runSettings, excludeFiles, false);
let options = {
cwd: cypressFolderPath,
ignore: ignoreFiles,
pattern: `**/*.+(${Constants.allowedFileTypes.join("|")})`
};
hashHelper.hashWrapper(options).then(function (data) {
const outputHash = crypto.createHash(Constants.hashingOptions.algo);
outputHash.update(data);
let packageJSON = {};

if (typeof runSettings.package_config_options === 'object') {
Object.assign(packageJSON, runSettings.package_config_options);
}

if (typeof runSettings.npm_dependencies === 'object') {
Object.assign(packageJSON, {
devDependencies: runSettings.npm_dependencies,
});
}

if (Object.keys(packageJSON).length > 0) {
let packageJSONString = JSON.stringify(packageJSON);
outputHash.update(packageJSONString);
}

if (
runSettings.cypress_config_file &&
runSettings.cypress_config_filename !== 'false'
) {
let cypressJSON = JSON.parse(
fs.readFileSync(runSettings.cypressConfigFilePath)
);
let cypressJSONString = JSON.stringify(cypressJSON);
outputHash.update(cypressJSONString);
}
resolve(outputHash.digest(Constants.hashingOptions.encoding));
}).catch(function (error) {
reject(error);
});
});
};

const checkUploadedMd5 = (bsConfig, args) => {
return new Promise(function (resolve) {
let obj = {
zipUrlPresent: false,
};
if (args["force-upload"]) {
return resolve(obj);
}
checkSpecsMd5(bsConfig.run_settings, args.exclude).then(function (md5data) {
Object.assign(obj, {md5sum: md5data});
let data = JSON.stringify({ zip_md5sum: md5data });

let options = {
url: config.checkMd5sum,
auth: {
user: bsConfig.auth.username,
password: bsConfig.auth.access_key
},
headers: {
'Content-Type': 'application/json',
"User-Agent": utils.getUserAgent(),
},
body: data
};

request.post(options, function (err, resp, body) {
if (err) {
resolve(obj);
} else {
let zipData = null;
try {
zipData = JSON.parse(body);
} catch (error) {
zipData = {};
}
if (resp.statusCode === 200 && !utils.isUndefined(zipData.zipUrl)) {
Object.assign(obj, zipData, {zipUrlPresent: true});
}
resolve(obj);
}
});
}).catch((error) => {
resolve({zipUrlPresent: false});
});
});
};

exports.checkUploadedMd5 = checkUploadedMd5;
1 change: 1 addition & 0 deletions bin/helpers/config.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ if(config.env !== "production") {
config.cypress_v1 = `${config.rails_host}/automate/cypress/v1`;
config.buildUrl = `${config.cypress_v1}/builds/`;
config.buildStopUrl = `${config.cypress_v1}/builds/stop/`;
config.checkMd5sum = `${config.cypress_v1}/md5sumcheck/`;
config.fileName = "tests.zip";
config.retries = 5;
config.networkErrorExitCode = 2;
Expand Down
Loading