From b01a8db0d3fe4c5a291515c234104fa0a6827c36 Mon Sep 17 00:00:00 2001 From: Archish Date: Mon, 6 Feb 2023 21:06:48 +0530 Subject: [PATCH 1/3] :art: added instrumentation for package diff and pushing package.json --- bin/commands/runs.js | 15 ++++++-- bin/helpers/archiver.js | 9 +++++ bin/helpers/package-diff.js | 71 +++++++++++++++++++++++++++++++++++++ 3 files changed, 93 insertions(+), 2 deletions(-) create mode 100644 bin/helpers/package-diff.js diff --git a/bin/commands/runs.js b/bin/commands/runs.js index 18d64936..c6bcbfda 100644 --- a/bin/commands/runs.js +++ b/bin/commands/runs.js @@ -19,7 +19,8 @@ const archiver = require("../helpers/archiver"), downloadBuildArtifacts = require('../helpers/buildArtifacts').downloadBuildArtifacts, downloadBuildStacktrace = require('../helpers/downloadBuildStacktrace').downloadBuildStacktrace, updateNotifier = require('update-notifier'), - pkg = require('../../package.json'); + pkg = require('../../package.json'), + packageDiff = require('../helpers/package-diff'); const { getStackTraceUrl } = require('../helpers/sync/syncSpecsLogs'); module.exports = function run(args, rawArgs) { @@ -164,7 +165,14 @@ module.exports = function run(args, rawArgs) { let test_zip_size = utils.fetchZipSize(path.join(process.cwd(), config.fileName)); let npm_zip_size = utils.fetchZipSize(path.join(process.cwd(), config.packageFileName)); - + + //Package diff + let isPackageDiff = false; + if(!md5data.zipUrlPresent){ + isPackageDiff = packageDiff.run(`package.json`, `${config.packageDirName}/package.json`); + logger.debug(`Package difference was ${isPackageDiff ? `found` : `not found`}`); + } + // Uploaded zip file logger.debug("Started uploading the test suite zip"); logger.debug("Started uploading the node_module zip"); @@ -270,6 +278,9 @@ module.exports = function run(args, rawArgs) { test_suite_zip_upload: md5data.zipUrlPresent ? 0 : 1, package_zip_upload: md5data.packageUrlPresent ? 0 : 1 }; + if(dataToSend.test_suite_zip_upload === 1 ){ + dataToSend['is_package_diff'] = isPackageDiff; + } if (!md5data.zipUrlPresent && zip.tests_upload_time) { dataToSend.test_suite_zip_size = parseFloat((test_zip_size / 1024).toFixed(2)); diff --git a/bin/helpers/archiver.js b/bin/helpers/archiver.js index 74d8e667..392abeac 100644 --- a/bin/helpers/archiver.js +++ b/bin/helpers/archiver.js @@ -1,4 +1,5 @@ 'use strict'; +const config = require('./config.js'); const fs = require("fs"), path = require("path"); @@ -76,6 +77,14 @@ const archiveSpecs = (runSettings, filePath, excludeFiles, md5data) => { archive.append(packageJSONString, {name: `${cypressAppendFilesZipLocation}browserstack-package.json`}); } + //Create copy of package.json + if(fs.existsSync('package.json')){ + let originalPackageJson = JSON.parse(fs.readFileSync('package.json')); + let originalPackageJsonString = JSON.stringify(originalPackageJson, null, 4); + archive.append(originalPackageJsonString, {name: `${cypressAppendFilesZipLocation}userPackage.json`}); + logger.debug(`Created copy of package.json in ${config.packageDirName} folder`) + } + // do not add cypress.json if arg provided is false if ( runSettings.cypress_config_file && diff --git a/bin/helpers/package-diff.js b/bin/helpers/package-diff.js new file mode 100644 index 00000000..c28e8ee4 --- /dev/null +++ b/bin/helpers/package-diff.js @@ -0,0 +1,71 @@ +'use strict' +const fs = require('fs') +const path = require('path') +const logger = require("./logger").winstonLogger; + +exports.run = (basePath, comparePath) => { + if (!basePath || !comparePath) { + logger.debug('Skipping package difference check.') + } + + const base = readModules(basePath) + const compare = readModules(comparePath) + + let isDiff = false; + Object.keys(base.deps).forEach(baseKey => { + if (baseKey in compare.deps) { + if (base.deps[baseKey] === compare.deps[baseKey]) { + //ignore this check + } else { + isDiff = true; + return; + } + } else { + isDiff = true; + return; + } + }) + return isDiff; +} +const readModules = (location) => { + const table = {} + + // Resolve package dependencies + if (location.indexOf('package.json') !== -1) { + const data = fs.readFileSync(location.replace(':dev', ''), 'utf-8') + let parsed + try { parsed = JSON.parse(data) } catch (e) { parsed = false } + if (!parsed) { return } + + const depsKey = location.indexOf(':dev') !== -1 ? 'devDependencies' : 'dependencies' + const deps = parsed[depsKey] ? parsed[depsKey] : (parsed.dependencies || parsed.devDependencies) + + Object.keys(deps).forEach(key => { + deps[key] = deps[key].replace(/\^|~/g, '') + }) + return { + name: `${location} {${depsKey}}`, + deps, + } + } + + fs.readdirSync(location) + .filter(name => name !== '.bin') + .map(name => { + const pkg = path.join(location, name, 'package.json') + const exists = fs.existsSync(pkg) + if (!exists) { return } + + const data = fs.readFileSync(pkg, 'utf-8') + let parsed + + try { parsed = JSON.parse(data) } catch (e) { parsed = false } + if (!parsed) { return } + + table[name] = parsed.version + }) + return { + name: location, + deps: table, +} +} From aa2e1cdd7c8498d78ef412d03ace8c83590c34c4 Mon Sep 17 00:00:00 2001 From: Archish Date: Tue, 7 Feb 2023 17:48:09 +0530 Subject: [PATCH 2/3] :art: added instrumentation for node_modules package --- bin/commands/runs.js | 4 +++- bin/helpers/utils.js | 31 +++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/bin/commands/runs.js b/bin/commands/runs.js index c6bcbfda..14e4e98e 100644 --- a/bin/commands/runs.js +++ b/bin/commands/runs.js @@ -159,12 +159,13 @@ module.exports = function run(args, rawArgs) { // Archive the spec files logger.debug("Started archiving test suite"); markBlockStart('zip.archive'); - return archiver.archive(bsConfig.run_settings, config.fileName, args.exclude, md5data).then(function (data) { + return archiver.archive(bsConfig.run_settings, config.fileName, args.exclude, md5data).then(async function (data) { logger.debug("Completed archiving test suite"); markBlockEnd('zip.archive'); let test_zip_size = utils.fetchZipSize(path.join(process.cwd(), config.fileName)); let npm_zip_size = utils.fetchZipSize(path.join(process.cwd(), config.packageFileName)); + let node_modules_size = await utils.fetchFolderSize(path.join(process.cwd(), "node_modules")) //Package diff let isPackageDiff = false; @@ -275,6 +276,7 @@ module.exports = function run(args, rawArgs) { build_id: data.build_id, test_zip_size: test_zip_size, npm_zip_size: npm_zip_size, + node_modules_size: node_modules_size, test_suite_zip_upload: md5data.zipUrlPresent ? 0 : 1, package_zip_upload: md5data.packageUrlPresent ? 0 : 1 }; diff --git a/bin/helpers/utils.js b/bin/helpers/utils.js index 6616816f..91dbcd59 100644 --- a/bin/helpers/utils.js +++ b/bin/helpers/utils.js @@ -8,6 +8,9 @@ const { v4: uuidv4 } = require('uuid'); const browserstack = require('browserstack-local'); const crypto = require('crypto'); const util = require('util'); +const { promisify } = require('util'); +const readdir = promisify(fs.readdir); +const stat = promisify(fs.stat); const usageReporting = require("./usageReporting"), logger = require("./logger").winstonLogger, @@ -1278,6 +1281,34 @@ exports.fetchZipSize = (fileName) => { } } +const getDirectorySize = async function(dir) { + try{ + const subdirs = (await readdir(dir)); + const files = await Promise.all(subdirs.map(async (subdir) => { + const res = path.resolve(dir, subdir); + const s = (await stat(res)); + return s.isDirectory() ? getDirectorySize(res) : (s.size); + })); + return files.reduce((a, f) => a+f, 0); + }catch(e){ + console.log(`Error ${e}`) + logger.debug('Failed to get file or directory.'); + return 0; + } +}; + +exports.fetchFolderSize = async (dir) => { + try { + if(fs.existsSync(dir)){ + return (await getDirectorySize(dir) / 1024 / 1024); + } + return 0; + } catch (error) { + logger.debug(`Failed to get directory size.`); + return 0; + } +} + exports.getVideoConfig = (cypressConfig) => { let conf = { video: true, From 97446bd85e3e39e342359ab0b865a43137420ea2 Mon Sep 17 00:00:00 2001 From: Archish Date: Thu, 9 Feb 2023 12:04:42 +0530 Subject: [PATCH 3/3] :art: added spec for folder dir --- bin/helpers/package-diff.js | 138 +++++++++++++++++++-------------- test/unit/bin/helpers/utils.js | 13 +++- 2 files changed, 90 insertions(+), 61 deletions(-) diff --git a/bin/helpers/package-diff.js b/bin/helpers/package-diff.js index c28e8ee4..16a6cd66 100644 --- a/bin/helpers/package-diff.js +++ b/bin/helpers/package-diff.js @@ -1,71 +1,93 @@ -'use strict' -const fs = require('fs') -const path = require('path') +"use strict"; +const fs = require("fs"); +const path = require("path"); const logger = require("./logger").winstonLogger; -exports.run = (basePath, comparePath) => { - if (!basePath || !comparePath) { - logger.debug('Skipping package difference check.') - } - - const base = readModules(basePath) - const compare = readModules(comparePath) +exports.run = (basePath, comparePath) => { + if (!basePath || !comparePath) { + logger.debug("Skipping package difference check."); + } - let isDiff = false; - Object.keys(base.deps).forEach(baseKey => { - if (baseKey in compare.deps) { - if (base.deps[baseKey] === compare.deps[baseKey]) { - //ignore this check - } else { - isDiff = true; - return; - } - } else { - isDiff = true; - return; - } - }) + let base; + let compare; + let isDiff = false; + try { + base = readModules(basePath); + compare = readModules(comparePath); + } catch (error) { + logger.debug('Unable to process package difference'); return isDiff; -} -const readModules = (location) => { - const table = {} - - // Resolve package dependencies - if (location.indexOf('package.json') !== -1) { - const data = fs.readFileSync(location.replace(':dev', ''), 'utf-8') - let parsed - try { parsed = JSON.parse(data) } catch (e) { parsed = false } - if (!parsed) { return } - - const depsKey = location.indexOf(':dev') !== -1 ? 'devDependencies' : 'dependencies' - const deps = parsed[depsKey] ? parsed[depsKey] : (parsed.dependencies || parsed.devDependencies) + } - Object.keys(deps).forEach(key => { - deps[key] = deps[key].replace(/\^|~/g, '') - }) - return { - name: `${location} {${depsKey}}`, - deps, + Object.keys(base.deps).forEach((baseKey) => { + if (baseKey in compare.deps) { + if (base.deps[baseKey] !== compare.deps[baseKey]) { + isDiff = true; + return; } + } else { + isDiff = true; + return; } - - fs.readdirSync(location) - .filter(name => name !== '.bin') - .map(name => { - const pkg = path.join(location, name, 'package.json') - const exists = fs.existsSync(pkg) - if (!exists) { return } + }); + return isDiff; +}; +const readModules = (location) => { + const table = {}; - const data = fs.readFileSync(pkg, 'utf-8') - let parsed + // Resolve package dependencies + if (location.indexOf("package.json") !== -1) { + const data = fs.readFileSync(location.replace(":dev", ""), "utf-8"); + let parsed; + try { + parsed = JSON.parse(data); + } catch (e) { + parsed = false; + } + if (!parsed) { + return; + } - try { parsed = JSON.parse(data) } catch (e) { parsed = false } - if (!parsed) { return } + const depsKey = + location.indexOf(":dev") !== -1 ? "devDependencies" : "dependencies"; + const deps = parsed[depsKey] + ? parsed[depsKey] + : parsed.dependencies || parsed.devDependencies; - table[name] = parsed.version - }) + Object.keys(deps).forEach((key) => { + deps[key] = deps[key].replace(/\^|~/g, ""); + }); return { + name: `${location} {${depsKey}}`, + deps, + }; + } + + fs.readdirSync(location) + .filter((name) => name !== ".bin") + .map((name) => { + const pkg = path.join(location, name, "package.json"); + const exists = fs.existsSync(pkg); + if (!exists) { + return; + } + + const data = fs.readFileSync(pkg, "utf-8"); + let parsed; + + try { + parsed = JSON.parse(data); + } catch (e) { + parsed = false; + } + if (!parsed) { + return; + } + + table[name] = parsed.version; + }); + return { name: location, deps: table, -} -} + }; +}; diff --git a/test/unit/bin/helpers/utils.js b/test/unit/bin/helpers/utils.js index db1317aa..de4ae1f0 100644 --- a/test/unit/bin/helpers/utils.js +++ b/test/unit/bin/helpers/utils.js @@ -1042,6 +1042,13 @@ describe('utils', () => { }); }); + describe('getDirectorySize', () => { + it('should return size of directory', async() => { + expect(await utils.fetchFolderSize('/absolute/path')).to + .be.equal(0); + }); + }); + describe('getLocalFlag', () => { it('should return false if connectionSettings is undefined', () => { expect(utils.getLocalFlag(undefined)).to.be.false; @@ -1834,7 +1841,7 @@ describe('utils', () => { describe('setCypressTestSuiteType', () => { it('sets correct cypressTestSuiteType when cypress.json is the cypress config file ', () => { - bsConfig = { + let bsConfig = { run_settings: { cypressConfigFilePath: 'cypress.json', }, @@ -1846,7 +1853,7 @@ describe('utils', () => { }); it('sets correct cypressTestSuiteType when cypress.config.js|.ts|.cjs|.mjs is the cypress config file ', () => { - bsConfig = { + let bsConfig = { run_settings: { cypressConfigFilePath: 'cypress.config.js', }, @@ -1880,7 +1887,7 @@ describe('utils', () => { }); it('by default assumes that CYPRESS_V9_AND_OLDER_TYPE is the test suite type', () => { - bsConfig = { + let bsConfig = { run_settings: {}, }; utils.setCypressTestSuiteType(bsConfig);