From b3303ce57b3e6f076ebebf2d8e27045816c2481c Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Wed, 1 Mar 2017 14:59:17 +0100 Subject: [PATCH 01/12] Added FQBN in the buildProperties Signed-off-by: Cristian Maglie --- src/arduino.cc/builder/builder_utils/utils.go | 6 ++++++ src/arduino.cc/builder/constants/constants.go | 1 + src/arduino.cc/builder/setup_build_properties.go | 1 + 3 files changed, 8 insertions(+) diff --git a/src/arduino.cc/builder/builder_utils/utils.go b/src/arduino.cc/builder/builder_utils/utils.go index d21420a3..96d71179 100644 --- a/src/arduino.cc/builder/builder_utils/utils.go +++ b/src/arduino.cc/builder/builder_utils/utils.go @@ -366,3 +366,9 @@ func ExecRecipeCollectStdErr(buildProperties properties.Map, recipe string, remo func RemoveHyphenMDDFlagFromGCCCommandLine(buildProperties properties.Map) { buildProperties[constants.BUILD_PROPERTIES_COMPILER_CPP_FLAGS] = strings.Replace(buildProperties[constants.BUILD_PROPERTIES_COMPILER_CPP_FLAGS], "-MMD", "", -1) } + +func GetCoreArchivePath(fqbn string) string { + fqbnToUnderscore := strings.Replace(fqbn, ":", "_", -1) + fqbnToUnderscore = strings.Replace(fqbnToUnderscore, "=", "_", -1) + return os.TempDir() + "/core_" + fqbnToUnderscore + ".a" +} diff --git a/src/arduino.cc/builder/constants/constants.go b/src/arduino.cc/builder/constants/constants.go index cdbdb72b..fdbf00cc 100644 --- a/src/arduino.cc/builder/constants/constants.go +++ b/src/arduino.cc/builder/constants/constants.go @@ -55,6 +55,7 @@ const BUILD_PROPERTIES_EXTRA_TIME_DST = "extra.time.dst" const BUILD_PROPERTIES_EXTRA_TIME_LOCAL = "extra.time.local" const BUILD_PROPERTIES_EXTRA_TIME_UTC = "extra.time.utc" const BUILD_PROPERTIES_EXTRA_TIME_ZONE = "extra.time.zone" +const BUILD_PROPERTIES_FQBN = "build.fqbn" const BUILD_PROPERTIES_INCLUDES = "includes" const BUILD_PROPERTIES_OBJECT_FILE = "object_file" const BUILD_PROPERTIES_OBJECT_FILES = "object_files" diff --git a/src/arduino.cc/builder/setup_build_properties.go b/src/arduino.cc/builder/setup_build_properties.go index e278a39d..02a272fc 100644 --- a/src/arduino.cc/builder/setup_build_properties.go +++ b/src/arduino.cc/builder/setup_build_properties.go @@ -70,6 +70,7 @@ func (s *SetupBuildProperties) Run(ctx *types.Context) error { buildProperties[constants.BUILD_PROPERTIES_RUNTIME_PLATFORM_PATH] = targetPlatform.Folder buildProperties[constants.BUILD_PROPERTIES_RUNTIME_HARDWARE_PATH] = filepath.Join(targetPlatform.Folder, "..") buildProperties[constants.BUILD_PROPERTIES_RUNTIME_IDE_VERSION] = ctx.ArduinoAPIVersion + buildProperties[constants.BUILD_PROPERTIES_FQBN] = ctx.FQBN buildProperties[constants.IDE_VERSION] = ctx.ArduinoAPIVersion buildProperties[constants.BUILD_PROPERTIES_RUNTIME_OS] = utils.PrettyOSName() From e36af020de238968e4fc08069288b01e80c9a36b Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Wed, 1 Mar 2017 15:00:35 +0100 Subject: [PATCH 02/12] [WIP] Archive compiled core and use in any compatible project --- src/arduino.cc/builder/builder_utils/utils.go | 91 +++++++++++++++++++ src/arduino.cc/builder/phases/core_builder.go | 14 +++ src/arduino.cc/builder/utils/utils.go | 12 +++ 3 files changed, 117 insertions(+) diff --git a/src/arduino.cc/builder/builder_utils/utils.go b/src/arduino.cc/builder/builder_utils/utils.go index 96d71179..196c1b63 100644 --- a/src/arduino.cc/builder/builder_utils/utils.go +++ b/src/arduino.cc/builder/builder_utils/utils.go @@ -32,6 +32,7 @@ package builder_utils import ( "bytes" "fmt" + "io" "os" "os/exec" "path/filepath" @@ -116,6 +117,34 @@ func findFilesInFolder(sourcePath string, extension string, recurse bool) ([]str return sources, nil } +func findAllFilesInFolder(sourcePath string, recurse bool) ([]string, error) { + files, err := utils.ReadDirFiltered(sourcePath, utils.FilterFiles()) + if err != nil { + return nil, i18n.WrapError(err) + } + var sources []string + for _, file := range files { + sources = append(sources, filepath.Join(sourcePath, file.Name())) + } + + if recurse { + folders, err := utils.ReadDirFiltered(sourcePath, utils.FilterDirs) + if err != nil { + return nil, i18n.WrapError(err) + } + + for _, folder := range folders { + otherSources, err := findAllFilesInFolder(filepath.Join(sourcePath, folder.Name()), recurse) + if err != nil { + return nil, i18n.WrapError(err) + } + sources = append(sources, otherSources...) + } + } + + return sources, nil +} + func compileFilesWithRecipe(objectFiles []string, sourcePath string, sources []string, buildPath string, buildProperties properties.Map, includes []string, recipe string, verbose bool, warningsLevel string, logger i18n.Logger) ([]string, error) { for _, source := range sources { objectFile, err := compileFileWithRecipe(sourcePath, source, buildPath, buildProperties, includes, recipe, verbose, warningsLevel, logger) @@ -258,6 +287,24 @@ func nonEmptyString(s string) bool { return s != constants.EMPTY_STRING } +func CheckIfRecompileIsAvoidable(corePath string, archiveFile string) bool { + archiveFileStat, err := os.Stat(archiveFile) + if err == nil { + files, err := findAllFilesInFolder(corePath, true) + if err != nil { + return false + } + for _, file := range files { + fileStat, err := os.Stat(file) + if err != nil || fileStat.ModTime().After(archiveFileStat.ModTime()) { + return false + } + } + return true + } + return false +} + func ArchiveCompiledFiles(buildPath string, archiveFile string, objectFiles []string, buildProperties properties.Map, verbose bool, logger i18n.Logger) (string, error) { archiveFilePath := filepath.Join(buildPath, archiveFile) @@ -367,6 +414,50 @@ func RemoveHyphenMDDFlagFromGCCCommandLine(buildProperties properties.Map) { buildProperties[constants.BUILD_PROPERTIES_COMPILER_CPP_FLAGS] = strings.Replace(buildProperties[constants.BUILD_PROPERTIES_COMPILER_CPP_FLAGS], "-MMD", "", -1) } +// CopyFile copies the contents of the file named src to the file named +// by dst. The file will be created if it does not already exist. If the +// destination file exists, all it's contents will be replaced by the contents +// of the source file. The file mode will be copied from the source and +// the copied data is synced/flushed to stable storage. +func CopyFile(src, dst string) (err error) { + in, err := os.Open(src) + if err != nil { + return + } + defer in.Close() + + out, err := os.Create(dst) + if err != nil { + return + } + defer func() { + if e := out.Close(); e != nil { + err = e + } + }() + + _, err = io.Copy(out, in) + if err != nil { + return + } + + err = out.Sync() + if err != nil { + return + } + + si, err := os.Stat(src) + if err != nil { + return + } + err = os.Chmod(dst, si.Mode()) + if err != nil { + return + } + + return +} + func GetCoreArchivePath(fqbn string) string { fqbnToUnderscore := strings.Replace(fqbn, ":", "_", -1) fqbnToUnderscore = strings.Replace(fqbnToUnderscore, "=", "_", -1) diff --git a/src/arduino.cc/builder/phases/core_builder.go b/src/arduino.cc/builder/phases/core_builder.go index 0a0c78c6..bfbc8057 100644 --- a/src/arduino.cc/builder/phases/core_builder.go +++ b/src/arduino.cc/builder/phases/core_builder.go @@ -84,6 +84,17 @@ func compileCore(buildPath string, buildProperties properties.Map, verbose bool, } } + targetArchivedCore := builder_utils.GetCoreArchivePath(buildProperties[constants.BUILD_PROPERTIES_FQBN]) + noNeedToRecompile := builder_utils.CheckIfRecompileIsAvoidable(coreFolder, targetArchivedCore) + + if noNeedToRecompile { + // use archived core + if verbose { + logger.Println(constants.LOG_LEVEL_INFO, "Using precompiled core") + } + return targetArchivedCore, variantObjectFiles, nil + } + coreObjectFiles, err := builder_utils.CompileFiles([]string{}, coreFolder, true, buildPath, buildProperties, includes, verbose, warningsLevel, logger) if err != nil { return "", nil, i18n.WrapError(err) @@ -94,5 +105,8 @@ func compileCore(buildPath string, buildProperties properties.Map, verbose bool, return "", nil, i18n.WrapError(err) } + // archive core.a + builder_utils.CopyFile(archiveFile, targetArchivedCore) + return archiveFile, variantObjectFiles, nil } diff --git a/src/arduino.cc/builder/utils/utils.go b/src/arduino.cc/builder/utils/utils.go index a3bd44e9..1b1823b4 100644 --- a/src/arduino.cc/builder/utils/utils.go +++ b/src/arduino.cc/builder/utils/utils.go @@ -150,6 +150,18 @@ func FilterFilesWithExtensions(extensions ...string) filterFiles { } } +func FilterFiles() filterFiles { + return func(files []os.FileInfo) []os.FileInfo { + var filtered []os.FileInfo + for _, file := range files { + if !file.IsDir() { + filtered = append(filtered, file) + } + } + return filtered + } +} + var SOURCE_CONTROL_FOLDERS = map[string]bool{"CVS": true, "RCS": true, ".git": true, ".github": true, ".svn": true, ".hg": true, ".bzr": true, ".vscode": true} func IsSCCSOrHiddenFile(file os.FileInfo) bool { From 1b629fde28090184b08600c0f60a1752f0dd4686 Mon Sep 17 00:00:00 2001 From: Martino Facchin Date: Fri, 10 Feb 2017 11:37:02 +0100 Subject: [PATCH 03/12] Fix tests that fails when core.a caching is enabled Some tests require that the core object files are recreated, but it doesn't happen when the prebuilt core is used. --- src/arduino.cc/builder/test/builder_test.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/arduino.cc/builder/test/builder_test.go b/src/arduino.cc/builder/test/builder_test.go index 3244c705..d0a34249 100644 --- a/src/arduino.cc/builder/test/builder_test.go +++ b/src/arduino.cc/builder/test/builder_test.go @@ -130,6 +130,10 @@ func TestBuilderSketchWithConfig(t *testing.T) { err := command.Run(ctx) NoError(t, err) + // Cleanup cached core + coreFile := builder_utils.GetCoreArchivePath(ctx.FQBN) + os.Remove(coreFile) + _, err = os.Stat(filepath.Join(buildPath, constants.FOLDER_CORE, "HardwareSerial.cpp.o")) NoError(t, err) _, err = os.Stat(filepath.Join(buildPath, constants.FOLDER_PREPROC, constants.FILE_CTAGS_TARGET_FOR_GCC_MINUS_E)) @@ -168,6 +172,10 @@ func TestBuilderBridgeTwice(t *testing.T) { err = command.Run(ctx) NoError(t, err) + // Cleanup cached core + coreFile := builder_utils.GetCoreArchivePath(ctx.FQBN) + os.Remove(coreFile) + _, err = os.Stat(filepath.Join(buildPath, constants.FOLDER_CORE, "HardwareSerial.cpp.o")) NoError(t, err) _, err = os.Stat(filepath.Join(buildPath, constants.FOLDER_PREPROC, constants.FILE_CTAGS_TARGET_FOR_GCC_MINUS_E)) From 2444448e7fa5dedabadc2ab8e16d10b0afc8caff Mon Sep 17 00:00:00 2001 From: Martino Facchin Date: Fri, 10 Feb 2017 11:55:02 +0100 Subject: [PATCH 04/12] Recreate the archive if ANY of the core files (including platform.txt) has changed --- src/arduino.cc/builder/phases/core_builder.go | 5 ++++- src/arduino.cc/builder/utils/utils.go | 10 ++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/arduino.cc/builder/phases/core_builder.go b/src/arduino.cc/builder/phases/core_builder.go index bfbc8057..0ef473d1 100644 --- a/src/arduino.cc/builder/phases/core_builder.go +++ b/src/arduino.cc/builder/phases/core_builder.go @@ -84,8 +84,11 @@ func compileCore(buildPath string, buildProperties properties.Map, verbose bool, } } + // Recreate the archive if ANY of the core files (including platform.txt) has changed + realCoreFolder := utils.GetParentFolder(coreFolder, 2) + targetArchivedCore := builder_utils.GetCoreArchivePath(buildProperties[constants.BUILD_PROPERTIES_FQBN]) - noNeedToRecompile := builder_utils.CheckIfRecompileIsAvoidable(coreFolder, targetArchivedCore) + noNeedToRecompile := builder_utils.CheckIfRecompileIsAvoidable(realCoreFolder, targetArchivedCore) if noNeedToRecompile { // use archived core diff --git a/src/arduino.cc/builder/utils/utils.go b/src/arduino.cc/builder/utils/utils.go index 1b1823b4..0aa2e811 100644 --- a/src/arduino.cc/builder/utils/utils.go +++ b/src/arduino.cc/builder/utils/utils.go @@ -365,6 +365,16 @@ func FindFilesInFolder(files *[]string, folder string, extensions CheckExtension return gohasissues.Walk(folder, walkFunc) } +func GetParentFolder(basefolder string, n int) string { + tempFolder := basefolder + i := 0 + for i < n { + tempFolder = filepath.Dir(tempFolder) + i++ + } + return tempFolder +} + func AppendIfNotPresent(target []string, elements ...string) []string { for _, element := range elements { if !SliceContains(target, element) { From 40372d7d7ee8c86a6867a6efe40406c9e3c856e7 Mon Sep 17 00:00:00 2001 From: Martino Facchin Date: Thu, 16 Feb 2017 13:26:25 +0100 Subject: [PATCH 05/12] Check for modifications also in referenced core --- src/arduino.cc/builder/phases/core_builder.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/arduino.cc/builder/phases/core_builder.go b/src/arduino.cc/builder/phases/core_builder.go index 0ef473d1..5ada42b8 100644 --- a/src/arduino.cc/builder/phases/core_builder.go +++ b/src/arduino.cc/builder/phases/core_builder.go @@ -36,6 +36,7 @@ import ( "arduino.cc/builder/types" "arduino.cc/builder/utils" "arduino.cc/properties" + "strings" ) type CoreBuilder struct{} @@ -67,6 +68,8 @@ func compileCore(buildPath string, buildProperties properties.Map, verbose bool, coreFolder := buildProperties[constants.BUILD_PROPERTIES_BUILD_CORE_PATH] variantFolder := buildProperties[constants.BUILD_PROPERTIES_BUILD_VARIANT_PATH] + targetCoreFolder := buildProperties[constants.BUILD_PROPERTIES_RUNTIME_PLATFORM_PATH] + includes := []string{} includes = append(includes, coreFolder) if variantFolder != constants.EMPTY_STRING { @@ -90,6 +93,11 @@ func compileCore(buildPath string, buildProperties properties.Map, verbose bool, targetArchivedCore := builder_utils.GetCoreArchivePath(buildProperties[constants.BUILD_PROPERTIES_FQBN]) noNeedToRecompile := builder_utils.CheckIfRecompileIsAvoidable(realCoreFolder, targetArchivedCore) + if !strings.EqualFold(realCoreFolder, targetCoreFolder) { + // targetCoreFolder is not a parent of realCoreFolder, so check it for modifications + noNeedToRecompile = noNeedToRecompile && builder_utils.CheckIfRecompileIsAvoidable(targetCoreFolder, targetArchivedCore) + } + if noNeedToRecompile { // use archived core if verbose { From 19751fe9aacb983043e1f8a580cf24e06a6b96c5 Mon Sep 17 00:00:00 2001 From: Martino Facchin Date: Wed, 1 Mar 2017 11:45:43 +0100 Subject: [PATCH 06/12] Generalize core changes checking routine Wipe the workspace if any of the core files has changed; this solves a long-standing problem for core developers about changes in platform.txt requiring a manual wipe to be correctly applied --- src/arduino.cc/builder/builder_utils/utils.go | 18 +++++++++++------- src/arduino.cc/builder/phases/core_builder.go | 10 ++-------- ...eout_build_path_if_build_options_changed.go | 14 +++++++++++++- 3 files changed, 26 insertions(+), 16 deletions(-) diff --git a/src/arduino.cc/builder/builder_utils/utils.go b/src/arduino.cc/builder/builder_utils/utils.go index 196c1b63..77f85203 100644 --- a/src/arduino.cc/builder/builder_utils/utils.go +++ b/src/arduino.cc/builder/builder_utils/utils.go @@ -287,22 +287,26 @@ func nonEmptyString(s string) bool { return s != constants.EMPTY_STRING } -func CheckIfRecompileIsAvoidable(corePath string, archiveFile string) bool { - archiveFileStat, err := os.Stat(archiveFile) +func CoreOrReferencedCoreHasChanged(corePath, targetCorePath, targetFile string) bool { + + targetFileStat, err := os.Stat(targetFile) if err == nil { files, err := findAllFilesInFolder(corePath, true) if err != nil { - return false + return true } for _, file := range files { fileStat, err := os.Stat(file) - if err != nil || fileStat.ModTime().After(archiveFileStat.ModTime()) { - return false + if err != nil || fileStat.ModTime().After(targetFileStat.ModTime()) { + return true } } - return true + if targetCorePath != constants.EMPTY_STRING && !strings.EqualFold(corePath, targetCorePath) { + return CoreOrReferencedCoreHasChanged(targetCorePath, constants.EMPTY_STRING, targetFile) + } + return false } - return false + return true } func ArchiveCompiledFiles(buildPath string, archiveFile string, objectFiles []string, buildProperties properties.Map, verbose bool, logger i18n.Logger) (string, error) { diff --git a/src/arduino.cc/builder/phases/core_builder.go b/src/arduino.cc/builder/phases/core_builder.go index 5ada42b8..b61ceeaa 100644 --- a/src/arduino.cc/builder/phases/core_builder.go +++ b/src/arduino.cc/builder/phases/core_builder.go @@ -36,7 +36,6 @@ import ( "arduino.cc/builder/types" "arduino.cc/builder/utils" "arduino.cc/properties" - "strings" ) type CoreBuilder struct{} @@ -91,14 +90,9 @@ func compileCore(buildPath string, buildProperties properties.Map, verbose bool, realCoreFolder := utils.GetParentFolder(coreFolder, 2) targetArchivedCore := builder_utils.GetCoreArchivePath(buildProperties[constants.BUILD_PROPERTIES_FQBN]) - noNeedToRecompile := builder_utils.CheckIfRecompileIsAvoidable(realCoreFolder, targetArchivedCore) + canUseArchivedCore := !builder_utils.CoreOrReferencedCoreHasChanged(realCoreFolder, targetCoreFolder, targetArchivedCore) - if !strings.EqualFold(realCoreFolder, targetCoreFolder) { - // targetCoreFolder is not a parent of realCoreFolder, so check it for modifications - noNeedToRecompile = noNeedToRecompile && builder_utils.CheckIfRecompileIsAvoidable(targetCoreFolder, targetArchivedCore) - } - - if noNeedToRecompile { + if canUseArchivedCore { // use archived core if verbose { logger.Println(constants.LOG_LEVEL_INFO, "Using precompiled core") diff --git a/src/arduino.cc/builder/wipeout_build_path_if_build_options_changed.go b/src/arduino.cc/builder/wipeout_build_path_if_build_options_changed.go index 5e4894c7..e08e7807 100644 --- a/src/arduino.cc/builder/wipeout_build_path_if_build_options_changed.go +++ b/src/arduino.cc/builder/wipeout_build_path_if_build_options_changed.go @@ -34,10 +34,12 @@ import ( "os" "path/filepath" + "arduino.cc/builder/builder_utils" "arduino.cc/builder/constants" "arduino.cc/builder/gohasissues" "arduino.cc/builder/i18n" "arduino.cc/builder/types" + "arduino.cc/builder/utils" "arduino.cc/properties" ) @@ -62,7 +64,17 @@ func (s *WipeoutBuildPathIfBuildOptionsChanged) Run(ctx *types.Context) error { delete(prevOpts, "sketchLocation") } - if opts.Equals(prevOpts) { + // check if any of the files contained in the core folders has changed + // since the json was generated - like platform.txt or similar + // if so, trigger a "safety" wipe + buildProperties := ctx.BuildProperties + targetCoreFolder := buildProperties[constants.BUILD_PROPERTIES_RUNTIME_PLATFORM_PATH] + coreFolder := buildProperties[constants.BUILD_PROPERTIES_BUILD_CORE_PATH] + realCoreFolder := utils.GetParentFolder(coreFolder, 2) + jsonPath := filepath.Join(ctx.BuildPath, constants.BUILD_OPTIONS_FILE) + coreHasChanged := builder_utils.CoreOrReferencedCoreHasChanged(realCoreFolder, targetCoreFolder, jsonPath) + + if opts.Equals(prevOpts) && !coreHasChanged { return nil } From 61a81f97341bd847c8051894e72cbcbb82d6fb2c Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Thu, 2 Mar 2017 16:44:09 +0100 Subject: [PATCH 07/12] Added test for core caching Signed-off-by: Cristian Maglie --- src/arduino.cc/builder/test/builder_test.go | 60 +++++++++++++++++++-- 1 file changed, 56 insertions(+), 4 deletions(-) diff --git a/src/arduino.cc/builder/test/builder_test.go b/src/arduino.cc/builder/test/builder_test.go index d0a34249..9144c384 100644 --- a/src/arduino.cc/builder/test/builder_test.go +++ b/src/arduino.cc/builder/test/builder_test.go @@ -30,14 +30,17 @@ package test import ( - "arduino.cc/builder" - "arduino.cc/builder/constants" - "arduino.cc/builder/types" - "github.com/stretchr/testify/require" "os" "os/exec" "path/filepath" "testing" + "time" + + "arduino.cc/builder" + "arduino.cc/builder/builder_utils" + "arduino.cc/builder/constants" + "arduino.cc/builder/types" + "github.com/stretchr/testify/require" ) func TestBuilderEmptySketch(t *testing.T) { @@ -410,3 +413,52 @@ func TestBuilderWithBuildPathInSketchDir(t *testing.T) { err = command.Run(ctx) NoError(t, err) } + +func TestBuilderCacheCoreAFile(t *testing.T) { + DownloadCoresAndToolsAndLibraries(t) + + ctx := &types.Context{ + HardwareFolders: []string{filepath.Join("..", "hardware"), "hardware", "downloaded_hardware"}, + ToolsFolders: []string{"downloaded_tools"}, + BuiltInLibrariesFolders: []string{"downloaded_libraries"}, + OtherLibrariesFolders: []string{"libraries"}, + SketchLocation: filepath.Join("sketch1", "sketch.ino"), + FQBN: "arduino:avr:uno", + ArduinoAPIVersion: "10801", + } + SetupBuildPath(t, ctx) + defer os.RemoveAll(ctx.BuildPath) + + // Cleanup cached core + coreFile := builder_utils.GetCoreArchivePath(ctx.FQBN) + os.Remove(coreFile) + + // Run build + bldr := builder.Builder{} + err := bldr.Run(ctx) + NoError(t, err) + coreStatBefore, err := os.Stat(coreFile) + require.NoError(t, err) + + // Run build again, to verify that the builder skips rebuilding core.a + err = bldr.Run(ctx) + NoError(t, err) + + coreStatAfterRebuild, err := os.Stat(coreFile) + require.NoError(t, err) + require.Equal(t, coreStatBefore.ModTime(), coreStatAfterRebuild.ModTime()) + + // Touch a file of the core and check if the builder invalidate the cache + time.Sleep(time.Second) + now := time.Now().Local() + err = os.Chtimes(filepath.Join("downloaded_hardware", "arduino", "avr", "cores", "arduino", "Arduino.h"), now, now) + require.NoError(t, err) + + // Run build again, to verify that the builder rebuilds core.a + err = bldr.Run(ctx) + NoError(t, err) + + coreStatAfterTouch, err := os.Stat(coreFile) + require.NoError(t, err) + require.NotEqual(t, coreStatBefore.ModTime(), coreStatAfterTouch.ModTime()) +} From ca9f34d160e82e072f897009d41c59bcbfec952b Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Fri, 3 Mar 2017 13:13:17 +0100 Subject: [PATCH 08/12] Cached core.a filename now depends on core path too Signed-off-by: Cristian Maglie --- src/arduino.cc/builder/builder_utils/utils.go | 10 +++- src/arduino.cc/builder/phases/core_builder.go | 2 +- src/arduino.cc/builder/test/builder_test.go | 59 +++++++++++++++---- 3 files changed, 58 insertions(+), 13 deletions(-) diff --git a/src/arduino.cc/builder/builder_utils/utils.go b/src/arduino.cc/builder/builder_utils/utils.go index 77f85203..06782e11 100644 --- a/src/arduino.cc/builder/builder_utils/utils.go +++ b/src/arduino.cc/builder/builder_utils/utils.go @@ -462,8 +462,14 @@ func CopyFile(src, dst string) (err error) { return } -func GetCoreArchivePath(fqbn string) string { +// GetCachedCoreArchiveFileName returns the filename to be used to store +// the global cached core.a. +func GetCachedCoreArchiveFileName(fqbn, coreFolder string) string { fqbnToUnderscore := strings.Replace(fqbn, ":", "_", -1) fqbnToUnderscore = strings.Replace(fqbnToUnderscore, "=", "_", -1) - return os.TempDir() + "/core_" + fqbnToUnderscore + ".a" + if absCoreFolder, err := filepath.Abs(coreFolder); err == nil { + coreFolder = absCoreFolder + } // silently continue if absolute path can't be detected + hash := utils.MD5Sum([]byte(coreFolder)) + return os.TempDir() + "/core_" + fqbnToUnderscore + "_" + hash + ".a" } diff --git a/src/arduino.cc/builder/phases/core_builder.go b/src/arduino.cc/builder/phases/core_builder.go index b61ceeaa..91a77231 100644 --- a/src/arduino.cc/builder/phases/core_builder.go +++ b/src/arduino.cc/builder/phases/core_builder.go @@ -89,7 +89,7 @@ func compileCore(buildPath string, buildProperties properties.Map, verbose bool, // Recreate the archive if ANY of the core files (including platform.txt) has changed realCoreFolder := utils.GetParentFolder(coreFolder, 2) - targetArchivedCore := builder_utils.GetCoreArchivePath(buildProperties[constants.BUILD_PROPERTIES_FQBN]) + targetArchivedCore := builder_utils.GetCachedCoreArchiveFileName(buildProperties[constants.BUILD_PROPERTIES_FQBN], realCoreFolder) canUseArchivedCore := !builder_utils.CoreOrReferencedCoreHasChanged(realCoreFolder, targetCoreFolder, targetArchivedCore) if canUseArchivedCore { diff --git a/src/arduino.cc/builder/test/builder_test.go b/src/arduino.cc/builder/test/builder_test.go index 9144c384..4e543532 100644 --- a/src/arduino.cc/builder/test/builder_test.go +++ b/src/arduino.cc/builder/test/builder_test.go @@ -62,6 +62,12 @@ func TestBuilderEmptySketch(t *testing.T) { ctx.DebugLevel = 10 + // Cleanup cached core + coreFolder := filepath.Join("downloaded_hardware", "arduino", "avr") + coreFile := builder_utils.GetCachedCoreArchiveFileName(ctx.FQBN, coreFolder) + os.Remove(coreFile) + + // Run builder command := builder.Builder{} err := command.Run(ctx) NoError(t, err) @@ -95,6 +101,12 @@ func TestBuilderBridge(t *testing.T) { buildPath := SetupBuildPath(t, ctx) defer os.RemoveAll(buildPath) + // Cleanup cached core + coreFolder := filepath.Join("downloaded_hardware", "arduino", "avr") + coreFile := builder_utils.GetCachedCoreArchiveFileName(ctx.FQBN, coreFolder) + os.Remove(coreFile) + + // Run builder command := builder.Builder{} err := command.Run(ctx) NoError(t, err) @@ -129,14 +141,16 @@ func TestBuilderSketchWithConfig(t *testing.T) { buildPath := SetupBuildPath(t, ctx) defer os.RemoveAll(buildPath) + // Cleanup cached core + coreFolder := filepath.Join("downloaded_hardware", "arduino", "avr") + coreFile := builder_utils.GetCachedCoreArchiveFileName(ctx.FQBN, coreFolder) + os.Remove(coreFile) + + // Run builder command := builder.Builder{} err := command.Run(ctx) NoError(t, err) - // Cleanup cached core - coreFile := builder_utils.GetCoreArchivePath(ctx.FQBN) - os.Remove(coreFile) - _, err = os.Stat(filepath.Join(buildPath, constants.FOLDER_CORE, "HardwareSerial.cpp.o")) NoError(t, err) _, err = os.Stat(filepath.Join(buildPath, constants.FOLDER_PREPROC, constants.FILE_CTAGS_TARGET_FOR_GCC_MINUS_E)) @@ -167,18 +181,21 @@ func TestBuilderBridgeTwice(t *testing.T) { buildPath := SetupBuildPath(t, ctx) defer os.RemoveAll(buildPath) + // Cleanup cached core + coreFolder := filepath.Join("downloaded_hardware", "arduino", "avr") + coreFile := builder_utils.GetCachedCoreArchiveFileName(ctx.FQBN, coreFolder) + os.Remove(coreFile) + + // Run builder command := builder.Builder{} err := command.Run(ctx) NoError(t, err) + // Run builder again command = builder.Builder{} err = command.Run(ctx) NoError(t, err) - // Cleanup cached core - coreFile := builder_utils.GetCoreArchivePath(ctx.FQBN) - os.Remove(coreFile) - _, err = os.Stat(filepath.Join(buildPath, constants.FOLDER_CORE, "HardwareSerial.cpp.o")) NoError(t, err) _, err = os.Stat(filepath.Join(buildPath, constants.FOLDER_PREPROC, constants.FILE_CTAGS_TARGET_FOR_GCC_MINUS_E)) @@ -211,6 +228,12 @@ func TestBuilderBridgeSAM(t *testing.T) { ctx.WarningsLevel = "all" + // Cleanup cached core + coreFolder := filepath.Join("downloaded_hardware", "arduino", "sam") + coreFile := builder_utils.GetCachedCoreArchiveFileName(ctx.FQBN, coreFolder) + os.Remove(coreFile) + + // Run builder command := builder.Builder{} err := command.Run(ctx) NoError(t, err) @@ -254,6 +277,12 @@ func TestBuilderBridgeRedBearLab(t *testing.T) { buildPath := SetupBuildPath(t, ctx) defer os.RemoveAll(buildPath) + // Cleanup cached core + coreFolder := filepath.Join("downloaded_hardware", "arduino", "avr") + coreFile := builder_utils.GetCachedCoreArchiveFileName(ctx.FQBN, coreFolder) + os.Remove(coreFile) + + // Run builder command := builder.Builder{} err := command.Run(ctx) NoError(t, err) @@ -404,10 +433,19 @@ func TestBuilderWithBuildPathInSketchDir(t *testing.T) { NoError(t, err) defer os.RemoveAll(ctx.BuildPath) + // Cleanup cached core + coreFolder := filepath.Join("downloaded_hardware", "arduino", "avr") + coreFile := builder_utils.GetCachedCoreArchiveFileName(ctx.FQBN, coreFolder) + os.Remove(coreFile) + + // Run build command := builder.Builder{} err = command.Run(ctx) NoError(t, err) + // Cleanup cached core + os.Remove(coreFile) + // Run build twice, to verify the build still works when the // build directory is present at the start err = command.Run(ctx) @@ -430,7 +468,8 @@ func TestBuilderCacheCoreAFile(t *testing.T) { defer os.RemoveAll(ctx.BuildPath) // Cleanup cached core - coreFile := builder_utils.GetCoreArchivePath(ctx.FQBN) + coreFolder := filepath.Join("downloaded_hardware", "arduino", "avr") + coreFile := builder_utils.GetCachedCoreArchiveFileName(ctx.FQBN, coreFolder) os.Remove(coreFile) // Run build @@ -451,7 +490,7 @@ func TestBuilderCacheCoreAFile(t *testing.T) { // Touch a file of the core and check if the builder invalidate the cache time.Sleep(time.Second) now := time.Now().Local() - err = os.Chtimes(filepath.Join("downloaded_hardware", "arduino", "avr", "cores", "arduino", "Arduino.h"), now, now) + err = os.Chtimes(filepath.Join(coreFolder, "cores", "arduino", "Arduino.h"), now, now) require.NoError(t, err) // Run build again, to verify that the builder rebuilds core.a From 0448b73a70bd9fb0cd5dfc6e4a6177630d4ef5a9 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Fri, 3 Mar 2017 13:33:34 +0100 Subject: [PATCH 09/12] Factored setup of builder tests Signed-off-by: Cristian Maglie --- src/arduino.cc/builder/test/builder_test.go | 150 ++++---------------- 1 file changed, 30 insertions(+), 120 deletions(-) diff --git a/src/arduino.cc/builder/test/builder_test.go b/src/arduino.cc/builder/test/builder_test.go index 4e543532..24368368 100644 --- a/src/arduino.cc/builder/test/builder_test.go +++ b/src/arduino.cc/builder/test/builder_test.go @@ -43,25 +43,27 @@ import ( "github.com/stretchr/testify/require" ) -func TestBuilderEmptySketch(t *testing.T) { - DownloadCoresAndToolsAndLibraries(t) - - ctx := &types.Context{ +func prepareBuilderTestContext(sketchPath, fqbn string) *types.Context { + return &types.Context{ + SketchLocation: sketchPath, + FQBN: fqbn, HardwareFolders: []string{filepath.Join("..", "hardware"), "hardware", "downloaded_hardware"}, ToolsFolders: []string{"downloaded_tools"}, BuiltInLibrariesFolders: []string{"downloaded_libraries"}, OtherLibrariesFolders: []string{"libraries"}, - SketchLocation: filepath.Join("sketch1", "sketch.ino"), - FQBN: "arduino:avr:uno", ArduinoAPIVersion: "10600", - Verbose: true, + Verbose: false, } +} +func TestBuilderEmptySketch(t *testing.T) { + DownloadCoresAndToolsAndLibraries(t) + + ctx := prepareBuilderTestContext(filepath.Join("sketch1", "sketch.ino"), "arduino:avr:uno") + ctx.DebugLevel = 10 buildPath := SetupBuildPath(t, ctx) defer os.RemoveAll(buildPath) - ctx.DebugLevel = 10 - // Cleanup cached core coreFolder := filepath.Join("downloaded_hardware", "arduino", "avr") coreFile := builder_utils.GetCachedCoreArchiveFileName(ctx.FQBN, coreFolder) @@ -87,16 +89,7 @@ func TestBuilderEmptySketch(t *testing.T) { func TestBuilderBridge(t *testing.T) { DownloadCoresAndToolsAndLibraries(t) - ctx := &types.Context{ - HardwareFolders: []string{filepath.Join("..", "hardware"), "hardware", "downloaded_hardware"}, - ToolsFolders: []string{"downloaded_tools"}, - BuiltInLibrariesFolders: []string{"downloaded_libraries"}, - OtherLibrariesFolders: []string{"libraries"}, - SketchLocation: filepath.Join("downloaded_libraries", "Bridge", "examples", "Bridge", "Bridge.ino"), - FQBN: "arduino:avr:leonardo", - ArduinoAPIVersion: "10600", - Verbose: true, - } + ctx := prepareBuilderTestContext(filepath.Join("downloaded_libraries", "Bridge", "examples", "Bridge", "Bridge.ino"), "arduino:avr:leonardo") buildPath := SetupBuildPath(t, ctx) defer os.RemoveAll(buildPath) @@ -128,15 +121,7 @@ func TestBuilderBridge(t *testing.T) { func TestBuilderSketchWithConfig(t *testing.T) { DownloadCoresAndToolsAndLibraries(t) - ctx := &types.Context{ - HardwareFolders: []string{filepath.Join("..", "hardware"), "hardware", "downloaded_hardware"}, - ToolsFolders: []string{"downloaded_tools"}, - BuiltInLibrariesFolders: []string{"downloaded_libraries"}, - OtherLibrariesFolders: []string{"libraries"}, - SketchLocation: filepath.Join("sketch_with_config", "sketch_with_config.ino"), - FQBN: "arduino:avr:leonardo", - ArduinoAPIVersion: "10600", - } + ctx := prepareBuilderTestContext(filepath.Join("sketch_with_config", "sketch_with_config.ino"), "arduino:avr:leonardo") buildPath := SetupBuildPath(t, ctx) defer os.RemoveAll(buildPath) @@ -168,15 +153,7 @@ func TestBuilderSketchWithConfig(t *testing.T) { func TestBuilderBridgeTwice(t *testing.T) { DownloadCoresAndToolsAndLibraries(t) - ctx := &types.Context{ - HardwareFolders: []string{filepath.Join("..", "hardware"), "hardware", "downloaded_hardware"}, - ToolsFolders: []string{"downloaded_tools"}, - BuiltInLibrariesFolders: []string{"downloaded_libraries"}, - OtherLibrariesFolders: []string{"libraries"}, - SketchLocation: filepath.Join("downloaded_libraries", "Bridge", "examples", "Bridge", "Bridge.ino"), - FQBN: "arduino:avr:leonardo", - ArduinoAPIVersion: "10600", - } + ctx := prepareBuilderTestContext(filepath.Join("downloaded_libraries", "Bridge", "examples", "Bridge", "Bridge.ino"), "arduino:avr:leonardo") buildPath := SetupBuildPath(t, ctx) defer os.RemoveAll(buildPath) @@ -213,15 +190,7 @@ func TestBuilderBridgeTwice(t *testing.T) { func TestBuilderBridgeSAM(t *testing.T) { DownloadCoresAndToolsAndLibraries(t) - ctx := &types.Context{ - HardwareFolders: []string{filepath.Join("..", "hardware"), "hardware", "downloaded_hardware"}, - ToolsFolders: []string{"downloaded_tools"}, - BuiltInLibrariesFolders: []string{"downloaded_libraries"}, - OtherLibrariesFolders: []string{"libraries"}, - SketchLocation: filepath.Join("downloaded_libraries", "Bridge", "examples", "Bridge", "Bridge.ino"), - FQBN: "arduino:sam:arduino_due_x_dbg", - ArduinoAPIVersion: "10600", - } + ctx := prepareBuilderTestContext(filepath.Join("downloaded_libraries", "Bridge", "examples", "Bridge", "Bridge.ino"), "arduino:sam:arduino_due_x_dbg") buildPath := SetupBuildPath(t, ctx) defer os.RemoveAll(buildPath) @@ -264,16 +233,9 @@ func TestBuilderBridgeSAM(t *testing.T) { func TestBuilderBridgeRedBearLab(t *testing.T) { DownloadCoresAndToolsAndLibraries(t) - ctx := &types.Context{ - HardwareFolders: []string{filepath.Join("..", "hardware"), "hardware", "downloaded_hardware", "downloaded_board_manager_stuff"}, - ToolsFolders: []string{"downloaded_tools", "downloaded_board_manager_stuff"}, - BuiltInLibrariesFolders: []string{"downloaded_libraries"}, - OtherLibrariesFolders: []string{"libraries"}, - SketchLocation: filepath.Join("downloaded_libraries", "Bridge", "examples", "Bridge", "Bridge.ino"), - FQBN: "RedBearLab:avr:blend", - ArduinoAPIVersion: "10600", - } - + ctx := prepareBuilderTestContext(filepath.Join("downloaded_libraries", "Bridge", "examples", "Bridge", "Bridge.ino"), "RedBearLab:avr:blend") + ctx.HardwareFolders = append(ctx.HardwareFolders, "downloaded_board_manager_stuff") + ctx.ToolsFolders = append(ctx.ToolsFolders, "downloaded_board_manager_stuff") buildPath := SetupBuildPath(t, ctx) defer os.RemoveAll(buildPath) @@ -304,15 +266,9 @@ func TestBuilderBridgeRedBearLab(t *testing.T) { func TestBuilderSketchNoFunctions(t *testing.T) { DownloadCoresAndToolsAndLibraries(t) - ctx := &types.Context{ - HardwareFolders: []string{filepath.Join("..", "hardware"), "hardware", "downloaded_hardware", "downloaded_board_manager_stuff"}, - ToolsFolders: []string{"downloaded_tools", "downloaded_board_manager_stuff"}, - BuiltInLibrariesFolders: []string{"downloaded_libraries"}, - OtherLibrariesFolders: []string{"libraries"}, - SketchLocation: filepath.Join("sketch_no_functions", "main.ino"), - FQBN: "RedBearLab:avr:blend", - ArduinoAPIVersion: "10600", - } + ctx := prepareBuilderTestContext(filepath.Join("sketch_no_functions", "main.ino"), "RedBearLab:avr:blend") + ctx.HardwareFolders = append(ctx.HardwareFolders, "downloaded_board_manager_stuff") + ctx.ToolsFolders = append(ctx.ToolsFolders, "downloaded_board_manager_stuff") buildPath := SetupBuildPath(t, ctx) defer os.RemoveAll(buildPath) @@ -325,15 +281,9 @@ func TestBuilderSketchNoFunctions(t *testing.T) { func TestBuilderSketchWithBackup(t *testing.T) { DownloadCoresAndToolsAndLibraries(t) - ctx := &types.Context{ - HardwareFolders: []string{filepath.Join("..", "hardware"), "hardware", "downloaded_hardware", "downloaded_board_manager_stuff"}, - ToolsFolders: []string{"downloaded_tools", "downloaded_board_manager_stuff"}, - BuiltInLibrariesFolders: []string{"downloaded_libraries"}, - OtherLibrariesFolders: []string{"libraries"}, - SketchLocation: filepath.Join("sketch_with_backup_files", "sketch.ino"), - FQBN: "arduino:avr:uno", - ArduinoAPIVersion: "10600", - } + ctx := prepareBuilderTestContext(filepath.Join("sketch_with_backup_files", "sketch.ino"), "arduino:avr:uno") + ctx.HardwareFolders = append(ctx.HardwareFolders, "downloaded_board_manager_stuff") + ctx.ToolsFolders = append(ctx.ToolsFolders, "downloaded_board_manager_stuff") buildPath := SetupBuildPath(t, ctx) defer os.RemoveAll(buildPath) @@ -346,15 +296,7 @@ func TestBuilderSketchWithBackup(t *testing.T) { func TestBuilderSketchWithOldLib(t *testing.T) { DownloadCoresAndToolsAndLibraries(t) - ctx := &types.Context{ - HardwareFolders: []string{filepath.Join("..", "hardware"), "hardware", "downloaded_hardware"}, - ToolsFolders: []string{"downloaded_tools"}, - BuiltInLibrariesFolders: []string{"downloaded_libraries"}, - OtherLibrariesFolders: []string{"libraries"}, - SketchLocation: filepath.Join("sketch_with_old_lib", "sketch.ino"), - FQBN: "arduino:avr:uno", - ArduinoAPIVersion: "10600", - } + ctx := prepareBuilderTestContext(filepath.Join("sketch_with_old_lib", "sketch.ino"), "arduino:avr:uno") buildPath := SetupBuildPath(t, ctx) defer os.RemoveAll(buildPath) @@ -367,15 +309,7 @@ func TestBuilderSketchWithOldLib(t *testing.T) { func TestBuilderSketchWithSubfolders(t *testing.T) { DownloadCoresAndToolsAndLibraries(t) - ctx := &types.Context{ - HardwareFolders: []string{filepath.Join("..", "hardware"), "hardware", "downloaded_hardware"}, - ToolsFolders: []string{"downloaded_tools"}, - BuiltInLibrariesFolders: []string{"downloaded_libraries"}, - OtherLibrariesFolders: []string{"libraries"}, - SketchLocation: filepath.Join("sketch_with_subfolders", "sketch_with_subfolders.ino"), - FQBN: "arduino:avr:uno", - ArduinoAPIVersion: "10600", - } + ctx := prepareBuilderTestContext(filepath.Join("sketch_with_subfolders", "sketch_with_subfolders.ino"), "arduino:avr:uno") buildPath := SetupBuildPath(t, ctx) defer os.RemoveAll(buildPath) @@ -388,15 +322,7 @@ func TestBuilderSketchWithSubfolders(t *testing.T) { func TestBuilderSketchBuildPathContainsUnusedPreviouslyCompiledLibrary(t *testing.T) { DownloadCoresAndToolsAndLibraries(t) - ctx := &types.Context{ - HardwareFolders: []string{filepath.Join("..", "hardware"), "hardware", "downloaded_hardware"}, - ToolsFolders: []string{"downloaded_tools"}, - BuiltInLibrariesFolders: []string{"downloaded_libraries"}, - OtherLibrariesFolders: []string{"libraries"}, - SketchLocation: filepath.Join("downloaded_libraries", "Bridge", "examples", "Bridge", "Bridge.ino"), - FQBN: "arduino:avr:leonardo", - ArduinoAPIVersion: "10600", - } + ctx := prepareBuilderTestContext(filepath.Join("downloaded_libraries", "Bridge", "examples", "Bridge", "Bridge.ino"), "arduino:avr:leonardo") buildPath := SetupBuildPath(t, ctx) defer os.RemoveAll(buildPath) @@ -417,16 +343,7 @@ func TestBuilderSketchBuildPathContainsUnusedPreviouslyCompiledLibrary(t *testin func TestBuilderWithBuildPathInSketchDir(t *testing.T) { DownloadCoresAndToolsAndLibraries(t) - ctx := &types.Context{ - HardwareFolders: []string{filepath.Join("..", "hardware"), "hardware", "downloaded_hardware"}, - ToolsFolders: []string{"downloaded_tools"}, - BuiltInLibrariesFolders: []string{"downloaded_libraries"}, - OtherLibrariesFolders: []string{"libraries"}, - SketchLocation: filepath.Join("sketch1", "sketch.ino"), - FQBN: "arduino:avr:uno", - ArduinoAPIVersion: "10600", - Verbose: true, - } + ctx := prepareBuilderTestContext(filepath.Join("sketch1", "sketch.ino"), "arduino:avr:uno") var err error ctx.BuildPath, err = filepath.Abs(filepath.Join("sketch1", "build")) @@ -455,15 +372,8 @@ func TestBuilderWithBuildPathInSketchDir(t *testing.T) { func TestBuilderCacheCoreAFile(t *testing.T) { DownloadCoresAndToolsAndLibraries(t) - ctx := &types.Context{ - HardwareFolders: []string{filepath.Join("..", "hardware"), "hardware", "downloaded_hardware"}, - ToolsFolders: []string{"downloaded_tools"}, - BuiltInLibrariesFolders: []string{"downloaded_libraries"}, - OtherLibrariesFolders: []string{"libraries"}, - SketchLocation: filepath.Join("sketch1", "sketch.ino"), - FQBN: "arduino:avr:uno", - ArduinoAPIVersion: "10801", - } + ctx := prepareBuilderTestContext(filepath.Join("sketch1", "sketch.ino"), "arduino:avr:uno") + SetupBuildPath(t, ctx) defer os.RemoveAll(ctx.BuildPath) From 4764159c30149d15beb92fc6149eb50d2b01441f Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Fri, 3 Mar 2017 14:49:19 +0100 Subject: [PATCH 10/12] The build caching of core.a is now optional thorugh a command line option Signed-off-by: Cristian Maglie --- src/arduino.cc/arduino-builder/main.go | 22 +++++++++++ .../add_additional_entries_to_context.go | 12 +++++- src/arduino.cc/builder/builder_utils/utils.go | 2 +- src/arduino.cc/builder/phases/core_builder.go | 39 +++++++++++++------ src/arduino.cc/builder/types/context.go | 2 + 5 files changed, 64 insertions(+), 13 deletions(-) diff --git a/src/arduino.cc/arduino-builder/main.go b/src/arduino.cc/arduino-builder/main.go index 83ef32cc..ec5a3d51 100644 --- a/src/arduino.cc/arduino-builder/main.go +++ b/src/arduino.cc/arduino-builder/main.go @@ -64,6 +64,7 @@ const FLAG_FQBN = "fqbn" const FLAG_IDE_VERSION = "ide-version" const FLAG_CORE_API_VERSION = "core-api-version" const FLAG_BUILD_PATH = "build-path" +const FLAG_BUILD_CACHE = "build-cache" const FLAG_VERBOSE = "verbose" const FLAG_QUIET = "quiet" const FLAG_DEBUG_LEVEL = "debug-level" @@ -126,6 +127,7 @@ var fqbnFlag *string var coreAPIVersionFlag *string var ideVersionFlag *string var buildPathFlag *string +var buildCachePathFlag *string var verboseFlag *bool var quietFlag *bool var debugLevelFlag *int @@ -148,6 +150,7 @@ func init() { coreAPIVersionFlag = flag.String(FLAG_CORE_API_VERSION, "10600", "version of core APIs (used to populate ARDUINO #define)") ideVersionFlag = flag.String(FLAG_IDE_VERSION, "10600", "[deprecated] use '"+FLAG_CORE_API_VERSION+"' instead") buildPathFlag = flag.String(FLAG_BUILD_PATH, "", "build path") + buildCachePathFlag = flag.String(FLAG_BUILD_CACHE, "", "builds of 'core.a' are saved into this folder to be cached and reused") verboseFlag = flag.Bool(FLAG_VERBOSE, false, "if 'true' prints lots of stuff") quietFlag = flag.Bool(FLAG_QUIET, false, "if 'true' doesn't print any warnings or progress or whatever") debugLevelFlag = flag.Int(FLAG_DEBUG_LEVEL, builder.DEFAULT_DEBUG_LEVEL, "Turns on debugging messages. The higher, the chattier") @@ -256,6 +259,25 @@ func main() { } ctx.BuildPath = buildPath + // FLAG_BUILD_CACHE + buildCachePath, err := gohasissues.Unquote(*buildCachePathFlag) + if err != nil { + printCompleteError(err) + } + if buildCachePath != "" { + _, err := os.Stat(buildCachePath) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + + err = utils.EnsureFolderExists(buildCachePath) + if err != nil { + printCompleteError(err) + } + } + ctx.BuildCachePath = buildCachePath + // FLAG_VID_PID if *vidPidFlag != "" { ctx.USBVidPid = *vidPidFlag diff --git a/src/arduino.cc/builder/add_additional_entries_to_context.go b/src/arduino.cc/builder/add_additional_entries_to_context.go index 91bc2a0e..795aea01 100644 --- a/src/arduino.cc/builder/add_additional_entries_to_context.go +++ b/src/arduino.cc/builder/add_additional_entries_to_context.go @@ -30,10 +30,11 @@ package builder import ( + "path/filepath" + "arduino.cc/builder/constants" "arduino.cc/builder/i18n" "arduino.cc/builder/types" - "path/filepath" ) type AddAdditionalEntriesToContext struct{} @@ -64,6 +65,15 @@ func (s *AddAdditionalEntriesToContext) Run(ctx *types.Context) error { ctx.CoreBuildPath = coreBuildPath } + if ctx.BuildCachePath != "" { + coreBuildCachePath, err := filepath.Abs(filepath.Join(ctx.BuildCachePath, constants.FOLDER_CORE)) + if err != nil { + return i18n.WrapError(err) + } + + ctx.CoreBuildCachePath = coreBuildCachePath + } + if ctx.WarningsLevel == "" { ctx.WarningsLevel = DEFAULT_WARNINGS_LEVEL } diff --git a/src/arduino.cc/builder/builder_utils/utils.go b/src/arduino.cc/builder/builder_utils/utils.go index 06782e11..97b6e355 100644 --- a/src/arduino.cc/builder/builder_utils/utils.go +++ b/src/arduino.cc/builder/builder_utils/utils.go @@ -471,5 +471,5 @@ func GetCachedCoreArchiveFileName(fqbn, coreFolder string) string { coreFolder = absCoreFolder } // silently continue if absolute path can't be detected hash := utils.MD5Sum([]byte(coreFolder)) - return os.TempDir() + "/core_" + fqbnToUnderscore + "_" + hash + ".a" + return "core_" + fqbnToUnderscore + "_" + hash + ".a" } diff --git a/src/arduino.cc/builder/phases/core_builder.go b/src/arduino.cc/builder/phases/core_builder.go index 91a77231..c906d042 100644 --- a/src/arduino.cc/builder/phases/core_builder.go +++ b/src/arduino.cc/builder/phases/core_builder.go @@ -30,6 +30,8 @@ package phases import ( + "path/filepath" + "arduino.cc/builder/builder_utils" "arduino.cc/builder/constants" "arduino.cc/builder/i18n" @@ -42,6 +44,7 @@ type CoreBuilder struct{} func (s *CoreBuilder) Run(ctx *types.Context) error { coreBuildPath := ctx.CoreBuildPath + coreBuildCachePath := ctx.CoreBuildCachePath buildProperties := ctx.BuildProperties verbose := ctx.Verbose warningsLevel := ctx.WarningsLevel @@ -52,7 +55,14 @@ func (s *CoreBuilder) Run(ctx *types.Context) error { return i18n.WrapError(err) } - archiveFile, objectFiles, err := compileCore(coreBuildPath, buildProperties, verbose, warningsLevel, logger) + if coreBuildCachePath != "" { + err := utils.EnsureFolderExists(coreBuildCachePath) + if err != nil { + return i18n.WrapError(err) + } + } + + archiveFile, objectFiles, err := compileCore(coreBuildPath, coreBuildCachePath, buildProperties, verbose, warningsLevel, logger) if err != nil { return i18n.WrapError(err) } @@ -63,7 +73,7 @@ func (s *CoreBuilder) Run(ctx *types.Context) error { return nil } -func compileCore(buildPath string, buildProperties properties.Map, verbose bool, warningsLevel string, logger i18n.Logger) (string, []string, error) { +func compileCore(buildPath string, buildCachePath string, buildProperties properties.Map, verbose bool, warningsLevel string, logger i18n.Logger) (string, []string, error) { coreFolder := buildProperties[constants.BUILD_PROPERTIES_BUILD_CORE_PATH] variantFolder := buildProperties[constants.BUILD_PROPERTIES_BUILD_VARIANT_PATH] @@ -89,15 +99,19 @@ func compileCore(buildPath string, buildProperties properties.Map, verbose bool, // Recreate the archive if ANY of the core files (including platform.txt) has changed realCoreFolder := utils.GetParentFolder(coreFolder, 2) - targetArchivedCore := builder_utils.GetCachedCoreArchiveFileName(buildProperties[constants.BUILD_PROPERTIES_FQBN], realCoreFolder) - canUseArchivedCore := !builder_utils.CoreOrReferencedCoreHasChanged(realCoreFolder, targetCoreFolder, targetArchivedCore) - - if canUseArchivedCore { - // use archived core - if verbose { - logger.Println(constants.LOG_LEVEL_INFO, "Using precompiled core") + var targetArchivedCore string + if buildCachePath != "" { + archivedCoreName := builder_utils.GetCachedCoreArchiveFileName(buildProperties[constants.BUILD_PROPERTIES_FQBN], realCoreFolder) + targetArchivedCore = filepath.Join(buildCachePath, archivedCoreName) + canUseArchivedCore := !builder_utils.CoreOrReferencedCoreHasChanged(realCoreFolder, targetCoreFolder, targetArchivedCore) + + if canUseArchivedCore { + // use archived core + if verbose { + logger.Println(constants.LOG_LEVEL_INFO, "Using precompiled core") + } + return targetArchivedCore, variantObjectFiles, nil } - return targetArchivedCore, variantObjectFiles, nil } coreObjectFiles, err := builder_utils.CompileFiles([]string{}, coreFolder, true, buildPath, buildProperties, includes, verbose, warningsLevel, logger) @@ -111,7 +125,10 @@ func compileCore(buildPath string, buildProperties properties.Map, verbose bool, } // archive core.a - builder_utils.CopyFile(archiveFile, targetArchivedCore) + if targetArchivedCore != "" { + logger.Println(constants.LOG_LEVEL_DEBUG, "Archiving built core (caching) in: "+targetArchivedCore) + builder_utils.CopyFile(archiveFile, targetArchivedCore) + } return archiveFile, variantObjectFiles, nil } diff --git a/src/arduino.cc/builder/types/context.go b/src/arduino.cc/builder/types/context.go index 2a8a3db4..5e4fc478 100644 --- a/src/arduino.cc/builder/types/context.go +++ b/src/arduino.cc/builder/types/context.go @@ -37,8 +37,10 @@ type Context struct { BuildProperties properties.Map BuildCore string BuildPath string + BuildCachePath string SketchBuildPath string CoreBuildPath string + CoreBuildCachePath string CoreArchiveFilePath string CoreObjectsFiles []string LibrariesBuildPath string From 4591ba3b7898d842be848c42583891e2bc9d216d Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Fri, 3 Mar 2017 14:50:24 +0100 Subject: [PATCH 11/12] Simplified builder test Now that caching is optional old tests may run as before (without the need to cleanup cache). Signed-off-by: Cristian Maglie --- src/arduino.cc/builder/test/builder_test.go | 66 ++++++--------------- src/arduino.cc/builder/test/helper.go | 20 +++++-- 2 files changed, 32 insertions(+), 54 deletions(-) diff --git a/src/arduino.cc/builder/test/builder_test.go b/src/arduino.cc/builder/test/builder_test.go index 24368368..bfcc7edd 100644 --- a/src/arduino.cc/builder/test/builder_test.go +++ b/src/arduino.cc/builder/test/builder_test.go @@ -55,6 +55,7 @@ func prepareBuilderTestContext(sketchPath, fqbn string) *types.Context { Verbose: false, } } + func TestBuilderEmptySketch(t *testing.T) { DownloadCoresAndToolsAndLibraries(t) @@ -64,11 +65,6 @@ func TestBuilderEmptySketch(t *testing.T) { buildPath := SetupBuildPath(t, ctx) defer os.RemoveAll(buildPath) - // Cleanup cached core - coreFolder := filepath.Join("downloaded_hardware", "arduino", "avr") - coreFile := builder_utils.GetCachedCoreArchiveFileName(ctx.FQBN, coreFolder) - os.Remove(coreFile) - // Run builder command := builder.Builder{} err := command.Run(ctx) @@ -94,11 +90,6 @@ func TestBuilderBridge(t *testing.T) { buildPath := SetupBuildPath(t, ctx) defer os.RemoveAll(buildPath) - // Cleanup cached core - coreFolder := filepath.Join("downloaded_hardware", "arduino", "avr") - coreFile := builder_utils.GetCachedCoreArchiveFileName(ctx.FQBN, coreFolder) - os.Remove(coreFile) - // Run builder command := builder.Builder{} err := command.Run(ctx) @@ -126,11 +117,6 @@ func TestBuilderSketchWithConfig(t *testing.T) { buildPath := SetupBuildPath(t, ctx) defer os.RemoveAll(buildPath) - // Cleanup cached core - coreFolder := filepath.Join("downloaded_hardware", "arduino", "avr") - coreFile := builder_utils.GetCachedCoreArchiveFileName(ctx.FQBN, coreFolder) - os.Remove(coreFile) - // Run builder command := builder.Builder{} err := command.Run(ctx) @@ -158,11 +144,6 @@ func TestBuilderBridgeTwice(t *testing.T) { buildPath := SetupBuildPath(t, ctx) defer os.RemoveAll(buildPath) - // Cleanup cached core - coreFolder := filepath.Join("downloaded_hardware", "arduino", "avr") - coreFile := builder_utils.GetCachedCoreArchiveFileName(ctx.FQBN, coreFolder) - os.Remove(coreFile) - // Run builder command := builder.Builder{} err := command.Run(ctx) @@ -191,17 +172,11 @@ func TestBuilderBridgeSAM(t *testing.T) { DownloadCoresAndToolsAndLibraries(t) ctx := prepareBuilderTestContext(filepath.Join("downloaded_libraries", "Bridge", "examples", "Bridge", "Bridge.ino"), "arduino:sam:arduino_due_x_dbg") + ctx.WarningsLevel = "all" buildPath := SetupBuildPath(t, ctx) defer os.RemoveAll(buildPath) - ctx.WarningsLevel = "all" - - // Cleanup cached core - coreFolder := filepath.Join("downloaded_hardware", "arduino", "sam") - coreFile := builder_utils.GetCachedCoreArchiveFileName(ctx.FQBN, coreFolder) - os.Remove(coreFile) - // Run builder command := builder.Builder{} err := command.Run(ctx) @@ -236,14 +211,10 @@ func TestBuilderBridgeRedBearLab(t *testing.T) { ctx := prepareBuilderTestContext(filepath.Join("downloaded_libraries", "Bridge", "examples", "Bridge", "Bridge.ino"), "RedBearLab:avr:blend") ctx.HardwareFolders = append(ctx.HardwareFolders, "downloaded_board_manager_stuff") ctx.ToolsFolders = append(ctx.ToolsFolders, "downloaded_board_manager_stuff") + buildPath := SetupBuildPath(t, ctx) defer os.RemoveAll(buildPath) - // Cleanup cached core - coreFolder := filepath.Join("downloaded_hardware", "arduino", "avr") - coreFile := builder_utils.GetCachedCoreArchiveFileName(ctx.FQBN, coreFolder) - os.Remove(coreFile) - // Run builder command := builder.Builder{} err := command.Run(ctx) @@ -273,6 +244,7 @@ func TestBuilderSketchNoFunctions(t *testing.T) { buildPath := SetupBuildPath(t, ctx) defer os.RemoveAll(buildPath) + // Run builder command := builder.Builder{} err := command.Run(ctx) require.Error(t, err) @@ -288,6 +260,7 @@ func TestBuilderSketchWithBackup(t *testing.T) { buildPath := SetupBuildPath(t, ctx) defer os.RemoveAll(buildPath) + // Run builder command := builder.Builder{} err := command.Run(ctx) NoError(t, err) @@ -301,6 +274,7 @@ func TestBuilderSketchWithOldLib(t *testing.T) { buildPath := SetupBuildPath(t, ctx) defer os.RemoveAll(buildPath) + // Run builder command := builder.Builder{} err := command.Run(ctx) NoError(t, err) @@ -314,6 +288,7 @@ func TestBuilderSketchWithSubfolders(t *testing.T) { buildPath := SetupBuildPath(t, ctx) defer os.RemoveAll(buildPath) + // Run builder command := builder.Builder{} err := command.Run(ctx) NoError(t, err) @@ -329,6 +304,7 @@ func TestBuilderSketchBuildPathContainsUnusedPreviouslyCompiledLibrary(t *testin NoError(t, os.MkdirAll(filepath.Join(buildPath, constants.FOLDER_LIBRARIES, "SPI"), os.FileMode(0755))) + // Run builder command := builder.Builder{} err := command.Run(ctx) NoError(t, err) @@ -350,19 +326,11 @@ func TestBuilderWithBuildPathInSketchDir(t *testing.T) { NoError(t, err) defer os.RemoveAll(ctx.BuildPath) - // Cleanup cached core - coreFolder := filepath.Join("downloaded_hardware", "arduino", "avr") - coreFile := builder_utils.GetCachedCoreArchiveFileName(ctx.FQBN, coreFolder) - os.Remove(coreFile) - // Run build command := builder.Builder{} err = command.Run(ctx) NoError(t, err) - // Cleanup cached core - os.Remove(coreFile) - // Run build twice, to verify the build still works when the // build directory is present at the start err = command.Run(ctx) @@ -376,24 +344,26 @@ func TestBuilderCacheCoreAFile(t *testing.T) { SetupBuildPath(t, ctx) defer os.RemoveAll(ctx.BuildPath) - - // Cleanup cached core - coreFolder := filepath.Join("downloaded_hardware", "arduino", "avr") - coreFile := builder_utils.GetCachedCoreArchiveFileName(ctx.FQBN, coreFolder) - os.Remove(coreFile) + SetupBuildCachePath(t, ctx) + defer os.RemoveAll(ctx.BuildCachePath) // Run build bldr := builder.Builder{} err := bldr.Run(ctx) NoError(t, err) - coreStatBefore, err := os.Stat(coreFile) + + // Pick timestamp of cached core + coreFolder := filepath.Join("downloaded_hardware", "arduino", "avr") + coreFileName := builder_utils.GetCachedCoreArchiveFileName(ctx.FQBN, coreFolder) + cachedCoreFile := filepath.Join(ctx.CoreBuildCachePath, coreFileName) + coreStatBefore, err := os.Stat(cachedCoreFile) require.NoError(t, err) // Run build again, to verify that the builder skips rebuilding core.a err = bldr.Run(ctx) NoError(t, err) - coreStatAfterRebuild, err := os.Stat(coreFile) + coreStatAfterRebuild, err := os.Stat(cachedCoreFile) require.NoError(t, err) require.Equal(t, coreStatBefore.ModTime(), coreStatAfterRebuild.ModTime()) @@ -407,7 +377,7 @@ func TestBuilderCacheCoreAFile(t *testing.T) { err = bldr.Run(ctx) NoError(t, err) - coreStatAfterTouch, err := os.Stat(coreFile) + coreStatAfterTouch, err := os.Stat(cachedCoreFile) require.NoError(t, err) require.NotEqual(t, coreStatBefore.ModTime(), coreStatAfterTouch.ModTime()) } diff --git a/src/arduino.cc/builder/test/helper.go b/src/arduino.cc/builder/test/helper.go index cd6e2b3d..dfb66338 100644 --- a/src/arduino.cc/builder/test/helper.go +++ b/src/arduino.cc/builder/test/helper.go @@ -31,17 +31,18 @@ package test import ( - "arduino.cc/builder/constants" - "arduino.cc/builder/types" - "arduino.cc/builder/utils" "bytes" "fmt" - "github.com/go-errors/errors" - "github.com/stretchr/testify/assert" "io/ioutil" "path/filepath" "testing" "text/template" + + "arduino.cc/builder/constants" + "arduino.cc/builder/types" + "arduino.cc/builder/utils" + "github.com/go-errors/errors" + "github.com/stretchr/testify/assert" ) func LoadAndInterpolate(t *testing.T, filename string, ctx *types.Context) string { @@ -78,12 +79,19 @@ func NoError(t *testing.T, err error, msgAndArgs ...interface{}) { } func SetupBuildPath(t *testing.T, ctx *types.Context) string { - buildPath, err := ioutil.TempDir(constants.EMPTY_STRING, "test") + buildPath, err := ioutil.TempDir(constants.EMPTY_STRING, "test_build_path") NoError(t, err) ctx.BuildPath = buildPath return buildPath } +func SetupBuildCachePath(t *testing.T, ctx *types.Context) string { + buildCachePath, err := ioutil.TempDir(constants.EMPTY_STRING, "test_build_cache") + NoError(t, err) + ctx.BuildCachePath = buildCachePath + return buildCachePath +} + type ByLibraryName []*types.Library func (s ByLibraryName) Len() int { From 1125e5380ec2f6480b64c662eafc74dbe2bef8a4 Mon Sep 17 00:00:00 2001 From: Martino Facchin Date: Fri, 17 Mar 2017 13:10:45 +0100 Subject: [PATCH 12/12] Remove TestWipeoutBuildPathIfBuildOptionsChangedBuildOptionsMatch test Build options json is not the optimal way to discover if something has changed in the build. Commit https://github.com/arduino/arduino-builder/pull/213/commits/19751fe9aacb983043e1f8a580cf24e06a6b96c5#diff-f796a16acff0c78b88ffe6618745edc6R67 enforces the check and also wipes if the core has changed "in place". Wiping more often gives better confidence that the actual compliation results reflect the backing storage situation. --- ...uild_path_if_build_options_changed_test.go | 38 ++----------------- 1 file changed, 4 insertions(+), 34 deletions(-) diff --git a/src/arduino.cc/builder/test/wipeout_build_path_if_build_options_changed_test.go b/src/arduino.cc/builder/test/wipeout_build_path_if_build_options_changed_test.go index 9354e2d2..f8fc520d 100644 --- a/src/arduino.cc/builder/test/wipeout_build_path_if_build_options_changed_test.go +++ b/src/arduino.cc/builder/test/wipeout_build_path_if_build_options_changed_test.go @@ -30,14 +30,15 @@ package test import ( + "os" + "path/filepath" + "testing" + "arduino.cc/builder" "arduino.cc/builder/gohasissues" "arduino.cc/builder/types" "arduino.cc/builder/utils" "github.com/stretchr/testify/require" - "os" - "path/filepath" - "testing" ) func TestWipeoutBuildPathIfBuildOptionsChanged(t *testing.T) { @@ -100,34 +101,3 @@ func TestWipeoutBuildPathIfBuildOptionsChangedNoPreviousBuildOptions(t *testing. _, err = os.Stat(filepath.Join(buildPath, "should_not_be_deleted.txt")) NoError(t, err) } - -func TestWipeoutBuildPathIfBuildOptionsChangedBuildOptionsMatch(t *testing.T) { - ctx := &types.Context{} - - buildPath := SetupBuildPath(t, ctx) - defer os.RemoveAll(buildPath) - - ctx.BuildOptionsJsonPrevious = "{ \"old\":\"old\" }" - ctx.BuildOptionsJson = "{ \"old\":\"old\" }" - - utils.TouchFile(filepath.Join(buildPath, "should_not_be_deleted.txt")) - - commands := []types.Command{ - &builder.WipeoutBuildPathIfBuildOptionsChanged{}, - } - - for _, command := range commands { - err := command.Run(ctx) - NoError(t, err) - } - - _, err := os.Stat(buildPath) - NoError(t, err) - - files, err := gohasissues.ReadDir(buildPath) - NoError(t, err) - require.Equal(t, 1, len(files)) - - _, err = os.Stat(filepath.Join(buildPath, "should_not_be_deleted.txt")) - NoError(t, err) -}