diff --git a/arduino/builder/archive_compiled_files.go b/arduino/builder/archive_compiled_files.go new file mode 100644 index 00000000000..b640e3e256e --- /dev/null +++ b/arduino/builder/archive_compiled_files.go @@ -0,0 +1,75 @@ +// This file is part of arduino-cli. +// +// Copyright 2023 ARDUINO SA (http://www.arduino.cc/) +// +// This software is released under the GNU General Public License version 3, +// which covers the main part of arduino-cli. +// The terms of this license can be found at: +// https://www.gnu.org/licenses/gpl-3.0.en.html +// +// You can be released from the requirements of the above licenses by purchasing +// a commercial license. Buying such a license is mandatory if you want to +// modify or otherwise use the software for commercial activities involving the +// Arduino software without disclosing the source code of your own applications. +// To purchase a commercial license, send an email to license@arduino.cc. + +package builder + +import ( + "github.com/arduino/go-paths-helper" + "github.com/pkg/errors" +) + +// ArchiveCompiledFiles fixdoc +func (b *Builder) archiveCompiledFiles(buildPath *paths.Path, archiveFile *paths.Path, objectFilesToArchive paths.PathList) (*paths.Path, error) { + archiveFilePath := buildPath.JoinPath(archiveFile) + + if b.onlyUpdateCompilationDatabase { + if b.logger.Verbose() { + b.logger.Info(tr("Skipping archive creation of: %[1]s", archiveFilePath)) + } + return archiveFilePath, nil + } + + if archiveFileStat, err := archiveFilePath.Stat(); err == nil { + rebuildArchive := false + for _, objectFile := range objectFilesToArchive { + objectFileStat, err := objectFile.Stat() + if err != nil || objectFileStat.ModTime().After(archiveFileStat.ModTime()) { + // need to rebuild the archive + rebuildArchive = true + break + } + } + + // something changed, rebuild the core archive + if rebuildArchive { + if err := archiveFilePath.Remove(); err != nil { + return nil, errors.WithStack(err) + } + } else { + if b.logger.Verbose() { + b.logger.Info(tr("Using previously compiled file: %[1]s", archiveFilePath)) + } + return archiveFilePath, nil + } + } + + for _, objectFile := range objectFilesToArchive { + properties := b.buildProperties.Clone() + properties.Set("archive_file", archiveFilePath.Base()) + properties.SetPath("archive_file_path", archiveFilePath) + properties.SetPath("object_file", objectFile) + + command, err := b.prepareCommandForRecipe(properties, "recipe.ar.pattern", false) + if err != nil { + return nil, errors.WithStack(err) + } + + if err := b.execCommand(command); err != nil { + return nil, errors.WithStack(err) + } + } + + return archiveFilePath, nil +} diff --git a/arduino/builder/builder.go b/arduino/builder/builder.go index 8f00d00a4c7..d3c7f2b73ca 100644 --- a/arduino/builder/builder.go +++ b/arduino/builder/builder.go @@ -19,15 +19,20 @@ import ( "errors" "fmt" "io" + "os" + "path/filepath" + "strings" "github.com/arduino/arduino-cli/arduino/builder/internal/compilation" "github.com/arduino/arduino-cli/arduino/builder/internal/detector" "github.com/arduino/arduino-cli/arduino/builder/internal/logger" "github.com/arduino/arduino-cli/arduino/builder/internal/progress" + "github.com/arduino/arduino-cli/arduino/builder/internal/utils" "github.com/arduino/arduino-cli/arduino/cores" "github.com/arduino/arduino-cli/arduino/libraries" "github.com/arduino/arduino-cli/arduino/libraries/librariesmanager" "github.com/arduino/arduino-cli/arduino/sketch" + "github.com/arduino/arduino-cli/executils" rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1" "github.com/arduino/go-paths-helper" "github.com/arduino/go-properties-orderedmap" @@ -268,19 +273,16 @@ func (b *Builder) preprocess() error { return err } b.Progress.CompleteStep() - b.Progress.PushProgress() if err := b.RunRecipe("recipe.hooks.prebuild", ".pattern", false); err != nil { return err } b.Progress.CompleteStep() - b.Progress.PushProgress() if err := b.prepareSketchBuildPath(); err != nil { return err } b.Progress.CompleteStep() - b.Progress.PushProgress() b.logIfVerbose(false, tr("Detecting libraries used...")) err := b.libsDetector.FindIncludes( @@ -297,18 +299,15 @@ func (b *Builder) preprocess() error { return err } b.Progress.CompleteStep() - b.Progress.PushProgress() b.warnAboutArchIncompatibleLibraries(b.libsDetector.ImportedLibraries()) b.Progress.CompleteStep() - b.Progress.PushProgress() b.logIfVerbose(false, tr("Generating function prototypes...")) if err := b.preprocessSketch(b.libsDetector.IncludeFolders()); err != nil { return err } b.Progress.CompleteStep() - b.Progress.PushProgress() return nil } @@ -337,11 +336,9 @@ func (b *Builder) Build() error { b.libsDetector.PrintUsedAndNotUsedLibraries(buildErr != nil) b.Progress.CompleteStep() - b.Progress.PushProgress() b.printUsedLibraries(b.libsDetector.ImportedLibraries()) b.Progress.CompleteStep() - b.Progress.PushProgress() if buildErr != nil { return buildErr @@ -350,13 +347,11 @@ func (b *Builder) Build() error { return err } b.Progress.CompleteStep() - b.Progress.PushProgress() if err := b.size(); err != nil { return err } b.Progress.CompleteStep() - b.Progress.PushProgress() return nil } @@ -368,115 +363,155 @@ func (b *Builder) build() error { return err } b.Progress.CompleteStep() - b.Progress.PushProgress() - if err := b.BuildSketch(b.libsDetector.IncludeFolders()); err != nil { + if err := b.buildSketch(b.libsDetector.IncludeFolders()); err != nil { return err } b.Progress.CompleteStep() - b.Progress.PushProgress() if err := b.RunRecipe("recipe.hooks.sketch.postbuild", ".pattern", true); err != nil { return err } b.Progress.CompleteStep() - b.Progress.PushProgress() b.logIfVerbose(false, tr("Compiling libraries...")) if err := b.RunRecipe("recipe.hooks.libraries.prebuild", ".pattern", false); err != nil { return err } b.Progress.CompleteStep() - b.Progress.PushProgress() if err := b.removeUnusedCompiledLibraries(b.libsDetector.ImportedLibraries()); err != nil { return err } b.Progress.CompleteStep() - b.Progress.PushProgress() if err := b.buildLibraries(b.libsDetector.IncludeFolders(), b.libsDetector.ImportedLibraries()); err != nil { return err } b.Progress.CompleteStep() - b.Progress.PushProgress() if err := b.RunRecipe("recipe.hooks.libraries.postbuild", ".pattern", true); err != nil { return err } b.Progress.CompleteStep() - b.Progress.PushProgress() b.logIfVerbose(false, tr("Compiling core...")) if err := b.RunRecipe("recipe.hooks.core.prebuild", ".pattern", false); err != nil { return err } b.Progress.CompleteStep() - b.Progress.PushProgress() if err := b.buildCore(); err != nil { return err } b.Progress.CompleteStep() - b.Progress.PushProgress() if err := b.RunRecipe("recipe.hooks.core.postbuild", ".pattern", true); err != nil { return err } b.Progress.CompleteStep() - b.Progress.PushProgress() b.logIfVerbose(false, tr("Linking everything together...")) if err := b.RunRecipe("recipe.hooks.linking.prelink", ".pattern", false); err != nil { return err } b.Progress.CompleteStep() - b.Progress.PushProgress() if err := b.link(); err != nil { return err } b.Progress.CompleteStep() - b.Progress.PushProgress() if err := b.RunRecipe("recipe.hooks.linking.postlink", ".pattern", true); err != nil { return err } b.Progress.CompleteStep() - b.Progress.PushProgress() if err := b.RunRecipe("recipe.hooks.objcopy.preobjcopy", ".pattern", false); err != nil { return err } b.Progress.CompleteStep() - b.Progress.PushProgress() if err := b.RunRecipe("recipe.objcopy.", ".pattern", true); err != nil { return err } b.Progress.CompleteStep() - b.Progress.PushProgress() if err := b.RunRecipe("recipe.hooks.objcopy.postobjcopy", ".pattern", true); err != nil { return err } b.Progress.CompleteStep() - b.Progress.PushProgress() - if err := b.MergeSketchWithBootloader(); err != nil { + if err := b.mergeSketchWithBootloader(); err != nil { return err } b.Progress.CompleteStep() - b.Progress.PushProgress() if err := b.RunRecipe("recipe.hooks.postbuild", ".pattern", true); err != nil { return err } b.Progress.CompleteStep() - b.Progress.PushProgress() if b.compilationDatabase != nil { b.compilationDatabase.SaveToFile() } return nil } + +func (b *Builder) prepareCommandForRecipe(buildProperties *properties.Map, recipe string, removeUnsetProperties bool) (*executils.Process, error) { + pattern := buildProperties.Get(recipe) + if pattern == "" { + return nil, fmt.Errorf(tr("%[1]s pattern is missing"), recipe) + } + + commandLine := buildProperties.ExpandPropsInString(pattern) + if removeUnsetProperties { + commandLine = properties.DeleteUnexpandedPropsFromString(commandLine) + } + + parts, err := properties.SplitQuotedString(commandLine, `"'`, false) + if err != nil { + return nil, err + } + + // if the overall commandline is too long for the platform + // try reducing the length by making the filenames relative + // and changing working directory to build.path + var relativePath string + if len(commandLine) > 30000 { + relativePath = buildProperties.Get("build.path") + for i, arg := range parts { + if _, err := os.Stat(arg); os.IsNotExist(err) { + continue + } + rel, err := filepath.Rel(relativePath, arg) + if err == nil && !strings.Contains(rel, "..") && len(rel) < len(arg) { + parts[i] = rel + } + } + } + + command, err := executils.NewProcess(nil, parts...) + if err != nil { + return nil, err + } + if relativePath != "" { + command.SetDir(relativePath) + } + + return command, nil +} + +func (b *Builder) execCommand(command *executils.Process) error { + if b.logger.Verbose() { + b.logger.Info(utils.PrintableCommand(command.GetArgs())) + command.RedirectStdoutTo(b.logger.Stdout()) + } + command.RedirectStderrTo(b.logger.Stderr()) + + if err := command.Start(); err != nil { + return err + } + + return command.Wait() +} diff --git a/arduino/builder/compilation.go b/arduino/builder/compilation.go new file mode 100644 index 00000000000..e3c70900bf6 --- /dev/null +++ b/arduino/builder/compilation.go @@ -0,0 +1,182 @@ +// This file is part of arduino-cli. +// +// Copyright 2023 ARDUINO SA (http://www.arduino.cc/) +// +// This software is released under the GNU General Public License version 3, +// which covers the main part of arduino-cli. +// The terms of this license can be found at: +// https://www.gnu.org/licenses/gpl-3.0.en.html +// +// You can be released from the requirements of the above licenses by purchasing +// a commercial license. Buying such a license is mandatory if you want to +// modify or otherwise use the software for commercial activities involving the +// Arduino software without disclosing the source code of your own applications. +// To purchase a commercial license, send an email to license@arduino.cc. + +package builder + +import ( + "bytes" + "fmt" + "runtime" + "strings" + "sync" + + "github.com/arduino/arduino-cli/arduino/builder/internal/utils" + "github.com/arduino/arduino-cli/arduino/globals" + "github.com/arduino/go-paths-helper" + "github.com/pkg/errors" +) + +func (b *Builder) compileFiles( + sourceDir *paths.Path, + buildPath *paths.Path, + recurse bool, + includes []string, +) (paths.PathList, error) { + validExtensions := []string{} + for ext := range globals.SourceFilesValidExtensions { + validExtensions = append(validExtensions, ext) + } + + sources, err := utils.FindFilesInFolder(sourceDir, recurse, validExtensions...) + if err != nil { + return nil, err + } + + b.Progress.AddSubSteps(len(sources)) + defer b.Progress.RemoveSubSteps() + + objectFiles := paths.NewPathList() + var objectFilesMux sync.Mutex + if len(sources) == 0 { + return objectFiles, nil + } + var errorsList []error + var errorsMux sync.Mutex + + queue := make(chan *paths.Path) + job := func(source *paths.Path) { + recipe := fmt.Sprintf("recipe%s.o.pattern", source.Ext()) + if !b.buildProperties.ContainsKey(recipe) { + recipe = fmt.Sprintf("recipe%s.o.pattern", globals.SourceFilesValidExtensions[source.Ext()]) + } + objectFile, err := b.compileFileWithRecipe(sourceDir, source, buildPath, includes, recipe) + if err != nil { + errorsMux.Lock() + errorsList = append(errorsList, err) + errorsMux.Unlock() + } else { + objectFilesMux.Lock() + objectFiles.Add(objectFile) + objectFilesMux.Unlock() + } + } + + // Spawn jobs runners + var wg sync.WaitGroup + if b.jobs == 0 { + b.jobs = runtime.NumCPU() + } + for i := 0; i < b.jobs; i++ { + wg.Add(1) + go func() { + for source := range queue { + job(source) + } + wg.Done() + }() + } + + // Feed jobs until error or done + for _, source := range sources { + errorsMux.Lock() + gotError := len(errorsList) > 0 + errorsMux.Unlock() + if gotError { + break + } + queue <- source + + b.Progress.CompleteStep() + } + close(queue) + wg.Wait() + if len(errorsList) > 0 { + // output the first error + return nil, errors.WithStack(errorsList[0]) + } + objectFiles.Sort() + return objectFiles, nil +} + +// CompileFilesRecursive fixdoc +func (b *Builder) compileFileWithRecipe( + sourcePath *paths.Path, + source *paths.Path, + buildPath *paths.Path, + includes []string, + recipe string, +) (*paths.Path, error) { + properties := b.buildProperties.Clone() + properties.Set("compiler.warning_flags", properties.Get("compiler.warning_flags."+b.logger.WarningsLevel())) + properties.Set("includes", strings.Join(includes, " ")) + properties.SetPath("source_file", source) + relativeSource, err := sourcePath.RelTo(source) + if err != nil { + return nil, errors.WithStack(err) + } + depsFile := buildPath.Join(relativeSource.String() + ".d") + objectFile := buildPath.Join(relativeSource.String() + ".o") + + properties.SetPath("object_file", objectFile) + err = objectFile.Parent().MkdirAll() + if err != nil { + return nil, errors.WithStack(err) + } + + objIsUpToDate, err := utils.ObjFileIsUpToDate(source, objectFile, depsFile) + if err != nil { + return nil, errors.WithStack(err) + } + + command, err := b.prepareCommandForRecipe(properties, recipe, false) + if err != nil { + return nil, errors.WithStack(err) + } + if b.compilationDatabase != nil { + b.compilationDatabase.Add(source, command) + } + if !objIsUpToDate && !b.onlyUpdateCompilationDatabase { + commandStdout, commandStderr := &bytes.Buffer{}, &bytes.Buffer{} + command.RedirectStdoutTo(commandStdout) + command.RedirectStderrTo(commandStderr) + + if b.logger.Verbose() { + b.logger.Info(utils.PrintableCommand(command.GetArgs())) + } + // Since this compile could be multithreaded, we first capture the command output + if err := command.Start(); err != nil { + return nil, err + } + err := command.Wait() + // and transfer all at once at the end... + if b.logger.Verbose() { + b.logger.WriteStdout(commandStdout.Bytes()) + } + b.logger.WriteStderr(commandStderr.Bytes()) + + // ...and then return the error + if err != nil { + return nil, errors.WithStack(err) + } + } else if b.logger.Verbose() { + if objIsUpToDate { + b.logger.Info(tr("Using previously compiled file: %[1]s", objectFile)) + } else { + b.logger.Info(tr("Skipping compile of: %[1]s", objectFile)) + } + } + + return objectFile, nil +} diff --git a/arduino/builder/core.go b/arduino/builder/core.go index b58d3dea806..7d61d962f18 100644 --- a/arduino/builder/core.go +++ b/arduino/builder/core.go @@ -71,13 +71,10 @@ func (b *Builder) compileCore() (*paths.Path, paths.PathList, error) { var err error variantObjectFiles := paths.NewPathList() if variantFolder != nil && variantFolder.IsDir() { - variantObjectFiles, err = utils.CompileFilesRecursive( - variantFolder, b.coreBuildPath, b.buildProperties, includes, - b.onlyUpdateCompilationDatabase, - b.compilationDatabase, - b.jobs, - b.logger, - b.Progress, + variantObjectFiles, err = b.compileFiles( + variantFolder, b.coreBuildPath, + true, /** recursive **/ + includes, ) if err != nil { return nil, nil, errors.WithStack(err) @@ -122,25 +119,16 @@ func (b *Builder) compileCore() (*paths.Path, paths.PathList, error) { } } - coreObjectFiles, err := utils.CompileFilesRecursive( - coreFolder, b.coreBuildPath, b.buildProperties, includes, - b.onlyUpdateCompilationDatabase, - b.compilationDatabase, - b.jobs, - b.logger, - b.Progress, + coreObjectFiles, err := b.compileFiles( + coreFolder, b.coreBuildPath, + true, /** recursive **/ + includes, ) if err != nil { return nil, nil, errors.WithStack(err) } - archiveFile, verboseInfo, err := utils.ArchiveCompiledFiles( - b.coreBuildPath, paths.New("core.a"), coreObjectFiles, b.buildProperties, - b.onlyUpdateCompilationDatabase, b.logger.Verbose(), b.logger.Stdout(), b.logger.Stderr(), - ) - if b.logger.Verbose() { - b.logger.Info(string(verboseInfo)) - } + archiveFile, err := b.archiveCompiledFiles(b.coreBuildPath, paths.New("core.a"), coreObjectFiles) if err != nil { return nil, nil, errors.WithStack(err) } diff --git a/arduino/builder/export_cmake.go b/arduino/builder/export_cmake.go index 6f3bc4e3dfd..e65776b4e0b 100644 --- a/arduino/builder/export_cmake.go +++ b/arduino/builder/export_cmake.go @@ -275,9 +275,9 @@ func (b *Builder) exportProjectCMake(importedLibraries libraries.List, includeFo var dynamicLibsFromGccMinusL []string var linkDirectories []string - extractCompileFlags(b.buildProperties, "recipe.c.combine.pattern", &defines, &dynamicLibsFromGccMinusL, &linkerflags, &linkDirectories) - extractCompileFlags(b.buildProperties, "recipe.c.o.pattern", &defines, &dynamicLibsFromGccMinusL, &linkerflags, &linkDirectories) - extractCompileFlags(b.buildProperties, "recipe.cpp.o.pattern", &defines, &dynamicLibsFromGccMinusL, &linkerflags, &linkDirectories) + b.extractCompileFlags(b.buildProperties, "recipe.c.combine.pattern", &defines, &dynamicLibsFromGccMinusL, &linkerflags, &linkDirectories) + b.extractCompileFlags(b.buildProperties, "recipe.c.o.pattern", &defines, &dynamicLibsFromGccMinusL, &linkerflags, &linkDirectories) + b.extractCompileFlags(b.buildProperties, "recipe.cpp.o.pattern", &defines, &dynamicLibsFromGccMinusL, &linkerflags, &linkDirectories) // Extract folders with .h in them for adding in include list headerFiles, _ := utils.FindFilesInFolder(cmakeFolder, true, validHeaderExtensions...) @@ -348,7 +348,7 @@ func (b *Builder) exportProjectCMake(importedLibraries libraries.List, includeFo return nil } -func extractCompileFlags(buildProperties *properties.Map, recipe string, defines, dynamicLibs, linkerflags, linkDirectories *[]string) { +func (b *Builder) extractCompileFlags(buildProperties *properties.Map, recipe string, defines, dynamicLibs, linkerflags, linkDirectories *[]string) { appendIfNotPresent := func(target []string, elements ...string) []string { for _, element := range elements { if !slices.Contains(target, element) { @@ -358,7 +358,7 @@ func extractCompileFlags(buildProperties *properties.Map, recipe string, defines return target } - command, _ := utils.PrepareCommandForRecipe(buildProperties, recipe, true) + command, _ := b.prepareCommandForRecipe(buildProperties, recipe, true) for _, arg := range command.GetArgs() { if strings.HasPrefix(arg, "-D") { diff --git a/arduino/builder/internal/progress/progress.go b/arduino/builder/internal/progress/progress.go index a3fa9b09d9b..4722813f6a8 100644 --- a/arduino/builder/internal/progress/progress.go +++ b/arduino/builder/internal/progress/progress.go @@ -53,10 +53,10 @@ func (p *Struct) RemoveSubSteps() { // CompleteStep fixdoc func (p *Struct) CompleteStep() { p.Progress += p.StepAmount + p.pushProgress() } -// PushProgress fixdoc -func (p *Struct) PushProgress() { +func (p *Struct) pushProgress() { if p.callback != nil { p.callback(&rpc.TaskProgress{ Percent: p.Progress, diff --git a/arduino/builder/internal/utils/utils.go b/arduino/builder/internal/utils/utils.go index 497f3de0668..efd40b307ea 100644 --- a/arduino/builder/internal/utils/utils.go +++ b/arduino/builder/internal/utils/utils.go @@ -16,25 +16,13 @@ package utils import ( - "bytes" - "fmt" - "io" "os" - "path/filepath" - "runtime" "strings" - "sync" "unicode" - "github.com/arduino/arduino-cli/arduino/builder/internal/compilation" - "github.com/arduino/arduino-cli/arduino/builder/internal/logger" - "github.com/arduino/arduino-cli/arduino/builder/internal/progress" - "github.com/arduino/arduino-cli/arduino/globals" - "github.com/arduino/arduino-cli/executils" "github.com/arduino/arduino-cli/i18n" f "github.com/arduino/arduino-cli/internal/algorithms" "github.com/arduino/go-paths-helper" - "github.com/arduino/go-properties-orderedmap" "github.com/pkg/errors" "github.com/sirupsen/logrus" "golang.org/x/text/runes" @@ -210,14 +198,6 @@ func FindFilesInFolder(dir *paths.Path, recurse bool, extensions ...string) (pat return dir.ReadDir(fileFilter) } -// nolint -const ( - Ignore = 0 // Redirect to null - Show = 1 // Show on stdout/stderr as normal - ShowIfVerbose = 2 // Show if verbose is set, Ignore otherwise - Capture = 3 // Capture into buffer -) - func printableArgument(arg string) string { if strings.ContainsAny(arg, "\"\\ \t") { arg = strings.ReplaceAll(arg, "\\", "\\\\") @@ -227,56 +207,14 @@ func printableArgument(arg string) string { return arg } -// Convert a command and argument slice back to a printable string. +// PrintableCommand Convert a command and argument slice back to a printable string. // This adds basic escaping which is sufficient for debug output, but // probably not for shell interpretation. This essentially reverses // ParseCommandLine. -func printableCommand(parts []string) string { +func PrintableCommand(parts []string) string { return strings.Join(f.Map(parts, printableArgument), " ") } -// ExecCommand fixdoc -func ExecCommand( - verbose bool, - stdoutWriter, stderrWriter io.Writer, - command *executils.Process, stdout int, stderr int, -) ([]byte, []byte, []byte, error) { - verboseInfoBuf := &bytes.Buffer{} - if verbose { - verboseInfoBuf.WriteString(printableCommand(command.GetArgs())) - } - - stdoutBuffer := &bytes.Buffer{} - if stdout == Capture { - command.RedirectStdoutTo(stdoutBuffer) - } else if stdout == Show || (stdout == ShowIfVerbose && verbose) { - if stdoutWriter != nil { - command.RedirectStdoutTo(stdoutWriter) - } else { - command.RedirectStdoutTo(os.Stdout) - } - } - - stderrBuffer := &bytes.Buffer{} - if stderr == Capture { - command.RedirectStderrTo(stderrBuffer) - } else if stderr == Show || (stderr == ShowIfVerbose && verbose) { - if stderrWriter != nil { - command.RedirectStderrTo(stderrWriter) - } else { - command.RedirectStderrTo(os.Stderr) - } - } - - err := command.Start() - if err != nil { - return verboseInfoBuf.Bytes(), nil, nil, errors.WithStack(err) - } - - err = command.Wait() - return verboseInfoBuf.Bytes(), stdoutBuffer.Bytes(), stderrBuffer.Bytes(), errors.WithStack(err) -} - // DirContentIsOlderThan returns true if the content of the given directory is // older than target file. If extensions are given, only the files with these // extensions are tested. @@ -302,330 +240,3 @@ func DirContentIsOlderThan(dir *paths.Path, target *paths.Path, extensions ...st } return true, nil } - -// PrepareCommandForRecipe fixdoc -func PrepareCommandForRecipe(buildProperties *properties.Map, recipe string, removeUnsetProperties bool) (*executils.Process, error) { - pattern := buildProperties.Get(recipe) - if pattern == "" { - return nil, errors.Errorf(tr("%[1]s pattern is missing"), recipe) - } - - commandLine := buildProperties.ExpandPropsInString(pattern) - if removeUnsetProperties { - commandLine = properties.DeleteUnexpandedPropsFromString(commandLine) - } - - parts, err := properties.SplitQuotedString(commandLine, `"'`, false) - if err != nil { - return nil, errors.WithStack(err) - } - - // if the overall commandline is too long for the platform - // try reducing the length by making the filenames relative - // and changing working directory to build.path - var relativePath string - if len(commandLine) > 30000 { - relativePath = buildProperties.Get("build.path") - for i, arg := range parts { - if _, err := os.Stat(arg); os.IsNotExist(err) { - continue - } - rel, err := filepath.Rel(relativePath, arg) - if err == nil && !strings.Contains(rel, "..") && len(rel) < len(arg) { - parts[i] = rel - } - } - } - - command, err := executils.NewProcess(nil, parts...) - if err != nil { - return nil, errors.WithStack(err) - } - if relativePath != "" { - command.SetDir(relativePath) - } - - return command, nil -} - -// CompileFiles fixdoc -func CompileFiles( - sourceDir, buildPath *paths.Path, - buildProperties *properties.Map, - includes []string, - onlyUpdateCompilationDatabase bool, - compilationDatabase *compilation.Database, - jobs int, - builderLogger *logger.BuilderLogger, - progress *progress.Struct, -) (paths.PathList, error) { - return compileFiles( - onlyUpdateCompilationDatabase, - compilationDatabase, - jobs, - sourceDir, - false, - buildPath, buildProperties, includes, - builderLogger, - progress, - ) -} - -// CompileFilesRecursive fixdoc -func CompileFilesRecursive( - sourceDir, buildPath *paths.Path, - buildProperties *properties.Map, - includes []string, - onlyUpdateCompilationDatabase bool, - compilationDatabase *compilation.Database, - jobs int, - builderLogger *logger.BuilderLogger, - progress *progress.Struct, -) (paths.PathList, error) { - return compileFiles( - onlyUpdateCompilationDatabase, - compilationDatabase, - jobs, - sourceDir, - true, - buildPath, buildProperties, includes, - builderLogger, - progress, - ) -} - -func compileFiles( - onlyUpdateCompilationDatabase bool, - compilationDatabase *compilation.Database, - jobs int, - sourceDir *paths.Path, - recurse bool, - buildPath *paths.Path, - buildProperties *properties.Map, - includes []string, - builderLogger *logger.BuilderLogger, - progress *progress.Struct, -) (paths.PathList, error) { - validExtensions := []string{} - for ext := range globals.SourceFilesValidExtensions { - validExtensions = append(validExtensions, ext) - } - - sources, err := FindFilesInFolder(sourceDir, recurse, validExtensions...) - if err != nil { - return nil, err - } - - progress.AddSubSteps(len(sources)) - defer progress.RemoveSubSteps() - - objectFiles := paths.NewPathList() - var objectFilesMux sync.Mutex - if len(sources) == 0 { - return objectFiles, nil - } - var errorsList []error - var errorsMux sync.Mutex - - queue := make(chan *paths.Path) - job := func(source *paths.Path) { - recipe := fmt.Sprintf("recipe%s.o.pattern", source.Ext()) - if !buildProperties.ContainsKey(recipe) { - recipe = fmt.Sprintf("recipe%s.o.pattern", globals.SourceFilesValidExtensions[source.Ext()]) - } - objectFile, verboseInfo, verboseStdout, stderr, err := compileFileWithRecipe( - compilationDatabase, - onlyUpdateCompilationDatabase, - sourceDir, source, buildPath, buildProperties, includes, recipe, - builderLogger, - ) - if builderLogger.Verbose() { - builderLogger.WriteStdout(verboseStdout) - builderLogger.Info(string(verboseInfo)) - } - builderLogger.WriteStderr(stderr) - if err != nil { - errorsMux.Lock() - errorsList = append(errorsList, err) - errorsMux.Unlock() - } else { - objectFilesMux.Lock() - objectFiles.Add(objectFile) - objectFilesMux.Unlock() - } - } - - // Spawn jobs runners - var wg sync.WaitGroup - if jobs == 0 { - jobs = runtime.NumCPU() - } - for i := 0; i < jobs; i++ { - wg.Add(1) - go func() { - for source := range queue { - job(source) - } - wg.Done() - }() - } - - // Feed jobs until error or done - for _, source := range sources { - errorsMux.Lock() - gotError := len(errorsList) > 0 - errorsMux.Unlock() - if gotError { - break - } - queue <- source - - progress.CompleteStep() - progress.PushProgress() - } - close(queue) - wg.Wait() - if len(errorsList) > 0 { - // output the first error - return nil, errors.WithStack(errorsList[0]) - } - objectFiles.Sort() - return objectFiles, nil -} - -func compileFileWithRecipe( - compilationDatabase *compilation.Database, - onlyUpdateCompilationDatabase bool, - sourcePath *paths.Path, - source *paths.Path, - buildPath *paths.Path, - buildProperties *properties.Map, - includes []string, - recipe string, - builderLogger *logger.BuilderLogger, -) (*paths.Path, []byte, []byte, []byte, error) { - verboseStdout, verboseInfo, errOut := &bytes.Buffer{}, &bytes.Buffer{}, &bytes.Buffer{} - - properties := buildProperties.Clone() - properties.Set("compiler.warning_flags", properties.Get("compiler.warning_flags."+builderLogger.WarningsLevel())) - properties.Set("includes", strings.Join(includes, " ")) - properties.SetPath("source_file", source) - relativeSource, err := sourcePath.RelTo(source) - if err != nil { - return nil, nil, nil, nil, errors.WithStack(err) - } - depsFile := buildPath.Join(relativeSource.String() + ".d") - objectFile := buildPath.Join(relativeSource.String() + ".o") - - properties.SetPath("object_file", objectFile) - err = objectFile.Parent().MkdirAll() - if err != nil { - return nil, nil, nil, nil, errors.WithStack(err) - } - - objIsUpToDate, err := ObjFileIsUpToDate(source, objectFile, depsFile) - if err != nil { - return nil, nil, nil, nil, errors.WithStack(err) - } - - command, err := PrepareCommandForRecipe(properties, recipe, false) - if err != nil { - return nil, nil, nil, nil, errors.WithStack(err) - } - if compilationDatabase != nil { - compilationDatabase.Add(source, command) - } - if !objIsUpToDate && !onlyUpdateCompilationDatabase { - // Since this compile could be multithreaded, we first capture the command output - info, stdout, stderr, err := ExecCommand( - builderLogger.Verbose(), - builderLogger.Stdout(), - builderLogger.Stderr(), - command, - Capture, - Capture, - ) - // and transfer all at once at the end... - if builderLogger.Verbose() { - verboseInfo.Write(info) - verboseStdout.Write(stdout) - } - errOut.Write(stderr) - - // ...and then return the error - if err != nil { - return nil, verboseInfo.Bytes(), verboseStdout.Bytes(), errOut.Bytes(), errors.WithStack(err) - } - } else if builderLogger.Verbose() { - if objIsUpToDate { - verboseInfo.WriteString(tr("Using previously compiled file: %[1]s", objectFile)) - } else { - verboseInfo.WriteString(tr("Skipping compile of: %[1]s", objectFile)) - } - } - - return objectFile, verboseInfo.Bytes(), verboseStdout.Bytes(), errOut.Bytes(), nil -} - -// ArchiveCompiledFiles fixdoc -func ArchiveCompiledFiles( - buildPath *paths.Path, archiveFile *paths.Path, objectFilesToArchive paths.PathList, buildProperties *properties.Map, - onlyUpdateCompilationDatabase, verbose bool, - stdoutWriter, stderrWriter io.Writer, -) (*paths.Path, []byte, error) { - verboseInfobuf := &bytes.Buffer{} - archiveFilePath := buildPath.JoinPath(archiveFile) - - if onlyUpdateCompilationDatabase { - if verbose { - verboseInfobuf.WriteString(tr("Skipping archive creation of: %[1]s", archiveFilePath)) - } - return archiveFilePath, verboseInfobuf.Bytes(), nil - } - - if archiveFileStat, err := archiveFilePath.Stat(); err == nil { - rebuildArchive := false - for _, objectFile := range objectFilesToArchive { - objectFileStat, err := objectFile.Stat() - if err != nil || objectFileStat.ModTime().After(archiveFileStat.ModTime()) { - // need to rebuild the archive - rebuildArchive = true - break - } - } - - // something changed, rebuild the core archive - if rebuildArchive { - if err := archiveFilePath.Remove(); err != nil { - return nil, nil, errors.WithStack(err) - } - } else { - if verbose { - verboseInfobuf.WriteString(tr("Using previously compiled file: %[1]s", archiveFilePath)) - } - return archiveFilePath, verboseInfobuf.Bytes(), nil - } - } - - for _, objectFile := range objectFilesToArchive { - properties := buildProperties.Clone() - properties.Set("archive_file", archiveFilePath.Base()) - properties.SetPath("archive_file_path", archiveFilePath) - properties.SetPath("object_file", objectFile) - - command, err := PrepareCommandForRecipe(properties, "recipe.ar.pattern", false) - if err != nil { - return nil, verboseInfobuf.Bytes(), errors.WithStack(err) - } - - verboseInfo, _, _, err := ExecCommand(verbose, stdoutWriter, stderrWriter, command, ShowIfVerbose /* stdout */, Show /* stderr */) - if verbose { - verboseInfobuf.WriteString(string(verboseInfo)) - } - if err != nil { - return nil, verboseInfobuf.Bytes(), errors.WithStack(err) - } - } - - return archiveFilePath, verboseInfobuf.Bytes(), nil -} diff --git a/arduino/builder/internal/utils/utils_test.go b/arduino/builder/internal/utils/utils_test.go index 1c31160f5d0..4af9613dbd4 100644 --- a/arduino/builder/internal/utils/utils_test.go +++ b/arduino/builder/internal/utils/utils_test.go @@ -39,7 +39,7 @@ func TestPrintableCommand(t *testing.T) { " \"specialchar-`~!@#$%^&*()-_=+[{]}\\\\|;:'\\\",<.>/?-argument\"" + " \"arg with spaces\" \"arg\twith\t\ttabs\"" + " lastarg" - result := printableCommand(parts) + result := PrintableCommand(parts) require.Equal(t, correct, result) } diff --git a/arduino/builder/libraries.go b/arduino/builder/libraries.go index 2499e5b9253..bbe4f514529 100644 --- a/arduino/builder/libraries.go +++ b/arduino/builder/libraries.go @@ -21,7 +21,6 @@ import ( "time" "github.com/arduino/arduino-cli/arduino/builder/cpp" - "github.com/arduino/arduino-cli/arduino/builder/internal/utils" "github.com/arduino/arduino-cli/arduino/libraries" f "github.com/arduino/arduino-cli/internal/algorithms" "github.com/arduino/go-paths-helper" @@ -68,7 +67,7 @@ func (b *Builder) findExpectedPrecompiledLibFolder( // Add fpu specifications if they exist // To do so, resolve recipe.cpp.o.pattern, // search for -mfpu=xxx -mfloat-abi=yyy and add to a subfolder - command, _ := utils.PrepareCommandForRecipe(buildProperties, "recipe.cpp.o.pattern", true) + command, _ := b.prepareCommandForRecipe(buildProperties, "recipe.cpp.o.pattern", true) fpuSpecs := "" for _, el := range command.GetArgs() { if strings.Contains(el, FpuCflag) { @@ -124,7 +123,6 @@ func (b *Builder) compileLibraries(libraries libraries.List, includes []string) objectFiles.AddAll(libraryObjectFiles) b.Progress.CompleteStep() - b.Progress.PushProgress() } return objectFiles, nil @@ -190,26 +188,16 @@ func (b *Builder) compileLibrary(library *libraries.Library, includes []string) } if library.Layout == libraries.RecursiveLayout { - libObjectFiles, err := utils.CompileFilesRecursive( - library.SourceDir, libraryBuildPath, b.buildProperties, includes, - b.onlyUpdateCompilationDatabase, - b.compilationDatabase, - b.jobs, - b.logger, - b.Progress, + libObjectFiles, err := b.compileFiles( + library.SourceDir, libraryBuildPath, + true, /** recursive **/ + includes, ) if err != nil { return nil, errors.WithStack(err) } if library.DotALinkage { - archiveFile, verboseInfo, err := utils.ArchiveCompiledFiles( - libraryBuildPath, paths.New(library.DirName+".a"), libObjectFiles, b.buildProperties, - b.onlyUpdateCompilationDatabase, b.logger.Verbose(), - b.logger.Stdout(), b.logger.Stderr(), - ) - if b.logger.Verbose() { - b.logger.Info(string(verboseInfo)) - } + archiveFile, err := b.archiveCompiledFiles(libraryBuildPath, paths.New(library.DirName+".a"), libObjectFiles) if err != nil { return nil, errors.WithStack(err) } @@ -221,13 +209,10 @@ func (b *Builder) compileLibrary(library *libraries.Library, includes []string) if library.UtilityDir != nil { includes = append(includes, cpp.WrapWithHyphenI(library.UtilityDir.String())) } - libObjectFiles, err := utils.CompileFiles( - library.SourceDir, libraryBuildPath, b.buildProperties, includes, - b.onlyUpdateCompilationDatabase, - b.compilationDatabase, - b.jobs, - b.logger, - b.Progress, + libObjectFiles, err := b.compileFiles( + library.SourceDir, libraryBuildPath, + false, /** recursive **/ + includes, ) if err != nil { return nil, errors.WithStack(err) @@ -236,13 +221,10 @@ func (b *Builder) compileLibrary(library *libraries.Library, includes []string) if library.UtilityDir != nil { utilityBuildPath := libraryBuildPath.Join("utility") - utilityObjectFiles, err := utils.CompileFiles( - library.UtilityDir, utilityBuildPath, b.buildProperties, includes, - b.onlyUpdateCompilationDatabase, - b.compilationDatabase, - b.jobs, - b.logger, - b.Progress, + utilityObjectFiles, err := b.compileFiles( + library.UtilityDir, utilityBuildPath, + false, /** recursive **/ + includes, ) if err != nil { return nil, errors.WithStack(err) diff --git a/arduino/builder/linker.go b/arduino/builder/linker.go index ba1b95718ee..caad00d02c9 100644 --- a/arduino/builder/linker.go +++ b/arduino/builder/linker.go @@ -18,7 +18,6 @@ package builder import ( "strings" - "github.com/arduino/arduino-cli/arduino/builder/internal/utils" f "github.com/arduino/arduino-cli/internal/algorithms" "github.com/arduino/go-paths-helper" "github.com/pkg/errors" @@ -72,15 +71,12 @@ func (b *Builder) link() error { properties.SetPath("archive_file_path", archive) properties.SetPath("object_file", object) - command, err := utils.PrepareCommandForRecipe(properties, "recipe.ar.pattern", false) + command, err := b.prepareCommandForRecipe(properties, "recipe.ar.pattern", false) if err != nil { return errors.WithStack(err) } - if verboseInfo, _, _, err := utils.ExecCommand(b.logger.Verbose(), b.logger.Stdout(), b.logger.Stderr(), command, utils.ShowIfVerbose /* stdout */, utils.Show /* stderr */); err != nil { - if b.logger.Verbose() { - b.logger.Info(string(verboseInfo)) - } + if err := b.execCommand(command); err != nil { return errors.WithStack(err) } } @@ -96,17 +92,10 @@ func (b *Builder) link() error { properties.Set("archive_file_path", b.buildArtifacts.coreArchiveFilePath.String()) properties.Set("object_files", objectFileList) - command, err := utils.PrepareCommandForRecipe(properties, "recipe.c.combine.pattern", false) + command, err := b.prepareCommandForRecipe(properties, "recipe.c.combine.pattern", false) if err != nil { return err } - verboseInfo, _, _, err := utils.ExecCommand(b.logger.Verbose(), b.logger.Stdout(), b.logger.Stderr(), command, utils.ShowIfVerbose /* stdout */, utils.Show /* stderr */) - if b.logger.Verbose() { - b.logger.Info(string(verboseInfo)) - } - if err != nil { - return err - } - return nil + return b.execCommand(command) } diff --git a/arduino/builder/recipe.go b/arduino/builder/recipe.go index 4cb11179188..57b137176f1 100644 --- a/arduino/builder/recipe.go +++ b/arduino/builder/recipe.go @@ -20,7 +20,6 @@ import ( "sort" "strings" - "github.com/arduino/arduino-cli/arduino/builder/internal/utils" properties "github.com/arduino/go-properties-orderedmap" "github.com/pkg/errors" "github.com/sirupsen/logrus" @@ -39,7 +38,7 @@ func (b *Builder) RunRecipe(prefix, suffix string, skipIfOnlyUpdatingCompilation for _, recipe := range recipes { logrus.Debugf(fmt.Sprintf("Running recipe: %s", recipe)) - command, err := utils.PrepareCommandForRecipe(properties, recipe, false) + command, err := b.prepareCommandForRecipe(properties, recipe, false) if err != nil { return errors.WithStack(err) } @@ -51,11 +50,7 @@ func (b *Builder) RunRecipe(prefix, suffix string, skipIfOnlyUpdatingCompilation return nil } - verboseInfo, _, _, err := utils.ExecCommand(b.logger.Verbose(), b.logger.Stdout(), b.logger.Stderr(), command, utils.ShowIfVerbose /* stdout */, utils.Show /* stderr */) - if b.logger.Verbose() { - b.logger.Info(string(verboseInfo)) - } - if err != nil { + if err := b.execCommand(command); err != nil { return errors.WithStack(err) } } diff --git a/arduino/builder/sizer.go b/arduino/builder/sizer.go index 1d8707df487..a4e362f7106 100644 --- a/arduino/builder/sizer.go +++ b/arduino/builder/sizer.go @@ -16,6 +16,7 @@ package builder import ( + "bytes" "encoding/json" "fmt" "regexp" @@ -72,16 +73,20 @@ func (b *Builder) size() error { } func (b *Builder) checkSizeAdvanced() (ExecutablesFileSections, error) { - command, err := utils.PrepareCommandForRecipe(b.buildProperties, "recipe.advanced_size.pattern", false) + command, err := b.prepareCommandForRecipe(b.buildProperties, "recipe.advanced_size.pattern", false) if err != nil { return nil, errors.New(tr("Error while determining sketch size: %s", err)) } - - verboseInfo, out, _, err := utils.ExecCommand(b.logger.Verbose(), b.logger.Stdout(), b.logger.Stderr(), command, utils.Capture /* stdout */, utils.Show /* stderr */) if b.logger.Verbose() { - b.logger.Info(string(verboseInfo)) + b.logger.Info(utils.PrintableCommand(command.GetArgs())) } - if err != nil { + out := &bytes.Buffer{} + command.RedirectStdoutTo(out) + command.RedirectStderrTo(b.logger.Stderr()) + if err := command.Start(); err != nil { + return nil, errors.New(tr("Error while determining sketch size: %s", err)) + } + if err := command.Wait(); err != nil { return nil, errors.New(tr("Error while determining sketch size: %s", err)) } @@ -100,7 +105,7 @@ func (b *Builder) checkSizeAdvanced() (ExecutablesFileSections, error) { } var resp AdvancedSizerResponse - if err := json.Unmarshal(out, &resp); err != nil { + if err := json.Unmarshal(out.Bytes(), &resp); err != nil { return nil, errors.New(tr("Error while determining sketch size: %s", err)) } @@ -204,20 +209,27 @@ func (b *Builder) checkSize() (ExecutablesFileSections, error) { } func (b *Builder) execSizeRecipe(properties *properties.Map) (textSize int, dataSize int, eepromSize int, resErr error) { - command, err := utils.PrepareCommandForRecipe(properties, "recipe.size.pattern", false) + command, err := b.prepareCommandForRecipe(properties, "recipe.size.pattern", false) if err != nil { resErr = fmt.Errorf(tr("Error while determining sketch size: %s"), err) return } - - verboseInfo, out, _, err := utils.ExecCommand(b.logger.Verbose(), b.logger.Stdout(), b.logger.Stderr(), command, utils.Capture /* stdout */, utils.Show /* stderr */) if b.logger.Verbose() { - b.logger.Info(string(verboseInfo)) + b.logger.Info(utils.PrintableCommand(command.GetArgs())) } - if err != nil { + commandStdout := &bytes.Buffer{} + command.RedirectStdoutTo(commandStdout) + command.RedirectStderrTo(b.logger.Stderr()) + if err := command.Start(); err != nil { resErr = fmt.Errorf(tr("Error while determining sketch size: %s"), err) return } + if err := command.Wait(); err != nil { + resErr = fmt.Errorf(tr("Error while determining sketch size: %s"), err) + return + } + + out := commandStdout.Bytes() // force multiline match prepending "(?m)" to the actual regexp // return an error if RECIPE_SIZE_REGEXP doesn't exist diff --git a/arduino/builder/sketch.go b/arduino/builder/sketch.go index f94b1b6bf16..3a7c21f3ded 100644 --- a/arduino/builder/sketch.go +++ b/arduino/builder/sketch.go @@ -24,7 +24,6 @@ import ( "strings" "github.com/arduino/arduino-cli/arduino/builder/cpp" - "github.com/arduino/arduino-cli/arduino/builder/internal/utils" "github.com/arduino/arduino-cli/i18n" f "github.com/arduino/arduino-cli/internal/algorithms" "github.com/arduino/go-paths-helper" @@ -174,21 +173,18 @@ func writeIfDifferent(source []byte, destPath *paths.Path) error { return nil } -// BuildSketch fixdoc -func (b *Builder) BuildSketch(includesFolders paths.PathList) error { +// buildSketch fixdoc +func (b *Builder) buildSketch(includesFolders paths.PathList) error { includes := f.Map(includesFolders.AsStrings(), cpp.WrapWithHyphenI) if err := b.sketchBuildPath.MkdirAll(); err != nil { return errors.WithStack(err) } - sketchObjectFiles, err := utils.CompileFiles( - b.sketchBuildPath, b.sketchBuildPath, b.buildProperties, includes, - b.onlyUpdateCompilationDatabase, - b.compilationDatabase, - b.jobs, - b.logger, - b.Progress, + sketchObjectFiles, err := b.compileFiles( + b.sketchBuildPath, b.sketchBuildPath, + false, /** recursive **/ + includes, ) if err != nil { return errors.WithStack(err) @@ -197,13 +193,10 @@ func (b *Builder) BuildSketch(includesFolders paths.PathList) error { // The "src/" subdirectory of a sketch is compiled recursively sketchSrcPath := b.sketchBuildPath.Join("src") if sketchSrcPath.IsDir() { - srcObjectFiles, err := utils.CompileFilesRecursive( - sketchSrcPath, sketchSrcPath, b.buildProperties, includes, - b.onlyUpdateCompilationDatabase, - b.compilationDatabase, - b.jobs, - b.logger, - b.Progress, + srcObjectFiles, err := b.compileFiles( + sketchSrcPath, sketchSrcPath, + true, /** recursive **/ + includes, ) if err != nil { return errors.WithStack(err) @@ -215,8 +208,8 @@ func (b *Builder) BuildSketch(includesFolders paths.PathList) error { return nil } -// MergeSketchWithBootloader fixdoc -func (b *Builder) MergeSketchWithBootloader() error { +// mergeSketchWithBootloader fixdoc +func (b *Builder) mergeSketchWithBootloader() error { if b.onlyUpdateCompilationDatabase { return nil }