[Ingest Manager] New agent structure (symlinks) (elastic#20400)

[Ingest Manager] New agent structure (symlinks) (elastic#20400)

dev-tools/mage/dmgbuilder.go

@@ -109,6 +109,11 @@ func (b *dmgBuilder) buildBeatPkg() error {
 
 	// Copy files into the packaging root and set their mode.
 	for _, f := range b.Files {
+		if f.Symlink {
+			// not supported, handling symlink in post/pre install scripts
+			continue
+		}
+
 		target := filepath.Join(beatPkgRoot, f.Target)
 		if err := Copy(f.Source, target); err != nil {
 			if f.SkipOnMissing && errors.Is(err, os.ErrNotExist) {

dev-tools/mage/pkgtypes.go

@@ -108,6 +108,7 @@ type PackageFile struct {
 	Dep           func(PackageSpec) error `yaml:"-" hash:"-" json:"-"`       // Dependency to invoke during Evaluate.
 	Owner         string                  `yaml:"owner,omitempty"`           // File Owner, for user and group name (rpm only).
 	SkipOnMissing bool                    `yaml:"skip_on_missing,omitempty"` // Prevents build failure if the file is missing.
+	Symlink       bool                    `yaml:"symlink"`                   // Symlink marks file as a symlink pointing from target to source.
 }
 
 // OSArchNames defines the names of architectures for use in packages.
@@ -476,6 +477,10 @@ func copyInstallScript(spec PackageSpec, script string, local *string) error {
 		*local = strings.TrimSuffix(*local, ".tmpl")
 	}
 
+	if strings.HasSuffix(*local, "."+spec.Name) {
+		*local = strings.TrimSuffix(*local, "."+spec.Name)
+	}
+
 	if err := spec.ExpandFile(script, createDir(*local)); err != nil {
 		return errors.Wrap(err, "failed to copy install script to package dir")
 	}
@@ -539,6 +544,11 @@ func PackageZip(spec PackageSpec) error {
 
 	// Add files to zip.
 	for _, pkgFile := range spec.Files {
+		if pkgFile.Symlink {
+			// not supported on zip archives
+			continue
+		}
+
 		if err := addFileToZip(w, baseDir, pkgFile); err != nil {
 			p, _ := filepath.Abs(pkgFile.Source)
 			return errors.Wrapf(err, "failed adding file=%+v to zip", p)
@@ -584,11 +594,32 @@ func PackageTarGz(spec PackageSpec) error {
 
 	// Add files to tar.
 	for _, pkgFile := range spec.Files {
+		if pkgFile.Symlink {
+			continue
+		}
+
 		if err := addFileToTar(w, baseDir, pkgFile); err != nil {
 			return errors.Wrapf(err, "failed adding file=%+v to tar", pkgFile)
 		}
 	}
 
+	// same for symlinks so they can point to files in tar
+	for _, pkgFile := range spec.Files {
+		if !pkgFile.Symlink {
+			continue
+		}
+
+		tmpdir, err := ioutil.TempDir("", "TmpSymlinkDropPath")
+		if err != nil {
+			return err
+		}
+		defer os.RemoveAll(tmpdir)
+
+		if err := addSymlinkToTar(tmpdir, w, baseDir, pkgFile); err != nil {
+			return errors.Wrapf(err, "failed adding file=%+v to tar", pkgFile)
+		}
+	}
+
 	if err := w.Close(); err != nil {
 		return err
 	}
@@ -882,6 +913,56 @@ func addFileToTar(ar *tar.Writer, baseDir string, pkgFile PackageFile) error {
 	})
 }
 
+// addSymlinkToTar adds a symlink file  to a tar archive.
+func addSymlinkToTar(tmpdir string, ar *tar.Writer, baseDir string, pkgFile PackageFile) error {
+	// create symlink we can work with later, header will be updated later
+	link := filepath.Join(tmpdir, "link")
+	target := tmpdir
+	if err := os.Symlink(target, link); err != nil {
+		return err
+	}
+
+	return filepath.Walk(link, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			if pkgFile.SkipOnMissing && os.IsNotExist(err) {
+				return nil
+			}
+
+			return err
+		}
+
+		header, err := tar.FileInfoHeader(info, info.Name())
+		if err != nil {
+			return err
+		}
+		header.Uname, header.Gname = "root", "root"
+		header.Uid, header.Gid = 0, 0
+
+		if info.Mode().IsRegular() && pkgFile.Mode > 0 {
+			header.Mode = int64(pkgFile.Mode & os.ModePerm)
+		} else if info.IsDir() {
+			header.Mode = int64(0755)
+		}
+
+		header.Name = filepath.Join(baseDir, pkgFile.Target)
+		if filepath.IsAbs(pkgFile.Target) {
+			header.Name = pkgFile.Target
+		}
+
+		header.Linkname = pkgFile.Source
+		header.Typeflag = tar.TypeSymlink
+
+		if mg.Verbose() {
+			log.Println("Adding", os.FileMode(header.Mode), header.Name)
+		}
+		if err := ar.WriteHeader(header); err != nil {
+			return err
+		}
+
+		return nil
+	})
+}
+
 // PackageDMG packages the Beat into a .dmg file containing an installer pkg
 // and uninstaller app.
 func PackageDMG(spec PackageSpec) error {

dev-tools/mage/settings.go

@@ -85,6 +85,7 @@ var (
 		"beat_doc_branch":   BeatDocBranch,
 		"beat_version":      BeatQualifiedVersion,
 		"commit":            CommitHash,
+		"commit_short":      CommitHashShort,
 		"date":              BuildDate,
 		"elastic_beats_dir": ElasticBeatsDir,
 		"go_version":        GoVersion,
@@ -239,6 +240,15 @@ func CommitHash() (string, error) {
 	return commitHash, err
 }
 
+// CommitHashShort returns the short length git commit hash.
+func CommitHashShort() (string, error) {
+	shortHash, err := CommitHash()
+	if len(shortHash) > 6 {
+		shortHash = shortHash[:6]
+	}
+	return shortHash, err
+}
+
 var (
 	elasticBeatsDirValue string
 	elasticBeatsDirErr   error