Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add support for .azdignore #4146

Closed
wants to merge 14 commits into from
10 changes: 7 additions & 3 deletions .devcontainer/devcontainer.json
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
"ghcr.io/devcontainers/features/go:1": {
"version": "1.21"
},
"ghcr.io/guiyomh/features/golangci-lint:0":{},
"ghcr.io/guiyomh/features/golangci-lint:0": {},
"ghcr.io/devcontainers/features/docker-in-docker:2.11.0": {
"version": "latest",
"moby": true
Expand All @@ -19,7 +19,6 @@
"ghcr.io/devcontainers/features/git:1": {},
"ghcr.io/devcontainers/features/github-cli:1": {},
"ghcr.io/devcontainers/features/terraform:1": {}

},
"customizations": {
"vscode": {
Expand All @@ -34,5 +33,10 @@
]
}
},
"postCreateCommand": "go install gotest.tools/gotestsum@latest"
"postCreateCommand": "go install gotest.tools/gotestsum@latest",
"runArgs": [
"--cap-add=SYS_PTRACE",
"--security-opt",
"seccomp=unconfined"
]
}
3 changes: 3 additions & 0 deletions .vscode/cspell.global.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ ignoreWords:
- azurewebsites
- azcore
- Azdo
- azdignore
- azdrelease
- aztfmod
- azurecaf
Expand All @@ -59,6 +60,7 @@ ignoreWords:
- dapr
- databricks
- dedb
- denormal
- devcenter
- devcontainer
- dnsz
Expand All @@ -71,6 +73,7 @@ ignoreWords:
- fdfp
- fics
- Frontdoor
- gitcli
- golobby
- graphsdk
- hndl
Expand Down
2 changes: 1 addition & 1 deletion cli/azd/.vscode/launch.json
Original file line number Diff line number Diff line change
Expand Up @@ -53,4 +53,4 @@
"default": "auth login --use-device-code"
}
]
}
}
13 changes: 11 additions & 2 deletions cli/azd/internal/repository/initializer.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (
"path/filepath"
"strings"

"github.com/azure/azure-dev/cli/azd/pkg/azdignore"
"github.com/azure/azure-dev/cli/azd/pkg/environment"
"github.com/azure/azure-dev/cli/azd/pkg/environment/azdcontext"
"github.com/azure/azure-dev/cli/azd/pkg/input"
Expand Down Expand Up @@ -63,7 +64,6 @@ func (i *Initializer) Initialize(
defer i.console.StopSpinner(ctx, stepMessage+"\n", input.GetStepResultFormat(err))

staging, err := os.MkdirTemp("", "az-dev-template")

if err != nil {
return fmt.Errorf("creating temp folder: %w", err)
}
Expand All @@ -86,6 +86,16 @@ func (i *Initializer) Initialize(
return err
}

ignoreMatcher, err := azdignore.ReadIgnoreFiles(staging)
if err != nil && !os.IsNotExist(err) {
return fmt.Errorf("reading .azdignore file: %w", err)
}

err = azdignore.RemoveIgnoredFiles(staging, ignoreMatcher)
if err != nil {
return fmt.Errorf("removing ignored files: %w", err)
}
Comment on lines +89 to +97
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Feels like the RemoveIgnoredFiles should internally determine the files to ignore from a file path instead of having to read them and pass them in.


skipStagingFiles, err := i.promptForDuplicates(ctx, staging, target)
if err != nil {
return err
Expand All @@ -102,7 +112,6 @@ func (i *Initializer) Initialize(
if _, shouldSkip := skipStagingFiles[src]; shouldSkip {
return true, nil
}

return false, nil
}
}
Expand Down
120 changes: 120 additions & 0 deletions cli/azd/pkg/azdignore/azdignore.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,120 @@
package azdignore

import (
"fmt"
"os"
"path/filepath"
"strings"

"github.com/denormal/go-gitignore"
)

// ReadIgnoreFiles reads all .azdignore files in the directory hierarchy, from the projectDir upwards,
// and returns a slice of gitignore.GitIgnore structures.
func ReadIgnoreFiles(projectDir string) ([]gitignore.GitIgnore, error) {
var ignoreMatchers []gitignore.GitIgnore

// Traverse upwards from the projectDir to the root directory
currentDir := projectDir
for {
ignoreFilePath := filepath.Join(currentDir, ".azdignore")
if _, err := os.Stat(ignoreFilePath); !os.IsNotExist(err) {
ignoreMatcher, err := gitignore.NewFromFile(ignoreFilePath)
if err != nil {
return nil, fmt.Errorf("error reading .azdignore file at %s: %w", ignoreFilePath, err)
}
ignoreMatchers = append([]gitignore.GitIgnore{ignoreMatcher}, ignoreMatchers...)
}

// Stop if we've reached the root directory
parentDir := filepath.Dir(currentDir)
if parentDir == currentDir {
break
}
currentDir = parentDir
}

return ignoreMatchers, nil
}

// ShouldIgnore checks if a file or directory should be ignored based on a slice of gitignore.GitIgnore structures.
func ShouldIgnore(path string, isDir bool, ignoreMatchers []gitignore.GitIgnore) bool {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nit: Consider not exporting this function (make lowercase) if its only used internally within the package.

for _, matcher := range ignoreMatchers {
match := matcher.Relative(path, isDir)
if match != nil && match.Ignore() {
return true
}
}
return false
}

// RemoveIgnoredFiles removes files and directories based on .azdignore rules using a pre-collected list of paths.
func RemoveIgnoredFiles(staging string, ignoreMatchers []gitignore.GitIgnore) error {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: consider renaming this to stagingPath to be more clear that it is a file path.

Suggested change
func RemoveIgnoredFiles(staging string, ignoreMatchers []gitignore.GitIgnore) error {
func RemoveIgnoredFiles(stagingPath string, ignoreMatchers []gitignore.GitIgnore) error {

if len(ignoreMatchers) == 0 {
return nil // No .azdignore files, no files to ignore
}

// Collect all file and directory paths
paths, err := CollectFilePaths(staging)
if err != nil {
return fmt.Errorf("collecting file paths: %w", err)
}

// Map to store directories that should be ignored, preventing their children from being processed
ignoredDirs := make(map[string]struct{})

// Iterate through collected paths and determine which to remove
for _, path := range paths {
relativePath, err := filepath.Rel(staging, path)
if err != nil {
return err
}

// Skip processing if the path is within an ignored directory
skip := false
for ignoredDir := range ignoredDirs {
if strings.HasPrefix(relativePath, ignoredDir) {
skip = true
break
}
}
if skip {
continue
}

isDir := false
info, err := os.Lstat(path)
if err == nil {
isDir = info.IsDir()
}

// Check if the file should be ignored
if ShouldIgnore(relativePath, isDir, ignoreMatchers) {
if isDir {
ignoredDirs[relativePath] = struct{}{}
if err := os.RemoveAll(path); err != nil {
return fmt.Errorf("removing directory %s: %w", path, err)
}
} else {
if err := os.Remove(path); err != nil {
return fmt.Errorf("removing file %s: %w", path, err)
}
}
}
}

return nil
}

// CollectFilePaths collects all file and directory paths under the given root directory.
func CollectFilePaths(root string) ([]string, error) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: Same as previous comment about not exporting function.

var paths []string
err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
paths = append(paths, path)
return nil
})
return paths, err
}
51 changes: 44 additions & 7 deletions cli/azd/pkg/project/project_utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"path/filepath"
"time"

"github.com/azure/azure-dev/cli/azd/pkg/azdignore"
"github.com/azure/azure-dev/cli/azd/pkg/rzip"
"github.com/otiai10/copy"
)
Expand All @@ -23,15 +24,23 @@ func createDeployableZip(projectName string, appName string, path string) (strin
return "", fmt.Errorf("failed when creating zip package to deploy %s: %w", appName, err)
}

if err := rzip.CreateFromDirectory(path, zipFile); err != nil {
// if we fail here just do our best to close things out and cleanup
// Read and honor the .azdignore files
ignoreMatchers, err := azdignore.ReadIgnoreFiles(path)
if err != nil && !os.IsNotExist(err) {
return "", fmt.Errorf("reading .azdignore files: %w", err)
}

// Create the zip file, excluding files that match the .azdignore rules
err = rzip.CreateFromDirectoryWithIgnore(path, zipFile, ignoreMatchers)
if err != nil {
// If we fail here, just do our best to close things out and cleanup
zipFile.Close()
os.Remove(zipFile.Name())
return "", err
}

if err := zipFile.Close(); err != nil {
// may fail but, again, we'll do our best to cleanup here.
// May fail, but again, we'll do our best to cleanup here.
os.Remove(zipFile.Name())
return "", err
}
Expand All @@ -48,12 +57,40 @@ type buildForZipOptions struct {
excludeConditions []excludeDirEntryCondition
}

// buildForZip is use by projects which build strategy is to only copy the source code into a folder which is later
// zipped for packaging. For example Python and Node framework languages. buildForZipOptions provides the specific
// details for each language which should not be ever copied.
// buildForZip is used by projects whose build strategy is to only copy the source code into a folder, which is later
// zipped for packaging. buildForZipOptions provides the specific details for each language regarding which files should
// not be copied.
func buildForZip(src, dst string, options buildForZipOptions) error {
// Add a global exclude condition for the .azdignore file
ignoreMatchers, err := azdignore.ReadIgnoreFiles(src)
if err != nil && !os.IsNotExist(err) {
return fmt.Errorf("reading .azdignore files: %w", err)
}

options.excludeConditions = append(options.excludeConditions, func(path string, file os.FileInfo) bool {
if len(ignoreMatchers) > 0 {
// Get the relative path from the source directory
relativePath, err := filepath.Rel(src, path)
if err != nil {
return false
}

// Check if the relative path should be ignored
isDir := file.IsDir()
if azdignore.ShouldIgnore(relativePath, isDir, ignoreMatchers) {
return true
}
}

// Always exclude .azdignore files
if filepath.Base(path) == ".azdignore" {
return true
}
Comment on lines +86 to +88
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Did we ever land on whether we were going to call this .zipignore or not? Or look into if app service has any existing behavior we could snap to?


return false
})

// these exclude conditions applies to all projects
// These exclude conditions apply to all projects
options.excludeConditions = append(options.excludeConditions, globalExcludeAzdFolder)

return copy.Copy(src, dst, copy.Options{
Expand Down
83 changes: 83 additions & 0 deletions cli/azd/pkg/rzip/rzip.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@ import (
"os"
"path/filepath"
"strings"

"github.com/denormal/go-gitignore"
)

func CreateFromDirectory(source string, buf *os.File) error {
Expand Down Expand Up @@ -56,3 +58,84 @@ func CreateFromDirectory(source string, buf *os.File) error {

return w.Close()
}

// CreateFromDirectoryWithIgnore creates a zip archive from the contents of a directory, excluding files
// that match any of the provided ignore rules.
func CreateFromDirectoryWithIgnore(srcDir string, writer io.Writer, ignoreMatchers []gitignore.GitIgnore) error {
zipWriter := zip.NewWriter(writer)
defer zipWriter.Close()

err := filepath.Walk(srcDir, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}

// Get the relative path of the file to ensure the root directory isn't included in the zip file
relativePath, err := filepath.Rel(srcDir, path)
if err != nil {
return err
}

// Skip the root directory itself
if relativePath == "." {
return nil
}

// Check if the file should be ignored based on the ignore matchers
isDir := info.IsDir()
if shouldIgnore(relativePath, isDir, ignoreMatchers) {
if isDir {
// If a directory should be ignored, skip its contents as well
return filepath.SkipDir
}
// Otherwise, just skip the file
return nil
}

// Add the file or directory to the zip archive
header, err := zip.FileInfoHeader(info)
if err != nil {
return err
}
header.Name = relativePath

// Ensure directories are properly handled
if isDir {
header.Name += "/"
} else {
header.Method = zip.Deflate
}

writer, err := zipWriter.CreateHeader(header)
if err != nil {
return err
}

if !isDir {
file, err := os.Open(path)
if err != nil {
return err
}
defer file.Close()

_, err = io.Copy(writer, file)
if err != nil {
return err
}
}

return nil
})

return err
}

// shouldIgnore determines whether a file or directory should be ignored based on the provided ignore matchers.
func shouldIgnore(relativePath string, isDir bool, ignoreMatchers []gitignore.GitIgnore) bool {
for _, matcher := range ignoreMatchers {
if match := matcher.Relative(relativePath, isDir); match != nil && match.Ignore() {
return true
}
}
return false
}
Loading
Loading