Use a general approach to access custom/static/builtin assets (#24022)

The idea is to use a Layered Asset File-system (modules/assetfs/layered.go)

For example: when there are 2 layers: "custom", "builtin", when access
to asset "my/page.tmpl", the Layered Asset File-system will first try to
use "custom" assets, if not found, then use "builtin" assets.

This approach will hugely simplify a lot of code, make them testable.

Other changes:

* Simplify the AssetsHandlerFunc code
* Simplify the `gitea embedded` sub-command code

---------

Co-authored-by: Jason Song <[email protected]>
Co-authored-by: Lunny Xiao <[email protected]>

cmd/embedded.go

@@ -1,18 +1,16 @@
 // Copyright 2020 The Gitea Authors. All rights reserved.
 // SPDX-License-Identifier: MIT
 
-//go:build bindata
-
 package cmd
 
 import (
 	"errors"
 	"fmt"
 	"os"
 	"path/filepath"
-	"sort"
 	"strings"
 
+	"code.gitea.io/gitea/modules/assetfs"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/options"
 	"code.gitea.io/gitea/modules/public"
@@ -89,24 +87,20 @@ var (
 		},
 	}
 
-	sections map[string]*section
-	assets   []asset
+	matchedAssetFiles []assetFile
 )
 
-type section struct {
-	Path  string
-	Names func() []string
-	IsDir func(string) (bool, error)
-	Asset func(string) ([]byte, error)
-}
-
-type asset struct {
-	Section *section
-	Name    string
-	Path    string
+type assetFile struct {
+	fs   *assetfs.LayeredFS
+	name string
+	path string
 }
 
 func initEmbeddedExtractor(c *cli.Context) error {
+	// FIXME: there is a bug, if the user runs `gitea embedded` with a different user or root,
+	// The setting.Init (loadRunModeFrom) will fail and do log.Fatal
+	// But the console logger has been deleted, so nothing is printed, the user sees nothing and Gitea just exits.
+
 	// Silence the console logger
 	log.DelNamedLogger("console")
 	log.DelNamedLogger(log.DEFAULT)
@@ -115,24 +109,14 @@ func initEmbeddedExtractor(c *cli.Context) error {
 	setting.InitProviderAllowEmpty()
 	setting.LoadCommonSettings()
 
-	pats, err := getPatterns(c.Args())
+	patterns, err := compileCollectPatterns(c.Args())
 	if err != nil {
 		return err
 	}
-	sections := make(map[string]*section, 3)
-
-	sections["public"] = &section{Path: "public", Names: public.AssetNames, IsDir: public.AssetIsDir, Asset: public.Asset}
-	sections["options"] = &section{Path: "options", Names: options.AssetNames, IsDir: options.AssetIsDir, Asset: options.Asset}
-	sections["templates"] = &section{Path: "templates", Names: templates.BuiltinAssetNames, IsDir: templates.BuiltinAssetIsDir, Asset: templates.BuiltinAsset}
 
-	for _, sec := range sections {
-		assets = append(assets, buildAssetList(sec, pats, c)...)
-	}
-
-	// Sort assets
-	sort.SliceStable(assets, func(i, j int) bool {
-		return assets[i].Path < assets[j].Path
-	})
+	collectAssetFilesByPattern(c, patterns, "options", options.BuiltinAssets())
+	collectAssetFilesByPattern(c, patterns, "public", public.BuiltinAssets())
+	collectAssetFilesByPattern(c, patterns, "templates", templates.BuiltinAssets())
 
 	return nil
 }
@@ -166,8 +150,8 @@ func runListDo(c *cli.Context) error {
 		return err
 	}
 
-	for _, a := range assets {
-		fmt.Println(a.Path)
+	for _, a := range matchedAssetFiles {
+		fmt.Println(a.path)
 	}
 
 	return nil
@@ -178,19 +162,19 @@ func runViewDo(c *cli.Context) error {
 		return err
 	}
 
-	if len(assets) == 0 {
-		return fmt.Errorf("No files matched the given pattern")
-	} else if len(assets) > 1 {
-		return fmt.Errorf("Too many files matched the given pattern; try to be more specific")
+	if len(matchedAssetFiles) == 0 {
+		return fmt.Errorf("no files matched the given pattern")
+	} else if len(matchedAssetFiles) > 1 {
+		return fmt.Errorf("too many files matched the given pattern, try to be more specific")
 	}
 
-	data, err := assets[0].Section.Asset(assets[0].Name)
+	data, err := matchedAssetFiles[0].fs.ReadFile(matchedAssetFiles[0].name)
 	if err != nil {
-		return fmt.Errorf("%s: %w", assets[0].Path, err)
+		return fmt.Errorf("%s: %w", matchedAssetFiles[0].path, err)
 	}
 
 	if _, err = os.Stdout.Write(data); err != nil {
-		return fmt.Errorf("%s: %w", assets[0].Path, err)
+		return fmt.Errorf("%s: %w", matchedAssetFiles[0].path, err)
 	}
 
 	return nil
@@ -202,7 +186,7 @@ func runExtractDo(c *cli.Context) error {
 	}
 
 	if len(c.Args()) == 0 {
-		return fmt.Errorf("A list of pattern of files to extract is mandatory (e.g. '**' for all)")
+		return fmt.Errorf("a list of pattern of files to extract is mandatory (e.g. '**' for all)")
 	}
 
 	destdir := "."
@@ -227,31 +211,31 @@ func runExtractDo(c *cli.Context) error {
 	if err != nil {
 		return fmt.Errorf("%s: %s", destdir, err)
 	} else if !fi.IsDir() {
-		return fmt.Errorf("%s is not a directory.", destdir)
+		return fmt.Errorf("destination %q is not a directory", destdir)
 	}
 
 	fmt.Printf("Extracting to %s:\n", destdir)
 
 	overwrite := c.Bool("overwrite")
 	rename := c.Bool("rename")
 
-	for _, a := range assets {
+	for _, a := range matchedAssetFiles {
 		if err := extractAsset(destdir, a, overwrite, rename); err != nil {
 			// Non-fatal error
-			fmt.Fprintf(os.Stderr, "%s: %v", a.Path, err)
+			fmt.Fprintf(os.Stderr, "%s: %v", a.path, err)
 		}
 	}
 
 	return nil
 }
 
-func extractAsset(d string, a asset, overwrite, rename bool) error {
-	dest := filepath.Join(d, filepath.FromSlash(a.Path))
+func extractAsset(d string, a assetFile, overwrite, rename bool) error {
+	dest := filepath.Join(d, filepath.FromSlash(a.path))
 	dir := filepath.Dir(dest)
 
-	data, err := a.Section.Asset(a.Name)
+	data, err := a.fs.ReadFile(a.name)
 	if err != nil {
-		return fmt.Errorf("%s: %w", a.Path, err)
+		return fmt.Errorf("%s: %w", a.path, err)
 	}
 
 	if err := os.MkdirAll(dir, os.ModePerm); err != nil {
@@ -272,7 +256,7 @@ func extractAsset(d string, a asset, overwrite, rename bool) error {
 		return fmt.Errorf("%s already exists, but it's not a regular file", dest)
 	} else if rename {
 		if err := util.Rename(dest, dest+".bak"); err != nil {
-			return fmt.Errorf("Error creating backup for %s: %w", dest, err)
+			return fmt.Errorf("error creating backup for %s: %w", dest, err)
 		}
 		// Attempt to respect file permissions mask (even if user:group will be set anew)
 		perms = fi.Mode()
@@ -293,40 +277,38 @@ func extractAsset(d string, a asset, overwrite, rename bool) error {
 	return nil
 }
 
-func buildAssetList(sec *section, globs []glob.Glob, c *cli.Context) []asset {
-	results := make([]asset, 0, 64)
-	for _, name := range sec.Names() {
-		if isdir, err := sec.IsDir(name); !isdir && err == nil {
-			if sec.Path == "public" &&
-				strings.HasPrefix(name, "vendor/") &&
-				!c.Bool("include-vendored") {
-				continue
-			}
-			matchName := sec.Path + "/" + name
-			for _, g := range globs {
-				if g.Match(matchName) {
-					results = append(results, asset{
-						Section: sec,
-						Name:    name,
-						Path:    sec.Path + "/" + name,
-					})
-					break
-				}
+func collectAssetFilesByPattern(c *cli.Context, globs []glob.Glob, path string, layer *assetfs.Layer) {
+	fs := assetfs.Layered(layer)
+	files, err := fs.ListAllFiles(".", true)
+	if err != nil {
+		log.Error("Error listing files in %q: %v", path, err)
+		return
+	}
+	for _, name := range files {
+		if path == "public" &&
+			strings.HasPrefix(name, "vendor/") &&
+			!c.Bool("include-vendored") {
+			continue
+		}
+		matchName := path + "/" + name
+		for _, g := range globs {
+			if g.Match(matchName) {
+				matchedAssetFiles = append(matchedAssetFiles, assetFile{fs: fs, name: name, path: path + "/" + name})
+				break
 			}
 		}
 	}
-	return results
 }
 
-func getPatterns(args []string) ([]glob.Glob, error) {
+func compileCollectPatterns(args []string) ([]glob.Glob, error) {
 	if len(args) == 0 {
 		args = []string{"**"}
 	}
 	pat := make([]glob.Glob, len(args))
 	for i := range args {
 		if g, err := glob.Compile(args[i], '/'); err != nil {
 			return nil, fmt.Errorf("'%s': Invalid glob pattern: %w", args[i], err)
-		} else {
+		} else { //nolint:revive
 			pat[i] = g
 		}
 	}