From 9e447e0cddddf746aad5b907438e6996ef675efc Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Sun, 9 Apr 2023 12:41:10 +0800 Subject: [PATCH 01/19] refactor --- cmd/embedded.go | 69 ++++---- cmd/embedded_stub.go | 29 ---- modules/assetfs/layered.go | 269 +++++++++++++++++++++++++++++ modules/auth/pam/pam_stub.go | 6 +- modules/context/context.go | 12 +- modules/doctor/paths.go | 3 +- modules/options/base.go | 118 ++----------- modules/options/dynamic.go | 25 +-- modules/options/options.go | 45 ----- modules/options/static.go | 95 +--------- modules/public/public.go | 78 ++++++--- modules/public/serve_dynamic.go | 15 +- modules/public/serve_static.go | 74 +------- modules/repository/init.go | 2 +- modules/setting/asset_dynamic.go | 8 + modules/setting/asset_static.go | 8 + modules/templates/base.go | 93 ++-------- modules/templates/dynamic.go | 72 +------- modules/templates/htmlrenderer.go | 62 ++----- modules/templates/mailer.go | 63 ++----- modules/templates/static.go | 113 +----------- modules/translation/translation.go | 14 +- modules/util/path.go | 19 +- modules/util/path_test.go | 2 +- modules/util/timer.go | 26 +++ modules/watcher/watcher.go | 114 ------------ routers/install/routes.go | 6 +- routers/web/devtest/devtest.go | 15 +- routers/web/web.go | 2 - 29 files changed, 512 insertions(+), 945 deletions(-) delete mode 100644 cmd/embedded_stub.go create mode 100644 modules/assetfs/layered.go delete mode 100644 modules/options/options.go create mode 100644 modules/setting/asset_dynamic.go create mode 100644 modules/setting/asset_static.go delete mode 100644 modules/watcher/watcher.go diff --git a/cmd/embedded.go b/cmd/embedded.go index d87fc0187c70d..4a10465f9a572 100644 --- a/cmd/embedded.go +++ b/cmd/embedded.go @@ -1,8 +1,6 @@ // Copyright 2020 The Gitea Authors. All rights reserved. // SPDX-License-Identifier: MIT -//go:build bindata - package cmd import ( @@ -13,6 +11,7 @@ import ( "sort" "strings" + "code.gitea.io/gitea/modules/assetfs" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/options" "code.gitea.io/gitea/modules/public" @@ -89,15 +88,12 @@ var ( }, } - sections map[string]*section - assets []asset + assets []asset ) type section struct { - Path string - Names func() []string - IsDir func(string) (bool, error) - Asset func(string) ([]byte, error) + Path string + AssetFS *assetfs.LayeredFS } type asset struct { @@ -121,9 +117,9 @@ func initEmbeddedExtractor(c *cli.Context) error { } sections := make(map[string]*section, 3) - sections["public"] = §ion{Path: "public", Names: public.AssetNames, IsDir: public.AssetIsDir, Asset: public.Asset} - sections["options"] = §ion{Path: "options", Names: options.AssetNames, IsDir: options.AssetIsDir, Asset: options.Asset} - sections["templates"] = §ion{Path: "templates", Names: templates.BuiltinAssetNames, IsDir: templates.BuiltinAssetIsDir, Asset: templates.BuiltinAsset} + sections["public"] = §ion{Path: "public", AssetFS: assetfs.Layered(public.BuiltinAssets())} + sections["options"] = §ion{Path: "options", AssetFS: assetfs.Layered(options.BuiltinAssets())} + sections["templates"] = §ion{Path: "templates", AssetFS: assetfs.Layered(templates.BuiltinAssets())} for _, sec := range sections { assets = append(assets, buildAssetList(sec, pats, c)...) @@ -178,13 +174,7 @@ func runViewDo(c *cli.Context) error { return err } - if len(assets) == 0 { - return fmt.Errorf("No files matched the given pattern") - } else if len(assets) > 1 { - return fmt.Errorf("Too many files matched the given pattern; try to be more specific") - } - - data, err := assets[0].Section.Asset(assets[0].Name) + data, err := assets[0].Section.AssetFS.ReadFile(assets[0].Name) if err != nil { return fmt.Errorf("%s: %w", assets[0].Path, err) } @@ -227,7 +217,7 @@ func runExtractDo(c *cli.Context) error { if err != nil { return fmt.Errorf("%s: %s", destdir, err) } else if !fi.IsDir() { - return fmt.Errorf("%s is not a directory.", destdir) + return fmt.Errorf("destination %q is not a directory", destdir) } fmt.Printf("Extracting to %s:\n", destdir) @@ -249,7 +239,7 @@ func extractAsset(d string, a asset, overwrite, rename bool) error { dest := filepath.Join(d, filepath.FromSlash(a.Path)) dir := filepath.Dir(dest) - data, err := a.Section.Asset(a.Name) + data, err := a.Section.AssetFS.ReadFile(a.Name) if err != nil { return fmt.Errorf("%s: %w", a.Path, err) } @@ -295,23 +285,26 @@ func extractAsset(d string, a asset, overwrite, rename bool) error { func buildAssetList(sec *section, globs []glob.Glob, c *cli.Context) []asset { results := make([]asset, 0, 64) - for _, name := range sec.Names() { - if isdir, err := sec.IsDir(name); !isdir && err == nil { - if sec.Path == "public" && - strings.HasPrefix(name, "vendor/") && - !c.Bool("include-vendored") { - continue - } - matchName := sec.Path + "/" + name - for _, g := range globs { - if g.Match(matchName) { - results = append(results, asset{ - Section: sec, - Name: name, - Path: sec.Path + "/" + name, - }) - break - } + files, err := sec.AssetFS.ListFiles(".", true) + if err != nil { + log.Error("Error listing files in %q: %v", sec.Path, err) + return results + } + for _, name := range files { + if sec.Path == "public" && + strings.HasPrefix(name, "vendor/") && + !c.Bool("include-vendored") { + continue + } + matchName := sec.Path + "/" + name + for _, g := range globs { + if g.Match(matchName) { + results = append(results, asset{ + Section: sec, + Name: name, + Path: sec.Path + "/" + name, + }) + break } } } @@ -326,7 +319,7 @@ func getPatterns(args []string) ([]glob.Glob, error) { for i := range args { if g, err := glob.Compile(args[i], '/'); err != nil { return nil, fmt.Errorf("'%s': Invalid glob pattern: %w", args[i], err) - } else { + } else { //nolint:revive pat[i] = g } } diff --git a/cmd/embedded_stub.go b/cmd/embedded_stub.go deleted file mode 100644 index 874df06f9d7fc..0000000000000 --- a/cmd/embedded_stub.go +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright 2020 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -//go:build !bindata - -package cmd - -import ( - "fmt" - "os" - - "github.com/urfave/cli" -) - -// Cmdembedded represents the available extract sub-command. -var ( - Cmdembedded = cli.Command{ - Name: "embedded", - Usage: "Extract embedded resources", - Description: "A command for extracting embedded resources, like templates and images", - Action: extractorNotImplemented, - } -) - -func extractorNotImplemented(c *cli.Context) error { - err := fmt.Errorf("Sorry: the 'embedded' subcommand is not available in builds without bindata") - fmt.Fprintf(os.Stderr, "%s\n", err) - return err -} diff --git a/modules/assetfs/layered.go b/modules/assetfs/layered.go new file mode 100644 index 0000000000000..0b5d0f817feba --- /dev/null +++ b/modules/assetfs/layered.go @@ -0,0 +1,269 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package assetfs + +import ( + "context" + "errors" + "fmt" + "io" + "io/fs" + "net/http" + "os" + "path/filepath" + "sort" + "time" + + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/process" + "code.gitea.io/gitea/modules/util" + + "github.com/fsnotify/fsnotify" +) + +// Layer represents a layer in a layered asset file-system. It has a name and works like http.FileSystem +type Layer struct { + name string + fs http.FileSystem + localPath string +} + +func (l *Layer) Name() string { + return l.name +} + +// Open opens the named file. The caller is responsible for closing the file. +func (l *Layer) Open(name string) (http.File, error) { + return l.fs.Open(name) +} + +// Local returns a new Layer with the given name, it serves files from the given local path. +func Local(name, base string, sub ...string) *Layer { + // TODO: the old behavior (StaticRootPath might not be absolute), not ideal, just keep the same as before + // Ideally, the caller should guarantee the base is absolute, guessing a relative path based on the current working directory is unreliable. + base, err := filepath.Abs(base) + if err != nil { + // This should never happen in a real system. If it happens, the user must have already been in trouble: the system is not able to resolve its own paths. + panic(fmt.Sprintf("Unable to get absolute path for %q: %v", base, err)) + } + root := util.FilePathJoinAbs(base, sub...) + return &Layer{name: name, fs: http.Dir(root), localPath: root} +} + +// Bindata returns a new Layer with the given name, it serves files from the given bindata asset. +func Bindata(name string, fs http.FileSystem) *Layer { + return &Layer{name: name, fs: fs} +} + +// LayeredFS is a layered asset file-system. It works like http.FileSystem, but it can have multiple layers. +// The first layer is the top layer, and it will be used first. +// If the file is not found in the top layer, it will be searched in the next layer. +type LayeredFS struct { + layers []*Layer +} + +// Layered returns a new LayeredFS with the given layers. The first layer is the top layer. +func Layered(layers ...*Layer) *LayeredFS { + return &LayeredFS{layers: layers} +} + +// Open opens the named file. The caller is responsible for closing the file. +func (l *LayeredFS) Open(name string) (http.File, error) { + for _, layer := range l.layers { + f, err := layer.Open(name) + if err == nil || !os.IsNotExist(err) { + return f, err + } + } + return nil, fs.ErrNotExist +} + +// ReadFile reads the named file. +func (l *LayeredFS) ReadFile(elems ...string) ([]byte, error) { + bs, _, err := l.ReadLayeredFile(elems...) + return bs, err +} + +// ReadLayeredFile reads the named file, and returns the layer name. +func (l *LayeredFS) ReadLayeredFile(elems ...string) ([]byte, string, error) { + name := util.PathJoinRel(elems...) + for _, layer := range l.layers { + f, err := layer.Open(name) + if os.IsNotExist(err) { + continue + } else if err != nil { + return nil, layer.name, err + } + bs, err := io.ReadAll(f) + _ = f.Close() + return bs, layer.name, err + } + return nil, "", fs.ErrNotExist +} + +// ListFiles lists files/directories in the given directory. The fileMode controls the returned files. +// * omitted: all files and directories will be returned. +// * true: only files will be returned. +// * false: only directories will be returned. +func (l *LayeredFS) ListFiles(name string, fileMode ...bool) ([]string, error) { + fileMap := map[string]bool{} + for _, layer := range l.layers { + f, err := layer.Open(name) + if os.IsNotExist(err) { + continue + } else if err != nil { + return nil, err + } + infos, err := f.Readdir(-1) + if err != nil { + _ = f.Close() + return nil, err + } + for _, info := range infos { + include := false + if len(fileMode) == 0 { + include = true + } else if len(fileMode) == 1 && fileMode[0] == !info.Mode().IsDir() { + include = true + } else if len(fileMode) > 1 { + return nil, errors.New("too many arguments") + } + if include { + fileMap[info.Name()] = true + } + } + _ = f.Close() + } + var files []string + for file := range fileMap { + files = append(files, file) + } + sort.Strings(files) + return files, nil +} + +// ListAllFiles returns files/directories in the given directory, including subdirectories, recursively. +// The fileMode controls the returned files: +// * omitted: all files and directories will be returned. +// * true: only files will be returned. +// * false: only directories will be returned. +func (l *LayeredFS) ListAllFiles(name string, fileMode ...bool) ([]string, error) { + return l.listAllFiles(l.layers, name, fileMode...) +} + +func (l *LayeredFS) listAllFiles(layers []*Layer, name string, fileMode ...bool) ([]string, error) { + fileMap := map[string]bool{} + var list func(dir string) error + list = func(dir string) error { + for _, layer := range layers { + f, err := layer.Open(dir) + if os.IsNotExist(err) { + continue + } else if err != nil { + return err + } + infos, err := f.Readdir(-1) + if err != nil { + _ = f.Close() + return err + } + _ = f.Close() + for _, info := range infos { + include := false + if len(fileMode) == 0 { + include = true + } else if len(fileMode) == 1 && fileMode[0] == !info.Mode().IsDir() { + include = true + } else if len(fileMode) > 1 { + return errors.New("too many arguments") + } + path := util.PathJoinRelX(dir, info.Name()) + if include { + fileMap[path] = true + } + if info.IsDir() { + if err = list(path); err != nil { + return err + } + } + } + } + return nil + } + if err := list(name); err != nil { + return nil, err + } + var files []string + for file := range fileMap { + files = append(files, file) + } + sort.Strings(files) + return files, nil +} + +// WatchLocalChanges watches local changes in the file-system. It's used to help to reload assets when the local file-system changes. +func (l *LayeredFS) WatchLocalChanges(ctx context.Context, callback func()) { + ctx, _, finished := process.GetManager().AddTypedContext(ctx, "Asset Local FileSystem Watcher", process.SystemProcessType, true) + defer finished() + + watcher, err := fsnotify.NewWatcher() + if err != nil { + log.Error("Unable to create watcher for asset local file-system: %v", err) + return + } + defer watcher.Close() + + for _, layer := range l.layers { + if layer.localPath == "" { + continue + } + layerDirs, err := l.listAllFiles([]*Layer{layer}, ".", false) + if err != nil { + log.Error("Unable to list directories for asset local file-system %q: %v", layer.localPath, err) + continue + } + for _, dir := range layerDirs { + if err = watcher.Add(util.FilePathJoinAbs(layer.localPath, dir)); err != nil { + log.Error("Unable to watch directory %s: %v", dir, err) + } + } + } + + debounce := util.Debounce(100 * time.Millisecond) + + for { + select { + case <-ctx.Done(): + return + case event, ok := <-watcher.Events: + if !ok { + return + } + log.Debug("Watched asset local file-system had event: %v", event) + debounce(callback) + case err, ok := <-watcher.Errors: + if !ok { + _ = watcher.Close() + return + } + log.Error("Watched asset local file-system had error: %v", err) + } + } +} + +// GetFileLayerName returns the name of the first-seen layer that contains the given file. +func (l *LayeredFS) GetFileLayerName(elems ...string) string { + name := util.PathJoinRel(elems...) + for _, layer := range l.layers { + f, err := layer.Open(name) + if os.IsNotExist(err) { + continue + } else if err != nil { + return "" + } + _ = f.Close() + return layer.name + } + return "" +} diff --git a/modules/auth/pam/pam_stub.go b/modules/auth/pam/pam_stub.go index a48e89860edb0..3631eeeda7b5a 100644 --- a/modules/auth/pam/pam_stub.go +++ b/modules/auth/pam/pam_stub.go @@ -14,5 +14,9 @@ var Supported = false // Auth not supported lack of pam tag func Auth(serviceName, userName, passwd string) (string, error) { - return "", errors.New("PAM not supported") + // bypass the lint on callers: SA4023: this comparison is always true (staticcheck) + if !Supported { + return "", errors.New("PAM not supported") + } + return "", nil } diff --git a/modules/context/context.go b/modules/context/context.go index bd561be0f5326..e2e120ba384ad 100644 --- a/modules/context/context.go +++ b/modules/context/context.go @@ -240,19 +240,15 @@ func (ctx *Context) HTML(status int, name base.TplName) { } line, _ := strconv.Atoi(lineStr) // Cannot error out as groups[2] is [1-9][0-9]* pos, _ := strconv.Atoi(posStr) // Cannot error out as groups[3] is [1-9][0-9]* - filename, filenameErr := templates.GetAssetFilename("templates/" + errorTemplateName + ".tmpl") - if filenameErr != nil { - filename = "(template) " + errorTemplateName - } + assetLayerName := templates.AssetFS().GetFileLayerName(errorTemplateName + ".tmpl") + filename := fmt.Sprintf("(%s) %s", assetLayerName, errorTemplateName) if errorTemplateName != string(name) { filename += " (subtemplate of " + string(name) + ")" } err = fmt.Errorf("failed to render %s, error: %w:\n%s", filename, err, templates.GetLineFromTemplate(errorTemplateName, line, target, pos)) } else { - filename, filenameErr := templates.GetAssetFilename("templates/" + execErr.Name + ".tmpl") - if filenameErr != nil { - filename = "(template) " + execErr.Name - } + assetLayerName := templates.AssetFS().GetFileLayerName(execErr.Name + ".tmpl") + filename := fmt.Sprintf("(%s) %s", assetLayerName, execErr.Name) if execErr.Name != string(name) { filename += " (subtemplate of " + string(name) + ")" } diff --git a/modules/doctor/paths.go b/modules/doctor/paths.go index 7b1c6ce9ad1c8..d7bf2539e760a 100644 --- a/modules/doctor/paths.go +++ b/modules/doctor/paths.go @@ -9,7 +9,6 @@ import ( "os" "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/options" "code.gitea.io/gitea/modules/setting" ) @@ -79,7 +78,7 @@ func checkConfigurationFiles(ctx context.Context, logger log.Logger, autofix boo {"Log Root Path", setting.Log.RootPath, true, true, true}, } - if options.IsDynamic() { + if !setting.HasBuiltinBindata { configurationFiles = append(configurationFiles, configurationFile{"Static File Root Path", setting.StaticRootPath, true, true, false}) } diff --git a/modules/options/base.go b/modules/options/base.go index 7882ed008159a..6c6e3839f462d 100644 --- a/modules/options/base.go +++ b/modules/options/base.go @@ -4,131 +4,39 @@ package options import ( - "fmt" - "io/fs" - "os" - "path/filepath" - - "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/assetfs" "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/modules/util" ) -var directories = make(directorySet) +func CustomAssets() *assetfs.Layer { + return assetfs.Local("custom", setting.CustomPath, "options") +} + +func AssetFS() *assetfs.LayeredFS { + return assetfs.Layered(CustomAssets(), BuiltinAssets()) +} // Locale reads the content of a specific locale from static/bindata or custom path. func Locale(name string) ([]byte, error) { - return fileFromOptionsDir("locale", name) + return AssetFS().ReadFile("locale", name) } // Readme reads the content of a specific readme from static/bindata or custom path. func Readme(name string) ([]byte, error) { - return fileFromOptionsDir("readme", name) + return AssetFS().ReadFile("readme", name) } // Gitignore reads the content of a gitignore locale from static/bindata or custom path. func Gitignore(name string) ([]byte, error) { - return fileFromOptionsDir("gitignore", name) + return AssetFS().ReadFile("gitignore", name) } // License reads the content of a specific license from static/bindata or custom path. func License(name string) ([]byte, error) { - return fileFromOptionsDir("license", name) + return AssetFS().ReadFile("license", name) } // Labels reads the content of a specific labels from static/bindata or custom path. func Labels(name string) ([]byte, error) { - return fileFromOptionsDir("label", name) -} - -// WalkLocales reads the content of a specific locale -func WalkLocales(callback func(path, name string, d fs.DirEntry, err error) error) error { - if IsDynamic() { - if err := walkAssetDir(filepath.Join(setting.StaticRootPath, "options", "locale"), callback); err != nil && !os.IsNotExist(err) { - return fmt.Errorf("failed to walk locales. Error: %w", err) - } - } - - if err := walkAssetDir(filepath.Join(setting.CustomPath, "options", "locale"), callback); err != nil && !os.IsNotExist(err) { - return fmt.Errorf("failed to walk locales. Error: %w", err) - } - return nil -} - -func walkAssetDir(root string, callback func(path, name string, d fs.DirEntry, err error) error) error { - if err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { - // name is the path relative to the root - name := path[len(root):] - if len(name) > 0 && name[0] == '/' { - name = name[1:] - } - if err != nil { - if os.IsNotExist(err) { - return callback(path, name, d, err) - } - return err - } - if util.CommonSkip(d.Name()) { - if d.IsDir() { - return fs.SkipDir - } - return nil - } - return callback(path, name, d, err) - }); err != nil && !os.IsNotExist(err) { - return fmt.Errorf("unable to get files for assets in %s: %w", root, err) - } - return nil -} - -// mustLocalPathAbs coverts a path to absolute path -// FIXME: the old behavior (StaticRootPath might not be absolute), not ideal, just keep the same as before -func mustLocalPathAbs(s string) string { - abs, err := filepath.Abs(s) - if err != nil { - // This should never happen in a real system. If it happens, the user must have already been in trouble: the system is not able to resolve its own paths. - log.Fatal("Unable to get absolute path for %q: %v", s, err) - } - return abs -} - -func joinLocalPaths(baseDirs []string, subDir string, elems ...string) (paths []string) { - abs := make([]string, len(elems)+2) - abs[1] = subDir - copy(abs[2:], elems) - for _, baseDir := range baseDirs { - abs[0] = mustLocalPathAbs(baseDir) - paths = append(paths, util.FilePathJoinAbs(abs...)) - } - return paths -} - -func listLocalDirIfExist(baseDirs []string, subDir string, elems ...string) (files []string, err error) { - for _, localPath := range joinLocalPaths(baseDirs, subDir, elems...) { - isDir, err := util.IsDir(localPath) - if err != nil { - return nil, fmt.Errorf("unable to check if path %q is a directory. %w", localPath, err) - } else if !isDir { - continue - } - - dirFiles, err := util.StatDir(localPath, true) - if err != nil { - return nil, fmt.Errorf("unable to read directory %q. %w", localPath, err) - } - files = append(files, dirFiles...) - } - return files, nil -} - -func readLocalFile(baseDirs []string, subDir string, elems ...string) ([]byte, error) { - for _, localPath := range joinLocalPaths(baseDirs, subDir, elems...) { - data, err := os.ReadFile(localPath) - if err == nil { - return data, nil - } else if !os.IsNotExist(err) { - log.Error("Unable to read file %q. Error: %v", localPath, err) - } - } - return nil, os.ErrNotExist + return AssetFS().ReadFile("label", name) } diff --git a/modules/options/dynamic.go b/modules/options/dynamic.go index 3d6261983f2de..085492d11c876 100644 --- a/modules/options/dynamic.go +++ b/modules/options/dynamic.go @@ -6,29 +6,10 @@ package options import ( + "code.gitea.io/gitea/modules/assetfs" "code.gitea.io/gitea/modules/setting" ) -// Dir returns all files from static or custom directory. -func Dir(name string) ([]string, error) { - if directories.Filled(name) { - return directories.Get(name), nil - } - - result, err := listLocalDirIfExist([]string{setting.CustomPath, setting.StaticRootPath}, "options", name) - if err != nil { - return nil, err - } - - return directories.AddAndGet(name, result), nil -} - -// fileFromOptionsDir is a helper to read files from custom or static path. -func fileFromOptionsDir(elems ...string) ([]byte, error) { - return readLocalFile([]string{setting.CustomPath, setting.StaticRootPath}, "options", elems...) -} - -// IsDynamic will return false when using embedded data (-tags bindata) -func IsDynamic() bool { - return true +func BuiltinAssets() *assetfs.Layer { + return assetfs.Local("builtin(static)", setting.StaticRootPath, "options") } diff --git a/modules/options/options.go b/modules/options/options.go deleted file mode 100644 index 17a8fa482e42e..0000000000000 --- a/modules/options/options.go +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright 2016 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package options - -type directorySet map[string][]string - -func (s directorySet) Add(key string, value []string) { - _, ok := s[key] - - if !ok { - s[key] = make([]string, 0, len(value)) - } - - s[key] = append(s[key], value...) -} - -func (s directorySet) Get(key string) []string { - _, ok := s[key] - - if ok { - result := []string{} - seen := map[string]string{} - - for _, val := range s[key] { - if _, ok := seen[val]; !ok { - result = append(result, val) - seen[val] = val - } - } - - return result - } - - return []string{} -} - -func (s directorySet) AddAndGet(key string, value []string) []string { - s.Add(key, value) - return s.Get(key) -} - -func (s directorySet) Filled(key string) bool { - return len(s[key]) > 0 -} diff --git a/modules/options/static.go b/modules/options/static.go index 0482dea6817ce..72b28e990e777 100644 --- a/modules/options/static.go +++ b/modules/options/static.go @@ -6,98 +6,9 @@ package options import ( - "fmt" - "io" - - "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/modules/assetfs" ) -// Dir returns all files from custom directory or bindata. -func Dir(name string) ([]string, error) { - if directories.Filled(name) { - return directories.Get(name), nil - } - - result, err := listLocalDirIfExist([]string{setting.CustomPath}, "options", name) - if err != nil { - return nil, err - } - - files, err := AssetDir(name) - if err != nil { - return []string{}, fmt.Errorf("unable to read embedded directory %q. %w", name, err) - } - - result = append(result, files...) - return directories.AddAndGet(name, result), nil -} - -func AssetDir(dirName string) ([]string, error) { - d, err := Assets.Open(dirName) - if err != nil { - return nil, err - } - defer d.Close() - - files, err := d.Readdir(-1) - if err != nil { - return nil, err - } - results := make([]string, 0, len(files)) - for _, file := range files { - results = append(results, file.Name()) - } - return results, nil -} - -// fileFromOptionsDir is a helper to read files from custom path or bindata. -func fileFromOptionsDir(elems ...string) ([]byte, error) { - // only try custom dir, no static dir - if data, err := readLocalFile([]string{setting.CustomPath}, "options", elems...); err == nil { - return data, nil - } - - f, err := Assets.Open(util.PathJoinRelX(elems...)) - if err != nil { - return nil, err - } - defer f.Close() - return io.ReadAll(f) -} - -func Asset(name string) ([]byte, error) { - f, err := Assets.Open("/" + name) - if err != nil { - return nil, err - } - defer f.Close() - return io.ReadAll(f) -} - -func AssetNames() []string { - realFS := Assets.(vfsgen۰FS) - results := make([]string, 0, len(realFS)) - for k := range realFS { - results = append(results, k[1:]) - } - return results -} - -func AssetIsDir(name string) (bool, error) { - if f, err := Assets.Open("/" + name); err != nil { - return false, err - } else { - defer f.Close() - if fi, err := f.Stat(); err != nil { - return false, err - } else { - return fi.IsDir(), nil - } - } -} - -// IsDynamic will return false when using embedded data (-tags bindata) -func IsDynamic() bool { - return false +func BuiltinAssets() *assetfs.Layer { + return assetfs.Bindata("builtin(bindata)", Assets) } diff --git a/modules/public/public.go b/modules/public/public.go index 2c96cf9e763b9..b74b1dcf20616 100644 --- a/modules/public/public.go +++ b/modules/public/public.go @@ -4,11 +4,15 @@ package public import ( + "bytes" + "io" "net/http" "os" - "path/filepath" + "path" "strings" + "time" + "code.gitea.io/gitea/modules/assetfs" "code.gitea.io/gitea/modules/container" "code.gitea.io/gitea/modules/httpcache" "code.gitea.io/gitea/modules/log" @@ -18,28 +22,36 @@ import ( // Options represents the available options to configure the handler. type Options struct { - Directory string - Prefix string CorsHandler func(http.Handler) http.Handler } // AssetsURLPathPrefix is the path prefix for static asset files const AssetsURLPathPrefix = "/assets/" +func CustomAssets() *assetfs.Layer { + return assetfs.Local("custom", setting.CustomPath, "public") +} + +func StaticAssets() *assetfs.Layer { + return assetfs.Local("static", setting.StaticRootPath, "public") +} + // AssetsHandlerFunc implements the static handler for serving custom or original assets. func AssetsHandlerFunc(opts *Options) http.HandlerFunc { - custPath := filepath.Join(setting.CustomPath, "public") - if !filepath.IsAbs(custPath) { - custPath = filepath.Join(setting.AppWorkPath, custPath) - } - if !filepath.IsAbs(opts.Directory) { - opts.Directory = filepath.Join(setting.AppWorkPath, opts.Directory) - } - if !strings.HasSuffix(opts.Prefix, "/") { - opts.Prefix += "/" + var assetFS http.FileSystem + if setting.HasBuiltinBindata { + assetFS = assetfs.Layered(CustomAssets(), StaticAssets(), BuiltinAssets()) // old behavior: always include StaticAssets + } else { + assetFS = assetfs.Layered(CustomAssets(), BuiltinAssets()) // now BuiltinAssets is StaticAssets } return func(resp http.ResponseWriter, req *http.Request) { + path := req.URL.Path + if !strings.HasPrefix(path, AssetsURLPathPrefix) { + return + } + path = strings.TrimPrefix(path, AssetsURLPathPrefix) + if req.Method != "GET" && req.Method != "HEAD" { resp.WriteHeader(http.StatusNotFound) return @@ -56,15 +68,7 @@ func AssetsHandlerFunc(opts *Options) http.HandlerFunc { } } - file := req.URL.Path[len(opts.Prefix):] - - // custom files - if opts.handle(resp, req, http.Dir(custPath), file) { - return - } - - // internal files - if opts.handle(resp, req, fileSystem(opts.Directory), file) { + if opts.handle(resp, req, assetFS, path) { return } @@ -85,7 +89,7 @@ func parseAcceptEncoding(val string) container.Set[string] { // setWellKnownContentType will set the Content-Type if the file is a well-known type. // See the comments of detectWellKnownMimeType func setWellKnownContentType(w http.ResponseWriter, file string) { - mimeType := detectWellKnownMimeType(filepath.Ext(file)) + mimeType := detectWellKnownMimeType(path.Ext(file)) if mimeType != "" { w.Header().Set("Content-Type", mimeType) } @@ -121,8 +125,34 @@ func (opts *Options) handle(w http.ResponseWriter, req *http.Request, fs http.Fi return true } - setWellKnownContentType(w, file) - serveContent(w, req, fi, fi.ModTime(), f) return true } + +type GzipBytesProvider interface { + GzipBytes() []byte +} + +// serveContent serve http content +func serveContent(w http.ResponseWriter, req *http.Request, fi os.FileInfo, modtime time.Time, content io.ReadSeeker) { + setWellKnownContentType(w, fi.Name()) + + encodings := parseAcceptEncoding(req.Header.Get("Accept-Encoding")) + if encodings.Contains("gzip") { + // try to provide gzip content directly from bindata (provided by vfsgen۰CompressedFileInfo) + if compressed, ok := fi.(GzipBytesProvider); ok { + rdGzip := bytes.NewReader(compressed.GzipBytes()) + // all gzipped static files (from bindata) are managed by Gitea, so we can make sure every file has the correct ext name + // then we can get the correct Content-Type, we do not need to do http.DetectContentType on the decompressed data + if w.Header().Get("Content-Type") == "" { + w.Header().Set("Content-Type", "application/octet-stream") + } + w.Header().Set("Content-Encoding", "gzip") + http.ServeContent(w, req, fi.Name(), modtime, rdGzip) + return + } + } + + http.ServeContent(w, req, fi.Name(), modtime, content) + return +} diff --git a/modules/public/serve_dynamic.go b/modules/public/serve_dynamic.go index cd74ee574366c..a668b17c34457 100644 --- a/modules/public/serve_dynamic.go +++ b/modules/public/serve_dynamic.go @@ -6,17 +6,10 @@ package public import ( - "io" - "net/http" - "os" - "time" + "code.gitea.io/gitea/modules/assetfs" + "code.gitea.io/gitea/modules/setting" ) -func fileSystem(dir string) http.FileSystem { - return http.Dir(dir) -} - -// serveContent serve http content -func serveContent(w http.ResponseWriter, req *http.Request, fi os.FileInfo, modtime time.Time, content io.ReadSeeker) { - http.ServeContent(w, req, fi.Name(), modtime, content) +func BuiltinAssets() *assetfs.Layer { + return assetfs.Local("builtin(static)", setting.StaticRootPath, "public") } diff --git a/modules/public/serve_static.go b/modules/public/serve_static.go index e85ca79253f74..b24bb8551c7ec 100644 --- a/modules/public/serve_static.go +++ b/modules/public/serve_static.go @@ -5,76 +5,8 @@ package public -import ( - "bytes" - "io" - "net/http" - "os" - "path/filepath" - "time" +var _ GzipBytesProvider = (*vfsgen۰CompressedFileInfo)(nil) - "code.gitea.io/gitea/modules/timeutil" -) - -// GlobalModTime provide a global mod time for embedded asset files -func GlobalModTime(filename string) time.Time { - return timeutil.GetExecutableModTime() -} - -func fileSystem(dir string) http.FileSystem { - return Assets -} - -func Asset(name string) ([]byte, error) { - f, err := Assets.Open("/" + name) - if err != nil { - return nil, err - } - defer f.Close() - return io.ReadAll(f) -} - -func AssetNames() []string { - realFS := Assets.(vfsgen۰FS) - results := make([]string, 0, len(realFS)) - for k := range realFS { - results = append(results, k[1:]) - } - return results -} - -func AssetIsDir(name string) (bool, error) { - if f, err := Assets.Open("/" + name); err != nil { - return false, err - } else { - defer f.Close() - if fi, err := f.Stat(); err != nil { - return false, err - } else { - return fi.IsDir(), nil - } - } -} - -// serveContent serve http content -func serveContent(w http.ResponseWriter, req *http.Request, fi os.FileInfo, modtime time.Time, content io.ReadSeeker) { - encodings := parseAcceptEncoding(req.Header.Get("Accept-Encoding")) - if encodings.Contains("gzip") { - if cf, ok := fi.(*vfsgen۰CompressedFileInfo); ok { - rdGzip := bytes.NewReader(cf.GzipBytes()) - // all static files are managed by Gitea, so we can make sure every file has the correct ext name - // then we can get the correct Content-Type, we do not need to do http.DetectContentType on the decompressed data - mimeType := detectWellKnownMimeType(filepath.Ext(fi.Name())) - if mimeType == "" { - mimeType = "application/octet-stream" - } - w.Header().Set("Content-Type", mimeType) - w.Header().Set("Content-Encoding", "gzip") - http.ServeContent(w, req, fi.Name(), modtime, rdGzip) - return - } - } - - http.ServeContent(w, req, fi.Name(), modtime, content) - return +func BuiltinAssets() *assetfs.Layer { + return assetfs.Bindata("builtin(bindata)", Assets) } diff --git a/modules/repository/init.go b/modules/repository/init.go index f9a33cd4f68c9..82a55b0ab7101 100644 --- a/modules/repository/init.go +++ b/modules/repository/init.go @@ -47,7 +47,7 @@ func LoadRepoConfig() { types := []string{"gitignore", "license", "readme", "label"} typeFiles := make([][]string, 4) for i, t := range types { - files, err := options.Dir(t) + files, err := options.AssetFS().ListFiles(t, true) if err != nil { log.Fatal("Failed to get %s files: %v", t, err) } diff --git a/modules/setting/asset_dynamic.go b/modules/setting/asset_dynamic.go new file mode 100644 index 0000000000000..2eb28833736b4 --- /dev/null +++ b/modules/setting/asset_dynamic.go @@ -0,0 +1,8 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +//go:build !bindata + +package setting + +const HasBuiltinBindata = false diff --git a/modules/setting/asset_static.go b/modules/setting/asset_static.go new file mode 100644 index 0000000000000..889fca9342415 --- /dev/null +++ b/modules/setting/asset_static.go @@ -0,0 +1,8 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +//go:build bindata + +package setting + +const HasBuiltinBindata = true diff --git a/modules/templates/base.go b/modules/templates/base.go index e0f8350afb11b..e95ce31cfcab0 100644 --- a/modules/templates/base.go +++ b/modules/templates/base.go @@ -4,14 +4,10 @@ package templates import ( - "fmt" - "io/fs" - "os" - "path/filepath" "strings" "time" - "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/assetfs" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" ) @@ -47,81 +43,30 @@ func BaseVars() Vars { } } -func getDirTemplateAssetNames(dir string) []string { - return getDirAssetNames(dir, false) +func AssetFS() *assetfs.LayeredFS { + return assetfs.Layered(CustomAssets(), BuiltinAssets()) } -func getDirAssetNames(dir string, mailer bool) []string { - var tmpls []string - - if mailer { - dir += filepath.Join(dir, "mail") - } - f, err := os.Stat(dir) - if err != nil { - if os.IsNotExist(err) { - return tmpls - } - log.Warn("Unable to check if templates dir %s is a directory. Error: %v", dir, err) - return tmpls - } - if !f.IsDir() { - log.Warn("Templates dir %s is a not directory.", dir) - return tmpls - } +func CustomAssets() *assetfs.Layer { + return assetfs.Local("custom", setting.CustomPath, "templates") +} - files, err := util.StatDir(dir) +func ListWebTemplateAssetNames(assets *assetfs.LayeredFS) ([]string, error) { + files, err := assets.ListAllFiles(".", true) if err != nil { - log.Warn("Failed to read %s templates dir. %v", dir, err) - return tmpls + return nil, err } - - prefix := "templates/" - if mailer { - prefix += "mail/" - } - for _, filePath := range files { - if !mailer && strings.HasPrefix(filePath, "mail/") { - continue - } - - if !strings.HasSuffix(filePath, ".tmpl") { - continue - } - - tmpls = append(tmpls, prefix+filePath) - } - return tmpls + return util.SliceRemoveAllFunc(files, func(file string) bool { + return strings.HasPrefix(file, "mail/") || !strings.HasSuffix(file, ".tmpl") + }), nil } -func walkAssetDir(root string, skipMail bool, callback func(path, name string, d fs.DirEntry, err error) error) error { - mailRoot := filepath.Join(root, "mail") - if err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { - name := path[len(root):] - if len(name) > 0 && name[0] == '/' { - name = name[1:] - } - if err != nil { - if os.IsNotExist(err) { - return callback(path, name, d, err) - } - return err - } - if skipMail && path == mailRoot && d.IsDir() { - return fs.SkipDir - } - if util.CommonSkip(d.Name()) { - if d.IsDir() { - return fs.SkipDir - } - return nil - } - if strings.HasSuffix(d.Name(), ".tmpl") || d.IsDir() { - return callback(path, name, d, err) - } - return nil - }); err != nil && !os.IsNotExist(err) { - return fmt.Errorf("unable to get files for template assets in %s: %w", root, err) +func ListMailTemplateAssetNames(assets *assetfs.LayeredFS) ([]string, error) { + files, err := assets.ListAllFiles(".", true) + if err != nil { + return nil, err } - return nil + return util.SliceRemoveAllFunc(files, func(file string) bool { + return !strings.HasPrefix(file, "mail/") || !strings.HasSuffix(file, ".tmpl") + }), nil } diff --git a/modules/templates/dynamic.go b/modules/templates/dynamic.go index 2f4f542e724e6..e1babd83c9438 100644 --- a/modules/templates/dynamic.go +++ b/modules/templates/dynamic.go @@ -6,76 +6,10 @@ package templates import ( - "io/fs" - "os" - "path/filepath" - + "code.gitea.io/gitea/modules/assetfs" "code.gitea.io/gitea/modules/setting" ) -// GetAsset returns asset content via name -func GetAsset(name string) ([]byte, error) { - bs, err := os.ReadFile(filepath.Join(setting.CustomPath, name)) - if err != nil && !os.IsNotExist(err) { - return nil, err - } else if err == nil { - return bs, nil - } - - return os.ReadFile(filepath.Join(setting.StaticRootPath, name)) -} - -// GetAssetFilename returns the filename of the provided asset -func GetAssetFilename(name string) (string, error) { - filename := filepath.Join(setting.CustomPath, name) - _, err := os.Stat(filename) - if err != nil && !os.IsNotExist(err) { - return filename, err - } else if err == nil { - return filename, nil - } - - filename = filepath.Join(setting.StaticRootPath, name) - _, err = os.Stat(filename) - return filename, err -} - -// walkTemplateFiles calls a callback for each template asset -func walkTemplateFiles(callback func(path, name string, d fs.DirEntry, err error) error) error { - if err := walkAssetDir(filepath.Join(setting.CustomPath, "templates"), true, callback); err != nil && !os.IsNotExist(err) { - return err - } - if err := walkAssetDir(filepath.Join(setting.StaticRootPath, "templates"), true, callback); err != nil && !os.IsNotExist(err) { - return err - } - return nil -} - -// GetTemplateAssetNames returns list of template names -func GetTemplateAssetNames() []string { - tmpls := getDirTemplateAssetNames(filepath.Join(setting.CustomPath, "templates")) - tmpls2 := getDirTemplateAssetNames(filepath.Join(setting.StaticRootPath, "templates")) - return append(tmpls, tmpls2...) -} - -func walkMailerTemplates(callback func(path, name string, d fs.DirEntry, err error) error) error { - if err := walkAssetDir(filepath.Join(setting.StaticRootPath, "templates", "mail"), false, callback); err != nil && !os.IsNotExist(err) { - return err - } - if err := walkAssetDir(filepath.Join(setting.CustomPath, "templates", "mail"), false, callback); err != nil && !os.IsNotExist(err) { - return err - } - return nil -} - -// BuiltinAsset will read the provided asset from the embedded assets -// (This always returns os.ErrNotExist) -func BuiltinAsset(name string) ([]byte, error) { - return nil, os.ErrNotExist -} - -// BuiltinAssetNames returns the names of the embedded assets -// (This always returns nil) -func BuiltinAssetNames() []string { - return nil +func BuiltinAssets() *assetfs.Layer { + return assetfs.Local("builtin(static)", setting.StaticRootPath, "templates") } diff --git a/modules/templates/htmlrenderer.go b/modules/templates/htmlrenderer.go index 4e7b09a9eccec..26dd365e4ca36 100644 --- a/modules/templates/htmlrenderer.go +++ b/modules/templates/htmlrenderer.go @@ -21,7 +21,6 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" - "code.gitea.io/gitea/modules/watcher" ) var ( @@ -66,20 +65,23 @@ func (h *HTMLRender) TemplateLookup(name string) (*template.Template, error) { } func (h *HTMLRender) CompileTemplates() error { - dirPrefix := "templates/" extSuffix := ".tmpl" tmpls := template.New("") - for _, path := range GetTemplateAssetNames() { - if !strings.HasSuffix(path, extSuffix) { + assets := AssetFS() + files, err := ListWebTemplateAssetNames(assets) + if err != nil { + return nil + } + for _, file := range files { + if !strings.HasSuffix(file, extSuffix) { continue } - name := strings.TrimPrefix(path, dirPrefix) - name = strings.TrimSuffix(name, extSuffix) + name := strings.TrimSuffix(file, extSuffix) tmpl := tmpls.New(filepath.ToSlash(name)) for _, fm := range NewFuncMap() { tmpl.Funcs(fm) } - buf, err := GetAsset(path) + buf, err := assets.ReadFile(file) if err != nil { return err } @@ -112,13 +114,10 @@ func HTMLRenderer(ctx context.Context) (context.Context, *HTMLRender) { log.Fatal("HTMLRenderer error: %v", err) } if !setting.IsProd { - watcher.CreateWatcher(ctx, "HTML Templates", &watcher.CreateWatcherOpts{ - PathsCallback: walkTemplateFiles, - BetweenCallback: func() { - if err := renderer.CompileTemplates(); err != nil { - log.Error("Template error: %v\n%s", err, log.Stack(2)) - } - }, + go AssetFS().WatchLocalChanges(ctx, func() { + if err := renderer.CompileTemplates(); err != nil { + log.Error("Template error: %v\n%s", err, log.Stack(2)) + } }) } return context.WithValue(ctx, rendererKey, renderer), renderer @@ -138,14 +137,8 @@ func handleGenericTemplateError(err error) (string, []interface{}) { } templateName, lineNumberStr, message := groups[1], groups[2], groups[3] - - filename, assetErr := GetAssetFilename("templates/" + templateName + ".tmpl") - if assetErr != nil { - return "", nil - } - + filename := fmt.Sprintf("%s (provided by %s)", templateName, AssetFS().GetFileLayerName(templateName+".tmpl")) lineNumber, _ := strconv.Atoi(lineNumberStr) - line := GetLineFromTemplate(templateName, lineNumber, "", -1) return "PANIC: Unable to compile templates!\n%s in template file %s at line %d:\n\n%s\nStacktrace:\n\n%s", []interface{}{message, filename, lineNumber, log.NewColoredValue(line, log.Reset), log.Stack(2)} @@ -158,16 +151,9 @@ func handleNotDefinedPanicError(err error) (string, []interface{}) { } templateName, lineNumberStr, functionName := groups[1], groups[2], groups[3] - functionName, _ = strconv.Unquote(`"` + functionName + `"`) - - filename, assetErr := GetAssetFilename("templates/" + templateName + ".tmpl") - if assetErr != nil { - return "", nil - } - + filename := fmt.Sprintf("%s (provided by %s)", templateName, AssetFS().GetFileLayerName(templateName+".tmpl")) lineNumber, _ := strconv.Atoi(lineNumberStr) - line := GetLineFromTemplate(templateName, lineNumber, functionName, -1) return "PANIC: Unable to compile templates!\nUndefined function %q in template file %s at line %d:\n\n%s", []interface{}{functionName, filename, lineNumber, log.NewColoredValue(line, log.Reset)} @@ -181,14 +167,8 @@ func handleUnexpected(err error) (string, []interface{}) { templateName, lineNumberStr, unexpected := groups[1], groups[2], groups[3] unexpected, _ = strconv.Unquote(`"` + unexpected + `"`) - - filename, assetErr := GetAssetFilename("templates/" + templateName + ".tmpl") - if assetErr != nil { - return "", nil - } - + filename := fmt.Sprintf("%s (provided by %s)", templateName, AssetFS().GetFileLayerName(templateName+".tmpl")) lineNumber, _ := strconv.Atoi(lineNumberStr) - line := GetLineFromTemplate(templateName, lineNumber, unexpected, -1) return "PANIC: Unable to compile templates!\nUnexpected %q in template file %s at line %d:\n\n%s", []interface{}{unexpected, filename, lineNumber, log.NewColoredValue(line, log.Reset)} @@ -201,14 +181,8 @@ func handleExpectedEnd(err error) (string, []interface{}) { } templateName, lineNumberStr, unexpected := groups[1], groups[2], groups[3] - - filename, assetErr := GetAssetFilename("templates/" + templateName + ".tmpl") - if assetErr != nil { - return "", nil - } - + filename := fmt.Sprintf("%s (provided by %s)", templateName, AssetFS().GetFileLayerName(templateName+".tmpl")) lineNumber, _ := strconv.Atoi(lineNumberStr) - line := GetLineFromTemplate(templateName, lineNumber, unexpected, -1) return "PANIC: Unable to compile templates!\nMissing end with unexpected %q in template file %s at line %d:\n\n%s", []interface{}{unexpected, filename, lineNumber, log.NewColoredValue(line, log.Reset)} @@ -218,7 +192,7 @@ const dashSeparator = "--------------------------------------------------------- // GetLineFromTemplate returns a line from a template with some context func GetLineFromTemplate(templateName string, targetLineNum int, target string, position int) string { - bs, err := GetAsset("templates/" + templateName + ".tmpl") + bs, err := AssetFS().ReadFile(templateName + ".tmpl") if err != nil { return fmt.Sprintf("(unable to read template file: %v)", err) } diff --git a/modules/templates/mailer.go b/modules/templates/mailer.go index d0c49e1025bf0..633ecee42d34a 100644 --- a/modules/templates/mailer.go +++ b/modules/templates/mailer.go @@ -6,15 +6,12 @@ package templates import ( "context" "html/template" - "io/fs" - "os" "strings" texttmpl "text/template" "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/modules/watcher" ) // mailSubjectTextFuncMap returns functions for injecting to text templates, it's only used for mail subject @@ -62,54 +59,23 @@ func Mailer(ctx context.Context) (*texttmpl.Template, *template.Template) { bodyTemplates.Funcs(funcs) } + assetFS := AssetFS() refreshTemplates := func() { - for _, assetPath := range BuiltinAssetNames() { - if !strings.HasPrefix(assetPath, "mail/") { - continue - } - - if !strings.HasSuffix(assetPath, ".tmpl") { - continue - } - - content, err := BuiltinAsset(assetPath) - if err != nil { - log.Warn("Failed to read embedded %s template. %v", assetPath, err) - continue - } - - assetName := strings.TrimPrefix(strings.TrimSuffix(assetPath, ".tmpl"), "mail/") - - log.Trace("Adding built-in mailer template for %s", assetName) - buildSubjectBodyTemplate(subjectTemplates, - bodyTemplates, - assetName, - content) + assetPaths, err := ListMailTemplateAssetNames(assetFS) + if err != nil { + log.Error("Failed to list mail templates: %v", err) + return } - if err := walkMailerTemplates(func(path, name string, d fs.DirEntry, err error) error { - if err != nil { - return err - } - if d.IsDir() { - return nil - } - - content, err := os.ReadFile(path) + for _, assetPath := range assetPaths { + content, providerName, err := assetFS.ReadLayeredFile(assetPath) if err != nil { - log.Warn("Failed to read custom %s template. %v", path, err) - return nil + log.Warn("Failed to read mail template %s: %v (by %s)", assetPath, err, providerName) + continue } - - assetName := strings.TrimSuffix(name, ".tmpl") - log.Trace("Adding mailer template for %s from %q", assetName, path) - buildSubjectBodyTemplate(subjectTemplates, - bodyTemplates, - assetName, - content) - return nil - }); err != nil && !os.IsNotExist(err) { - log.Warn("Error whilst walking mailer templates directories. %v", err) + tmplName := strings.TrimPrefix(strings.TrimSuffix(assetPath, ".tmpl"), "mail/") + log.Debug("Adding mail template %s: %s (by %s)", tmplName, assetPath, providerName) + buildSubjectBodyTemplate(subjectTemplates, bodyTemplates, tmplName, content) } } @@ -118,10 +84,7 @@ func Mailer(ctx context.Context) (*texttmpl.Template, *template.Template) { if !setting.IsProd { // Now subjectTemplates and bodyTemplates are both synchronized // thus it is safe to call refresh from a different goroutine - watcher.CreateWatcher(ctx, "Mailer Templates", &watcher.CreateWatcherOpts{ - PathsCallback: walkMailerTemplates, - BetweenCallback: refreshTemplates, - }) + go assetFS.WatchLocalChanges(ctx, refreshTemplates) } return subjectTemplates, bodyTemplates diff --git a/modules/templates/static.go b/modules/templates/static.go index 7ebb327ae6ebd..48a14580622f9 100644 --- a/modules/templates/static.go +++ b/modules/templates/static.go @@ -5,115 +5,6 @@ package templates -import ( - "html/template" - "io" - "io/fs" - "os" - "path" - "path/filepath" - "strings" - texttmpl "text/template" - "time" - - "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/modules/timeutil" -) - -var ( - subjectTemplates = texttmpl.New("") - bodyTemplates = template.New("") -) - -// GlobalModTime provide a global mod time for embedded asset files -func GlobalModTime(filename string) time.Time { - return timeutil.GetExecutableModTime() -} - -// GetAssetFilename returns the filename of the provided asset -func GetAssetFilename(name string) (string, error) { - filename := filepath.Join(setting.CustomPath, name) - _, err := os.Stat(filename) - if err != nil && !os.IsNotExist(err) { - return name, err - } else if err == nil { - return filename, nil - } - return "(builtin) " + name, nil -} - -// GetAsset get a special asset, only for chi -func GetAsset(name string) ([]byte, error) { - bs, err := os.ReadFile(filepath.Join(setting.CustomPath, name)) - if err != nil && !os.IsNotExist(err) { - return nil, err - } else if err == nil { - return bs, nil - } - return BuiltinAsset(strings.TrimPrefix(name, "templates/")) -} - -// GetFiles calls a callback for each template asset -func walkTemplateFiles(callback func(path, name string, d fs.DirEntry, err error) error) error { - if err := walkAssetDir(filepath.Join(setting.CustomPath, "templates"), true, callback); err != nil && !os.IsNotExist(err) { - return err - } - return nil -} - -// GetTemplateAssetNames only for chi -func GetTemplateAssetNames() []string { - realFS := Assets.(vfsgen۰FS) - tmpls := make([]string, 0, len(realFS)) - for k := range realFS { - if strings.HasPrefix(k, "/mail/") { - continue - } - tmpls = append(tmpls, "templates/"+k[1:]) - } - - customDir := path.Join(setting.CustomPath, "templates") - customTmpls := getDirTemplateAssetNames(customDir) - return append(tmpls, customTmpls...) -} - -func walkMailerTemplates(callback func(path, name string, d fs.DirEntry, err error) error) error { - if err := walkAssetDir(filepath.Join(setting.CustomPath, "templates", "mail"), false, callback); err != nil && !os.IsNotExist(err) { - return err - } - return nil -} - -// BuiltinAsset reads the provided asset from the builtin embedded assets -func BuiltinAsset(name string) ([]byte, error) { - f, err := Assets.Open("/" + name) - if err != nil { - return nil, err - } - defer f.Close() - return io.ReadAll(f) -} - -// BuiltinAssetNames returns the names of the built-in embedded assets -func BuiltinAssetNames() []string { - realFS := Assets.(vfsgen۰FS) - results := make([]string, 0, len(realFS)) - for k := range realFS { - results = append(results, k[1:]) - } - return results -} - -// BuiltinAssetIsDir returns if a provided asset is a directory -func BuiltinAssetIsDir(name string) (bool, error) { - if f, err := Assets.Open("/" + name); err != nil { - return false, err - } else { - defer f.Close() - if fi, err := f.Stat(); err != nil { - return false, err - } else { - return fi.IsDir(), nil - } - } +func BuiltinAssets() *assetfs.Layer { + return assetfs.Bindata("builtin(bindata)", Assets) } diff --git a/modules/translation/translation.go b/modules/translation/translation.go index 3165390c3238c..331da0f965c38 100644 --- a/modules/translation/translation.go +++ b/modules/translation/translation.go @@ -13,7 +13,6 @@ import ( "code.gitea.io/gitea/modules/options" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/translation/i18n" - "code.gitea.io/gitea/modules/watcher" "golang.org/x/text/language" ) @@ -58,7 +57,7 @@ func InitLocales(ctx context.Context) { refreshLocales := func() { i18n.ResetDefaultLocales() - localeNames, err := options.Dir("locale") + localeNames, err := options.AssetFS().ListFiles("locale", true) if err != nil { log.Fatal("Failed to list locale files: %v", err) } @@ -118,13 +117,10 @@ func InitLocales(ctx context.Context) { }) if !setting.IsProd { - watcher.CreateWatcher(ctx, "Locales", &watcher.CreateWatcherOpts{ - PathsCallback: options.WalkLocales, - BetweenCallback: func() { - lock.Lock() - defer lock.Unlock() - refreshLocales() - }, + go options.AssetFS().WatchLocalChanges(ctx, func() { + lock.Lock() + defer lock.Unlock() + refreshLocales() }) } } diff --git a/modules/util/path.go b/modules/util/path.go index 37d06e9813e07..1a68bc748802d 100644 --- a/modules/util/path.go +++ b/modules/util/path.go @@ -74,29 +74,28 @@ const pathSeparator = string(os.PathSeparator) // // {`/foo`, ``, `bar`} => `/foo/bar` // {`/foo`, `..`, `bar`} => `/foo/bar` -func FilePathJoinAbs(elem ...string) string { - elems := make([]string, len(elem)) +func FilePathJoinAbs(base string, sub ...string) string { + elems := make([]string, 1, len(sub)+1) - // POISX filesystem can have `\` in file names. Windows: `\` and `/` are both used for path separators + // POSIX filesystem can have `\` in file names. Windows: `\` and `/` are both used for path separators // to keep the behavior consistent, we do not allow `\` in file names, replace all `\` with `/` if isOSWindows() { - elems[0] = filepath.Clean(elem[0]) + elems[0] = filepath.Clean(base) } else { - elems[0] = filepath.Clean(strings.ReplaceAll(elem[0], "\\", pathSeparator)) + elems[0] = filepath.Clean(strings.ReplaceAll(base, "\\", pathSeparator)) } if !filepath.IsAbs(elems[0]) { // This shouldn't happen. If there is really necessary to pass in relative path, return the full path with filepath.Abs() instead panic(fmt.Sprintf("FilePathJoinAbs: %q (for path %v) is not absolute, do not guess a relative path based on current working directory", elems[0], elems)) } - - for i := 1; i < len(elem); i++ { - if elem[i] == "" { + for _, s := range sub { + if s == "" { continue } if isOSWindows() { - elems[i] = filepath.Clean(pathSeparator + elem[i]) + elems = append(elems, filepath.Clean(pathSeparator+s)) } else { - elems[i] = filepath.Clean(pathSeparator + strings.ReplaceAll(elem[i], "\\", pathSeparator)) + elems = append(elems, filepath.Clean(pathSeparator+strings.ReplaceAll(s, "\\", pathSeparator))) } } // the elems[0] must be an absolute path, just join them together diff --git a/modules/util/path_test.go b/modules/util/path_test.go index 1d27c9bf0c0f8..6a38bf4ace863 100644 --- a/modules/util/path_test.go +++ b/modules/util/path_test.go @@ -207,6 +207,6 @@ func TestCleanPath(t *testing.T) { } } for _, c := range cases { - assert.Equal(t, c.expected, FilePathJoinAbs(c.elems...), "case: %v", c.elems) + assert.Equal(t, c.expected, FilePathJoinAbs(c.elems[0], c.elems[1:]...), "case: %v", c.elems) } } diff --git a/modules/util/timer.go b/modules/util/timer.go index daf96bda7e236..629722246de33 100644 --- a/modules/util/timer.go +++ b/modules/util/timer.go @@ -4,6 +4,7 @@ package util import ( + "sync" "time" ) @@ -18,3 +19,28 @@ func StopTimer(t *time.Timer) bool { } return stopped } + +func Debounce(d time.Duration) func(f func()) { + type debouncer struct { + mu sync.Mutex + t *time.Timer + } + db := &debouncer{} + + return func(f func()) { + db.mu.Lock() + defer db.mu.Unlock() + + if db.t != nil { + db.t.Stop() + } + trigger := db.t + db.t = time.AfterFunc(d, func() { + db.mu.Lock() + defer db.mu.Unlock() + if trigger == db.t { + f() + } + }) + } +} diff --git a/modules/watcher/watcher.go b/modules/watcher/watcher.go deleted file mode 100644 index 75d062d7aaaa3..0000000000000 --- a/modules/watcher/watcher.go +++ /dev/null @@ -1,114 +0,0 @@ -// Copyright 2022 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package watcher - -import ( - "context" - "io/fs" - "os" - - "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/process" - - "github.com/fsnotify/fsnotify" -) - -// CreateWatcherOpts are options to configure the watcher -type CreateWatcherOpts struct { - // PathsCallback is used to set the required paths to watch - PathsCallback func(func(path, name string, d fs.DirEntry, err error) error) error - - // BeforeCallback is called before any files are watched - BeforeCallback func() - - // Between Callback is called between after a watched event has occurred - BetweenCallback func() - - // AfterCallback is called as this watcher ends - AfterCallback func() -} - -// CreateWatcher creates a watcher labelled with the provided description and running with the provided options. -// The created watcher will create a subcontext from the provided ctx and register it with the process manager. -func CreateWatcher(ctx context.Context, desc string, opts *CreateWatcherOpts) { - go run(ctx, desc, opts) -} - -func run(ctx context.Context, desc string, opts *CreateWatcherOpts) { - if opts.BeforeCallback != nil { - opts.BeforeCallback() - } - if opts.AfterCallback != nil { - defer opts.AfterCallback() - } - ctx, _, finished := process.GetManager().AddTypedContext(ctx, "Watcher: "+desc, process.SystemProcessType, true) - defer finished() - - log.Trace("Watcher loop starting for %s", desc) - defer log.Trace("Watcher loop ended for %s", desc) - - watcher, err := fsnotify.NewWatcher() - if err != nil { - log.Error("Unable to create watcher for %s: %v", desc, err) - return - } - if err := opts.PathsCallback(func(path, _ string, d fs.DirEntry, err error) error { - if err != nil && !os.IsNotExist(err) { - return err - } - log.Trace("Watcher: %s watching %q", desc, path) - _ = watcher.Add(path) - return nil - }); err != nil { - log.Error("Unable to create watcher for %s: %v", desc, err) - _ = watcher.Close() - return - } - - // Note we don't call the BetweenCallback here - - for { - select { - case event, ok := <-watcher.Events: - if !ok { - _ = watcher.Close() - return - } - log.Debug("Watched file for %s had event: %v", desc, event) - case err, ok := <-watcher.Errors: - if !ok { - _ = watcher.Close() - return - } - log.Error("Error whilst watching files for %s: %v", desc, err) - case <-ctx.Done(): - _ = watcher.Close() - return - } - - // Recreate the watcher - only call the BetweenCallback after the new watcher is set-up - _ = watcher.Close() - watcher, err = fsnotify.NewWatcher() - if err != nil { - log.Error("Unable to create watcher for %s: %v", desc, err) - return - } - if err := opts.PathsCallback(func(path, _ string, _ fs.DirEntry, err error) error { - if err != nil { - return err - } - _ = watcher.Add(path) - return nil - }); err != nil { - log.Error("Unable to create watcher for %s: %v", desc, err) - _ = watcher.Close() - return - } - - // Inform our BetweenCallback that there has been an event - if opts.BetweenCallback != nil { - opts.BetweenCallback() - } - } -} diff --git a/routers/install/routes.go b/routers/install/routes.go index 82d9c34b41f18..6b8222abd239d 100644 --- a/routers/install/routes.go +++ b/routers/install/routes.go @@ -8,7 +8,6 @@ import ( "fmt" "html" "net/http" - "path" "code.gitea.io/gitea/modules/httpcache" "code.gitea.io/gitea/modules/log" @@ -89,10 +88,7 @@ func Routes(ctx goctx.Context) *web.Route { r.Use(middle) } - r.Use(web.WrapWithPrefix(public.AssetsURLPathPrefix, public.AssetsHandlerFunc(&public.Options{ - Directory: path.Join(setting.StaticRootPath, "public"), - Prefix: public.AssetsURLPathPrefix, - }), "InstallAssetsHandler")) + r.Use(web.WrapWithPrefix(public.AssetsURLPathPrefix, public.AssetsHandlerFunc(&public.Options{}), "InstallAssetsHandler")) r.Use(session.Sessioner(session.Options{ Provider: setting.SessionConfig.Provider, diff --git a/routers/web/devtest/devtest.go b/routers/web/devtest/devtest.go index eb77d0b927b6b..784940909a612 100644 --- a/routers/web/devtest/devtest.go +++ b/routers/web/devtest/devtest.go @@ -15,15 +15,16 @@ import ( // List all devtest templates, they will be used for e2e tests for the UI components func List(ctx *context.Context) { - templateNames := templates.GetTemplateAssetNames() + templateNames, err := templates.AssetFS().ListFiles("devtest", true) + if err != nil { + ctx.ServerError("AssetFS().ListFiles", err) + return + } var subNames []string - const prefix = "templates/devtest/" for _, tmplName := range templateNames { - if strings.HasPrefix(tmplName, prefix) { - subName := strings.TrimSuffix(strings.TrimPrefix(tmplName, prefix), ".tmpl") - if subName != "list" { - subNames = append(subNames, subName) - } + subName := strings.TrimSuffix(tmplName, ".tmpl") + if subName != "list" { + subNames = append(subNames, subName) } } ctx.Data["SubNames"] = subNames diff --git a/routers/web/web.go b/routers/web/web.go index cee8bafcdba1a..6b217fc9b1b83 100644 --- a/routers/web/web.go +++ b/routers/web/web.go @@ -104,8 +104,6 @@ func Routes(ctx gocontext.Context) *web.Route { routes := web.NewRoute() routes.Use(web.WrapWithPrefix(public.AssetsURLPathPrefix, public.AssetsHandlerFunc(&public.Options{ - Directory: path.Join(setting.StaticRootPath, "public"), - Prefix: public.AssetsURLPathPrefix, CorsHandler: CorsHandler(), }), "AssetsHandler")) From bbfef58a5f2180fe9c58a639bc853826634c219e Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Sun, 9 Apr 2023 21:55:00 +0800 Subject: [PATCH 02/19] fix var name, fix embedded assets check --- cmd/embedded.go | 8 +++++++- modules/templates/mailer.go | 6 +++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/cmd/embedded.go b/cmd/embedded.go index 4a10465f9a572..9e252c06aaddc 100644 --- a/cmd/embedded.go +++ b/cmd/embedded.go @@ -174,6 +174,12 @@ func runViewDo(c *cli.Context) error { return err } + if len(assets) == 0 { + return fmt.Errorf("no files matched the given pattern") + } else if len(assets) > 1 { + return fmt.Errorf("too many files matched the given pattern, try to be more specific") + } + data, err := assets[0].Section.AssetFS.ReadFile(assets[0].Name) if err != nil { return fmt.Errorf("%s: %w", assets[0].Path, err) @@ -288,7 +294,7 @@ func buildAssetList(sec *section, globs []glob.Glob, c *cli.Context) []asset { files, err := sec.AssetFS.ListFiles(".", true) if err != nil { log.Error("Error listing files in %q: %v", sec.Path, err) - return results + return nil } for _, name := range files { if sec.Path == "public" && diff --git a/modules/templates/mailer.go b/modules/templates/mailer.go index 633ecee42d34a..dc0045beca741 100644 --- a/modules/templates/mailer.go +++ b/modules/templates/mailer.go @@ -68,13 +68,13 @@ func Mailer(ctx context.Context) (*texttmpl.Template, *template.Template) { } for _, assetPath := range assetPaths { - content, providerName, err := assetFS.ReadLayeredFile(assetPath) + content, layerName, err := assetFS.ReadLayeredFile(assetPath) if err != nil { - log.Warn("Failed to read mail template %s: %v (by %s)", assetPath, err, providerName) + log.Warn("Failed to read mail template %s: %v (by %s)", assetPath, err, layerName) continue } tmplName := strings.TrimPrefix(strings.TrimSuffix(assetPath, ".tmpl"), "mail/") - log.Debug("Adding mail template %s: %s (by %s)", tmplName, assetPath, providerName) + log.Debug("Adding mail template %s: %s (by %s)", tmplName, assetPath, layerName) buildSubjectBodyTemplate(subjectTemplates, bodyTemplates, tmplName, content) } } From 716bfd51c6d63c286c8128346f1b611a441289c4 Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Sun, 9 Apr 2023 22:25:35 +0800 Subject: [PATCH 03/19] fix bindata build, fix svg assets loading --- modules/public/public.go | 13 ++++++++----- modules/public/serve_static.go | 12 ++++++++++++ modules/svg/discover_bindata.go | 30 ------------------------------ modules/svg/discover_nobindata.go | 29 ----------------------------- modules/svg/svg.go | 29 ++++++++++++++++++++++------- modules/templates/static.go | 12 ++++++++++++ routers/init.go | 2 +- routers/install/setting.go | 1 - 8 files changed, 55 insertions(+), 73 deletions(-) delete mode 100644 modules/svg/discover_bindata.go delete mode 100644 modules/svg/discover_nobindata.go diff --git a/modules/public/public.go b/modules/public/public.go index b74b1dcf20616..a68e44fa66168 100644 --- a/modules/public/public.go +++ b/modules/public/public.go @@ -36,14 +36,17 @@ func StaticAssets() *assetfs.Layer { return assetfs.Local("static", setting.StaticRootPath, "public") } -// AssetsHandlerFunc implements the static handler for serving custom or original assets. -func AssetsHandlerFunc(opts *Options) http.HandlerFunc { - var assetFS http.FileSystem +func AssetFS() *assetfs.LayeredFS { if setting.HasBuiltinBindata { - assetFS = assetfs.Layered(CustomAssets(), StaticAssets(), BuiltinAssets()) // old behavior: always include StaticAssets + return assetfs.Layered(CustomAssets(), StaticAssets(), BuiltinAssets()) // old behavior: always include StaticAssets } else { - assetFS = assetfs.Layered(CustomAssets(), BuiltinAssets()) // now BuiltinAssets is StaticAssets + return assetfs.Layered(CustomAssets(), BuiltinAssets()) // now BuiltinAssets is StaticAssets } +} + +// AssetsHandlerFunc implements the static handler for serving custom or original assets. +func AssetsHandlerFunc(opts *Options) http.HandlerFunc { + assetFS := AssetFS() return func(resp http.ResponseWriter, req *http.Request) { path := req.URL.Path diff --git a/modules/public/serve_static.go b/modules/public/serve_static.go index b24bb8551c7ec..e79085021eab1 100644 --- a/modules/public/serve_static.go +++ b/modules/public/serve_static.go @@ -5,8 +5,20 @@ package public +import ( + "time" + + "code.gitea.io/gitea/modules/assetfs" + "code.gitea.io/gitea/modules/timeutil" +) + var _ GzipBytesProvider = (*vfsgen۰CompressedFileInfo)(nil) +// GlobalModTime provide a global mod time for embedded asset files +func GlobalModTime(filename string) time.Time { + return timeutil.GetExecutableModTime() +} + func BuiltinAssets() *assetfs.Layer { return assetfs.Bindata("builtin(bindata)", Assets) } diff --git a/modules/svg/discover_bindata.go b/modules/svg/discover_bindata.go deleted file mode 100644 index b6abd294f17d4..0000000000000 --- a/modules/svg/discover_bindata.go +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright 2020 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -//go:build bindata - -package svg - -import ( - "path/filepath" - - "code.gitea.io/gitea/modules/public" -) - -// Discover returns a map of discovered SVG icons in bindata -func Discover() map[string]string { - svgs := make(map[string]string) - - for _, file := range public.AssetNames() { - matched, _ := filepath.Match("img/svg/*.svg", file) - if matched { - content, err := public.Asset(file) - if err == nil { - filename := filepath.Base(file) - svgs[filename[:len(filename)-4]] = string(content) - } - } - } - - return svgs -} diff --git a/modules/svg/discover_nobindata.go b/modules/svg/discover_nobindata.go deleted file mode 100644 index da7ab7b98f5de..0000000000000 --- a/modules/svg/discover_nobindata.go +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright 2020 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -//go:build !bindata - -package svg - -import ( - "os" - "path/filepath" - - "code.gitea.io/gitea/modules/setting" -) - -// Discover returns a map of discovered SVG icons in the file system -func Discover() map[string]string { - svgs := make(map[string]string) - - files, _ := filepath.Glob(filepath.Join(setting.StaticRootPath, "public", "img", "svg", "*.svg")) - for _, file := range files { - content, err := os.ReadFile(file) - if err == nil { - filename := filepath.Base(file) - svgs[filename[:len(filename)-4]] = string(content) - } - } - - return svgs -} diff --git a/modules/svg/svg.go b/modules/svg/svg.go index b74ee3535800d..071340764e049 100644 --- a/modules/svg/svg.go +++ b/modules/svg/svg.go @@ -6,15 +6,18 @@ package svg import ( "fmt" "html/template" + "path" "regexp" "strings" "code.gitea.io/gitea/modules/html" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/public" ) var ( // SVGs contains discovered SVGs - SVGs map[string]string + SVGs = map[string]string{} widthRe = regexp.MustCompile(`width="[0-9]+?"`) heightRe = regexp.MustCompile(`height="[0-9]+?"`) @@ -23,17 +26,29 @@ var ( const defaultSize = 16 // Init discovers SVGs and populates the `SVGs` variable -func Init() { - SVGs = Discover() +func Init() error { + files, err := public.AssetFS().ListFiles("img/svg") + if err != nil { + return err + } // Remove `xmlns` because inline SVG does not need it - r := regexp.MustCompile(`(]*?)\s+xmlns="[^"]*"`) - for name, svg := range SVGs { - SVGs[name] = r.ReplaceAllString(svg, "$1") + reXmlns := regexp.MustCompile(`(]*?)\s+xmlns="[^"]*"`) + for _, file := range files { + if path.Ext(file) != ".svg" { + continue + } + bs, err := public.AssetFS().ReadFile("img/svg", file) + if err != nil { + log.Error("Failed to read SVG file %s: %v", file, err) + } else { + SVGs[file[:len(file)-4]] = reXmlns.ReplaceAllString(string(bs), "$1") + } } + return nil } -// Render render icons - arguments icon name (string), size (int), class (string) +// RenderHTML renders icons - arguments icon name (string), size (int), class (string) func RenderHTML(icon string, others ...interface{}) template.HTML { size, class := html.ParseSizeAndClass(defaultSize, "", others...) diff --git a/modules/templates/static.go b/modules/templates/static.go index 48a14580622f9..b5a7e561ec065 100644 --- a/modules/templates/static.go +++ b/modules/templates/static.go @@ -5,6 +5,18 @@ package templates +import ( + "time" + + "code.gitea.io/gitea/modules/assetfs" + "code.gitea.io/gitea/modules/timeutil" +) + +// GlobalModTime provide a global mod time for embedded asset files +func GlobalModTime(filename string) time.Time { + return timeutil.GetExecutableModTime() +} + func BuiltinAssets() *assetfs.Layer { return assetfs.Bindata("builtin(bindata)", Assets) } diff --git a/routers/init.go b/routers/init.go index 8cf53fc108de0..e143f72e00fb4 100644 --- a/routers/init.go +++ b/routers/init.go @@ -172,7 +172,7 @@ func GlobalInitInstalled(ctx context.Context) { mustInit(ssh.Init) auth.Init() - svg.Init() + mustInit(svg.Init) actions_service.Init() diff --git a/routers/install/setting.go b/routers/install/setting.go index 68984f1e7837d..44806ce46554b 100644 --- a/routers/install/setting.go +++ b/routers/install/setting.go @@ -47,6 +47,5 @@ func reloadSettings(ctx context.Context) { } else { log.Fatal("ORM engine initialization failed: %v", err) } - svg.Init() } } From 1ef9c41dc67359b2a8e0865c58250e96c1f56787 Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Sun, 9 Apr 2023 22:33:14 +0800 Subject: [PATCH 04/19] fine tune --- modules/assetfs/layered.go | 6 ++---- modules/templates/mailer.go | 4 ++-- modules/util/timer.go | 5 +++-- routers/init.go | 7 ------- routers/install/setting.go | 2 +- 5 files changed, 8 insertions(+), 16 deletions(-) diff --git a/modules/assetfs/layered.go b/modules/assetfs/layered.go index 0b5d0f817feba..97576e312a4d5 100644 --- a/modules/assetfs/layered.go +++ b/modules/assetfs/layered.go @@ -116,8 +116,8 @@ func (l *LayeredFS) ListFiles(name string, fileMode ...bool) ([]string, error) { return nil, err } infos, err := f.Readdir(-1) + _ = f.Close() if err != nil { - _ = f.Close() return nil, err } for _, info := range infos { @@ -133,7 +133,6 @@ func (l *LayeredFS) ListFiles(name string, fileMode ...bool) ([]string, error) { fileMap[info.Name()] = true } } - _ = f.Close() } var files []string for file := range fileMap { @@ -164,11 +163,10 @@ func (l *LayeredFS) listAllFiles(layers []*Layer, name string, fileMode ...bool) return err } infos, err := f.Readdir(-1) + _ = f.Close() if err != nil { - _ = f.Close() return err } - _ = f.Close() for _, info := range infos { include := false if len(fileMode) == 0 { diff --git a/modules/templates/mailer.go b/modules/templates/mailer.go index dc0045beca741..280ac0e587e26 100644 --- a/modules/templates/mailer.go +++ b/modules/templates/mailer.go @@ -70,11 +70,11 @@ func Mailer(ctx context.Context) (*texttmpl.Template, *template.Template) { for _, assetPath := range assetPaths { content, layerName, err := assetFS.ReadLayeredFile(assetPath) if err != nil { - log.Warn("Failed to read mail template %s: %v (by %s)", assetPath, err, layerName) + log.Warn("Failed to read mail template %s by %s: %v", assetPath, layerName, err) continue } tmplName := strings.TrimPrefix(strings.TrimSuffix(assetPath, ".tmpl"), "mail/") - log.Debug("Adding mail template %s: %s (by %s)", tmplName, assetPath, layerName) + log.Trace("Adding mail template %s: %s by %s", tmplName, assetPath, layerName) buildSubjectBodyTemplate(subjectTemplates, bodyTemplates, tmplName, content) } } diff --git a/modules/util/timer.go b/modules/util/timer.go index 629722246de33..dc039bc0c9ae8 100644 --- a/modules/util/timer.go +++ b/modules/util/timer.go @@ -34,13 +34,14 @@ func Debounce(d time.Duration) func(f func()) { if db.t != nil { db.t.Stop() } - trigger := db.t - db.t = time.AfterFunc(d, func() { + var trigger *time.Timer + trigger = time.AfterFunc(d, func() { db.mu.Lock() defer db.mu.Unlock() if trigger == db.t { f() } }) + db.t = trigger } } diff --git a/routers/init.go b/routers/init.go index e143f72e00fb4..c539975acadcc 100644 --- a/routers/init.go +++ b/routers/init.go @@ -71,13 +71,6 @@ func mustInitCtx(ctx context.Context, fn func(ctx context.Context) error) { } } -// InitGitServices init new services for git, this is also called in `contrib/pr/checkout.go` -func InitGitServices() { - setting.LoadSettings() - mustInit(storage.Init) - mustInit(repo_service.Init) -} - func syncAppConfForGit(ctx context.Context) error { runtimeState := new(system.RuntimeState) if err := system.AppState.Get(runtimeState); err != nil { diff --git a/routers/install/setting.go b/routers/install/setting.go index 44806ce46554b..dadefa26a2930 100644 --- a/routers/install/setting.go +++ b/routers/install/setting.go @@ -30,7 +30,7 @@ func PreloadSettings(ctx context.Context) bool { } setting.LoadSettingsForInstall() - svg.Init() + _ = svg.Init() } return !setting.InstallLock From 96a818da386283d911a39e02f0067692e42ad97e Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Sun, 9 Apr 2023 23:39:27 +0800 Subject: [PATCH 05/19] fine tune --- modules/assetfs/layered.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/modules/assetfs/layered.go b/modules/assetfs/layered.go index 97576e312a4d5..ef68018749066 100644 --- a/modules/assetfs/layered.go +++ b/modules/assetfs/layered.go @@ -238,11 +238,10 @@ func (l *LayeredFS) WatchLocalChanges(ctx context.Context, callback func()) { if !ok { return } - log.Debug("Watched asset local file-system had event: %v", event) + log.Trace("Watched asset local file-system had event: %v", event) debounce(callback) case err, ok := <-watcher.Errors: if !ok { - _ = watcher.Close() return } log.Error("Watched asset local file-system had error: %v", err) From bb7b7f7828b5b921539feb5f1af132b075c5261a Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Mon, 10 Apr 2023 09:15:35 +0800 Subject: [PATCH 06/19] fix logic for public assets --- modules/public/public.go | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/modules/public/public.go b/modules/public/public.go index a68e44fa66168..a61b064c505ed 100644 --- a/modules/public/public.go +++ b/modules/public/public.go @@ -32,16 +32,8 @@ func CustomAssets() *assetfs.Layer { return assetfs.Local("custom", setting.CustomPath, "public") } -func StaticAssets() *assetfs.Layer { - return assetfs.Local("static", setting.StaticRootPath, "public") -} - func AssetFS() *assetfs.LayeredFS { - if setting.HasBuiltinBindata { - return assetfs.Layered(CustomAssets(), StaticAssets(), BuiltinAssets()) // old behavior: always include StaticAssets - } else { - return assetfs.Layered(CustomAssets(), BuiltinAssets()) // now BuiltinAssets is StaticAssets - } + return assetfs.Layered(CustomAssets(), BuiltinAssets()) } // AssetsHandlerFunc implements the static handler for serving custom or original assets. From 0a44b9cdb9aea759899fe5d071c38d935a102246 Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Mon, 10 Apr 2023 09:20:29 +0800 Subject: [PATCH 07/19] fine tune listAllFiles --- modules/assetfs/layered.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/assetfs/layered.go b/modules/assetfs/layered.go index ef68018749066..868495c248bbe 100644 --- a/modules/assetfs/layered.go +++ b/modules/assetfs/layered.go @@ -148,10 +148,10 @@ func (l *LayeredFS) ListFiles(name string, fileMode ...bool) ([]string, error) { // * true: only files will be returned. // * false: only directories will be returned. func (l *LayeredFS) ListAllFiles(name string, fileMode ...bool) ([]string, error) { - return l.listAllFiles(l.layers, name, fileMode...) + return listAllFiles(l.layers, name, fileMode...) } -func (l *LayeredFS) listAllFiles(layers []*Layer, name string, fileMode ...bool) ([]string, error) { +func listAllFiles(layers []*Layer, name string, fileMode ...bool) ([]string, error) { fileMap := map[string]bool{} var list func(dir string) error list = func(dir string) error { @@ -216,7 +216,7 @@ func (l *LayeredFS) WatchLocalChanges(ctx context.Context, callback func()) { if layer.localPath == "" { continue } - layerDirs, err := l.listAllFiles([]*Layer{layer}, ".", false) + layerDirs, err := listAllFiles([]*Layer{layer}, ".", false) if err != nil { log.Error("Unable to list directories for asset local file-system %q: %v", layer.localPath, err) continue From 5aab370a5a7cc61b022bb1fb3cf6cf8192b8ef73 Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Mon, 10 Apr 2023 15:39:34 +0800 Subject: [PATCH 08/19] simplify AssetsHandlerFunc --- modules/public/public.go | 33 +++++++-------------------------- routers/install/routes.go | 2 +- routers/web/web.go | 4 +--- 3 files changed, 9 insertions(+), 30 deletions(-) diff --git a/modules/public/public.go b/modules/public/public.go index a61b064c505ed..0c0e6dc1cc804 100644 --- a/modules/public/public.go +++ b/modules/public/public.go @@ -20,14 +20,6 @@ import ( "code.gitea.io/gitea/modules/util" ) -// Options represents the available options to configure the handler. -type Options struct { - CorsHandler func(http.Handler) http.Handler -} - -// AssetsURLPathPrefix is the path prefix for static asset files -const AssetsURLPathPrefix = "/assets/" - func CustomAssets() *assetfs.Layer { return assetfs.Local("custom", setting.CustomPath, "public") } @@ -37,33 +29,22 @@ func AssetFS() *assetfs.LayeredFS { } // AssetsHandlerFunc implements the static handler for serving custom or original assets. -func AssetsHandlerFunc(opts *Options) http.HandlerFunc { +func AssetsHandlerFunc(prefix string) http.HandlerFunc { assetFS := AssetFS() - + prefix = strings.TrimSuffix(prefix, "/") + "/" return func(resp http.ResponseWriter, req *http.Request) { - path := req.URL.Path - if !strings.HasPrefix(path, AssetsURLPathPrefix) { + subPath := req.URL.Path + if !strings.HasPrefix(subPath, prefix) { return } - path = strings.TrimPrefix(path, AssetsURLPathPrefix) + subPath = strings.TrimPrefix(subPath, prefix) if req.Method != "GET" && req.Method != "HEAD" { resp.WriteHeader(http.StatusNotFound) return } - if opts.CorsHandler != nil { - var corsSent bool - opts.CorsHandler(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { - corsSent = true - })).ServeHTTP(resp, req) - // If CORS is not sent, the response must have been written by other handlers - if !corsSent { - return - } - } - - if opts.handle(resp, req, assetFS, path) { + if handleRequest(resp, req, assetFS, subPath) { return } @@ -90,7 +71,7 @@ func setWellKnownContentType(w http.ResponseWriter, file string) { } } -func (opts *Options) handle(w http.ResponseWriter, req *http.Request, fs http.FileSystem, file string) bool { +func handleRequest(w http.ResponseWriter, req *http.Request, fs http.FileSystem, file string) bool { // actually, fs (http.FileSystem) is designed to be a safe interface, relative paths won't bypass its parent directory, it's also fine to do a clean here f, err := fs.Open(util.PathJoinRelX(file)) if err != nil { diff --git a/routers/install/routes.go b/routers/install/routes.go index 6b8222abd239d..df82ba2e4c3e7 100644 --- a/routers/install/routes.go +++ b/routers/install/routes.go @@ -88,7 +88,7 @@ func Routes(ctx goctx.Context) *web.Route { r.Use(middle) } - r.Use(web.WrapWithPrefix(public.AssetsURLPathPrefix, public.AssetsHandlerFunc(&public.Options{}), "InstallAssetsHandler")) + r.Use(web.WrapWithPrefix("/assets/", public.AssetsHandlerFunc("/assets/"), "AssetsHandler")) r.Use(session.Sessioner(session.Options{ Provider: setting.SessionConfig.Provider, diff --git a/routers/web/web.go b/routers/web/web.go index 6b217fc9b1b83..a4a1b7113c4d9 100644 --- a/routers/web/web.go +++ b/routers/web/web.go @@ -103,9 +103,7 @@ func buildAuthGroup() *auth_service.Group { func Routes(ctx gocontext.Context) *web.Route { routes := web.NewRoute() - routes.Use(web.WrapWithPrefix(public.AssetsURLPathPrefix, public.AssetsHandlerFunc(&public.Options{ - CorsHandler: CorsHandler(), - }), "AssetsHandler")) + routes.Use(web.WrapWithPrefix("/assets/", web.Wrap(CorsHandler(), public.AssetsHandlerFunc("/assets/")), "AssetsHandler")) sessioner := session.Sessioner(session.Options{ Provider: setting.SessionConfig.Provider, From c04d21d4697f9fd74595620252fbd014881198d7 Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Mon, 10 Apr 2023 17:08:53 +0800 Subject: [PATCH 09/19] fix merge --- modules/repository/init.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/repository/init.go b/modules/repository/init.go index 38dd8a0c4f969..cb353f24968a7 100644 --- a/modules/repository/init.go +++ b/modules/repository/init.go @@ -79,7 +79,7 @@ func LoadRepoConfig() error { typeFiles := make([]optionFileList, len(types)) for i, t := range types { var err error - if typeFiles[i].all, err = options.Dir(t); err != nil { + if typeFiles[i].all, err = options.AssetFS().ListFiles(t, true); err != nil { return fmt.Errorf("failed to list %s files: %w", t, err) } sort.Strings(typeFiles[i].all) From c976533c61d18b12e16150b1169209e485c4b8e1 Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Mon, 10 Apr 2023 18:05:42 +0800 Subject: [PATCH 10/19] add tests --- modules/assetfs/layered_test.go | 108 ++++++++++++++++++++++++++++++++ modules/util/timer_test.go | 29 +++++++++ 2 files changed, 137 insertions(+) create mode 100644 modules/assetfs/layered_test.go create mode 100644 modules/util/timer_test.go diff --git a/modules/assetfs/layered_test.go b/modules/assetfs/layered_test.go new file mode 100644 index 0000000000000..c443dd4c73bfc --- /dev/null +++ b/modules/assetfs/layered_test.go @@ -0,0 +1,108 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package assetfs + +import ( + "io" + "io/fs" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestLayered(t *testing.T) { + dir := filepath.Join(t.TempDir(), "assetfs-layers") + dir1 := filepath.Join(dir, "l1") + dir2 := filepath.Join(dir, "l2") + + mkdir := func(elems ...string) { + assert.NoError(t, os.MkdirAll(filepath.Join(elems...), 0o755)) + } + write := func(content string, elems ...string) { + assert.NoError(t, os.WriteFile(filepath.Join(elems...), []byte(content), 0o644)) + } + + // d1 & f1: only in "l1"; d2 & f2: only in "l2" + // da & fa: in both "l1" and "l2" + mkdir(dir1, "d1") + mkdir(dir1, "da") + mkdir(dir1, "da/sub1") + + mkdir(dir2, "d2") + mkdir(dir2, "da") + mkdir(dir2, "da/sub2") + + write("f1", dir1, "f1") + write("fa-1", dir1, "fa") + write("d1-f", dir1, "d1/f") + write("da-f-1", dir1, "da/f") + + write("f2", dir2, "f2") + write("fa-2", dir2, "fa") + write("d2-f", dir2, "d2/f") + write("da-f-2", dir2, "da/f") + + assets := Layered(Local("l1", dir1), Local("l2", dir2)) + + f, err := assets.Open("f1") + assert.NoError(t, err) + bs, err := io.ReadAll(f) + assert.NoError(t, err) + assert.EqualValues(t, "f1", string(bs)) + _ = f.Close() + + assertRead := func(expected string, expectedErr error, elems ...string) { + bs, err := assets.ReadFile(elems...) + if err != nil { + assert.ErrorAs(t, err, &expectedErr) + } else { + assert.NoError(t, err) + assert.Equal(t, expected, string(bs)) + } + } + assertRead("f1", nil, "f1") + assertRead("f2", nil, "f2") + assertRead("fa-1", nil, "fa") + + assertRead("d1-f", nil, "d1/f") + assertRead("d2-f", nil, "d2/f") + assertRead("da-f-1", nil, "da/f") + + assertRead("", fs.ErrNotExist, "no-such") + + files, err := assets.ListFiles(".", true) + assert.NoError(t, err) + assert.EqualValues(t, []string{"f1", "f2", "fa"}, files) + + files, err = assets.ListFiles(".", false) + assert.NoError(t, err) + assert.EqualValues(t, []string{"d1", "d2", "da"}, files) + + files, err = assets.ListFiles(".") + assert.NoError(t, err) + assert.EqualValues(t, []string{"d1", "d2", "da", "f1", "f2", "fa"}, files) + + files, err = assets.ListAllFiles(".", true) + assert.NoError(t, err) + assert.EqualValues(t, []string{"d1/f", "d2/f", "da/f", "f1", "f2", "fa"}, files) + + files, err = assets.ListAllFiles(".", false) + assert.NoError(t, err) + assert.EqualValues(t, []string{"d1", "d2", "da", "da/sub1", "da/sub2"}, files) + + files, err = assets.ListAllFiles(".") + assert.NoError(t, err) + assert.EqualValues(t, []string{ + "d1", "d1/f", + "d2", "d2/f", + "da", "da/f", "da/sub1", "da/sub2", + "f1", "f2", "fa", + }, files) + + assert.Empty(t, assets.GetFileLayerName("no-such")) + assert.EqualValues(t, "l1", assets.GetFileLayerName("f1")) + assert.EqualValues(t, "l2", assets.GetFileLayerName("f2")) +} diff --git a/modules/util/timer_test.go b/modules/util/timer_test.go new file mode 100644 index 0000000000000..aacefeba1ea9c --- /dev/null +++ b/modules/util/timer_test.go @@ -0,0 +1,29 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package util + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestDebounce(t *testing.T) { + c := 0 + d := Debounce(50 * time.Millisecond) + d(func() { c++ }) + assert.EqualValues(t, 0, c) + d(func() { c++ }) + d(func() { c++ }) + time.Sleep(100 * time.Millisecond) + assert.EqualValues(t, 1, c) + d(func() { c++ }) + assert.EqualValues(t, 1, c) + d(func() { c++ }) + d(func() { c++ }) + d(func() { c++ }) + time.Sleep(100 * time.Millisecond) + assert.EqualValues(t, 2, c) +} From 934c3afc8a5cdf594b4a38d448f706b11593694c Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Mon, 10 Apr 2023 19:38:32 +0800 Subject: [PATCH 11/19] fix embedded sub-cmd --- cmd/embedded.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/embedded.go b/cmd/embedded.go index 9e252c06aaddc..e9705858a211e 100644 --- a/cmd/embedded.go +++ b/cmd/embedded.go @@ -291,7 +291,7 @@ func extractAsset(d string, a asset, overwrite, rename bool) error { func buildAssetList(sec *section, globs []glob.Glob, c *cli.Context) []asset { results := make([]asset, 0, 64) - files, err := sec.AssetFS.ListFiles(".", true) + files, err := sec.AssetFS.ListAllFiles(".", true) if err != nil { log.Error("Error listing files in %q: %v", sec.Path, err) return nil From dc3d7b03d5d98e125b7d3190d59f2d866bb67729 Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Mon, 10 Apr 2023 20:47:47 +0800 Subject: [PATCH 12/19] fix test --- modules/util/timer_test.go | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/modules/util/timer_test.go b/modules/util/timer_test.go index aacefeba1ea9c..602800c2482c4 100644 --- a/modules/util/timer_test.go +++ b/modules/util/timer_test.go @@ -4,6 +4,7 @@ package util import ( + "sync/atomic" "testing" "time" @@ -11,19 +12,19 @@ import ( ) func TestDebounce(t *testing.T) { - c := 0 + var c int64 d := Debounce(50 * time.Millisecond) - d(func() { c++ }) - assert.EqualValues(t, 0, c) - d(func() { c++ }) - d(func() { c++ }) + d(func() { atomic.AddInt64(&c, 1) }) + assert.EqualValues(t, 0, atomic.LoadInt64(&c)) + d(func() { atomic.AddInt64(&c, 1) }) + d(func() { atomic.AddInt64(&c, 1) }) time.Sleep(100 * time.Millisecond) - assert.EqualValues(t, 1, c) - d(func() { c++ }) - assert.EqualValues(t, 1, c) - d(func() { c++ }) - d(func() { c++ }) - d(func() { c++ }) + assert.EqualValues(t, 1, atomic.LoadInt64(&c)) + d(func() { atomic.AddInt64(&c, 1) }) + assert.EqualValues(t, 1, atomic.LoadInt64(&c)) + d(func() { atomic.AddInt64(&c, 1) }) + d(func() { atomic.AddInt64(&c, 1) }) + d(func() { atomic.AddInt64(&c, 1) }) time.Sleep(100 * time.Millisecond) - assert.EqualValues(t, 2, c) + assert.EqualValues(t, 2, atomic.LoadInt64(&c)) } From cce1a68be292f6d0541fd972ea9eb8937436c242 Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Mon, 10 Apr 2023 23:32:30 +0800 Subject: [PATCH 13/19] skip OS-related hidden files --- modules/assetfs/layered.go | 2 ++ modules/assetfs/layered_test.go | 1 + 2 files changed, 3 insertions(+) diff --git a/modules/assetfs/layered.go b/modules/assetfs/layered.go index 868495c248bbe..3d8b0f326c1a5 100644 --- a/modules/assetfs/layered.go +++ b/modules/assetfs/layered.go @@ -129,6 +129,7 @@ func (l *LayeredFS) ListFiles(name string, fileMode ...bool) ([]string, error) { } else if len(fileMode) > 1 { return nil, errors.New("too many arguments") } + include = include && !util.CommonSkip(info.Name()) if include { fileMap[info.Name()] = true } @@ -177,6 +178,7 @@ func listAllFiles(layers []*Layer, name string, fileMode ...bool) ([]string, err return errors.New("too many arguments") } path := util.PathJoinRelX(dir, info.Name()) + include = include && !util.CommonSkip(info.Name()) if include { fileMap[path] = true } diff --git a/modules/assetfs/layered_test.go b/modules/assetfs/layered_test.go index c443dd4c73bfc..b82111e745e81 100644 --- a/modules/assetfs/layered_test.go +++ b/modules/assetfs/layered_test.go @@ -35,6 +35,7 @@ func TestLayered(t *testing.T) { mkdir(dir2, "da") mkdir(dir2, "da/sub2") + write("dummy", dir1, ".DS_Store") write("f1", dir1, "f1") write("fa-1", dir1, "fa") write("d1-f", dir1, "d1/f") From aa81b8e3b0de2587669112b8b6a35850ee1a9337 Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Tue, 11 Apr 2023 11:19:15 +0800 Subject: [PATCH 14/19] apply patch --- cmd/embedded.go | 43 +++++++++++++++++++------------------------ 1 file changed, 19 insertions(+), 24 deletions(-) diff --git a/cmd/embedded.go b/cmd/embedded.go index e9705858a211e..22cf107788075 100644 --- a/cmd/embedded.go +++ b/cmd/embedded.go @@ -91,13 +91,8 @@ var ( assets []asset ) -type section struct { - Path string - AssetFS *assetfs.LayeredFS -} - type asset struct { - Section *section + AssetFS *assetfs.LayeredFS Name string Path string } @@ -115,14 +110,14 @@ func initEmbeddedExtractor(c *cli.Context) error { if err != nil { return err } - sections := make(map[string]*section, 3) - - sections["public"] = §ion{Path: "public", AssetFS: assetfs.Layered(public.BuiltinAssets())} - sections["options"] = §ion{Path: "options", AssetFS: assetfs.Layered(options.BuiltinAssets())} - sections["templates"] = §ion{Path: "templates", AssetFS: assetfs.Layered(templates.BuiltinAssets())} + fss := map[string]*assetfs.LayeredFS{ + "public": assetfs.Layered(public.BuiltinAssets()), + "options": assetfs.Layered(options.BuiltinAssets()), + "templates": assetfs.Layered(templates.BuiltinAssets()), + } - for _, sec := range sections { - assets = append(assets, buildAssetList(sec, pats, c)...) + for p, fs := range fss { + assets = append(assets, buildAssetList(p, fs, pats, c)...) } // Sort assets @@ -180,7 +175,7 @@ func runViewDo(c *cli.Context) error { return fmt.Errorf("too many files matched the given pattern, try to be more specific") } - data, err := assets[0].Section.AssetFS.ReadFile(assets[0].Name) + data, err := assets[0].AssetFS.ReadFile(assets[0].Name) if err != nil { return fmt.Errorf("%s: %w", assets[0].Path, err) } @@ -198,7 +193,7 @@ func runExtractDo(c *cli.Context) error { } if len(c.Args()) == 0 { - return fmt.Errorf("A list of pattern of files to extract is mandatory (e.g. '**' for all)") + return fmt.Errorf("a list of pattern of files to extract is mandatory (e.g. '**' for all)") } destdir := "." @@ -245,7 +240,7 @@ func extractAsset(d string, a asset, overwrite, rename bool) error { dest := filepath.Join(d, filepath.FromSlash(a.Path)) dir := filepath.Dir(dest) - data, err := a.Section.AssetFS.ReadFile(a.Name) + data, err := a.AssetFS.ReadFile(a.Name) if err != nil { return fmt.Errorf("%s: %w", a.Path, err) } @@ -268,7 +263,7 @@ func extractAsset(d string, a asset, overwrite, rename bool) error { return fmt.Errorf("%s already exists, but it's not a regular file", dest) } else if rename { if err := util.Rename(dest, dest+".bak"); err != nil { - return fmt.Errorf("Error creating backup for %s: %w", dest, err) + return fmt.Errorf("error creating backup for %s: %w", dest, err) } // Attempt to respect file permissions mask (even if user:group will be set anew) perms = fi.Mode() @@ -289,26 +284,26 @@ func extractAsset(d string, a asset, overwrite, rename bool) error { return nil } -func buildAssetList(sec *section, globs []glob.Glob, c *cli.Context) []asset { +func buildAssetList(path string, assetFS *assetfs.LayeredFS, globs []glob.Glob, c *cli.Context) []asset { results := make([]asset, 0, 64) - files, err := sec.AssetFS.ListAllFiles(".", true) + files, err := assetFS.ListAllFiles(".", true) if err != nil { - log.Error("Error listing files in %q: %v", sec.Path, err) + log.Error("Error listing files in %q: %v", path, err) return nil } for _, name := range files { - if sec.Path == "public" && + if path == "public" && strings.HasPrefix(name, "vendor/") && !c.Bool("include-vendored") { continue } - matchName := sec.Path + "/" + name + matchName := path + "/" + name for _, g := range globs { if g.Match(matchName) { results = append(results, asset{ - Section: sec, + AssetFS: assetFS, Name: name, - Path: sec.Path + "/" + name, + Path: path + "/" + name, }) break } From 7a8bbeb894f5ed9708df2f9c7be121cfbd29b482 Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Tue, 11 Apr 2023 11:26:05 +0800 Subject: [PATCH 15/19] refactor embedded --- cmd/embedded.go | 76 +++++++++++++++++++++---------------------------- 1 file changed, 32 insertions(+), 44 deletions(-) diff --git a/cmd/embedded.go b/cmd/embedded.go index 22cf107788075..cee8928ce08db 100644 --- a/cmd/embedded.go +++ b/cmd/embedded.go @@ -8,7 +8,6 @@ import ( "fmt" "os" "path/filepath" - "sort" "strings" "code.gitea.io/gitea/modules/assetfs" @@ -88,16 +87,20 @@ var ( }, } - assets []asset + matchedAssetFiles []assetFile ) -type asset struct { - AssetFS *assetfs.LayeredFS - Name string - Path string +type assetFile struct { + fs *assetfs.LayeredFS + name string + path string } func initEmbeddedExtractor(c *cli.Context) error { + // FIXME: there is a bug, if the user runs `gitea embedded` with a different user or root, + // The setting.Init (loadRunModeFrom) will fail and do log.Fatal + // But the console logger has been deleted, so nothing is printed, the user sees nothing and Gitea just exits. + // Silence the console logger log.DelNamedLogger("console") log.DelNamedLogger(log.DEFAULT) @@ -106,24 +109,14 @@ func initEmbeddedExtractor(c *cli.Context) error { setting.InitProviderAllowEmpty() setting.LoadCommonSettings() - pats, err := getPatterns(c.Args()) + patterns, err := compileCollectPatterns(c.Args()) if err != nil { return err } - fss := map[string]*assetfs.LayeredFS{ - "public": assetfs.Layered(public.BuiltinAssets()), - "options": assetfs.Layered(options.BuiltinAssets()), - "templates": assetfs.Layered(templates.BuiltinAssets()), - } - for p, fs := range fss { - assets = append(assets, buildAssetList(p, fs, pats, c)...) - } - - // Sort assets - sort.SliceStable(assets, func(i, j int) bool { - return assets[i].Path < assets[j].Path - }) + collectAssetFilesByPattern(c, patterns, "options", options.BuiltinAssets()) + collectAssetFilesByPattern(c, patterns, "public", public.BuiltinAssets()) + collectAssetFilesByPattern(c, patterns, "templates", templates.BuiltinAssets()) return nil } @@ -157,8 +150,8 @@ func runListDo(c *cli.Context) error { return err } - for _, a := range assets { - fmt.Println(a.Path) + for _, a := range matchedAssetFiles { + fmt.Println(a.path) } return nil @@ -169,19 +162,19 @@ func runViewDo(c *cli.Context) error { return err } - if len(assets) == 0 { + if len(matchedAssetFiles) == 0 { return fmt.Errorf("no files matched the given pattern") - } else if len(assets) > 1 { + } else if len(matchedAssetFiles) > 1 { return fmt.Errorf("too many files matched the given pattern, try to be more specific") } - data, err := assets[0].AssetFS.ReadFile(assets[0].Name) + data, err := matchedAssetFiles[0].fs.ReadFile(matchedAssetFiles[0].name) if err != nil { - return fmt.Errorf("%s: %w", assets[0].Path, err) + return fmt.Errorf("%s: %w", matchedAssetFiles[0].path, err) } if _, err = os.Stdout.Write(data); err != nil { - return fmt.Errorf("%s: %w", assets[0].Path, err) + return fmt.Errorf("%s: %w", matchedAssetFiles[0].path, err) } return nil @@ -226,23 +219,23 @@ func runExtractDo(c *cli.Context) error { overwrite := c.Bool("overwrite") rename := c.Bool("rename") - for _, a := range assets { + for _, a := range matchedAssetFiles { if err := extractAsset(destdir, a, overwrite, rename); err != nil { // Non-fatal error - fmt.Fprintf(os.Stderr, "%s: %v", a.Path, err) + fmt.Fprintf(os.Stderr, "%s: %v", a.path, err) } } return nil } -func extractAsset(d string, a asset, overwrite, rename bool) error { - dest := filepath.Join(d, filepath.FromSlash(a.Path)) +func extractAsset(d string, a assetFile, overwrite, rename bool) error { + dest := filepath.Join(d, filepath.FromSlash(a.path)) dir := filepath.Dir(dest) - data, err := a.AssetFS.ReadFile(a.Name) + data, err := a.fs.ReadFile(a.name) if err != nil { - return fmt.Errorf("%s: %w", a.Path, err) + return fmt.Errorf("%s: %w", a.path, err) } if err := os.MkdirAll(dir, os.ModePerm); err != nil { @@ -284,12 +277,12 @@ func extractAsset(d string, a asset, overwrite, rename bool) error { return nil } -func buildAssetList(path string, assetFS *assetfs.LayeredFS, globs []glob.Glob, c *cli.Context) []asset { - results := make([]asset, 0, 64) - files, err := assetFS.ListAllFiles(".", true) +func collectAssetFilesByPattern(c *cli.Context, globs []glob.Glob, path string, layer *assetfs.Layer) { + fs := assetfs.Layered(layer) + files, err := fs.ListAllFiles(".", true) if err != nil { log.Error("Error listing files in %q: %v", path, err) - return nil + return } for _, name := range files { if path == "public" && @@ -300,19 +293,14 @@ func buildAssetList(path string, assetFS *assetfs.LayeredFS, globs []glob.Glob, matchName := path + "/" + name for _, g := range globs { if g.Match(matchName) { - results = append(results, asset{ - AssetFS: assetFS, - Name: name, - Path: path + "/" + name, - }) + matchedAssetFiles = append(matchedAssetFiles, assetFile{fs: fs, name: name, path: path + "/" + name}) break } } } - return results } -func getPatterns(args []string) ([]glob.Glob, error) { +func compileCollectPatterns(args []string) ([]glob.Glob, error) { if len(args) == 0 { args = []string{"**"} } From 4229674b590a90d772ef2b7e5a2286c940f2bea0 Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Tue, 11 Apr 2023 11:30:33 +0800 Subject: [PATCH 16/19] fix comment --- modules/assetfs/layered.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/modules/assetfs/layered.go b/modules/assetfs/layered.go index 3d8b0f326c1a5..71fc677690667 100644 --- a/modules/assetfs/layered.go +++ b/modules/assetfs/layered.go @@ -106,6 +106,7 @@ func (l *LayeredFS) ReadLayeredFile(elems ...string) ([]byte, string, error) { // * omitted: all files and directories will be returned. // * true: only files will be returned. // * false: only directories will be returned. +// The returned files are sorted by name. func (l *LayeredFS) ListFiles(name string, fileMode ...bool) ([]string, error) { fileMap := map[string]bool{} for _, layer := range l.layers { @@ -143,11 +144,12 @@ func (l *LayeredFS) ListFiles(name string, fileMode ...bool) ([]string, error) { return files, nil } -// ListAllFiles returns files/directories in the given directory, including subdirectories, recursively. +// ListAllFiles returns files/directories in the given directory, including subdirectories, recursively. // The fileMode controls the returned files: // * omitted: all files and directories will be returned. // * true: only files will be returned. // * false: only directories will be returned. +// The returned files are sorted by name. func (l *LayeredFS) ListAllFiles(name string, fileMode ...bool) ([]string, error) { return listAllFiles(l.layers, name, fileMode...) } From d56b8bc98407dcb8dbe74e419aec50d462de7cdf Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Wed, 12 Apr 2023 14:41:18 +0800 Subject: [PATCH 17/19] Update modules/assetfs/layered.go Co-authored-by: Jason Song --- modules/assetfs/layered.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/assetfs/layered.go b/modules/assetfs/layered.go index 71fc677690667..423cd5e7c8be3 100644 --- a/modules/assetfs/layered.go +++ b/modules/assetfs/layered.go @@ -136,7 +136,7 @@ func (l *LayeredFS) ListFiles(name string, fileMode ...bool) ([]string, error) { } } } - var files []string + files := make([]string, 0, len(fileMap)) for file := range fileMap { files = append(files, file) } From f563728851a31c0c4bf50d40abc2c69fab35047d Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Wed, 12 Apr 2023 14:48:51 +0800 Subject: [PATCH 18/19] Update modules/util/timer.go --- modules/util/timer.go | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/util/timer.go b/modules/util/timer.go index dc039bc0c9ae8..d598fde73ad27 100644 --- a/modules/util/timer.go +++ b/modules/util/timer.go @@ -40,6 +40,7 @@ func Debounce(d time.Duration) func(f func()) { defer db.mu.Unlock() if trigger == db.t { f() + db.t = nil } }) db.t = trigger From 8980b134936f03ae27524fcf17578cc2447d36b7 Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Wed, 12 Apr 2023 14:54:07 +0800 Subject: [PATCH 19/19] improve fileMode handling, use helper functions --- modules/assetfs/layered.go | 64 ++++++++++++++++---------------------- 1 file changed, 27 insertions(+), 37 deletions(-) diff --git a/modules/assetfs/layered.go b/modules/assetfs/layered.go index 423cd5e7c8be3..d032160a6feca 100644 --- a/modules/assetfs/layered.go +++ b/modules/assetfs/layered.go @@ -5,7 +5,6 @@ package assetfs import ( "context" - "errors" "fmt" "io" "io/fs" @@ -102,6 +101,29 @@ func (l *LayeredFS) ReadLayeredFile(elems ...string) ([]byte, string, error) { return nil, "", fs.ErrNotExist } +func shouldInclude(info fs.FileInfo, fileMode ...bool) bool { + if util.CommonSkip(info.Name()) { + return false + } + if len(fileMode) == 0 { + return true + } else if len(fileMode) == 1 { + return fileMode[0] == !info.Mode().IsDir() + } + panic("too many arguments for fileMode in shouldInclude") +} + +func readDir(layer *Layer, name string) ([]fs.FileInfo, error) { + f, err := layer.Open(name) + if os.IsNotExist(err) { + return nil, nil + } else if err != nil { + return nil, err + } + defer f.Close() + return f.Readdir(-1) +} + // ListFiles lists files/directories in the given directory. The fileMode controls the returned files. // * omitted: all files and directories will be returned. // * true: only files will be returned. @@ -110,28 +132,12 @@ func (l *LayeredFS) ReadLayeredFile(elems ...string) ([]byte, string, error) { func (l *LayeredFS) ListFiles(name string, fileMode ...bool) ([]string, error) { fileMap := map[string]bool{} for _, layer := range l.layers { - f, err := layer.Open(name) - if os.IsNotExist(err) { - continue - } else if err != nil { - return nil, err - } - infos, err := f.Readdir(-1) - _ = f.Close() + infos, err := readDir(layer, name) if err != nil { return nil, err } for _, info := range infos { - include := false - if len(fileMode) == 0 { - include = true - } else if len(fileMode) == 1 && fileMode[0] == !info.Mode().IsDir() { - include = true - } else if len(fileMode) > 1 { - return nil, errors.New("too many arguments") - } - include = include && !util.CommonSkip(info.Name()) - if include { + if shouldInclude(info, fileMode...) { fileMap[info.Name()] = true } } @@ -159,29 +165,13 @@ func listAllFiles(layers []*Layer, name string, fileMode ...bool) ([]string, err var list func(dir string) error list = func(dir string) error { for _, layer := range layers { - f, err := layer.Open(dir) - if os.IsNotExist(err) { - continue - } else if err != nil { - return err - } - infos, err := f.Readdir(-1) - _ = f.Close() + infos, err := readDir(layer, dir) if err != nil { return err } for _, info := range infos { - include := false - if len(fileMode) == 0 { - include = true - } else if len(fileMode) == 1 && fileMode[0] == !info.Mode().IsDir() { - include = true - } else if len(fileMode) > 1 { - return errors.New("too many arguments") - } path := util.PathJoinRelX(dir, info.Name()) - include = include && !util.CommonSkip(info.Name()) - if include { + if shouldInclude(info, fileMode...) { fileMap[path] = true } if info.IsDir() {