From ec18d9b535cb9f9d0be8b4d84a1ef43f99347f13 Mon Sep 17 00:00:00 2001 From: dancheg97 Date: Tue, 20 Jun 2023 23:08:48 +0300 Subject: [PATCH 001/124] added arch packages implementation --- go.mod | 4 +- go.sum | 5 + models/packages/descriptor.go | 3 + models/packages/package.go | 6 + modules/packages/arch/metadata.go | 255 +++++++++++++++ options/locale/locale_en-US.ini | 4 + public/img/svg/gitea-arch.svg | 1 + routers/api/packages/api.go | 14 + routers/api/packages/arch/arch.go | 416 +++++++++++++++++++++++++ routers/api/packages/arch/connector.go | 133 ++++++++ routers/init.go | 2 + templates/package/content/arch.tmpl | 105 +++++++ templates/package/metadata/arch.tmpl | 3 + templates/package/view.tmpl | 2 + 14 files changed, 952 insertions(+), 1 deletion(-) create mode 100644 modules/packages/arch/metadata.go create mode 100644 public/img/svg/gitea-arch.svg create mode 100644 routers/api/packages/arch/arch.go create mode 100644 routers/api/packages/arch/connector.go create mode 100644 templates/package/content/arch.tmpl create mode 100644 templates/package/metadata/arch.tmpl diff --git a/go.mod b/go.mod index 7b7a51efbb7f..7d783a75242c 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,9 @@ require ( gitea.com/lunny/levelqueue v0.4.2-0.20220729054728-f020868cc2f7 github.com/42wim/sshsig v0.0.0-20211121163825-841cf5bbc121 github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 + github.com/DataDog/zstd v1.4.5 github.com/NYTimes/gziphandler v1.1.1 + github.com/ProtonMail/gopenpgp/v2 v2.7.1 github.com/PuerkitoBio/goquery v1.8.1 github.com/alecthomas/chroma/v2 v2.7.0 github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb @@ -131,12 +133,12 @@ require ( git.sr.ht/~mariusor/go-xsd-duration v0.0.0-20220703122237-02e73435a078 // indirect github.com/ClickHouse/ch-go v0.55.0 // indirect github.com/ClickHouse/clickhouse-go/v2 v2.9.1 // indirect - github.com/DataDog/zstd v1.4.5 // indirect github.com/Masterminds/goutils v1.1.1 // indirect github.com/Masterminds/semver/v3 v3.2.0 // indirect github.com/Masterminds/sprig/v3 v3.2.3 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect github.com/ProtonMail/go-crypto v0.0.0-20230528122434-6f98819771a1 // indirect + github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f // indirect github.com/RoaringBitmap/roaring v1.2.3 // indirect github.com/acomagu/bufpipe v1.0.4 // indirect github.com/andybalholm/brotli v1.0.5 // indirect diff --git a/go.sum b/go.sum index c3ac719f3f4a..686b5505c35a 100644 --- a/go.sum +++ b/go.sum @@ -105,8 +105,13 @@ github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cq github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8/go.mod h1:I0gYDMZ6Z5GRU7l58bNFSkPTFN6Yl12dsUlAZ8xy98g= +github.com/ProtonMail/go-crypto v0.0.0-20230321155629-9a39f2531310/go.mod h1:8TI4H3IbrackdNgv+92dI+rhpCaLqM0IfpgCgenFvRE= github.com/ProtonMail/go-crypto v0.0.0-20230528122434-6f98819771a1 h1:JMDGhoQvXNTqH6Y3MC0IUw6tcZvaUdujNqzK2HYWZc8= github.com/ProtonMail/go-crypto v0.0.0-20230528122434-6f98819771a1/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0= +github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f h1:tCbYj7/299ekTTXpdwKYF8eBlsYsDVoggDAuAjoK66k= +github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f/go.mod h1:gcr0kNtGBqin9zDW9GOHcVntrwnjrK+qdJ06mWYBybw= +github.com/ProtonMail/gopenpgp/v2 v2.7.1 h1:Awsg7MPc2gD3I7IFac2qE3Gdls0lZW8SzrFZ3k1oz0s= +github.com/ProtonMail/gopenpgp/v2 v2.7.1/go.mod h1:/BU5gfAVwqyd8EfC3Eu7zmuhwYQpKs+cGD8M//iiaxs= github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAcwmWM= github.com/PuerkitoBio/goquery v1.8.1/go.mod h1:Q8ICL1kNUJ2sXGoAhPGUdYDJvgQgHzJsnnd3H7Ho5jQ= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= diff --git a/models/packages/descriptor.go b/models/packages/descriptor.go index ee35ffe0f2a3..4a03006de47a 100644 --- a/models/packages/descriptor.go +++ b/models/packages/descriptor.go @@ -13,6 +13,7 @@ import ( user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/json" "code.gitea.io/gitea/modules/packages/alpine" + "code.gitea.io/gitea/modules/packages/arch" "code.gitea.io/gitea/modules/packages/cargo" "code.gitea.io/gitea/modules/packages/chef" "code.gitea.io/gitea/modules/packages/composer" @@ -140,6 +141,8 @@ func GetPackageDescriptor(ctx context.Context, pv *PackageVersion) (*PackageDesc switch p.Type { case TypeAlpine: metadata = &alpine.VersionMetadata{} + case TypeArch: + metadata = &arch.Metadata{} case TypeCargo: metadata = &cargo.Metadata{} case TypeChef: diff --git a/models/packages/package.go b/models/packages/package.go index 380a076f9dfe..99c478fa79b0 100644 --- a/models/packages/package.go +++ b/models/packages/package.go @@ -31,6 +31,7 @@ type Type string // List of supported packages const ( TypeAlpine Type = "alpine" + TypeArch Type = "arch" TypeCargo Type = "cargo" TypeChef Type = "chef" TypeComposer Type = "composer" @@ -55,6 +56,7 @@ const ( var TypeList = []Type{ TypeAlpine, + TypeArch, TypeCargo, TypeChef, TypeComposer, @@ -82,6 +84,8 @@ func (pt Type) Name() string { switch pt { case TypeAlpine: return "Alpine" + case TypeArch: + return "Arch" case TypeCargo: return "Cargo" case TypeChef: @@ -131,6 +135,8 @@ func (pt Type) SVGName() string { switch pt { case TypeAlpine: return "gitea-alpine" + case TypeArch: + return "gitea-arch" case TypeCargo: return "gitea-cargo" case TypeChef: diff --git a/modules/packages/arch/metadata.go b/modules/packages/arch/metadata.go new file mode 100644 index 000000000000..5433138a67f7 --- /dev/null +++ b/modules/packages/arch/metadata.go @@ -0,0 +1,255 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package arch + +import ( + "crypto/md5" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io/fs" + "os" + "path" + "strconv" + "strings" + "time" + + "github.com/DataDog/zstd" + "github.com/mholt/archiver/v3" +) + +// Metadata for arch package. +type Metadata struct { + Filename string + Name string + Base string + Version string + Description string + CompressedSize int64 + CompressedSizeMib string + InstalledSize int64 + InstalledSizeMib string + MD5 string + SHA256 string + URL string + BuildDate int64 + BuildDateStr string + BaseDomain string + Packager string + Provides []string + License []string + Arch []string + Depends []string + OptDepends []string + MakeDepends []string + CheckDepends []string + Backup []string +} + +// Function that recieves arch package archive data and returns it's metadata. +func EjectMetadata(filename, domain string, pkg []byte) (*Metadata, error) { + d, err := zstd.Decompress(nil, pkg) + if err != nil { + return nil, err + } + splt := strings.Split(string(d), "PKGINFO") + if len(splt) < 2 { + return nil, errors.New("unable to eject .PKGINFO from archive") + } + raw := splt[1][0:10000] + inssize := int64(len(pkg)) + compsize := ejectInt64(raw, "size") + unixbuilddate := ejectInt64(raw, "builddate") + return &Metadata{ + Filename: filename, + Name: ejectString(raw, "pkgname"), + Base: ejectString(raw, "pkgbase"), + Version: ejectString(raw, "pkgver"), + Description: ejectString(raw, "pkgdesc"), + CompressedSize: inssize, + CompressedSizeMib: ByteCountSI(inssize), + InstalledSize: compsize, + InstalledSizeMib: ByteCountSI(compsize), + MD5: md5sum(pkg), + SHA256: sha256sum(pkg), + URL: ejectString(raw, "url"), + BuildDate: unixbuilddate, + BuildDateStr: ReadableTime(unixbuilddate), + BaseDomain: domain, + Packager: ejectString(raw, "packager"), + Provides: ejectStrings(raw, "provides"), + License: ejectStrings(raw, "license"), + Arch: ejectStrings(raw, "arch"), + Depends: ejectStrings(raw, "depend"), + OptDepends: ejectStrings(raw, "optdepend"), + MakeDepends: ejectStrings(raw, "makedepend"), + CheckDepends: ejectStrings(raw, "checkdepend"), + Backup: ejectStrings(raw, "backup"), + }, err +} + +func ejectString(raw, field string) string { + splitted := strings.Split(raw, "\n"+field+" = ") + if len(splitted) < 2 { + return `` + } + return strings.Split(splitted[1], "\n")[0] +} + +func ejectStrings(raw, field string) []string { + splitted := strings.Split(raw, "\n"+field+" = ") + if len(splitted) < 2 { + return nil + } + var rez []string + for i, v := range splitted { + if i == 0 { + continue + } + rez = append(rez, strings.Split(v, "\n")[0]) + } + return rez +} + +func ejectInt64(raw, field string) int64 { + splitted := strings.Split(raw, "\n"+field+" = ") + if len(splitted) < 2 { + return 0 + } + i, err := strconv.ParseInt(strings.Split(splitted[1], "\n")[0], 10, 64) + if err != nil { + return 0 + } + return i +} + +func ByteCountSI(b int64) string { + const unit = 1000 + if b < unit { + return fmt.Sprintf("%d B", b) + } + div, exp := int64(unit), 0 + for n := b / unit; n >= unit; n /= unit { + div *= unit + exp++ + } + return fmt.Sprintf("%.1f %cB", float64(b)/float64(div), "kMGTPE"[exp]) +} + +func ReadableTime(unix int64) string { + return time.Unix(unix, 0).Format(time.DateTime) +} + +func md5sum(data []byte) string { + sum := md5.Sum(data) + return hex.EncodeToString(sum[:]) +} + +func sha256sum(data []byte) string { + sum := sha256.Sum256(data) + return hex.EncodeToString(sum[:]) +} + +// This function returns pacman package description in unarchived raw database +// format. +func (m *Metadata) GetDbDesc() string { + return strings.Join(rmEmptyStrings([]string{ + formatField("FILENAME", m.Filename), + formatField("NAME", m.Name), + formatField("BASE", m.Base), + formatField("VERSION", m.Version), + formatField("DESC", m.Description), + formatField("CSIZE", m.CompressedSize), + formatField("ISIZE", m.InstalledSize), + formatField("MD5SUM", m.MD5), + formatField("SHA256SUM", m.SHA256), + formatField("URL", m.URL), + formatField("LICENSE", m.License), + formatField("ARCH", m.Arch), + formatField("BUILDDATE", m.BuildDate), + formatField("PACKAGER", m.Packager), + formatField("PROVIDES", m.Provides), + formatField("DEPENDS", m.Depends), + formatField("OPTDEPENDS", m.OptDepends), + formatField("MAKEDEPENDS", m.MakeDepends), + formatField("CHECKDEPENDS", m.CheckDepends), + }), "\n\n") + "\n\n" +} + +func formatField(field string, value any) string { + switch value := value.(type) { + case []string: + if value == nil { + return `` + } + val := strings.Join(value, "\n") + return fmt.Sprintf("%%%s%%\n%s", field, val) + case string: + return fmt.Sprintf("%%%s%%\n%s", field, value) + case int64: + return fmt.Sprintf("%%%s%%\n%d", field, value) + } + return `` +} + +func rmEmptyStrings(s []string) []string { + var r []string + for _, str := range s { + if str != "" { + r = append(r, str) + } + } + return r +} + +// Function takes path to directory with pacman database and updates package +// it with current metadata. +func (m *Metadata) PutToDb(dir string, mode fs.FileMode) error { + descdir := path.Join(dir, m.Name+"-"+m.Version) + err := os.MkdirAll(descdir, mode) + if err != nil { + return err + } + return os.WriteFile(path.Join(descdir, "desc"), []byte(m.GetDbDesc()), mode) +} + +// Function takes raw database archive bytes and destination directory as +// arguements and unpacks database contents to destination directory. +func UnpackDb(src, dst string) error { + return archiver.DefaultTarGz.Unarchive(src, dst) +} + +// Function takes path to source directory with raw pacman description files +// for pacman database, creates db.tar.gz archive and related symlink for +// provided path. +func PackDb(src, dst string) error { + if !strings.HasSuffix(dst, ".db.tar.gz") { + return fmt.Errorf("dst should end with '.db.tar.gz': %s", dst) + } + symlink := strings.TrimSuffix(dst, ".tar.gz") + if _, err := os.Stat(dst); err == nil { + err = os.RemoveAll(dst) + if err != nil { + return err + } + err = os.RemoveAll(symlink) + if err != nil { + return err + } + } + des, err := os.ReadDir(src) + if err != nil { + return err + } + var pkgdescs []string + for _, de := range des { + pkgdescs = append(pkgdescs, path.Join(src, de.Name())) + } + err = archiver.DefaultTarGz.Archive(pkgdescs, dst) + if err != nil { + return err + } + return os.Symlink(dst, symlink) +} diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 6cab7c0cbb09..8d9b17c1e969 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -3254,6 +3254,10 @@ alpine.repository = Repository Info alpine.repository.branches = Branches alpine.repository.repositories = Repositories alpine.repository.architectures = Architectures +arch.pacmanconf = Add server to pacman.conf: +arch.pacmansync = And sync package with pacman: +arch.documentation = For more information on the Arch registry, see the documentation. +arch.properties = Package properties cargo.registry = Setup this registry in the Cargo configuration file (for example ~/.cargo/config.toml): cargo.install = To install the package using Cargo, run the following command: cargo.documentation = For more information on the Cargo registry, see the documentation. diff --git a/public/img/svg/gitea-arch.svg b/public/img/svg/gitea-arch.svg new file mode 100644 index 000000000000..ac50ffdf2618 --- /dev/null +++ b/public/img/svg/gitea-arch.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/routers/api/packages/api.go b/routers/api/packages/api.go index fa7f66f3ab38..bbc6ce278277 100644 --- a/routers/api/packages/api.go +++ b/routers/api/packages/api.go @@ -15,6 +15,7 @@ import ( "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/web" "code.gitea.io/gitea/routers/api/packages/alpine" + "code.gitea.io/gitea/routers/api/packages/arch" "code.gitea.io/gitea/routers/api/packages/cargo" "code.gitea.io/gitea/routers/api/packages/chef" "code.gitea.io/gitea/routers/api/packages/composer" @@ -752,3 +753,16 @@ func ContainerRoutes() *web.Route { return r } + +// Routes for arch packages. +func ArchRoutes() *web.Route { + r := web.NewRoute() + + r.Use(context.PackageContexter()) + + r.Put("/push", arch.Push) + r.Get("/{distro}/{arch}/{owner}/{file}", arch.Get) + r.Get("/{distro}/{arch}/{file}", arch.Get) + + return r +} diff --git a/routers/api/packages/arch/arch.go b/routers/api/packages/arch/arch.go new file mode 100644 index 000000000000..1948bbee0754 --- /dev/null +++ b/routers/api/packages/arch/arch.go @@ -0,0 +1,416 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package arch + +import ( + "bytes" + "encoding/hex" + "errors" + "fmt" + "io" + "net/http" + "os" + "path" + "strings" + + "code.gitea.io/gitea/models/db" + packages_model "code.gitea.io/gitea/models/packages" + repo_model "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/json" + packages_module "code.gitea.io/gitea/modules/packages" + arch_module "code.gitea.io/gitea/modules/packages/arch" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/timeutil" + "code.gitea.io/gitea/routers/api/packages/helper" + packages_service "code.gitea.io/gitea/services/packages" + + "github.com/ProtonMail/gopenpgp/v2/crypto" + "github.com/google/uuid" +) + +// Push new package to arch package registry. +func Push(ctx *context.Context) { + // Creating connector that will help with keys/blobs. + connector := Connector{ctx: ctx} + + // Getting some information related to package from headers. + filename := ctx.Req.Header.Get("filename") + email := ctx.Req.Header.Get("email") + sign := ctx.Req.Header.Get("sign") + owner := ctx.Req.Header.Get("owner") + distro := ctx.Req.Header.Get("distro") + + // Decoding package signature. + sigdata, err := hex.DecodeString(sign) + if err != nil { + apiError(ctx, http.StatusBadRequest, err) + return + } + pgpsig := crypto.NewPGPSignature(sigdata) + + // Validating that user is allowed to push to specified namespace. + err = connector.ValidateNamespace(owner, email) + if err != nil { + apiError(ctx, http.StatusBadRequest, err) + return + } + + // Getting GPG keys related to specific user. After keys have been recieved, + // this function will find one key related to email provided in request. + armoredKeys, err := connector.GetValidKeys(email) + if err != nil { + apiError(ctx, http.StatusBadRequest, err) + return + } + var matchedKeyring *crypto.KeyRing + for _, armor := range armoredKeys { + pgpkey, err := crypto.NewKeyFromArmored(armor) + if err != nil { + apiError(ctx, http.StatusBadRequest, err) + return + } + keyring, err := crypto.NewKeyRing(pgpkey) + if err != nil { + apiError(ctx, http.StatusBadRequest, err) + return + } + for _, idnt := range keyring.GetIdentities() { + if idnt.Email == email { + matchedKeyring = keyring + break + } + } + if matchedKeyring != nil { + break + } + } + if matchedKeyring == nil { + msg := "GPG key related to " + email + " not found" + apiError(ctx, http.StatusBadRequest, msg) + return + } + + // Read package to memory and create plain GPG message to validate signature. + pkgdata, err := io.ReadAll(ctx.Req.Body) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + defer ctx.Req.Body.Close() + + pgpmes := crypto.NewPlainMessage(pkgdata) + + // Validate package signature with user's GPG key related to his email. + err = matchedKeyring.VerifyDetached(pgpmes, pgpsig, crypto.GetUnixTime()) + if err != nil { + apiError(ctx, http.StatusUnauthorized, "unable to validate package signature") + return + } + + // Create temporary directory for arch database operations. + tmpdir := path.Join(setting.Repository.Upload.TempPath, uuid.New().String()) + err = os.MkdirAll(tmpdir, os.ModePerm) + if err != nil { + apiError(ctx, http.StatusInternalServerError, "unable to create tmp path") + return + } + defer os.RemoveAll(tmpdir) + + // Parse metadata contained in arch package archive. + md, err := arch_module.EjectMetadata(filename, setting.Domain, pkgdata) + if err != nil { + apiError(ctx, http.StatusBadRequest, err) + return + } + + // Arch database related filenames, pathes and folders. + dbname := Join(owner, distro, setting.Domain, "db.tar.gz") + dbpath := path.Join(tmpdir, dbname) + dbfolder := path.Join(tmpdir, dbname) + ".folder" + dbsymlink := strings.TrimSuffix(dbname, ".tar.gz") + dbsymlinkpath := path.Join(tmpdir, dbsymlink) + + // Get existing arch package database, related to specific userspace from + // file storage, and save it on disk, then unpack it's contents to related + // folder. If database is not found in storage, create empty directory to + // store package related information. + dbdata, err := connector.Get(dbname) + if err == nil { + err = os.WriteFile(dbpath, dbdata, os.ModePerm) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + err = arch_module.UnpackDb(dbpath, dbfolder) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + } + if err != nil { + err = os.MkdirAll(dbfolder, os.ModePerm) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + } + + // Update database folder with metadata for new package. + err = md.PutToDb(dbfolder, os.ModePerm) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + // Create database archive and related symlink. + err = arch_module.PackDb(dbfolder, dbpath) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + // Save namespace related arch repository database. + f, err := os.Open(dbpath) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + defer f.Close() + dbfi, err := f.Stat() + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + err = connector.Save(dbname, f, dbfi.Size()) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + // Save namespace related arch repository db archive. + f, err = os.Open(dbsymlinkpath) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + defer f.Close() + dbarchivefi, err := f.Stat() + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + err = connector.Save(dbsymlink, f, dbarchivefi.Size()) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + // Create package in database. + pkg, err := packages_model.TryInsertPackage(ctx, &packages_model.Package{ + OwnerID: connector.user.ID, + Type: packages_model.TypeArch, + Name: md.Name, + LowerName: strings.ToLower(md.Name), + }) + if errors.Is(err, packages_model.ErrDuplicatePackage) { + pkg, err = packages_model.GetPackageByName( + ctx, connector.user.ID, + packages_model.TypeArch, md.Name, + ) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + } + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + // Check if repository for package with provided owner exists. + repo, err := repo_model.GetRepositoryByOwnerAndName(ctx, owner, md.Name) + if err == nil { + err = packages_model.SetRepositoryLink(ctx, pkg.ID, repo.ID) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + } + + // Create new package version in database. + rawjsonmetadata, err := json.Marshal(&md) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + ver, err := packages_model.GetOrInsertVersion(ctx, &packages_model.PackageVersion{ + PackageID: pkg.ID, + CreatorID: connector.user.ID, + Version: md.Version, + LowerVersion: strings.ToLower(md.Version), + CreatedUnix: timeutil.TimeStampNow(), + MetadataJSON: string(rawjsonmetadata), + }) + if err != nil { + if errors.Is(err, packages_model.ErrDuplicatePackageVersion) { + apiError(ctx, http.StatusConflict, err) + return + } + apiError(ctx, http.StatusInternalServerError, err) + return + } + + // Create package blob and db file for package file. + pkgreader := bytes.NewReader(pkgdata) + fbuf, err := packages_module.CreateHashedBufferFromReader(pkgreader) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + defer fbuf.Close() + + filepb, ok, err := packages_model.GetOrInsertBlob( + ctx, packages_service.NewPackageBlob(fbuf), + ) + if err != nil { + apiError(ctx, http.StatusInternalServerError, fmt.Errorf("%v %t", err, ok)) + return + } + err = connector.Save(filepb.HashSHA256, fbuf, filepb.Size) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + _, err = packages_model.TryInsertFile(ctx, &packages_model.PackageFile{ + VersionID: ver.ID, + BlobID: filepb.ID, + Name: filename, + LowerName: strings.ToLower(filename), + CompositeKey: distro + "-" + filename, + IsLead: true, + CreatedUnix: timeutil.TimeStampNow(), + }) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + // Create package blob for package signature. + sigreader := bytes.NewReader(sigdata) + sbuf, err := packages_module.CreateHashedBufferFromReader(sigreader) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + defer fbuf.Close() + + sigpb, ok, err := packages_model.GetOrInsertBlob( + ctx, packages_service.NewPackageBlob(sbuf), + ) + if err != nil { + apiError(ctx, http.StatusInternalServerError, fmt.Errorf("%v %t", err, ok)) + return + } + err = connector.Save(sigpb.HashSHA256, sbuf, sigpb.Size) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + _, err = packages_model.TryInsertFile(ctx, &packages_model.PackageFile{ + VersionID: ver.ID, + BlobID: sigpb.ID, + Name: filename + ".sig", + LowerName: strings.ToLower(filename + ".sig"), + CompositeKey: distro + "-" + filename + ".sig", + IsLead: false, + CreatedUnix: timeutil.TimeStampNow(), + }) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + ctx.Status(http.StatusOK) +} + +// Get file from arch package registry. +func Get(ctx *context.Context) { + filename := ctx.Params("file") + owner := ctx.Params("owner") + distro := ctx.Params("distro") + // arch := ctx.Params("arch") + + cs := packages_module.NewContentStore() + + if strings.HasSuffix(filename, "tar.zst") || + strings.HasSuffix(filename, "zst.sig") { + db := db.GetEngine(ctx) + + pkgfile := &packages_model.PackageFile{ + CompositeKey: distro + "-" + filename, + } + ok, err := db.Get(pkgfile) + if err != nil || !ok { + apiError( + ctx, http.StatusInternalServerError, + fmt.Errorf("%+v %t", err, ok), + ) + return + } + + blob, err := packages_model.GetBlobByID(ctx, pkgfile.BlobID) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + obj, err := cs.Get(packages_module.BlobHash256Key(blob.HashSHA256)) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + data, err := io.ReadAll(obj) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + _, err = ctx.Resp.Write(data) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + ctx.Resp.WriteHeader(http.StatusOK) + + return + } + obj, err := cs.Get(packages_module.BlobHash256Key(Join(owner, distro, filename))) + if err != nil { + apiError(ctx, http.StatusNotFound, err) + } + + data, err := io.ReadAll(obj) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + _, err = ctx.Resp.Write(data) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + ctx.Resp.WriteHeader(http.StatusOK) +} + +func apiError(ctx *context.Context, status int, obj interface{}) { + helper.LogAndProcessError(ctx, status, obj, func(message string) { + ctx.PlainText(status, message) + }) +} diff --git a/routers/api/packages/arch/connector.go b/routers/api/packages/arch/connector.go new file mode 100644 index 000000000000..e19215c5a19a --- /dev/null +++ b/routers/api/packages/arch/connector.go @@ -0,0 +1,133 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package arch + +import ( + "errors" + "io" + + "code.gitea.io/gitea/models/asymkey" + "code.gitea.io/gitea/models/db" + organization_model "code.gitea.io/gitea/models/organization" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/log" + packages_module "code.gitea.io/gitea/modules/packages" +) + +// Connector helps to retrieve GPG keys related to package validation and +// manage blobs related to specific user spaces: +// 1 - Check if user is allowed to push package to specific namespace. +// 2 - Retrieving GPG keys related to provided email. +// 3 - Get/put arch arch package/signature/database files to connected file +// storage. +type Connector struct { + ctx *context.Context + user *user_model.User + org *organization_model.Organization +} + +// This function will find user related to provided email adress and check if +// he is able to push packages to provided namespace (user/organization/or +// empty namespace allowed for admin users). +func (c *Connector) ValidateNamespace(namespace, email string) error { + var err error + c.user, err = user_model.GetUserByEmail(c.ctx, email) + if err != nil { + log.Error("unable to get user with email: %s %v", email, err) + return err + } + + if namespace == "" && c.user.IsAdmin { + c.org = (*organization_model.Organization)(c.user) + return nil + } + + if c.user.Name != namespace && c.org == nil { + c.org, err = organization_model.GetOrgByName(c.ctx, namespace) + if err != nil { + log.Error("unable to organization: %s %v", namespace, err) + return err + } + ismember, err := c.org.IsOrgMember(c.user.ID) + if err != nil { + log.Error( + "unable to check if user belongs to organization: %s %s %v", + c.user.Name, email, err, + ) + return err + } + if !ismember { + log.Error("user %s is not member of organization: %s", c.user.Name, email) + return errors.New("user is not member of organization: " + namespace) + } + } else { + c.org = (*organization_model.Organization)(c.user) + } + return nil +} + +// This function will try to find user related to specific email. And check +// that user is allowed to push to 'owner' namespace (package owner, could +// be empty, user or organization). +// After namespace check, this function +func (c *Connector) GetValidKeys(email string) ([]string, error) { + keys, err := asymkey.ListGPGKeys(c.ctx, c.user.ID, db.ListOptions{ + ListAll: true, + }) + if err != nil { + log.Error("unable to get keys related to user: %v", err) + return nil, errors.New("unable to get public keys") + } + if len(keys) == 0 { + log.Error("no keys related to user") + return nil, errors.New("no keys for user with email: " + email) + } + + var keyarmors []string + for _, key := range keys { + k, err := asymkey.GetGPGImportByKeyID(key.KeyID) + if err != nil { + log.Error("unable to import GPG key by ID: %v", err) + return nil, errors.New("internal error") + } + keyarmors = append(keyarmors, k.Content) + } + + return keyarmors, nil +} + +// Get specific file content from content storage. +func (c *Connector) Get(key string) ([]byte, error) { + cs := packages_module.NewContentStore() + obj, err := cs.Get(packages_module.BlobHash256Key(key)) + if err != nil { + return nil, err + } + return io.ReadAll(obj) +} + +// Save contents related to specific arch package. +func (c *Connector) Save(key string, content io.Reader, size int64) error { + cs := packages_module.NewContentStore() + return cs.Save(packages_module.BlobHash256Key(key), content, size) +} + +// Join database or package names to prevent collisions with same packages in +// different user spaces. Skips empty strings and returns name joined with +// dots. +func Join(s ...string) string { + rez := "" + for i, v := range s { + if v == "" { + continue + } + if i+1 == len(s) { + rez += v + continue + } + rez += v + "." + } + return rez +} diff --git a/routers/init.go b/routers/init.go index 54e8d2b8b39e..668efdcb859d 100644 --- a/routers/init.go +++ b/routers/init.go @@ -190,6 +190,8 @@ func NormalRoutes() *web.Route { r.Mount("/api/packages", packages_router.CommonRoutes()) // This implements the OCI API (Note this is not preceded by /api but is instead /v2) r.Mount("/v2", packages_router.ContainerRoutes()) + // Arch package routes + r.Mount("/api/packages/arch", packages_router.ArchRoutes()) } if setting.Actions.Enabled { diff --git a/templates/package/content/arch.tmpl b/templates/package/content/arch.tmpl new file mode 100644 index 000000000000..0ecce3550946 --- /dev/null +++ b/templates/package/content/arch.tmpl @@ -0,0 +1,105 @@ +{{if eq .PackageDescriptor.Package.Type "arch"}} +

{{.locale.Tr "packages.installation"}}

+
+
+ +
+ + 
[{{.PackageDescriptor.Owner.LowerName}}.{{.PackageDescriptor.Metadata.BaseDomain}}]
+Server = https://{{.PackageDescriptor.Metadata.BaseDomain}}/api/packages/arch/(distribution)/(architecture)/{{.PackageDescriptor.Owner.LowerName}}
+
+ +
+ + 
pacman -S {{.PackageDescriptor.Package.LowerName}}
+
+ +
+ +
+
+
+ +

{{.locale.Tr "packages.arch.properties"}}

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + {{range $key := .PackageDescriptor.Metadata.Provides}} + + + + + {{end}} + + {{range $key := .PackageDescriptor.Metadata.Arch}} + + + + + {{end}} + + {{range $key := .PackageDescriptor.Metadata.Depends}} + + + + + {{end}} + + {{range $key := .PackageDescriptor.Metadata.OptDepends}} + + + + + {{end}} + + {{range $key := .PackageDescriptor.Metadata.MakeDepends}} + + + + + {{end}} + + {{range $key := .PackageDescriptor.Metadata.CheckDepends}} + + + + + {{end}} + + {{range $key := .PackageDescriptor.Metadata.Backup}} + + + + + {{end}} + +
Description
{{.PackageDescriptor.Metadata.Description}}
Compressed size
{{.PackageDescriptor.Metadata.CompressedSizeMib}}
Installed size
{{.PackageDescriptor.Metadata.InstalledSizeMib}}
Official URL
 {{.PackageDescriptor.Metadata.URL}}
Build date
{{.PackageDescriptor.Metadata.BuildDateStr}}
Packager
{{.PackageDescriptor.Metadata.Packager}}
Provides
{{$key}}
Architecture
{{$key}}
Depends
{{$key}}
Optional depends
{{$key}}
Make depends
{{$key}}
Check depends
{{$key}}
Backup file
{{$key}}
+
+ +{{end}} diff --git a/templates/package/metadata/arch.tmpl b/templates/package/metadata/arch.tmpl new file mode 100644 index 000000000000..8cabeda06a5a --- /dev/null +++ b/templates/package/metadata/arch.tmpl @@ -0,0 +1,3 @@ +{{if eq .PackageDescriptor.Package.Type "arch"}} + {{range .PackageDescriptor.Metadata.License}}
{{svg "octicon-law" 16 "gt-mr-3"}} {{.}}
{{end}} +{{end}} diff --git a/templates/package/view.tmpl b/templates/package/view.tmpl index 755c93fde301..589068114910 100644 --- a/templates/package/view.tmpl +++ b/templates/package/view.tmpl @@ -18,6 +18,7 @@
{{template "package/content/alpine" .}} + {{template "package/content/arch" .}} {{template "package/content/cargo" .}} {{template "package/content/chef" .}} {{template "package/content/composer" .}} @@ -49,6 +50,7 @@
{{svg "octicon-calendar" 16 "gt-mr-3"}} {{TimeSinceUnix .PackageDescriptor.Version.CreatedUnix $.locale}}
{{svg "octicon-download" 16 "gt-mr-3"}} {{.PackageDescriptor.Version.DownloadCount}}
{{template "package/metadata/alpine" .}} + {{template "package/metadata/arch" .}} {{template "package/metadata/cargo" .}} {{template "package/metadata/chef" .}} {{template "package/metadata/composer" .}} From d24e41a6fe9ed2aba955c487b7fab09a4306603e Mon Sep 17 00:00:00 2001 From: dancheg97 Date: Wed, 21 Jun 2023 00:15:56 +0300 Subject: [PATCH 002/124] fixed db download failure and incorrect package owner assigned on push --- routers/api/packages/arch/arch.go | 41 +++++++++++++++++-------------- 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/routers/api/packages/arch/arch.go b/routers/api/packages/arch/arch.go index 1948bbee0754..0918d98e5b8d 100644 --- a/routers/api/packages/arch/arch.go +++ b/routers/api/packages/arch/arch.go @@ -209,14 +209,14 @@ func Push(ctx *context.Context) { // Create package in database. pkg, err := packages_model.TryInsertPackage(ctx, &packages_model.Package{ - OwnerID: connector.user.ID, + OwnerID: connector.org.ID, Type: packages_model.TypeArch, Name: md.Name, LowerName: strings.ToLower(md.Name), }) if errors.Is(err, packages_model.ErrDuplicatePackage) { pkg, err = packages_model.GetPackageByName( - ctx, connector.user.ID, + ctx, connector.org.ID, packages_model.TypeArch, md.Name, ) if err != nil { @@ -248,7 +248,7 @@ func Push(ctx *context.Context) { ver, err := packages_model.GetOrInsertVersion(ctx, &packages_model.PackageVersion{ PackageID: pkg.ID, - CreatorID: connector.user.ID, + CreatorID: connector.org.ID, Version: md.Version, LowerVersion: strings.ToLower(md.Version), CreatedUnix: timeutil.TimeStampNow(), @@ -347,8 +347,7 @@ func Get(ctx *context.Context) { cs := packages_module.NewContentStore() - if strings.HasSuffix(filename, "tar.zst") || - strings.HasSuffix(filename, "zst.sig") { + if strings.HasSuffix(filename, "tar.zst") || strings.HasSuffix(filename, "zst.sig") { db := db.GetEngine(ctx) pkgfile := &packages_model.PackageFile{ @@ -390,23 +389,27 @@ func Get(ctx *context.Context) { return } - obj, err := cs.Get(packages_module.BlobHash256Key(Join(owner, distro, filename))) - if err != nil { - apiError(ctx, http.StatusNotFound, err) - } + if strings.HasSuffix(filename, ".db.tar.gz") || strings.HasSuffix(filename, ".db") { + filename = strings.TrimPrefix(filename, owner+".") + obj, err := cs.Get(packages_module.BlobHash256Key(Join(owner, distro, filename))) + if err != nil { + apiError(ctx, http.StatusNotFound, err) + } - data, err := io.ReadAll(obj) - if err != nil { - apiError(ctx, http.StatusInternalServerError, err) - return - } + data, err := io.ReadAll(obj) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } - _, err = ctx.Resp.Write(data) - if err != nil { - apiError(ctx, http.StatusInternalServerError, err) - return + _, err = ctx.Resp.Write(data) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + ctx.Resp.WriteHeader(http.StatusOK) } - ctx.Resp.WriteHeader(http.StatusOK) + ctx.Resp.WriteHeader(http.StatusNotFound) } func apiError(ctx *context.Context, status int, obj interface{}) { From 6ec9686162e4ba10e418ebb756da712020cddd7e Mon Sep 17 00:00:00 2001 From: dancheg97 Date: Wed, 21 Jun 2023 01:44:15 +0300 Subject: [PATCH 003/124] updated icon name and centered viewbox --- models/packages/package.go | 2 +- public/img/svg/gitea-arch.svg | 1 - public/img/svg/octicon-arch.svg | 1 + 3 files changed, 2 insertions(+), 2 deletions(-) delete mode 100644 public/img/svg/gitea-arch.svg create mode 100644 public/img/svg/octicon-arch.svg diff --git a/models/packages/package.go b/models/packages/package.go index 99c478fa79b0..0d0adc03f66b 100644 --- a/models/packages/package.go +++ b/models/packages/package.go @@ -136,7 +136,7 @@ func (pt Type) SVGName() string { case TypeAlpine: return "gitea-alpine" case TypeArch: - return "gitea-arch" + return "octicon-arch" case TypeCargo: return "gitea-cargo" case TypeChef: diff --git a/public/img/svg/gitea-arch.svg b/public/img/svg/gitea-arch.svg deleted file mode 100644 index ac50ffdf2618..000000000000 --- a/public/img/svg/gitea-arch.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/public/img/svg/octicon-arch.svg b/public/img/svg/octicon-arch.svg new file mode 100644 index 000000000000..3a420d58dbf9 --- /dev/null +++ b/public/img/svg/octicon-arch.svg @@ -0,0 +1 @@ + \ No newline at end of file From 6328a92c0b76a2c706dfdf81d84df12000ca0c3e Mon Sep 17 00:00:00 2001 From: dancheg97 Date: Wed, 21 Jun 2023 13:03:57 +0300 Subject: [PATCH 004/124] generated optimized svg arch icon --- public/img/svg/octicon-arch.svg | 2 +- web_src/svg/octicon-arch.svg | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 web_src/svg/octicon-arch.svg diff --git a/public/img/svg/octicon-arch.svg b/public/img/svg/octicon-arch.svg index 3a420d58dbf9..264994d15067 100644 --- a/public/img/svg/octicon-arch.svg +++ b/public/img/svg/octicon-arch.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/web_src/svg/octicon-arch.svg b/web_src/svg/octicon-arch.svg new file mode 100644 index 000000000000..3a420d58dbf9 --- /dev/null +++ b/web_src/svg/octicon-arch.svg @@ -0,0 +1 @@ + \ No newline at end of file From 808a7aed6725661fb43e59f2e319a4dd2d61b44e Mon Sep 17 00:00:00 2001 From: dancheg97 Date: Wed, 21 Jun 2023 14:05:25 +0300 Subject: [PATCH 005/124] replaced zstd with archiver and limit reader to speed up metadata ejection and remove windows incompatible dependency --- go.mod | 2 +- modules/packages/arch/metadata.go | 15 ++++++++++----- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 7d783a75242c..8e297a6a16ee 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,6 @@ require ( gitea.com/lunny/levelqueue v0.4.2-0.20220729054728-f020868cc2f7 github.com/42wim/sshsig v0.0.0-20211121163825-841cf5bbc121 github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 - github.com/DataDog/zstd v1.4.5 github.com/NYTimes/gziphandler v1.1.1 github.com/ProtonMail/gopenpgp/v2 v2.7.1 github.com/PuerkitoBio/goquery v1.8.1 @@ -133,6 +132,7 @@ require ( git.sr.ht/~mariusor/go-xsd-duration v0.0.0-20220703122237-02e73435a078 // indirect github.com/ClickHouse/ch-go v0.55.0 // indirect github.com/ClickHouse/clickhouse-go/v2 v2.9.1 // indirect + github.com/DataDog/zstd v1.4.5 // indirect github.com/Masterminds/goutils v1.1.1 // indirect github.com/Masterminds/semver/v3 v3.2.0 // indirect github.com/Masterminds/sprig/v3 v3.2.3 // indirect diff --git a/modules/packages/arch/metadata.go b/modules/packages/arch/metadata.go index 5433138a67f7..e0fba239e2a8 100644 --- a/modules/packages/arch/metadata.go +++ b/modules/packages/arch/metadata.go @@ -4,11 +4,13 @@ package arch import ( + "bytes" "crypto/md5" "crypto/sha256" "encoding/hex" "errors" "fmt" + "io" "io/fs" "os" "path" @@ -16,7 +18,6 @@ import ( "strings" "time" - "github.com/DataDog/zstd" "github.com/mholt/archiver/v3" ) @@ -50,11 +51,15 @@ type Metadata struct { // Function that recieves arch package archive data and returns it's metadata. func EjectMetadata(filename, domain string, pkg []byte) (*Metadata, error) { - d, err := zstd.Decompress(nil, pkg) + pkgreader := io.LimitReader(bytes.NewReader(pkg), 250000) + var buf bytes.Buffer + err := archiver.DefaultZstd.Decompress(pkgreader, &buf) if err != nil { - return nil, err + if !errors.Is(err, io.ErrUnexpectedEOF) { + return nil, err + } } - splt := strings.Split(string(d), "PKGINFO") + splt := strings.Split(buf.String(), "PKGINFO") if len(splt) < 2 { return nil, errors.New("unable to eject .PKGINFO from archive") } @@ -87,7 +92,7 @@ func EjectMetadata(filename, domain string, pkg []byte) (*Metadata, error) { MakeDepends: ejectStrings(raw, "makedepend"), CheckDepends: ejectStrings(raw, "checkdepend"), Backup: ejectStrings(raw, "backup"), - }, err + }, nil } func ejectString(raw, field string) string { From b354f027ff03ff50cf4df8ba5a36281feb10a6c4 Mon Sep 17 00:00:00 2001 From: dancheg97 Date: Wed, 21 Jun 2023 23:30:07 +0300 Subject: [PATCH 006/124] code refactoring and package version publisher id correction --- modules/packages/arch/metadata.go | 18 ++ modules/packages/content_store.go | 15 ++ routers/api/packages/arch/arch.go | 344 ++++--------------------- routers/api/packages/arch/connector.go | 133 ---------- services/packages/arch/db_manager.go | 77 ++++++ services/packages/arch/file_manager.go | 190 ++++++++++++++ services/packages/arch/verificator.go | 110 ++++++++ 7 files changed, 453 insertions(+), 434 deletions(-) delete mode 100644 routers/api/packages/arch/connector.go create mode 100644 services/packages/arch/db_manager.go create mode 100644 services/packages/arch/file_manager.go create mode 100644 services/packages/arch/verificator.go diff --git a/modules/packages/arch/metadata.go b/modules/packages/arch/metadata.go index e0fba239e2a8..4b3631f2f20d 100644 --- a/modules/packages/arch/metadata.go +++ b/modules/packages/arch/metadata.go @@ -258,3 +258,21 @@ func PackDb(src, dst string) error { } return os.Symlink(dst, symlink) } + +// Join database or package names to prevent collisions with same packages in +// different user spaces. Skips empty strings and returns name joined with +// dots. +func Join(s ...string) string { + rez := "" + for i, v := range s { + if v == "" { + continue + } + if i+1 == len(s) { + rez += v + continue + } + rez += v + "." + } + return rez +} diff --git a/modules/packages/content_store.go b/modules/packages/content_store.go index 1181fa4d528b..4f53491022ae 100644 --- a/modules/packages/content_store.go +++ b/modules/packages/content_store.go @@ -4,6 +4,7 @@ package packages import ( + "bytes" "io" "path" "strings" @@ -63,3 +64,17 @@ func RelativePathToKey(relativePath string) (BlobHash256Key, error) { return BlobHash256Key(parts[2]), nil } + +// Save data with specified string key. +func (s *ContentStore) SaveStrBytes(key string, data []byte) error { + return s.Save(BlobHash256Key(key), bytes.NewReader(data), int64(len(data))) +} + +// Get data related to provided key. +func (s *ContentStore) GetStrBytes(key string) ([]byte, error) { + obj, err := s.Get(BlobHash256Key(key)) + if err != nil { + return nil, err + } + return io.ReadAll(obj) +} diff --git a/routers/api/packages/arch/arch.go b/routers/api/packages/arch/arch.go index 0918d98e5b8d..d7b54f0638a7 100644 --- a/routers/api/packages/arch/arch.go +++ b/routers/api/packages/arch/arch.go @@ -4,43 +4,27 @@ package arch import ( - "bytes" "encoding/hex" - "errors" - "fmt" "io" "net/http" - "os" - "path" "strings" - "code.gitea.io/gitea/models/db" - packages_model "code.gitea.io/gitea/models/packages" - repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/modules/context" - "code.gitea.io/gitea/modules/json" - packages_module "code.gitea.io/gitea/modules/packages" arch_module "code.gitea.io/gitea/modules/packages/arch" "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/modules/timeutil" "code.gitea.io/gitea/routers/api/packages/helper" - packages_service "code.gitea.io/gitea/services/packages" - - "github.com/ProtonMail/gopenpgp/v2/crypto" - "github.com/google/uuid" + arch_service "code.gitea.io/gitea/services/packages/arch" ) // Push new package to arch package registry. func Push(ctx *context.Context) { - // Creating connector that will help with keys/blobs. - connector := Connector{ctx: ctx} - - // Getting some information related to package from headers. - filename := ctx.Req.Header.Get("filename") - email := ctx.Req.Header.Get("email") - sign := ctx.Req.Header.Get("sign") - owner := ctx.Req.Header.Get("owner") - distro := ctx.Req.Header.Get("distro") + var ( + filename = ctx.Req.Header.Get("filename") + email = ctx.Req.Header.Get("email") + sign = ctx.Req.Header.Get("sign") + owner = ctx.Req.Header.Get("owner") + distro = ctx.Req.Header.Get("distro") + ) // Decoding package signature. sigdata, err := hex.DecodeString(sign) @@ -48,49 +32,6 @@ func Push(ctx *context.Context) { apiError(ctx, http.StatusBadRequest, err) return } - pgpsig := crypto.NewPGPSignature(sigdata) - - // Validating that user is allowed to push to specified namespace. - err = connector.ValidateNamespace(owner, email) - if err != nil { - apiError(ctx, http.StatusBadRequest, err) - return - } - - // Getting GPG keys related to specific user. After keys have been recieved, - // this function will find one key related to email provided in request. - armoredKeys, err := connector.GetValidKeys(email) - if err != nil { - apiError(ctx, http.StatusBadRequest, err) - return - } - var matchedKeyring *crypto.KeyRing - for _, armor := range armoredKeys { - pgpkey, err := crypto.NewKeyFromArmored(armor) - if err != nil { - apiError(ctx, http.StatusBadRequest, err) - return - } - keyring, err := crypto.NewKeyRing(pgpkey) - if err != nil { - apiError(ctx, http.StatusBadRequest, err) - return - } - for _, idnt := range keyring.GetIdentities() { - if idnt.Email == email { - matchedKeyring = keyring - break - } - } - if matchedKeyring != nil { - break - } - } - if matchedKeyring == nil { - msg := "GPG key related to " + email + " not found" - apiError(ctx, http.StatusBadRequest, msg) - return - } // Read package to memory and create plain GPG message to validate signature. pkgdata, err := io.ReadAll(ctx.Req.Body) @@ -100,23 +41,19 @@ func Push(ctx *context.Context) { } defer ctx.Req.Body.Close() - pgpmes := crypto.NewPlainMessage(pkgdata) - - // Validate package signature with user's GPG key related to his email. - err = matchedKeyring.VerifyDetached(pgpmes, pgpsig, crypto.GetUnixTime()) + // Get user and organization owning arch package. + user, org, err := arch_service.IdentifyOwner(ctx, owner, email) if err != nil { - apiError(ctx, http.StatusUnauthorized, "unable to validate package signature") + apiError(ctx, http.StatusUnauthorized, err) return } - // Create temporary directory for arch database operations. - tmpdir := path.Join(setting.Repository.Upload.TempPath, uuid.New().String()) - err = os.MkdirAll(tmpdir, os.ModePerm) + // Validate package signature with user's GnuPG key. + err = arch_service.ValidatePackageSignature(ctx, pkgdata, sigdata, user) if err != nil { - apiError(ctx, http.StatusInternalServerError, "unable to create tmp path") + apiError(ctx, http.StatusUnauthorized, err) return } - defer os.RemoveAll(tmpdir) // Parse metadata contained in arch package archive. md, err := arch_module.EjectMetadata(filename, setting.Domain, pkgdata) @@ -125,211 +62,43 @@ func Push(ctx *context.Context) { return } - // Arch database related filenames, pathes and folders. - dbname := Join(owner, distro, setting.Domain, "db.tar.gz") - dbpath := path.Join(tmpdir, dbname) - dbfolder := path.Join(tmpdir, dbname) + ".folder" - dbsymlink := strings.TrimSuffix(dbname, ".tar.gz") - dbsymlinkpath := path.Join(tmpdir, dbsymlink) - - // Get existing arch package database, related to specific userspace from - // file storage, and save it on disk, then unpack it's contents to related - // folder. If database is not found in storage, create empty directory to - // store package related information. - dbdata, err := connector.Get(dbname) - if err == nil { - err = os.WriteFile(dbpath, dbdata, os.ModePerm) - if err != nil { - apiError(ctx, http.StatusInternalServerError, err) - return - } - err = arch_module.UnpackDb(dbpath, dbfolder) - if err != nil { - apiError(ctx, http.StatusInternalServerError, err) - return - } - } - if err != nil { - err = os.MkdirAll(dbfolder, os.ModePerm) - if err != nil { - apiError(ctx, http.StatusInternalServerError, err) - return - } - } - - // Update database folder with metadata for new package. - err = md.PutToDb(dbfolder, os.ModePerm) + // Get package property from DB if exists/create new one. + dbpkg, err := arch_service.CreateGetPackage(ctx, org, md.Name) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return } - // Create database archive and related symlink. - err = arch_module.PackDb(dbfolder, dbpath) + // Create or get package version from DB if exists/create new one. + dbpkgver, err := arch_service.CreateGetPackageVersion(ctx, md, dbpkg, user) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return } - // Save namespace related arch repository database. - f, err := os.Open(dbpath) - if err != nil { - apiError(ctx, http.StatusInternalServerError, err) - return - } - defer f.Close() - dbfi, err := f.Stat() - if err != nil { - apiError(ctx, http.StatusInternalServerError, err) - return - } - err = connector.Save(dbname, f, dbfi.Size()) + // Automatically connect repository for provided package if name matched. + err = arch_service.RepositoryAutoconnect(ctx, owner, md.Name, dbpkg) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return } - // Save namespace related arch repository db archive. - f, err = os.Open(dbsymlinkpath) - if err != nil { - apiError(ctx, http.StatusInternalServerError, err) - return - } - defer f.Close() - dbarchivefi, err := f.Stat() - if err != nil { - apiError(ctx, http.StatusInternalServerError, err) - return - } - err = connector.Save(dbsymlink, f, dbarchivefi.Size()) + // Save package file data to gitea storage and update database. + err = arch_service.SavePackageFile(ctx, pkgdata, distro, filename, dbpkgver.ID) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return } - // Create package in database. - pkg, err := packages_model.TryInsertPackage(ctx, &packages_model.Package{ - OwnerID: connector.org.ID, - Type: packages_model.TypeArch, - Name: md.Name, - LowerName: strings.ToLower(md.Name), - }) - if errors.Is(err, packages_model.ErrDuplicatePackage) { - pkg, err = packages_model.GetPackageByName( - ctx, connector.org.ID, - packages_model.TypeArch, md.Name, - ) - if err != nil { - apiError(ctx, http.StatusInternalServerError, err) - return - } - } + // Save package signature data to gitea storage and update database. + err = arch_service.SavePackageFile(ctx, sigdata, distro, filename+".sig", dbpkgver.ID) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return } - // Check if repository for package with provided owner exists. - repo, err := repo_model.GetRepositoryByOwnerAndName(ctx, owner, md.Name) - if err == nil { - err = packages_model.SetRepositoryLink(ctx, pkg.ID, repo.ID) - if err != nil { - apiError(ctx, http.StatusInternalServerError, err) - return - } - } - - // Create new package version in database. - rawjsonmetadata, err := json.Marshal(&md) - if err != nil { - apiError(ctx, http.StatusInternalServerError, err) - return - } - - ver, err := packages_model.GetOrInsertVersion(ctx, &packages_model.PackageVersion{ - PackageID: pkg.ID, - CreatorID: connector.org.ID, - Version: md.Version, - LowerVersion: strings.ToLower(md.Version), - CreatedUnix: timeutil.TimeStampNow(), - MetadataJSON: string(rawjsonmetadata), - }) - if err != nil { - if errors.Is(err, packages_model.ErrDuplicatePackageVersion) { - apiError(ctx, http.StatusConflict, err) - return - } - apiError(ctx, http.StatusInternalServerError, err) - return - } - - // Create package blob and db file for package file. - pkgreader := bytes.NewReader(pkgdata) - fbuf, err := packages_module.CreateHashedBufferFromReader(pkgreader) - if err != nil { - apiError(ctx, http.StatusInternalServerError, err) - return - } - defer fbuf.Close() - - filepb, ok, err := packages_model.GetOrInsertBlob( - ctx, packages_service.NewPackageBlob(fbuf), - ) - if err != nil { - apiError(ctx, http.StatusInternalServerError, fmt.Errorf("%v %t", err, ok)) - return - } - err = connector.Save(filepb.HashSHA256, fbuf, filepb.Size) - if err != nil { - apiError(ctx, http.StatusInternalServerError, err) - return - } - - _, err = packages_model.TryInsertFile(ctx, &packages_model.PackageFile{ - VersionID: ver.ID, - BlobID: filepb.ID, - Name: filename, - LowerName: strings.ToLower(filename), - CompositeKey: distro + "-" + filename, - IsLead: true, - CreatedUnix: timeutil.TimeStampNow(), - }) - if err != nil { - apiError(ctx, http.StatusInternalServerError, err) - return - } - - // Create package blob for package signature. - sigreader := bytes.NewReader(sigdata) - sbuf, err := packages_module.CreateHashedBufferFromReader(sigreader) - if err != nil { - apiError(ctx, http.StatusInternalServerError, err) - return - } - defer fbuf.Close() - - sigpb, ok, err := packages_model.GetOrInsertBlob( - ctx, packages_service.NewPackageBlob(sbuf), - ) - if err != nil { - apiError(ctx, http.StatusInternalServerError, fmt.Errorf("%v %t", err, ok)) - return - } - err = connector.Save(sigpb.HashSHA256, sbuf, sigpb.Size) - if err != nil { - apiError(ctx, http.StatusInternalServerError, err) - return - } - - _, err = packages_model.TryInsertFile(ctx, &packages_model.PackageFile{ - VersionID: ver.ID, - BlobID: sigpb.ID, - Name: filename + ".sig", - LowerName: strings.ToLower(filename + ".sig"), - CompositeKey: distro + "-" + filename + ".sig", - IsLead: false, - CreatedUnix: timeutil.TimeStampNow(), - }) + // Update pacman databases with new package. + err = arch_service.UpdatePacmanDatabases(ctx, md, distro, owner) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return @@ -340,65 +109,38 @@ func Push(ctx *context.Context) { // Get file from arch package registry. func Get(ctx *context.Context) { - filename := ctx.Params("file") - owner := ctx.Params("owner") - distro := ctx.Params("distro") - // arch := ctx.Params("arch") - - cs := packages_module.NewContentStore() - - if strings.HasSuffix(filename, "tar.zst") || strings.HasSuffix(filename, "zst.sig") { - db := db.GetEngine(ctx) - - pkgfile := &packages_model.PackageFile{ - CompositeKey: distro + "-" + filename, - } - ok, err := db.Get(pkgfile) - if err != nil || !ok { - apiError( - ctx, http.StatusInternalServerError, - fmt.Errorf("%+v %t", err, ok), - ) - return - } - - blob, err := packages_model.GetBlobByID(ctx, pkgfile.BlobID) - if err != nil { - apiError(ctx, http.StatusInternalServerError, err) - return - } + var ( + file = ctx.Params("file") + owner = ctx.Params("owner") + distro = ctx.Params("distro") + arch = ctx.Params("arch") + ) - obj, err := cs.Get(packages_module.BlobHash256Key(blob.HashSHA256)) + // Packages are stored in different way from pacman databases, and loaded + // with LoadPackageFile function. + if strings.HasSuffix(file, "tar.zst") || strings.HasSuffix(file, "zst.sig") { + pkgdata, err := arch_service.LoadPackageFile(ctx, distro, file) if err != nil { - apiError(ctx, http.StatusInternalServerError, err) + apiError(ctx, http.StatusNotFound, err) return } - data, err := io.ReadAll(obj) + _, err = ctx.Resp.Write(pkgdata) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return } - _, err = ctx.Resp.Write(data) - if err != nil { - apiError(ctx, http.StatusInternalServerError, err) - return - } ctx.Resp.WriteHeader(http.StatusOK) - return } - if strings.HasSuffix(filename, ".db.tar.gz") || strings.HasSuffix(filename, ".db") { - filename = strings.TrimPrefix(filename, owner+".") - obj, err := cs.Get(packages_module.BlobHash256Key(Join(owner, distro, filename))) - if err != nil { - apiError(ctx, http.StatusNotFound, err) - } - data, err := io.ReadAll(obj) + // Pacman databases are stored directly in gitea file storage and could be + // loaded with name as a key. + if strings.HasSuffix(file, ".db.tar.gz") || strings.HasSuffix(file, ".db") { + data, err := arch_service.LoadPacmanDatabase(ctx, owner, distro, arch, file) if err != nil { - apiError(ctx, http.StatusInternalServerError, err) + apiError(ctx, http.StatusNotFound, err) return } diff --git a/routers/api/packages/arch/connector.go b/routers/api/packages/arch/connector.go deleted file mode 100644 index e19215c5a19a..000000000000 --- a/routers/api/packages/arch/connector.go +++ /dev/null @@ -1,133 +0,0 @@ -// Copyright 2023 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package arch - -import ( - "errors" - "io" - - "code.gitea.io/gitea/models/asymkey" - "code.gitea.io/gitea/models/db" - organization_model "code.gitea.io/gitea/models/organization" - user_model "code.gitea.io/gitea/models/user" - "code.gitea.io/gitea/modules/context" - "code.gitea.io/gitea/modules/log" - packages_module "code.gitea.io/gitea/modules/packages" -) - -// Connector helps to retrieve GPG keys related to package validation and -// manage blobs related to specific user spaces: -// 1 - Check if user is allowed to push package to specific namespace. -// 2 - Retrieving GPG keys related to provided email. -// 3 - Get/put arch arch package/signature/database files to connected file -// storage. -type Connector struct { - ctx *context.Context - user *user_model.User - org *organization_model.Organization -} - -// This function will find user related to provided email adress and check if -// he is able to push packages to provided namespace (user/organization/or -// empty namespace allowed for admin users). -func (c *Connector) ValidateNamespace(namespace, email string) error { - var err error - c.user, err = user_model.GetUserByEmail(c.ctx, email) - if err != nil { - log.Error("unable to get user with email: %s %v", email, err) - return err - } - - if namespace == "" && c.user.IsAdmin { - c.org = (*organization_model.Organization)(c.user) - return nil - } - - if c.user.Name != namespace && c.org == nil { - c.org, err = organization_model.GetOrgByName(c.ctx, namespace) - if err != nil { - log.Error("unable to organization: %s %v", namespace, err) - return err - } - ismember, err := c.org.IsOrgMember(c.user.ID) - if err != nil { - log.Error( - "unable to check if user belongs to organization: %s %s %v", - c.user.Name, email, err, - ) - return err - } - if !ismember { - log.Error("user %s is not member of organization: %s", c.user.Name, email) - return errors.New("user is not member of organization: " + namespace) - } - } else { - c.org = (*organization_model.Organization)(c.user) - } - return nil -} - -// This function will try to find user related to specific email. And check -// that user is allowed to push to 'owner' namespace (package owner, could -// be empty, user or organization). -// After namespace check, this function -func (c *Connector) GetValidKeys(email string) ([]string, error) { - keys, err := asymkey.ListGPGKeys(c.ctx, c.user.ID, db.ListOptions{ - ListAll: true, - }) - if err != nil { - log.Error("unable to get keys related to user: %v", err) - return nil, errors.New("unable to get public keys") - } - if len(keys) == 0 { - log.Error("no keys related to user") - return nil, errors.New("no keys for user with email: " + email) - } - - var keyarmors []string - for _, key := range keys { - k, err := asymkey.GetGPGImportByKeyID(key.KeyID) - if err != nil { - log.Error("unable to import GPG key by ID: %v", err) - return nil, errors.New("internal error") - } - keyarmors = append(keyarmors, k.Content) - } - - return keyarmors, nil -} - -// Get specific file content from content storage. -func (c *Connector) Get(key string) ([]byte, error) { - cs := packages_module.NewContentStore() - obj, err := cs.Get(packages_module.BlobHash256Key(key)) - if err != nil { - return nil, err - } - return io.ReadAll(obj) -} - -// Save contents related to specific arch package. -func (c *Connector) Save(key string, content io.Reader, size int64) error { - cs := packages_module.NewContentStore() - return cs.Save(packages_module.BlobHash256Key(key), content, size) -} - -// Join database or package names to prevent collisions with same packages in -// different user spaces. Skips empty strings and returns name joined with -// dots. -func Join(s ...string) string { - rez := "" - for i, v := range s { - if v == "" { - continue - } - if i+1 == len(s) { - rez += v - continue - } - rez += v + "." - } - return rez -} diff --git a/services/packages/arch/db_manager.go b/services/packages/arch/db_manager.go new file mode 100644 index 000000000000..163246a1e133 --- /dev/null +++ b/services/packages/arch/db_manager.go @@ -0,0 +1,77 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package arch + +import ( + "errors" + "fmt" + "strings" + + org "code.gitea.io/gitea/models/organization" + pkg "code.gitea.io/gitea/models/packages" + "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/json" + "code.gitea.io/gitea/modules/packages/arch" + "code.gitea.io/gitea/modules/timeutil" +) + +// This function will create new package in database, if it does not exist it +// will get existing and return it back to user. +func CreateGetPackage(ctx *context.Context, o *org.Organization, name string) (*pkg.Package, error) { + pack, err := pkg.TryInsertPackage(ctx, &pkg.Package{ + OwnerID: o.ID, + Type: pkg.TypeArch, + Name: name, + LowerName: strings.ToLower(name), + }) + if errors.Is(err, pkg.ErrDuplicatePackage) { + pack, err = pkg.GetPackageByName(ctx, o.ID, pkg.TypeArch, name) + if err != nil { + return nil, fmt.Errorf("unable to get package %s in organization %s", name, o.Name) + } + } + if err != nil { + return nil, err + } + return pack, nil +} + +// This function will create new version for package, or find and return existing. +func CreateGetPackageVersion(ctx *context.Context, md *arch.Metadata, p *pkg.Package, u *user.User) (*pkg.PackageVersion, error) { + rawjsonmetadata, err := json.Marshal(&md) + if err != nil { + return nil, err + } + + ver, err := pkg.GetOrInsertVersion(ctx, &pkg.PackageVersion{ + PackageID: p.ID, + CreatorID: u.ID, + Version: md.Version, + LowerVersion: strings.ToLower(md.Version), + CreatedUnix: timeutil.TimeStampNow(), + MetadataJSON: string(rawjsonmetadata), + }) + if err != nil { + if errors.Is(err, pkg.ErrDuplicatePackageVersion) { + return ver, nil + } + return nil, err + } + return ver, nil +} + +// Automatically connect repository to pushed package, if package with provided +// with provided name exists in namespace scope. +func RepositoryAutoconnect(ctx *context.Context, owner, repository string, p *pkg.Package) error { + repo, err := repo.GetRepositoryByOwnerAndName(ctx, owner, repository) + if err == nil { + err = pkg.SetRepositoryLink(ctx, p.ID, repo.ID) + if err != nil { + return err + } + } + return nil +} diff --git a/services/packages/arch/file_manager.go b/services/packages/arch/file_manager.go new file mode 100644 index 000000000000..8a935560b9a4 --- /dev/null +++ b/services/packages/arch/file_manager.go @@ -0,0 +1,190 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package arch + +import ( + "bytes" + "fmt" + "io" + "os" + "path" + "strings" + + "code.gitea.io/gitea/models/db" + pkg_mdl "code.gitea.io/gitea/models/packages" + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/packages" + "code.gitea.io/gitea/modules/packages/arch" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/timeutil" + pkg_svc "code.gitea.io/gitea/services/packages" + "github.com/google/uuid" +) + +// Save package file to content store for the provided version id and specified distribution. +func SavePackageFile(ctx *context.Context, data []byte, distro, filename string, pkgverid int64) error { + buf, err := packages.CreateHashedBufferFromReader(bytes.NewReader(data)) + if err != nil { + return err + } + defer buf.Close() + + blob, _, err := pkg_mdl.GetOrInsertBlob(ctx, pkg_svc.NewPackageBlob(buf)) + if err != nil { + return err + } + + cs := packages.NewContentStore() + err = cs.Save(packages.BlobHash256Key(blob.HashSHA256), buf, blob.Size) + if err != nil { + return err + } + + _, err = pkg_mdl.TryInsertFile(ctx, &pkg_mdl.PackageFile{ + VersionID: pkgverid, + BlobID: blob.ID, + Name: filename, + LowerName: strings.ToLower(filename), + CompositeKey: distro + "-" + filename, + CreatedUnix: timeutil.TimeStampNow(), + }) + return err +} + +// Get data related to provided file name and distribution. +func LoadPackageFile(ctx *context.Context, distro, file string) ([]byte, error) { + db := db.GetEngine(ctx) + + pkgfile := &pkg_mdl.PackageFile{CompositeKey: distro + "-" + file} + + ok, err := db.Get(pkgfile) + if err != nil || !ok { + return nil, fmt.Errorf("%+v %t", err, ok) + } + + blob, err := pkg_mdl.GetBlobByID(ctx, pkgfile.BlobID) + if err != nil { + return nil, err + } + + cs := packages.NewContentStore() + + obj, err := cs.Get(packages.BlobHash256Key(blob.HashSHA256)) + if err != nil { + return nil, err + } + + return io.ReadAll(obj) +} + +// Get data related to pacman database file or symlink. +func LoadPacmanDatabase(ctx *context.Context, owner, distro, architecture, file string) ([]byte, error) { + + cs := packages.NewContentStore() + + file = strings.TrimPrefix(file, owner+".") + + obj, err := cs.Get(packages.BlobHash256Key(arch.Join(owner, distro, architecture, file))) + if err != nil { + return nil, err + } + + return io.ReadAll(obj) +} + +// This function will update information about package in related pacman databases +// or create them if they do not exist. +func UpdatePacmanDatabases(ctx *context.Context, md *arch.Metadata, distro, owner string) error { + // Create temporary directory for arch database operations. + tmpdir := path.Join(setting.Repository.Upload.TempPath, uuid.New().String()) + err := os.MkdirAll(tmpdir, os.ModePerm) + if err != nil { + return err + } + defer os.RemoveAll(tmpdir) + + // If architecure is not specified or any, package will be automatically + // saved to databases with most popular architectures. + var architectures = md.Arch + if len(md.Arch) == 0 || md.Arch[0] == "any" { + architectures = []string{ + "x86_64", "arm", "i686", "pentium4", + "armv7h", "armv6h", "aarch64", "riscv64", + } + } + + cs := packages.NewContentStore() + + for _, architecture := range architectures { + var ( + db = arch.Join(owner, distro, architecture, setting.Domain, "db.tar.gz") + dbpth = path.Join(tmpdir, db) + dbf = path.Join(tmpdir, db) + ".folder" + sbsl = strings.TrimSuffix(db, ".tar.gz") + slpth = path.Join(tmpdir, sbsl) + ) + + // Get existing pacman database, or create empty folder for it. + dbdata, err := cs.GetStrBytes(db) + if err == nil { + err = os.WriteFile(dbpth, dbdata, os.ModePerm) + if err != nil { + return err + } + err = arch.UnpackDb(dbpth, dbf) + if err != nil { + return err + } + } + if err != nil { + err = os.MkdirAll(dbf, os.ModePerm) + if err != nil { + return err + } + } + + // Update database folder with metadata for new package. + err = md.PutToDb(dbf, os.ModePerm) + if err != nil { + return err + } + + // Create database archive and related symlink. + err = arch.PackDb(dbf, dbpth) + if err != nil { + return err + } + + // Save database file. + f, err := os.Open(dbpth) + if err != nil { + return err + } + defer f.Close() + dbfi, err := f.Stat() + if err != nil { + return err + } + err = cs.Save(packages.BlobHash256Key(db), f, dbfi.Size()) + if err != nil { + return err + } + + // Save database symlink file. + f, err = os.Open(slpth) + if err != nil { + return err + } + defer f.Close() + dbarchivefi, err := f.Stat() + if err != nil { + return err + } + err = cs.Save(packages.BlobHash256Key(sbsl), f, dbarchivefi.Size()) + if err != nil { + return err + } + } + return nil +} diff --git a/services/packages/arch/verificator.go b/services/packages/arch/verificator.go new file mode 100644 index 000000000000..4a16fcffb1c0 --- /dev/null +++ b/services/packages/arch/verificator.go @@ -0,0 +1,110 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package arch + +import ( + "errors" + "fmt" + + "code.gitea.io/gitea/models/asymkey" + "code.gitea.io/gitea/models/db" + org "code.gitea.io/gitea/models/organization" + "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/context" + "github.com/ProtonMail/gopenpgp/v2/crypto" +) + +type IdentidyOwnerParameters struct { + *context.Context + Owner string + Email string +} + +// This function will find user related to provided email adress and check if +// he is able to push packages to provided namespace (user/organization/or +// empty namespace allowed for admin users). Function will return user making +// operation, organization or user owning the package. +func IdentifyOwner(ctx *context.Context, owner, email string) (*user.User, *org.Organization, error) { + u, err := user.GetUserByEmail(ctx, email) + if err != nil { + return nil, nil, fmt.Errorf("unable to find user with email %s, %v", email, err) + } + + if owner == "" && u.IsAdmin { + return u, (*org.Organization)(u), nil + } + + if owner == u.Name { + return u, (*org.Organization)(u), nil + } + + if u.Name != owner { + org, err := org.GetOrgByName(ctx, owner) + if err != nil { + return nil, nil, fmt.Errorf("unable to get organization: %s, %v", owner, err) + } + ismember, err := org.IsOrgMember(u.ID) + if err != nil { + return nil, nil, fmt.Errorf("unable to check if user %s belongs to organization %s: %v", u.Name, org.Name, err) + } + if !ismember { + return nil, nil, fmt.Errorf("user %s is not member of organization %s", u.Name, org.Name) + } + return u, org, nil + } + return nil, nil, fmt.Errorf("unknown package owner") +} + +// Validate package signature with owner's GnuPG keys stored in gitea's database. +func ValidatePackageSignature(ctx *context.Context, pkg, sign []byte, u *user.User) error { + keys, err := asymkey.ListGPGKeys(ctx, u.ID, db.ListOptions{ + ListAll: true, + }) + if err != nil { + return errors.New("unable to get public keys") + } + if len(keys) == 0 { + return errors.New("no keys for user with email: " + u.Email) + } + + var keyarmors []string + for _, key := range keys { + k, err := asymkey.GetGPGImportByKeyID(key.KeyID) + if err != nil { + return errors.New("unable to import GPG key armor") + } + keyarmors = append(keyarmors, k.Content) + } + + var matchedKeyring *crypto.KeyRing + for _, armor := range keyarmors { + pgpkey, err := crypto.NewKeyFromArmored(armor) + if err != nil { + return fmt.Errorf("unable to get keys for %s: %v", u.Name, err) + } + keyring, err := crypto.NewKeyRing(pgpkey) + if err != nil { + return fmt.Errorf("unable to form keyring %s: %v", u.Name, err) + } + for _, idnt := range keyring.GetIdentities() { + if idnt.Email == u.Email { + matchedKeyring = keyring + break + } + } + if matchedKeyring != nil { + break + } + } + if matchedKeyring == nil { + return fmt.Errorf("GPG key related to %s not found", u.Email) + } + + var ( + pgpmes = crypto.NewPlainMessage(pkg) + pgpsig = crypto.NewPGPSignature(sign) + ) + + return matchedKeyring.VerifyDetached(pgpmes, pgpsig, crypto.GetUnixTime()) +} From af99efb0a0df1218bf33a79af504d2165dbd8298 Mon Sep 17 00:00:00 2001 From: dancheg97 Date: Thu, 22 Jun 2023 09:29:08 +0300 Subject: [PATCH 007/124] moved arch package routes to the same scope with other packages --- routers/api/packages/api.go | 17 ++++------------- routers/api/packages/arch/arch.go | 2 +- routers/init.go | 2 -- 3 files changed, 5 insertions(+), 16 deletions(-) diff --git a/routers/api/packages/api.go b/routers/api/packages/api.go index bbc6ce278277..7b2fb7b125fe 100644 --- a/routers/api/packages/api.go +++ b/routers/api/packages/api.go @@ -123,6 +123,10 @@ func CommonRoutes() *web.Route { }) }) }, reqPackageAccess(perm.AccessModeRead)) + r.Group("/arch", func() { + r.Put("/push", arch.Push) + r.Get("/{distro}/{arch}/{file}", arch.Get) + }) r.Group("/cargo", func() { r.Group("/api/v1/crates", func() { r.Get("", cargo.SearchPackages) @@ -753,16 +757,3 @@ func ContainerRoutes() *web.Route { return r } - -// Routes for arch packages. -func ArchRoutes() *web.Route { - r := web.NewRoute() - - r.Use(context.PackageContexter()) - - r.Put("/push", arch.Push) - r.Get("/{distro}/{arch}/{owner}/{file}", arch.Get) - r.Get("/{distro}/{arch}/{file}", arch.Get) - - return r -} diff --git a/routers/api/packages/arch/arch.go b/routers/api/packages/arch/arch.go index d7b54f0638a7..297430337f90 100644 --- a/routers/api/packages/arch/arch.go +++ b/routers/api/packages/arch/arch.go @@ -19,10 +19,10 @@ import ( // Push new package to arch package registry. func Push(ctx *context.Context) { var ( + owner = ctx.Params("username") filename = ctx.Req.Header.Get("filename") email = ctx.Req.Header.Get("email") sign = ctx.Req.Header.Get("sign") - owner = ctx.Req.Header.Get("owner") distro = ctx.Req.Header.Get("distro") ) diff --git a/routers/init.go b/routers/init.go index 668efdcb859d..54e8d2b8b39e 100644 --- a/routers/init.go +++ b/routers/init.go @@ -190,8 +190,6 @@ func NormalRoutes() *web.Route { r.Mount("/api/packages", packages_router.CommonRoutes()) // This implements the OCI API (Note this is not preceded by /api but is instead /v2) r.Mount("/v2", packages_router.ContainerRoutes()) - // Arch package routes - r.Mount("/api/packages/arch", packages_router.ArchRoutes()) } if setting.Actions.Enabled { From e75ddfd1db278f01f274d91c3d371de19568848c Mon Sep 17 00:00:00 2001 From: dancheg97 Date: Thu, 22 Jun 2023 10:33:23 +0300 Subject: [PATCH 008/124] reused existing services/packages functions for package and signature files/blobs creation --- routers/api/packages/arch/arch.go | 42 ++++++------- services/packages/arch/db_manager.go | 86 +++++++++++++------------- services/packages/arch/file_manager.go | 33 ---------- 3 files changed, 64 insertions(+), 97 deletions(-) diff --git a/routers/api/packages/arch/arch.go b/routers/api/packages/arch/arch.go index 297430337f90..f93c80878b3a 100644 --- a/routers/api/packages/arch/arch.go +++ b/routers/api/packages/arch/arch.go @@ -62,36 +62,36 @@ func Push(ctx *context.Context) { return } - // Get package property from DB if exists/create new one. - dbpkg, err := arch_service.CreateGetPackage(ctx, org, md.Name) + // Save file related to arch package. + pkgid, err := arch_service.SaveFile(ctx, &arch_service.SaveFileParams{ + Organization: org, + User: user, + Metadata: md, + Filename: filename, + Data: pkgdata, + Distro: distro, + }) if err != nil { - apiError(ctx, http.StatusInternalServerError, err) + apiError(ctx, http.StatusBadRequest, err) return } - // Create or get package version from DB if exists/create new one. - dbpkgver, err := arch_service.CreateGetPackageVersion(ctx, md, dbpkg, user) + // Save file related to arch package signature. + _, err = arch_service.SaveFile(ctx, &arch_service.SaveFileParams{ + Organization: org, + User: user, + Metadata: md, + Data: sigdata, + Filename: filename + ".sig", + Distro: distro, + }) if err != nil { - apiError(ctx, http.StatusInternalServerError, err) + apiError(ctx, http.StatusBadRequest, err) return } // Automatically connect repository for provided package if name matched. - err = arch_service.RepositoryAutoconnect(ctx, owner, md.Name, dbpkg) - if err != nil { - apiError(ctx, http.StatusInternalServerError, err) - return - } - - // Save package file data to gitea storage and update database. - err = arch_service.SavePackageFile(ctx, pkgdata, distro, filename, dbpkgver.ID) - if err != nil { - apiError(ctx, http.StatusInternalServerError, err) - return - } - - // Save package signature data to gitea storage and update database. - err = arch_service.SavePackageFile(ctx, sigdata, distro, filename+".sig", dbpkgver.ID) + err = arch_service.RepositoryAutoconnect(ctx, owner, md.Name, pkgid) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return diff --git a/services/packages/arch/db_manager.go b/services/packages/arch/db_manager.go index 163246a1e133..99d1c051326f 100644 --- a/services/packages/arch/db_manager.go +++ b/services/packages/arch/db_manager.go @@ -4,71 +4,71 @@ package arch import ( - "errors" - "fmt" - "strings" + "bytes" org "code.gitea.io/gitea/models/organization" pkg "code.gitea.io/gitea/models/packages" "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/context" - "code.gitea.io/gitea/modules/json" + "code.gitea.io/gitea/modules/packages" "code.gitea.io/gitea/modules/packages/arch" - "code.gitea.io/gitea/modules/timeutil" + svc "code.gitea.io/gitea/services/packages" ) -// This function will create new package in database, if it does not exist it -// will get existing and return it back to user. -func CreateGetPackage(ctx *context.Context, o *org.Organization, name string) (*pkg.Package, error) { - pack, err := pkg.TryInsertPackage(ctx, &pkg.Package{ - OwnerID: o.ID, - Type: pkg.TypeArch, - Name: name, - LowerName: strings.ToLower(name), - }) - if errors.Is(err, pkg.ErrDuplicatePackage) { - pack, err = pkg.GetPackageByName(ctx, o.ID, pkg.TypeArch, name) - if err != nil { - return nil, fmt.Errorf("unable to get package %s in organization %s", name, o.Name) - } - } - if err != nil { - return nil, err - } - return pack, nil +// Parameters required to save new arch package. +type SaveFileParams struct { + *org.Organization + *user.User + *arch.Metadata + Data []byte + Filename string + Distro string } -// This function will create new version for package, or find and return existing. -func CreateGetPackageVersion(ctx *context.Context, md *arch.Metadata, p *pkg.Package, u *user.User) (*pkg.PackageVersion, error) { - rawjsonmetadata, err := json.Marshal(&md) +// This function create new package, version and package file properties in +// database, and write blob to file storage. If package/version/blob exists it +// will overwrite existing data. Package id and error will be returned. +func SaveFile(ctx *context.Context, p *SaveFileParams) (int64, error) { + buf, err := packages.CreateHashedBufferFromReader(bytes.NewReader(p.Data)) if err != nil { - return nil, err + return 0, err } + defer buf.Close() - ver, err := pkg.GetOrInsertVersion(ctx, &pkg.PackageVersion{ - PackageID: p.ID, - CreatorID: u.ID, - Version: md.Version, - LowerVersion: strings.ToLower(md.Version), - CreatedUnix: timeutil.TimeStampNow(), - MetadataJSON: string(rawjsonmetadata), - }) + pv, _, err := svc.CreatePackageOrAddFileToExisting( + &svc.PackageCreationInfo{ + PackageInfo: svc.PackageInfo{ + Owner: p.Organization.AsUser(), + PackageType: pkg.TypeArch, + Name: p.Metadata.Name, + Version: p.Metadata.Version, + }, + Creator: p.User, + Metadata: p.Metadata, + }, + &svc.PackageFileCreationInfo{ + PackageFileInfo: svc.PackageFileInfo{ + Filename: p.Filename, + CompositeKey: p.Distro + "-" + p.Filename, + }, + Creator: p.User, + Data: buf, + OverwriteExisting: true, + }, + ) if err != nil { - if errors.Is(err, pkg.ErrDuplicatePackageVersion) { - return ver, nil - } - return nil, err + return 0, err } - return ver, nil + return pv.PackageID, nil } // Automatically connect repository to pushed package, if package with provided // with provided name exists in namespace scope. -func RepositoryAutoconnect(ctx *context.Context, owner, repository string, p *pkg.Package) error { +func RepositoryAutoconnect(ctx *context.Context, owner, repository string, pkgid int64) error { repo, err := repo.GetRepositoryByOwnerAndName(ctx, owner, repository) if err == nil { - err = pkg.SetRepositoryLink(ctx, p.ID, repo.ID) + err = pkg.SetRepositoryLink(ctx, pkgid, repo.ID) if err != nil { return err } diff --git a/services/packages/arch/file_manager.go b/services/packages/arch/file_manager.go index 8a935560b9a4..86a84338d8d5 100644 --- a/services/packages/arch/file_manager.go +++ b/services/packages/arch/file_manager.go @@ -4,7 +4,6 @@ package arch import ( - "bytes" "fmt" "io" "os" @@ -17,41 +16,9 @@ import ( "code.gitea.io/gitea/modules/packages" "code.gitea.io/gitea/modules/packages/arch" "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/modules/timeutil" - pkg_svc "code.gitea.io/gitea/services/packages" "github.com/google/uuid" ) -// Save package file to content store for the provided version id and specified distribution. -func SavePackageFile(ctx *context.Context, data []byte, distro, filename string, pkgverid int64) error { - buf, err := packages.CreateHashedBufferFromReader(bytes.NewReader(data)) - if err != nil { - return err - } - defer buf.Close() - - blob, _, err := pkg_mdl.GetOrInsertBlob(ctx, pkg_svc.NewPackageBlob(buf)) - if err != nil { - return err - } - - cs := packages.NewContentStore() - err = cs.Save(packages.BlobHash256Key(blob.HashSHA256), buf, blob.Size) - if err != nil { - return err - } - - _, err = pkg_mdl.TryInsertFile(ctx, &pkg_mdl.PackageFile{ - VersionID: pkgverid, - BlobID: blob.ID, - Name: filename, - LowerName: strings.ToLower(filename), - CompositeKey: distro + "-" + filename, - CreatedUnix: timeutil.TimeStampNow(), - }) - return err -} - // Get data related to provided file name and distribution. func LoadPackageFile(ctx *context.Context, distro, file string) ([]byte, error) { db := db.GetEngine(ctx) From ae79dbd7b9bb46114838b0748538fe617d6de03b Mon Sep 17 00:00:00 2001 From: dancheg97 Date: Thu, 22 Jun 2023 11:43:30 +0300 Subject: [PATCH 009/124] package UI corrections --- templates/package/content/arch.tmpl | 20 ++------------------ templates/package/metadata/arch.tmpl | 3 ++- 2 files changed, 4 insertions(+), 19 deletions(-) diff --git a/templates/package/content/arch.tmpl b/templates/package/content/arch.tmpl index 0ecce3550946..f0f967778ee2 100644 --- a/templates/package/content/arch.tmpl +++ b/templates/package/content/arch.tmpl @@ -24,32 +24,16 @@ Server = https://{{.PackageDescriptor.Metadata.BaseDomain}}/api/packages/arch/(d
- - - - - - - - - - - - - - - - - + - + {{range $key := .PackageDescriptor.Metadata.Provides}} diff --git a/templates/package/metadata/arch.tmpl b/templates/package/metadata/arch.tmpl index 8cabeda06a5a..33e5fda9bd30 100644 --- a/templates/package/metadata/arch.tmpl +++ b/templates/package/metadata/arch.tmpl @@ -1,3 +1,4 @@ {{if eq .PackageDescriptor.Package.Type "arch"}} - {{range .PackageDescriptor.Metadata.License}}
{{svg "octicon-law" 16 "gt-mr-3"}} {{.}}
{{end}} + {{range .PackageDescriptor.Metadata.License}}
{{svg "octicon-law" 16 "gt-mr-3"}} {{.}}
{{end}} + {{if .PackageDescriptor.Metadata.URL}}
{{svg "octicon-link-external" 16 "mr-3"}} {{.locale.Tr "packages.details.project_site"}}
{{end}} {{end}} From a5784f68302ab8acbdd94b64a69ce460672ed0a1 Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Fri, 23 Jun 2023 00:26:49 +0300 Subject: [PATCH 010/124] Update web_src/svg/octicon-arch.svg Co-authored-by: silverwind --- web_src/svg/octicon-arch.svg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web_src/svg/octicon-arch.svg b/web_src/svg/octicon-arch.svg index 3a420d58dbf9..c658508989f6 100644 --- a/web_src/svg/octicon-arch.svg +++ b/web_src/svg/octicon-arch.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file From f31e52d8cf7e2d30bac0593a957748080ba68d51 Mon Sep 17 00:00:00 2001 From: dancheg97 Date: Fri, 23 Jun 2023 03:41:08 +0300 Subject: [PATCH 011/124] corrected arch icon reference and optimization --- models/packages/package.go | 2 +- public/img/svg/gitea-arch.svg | 1 + public/img/svg/octicon-arch.svg => web_src/svg/gitea-arch.svg | 0 web_src/svg/octicon-arch.svg | 1 - 4 files changed, 2 insertions(+), 2 deletions(-) create mode 100644 public/img/svg/gitea-arch.svg rename public/img/svg/octicon-arch.svg => web_src/svg/gitea-arch.svg (100%) delete mode 100644 web_src/svg/octicon-arch.svg diff --git a/models/packages/package.go b/models/packages/package.go index 0d0adc03f66b..99c478fa79b0 100644 --- a/models/packages/package.go +++ b/models/packages/package.go @@ -136,7 +136,7 @@ func (pt Type) SVGName() string { case TypeAlpine: return "gitea-alpine" case TypeArch: - return "octicon-arch" + return "gitea-arch" case TypeCargo: return "gitea-cargo" case TypeChef: diff --git a/public/img/svg/gitea-arch.svg b/public/img/svg/gitea-arch.svg new file mode 100644 index 000000000000..c6173d67db99 --- /dev/null +++ b/public/img/svg/gitea-arch.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/public/img/svg/octicon-arch.svg b/web_src/svg/gitea-arch.svg similarity index 100% rename from public/img/svg/octicon-arch.svg rename to web_src/svg/gitea-arch.svg diff --git a/web_src/svg/octicon-arch.svg b/web_src/svg/octicon-arch.svg deleted file mode 100644 index c658508989f6..000000000000 --- a/web_src/svg/octicon-arch.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file From 41667b2dd1321f257f70910330973533578ca154 Mon Sep 17 00:00:00 2001 From: dancheg97 Date: Fri, 23 Jun 2023 03:59:28 +0300 Subject: [PATCH 012/124] changed signature verification library to reuse existing dependency --- go.mod | 2 -- go.sum | 5 ---- services/packages/arch/verificator.go | 36 +++++++++------------------ 3 files changed, 12 insertions(+), 31 deletions(-) diff --git a/go.mod b/go.mod index 8e297a6a16ee..7b7a51efbb7f 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,6 @@ require ( github.com/42wim/sshsig v0.0.0-20211121163825-841cf5bbc121 github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 github.com/NYTimes/gziphandler v1.1.1 - github.com/ProtonMail/gopenpgp/v2 v2.7.1 github.com/PuerkitoBio/goquery v1.8.1 github.com/alecthomas/chroma/v2 v2.7.0 github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb @@ -138,7 +137,6 @@ require ( github.com/Masterminds/sprig/v3 v3.2.3 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect github.com/ProtonMail/go-crypto v0.0.0-20230528122434-6f98819771a1 // indirect - github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f // indirect github.com/RoaringBitmap/roaring v1.2.3 // indirect github.com/acomagu/bufpipe v1.0.4 // indirect github.com/andybalholm/brotli v1.0.5 // indirect diff --git a/go.sum b/go.sum index 686b5505c35a..c3ac719f3f4a 100644 --- a/go.sum +++ b/go.sum @@ -105,13 +105,8 @@ github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cq github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8/go.mod h1:I0gYDMZ6Z5GRU7l58bNFSkPTFN6Yl12dsUlAZ8xy98g= -github.com/ProtonMail/go-crypto v0.0.0-20230321155629-9a39f2531310/go.mod h1:8TI4H3IbrackdNgv+92dI+rhpCaLqM0IfpgCgenFvRE= github.com/ProtonMail/go-crypto v0.0.0-20230528122434-6f98819771a1 h1:JMDGhoQvXNTqH6Y3MC0IUw6tcZvaUdujNqzK2HYWZc8= github.com/ProtonMail/go-crypto v0.0.0-20230528122434-6f98819771a1/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0= -github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f h1:tCbYj7/299ekTTXpdwKYF8eBlsYsDVoggDAuAjoK66k= -github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f/go.mod h1:gcr0kNtGBqin9zDW9GOHcVntrwnjrK+qdJ06mWYBybw= -github.com/ProtonMail/gopenpgp/v2 v2.7.1 h1:Awsg7MPc2gD3I7IFac2qE3Gdls0lZW8SzrFZ3k1oz0s= -github.com/ProtonMail/gopenpgp/v2 v2.7.1/go.mod h1:/BU5gfAVwqyd8EfC3Eu7zmuhwYQpKs+cGD8M//iiaxs= github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAcwmWM= github.com/PuerkitoBio/goquery v1.8.1/go.mod h1:Q8ICL1kNUJ2sXGoAhPGUdYDJvgQgHzJsnnd3H7Ho5jQ= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= diff --git a/services/packages/arch/verificator.go b/services/packages/arch/verificator.go index 4a16fcffb1c0..35694aba3969 100644 --- a/services/packages/arch/verificator.go +++ b/services/packages/arch/verificator.go @@ -4,15 +4,17 @@ package arch import ( + "bytes" "errors" "fmt" + "strings" "code.gitea.io/gitea/models/asymkey" "code.gitea.io/gitea/models/db" org "code.gitea.io/gitea/models/organization" "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/context" - "github.com/ProtonMail/gopenpgp/v2/crypto" + "github.com/keybase/go-crypto/openpgp" ) type IdentidyOwnerParameters struct { @@ -77,34 +79,20 @@ func ValidatePackageSignature(ctx *context.Context, pkg, sign []byte, u *user.Us keyarmors = append(keyarmors, k.Content) } - var matchedKeyring *crypto.KeyRing + var trace []error for _, armor := range keyarmors { - pgpkey, err := crypto.NewKeyFromArmored(armor) + kr, err := openpgp.ReadArmoredKeyRing(strings.NewReader(armor)) if err != nil { - return fmt.Errorf("unable to get keys for %s: %v", u.Name, err) + trace = append(trace, fmt.Errorf("unable to get keys for %s: %v", u.Name, err)) + continue } - keyring, err := crypto.NewKeyRing(pgpkey) + _, err = openpgp.CheckDetachedSignature(kr, bytes.NewReader(pkg), bytes.NewReader(sign)) if err != nil { - return fmt.Errorf("unable to form keyring %s: %v", u.Name, err) - } - for _, idnt := range keyring.GetIdentities() { - if idnt.Email == u.Email { - matchedKeyring = keyring - break - } - } - if matchedKeyring != nil { - break + trace = append(trace, err) + continue } + return nil } - if matchedKeyring == nil { - return fmt.Errorf("GPG key related to %s not found", u.Email) - } - - var ( - pgpmes = crypto.NewPlainMessage(pkg) - pgpsig = crypto.NewPGPSignature(sign) - ) - return matchedKeyring.VerifyDetached(pgpmes, pgpsig, crypto.GetUnixTime()) + return errors.Join(trace...) } From e399ce845256cbb8c7290abbcbea21c9aff43c4a Mon Sep 17 00:00:00 2001 From: dancheg97 Date: Sat, 24 Jun 2023 13:37:33 +0300 Subject: [PATCH 013/124] opdated operation pacman database craetion to form or update database file in memory, added download counter related to pkg.tar.zst files --- modules/packages/arch/metadata.go | 137 +++++++++++++++---------- modules/packages/content_store.go | 15 --- routers/api/packages/arch/arch.go | 7 +- services/packages/arch/file_manager.go | 78 ++++---------- 4 files changed, 112 insertions(+), 125 deletions(-) diff --git a/modules/packages/arch/metadata.go b/modules/packages/arch/metadata.go index 4b3631f2f20d..d42ef1e2ffe7 100644 --- a/modules/packages/arch/metadata.go +++ b/modules/packages/arch/metadata.go @@ -4,16 +4,16 @@ package arch import ( + "archive/tar" "bytes" + "compress/gzip" "crypto/md5" "crypto/sha256" "encoding/hex" "errors" "fmt" "io" - "io/fs" "os" - "path" "strconv" "strings" "time" @@ -209,56 +209,6 @@ func rmEmptyStrings(s []string) []string { return r } -// Function takes path to directory with pacman database and updates package -// it with current metadata. -func (m *Metadata) PutToDb(dir string, mode fs.FileMode) error { - descdir := path.Join(dir, m.Name+"-"+m.Version) - err := os.MkdirAll(descdir, mode) - if err != nil { - return err - } - return os.WriteFile(path.Join(descdir, "desc"), []byte(m.GetDbDesc()), mode) -} - -// Function takes raw database archive bytes and destination directory as -// arguements and unpacks database contents to destination directory. -func UnpackDb(src, dst string) error { - return archiver.DefaultTarGz.Unarchive(src, dst) -} - -// Function takes path to source directory with raw pacman description files -// for pacman database, creates db.tar.gz archive and related symlink for -// provided path. -func PackDb(src, dst string) error { - if !strings.HasSuffix(dst, ".db.tar.gz") { - return fmt.Errorf("dst should end with '.db.tar.gz': %s", dst) - } - symlink := strings.TrimSuffix(dst, ".tar.gz") - if _, err := os.Stat(dst); err == nil { - err = os.RemoveAll(dst) - if err != nil { - return err - } - err = os.RemoveAll(symlink) - if err != nil { - return err - } - } - des, err := os.ReadDir(src) - if err != nil { - return err - } - var pkgdescs []string - for _, de := range des { - pkgdescs = append(pkgdescs, path.Join(src, de.Name())) - } - err = archiver.DefaultTarGz.Archive(pkgdescs, dst) - if err != nil { - return err - } - return os.Symlink(dst, symlink) -} - // Join database or package names to prevent collisions with same packages in // different user spaces. Skips empty strings and returns name joined with // dots. @@ -276,3 +226,86 @@ func Join(s ...string) string { } return rez } + +// Add or update existing package entry in database archived data. +func UpdatePacmanDbEntry(db []byte, md *Metadata) ([]byte, error) { + // Read existing entries in archive. + entries, err := readEntries(db) + if err != nil { + return nil, err + } + + // Add new package entry to list. + entries[md.Name+"-"+md.Version+"/desc"] = []byte(md.GetDbDesc()) + + fmt.Println(entries) + + var out bytes.Buffer + + // Write entries to new buffer and return it. + err = writeToArchive(entries, &out) + if err != nil { + return nil, err + } + + return out.Bytes(), nil +} + +// Read database entries containing in pacman archive. +func readEntries(dbarchive []byte) (map[string][]byte, error) { + gzf, err := gzip.NewReader(bytes.NewReader(dbarchive)) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + + var entries = map[string][]byte{} + + tarReader := tar.NewReader(gzf) + for { + header, err := tarReader.Next() + if err == io.EOF { + break + } + + if err != nil { + return nil, err + } + + if header.Typeflag == tar.TypeReg { + content, err := io.ReadAll(tarReader) + if err != nil { + return nil, err + } + entries[header.Name] = content + } + } + return entries, nil +} + +// Write pacman package entries to empty buffer. +func writeToArchive(files map[string][]byte, buf io.Writer) error { + gw := gzip.NewWriter(buf) + defer gw.Close() + tw := tar.NewWriter(gw) + defer tw.Close() + + for name, content := range files { + hdr := &tar.Header{ + Name: name, + Size: int64(len(content)), + Mode: int64(os.ModePerm), + } + + err := tw.WriteHeader(hdr) + if err != nil { + return err + } + + _, err = io.Copy(tw, bytes.NewReader(content)) + if err != nil { + return err + } + } + return nil +} diff --git a/modules/packages/content_store.go b/modules/packages/content_store.go index 4f53491022ae..1181fa4d528b 100644 --- a/modules/packages/content_store.go +++ b/modules/packages/content_store.go @@ -4,7 +4,6 @@ package packages import ( - "bytes" "io" "path" "strings" @@ -64,17 +63,3 @@ func RelativePathToKey(relativePath string) (BlobHash256Key, error) { return BlobHash256Key(parts[2]), nil } - -// Save data with specified string key. -func (s *ContentStore) SaveStrBytes(key string, data []byte) error { - return s.Save(BlobHash256Key(key), bytes.NewReader(data), int64(len(data))) -} - -// Get data related to provided key. -func (s *ContentStore) GetStrBytes(key string) ([]byte, error) { - obj, err := s.Get(BlobHash256Key(key)) - if err != nil { - return nil, err - } - return io.ReadAll(obj) -} diff --git a/routers/api/packages/arch/arch.go b/routers/api/packages/arch/arch.go index f93c80878b3a..18cd9ef5b3b7 100644 --- a/routers/api/packages/arch/arch.go +++ b/routers/api/packages/arch/arch.go @@ -33,7 +33,7 @@ func Push(ctx *context.Context) { return } - // Read package to memory and create plain GPG message to validate signature. + // Read package to memory for signature validation. pkgdata, err := io.ReadAll(ctx.Req.Body) if err != nil { apiError(ctx, http.StatusInternalServerError, err) @@ -48,7 +48,7 @@ func Push(ctx *context.Context) { return } - // Validate package signature with user's GnuPG key. + // Validate package signature with any of user's GnuPG keys. err = arch_service.ValidatePackageSignature(ctx, pkgdata, sigdata, user) if err != nil { apiError(ctx, http.StatusUnauthorized, err) @@ -149,8 +149,11 @@ func Get(ctx *context.Context) { apiError(ctx, http.StatusInternalServerError, err) return } + ctx.Resp.WriteHeader(http.StatusOK) + return } + ctx.Resp.WriteHeader(http.StatusNotFound) } diff --git a/services/packages/arch/file_manager.go b/services/packages/arch/file_manager.go index 86a84338d8d5..c39e11dcd3ce 100644 --- a/services/packages/arch/file_manager.go +++ b/services/packages/arch/file_manager.go @@ -4,6 +4,7 @@ package arch import ( + "bytes" "fmt" "io" "os" @@ -19,7 +20,8 @@ import ( "github.com/google/uuid" ) -// Get data related to provided file name and distribution. +// Get data related to provided file name and distribution, and update download +// counter if actual package file is retrieved from database. func LoadPackageFile(ctx *context.Context, distro, file string) ([]byte, error) { db := db.GetEngine(ctx) @@ -35,6 +37,13 @@ func LoadPackageFile(ctx *context.Context, distro, file string) ([]byte, error) return nil, err } + if strings.HasSuffix(file, ".pkg.tar.zst") { + err = pkg_mdl.IncrementDownloadCounter(ctx, pkgfile.VersionID) + if err != nil { + return nil, err + } + } + cs := packages.NewContentStore() obj, err := cs.Get(packages.BlobHash256Key(blob.HashSHA256)) @@ -47,12 +56,13 @@ func LoadPackageFile(ctx *context.Context, distro, file string) ([]byte, error) // Get data related to pacman database file or symlink. func LoadPacmanDatabase(ctx *context.Context, owner, distro, architecture, file string) ([]byte, error) { - cs := packages.NewContentStore() file = strings.TrimPrefix(file, owner+".") - obj, err := cs.Get(packages.BlobHash256Key(arch.Join(owner, distro, architecture, file))) + dbname := strings.TrimSuffix(arch.Join(owner, distro, architecture, file), ".tar.gz") + + obj, err := cs.Get(packages.BlobHash256Key(dbname)) if err != nil { return nil, err } @@ -72,7 +82,7 @@ func UpdatePacmanDatabases(ctx *context.Context, md *arch.Metadata, distro, owne defer os.RemoveAll(tmpdir) // If architecure is not specified or any, package will be automatically - // saved to databases with most popular architectures. + // saved to pacman databases with most popular architectures. var architectures = md.Arch if len(md.Arch) == 0 || md.Arch[0] == "any" { architectures = []string{ @@ -83,75 +93,31 @@ func UpdatePacmanDatabases(ctx *context.Context, md *arch.Metadata, distro, owne cs := packages.NewContentStore() + // Update pacman database files for each architecture. for _, architecture := range architectures { - var ( - db = arch.Join(owner, distro, architecture, setting.Domain, "db.tar.gz") - dbpth = path.Join(tmpdir, db) - dbf = path.Join(tmpdir, db) + ".folder" - sbsl = strings.TrimSuffix(db, ".tar.gz") - slpth = path.Join(tmpdir, sbsl) - ) - - // Get existing pacman database, or create empty folder for it. - dbdata, err := cs.GetStrBytes(db) - if err == nil { - err = os.WriteFile(dbpth, dbdata, os.ModePerm) - if err != nil { - return err - } - err = arch.UnpackDb(dbpth, dbf) - if err != nil { - return err - } - } - if err != nil { - err = os.MkdirAll(dbf, os.ModePerm) - if err != nil { - return err - } - } + db := arch.Join(owner, distro, architecture, setting.Domain, "db") + dbkey := packages.BlobHash256Key(db) - // Update database folder with metadata for new package. - err = md.PutToDb(dbf, os.ModePerm) + o, err := cs.Get(dbkey) if err != nil { return err } - // Create database archive and related symlink. - err = arch.PackDb(dbf, dbpth) + data, err := io.ReadAll(o) if err != nil { return err } - // Save database file. - f, err := os.Open(dbpth) - if err != nil { - return err - } - defer f.Close() - dbfi, err := f.Stat() - if err != nil { - return err - } - err = cs.Save(packages.BlobHash256Key(db), f, dbfi.Size()) + udata, err := arch.UpdatePacmanDbEntry(data, md) if err != nil { return err } - // Save database symlink file. - f, err = os.Open(slpth) - if err != nil { - return err - } - defer f.Close() - dbarchivefi, err := f.Stat() - if err != nil { - return err - } - err = cs.Save(packages.BlobHash256Key(sbsl), f, dbarchivefi.Size()) + err = cs.Save(dbkey, bytes.NewReader(udata), int64(len(udata))) if err != nil { return err } } + return nil } From e2b12f2db705a8d5e28f302639d07e91c52b42af Mon Sep 17 00:00:00 2001 From: dancheg97 Date: Sat, 24 Jun 2023 17:27:39 +0300 Subject: [PATCH 014/124] fixed error when attempting to read empty database --- modules/packages/arch/metadata.go | 2 +- services/packages/arch/file_manager.go | 20 ++++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/modules/packages/arch/metadata.go b/modules/packages/arch/metadata.go index d42ef1e2ffe7..a76f57c8f7d8 100644 --- a/modules/packages/arch/metadata.go +++ b/modules/packages/arch/metadata.go @@ -256,7 +256,7 @@ func readEntries(dbarchive []byte) (map[string][]byte, error) { gzf, err := gzip.NewReader(bytes.NewReader(dbarchive)) if err != nil { fmt.Println(err) - os.Exit(1) + return map[string][]byte{}, nil } var entries = map[string][]byte{} diff --git a/services/packages/arch/file_manager.go b/services/packages/arch/file_manager.go index c39e11dcd3ce..aed7beca44c8 100644 --- a/services/packages/arch/file_manager.go +++ b/services/packages/arch/file_manager.go @@ -98,22 +98,22 @@ func UpdatePacmanDatabases(ctx *context.Context, md *arch.Metadata, distro, owne db := arch.Join(owner, distro, architecture, setting.Domain, "db") dbkey := packages.BlobHash256Key(db) - o, err := cs.Get(dbkey) - if err != nil { - return err - } - - data, err := io.ReadAll(o) - if err != nil { - return err + var dbdata []byte + + dbobj, err := cs.Get(dbkey) + if err == nil { + dbdata, err = io.ReadAll(dbobj) + if err != nil { + return err + } } - udata, err := arch.UpdatePacmanDbEntry(data, md) + newdata, err := arch.UpdatePacmanDbEntry(dbdata, md) if err != nil { return err } - err = cs.Save(dbkey, bytes.NewReader(udata), int64(len(udata))) + err = cs.Save(dbkey, bytes.NewReader(newdata), int64(len(newdata))) if err != nil { return err } From c37231189b134528b1b5ac0471408cf055bf3ca8 Mon Sep 17 00:00:00 2001 From: dancheg97 Date: Sat, 24 Jun 2023 18:08:18 +0300 Subject: [PATCH 015/124] corrected package database connection description and documentation reference in UI --- modules/packages/arch/metadata.go | 3 --- options/locale/locale_en-US.ini | 6 +++--- routers/api/packages/arch/arch.go | 2 +- templates/package/content/arch.tmpl | 6 +++--- 4 files changed, 7 insertions(+), 10 deletions(-) diff --git a/modules/packages/arch/metadata.go b/modules/packages/arch/metadata.go index a76f57c8f7d8..f0d942f4c024 100644 --- a/modules/packages/arch/metadata.go +++ b/modules/packages/arch/metadata.go @@ -238,8 +238,6 @@ func UpdatePacmanDbEntry(db []byte, md *Metadata) ([]byte, error) { // Add new package entry to list. entries[md.Name+"-"+md.Version+"/desc"] = []byte(md.GetDbDesc()) - fmt.Println(entries) - var out bytes.Buffer // Write entries to new buffer and return it. @@ -255,7 +253,6 @@ func UpdatePacmanDbEntry(db []byte, md *Metadata) ([]byte, error) { func readEntries(dbarchive []byte) (map[string][]byte, error) { gzf, err := gzip.NewReader(bytes.NewReader(dbarchive)) if err != nil { - fmt.Println(err) return map[string][]byte{}, nil } diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 8d9b17c1e969..0b919ed8a5c6 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -3254,9 +3254,9 @@ alpine.repository = Repository Info alpine.repository.branches = Branches alpine.repository.repositories = Repositories alpine.repository.architectures = Architectures -arch.pacmanconf = Add server to pacman.conf: -arch.pacmansync = And sync package with pacman: -arch.documentation = For more information on the Arch registry, see the documentation. +arch.pacmanconf = Add server to pacman.conf with related distribution and architecture: +arch.pacmansync = Sync package with pacman: +arch.documentation = For more information on the arch mirrors, see the documentation. arch.properties = Package properties cargo.registry = Setup this registry in the Cargo configuration file (for example ~/.cargo/config.toml): cargo.install = To install the package using Cargo, run the following command: diff --git a/routers/api/packages/arch/arch.go b/routers/api/packages/arch/arch.go index 18cd9ef5b3b7..db34529da831 100644 --- a/routers/api/packages/arch/arch.go +++ b/routers/api/packages/arch/arch.go @@ -111,7 +111,7 @@ func Push(ctx *context.Context) { func Get(ctx *context.Context) { var ( file = ctx.Params("file") - owner = ctx.Params("owner") + owner = ctx.Params("username") distro = ctx.Params("distro") arch = ctx.Params("arch") ) diff --git a/templates/package/content/arch.tmpl b/templates/package/content/arch.tmpl index f0f967778ee2..8f5e4337a4b9 100644 --- a/templates/package/content/arch.tmpl +++ b/templates/package/content/arch.tmpl @@ -6,16 +6,16 @@ 
[{{.PackageDescriptor.Owner.LowerName}}.{{.PackageDescriptor.Metadata.BaseDomain}}]
-Server = https://{{.PackageDescriptor.Metadata.BaseDomain}}/api/packages/arch/(distribution)/(architecture)/{{.PackageDescriptor.Owner.LowerName}}
+Server = https://{{.PackageDescriptor.Metadata.BaseDomain}}/api/packages/{{.PackageDescriptor.Owner.LowerName}}/arch/archlinux/x86_64
- 
pacman -S {{.PackageDescriptor.Package.LowerName}}
+ 
pacman -Sy {{.PackageDescriptor.Package.LowerName}}
- +
From 5beeb2468418911ae3f4675a0efc1df20fa86e90 Mon Sep 17 00:00:00 2001 From: dancheg97 Date: Sun, 25 Jun 2023 13:15:29 +0300 Subject: [PATCH 016/124] added method to remove package verified with user's GPG key, removed unnecessary directory creation --- modules/packages/arch/metadata.go | 22 ++++++++ routers/api/packages/api.go | 1 + routers/api/packages/arch/arch.go | 73 +++++++++++++++++++++++++- services/packages/arch/db_manager.go | 22 ++++++++ services/packages/arch/file_manager.go | 64 ++++++++++++++++------ services/packages/arch/verificator.go | 2 +- 6 files changed, 165 insertions(+), 19 deletions(-) diff --git a/modules/packages/arch/metadata.go b/modules/packages/arch/metadata.go index f0d942f4c024..bf46ec95eb18 100644 --- a/modules/packages/arch/metadata.go +++ b/modules/packages/arch/metadata.go @@ -249,6 +249,28 @@ func UpdatePacmanDbEntry(db []byte, md *Metadata) ([]byte, error) { return out.Bytes(), nil } +// Add or update existing package entry in database archived data. +func RemoveDbEntry(db []byte, pkg, ver string) ([]byte, error) { + // Read existing entries in archive. + entries, err := readEntries(db) + if err != nil { + return nil, err + } + + // Add new package entry to list. + delete(entries, pkg+"-"+ver+"/desc") + + var out bytes.Buffer + + // Write entries to new buffer and return it. + err = writeToArchive(entries, &out) + if err != nil { + return nil, err + } + + return out.Bytes(), nil +} + // Read database entries containing in pacman archive. func readEntries(dbarchive []byte) (map[string][]byte, error) { gzf, err := gzip.NewReader(bytes.NewReader(dbarchive)) diff --git a/routers/api/packages/api.go b/routers/api/packages/api.go index 7b2fb7b125fe..bf8cc7570bca 100644 --- a/routers/api/packages/api.go +++ b/routers/api/packages/api.go @@ -125,6 +125,7 @@ func CommonRoutes() *web.Route { }, reqPackageAccess(perm.AccessModeRead)) r.Group("/arch", func() { r.Put("/push", arch.Push) + r.Delete("/remove", arch.Remove) r.Get("/{distro}/{arch}/{file}", arch.Get) }) r.Group("/cargo", func() { diff --git a/routers/api/packages/arch/arch.go b/routers/api/packages/arch/arch.go index db34529da831..f859a58731f2 100644 --- a/routers/api/packages/arch/arch.go +++ b/routers/api/packages/arch/arch.go @@ -8,6 +8,7 @@ import ( "io" "net/http" "strings" + "time" "code.gitea.io/gitea/modules/context" arch_module "code.gitea.io/gitea/modules/packages/arch" @@ -49,7 +50,7 @@ func Push(ctx *context.Context) { } // Validate package signature with any of user's GnuPG keys. - err = arch_service.ValidatePackageSignature(ctx, pkgdata, sigdata, user) + err = arch_service.ValidateSignature(ctx, pkgdata, sigdata, user) if err != nil { apiError(ctx, http.StatusUnauthorized, err) return @@ -157,6 +158,76 @@ func Get(ctx *context.Context) { ctx.Resp.WriteHeader(http.StatusNotFound) } +// Remove package and all it's versions from gitea. +func Remove(ctx *context.Context) { + var ( + owner = ctx.Params("username") + email = ctx.Req.Header.Get("email") + distro = ctx.Req.Header.Get("distro") + target = ctx.Req.Header.Get("target") + stime = ctx.Req.Header.Get("time") + version = ctx.Req.Header.Get("version") + arch = strings.Split(ctx.Req.Header.Get("arch"), " ") + ) + + // Parse sent time and check if it is within last minute. + t, err := time.Parse(time.RFC3339, stime) + if err != nil { + apiError(ctx, http.StatusBadRequest, err) + return + } + + if time.Since(t) > time.Minute { + apiError(ctx, http.StatusUnauthorized, "outdated message") + return + } + + // Get user owning the package. + user, org, err := arch_service.IdentifyOwner(ctx, owner, email) + if err != nil { + apiError(ctx, http.StatusUnauthorized, err) + return + } + + // Read signature data from request body. + sigdata, err := io.ReadAll(ctx.Req.Body) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + defer ctx.Req.Body.Close() + + // Validate package signature with any of user's GnuPG keys. + mesdata := []byte(stime + owner + target) + err = arch_service.ValidateSignature(ctx, mesdata, sigdata, user) + if err != nil { + apiError(ctx, http.StatusUnauthorized, err) + return + } + + // Remove package files and pacman database entry. + err = arch_service.RemovePackage(ctx, &arch_service.RemoveParameters{ + User: user, + Organization: org, + Owner: owner, + Name: target, + Version: version, + }) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + // Remove pacman database entries related to package. + err = arch_service.RemoveDbEntry(ctx, arch, owner, distro, target, version) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + ctx.Resp.WriteHeader(http.StatusOK) +} + func apiError(ctx *context.Context, status int, obj interface{}) { helper.LogAndProcessError(ctx, status, obj, func(message string) { ctx.PlainText(status, message) diff --git a/services/packages/arch/db_manager.go b/services/packages/arch/db_manager.go index 99d1c051326f..f826e0424e0e 100644 --- a/services/packages/arch/db_manager.go +++ b/services/packages/arch/db_manager.go @@ -5,6 +5,7 @@ package arch import ( "bytes" + "strings" org "code.gitea.io/gitea/models/organization" pkg "code.gitea.io/gitea/models/packages" @@ -75,3 +76,24 @@ func RepositoryAutoconnect(ctx *context.Context, owner, repository string, pkgid } return nil } + +type RemoveParameters struct { + *user.User + *org.Organization + Owner string + Name string + Version string +} + +// Remove package and it's blobs from gitea. +func RemovePackage(ctx *context.Context, p *RemoveParameters) error { + tpkg, err := pkg.GetPackageByName(ctx, p.Organization.ID, pkg.TypeArch, p.Name) + if err != nil { + return err + } + return svc.RemovePackageVersion(p.User, &pkg.PackageVersion{ + PackageID: tpkg.ID, + Version: p.Version, + LowerVersion: strings.ToLower(p.Version), + }) +} diff --git a/services/packages/arch/file_manager.go b/services/packages/arch/file_manager.go index aed7beca44c8..4f146baa4da8 100644 --- a/services/packages/arch/file_manager.go +++ b/services/packages/arch/file_manager.go @@ -7,8 +7,6 @@ import ( "bytes" "fmt" "io" - "os" - "path" "strings" "code.gitea.io/gitea/models/db" @@ -17,7 +15,6 @@ import ( "code.gitea.io/gitea/modules/packages" "code.gitea.io/gitea/modules/packages/arch" "code.gitea.io/gitea/modules/setting" - "github.com/google/uuid" ) // Get data related to provided file name and distribution, and update download @@ -73,28 +70,16 @@ func LoadPacmanDatabase(ctx *context.Context, owner, distro, architecture, file // This function will update information about package in related pacman databases // or create them if they do not exist. func UpdatePacmanDatabases(ctx *context.Context, md *arch.Metadata, distro, owner string) error { - // Create temporary directory for arch database operations. - tmpdir := path.Join(setting.Repository.Upload.TempPath, uuid.New().String()) - err := os.MkdirAll(tmpdir, os.ModePerm) - if err != nil { - return err - } - defer os.RemoveAll(tmpdir) - // If architecure is not specified or any, package will be automatically // saved to pacman databases with most popular architectures. - var architectures = md.Arch if len(md.Arch) == 0 || md.Arch[0] == "any" { - architectures = []string{ - "x86_64", "arm", "i686", "pentium4", - "armv7h", "armv6h", "aarch64", "riscv64", - } + md.Arch = popularArchitectures() } cs := packages.NewContentStore() // Update pacman database files for each architecture. - for _, architecture := range architectures { + for _, architecture := range md.Arch { db := arch.Join(owner, distro, architecture, setting.Domain, "db") dbkey := packages.BlobHash256Key(db) @@ -121,3 +106,48 @@ func UpdatePacmanDatabases(ctx *context.Context, md *arch.Metadata, distro, owne return nil } + +func RemoveDbEntry(ctx *context.Context, architectures []string, owner, distro, pkg, ver string) error { + cs := packages.NewContentStore() + + // If architecures are not specified or any, package will be automatically + // removed from pacman databases with most popular architectures. + if len(architectures) == 0 || architectures[0] == "any" { + architectures = popularArchitectures() + } + + for _, architecture := range architectures { + db := arch.Join(owner, distro, architecture, setting.Domain, "db") + dbkey := packages.BlobHash256Key(db) + + var dbdata []byte + + dbobj, err := cs.Get(dbkey) + if err != nil { + return err + } + + dbdata, err = io.ReadAll(dbobj) + if err != nil { + return err + } + + newdata, err := arch.RemoveDbEntry(dbdata, pkg, ver) + if err != nil { + return err + } + + err = cs.Save(dbkey, bytes.NewReader(newdata), int64(len(newdata))) + if err != nil { + return err + } + } + return nil +} + +func popularArchitectures() []string { + return []string{ + "x86_64", "arm", "i686", "pentium4", + "armv7h", "armv6h", "aarch64", "riscv64", + } +} diff --git a/services/packages/arch/verificator.go b/services/packages/arch/verificator.go index 35694aba3969..31e072a644cd 100644 --- a/services/packages/arch/verificator.go +++ b/services/packages/arch/verificator.go @@ -59,7 +59,7 @@ func IdentifyOwner(ctx *context.Context, owner, email string) (*user.User, *org. } // Validate package signature with owner's GnuPG keys stored in gitea's database. -func ValidatePackageSignature(ctx *context.Context, pkg, sign []byte, u *user.User) error { +func ValidateSignature(ctx *context.Context, pkg, sign []byte, u *user.User) error { keys, err := asymkey.ListGPGKeys(ctx, u.ID, db.ListOptions{ ListAll: true, }) From 04a3ddd11a3910e78d9df43d475465318197419f Mon Sep 17 00:00:00 2001 From: dancheg97 Date: Sun, 25 Jun 2023 17:41:00 +0300 Subject: [PATCH 017/124] changed function retrieving package version when deleting package version and related files --- services/packages/arch/db_manager.go | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/services/packages/arch/db_manager.go b/services/packages/arch/db_manager.go index f826e0424e0e..b893d70c9c49 100644 --- a/services/packages/arch/db_manager.go +++ b/services/packages/arch/db_manager.go @@ -5,7 +5,6 @@ package arch import ( "bytes" - "strings" org "code.gitea.io/gitea/models/organization" pkg "code.gitea.io/gitea/models/packages" @@ -87,13 +86,9 @@ type RemoveParameters struct { // Remove package and it's blobs from gitea. func RemovePackage(ctx *context.Context, p *RemoveParameters) error { - tpkg, err := pkg.GetPackageByName(ctx, p.Organization.ID, pkg.TypeArch, p.Name) + ver, err := pkg.GetVersionByNameAndVersion(ctx, p.Organization.ID, pkg.TypeArch, p.Name, p.Version) if err != nil { return err } - return svc.RemovePackageVersion(p.User, &pkg.PackageVersion{ - PackageID: tpkg.ID, - Version: p.Version, - LowerVersion: strings.ToLower(p.Version), - }) + return svc.RemovePackageVersion(p.User, ver) } From 749a106d06be458b278f8d13465f02d45d781a7b Mon Sep 17 00:00:00 2001 From: dancheg97 Date: Sun, 25 Jun 2023 17:45:16 +0300 Subject: [PATCH 018/124] updated remove function description --- routers/api/packages/arch/arch.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routers/api/packages/arch/arch.go b/routers/api/packages/arch/arch.go index f859a58731f2..6a4f243943b2 100644 --- a/routers/api/packages/arch/arch.go +++ b/routers/api/packages/arch/arch.go @@ -158,7 +158,7 @@ func Get(ctx *context.Context) { ctx.Resp.WriteHeader(http.StatusNotFound) } -// Remove package and all it's versions from gitea. +// Remove specific package version, related files and pacman database entry. func Remove(ctx *context.Context) { var ( owner = ctx.Params("username") From 1d1a5a312f94c885ec0453dbe942b037e70e88a3 Mon Sep 17 00:00:00 2001 From: dancheg97 Date: Sun, 25 Jun 2023 23:05:02 +0300 Subject: [PATCH 019/124] added function removing old package entries from pacman database to avoid duplication error --- modules/packages/arch/metadata.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/modules/packages/arch/metadata.go b/modules/packages/arch/metadata.go index bf46ec95eb18..6b545ca95e91 100644 --- a/modules/packages/arch/metadata.go +++ b/modules/packages/arch/metadata.go @@ -235,6 +235,9 @@ func UpdatePacmanDbEntry(db []byte, md *Metadata) ([]byte, error) { return nil, err } + // Remove entries related old package versions. + entries = CleanOldEntries(entries, md.Name) + // Add new package entry to list. entries[md.Name+"-"+md.Version+"/desc"] = []byte(md.GetDbDesc()) @@ -249,6 +252,19 @@ func UpdatePacmanDbEntry(db []byte, md *Metadata) ([]byte, error) { return out.Bytes(), nil } +// Clean entries for old package versions from pacman database. +func CleanOldEntries(entries map[string][]byte, pkg string) map[string][]byte { + out := map[string][]byte{} + for entry, value := range entries { + splt := strings.Split(entry, "-") + basename := strings.Join(splt[0:len(splt)-2], "-") + if pkg != basename { + out[entry] = value + } + } + return out +} + // Add or update existing package entry in database archived data. func RemoveDbEntry(db []byte, pkg, ver string) ([]byte, error) { // Read existing entries in archive. From f1fae9ada60eb35f84c9d7208507e65fd83b8271 Mon Sep 17 00:00:00 2001 From: dancheg97 Date: Mon, 26 Jun 2023 12:28:14 +0300 Subject: [PATCH 020/124] changed procedure of package metadata ejection, removed unnecessary fields and unnecessary string conversions --- modules/packages/arch/metadata.go | 202 ++++++++++++++---------------- 1 file changed, 93 insertions(+), 109 deletions(-) diff --git a/modules/packages/arch/metadata.go b/modules/packages/arch/metadata.go index 6b545ca95e91..70a77e8b3131 100644 --- a/modules/packages/arch/metadata.go +++ b/modules/packages/arch/metadata.go @@ -10,141 +10,125 @@ import ( "crypto/md5" "crypto/sha256" "encoding/hex" - "errors" "fmt" "io" "os" "strconv" "strings" - "time" "github.com/mholt/archiver/v3" ) // Metadata for arch package. type Metadata struct { - Filename string - Name string - Base string - Version string - Description string - CompressedSize int64 - CompressedSizeMib string - InstalledSize int64 - InstalledSizeMib string - MD5 string - SHA256 string - URL string - BuildDate int64 - BuildDateStr string - BaseDomain string - Packager string - Provides []string - License []string - Arch []string - Depends []string - OptDepends []string - MakeDepends []string - CheckDepends []string - Backup []string + Filename string + Name string + Base string + Version string + Description string + CompressedSize int64 + InstalledSize int64 + MD5 string + SHA256 string + URL string + BuildDate int64 + BaseDomain string + Packager string + Provides []string + License []string + Arch []string + Depends []string + OptDepends []string + MakeDepends []string + CheckDepends []string + Backup []string } // Function that recieves arch package archive data and returns it's metadata. func EjectMetadata(filename, domain string, pkg []byte) (*Metadata, error) { - pkgreader := io.LimitReader(bytes.NewReader(pkg), 250000) - var buf bytes.Buffer - err := archiver.DefaultZstd.Decompress(pkgreader, &buf) + pkginfo, err := getPkginfo(pkg) if err != nil { - if !errors.Is(err, io.ErrUnexpectedEOF) { - return nil, err - } - } - splt := strings.Split(buf.String(), "PKGINFO") - if len(splt) < 2 { - return nil, errors.New("unable to eject .PKGINFO from archive") + return nil, err } - raw := splt[1][0:10000] - inssize := int64(len(pkg)) - compsize := ejectInt64(raw, "size") - unixbuilddate := ejectInt64(raw, "builddate") - return &Metadata{ - Filename: filename, - Name: ejectString(raw, "pkgname"), - Base: ejectString(raw, "pkgbase"), - Version: ejectString(raw, "pkgver"), - Description: ejectString(raw, "pkgdesc"), - CompressedSize: inssize, - CompressedSizeMib: ByteCountSI(inssize), - InstalledSize: compsize, - InstalledSizeMib: ByteCountSI(compsize), - MD5: md5sum(pkg), - SHA256: sha256sum(pkg), - URL: ejectString(raw, "url"), - BuildDate: unixbuilddate, - BuildDateStr: ReadableTime(unixbuilddate), - BaseDomain: domain, - Packager: ejectString(raw, "packager"), - Provides: ejectStrings(raw, "provides"), - License: ejectStrings(raw, "license"), - Arch: ejectStrings(raw, "arch"), - Depends: ejectStrings(raw, "depend"), - OptDepends: ejectStrings(raw, "optdepend"), - MakeDepends: ejectStrings(raw, "makedepend"), - CheckDepends: ejectStrings(raw, "checkdepend"), - Backup: ejectStrings(raw, "backup"), - }, nil -} - -func ejectString(raw, field string) string { - splitted := strings.Split(raw, "\n"+field+" = ") - if len(splitted) < 2 { - return `` + var md = Metadata{ + Filename: filename, + BaseDomain: domain, + CompressedSize: int64(len(pkg)), + MD5: md5sum(pkg), + SHA256: sha256sum(pkg), } - return strings.Split(splitted[1], "\n")[0] -} - -func ejectStrings(raw, field string) []string { - splitted := strings.Split(raw, "\n"+field+" = ") - if len(splitted) < 2 { - return nil - } - var rez []string - for i, v := range splitted { - if i == 0 { + for _, line := range strings.Split(pkginfo, "\n") { + splt := strings.Split(line, " = ") + if len(splt) != 2 { continue } - rez = append(rez, strings.Split(v, "\n")[0]) + switch splt[0] { + case "pkgname": + md.Name = splt[1] + case "pkgbase": + md.Base = splt[1] + case "pkgver": + md.Version = splt[1] + case "pkgdesc": + md.Description = splt[1] + case "url": + md.URL = splt[1] + case "packager": + md.Packager = splt[1] + case "builddate": + num, err := strconv.ParseInt(splt[1], 10, 64) + if err != nil { + return nil, err + } + md.BuildDate = num + case "size": + num, err := strconv.ParseInt(splt[1], 10, 64) + if err != nil { + return nil, err + } + md.InstalledSize = num + case "provides": + md.Provides = append(md.Provides, splt[1]) + case "license": + md.License = append(md.License, splt[1]) + case "arch": + md.Arch = append(md.Arch, splt[1]) + case "depend": + md.Depends = append(md.Depends, splt[1]) + case "optdepend": + md.Depends = append(md.OptDepends, splt[1]) + case "makedepend": + md.Depends = append(md.MakeDepends, splt[1]) + case "checkdepend": + md.Depends = append(md.CheckDepends, splt[1]) + case "backup": + md.Depends = append(md.Backup, splt[1]) + } } - return rez + return &md, nil } -func ejectInt64(raw, field string) int64 { - splitted := strings.Split(raw, "\n"+field+" = ") - if len(splitted) < 2 { - return 0 - } - i, err := strconv.ParseInt(strings.Split(splitted[1], "\n")[0], 10, 64) +func getPkginfo(data []byte) (string, error) { + pkgreader := io.LimitReader(bytes.NewReader(data), 250000) + zstd := archiver.NewTarZstd() + err := zstd.Open(pkgreader, int64(250000)) if err != nil { - return 0 - } - return i -} - -func ByteCountSI(b int64) string { - const unit = 1000 - if b < unit { - return fmt.Sprintf("%d B", b) + return ``, err } - div, exp := int64(unit), 0 - for n := b / unit; n >= unit; n /= unit { - div *= unit - exp++ + for { + f, err := zstd.Read() + if err != nil { + return ``, err + } + if f.Name() != ".PKGINFO" { + continue + } + b, err := io.ReadAll(f) + if err != nil { + return ``, err + } + return string(b), nil } - return fmt.Sprintf("%.1f %cB", float64(b)/float64(div), "kMGTPE"[exp]) -} - -func ReadableTime(unix int64) string { - return time.Unix(unix, 0).Format(time.DateTime) } func md5sum(data []byte) string { From 39e42f7c2bfffdc9aab598995d58f4945294e924 Mon Sep 17 00:00:00 2001 From: dancheg97 Date: Mon, 26 Jun 2023 17:10:04 +0300 Subject: [PATCH 021/124] fixed dependencies appended to wrong field in metadata --- modules/packages/arch/metadata.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/packages/arch/metadata.go b/modules/packages/arch/metadata.go index 70a77e8b3131..a55df76f25d1 100644 --- a/modules/packages/arch/metadata.go +++ b/modules/packages/arch/metadata.go @@ -96,11 +96,11 @@ func EjectMetadata(filename, domain string, pkg []byte) (*Metadata, error) { case "depend": md.Depends = append(md.Depends, splt[1]) case "optdepend": - md.Depends = append(md.OptDepends, splt[1]) + md.OptDepends = append(md.OptDepends, splt[1]) case "makedepend": - md.Depends = append(md.MakeDepends, splt[1]) + md.MakeDepends = append(md.MakeDepends, splt[1]) case "checkdepend": - md.Depends = append(md.CheckDepends, splt[1]) + md.CheckDepends = append(md.CheckDepends, splt[1]) case "backup": md.Depends = append(md.Backup, splt[1]) } From 2873debef0ecff86914a9d397c214a2c91defeac Mon Sep 17 00:00:00 2001 From: dancheg97 Date: Wed, 28 Jun 2023 22:12:01 +0300 Subject: [PATCH 022/124] documentation for arch packages, additional signature in arch package upload for better security --- docs/content/doc/usage/packages/arch.en-us.md | 184 ++++++++++++++++++ routers/api/packages/arch/arch.go | 50 ++++- 2 files changed, 224 insertions(+), 10 deletions(-) create mode 100644 docs/content/doc/usage/packages/arch.en-us.md diff --git a/docs/content/doc/usage/packages/arch.en-us.md b/docs/content/doc/usage/packages/arch.en-us.md new file mode 100644 index 000000000000..36366ea006f9 --- /dev/null +++ b/docs/content/doc/usage/packages/arch.en-us.md @@ -0,0 +1,184 @@ +--- +date: "2016-11-08T16:00:00+02:00" +title: "Title" +weight: 10 +toc: true +draft: false +menu: + sidebar: + parent: "packages" + name: "Arch" + weight: 10 + identifier: "arch" +--- + +# Arch package registry + +Gitea has arch package registry, which can act as a fully working [arch linux mirror](https://wiki.archlinux.org/title/mirrors) and connected directly in `/etc/pacman.conf`. Gitea automatically creates pacman database for packages in user space when new arch package is uploaded. + +**Table of Contents** + +{{< toc >}} + +## Requirements + +You can install packages in any environment with [pacman](https://wiki.archlinux.org/title/Pacman). Alternatively you can use [pack](https://fmnx.su/core/pack) which connects specified registries automatically and provides simple interface for package uploads and deletions. + +## Install packages + +First, you need to update your pacman configuration, adding following lines: + +```conf +[{owner}.{domain}] +Server = https://{domain}/api/packages/{owner}/arch/{distribution}/{architecture} +``` + +Then, you can run pacman sync command (with -y flag to load connected database file), to install your package. + +```sh +pacman -Sy package +``` + +## GPG Verification + +Upload and remove operation are validated with [GnuPG](https://gnupg.org/). First, you need to export and upload your public gpg key to `SSH/GPG Keys` in account settings. This works similarly with SSH key. You can export gpg key with command: + +```sh +gpg --armor --export +``` + +``` +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQENBGSYoJUBCADSJ6v8Egst/gNJVC2206o8JqTzRBxTULKm/DH5J7AzrhJBxC2/ +... + +-----END PGP PUBLIC KEY BLOCK----- +``` + +## Upload packages + +1. Ensure, that your package have been signed with your gpg key (more about arch package signing)[https://wiki.archlinux.org/title/DeveloperWiki:Package_signing]. You can do that by running following command: + +```sh +gpg --verify package-ver-1-x86_64.pkg.tar.zst.sig +``` + +2. Sign message metadata, which consists of package owner (namespace in gitea), package file name and send time. You can do that by running following command: + +```sh +echo -n {owner}{package}-1-1-x86_64.pkg.tar.zst$(date --rfc-3339=seconds | tr " " T) >> md +gpg --detach-sign md +``` + +3. Decode message and metadata signatures to hex, by running following commands, save output somewhere. + +```sh +xxd -p md.sig >> md.sig.hex +xxd -p package-1-1-x86_64.pkg.tar.zst.sig >> pkg.sig.hex +``` + +4. Paste your parameters and push package with [curl](https://curl.se/). Important, that time should be the same with metadata (signed md file), since this value is verified with GnuPG. + +```sh +curl -X PUT \ + 'https://{domain}/api/packages/{owner}/arch/push' \ + --header 'filename: {package}-1-1-x86_64.pkg.tar.zst' \ + --header 'email: dancheg97@fmnx.su' \ + --header 'distro: archlinux' \ + --header 'time: {metadata-time}' \ + --header 'pkgsign: {package-signature-hex}' \ + --header 'metasign: {metadata-signature-hex}' \ + --header 'Content-Type: application/octet-stream' \ + --data-binary '@/path/to/package/file/{package}-1-1-x86_64.pkg.tar.zst' +``` + +Full script for package upload: + +```sh +owner=user +package=package-0.1.0-1-x86_64.pkg.tar.zst +email=user@example.com + +time=`date --rfc-3339=seconds | tr " " T` +pkgsignhex=`xxd -p $package.sig | tr -d "\n"` + +echo -n $owner$package$time >> mddata +gpg --detach-sign mddata +mdsignhex=`xxd -p mddata.sig | tr -d "\n"` + +curl -X PUT \ + http://{domain}/api/packages/$owner/arch/push \ + --header "filename: $package" \ + --header "email: $email" \ + --header "time: $time" \ + --header "distro: archlinux" \ + --header "metasign: $mdsignhex" \ + --header "pkgsign: $pkgsignhex" \ + --header 'Content-Type: application/octet-stream' \ + --data-binary @$package +``` + +Alternatively, you can install [pack](https://fmnx.su/core/pack) and execute push command. Pack is automatically handling all gpg/http related operations: + +```sh +pack -P {domain}/{owner}/{package} +``` + +## Delete packages + +1. Prepare signature for delete message. + +```sh +echo -n {owner}{package}$(date --rfc-3339=seconds | tr " " T) >> md +gpg --detach-sign md +``` + +2. Send delete message with [curl](https://curl.se/). Time should be the same with saved in `md` file. + +```sh +curl -X DELETE \ + http://localhost:3000/api/packages/{user}/arch/remove \ + --header "username: {user}" \ + --header "email: user@email.com" \ + --header "distro: archlinux" \ + --header "target: package" \ + --header "time: {rmtime}" \ + --header "version: {version-release}" \ + --header "arch: x86_64" \ + --header 'Content-Type: application/octet-stream' \ + --data-binary @md.sig +``` + +Full script for package deletion: + +```sh +owner=user +package=package +version=0.1.0-1 +email=user@example.com +arch=x86_64 +time=`date --rfc-3339=seconds | tr " " T` + +sudo rm -rf md md.sig +echo -n $owner$package$time >> md +gpg --detach-sign md + +curl -X DELETE \ + http://{domain}/api/packages/$owner/arch/remove \ + --header "username: $owner" \ + --header "email: $email" \ + --header "distro: archlinux" \ + --header "target: $package" \ + --header "time: $time" \ + --header "version: $version" \ + --header "arch: $arch" \ + --header 'Content-Type: application/octet-stream' \ + --data-binary @md.sig +``` + +Alternatively, you can use [pack](https://fmnx.su/core/pack) to execute remote delete operations: + +```sh +pack -R {domain}/{owner}/{package}@{version-release} +``` diff --git a/routers/api/packages/arch/arch.go b/routers/api/packages/arch/arch.go index 6a4f243943b2..7e40b79b3993 100644 --- a/routers/api/packages/arch/arch.go +++ b/routers/api/packages/arch/arch.go @@ -23,12 +23,14 @@ func Push(ctx *context.Context) { owner = ctx.Params("username") filename = ctx.Req.Header.Get("filename") email = ctx.Req.Header.Get("email") - sign = ctx.Req.Header.Get("sign") distro = ctx.Req.Header.Get("distro") + sendtime = ctx.Req.Header.Get("time") + pkgsign = ctx.Req.Header.Get("pkgsign") + metasign = ctx.Req.Header.Get("metasign") ) // Decoding package signature. - sigdata, err := hex.DecodeString(sign) + sigdata, err := hex.DecodeString(pkgsign) if err != nil { apiError(ctx, http.StatusBadRequest, err) return @@ -49,6 +51,34 @@ func Push(ctx *context.Context) { return } + // Decoding time when message was created. + t, err := time.Parse(time.RFC3339, sendtime) + if err != nil { + apiError(ctx, http.StatusBadRequest, err) + return + } + + if time.Since(t) > time.Hour { + apiError(ctx, http.StatusUnauthorized, "outdated message") + return + } + + // Decoding signature related to metadata. + msigdata, err := hex.DecodeString(metasign) + if err != nil { + apiError(ctx, http.StatusBadRequest, err) + return + } + + // Validating metadata signature, to ensure that operation push operation + // is initiated by original package owner. + sendmetadata := []byte(owner + filename + sendtime) + err = arch_service.ValidateSignature(ctx, sendmetadata, msigdata, user) + if err != nil { + apiError(ctx, http.StatusUnauthorized, err) + return + } + // Validate package signature with any of user's GnuPG keys. err = arch_service.ValidateSignature(ctx, pkgdata, sigdata, user) if err != nil { @@ -198,13 +228,20 @@ func Remove(ctx *context.Context) { defer ctx.Req.Body.Close() // Validate package signature with any of user's GnuPG keys. - mesdata := []byte(stime + owner + target) + mesdata := []byte(owner + target + stime) err = arch_service.ValidateSignature(ctx, mesdata, sigdata, user) if err != nil { apiError(ctx, http.StatusUnauthorized, err) return } + // Remove pacman database entries related to package. + err = arch_service.RemoveDbEntry(ctx, arch, owner, distro, target, version) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + // Remove package files and pacman database entry. err = arch_service.RemovePackage(ctx, &arch_service.RemoveParameters{ User: user, @@ -218,13 +255,6 @@ func Remove(ctx *context.Context) { return } - // Remove pacman database entries related to package. - err = arch_service.RemoveDbEntry(ctx, arch, owner, distro, target, version) - if err != nil { - apiError(ctx, http.StatusInternalServerError, err) - return - } - ctx.Resp.WriteHeader(http.StatusOK) } From 6d1037d506d59b0569e8fed036a6ba0650611eb8 Mon Sep 17 00:00:00 2001 From: dancheg97 Date: Sat, 1 Jul 2023 17:13:21 +0300 Subject: [PATCH 023/124] db desc file for pacman database is saved in the same space with package --- routers/api/packages/arch/arch.go | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/routers/api/packages/arch/arch.go b/routers/api/packages/arch/arch.go index 7e40b79b3993..52dfad0641f3 100644 --- a/routers/api/packages/arch/arch.go +++ b/routers/api/packages/arch/arch.go @@ -103,7 +103,7 @@ func Push(ctx *context.Context) { Distro: distro, }) if err != nil { - apiError(ctx, http.StatusBadRequest, err) + apiError(ctx, http.StatusInternalServerError, err) return } @@ -117,19 +117,26 @@ func Push(ctx *context.Context) { Distro: distro, }) if err != nil { - apiError(ctx, http.StatusBadRequest, err) + apiError(ctx, http.StatusInternalServerError, err) return } - // Automatically connect repository for provided package if name matched. - err = arch_service.RepositoryAutoconnect(ctx, owner, md.Name, pkgid) + // Save file related to arch package description. + _, err = arch_service.SaveFile(ctx, &arch_service.SaveFileParams{ + Organization: org, + User: user, + Metadata: md, + Data: []byte(md.GetDbDesc()), + Filename: filename + ".desc", + Distro: distro, + }) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return } - // Update pacman databases with new package. - err = arch_service.UpdatePacmanDatabases(ctx, md, distro, owner) + // Automatically connect repository for provided package if name matched. + err = arch_service.RepositoryAutoconnect(ctx, owner, md.Name, pkgid) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return From 56772de7306bc97ad694f9a178e77b20d11f65c4 Mon Sep 17 00:00:00 2001 From: dancheg97 Date: Sun, 2 Jul 2023 23:03:34 +0300 Subject: [PATCH 024/124] corrected package blob structure, pacman database is created on get request automatically from package metadata in database --- docs/content/doc/usage/packages/arch.en-us.md | 2 - modules/packages/arch/metadata.go | 132 ++++----------- routers/api/packages/arch/arch.go | 64 ++------ services/packages/arch/db_manager.go | 125 +++++++++++--- services/packages/arch/file_manager.go | 153 ------------------ 5 files changed, 150 insertions(+), 326 deletions(-) delete mode 100644 services/packages/arch/file_manager.go diff --git a/docs/content/doc/usage/packages/arch.en-us.md b/docs/content/doc/usage/packages/arch.en-us.md index 36366ea006f9..d2d02ec1ad22 100644 --- a/docs/content/doc/usage/packages/arch.en-us.md +++ b/docs/content/doc/usage/packages/arch.en-us.md @@ -141,11 +141,9 @@ curl -X DELETE \ http://localhost:3000/api/packages/{user}/arch/remove \ --header "username: {user}" \ --header "email: user@email.com" \ - --header "distro: archlinux" \ --header "target: package" \ --header "time: {rmtime}" \ --header "version: {version-release}" \ - --header "arch: x86_64" \ --header 'Content-Type: application/octet-stream' \ --data-binary @md.sig ``` diff --git a/modules/packages/arch/metadata.go b/modules/packages/arch/metadata.go index a55df76f25d1..629d94b22ba1 100644 --- a/modules/packages/arch/metadata.go +++ b/modules/packages/arch/metadata.go @@ -21,31 +21,32 @@ import ( // Metadata for arch package. type Metadata struct { - Filename string - Name string - Base string - Version string - Description string - CompressedSize int64 - InstalledSize int64 - MD5 string - SHA256 string - URL string - BuildDate int64 - BaseDomain string - Packager string - Provides []string - License []string - Arch []string - Depends []string - OptDepends []string - MakeDepends []string - CheckDepends []string - Backup []string + Filename string `json:"filename"` + Name string `json:"name"` + Base string `json:"base"` + Version string `json:"version"` + Description string `json:"description"` + CompressedSize int64 `json:"compressed-size"` + InstalledSize int64 `json:"installed-size"` + MD5 string `json:"md5"` + SHA256 string `json:"sha256"` + URL string `json:"url"` + BuildDate int64 `json:"build-date"` + BaseDomain string `json:"base-domain"` + Packager string `json:"packager"` + Distribution string `json:"distribution"` + Provides []string `json:"provides"` + License []string `json:"license"` + Arch []string `json:"arch"` + Depends []string `json:"depends"` + OptDepends []string `json:"opt-depends"` + MakeDepends []string `json:"make-depends"` + CheckDepends []string `json:"check-depends"` + Backup []string `json:"backup"` } // Function that recieves arch package archive data and returns it's metadata. -func EjectMetadata(filename, domain string, pkg []byte) (*Metadata, error) { +func EjectMetadata(filename, distribution, domain string, pkg []byte) (*Metadata, error) { pkginfo, err := getPkginfo(pkg) if err != nil { return nil, err @@ -56,6 +57,7 @@ func EjectMetadata(filename, domain string, pkg []byte) (*Metadata, error) { CompressedSize: int64(len(pkg)), MD5: md5sum(pkg), SHA256: sha256sum(pkg), + Distribution: distribution, } for _, line := range strings.Split(pkginfo, "\n") { splt := strings.Split(line, " = ") @@ -211,59 +213,18 @@ func Join(s ...string) string { return rez } -// Add or update existing package entry in database archived data. -func UpdatePacmanDbEntry(db []byte, md *Metadata) ([]byte, error) { - // Read existing entries in archive. - entries, err := readEntries(db) - if err != nil { - return nil, err - } - - // Remove entries related old package versions. - entries = CleanOldEntries(entries, md.Name) - - // Add new package entry to list. - entries[md.Name+"-"+md.Version+"/desc"] = []byte(md.GetDbDesc()) - - var out bytes.Buffer - - // Write entries to new buffer and return it. - err = writeToArchive(entries, &out) - if err != nil { - return nil, err - } - - return out.Bytes(), nil -} +// Create pacman database archive based on provided package metadata structs. +func CreatePacmanDb(mds []*Metadata) ([]byte, error) { + entries := make(map[string][]byte) -// Clean entries for old package versions from pacman database. -func CleanOldEntries(entries map[string][]byte, pkg string) map[string][]byte { - out := map[string][]byte{} - for entry, value := range entries { - splt := strings.Split(entry, "-") - basename := strings.Join(splt[0:len(splt)-2], "-") - if pkg != basename { - out[entry] = value - } + for _, md := range mds { + entries[md.Name+"-"+md.Version+"/desc"] = []byte(md.GetDbDesc()) } - return out -} - -// Add or update existing package entry in database archived data. -func RemoveDbEntry(db []byte, pkg, ver string) ([]byte, error) { - // Read existing entries in archive. - entries, err := readEntries(db) - if err != nil { - return nil, err - } - - // Add new package entry to list. - delete(entries, pkg+"-"+ver+"/desc") var out bytes.Buffer // Write entries to new buffer and return it. - err = writeToArchive(entries, &out) + err := writeToArchive(entries, &out) if err != nil { return nil, err } @@ -271,38 +232,7 @@ func RemoveDbEntry(db []byte, pkg, ver string) ([]byte, error) { return out.Bytes(), nil } -// Read database entries containing in pacman archive. -func readEntries(dbarchive []byte) (map[string][]byte, error) { - gzf, err := gzip.NewReader(bytes.NewReader(dbarchive)) - if err != nil { - return map[string][]byte{}, nil - } - - var entries = map[string][]byte{} - - tarReader := tar.NewReader(gzf) - for { - header, err := tarReader.Next() - if err == io.EOF { - break - } - - if err != nil { - return nil, err - } - - if header.Typeflag == tar.TypeReg { - content, err := io.ReadAll(tarReader) - if err != nil { - return nil, err - } - entries[header.Name] = content - } - } - return entries, nil -} - -// Write pacman package entries to empty buffer. +// Write pacman package entries to tarball. func writeToArchive(files map[string][]byte, buf io.Writer) error { gw := gzip.NewWriter(buf) defer gw.Close() diff --git a/routers/api/packages/arch/arch.go b/routers/api/packages/arch/arch.go index 52dfad0641f3..5ed794fedf03 100644 --- a/routers/api/packages/arch/arch.go +++ b/routers/api/packages/arch/arch.go @@ -4,6 +4,7 @@ package arch import ( + "bytes" "encoding/hex" "io" "net/http" @@ -87,7 +88,7 @@ func Push(ctx *context.Context) { } // Parse metadata contained in arch package archive. - md, err := arch_module.EjectMetadata(filename, setting.Domain, pkgdata) + md, err := arch_module.EjectMetadata(filename, distro, setting.Domain, pkgdata) if err != nil { apiError(ctx, http.StatusBadRequest, err) return @@ -121,20 +122,6 @@ func Push(ctx *context.Context) { return } - // Save file related to arch package description. - _, err = arch_service.SaveFile(ctx, &arch_service.SaveFileParams{ - Organization: org, - User: user, - Metadata: md, - Data: []byte(md.GetDbDesc()), - Filename: filename + ".desc", - Distro: distro, - }) - if err != nil { - apiError(ctx, http.StatusInternalServerError, err) - return - } - // Automatically connect repository for provided package if name matched. err = arch_service.RepositoryAutoconnect(ctx, owner, md.Name, pkgid) if err != nil { @@ -157,38 +144,32 @@ func Get(ctx *context.Context) { // Packages are stored in different way from pacman databases, and loaded // with LoadPackageFile function. if strings.HasSuffix(file, "tar.zst") || strings.HasSuffix(file, "zst.sig") { - pkgdata, err := arch_service.LoadPackageFile(ctx, distro, file) + pkgdata, err := arch_service.LoadFile(ctx, distro, file) if err != nil { apiError(ctx, http.StatusNotFound, err) return } - _, err = ctx.Resp.Write(pkgdata) - if err != nil { - apiError(ctx, http.StatusInternalServerError, err) - return - } - - ctx.Resp.WriteHeader(http.StatusOK) + ctx.ServeContent(bytes.NewReader(pkgdata), &context.ServeHeaderOptions{ + Filename: file, + CacheDuration: time.Minute * 5, + }) return } - // Pacman databases are stored directly in gitea file storage and could be - // loaded with name as a key. + // Pacman databases is not stored in gitea's storage, it is created for + // incoming request and cached. if strings.HasSuffix(file, ".db.tar.gz") || strings.HasSuffix(file, ".db") { - data, err := arch_service.LoadPacmanDatabase(ctx, owner, distro, arch, file) - if err != nil { - apiError(ctx, http.StatusNotFound, err) - return - } - - _, err = ctx.Resp.Write(data) + db, err := arch_service.CreatePacmanDb(ctx, owner, arch, distro) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return } - ctx.Resp.WriteHeader(http.StatusOK) + ctx.ServeContent(bytes.NewReader(db), &context.ServeHeaderOptions{ + Filename: file, + CacheDuration: time.Minute * 5, + }) return } @@ -200,11 +181,9 @@ func Remove(ctx *context.Context) { var ( owner = ctx.Params("username") email = ctx.Req.Header.Get("email") - distro = ctx.Req.Header.Get("distro") target = ctx.Req.Header.Get("target") stime = ctx.Req.Header.Get("time") version = ctx.Req.Header.Get("version") - arch = strings.Split(ctx.Req.Header.Get("arch"), " ") ) // Parse sent time and check if it is within last minute. @@ -242,21 +221,8 @@ func Remove(ctx *context.Context) { return } - // Remove pacman database entries related to package. - err = arch_service.RemoveDbEntry(ctx, arch, owner, distro, target, version) - if err != nil { - apiError(ctx, http.StatusInternalServerError, err) - return - } - // Remove package files and pacman database entry. - err = arch_service.RemovePackage(ctx, &arch_service.RemoveParameters{ - User: user, - Organization: org, - Owner: owner, - Name: target, - Version: version, - }) + err = arch_service.RemovePackage(ctx, org.AsUser(), target, version) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return diff --git a/services/packages/arch/db_manager.go b/services/packages/arch/db_manager.go index b893d70c9c49..361f5e74776e 100644 --- a/services/packages/arch/db_manager.go +++ b/services/packages/arch/db_manager.go @@ -5,25 +5,31 @@ package arch import ( "bytes" + "fmt" + "io" + "strings" - org "code.gitea.io/gitea/models/organization" - pkg "code.gitea.io/gitea/models/packages" + "code.gitea.io/gitea/models/db" + org_model "code.gitea.io/gitea/models/organization" + pkg_model "code.gitea.io/gitea/models/packages" "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/json" "code.gitea.io/gitea/modules/packages" "code.gitea.io/gitea/modules/packages/arch" - svc "code.gitea.io/gitea/services/packages" + pkg_service "code.gitea.io/gitea/services/packages" ) // Parameters required to save new arch package. type SaveFileParams struct { - *org.Organization + *org_model.Organization *user.User *arch.Metadata Data []byte Filename string Distro string + IsLead bool } // This function create new package, version and package file properties in @@ -36,25 +42,26 @@ func SaveFile(ctx *context.Context, p *SaveFileParams) (int64, error) { } defer buf.Close() - pv, _, err := svc.CreatePackageOrAddFileToExisting( - &svc.PackageCreationInfo{ - PackageInfo: svc.PackageInfo{ + pv, _, err := pkg_service.CreatePackageOrAddFileToExisting( + &pkg_service.PackageCreationInfo{ + PackageInfo: pkg_service.PackageInfo{ Owner: p.Organization.AsUser(), - PackageType: pkg.TypeArch, + PackageType: pkg_model.TypeArch, Name: p.Metadata.Name, Version: p.Metadata.Version, }, Creator: p.User, Metadata: p.Metadata, }, - &svc.PackageFileCreationInfo{ - PackageFileInfo: svc.PackageFileInfo{ + &pkg_service.PackageFileCreationInfo{ + PackageFileInfo: pkg_service.PackageFileInfo{ Filename: p.Filename, CompositeKey: p.Distro + "-" + p.Filename, }, Creator: p.User, Data: buf, OverwriteExisting: true, + IsLead: p.IsLead, }, ) if err != nil { @@ -63,12 +70,46 @@ func SaveFile(ctx *context.Context, p *SaveFileParams) (int64, error) { return pv.PackageID, nil } +// Get data related to provided file name and distribution, and update download +// counter if actual package file is retrieved from database. +func LoadFile(ctx *context.Context, distro, file string) ([]byte, error) { + db := db.GetEngine(ctx) + + pkgfile := &pkg_model.PackageFile{CompositeKey: distro + "-" + file} + + ok, err := db.Get(pkgfile) + if err != nil || !ok { + return nil, fmt.Errorf("%+v %t", err, ok) + } + + blob, err := pkg_model.GetBlobByID(ctx, pkgfile.BlobID) + if err != nil { + return nil, err + } + + if strings.HasSuffix(file, ".pkg.tar.zst") { + err = pkg_model.IncrementDownloadCounter(ctx, pkgfile.VersionID) + if err != nil { + return nil, err + } + } + + cs := packages.NewContentStore() + + obj, err := cs.Get(packages.BlobHash256Key(blob.HashSHA256)) + if err != nil { + return nil, err + } + + return io.ReadAll(obj) +} + // Automatically connect repository to pushed package, if package with provided // with provided name exists in namespace scope. func RepositoryAutoconnect(ctx *context.Context, owner, repository string, pkgid int64) error { repo, err := repo.GetRepositoryByOwnerAndName(ctx, owner, repository) if err == nil { - err = pkg.SetRepositoryLink(ctx, pkgid, repo.ID) + err = pkg_model.SetRepositoryLink(ctx, pkgid, repo.ID) if err != nil { return err } @@ -76,19 +117,61 @@ func RepositoryAutoconnect(ctx *context.Context, owner, repository string, pkgid return nil } -type RemoveParameters struct { - *user.User - *org.Organization - Owner string - Name string - Version string +// This function is collecting information about packages in some organization/ +// user space, and created pacman database archive based on package metadata. +func CreatePacmanDb(ctx *context.Context, owner, architecture, distro string) ([]byte, error) { + u, err := user.GetUserByName(ctx, owner) + if err != nil { + return nil, err + } + + pkgs, err := pkg_model.GetPackagesByType(ctx, u.ID, pkg_model.TypeArch) + if err != nil { + return nil, err + } + + var mds []*arch.Metadata + + for _, pkg := range pkgs { + vers, err := pkg_model.GetVersionsByPackageName(ctx, u.ID, pkg_model.TypeArch, pkg.Name) + if err != nil { + return nil, err + } + for i := len(vers) - 1; i >= 0; i-- { + var md arch.Metadata + err = json.Unmarshal([]byte(vers[i].MetadataJSON), &md) + if err != nil { + return nil, err + } + if checkArchitecture(architecture, md.Arch) && md.Distribution == distro { + mds = append(mds, &md) + break + } + } + } + + return arch.CreatePacmanDb(mds) } -// Remove package and it's blobs from gitea. -func RemovePackage(ctx *context.Context, p *RemoveParameters) error { - ver, err := pkg.GetVersionByNameAndVersion(ctx, p.Organization.ID, pkg.TypeArch, p.Name, p.Version) +// Remove specific package version related to provided user or organization. +func RemovePackage(ctx *context.Context, u *user.User, name, version string) error { + ver, err := pkg_model.GetVersionByNameAndVersion(ctx, u.ID, pkg_model.TypeArch, name, version) if err != nil { return err } - return svc.RemovePackageVersion(p.User, ver) + + return pkg_service.RemovePackageVersion(u, ver) +} + +// Check wether package architecture is relevant for requestsed database. +func checkArchitecture(architecture string, list []string) bool { + for _, v := range list { + if v == "any" { + return true + } + if v == architecture { + return true + } + } + return false } diff --git a/services/packages/arch/file_manager.go b/services/packages/arch/file_manager.go deleted file mode 100644 index 4f146baa4da8..000000000000 --- a/services/packages/arch/file_manager.go +++ /dev/null @@ -1,153 +0,0 @@ -// Copyright 2023 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package arch - -import ( - "bytes" - "fmt" - "io" - "strings" - - "code.gitea.io/gitea/models/db" - pkg_mdl "code.gitea.io/gitea/models/packages" - "code.gitea.io/gitea/modules/context" - "code.gitea.io/gitea/modules/packages" - "code.gitea.io/gitea/modules/packages/arch" - "code.gitea.io/gitea/modules/setting" -) - -// Get data related to provided file name and distribution, and update download -// counter if actual package file is retrieved from database. -func LoadPackageFile(ctx *context.Context, distro, file string) ([]byte, error) { - db := db.GetEngine(ctx) - - pkgfile := &pkg_mdl.PackageFile{CompositeKey: distro + "-" + file} - - ok, err := db.Get(pkgfile) - if err != nil || !ok { - return nil, fmt.Errorf("%+v %t", err, ok) - } - - blob, err := pkg_mdl.GetBlobByID(ctx, pkgfile.BlobID) - if err != nil { - return nil, err - } - - if strings.HasSuffix(file, ".pkg.tar.zst") { - err = pkg_mdl.IncrementDownloadCounter(ctx, pkgfile.VersionID) - if err != nil { - return nil, err - } - } - - cs := packages.NewContentStore() - - obj, err := cs.Get(packages.BlobHash256Key(blob.HashSHA256)) - if err != nil { - return nil, err - } - - return io.ReadAll(obj) -} - -// Get data related to pacman database file or symlink. -func LoadPacmanDatabase(ctx *context.Context, owner, distro, architecture, file string) ([]byte, error) { - cs := packages.NewContentStore() - - file = strings.TrimPrefix(file, owner+".") - - dbname := strings.TrimSuffix(arch.Join(owner, distro, architecture, file), ".tar.gz") - - obj, err := cs.Get(packages.BlobHash256Key(dbname)) - if err != nil { - return nil, err - } - - return io.ReadAll(obj) -} - -// This function will update information about package in related pacman databases -// or create them if they do not exist. -func UpdatePacmanDatabases(ctx *context.Context, md *arch.Metadata, distro, owner string) error { - // If architecure is not specified or any, package will be automatically - // saved to pacman databases with most popular architectures. - if len(md.Arch) == 0 || md.Arch[0] == "any" { - md.Arch = popularArchitectures() - } - - cs := packages.NewContentStore() - - // Update pacman database files for each architecture. - for _, architecture := range md.Arch { - db := arch.Join(owner, distro, architecture, setting.Domain, "db") - dbkey := packages.BlobHash256Key(db) - - var dbdata []byte - - dbobj, err := cs.Get(dbkey) - if err == nil { - dbdata, err = io.ReadAll(dbobj) - if err != nil { - return err - } - } - - newdata, err := arch.UpdatePacmanDbEntry(dbdata, md) - if err != nil { - return err - } - - err = cs.Save(dbkey, bytes.NewReader(newdata), int64(len(newdata))) - if err != nil { - return err - } - } - - return nil -} - -func RemoveDbEntry(ctx *context.Context, architectures []string, owner, distro, pkg, ver string) error { - cs := packages.NewContentStore() - - // If architecures are not specified or any, package will be automatically - // removed from pacman databases with most popular architectures. - if len(architectures) == 0 || architectures[0] == "any" { - architectures = popularArchitectures() - } - - for _, architecture := range architectures { - db := arch.Join(owner, distro, architecture, setting.Domain, "db") - dbkey := packages.BlobHash256Key(db) - - var dbdata []byte - - dbobj, err := cs.Get(dbkey) - if err != nil { - return err - } - - dbdata, err = io.ReadAll(dbobj) - if err != nil { - return err - } - - newdata, err := arch.RemoveDbEntry(dbdata, pkg, ver) - if err != nil { - return err - } - - err = cs.Save(dbkey, bytes.NewReader(newdata), int64(len(newdata))) - if err != nil { - return err - } - } - return nil -} - -func popularArchitectures() []string { - return []string{ - "x86_64", "arm", "i686", "pentium4", - "armv7h", "armv6h", "aarch64", "riscv64", - } -} From 6d7c77071e46ca1eb4ed952bc873f8ab3a214ec7 Mon Sep 17 00:00:00 2001 From: dancheg97 Date: Mon, 3 Jul 2023 16:39:45 +0300 Subject: [PATCH 025/124] build date, installed and compressed size UI view --- templates/package/content/arch.tmpl | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/templates/package/content/arch.tmpl b/templates/package/content/arch.tmpl index 8f5e4337a4b9..8483898d1e40 100644 --- a/templates/package/content/arch.tmpl +++ b/templates/package/content/arch.tmpl @@ -33,6 +33,21 @@ Server = https://{{.PackageDescriptor.Metadata.BaseDomain}}/api/packages/{{.Pack + + + + + + + + + + + + + + + {{range $key := .PackageDescriptor.Metadata.Provides}} From 568c3b12b99a410a3b82f7b4c0be1553fb3de7b0 Mon Sep 17 00:00:00 2001 From: dancheg97 Date: Tue, 4 Jul 2023 20:02:32 +0300 Subject: [PATCH 026/124] test for package create, get and delete operations --- docs/content/doc/usage/packages/arch.en-us.md | 4 +- routers/api/packages/arch/arch.go | 16 +- tests/integration/api_packages_arch_test.go | 332 ++++++++++++++++++ 3 files changed, 341 insertions(+), 11 deletions(-) create mode 100644 tests/integration/api_packages_arch_test.go diff --git a/docs/content/doc/usage/packages/arch.en-us.md b/docs/content/doc/usage/packages/arch.en-us.md index d2d02ec1ad22..b67266338271 100644 --- a/docs/content/doc/usage/packages/arch.en-us.md +++ b/docs/content/doc/usage/packages/arch.en-us.md @@ -67,7 +67,7 @@ gpg --verify package-ver-1-x86_64.pkg.tar.zst.sig 2. Sign message metadata, which consists of package owner (namespace in gitea), package file name and send time. You can do that by running following command: ```sh -echo -n {owner}{package}-1-1-x86_64.pkg.tar.zst$(date --rfc-3339=seconds | tr " " T) >> md +echo -n {owner}{package}$(date --rfc-3339=seconds | tr " " T) >> md gpg --detach-sign md ``` @@ -166,11 +166,9 @@ curl -X DELETE \ http://{domain}/api/packages/$owner/arch/remove \ --header "username: $owner" \ --header "email: $email" \ - --header "distro: archlinux" \ --header "target: $package" \ --header "time: $time" \ --header "version: $version" \ - --header "arch: $arch" \ --header 'Content-Type: application/octet-stream' \ --data-binary @md.sig ``` diff --git a/routers/api/packages/arch/arch.go b/routers/api/packages/arch/arch.go index 5ed794fedf03..b214c35261b6 100644 --- a/routers/api/packages/arch/arch.go +++ b/routers/api/packages/arch/arch.go @@ -71,9 +71,16 @@ func Push(ctx *context.Context) { return } + // Parse metadata contained in arch package archive. + md, err := arch_module.EjectMetadata(filename, distro, setting.Domain, pkgdata) + if err != nil { + apiError(ctx, http.StatusBadRequest, err) + return + } + // Validating metadata signature, to ensure that operation push operation // is initiated by original package owner. - sendmetadata := []byte(owner + filename + sendtime) + sendmetadata := []byte(owner + md.Name + sendtime) err = arch_service.ValidateSignature(ctx, sendmetadata, msigdata, user) if err != nil { apiError(ctx, http.StatusUnauthorized, err) @@ -87,13 +94,6 @@ func Push(ctx *context.Context) { return } - // Parse metadata contained in arch package archive. - md, err := arch_module.EjectMetadata(filename, distro, setting.Domain, pkgdata) - if err != nil { - apiError(ctx, http.StatusBadRequest, err) - return - } - // Save file related to arch package. pkgid, err := arch_service.SaveFile(ctx, &arch_service.SaveFileParams{ Organization: org, diff --git a/tests/integration/api_packages_arch_test.go b/tests/integration/api_packages_arch_test.go new file mode 100644 index 000000000000..88a9fc10c26b --- /dev/null +++ b/tests/integration/api_packages_arch_test.go @@ -0,0 +1,332 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package integration + +import ( + "bytes" + "encoding/base64" + "encoding/hex" + "fmt" + "io" + "net/http" + "path" + "testing" + + auth_model "code.gitea.io/gitea/models/auth" + "code.gitea.io/gitea/models/unittest" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/setting" + api "code.gitea.io/gitea/modules/structs" + "code.gitea.io/gitea/tests" + "github.com/stretchr/testify/assert" +) + +func TestPackageArch(t *testing.T) { + const pkg = `KLUv/QRYPQ8BWoM5Q0aQriRpAwIANx0x6IHGcK3rM6A/BEBTh6DDFs4wGLaRRiRyf3f3Jj8RQily +i6KqPmMmDr1trRFCJHLRBHKMnQsHagvJ4UABCgTsA+QDD6VE+myp6RlHFEUtjl6KsWjtuJ1NkyjW +udbrkGuys5M7mlrdpJ/eTp/coaikr0vXcbvsYtP6OZJVclavK0o7VcnUPZD6olZL6Y6kr5O+cdbX +mbnW9403U6rdoOgNRTHcNI/khy8t09qty1AUF8wOfYi7j69LtMJ2+aKqlMay1owbXGNrx9XOClmz +YH0dejQUa4kfvSil5ZwaqvXBGh+Nwbc+kLqGjjFlKmmabypZqvV1a910pqklv2lbM4Q9KkrbKLZD +4bFlGbnG41JL54+eKH1r/uhLUSs2xtTq99KJXmwqUUsnannJi5FJLjLlaHkj2LhupT8N2cY2gklr +G2XH2UYuUw/V+r2EoLvw4bfChtN3mdaWb1+Xb6eyVKJW0zpnhQ1lR1Gs462yMk0lj26opRbWDnUS +LtVo/sHUkjQWtZTCWM/oZ71Po5taSmmnNWProEqWlH5dpopSmGI9Z73voEM1mGJd7z9U0k7Kv6hz +WY/V2mqNPTv4zefrsB18MMUa+3UNHc3+hUOttu5CXL4XPZUoprFTr23c3NxYP9RSrF/mehuKuRP/ +LuqhVmdZ8XvpYNTSFx7JphwqaVrxM70U1RrLzxH9x7T0C6UZthRTSz/YvZYueUzasXvPvepewzWH +EhunorVb4fjHSYtiIDdrjppqSyVtNQvWLinNGqrVHDGVdP4Y9c3Z0VTrOaGcKIeiWvutsNtb3Hnp +thrrwVbrRqtzHfo34lBPEX3JGl0xoxbVHnhb42z/Qtz9wMuSnpVRS0N5xrYo7OGulsZEHedjwqgd +lktDK+yIU2W1NbfDsinW72VrqEQtC9bY5Tf12o7XwXZYMtXU6ozp0wp/KUVpfCu+EWyX3zmU0klr +plj79Cc3lJ2GxAh1zbqRmLYUq/jGLMUqoqYs1RpnWpuu17bHCtvhjSpufzUmzVSiWsuHSphKWjt8 +6TrGmUYVwmadqKWxlbTV2lrY8h0/xzDBiC6ly5i+ETVLJJzoeUTCnRtZN+1f3I+RbKe/l/Czy1jW +mt9LJDCs0f7NmGGCgWn9bKsR1HmMBAT9gf5AByESt8oKIcz4IxIMEwyJelNLMQ65yTDBkLXmfF3G +SSThsz5JMEww7jPlkTwSh4NhDBMMzy4a4YEIq4nOBgSNh+MPpC52gPFAEfqzloDh5FOnhOMg6KwR +LTqlYc0xTGPwwRwN5hyuPH/+gIxb2mOt2FAai8OhrMdzNJLm3qyEJQ2zHIwOhgnGjDCzNPfgE1F+ +pDAnGMfEMMF44HUOZI1gzRgmGNcl7J4DWSMcyEbqLsMEw6ejDmSNaAwTjOzmrQ0j13rXKuK6jKPF +KKLRSBpbjKUlyrKsRVEYg1GUpbGKYCgKs9YqggUlWMESSqRgCRmIQAXOIc57yTycRM57yWzNt97r +7VjnvXTVeS/jg95LKknnvYzcmh9X6/ed99JZ7ryX72Vz3svVU7bO/TaKNaSeaW3eS+Wb5r1sDZ1r +KDbvZVPM5I1l6qwlj5K/l5GpWndIfy9PrGSx1lZ7L5WPXHbRmWI6Or9RiowpRbWeN76XSz8wO4s/ +Jsz/ZqbWe4lt5P5ozSVOheNW4OvxpXptjKqE7+VSraKpxdp9pe46m7SuH+H71Q711ZgFnYp2K5a7 +1x+1r1a767M5+F629ovHniOPNb1RI76G33tp33uZ3Ei+95JHCFb53kvHQ61ur3lFLWq18kgYlB5M +rc7XTexy45YIHYxTkWAKaYS6CLv3siXvJav/XiaMBVOsMY2yESWeUPp0clEPdhO3XN6Fko51QhGH +HtRYWwtpxSgUoxitmKnVsGnFKBpTbbFui2G/KB6RYmqp1Rpjl9H54xeFkgLFYsEaVinddBSRerCG +IgrqtjZjUFw6U5+03peot5MciusWOxRMtaYVY+pQtI6QZsE6/8KU0rjiZ8w6FHsu22e6eY7qtc9q +zTLTV54+c/PbsmD2dtS9Q7Wm/tjIk6gFeWj0PPB1tm3Oj635iTjI4/zD1Oq3Lv7TYGp9BL/RFxfR +a657mGJt/2JRrHEjrei8pTayMkra5UdLL0ppBeIvQbmTORUoJ0ppKrFgnUB5MaVYsPbsoo0aykmi +WMMaG8rXp/PruKnoXmO/D3H7tuk7kbSdsWJtI8ZMWZxEGlXcJnbTH4gd5qtxjcXhKIZZrmLWi25T +r5O09ctUopgFa8+/qD/W3SZbxrHYHO4oR3JVQy27mmOT5rpNurluP9PoQre43ky/M3QUr65WscyR +oppq7diIOpBpxEdsToXCboXL/prSGHIOydLRn9fz0zqXdvnZURwWrCFYY9fcD3bxute25s7Q2Vkh +9Tx5k9IfNt3Fxg79nB1jx+30Ja3YqbIpFswh/lJnWptIfgi/YsGuZ+qwZb2ZVgdjC3Mo6+FUNOVW +LMUkjqNu9q2yQrBKz9Bd98DrInXFyQhKB5vSSyX95FqzR/Kp1dfVHLn2WemjUdZyi3I4VlOrqZ1O +mz8fHS0Hw7yXTiNRrSGGNSbd3kunoZoFdCskO+33kmkoZtEao3BwIzu/olrnd97L5h2m2OQ5u5Mk +rbrDIgmrN6UoiQXryHXuzOgw1RTVujXRm942FJ2mWM8XHaaUVvpChykGncW68ViO5Wgsp4KxCkY1 +lXRDK4QOawidphwNnaaWhiDOSRxHzUM9cNQwlTiOGhb//W0Splo3t5HsfhuVdI6k434bhalm0dr5 +hlaYaW0WplZjSNZGmVw2CXSyCU+OtIb4J7atm+jVXK9NHNrYwnrtg9lqWeeHZLUWsopaTd2cUNSP +UaZO3yqqTU1GN9dXb0MgEKaW/hutjtDPHbk5SVTz+bkpprNT1uS7aSPZffTZFiIZxTQEAnkvlbUU +q8+afa6OU0VRTLvjMPa5LMUo5qFaYducdHbsuOxQiZz5Ydqhn/HLR1sNptZivSwHq6Te8rqiVjut +Eral5XXr+SXE0CpqsW5+oVWUo/1C+9AyqqFa0tHzZQtH962zylqqQ52dzqrFOm699yd1EzoLQWeT +W22VNlGPfjCmbutqjts7OXVbmcauRuq2TkfdViYddVsfimk3oxs5T2uE3xLWR6JWT4lm6GZEaV+L +6ZMtfTDl6Gnp41HU6bnSRmdbKQqPlapH24jTHitlKOjTnJ22zDDStpxQ8I0g7UodbTma/WRnP8tI +UUspFmzmmTTm2GnVUk0pfRvLlNrQb+pedPW+nuqL4OglXd/C/PShmTCLpz8Z9tkzujgdUImzqRXJ +EIxix+ukdFMpTJkXUrmQDqZYN2lpF8kXZ8MiSxzFlKuWYlRiwajjEzOZkpRRUr9QjPrFA7kOMu83 +19+6niLalNLUakm+bAxTh/HBNM9dUtFa0m5LUaRxmiPTRabEOJoJTkmaMs4LS+rNfPD0Xkbqd0pu +pO5O0ZyWnJNa+vpLTkLTg6a7Tp/91Ewt09pM7xXT2Kdvpjz5BGd03HRxkuA0LYlSlDL9xBTrB5la +E8PYpFiIopa0fh9in0OYUjpTfwkf5L2cFTrImqz0XuJ/t0r5usQ6cjZK16W/sR322U5q7aT3UlGs +pSjb6TlqcYrmw9F7+XAUvZcRvNFDaMQyfho1FLNgDt3NrnFAZ06FUtFQimr3s9JN89nd6u3dk99L +x3SeH0wpzSh22W+UrYPd1GtxBLc5PWOmWLcHY6ZWPxgnd6otsVHF76VaKlYcKX4v30ultL0v8Xu5 +VNIVFdB5/F4uv9CxnRVCxwvGDjOHIY3g+14yi4Tvw+97mTSIu+8Vc1EKiGWKYe1YRva9tJGp3pYu +36ooxr6XC7+076XjzYg6rY99LxubjmWNnVuH0D2076WD9qFmzL6Xq7Od1G1VD8dJ6a/lT7M6Tl0S +00jzuTpFsZ70vVRJy+bw6Xu55Pw4tOWGLo0biL7MThv6XrZFrc4QBt3NlFZ44nqW32Sn7yV9LxUF +R/Jz5KhrKo5MO9/LRyRtrJPO99Khk7WemrniVDk7dbNlKN1UUb6XLIEycQ9DckKxvBvldtyiqPgr +/veSSfw5svSXfp7+e/lgLv/V+e/lShwOxkD/9zJpND7893KBv3wkUve8l82fHEYshgmGXzxOIgkL +bHN+GCYY8kZG0PquBMMEA2KPLFGMwXDo44hRgchjRDcBowJNYY2lrGUCFLRAAWDWMliAgAVg8AN0 +gyKo4QUXuIAJInwhB2ZQf6AAD5QIhBU6oIcdbCADDAiSI9DQhScAgggiSCIJRGBBCRgoQQhd+BGE +4Ygj0IFJAiMGVCCCEQxRBRD8YA6HQMQeklCDA6AQgYsoVCihAQLsYQpIEMERUojDQ/tAgQpIgFGA +g4+YgxTYcHEFGdhBBVwwxAaOiLgg7GEBMOCAF86wRAxqEIACdGELjBAuMIANJBQsujkCCEAYgTNE +GxLQgyGkcDTBC0UwbrAwgyCSqBDAAR4cP/hAFhJxhz5g4AEz8AHOwwoaQIQlviCEJchBD3UYgxiy +QAc0EEFywwkdSD4M4gKFGIQF0hAHLdDgogcIyAEaGECANyzBBcYEHuCCETBBiVmEFKzBCUqgiSDD +i0Ii9FCABIRgACfQAhx6IIADGGECC2uwZClrkYAEL4iBCtQSM75Y0GLSJZfrcV4+zj89uUwjuB98 +0RZ2x4nwgy8urlN3erp7cWH/Yv6FonSKWa9fSDw1Kz3SmoskF5OypqnFlybn1qnha+u1K9Zpn/40 +HzRlxWKF1Ll1SlTH6+qLOFRjrWw57FKKRlsacDnK8WhctWa9T8OlqNKmwTyl8ZyKpD0qGuyhHg/P +9JPVksfXROaMYrNg/TlpHVknTtfV40vtK2vHUZ1TwvCzdTSCmWYczcS05c7hVCytIlEsOx+d6MRp +QhQ8Izji0FoPTg9OSS9Mm5bi7p5aHiu5SmGYWhZJK0ybmo7UIeieW4jDAr60NsInnJUYN7JCmqZC +/Fq1khMYu/T7kAf+A3EHsrDDei3M9zlkWSg7impQdlp+6Dlk0erR3aWpJLoqO8JfGnWQjBLPUmuo +lSWGJU019WhALDl0wmcka+OmxHAjnTtfYth86bmMVXwpiVKTipSo1vJztPrhaD0g1UbM/gWNFPWA +8McIJm21i7yYw03O19/LbnJusMUaW63j1xu7yRnnlnRynWGcu03r184KI/RzY/K8smLBWlrP013H +pecXc6JYX5eTlrOEIzhZWkcmuc6SVQmrMcKNqdXw20b5upzvrNld3Lh5sItSKCpJgWWFi8VxcrfL +/TYMq/gRsdZRPDGts+H2mqY5pyJhFU35/NnRr3uw0pAa3YMTtTpex9el6dEx++ccn2gy4hYxhhg7 +HF38ldL7YPfctVRTjuj8rmQ5k9yN1MmIgzSTUziJqbvZkTDD7rj5TfrsrpPQYzM5xcI0k6/jSLLu +o48+yo50vD/yjxjHdO1Dvx/9ZlyffjiCQDitR4KdIznSQ+tE1un6+b40peiIkxZ/bMgjPRg31Jlx +tcO3MmKLK45edHQ0PQXD3kt5I89O2un5b4RPOPRtJC9Mivz4oZR2+Doa9iiF34vs0MfX2YiNWAkd +jdiF+XQaazDiUE2tWKeDUawWlTTLOiIZ9NNdZxPVnhy52jhsI+i6mVBIjdB1E3LzIWn81h74rVtP +jp4cORyqNZdOrY6ikhU11DN2DVM6UrON3OzxuuebBi61LJU0taxHa4aoqJNFmlyip0nnmEJDQpMU +JB1DEUAgGHg4Go8LxpNq+xOAQBBHoTgQCAOBQCAQCAqKI4Gkh0mIzMzIyEiSpNAcs1q6QNNKdO0V +6C4jRKBDbQd0z9CAjuuUAzrlsc8dnGYEunQypynOcG6GDNCxvAaoq/gM7hh0gdR1QKA7pw0QaDok +5OibuXZ4XrD68t6NgS4GnQHfe5kIXzhcijLa5Hf/VV0lQni2biDXHr3eGEBmHiXKJXc5zWMjIO4k +fFUvLrLnRprRrNYzMC0SC4wlLtl6mJOKeexFFMks5ez8sRoVjX2Z9803JRqOj8NAjDiIGzZtJEFa +zLpEHpszYuSdT7VappDg/O0phV0K4LmXvEA/8LhBFZAQmNjBYOKl/iUVhsbEH9CyFimWn5m0+nSK +cLRVm0eQrioSNNHjmtCEUNmYT3KK/vlmwC9xZGwlpGX5CB3JCtKFJhUMRYs0QQXHafEqwdQ4ibYh +O0gs39VmjojW2JMWUPsuIwASABVNhWVUBmi4ojD8xY7qbMAr0aqNZeG+inR3fZHwn05MIdn1MrAQ +TyGfxKY3eciClcScRFwSGvE40U/PNA2nnELw2Iwjz2djO6C/lqF8WoEmYHnwxaAK960JyEu4aNpr +UPq7j8JjSg/hHmf1ySBbcqXxfr1odYRpe9BmblpASW+BNWT9ml3tuCMl94+b7e3tt1QNjY3y+yEF +HBJrji0lHOWJ8sgZ02HqEJL8m/pbzqL/siUayjR8D1TFoM0VsHSkcjD1MzDtsf87kaNZ0HuKA5m9 +StwYlxZgnHS7kIl/nustI9sFt4TlT+bBP86sf/Ly23bboAeddYAxDjd9L2mAxHJ9x7JM8olacOzj +rKb2GPLTW4aGhq1nSz7OLqZeZWhAGmdkRZPuNhj7W5OgJcU49ahcnRIWJntjACpGBId4MbJjjtrX +pJTTYcjJB5oRSyHRZ2lkbhbrV5jUS6JRnOchofXpJ7sHH9f8yyjdUXWpc6gkzZAtKWMva6QSe7Yf +01Zoq3FstZ0M1MymPulFVK2I+ORuLD8jpez5usPDWixyYmockuX8pRqFud5TWRBhEjfItmPGeb1J +s0jrf3C8+TMzwyodrsAC+URiNaOEnMMz4tRLgraoS8Cd86cdJcVAshPx7AqqiE5lEuc6c2thpSJV +898cAgjtjbTNAmPQ0ZMgTdseXkDWRZH663ZE1CcYP+onRHf5/5zpcVk8UJiyZ3agT6eTMZP0hZP1 +DJBL/NGJVvwQJIZpoJFwFlDMw5JMexEGcERxGFyQMozuE7DEsflKfEgHxq1OkulcJIe2ROj6CdeV +eOFDWB8qWfnqhyKyV3IyJj5FT4I1w7acqmyJn5yNrmIkcFQJlGgpY3lA/LKmjjUIulvQcIjtp487 +aRqDDbo3K4J257r1xnySQvnbV3tm7TLyDOr373xVmBdDBmYkQdEqpIq0JUfuFkoMNYAbyhryobMs +vEYg/WRAb3UjAiOSohRrKYcbDgMd6+6yhGbTLHpcuLfWctciDam8I+VwkJ7viftFSxcLYt+8gTKh +yEBXhIugsqChV0vZ4BlhM6Lmu7/SYP6tLhoycfXhESvb0Enn4qCPS5W1GDN3kaQLGsJDXXtoF0cM +28HAwBItnwPMR1avGEOAgBaMv5u1uDrgHpIlRkmlJ4Vs+m9W+kOs/7Idkjvjf4cl08kiEkq8JxBp +tFd62eDlTy8zzuoQ5q1gGKsLmSMDWI2AvCDP2UFGsh19Io1FlS8OVsWy/q98sF+Pa+Z1SWOOLmTW +cF5NhRRXq3jWG+zkVVOGZnxToexPwewJ6KrDMrhy+/iC9YhVL7dvqMvfOoVBi2X5hFYdwPnDQHqL +CSbCEyUm14SHTUH5bCAtewKhlbDogflLEquYqVcp7gShHZmhV2m1Qi/A+jQHGWI3uNS7+OhmlSpL +qjdaqtisRULhlOPj4dzpYXh4IUdGuHL+JBnygS6+2DtK6NqYRWaru4ek6e23BYjMWAWhZptLaufU +QLvSie7c86f1j8aQOJu2in2TARwQcsvsjvXBDJAHjzEr1ZBFKoafkVy4a7MAWVw4zamGWMnF95r3 +mQrvjduOmZ+bQDgxZg6cv2scLEZAyaqTihukzHJGetU+j4bf8MS5GvJgCAxVHEwkFT5wuchsOGgt +Vkg+p8I9Jv4Q9I8DvW4bDAgmXK/o89kNHnegFJJLG/IkkpHyfJ+uVFqBUf6n/S0LWXNyYz879L8S +tYzLWpWX3pBnQYkaB1Lz4qwdVx6HddS6/1Q4vrNOVUIEFGYcbhYKt3Hkcr3713sE7CIrk3741lta +fOhW2kVOSPWDCGgbFDFYIkHHDuzETAnMuzrOK38nvTSHg3XzcRNGizl9LrNQ5HS73xsV2HNmsPME +kGqquX1upDtkwqORcUgkCZcXjGOByVg/KDF/7ACiSdEWWbQcAWdnwDwbZEQrNMYKHG6idvfZdKkf +SFGkLkyrPw4qHsx3PGQ4r8Qg86VfWq8jELChbU0+WZ802EZ2s63U8W2vgCC5D1oDnrtIdyptGVmg +a7ScPlmYFTHLMyRcUDwmkhrCwsnpjvEo92PtZ2706XUJU61AkBh6guNHkuU4KiPSYq/ldiya9dBf +4iNCPKovP+duoIebNcwihVY/ajVxPRzQlqWyB2UC6lQpiYs6TqL+pGriKL8SANQAddkqw1uZRZca +9YtRDLqIQIgIGBMTQJm9YlRk/JppEnJnXg0iqk3YayCEaHX0Q4vR6ynZdF2RwJshSyAQ9hG8hnc4 +nKsG9+fr63OdwEH51tF5Jm2AzPwKesEmh2izqcC+GLwWFFlvbwioavZyUrY9Lv2aB5dvlS+AAazY +fFpefnpZD9JKNNH3gisS+Gy481c+dkih03vcHzvajk6KHFhyDtzoMZBUdui6L2necoBR/ztieQmC +IKMh9i9ZaTY5y4x0VIqFcuAmcuWvzYjXakmxR8Lv+HZ0mjnQQadtXGj1WFFgAdn2whaGe0LgivQS +GLBVasJoFm0037ylI93QcB5hdWeil6AyuIuFrFl7Ga3V1sPT2Mxb9vTK+kA75YU7nbMXCddl6V4f +GLvzjQzFbAVFLhO8m3KvNtnj4+UtN1AgJYaq3uJyKRshSD2OKHZTzrACCHATvb4fmAuMMAYc5/kK +J7W03jgWM4aYEI84KX0YfU2DyZ7rhmhjzIka6XzVK5vQTNQeNvSKuc6/k6dWyq9j6QidvqMRdhe0 +6e8lf3r409UR4To2FiBVAs35Dq96xZHxcejDZWoz6gXw7B7W6mW1mOaNGIujR+OWKOFnRjQXEHhf +ZyIse/zizc3MhUYx9uMvvfNn2lFsGOPcal7lKyMFg6EXgup63tuZDFVN1+o+0fqniDGRUOpPrTox +AW+COhx01z6oqC+ezJqGBVaQVxcP7Z4NH2MRMps1qONWRA90aZFDAQAuigUiWrVdVVtihZvmT9cy +m7kqbjmn+g3s20T0GstQnQiSPQfyBapoy5OKVNCg8FpLXx28R6WPgNb84E6Z1xfWz1rjRIT1hSTV +VE3d3Ip6P9QUT/ZEbD4gdN15MXwtavtDstNyC3KxM97gvv11aYk7Vfv+gGRraqXaduvSvvZISBvf +Z6Y+JAhq0kJyDg/jyF8kdZu5ekg6e6sQpegqfZyFHPFzn5b7I82IWM4pJDSOaQoTWwBHrVcgHCLj +nFsbpsv/IyqO7iFDq5PSi6O/+jtDjik5Q+jzBQwJWzDmf4n7OWChlriDG3vMV77g/do+zQSrFcJs +v4PdZT2yj0fGP9uRg5rP3QMXCPmBm8CTTgARPFRsCGQm96JpDoXsE4lC7d3JALoIwOciH8OrrHvH +oFahjl4xTaUcQGlagt2n/rSdlwPWvWT1EZ6qvR50hI60Ces5ASeUK95/k9u5MpSAWHc5vR3MH6aY +jDo2YK9YRZTWOPa1kq0/hFrZz0oh7DqppUSNe816GWXKolShkYsG15isKZbAUG7kO4pMbKkEwUk4 +OWH3kOaY+ILgsPhNCZQ6zl+DXOetnQEohuij9hYK2uZSe27M6mQ/bp9YZRIG9Jn0TyBGclNHil4/ +mAV0SN9JiXC1rVwuFs7hW4S3TKc8WMT+GPwyipjTXJ1ZCs+0k5k9AQX9NkhgMyqRyuhlTvZKQjRA +/U7ixMjDgcLfpZbNR6tRZAnIAo2ajV8JQy1hyilwoLWOSxdItDooWbIO4QcbuxNJPgxScxl/YWmN +onjDyr8ZkJApZe1ccv0evuAcPSEDtL8PjQZbTsBN7bgvIzgdmxEsS8cNdjzS+5pnLYtDj05SBPIV +KrE0HrQUZaawJ4phPKlwvqz9APiRTCEDumLkChL7XM+YImUvfunEUxhFIZar4gEEH9bgDF0DGnOV +NwmrzcDs94UO3cSuSAnrzkeTPhX8e3OcdW9yYd/FHN0AKrgGsQR11gTcqJh8Q281gD/a2r4UlWOu +8olrS3LM5Wq2+UpnwTOnPVhsppr1+hxUdjNP+Yw/EfG+2hW8ATzbnnXlbkMzO10MSiZHlF3EcTdS +EQILYngHjVCHUigFFVGTh32qfV2kuR6p0ZRE6/xu3BpVudNTKWHggA/32j9b6i4InmAY9iKAKKd0 +OaP2/q1L/BPCBk3C4PvzkV62k4KEzVTLlrId5WpSprheGFGBVoBNeK2vbf5VqA9UYCkILAos+uI7 +KQtz3GgE0WYGSSsdv4EepaKjbVebOCIMleKwjPqtJRMQNevBClT44kph9auwddj0Y1dkrTlGNL28 +q1zZqGpqF62H3kPqbaGyLRxrXBPcdbhYyqTMseWQo3sI+zzAZzCq+vUgcGqpZoOWh+xSMfHZiBfQ +V7oLSRmiOLflBl4V2+AYOEJzQ9IFVokUdfQGnlhsJqr4Qi6sqPn2XB9q7pYt5A7uKV+L+AcviJvO +ELzSY3vD8Q0r/E4vrUU/equw+bLeitPWh9ZqU5+Cw3AraSY5iAuGSDFh7BjFLdWVbyKGPSluradQ +n7q9h8fDK/ImJBBk7T0Pz4CM1fQXtnJFzVNLJd7w9QvVZGWkme+R/HZTfmh3y/iyokT0sgXw0ln9 +qTt1UvMYd10AMmoCY3i1T+4LgOQHSAzKOl839oDJ5FsDcZ6fgFH+nqOi2uuFVLFkyGEPl48ZOkJu +5tHLY0emfnz3eSuP/quXDSME9ofKRLozMLkc3GW1yw5XbWbWgcQqIYxBc9dRQgZGnv4b2ZIt1fJQ +mC30pfq2ztjpLcoUYFu5ebJ8/JTKTF6Z1LBA+fHxo27lTE73/wnx1NaFNAW6j+AaPIHEAX3kC4Zp +yZSoIXuvCKZpeyVy2CYeNN0edJfrSsiTVDS4Z3hAA9Ub5H9ZfxtHpYf0wYlFFa8JkVAgvaVyN5eW +5GCDGiLto6zdwDB6ZzqmdCHPrMfj5bbGG8ut2eUidRVasj6xZmhi1fCg6kEXX6rr4KE3K4H2uRAx +lLdjO3+zp7yTMIE2sRmvPEPVkfARrDuB+psg84sRMiNguXTrCOAwzL11NnU0Mh+gRYrECd1KbBb2 +6Am48uG79aTdB8eKnON+DqSDZOrQQWidRwB8plDbdhZZNoZZRpZqYQejAIz9K9ltwiKP+UUvgTWC +OAJ8Cdjbme7u0v5dByy7MQ7hhgLptxFslsdCBa119odRyGifx02Z3hPByM/iUPjmxWioiFNc2Sut +pdxaFAxcy9iX4aDaSvXchsOm6uVQq+roGxWUs25F5uN1hRsHPn9HVCMvwCKCywY0wvroPUUdtolq +E0QugmLyESwQCaLIBFElQfRqirLVHbZs5IQg2c4ohjxe8dVZ1SeXhWA0lhNd54zUZSWgU6W8kLDd +sLkw6hyju9t3QVfqi0JGZ5xa1ZTRwVQEIQH6LQ==` + + const signature = `iQGzBAABCAAdFiEEQ6WNuVKuWHQJqhdSETT46/mKoG8FAmSi2NYACgkQETT46/mKoG8ccgwAiyWE +lWTZ11Ah5AzQruNwVrC8FjW5EVP4BmWnpppuTF4pNlCaBNKh1saj2WMY3UYjwqsRGFuxJXrREU0O +l8m376ilOzoHMd4T5WBCOoJnuLu17HNElwe0YtnKbImwM2oYy/QdQP26GC6kiZ0Rmwia+tK9CvgI +epmK9MOacjs/TakA0z1+1itD+nVYpb3YEDTyHu0eMMtKeBwVIco26fmFa4ilyhIU2PzwxQnUd6sB +csrS4nvqPAS/a6Ubp9N1g3DYlz/Tzxz1F0zpBuSToQ1k/XEONI8HW4QkUpc3XFnAXNgw/ipAyC1j +3dmAfEV5lT93cCdjoxQ4vrygoWbuAfAyCcgs8iiDsXl+4tRhc0aMM+/CyYh8cUiVemldgCqEcFLF +Ta5/MiibmkmseToY8VFmNJiMDaey7Pu2smpbQXpOVhHUBQBru1VGm7G9N1QP6tp57z+xw7ombXzz +7R/6iYoqN7CdRK5IstoWoNV1qYdxmlpf1zZBYU7NFm/9OyRnHfhy` + + const mdsign = `iQGzBAABCAAdFiEEQ6WNuVKuWHQJqhdSETT46/mKoG8FAmSkT2UACgkQETT46/mKoG8s+wv/URHr +D7IvXVzcXzaaQhRKRDdywUPey4Yb+QSHl4eJ93qnaRslTdDc1wR4PGpZBahXGhXKGIBf9C+MSsmB +bUWIL05Bia3myBC1/oivm8or29vL6elt0Q+rNaOACMmVpGIRt6MxnI5KPhCPstv14p2y09ROf6/r +dQvFc1jeoNeS4JbcuA4bzABCMjZm822Z3ywYDtIJzGGMqbsGNSChYrY1NKdf60e24cJH9enRJUNX +k/qv2tUToChe+trVjx2EAOlbaenGHd1RjpRQk+CmEwNdhk8OKR6SEayynVYvU04QoTc/w170YXQp +G9oBJsMHOjlSIiZSByLe1kj1qsyGHfgZSg+C/igfGl2PXA4smgbEem4Y+P0j/i5Db6nQ81vZKZH0 +sNBESpX3kvN8rAR0cmHnGlelCklILv93SWZHL6+4yZaXVcm78Dfc905B1HPONb/zY5hqQ/CDuMK8 +ej9Z9qp6STZxDfr6HUNNMWCKgel/li9ptNlp3dOONEJHpJvGSGCk` + + const gpgkey = `-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQGNBGR6EBQBDACseiWT4OrE4FW7RuKmVN+ETK+yXUdViB2lWJRJrdm/XsjYKRvw +uFqDHDqZDOTV8cy93JzVa2xAIfa+aWdNrXyoZFOVgDdtuW00RrbNwztDDIDS5yy4 +bOX0pIv0UY3KgPX0N8zr+mn6wdpRqYvSx3tZMr+kAeXOnWQB1dsStixL8lO+iq3j +oPbOkMVxR+3ydQUz8f5xycKWSu8zU4L/OdCsrc7VN4UpeQcaOL2g+X09Y+1d96eG +u4knTGvUbix07WLNM8X6KtCtmAZW6qPfOM7pHWtqN3U4GrKQHHLh4xWYdT6yPqKA +WgngqqHMXinqC9+HPLKkWjHnbri0w3t1pS6WraHmcuYOZJepfEjacWxBpjgoP/PD +vrb9uUU72o6jyYFkoH/ji1dw5b/uaNR3FW+fuLHY0ymGFMr7BgWE7XK0X5ikLVjC +waOPowl7B3wbR7nIz7cWwDcC1W6rBaqltuaya5yWU231rC7YahOABJM0unBDyCpX +u4VfHNYeUMzOad8AEQEAAbQjRGFuaWxhIEZvbWlueWtoIDxkYW5jaGVnOTdAZm1u +eC5zdT6JAdQEEwEIAD4WIQRDpY25Uq5YdAmqF1IRNPjr+YqgbwUCZHoQFAIbAwUJ +A8JnAAULCQgHAgYVCgkICwIEFgIDAQIeAQIXgAAKCRARNPjr+Yqgby4pC/9Valyu +VmoWYbqFLkQhGdpsgyfDnIZ4VpZGZ4nWMnaOhQAzgcRnDYCvPryQYhTf87NzoCif +QhP4RMD7n3EmZNXxEINIKLgvWkp6ts5YquR6e+PhiQBoVhNOg1o7L/g4WCnyXqQM +68mla9jEBD+qKVQD9EIGS0HN/645ch39S3mMUFEGg4NDoRvoQcas3/3sbl1tyqTj +Q+oSHp5nRQVw+BtGBU4mu81gM75isf5fdRkYJdcAufQJbiTkEy1qhBTB2WpqBJe6 +a4KzhcpDDv8LXhanRUioRVrgUphlAUf9PCV3z6TLirDqm2tt4op1Sd/HnBQYOzs4 +ubBryS7bw48uAVEeJ4Ri3N0aHJDmtK4R56fhUBUMOeZb3Rg+erTsJ/V1U2aJxoF8 +i+w9aZcuA3CAMOyg5r8ILs+qD5ZpWDKUyWWa2toJK4+OXpzEUo6Mr9c4lsHLitvQ +zXHfgFSu7xZvi7jwBV09GM07C9Zv70f/4L2d+Qhj6udfFM44DFE+3Tzi9025AY0E +ZHoQFAEMAMY4zcCR8OR6mkcvVKsuDOdTy4LzSUPNL5WnoIICBbWFBVpU4T0AY+5E +E+3JxNDpgzzQp7YNNyO02Kkwo662bCYIMC0xa3siIeZZb82l/8+L68w/4wXDUXFF +qP5cWyw5J0zPLt7nRpEFlG3RbHJn6ndSL7uDTxMaJtREqyDvfuHhLcRqjAtnTj7W +SHumvIAacP1IWrcjfJVv6aBZzg3y31wx7KF0cx2CLAGBKC6DXhZdyLSzqhtgy0OY +ROf7x7sP/xu/UqStWVsQIj0/mxoNYoNH5g8Paa/80zIXZZlqvN1FycDH+U761hUd +1L2Gl5TBJy1gCoUGtbwog2LBdc76tY/IcHd6+8cUdxpKV7sCbTmLXI0sQm8imjez +JbG6dC/B5U2AdTdpVd1KIqC1SCZHG9dsnXvJDB6pKwDCcPC0Dl59lwHaD9b08YMP +pAA1J9rW6T682cGMsLHO3ipUaGnf1z4gA2IGhIsqgl00V4W2Es3gHzkAD6YJ9+H6 +hMRJCRaSZwARAQABiQG8BBgBCAAmFiEEQ6WNuVKuWHQJqhdSETT46/mKoG8FAmR6 +EBQCGwwFCQPCZwAACgkQETT46/mKoG8XxgwAqKYu0/irkZzhbLUiZPljE1dZSozX +bd0HtgizWt7iOiUdJzJrM9WGaXJ9hYykYp0IlDID5HNG+ZMgqS18XRAbUqjTalav +0ftjQdx25mUplCBOJmP7bciKjGUS2BLImjhMAHgwyf0MAQJApiBvH3kGLOEsL3Lh +uNwB04+iPFKYjIv6A4abRo4LnMy3we7o/IOwFqsSMZKKVWjY9q+au4h1q8h4YI70 +kDp+wItT92I90OMRHnSs88MrP8aMvVzI6Arph/5tVkthm0olO/TtSFC6ki4H3LdI +3Yzrx42A0MGeZQrQ49gydFFDreISYwU99AgIk33j85Ut3hkCriW5LbgOqHY21Fbs +LzlPDJpA9VZIQMcLZW0pFISTyWkGsSf0P1J8Z03RHzU8L5ROxonIyMriw7thMK/s +ZmaX8KGlqBgJ/6s6/8ERh7IbNvNu+PolTI5+PXDTofZ8ssKIcALQgF9zjaVIORfF +8Iu2wss37Re8PI2n7enU+iNe1UN81ZMojf/V +=5NFR +-----END PGP PUBLIC KEY BLOCK-----` + + const db = `H4sIAAAAAAAA/+yST2/TQBDFc55PkcscY++s9y9CCGO71CIpyCYcuKC1106sUNeyXanw6REpgjZX +VCHU/C6zs3rat293Rtf74bBb0YpC30z14glgjDGt9bEyxk4rUw/Wx30iraPFkj3FZU65nWY3Lthf +e52G+0/Ai3ydXcWbDOHBKKzujPqiRDAcdsHsxuD7NAPgIxkAvonLR/2nrCjz91cItCIATLMyQUib +qR67Ye5uegBMyvxzhmB0JAEwv+8YAG5SWW43CIaT9555LmsvIm+0p4i3kVMtk5o5B4DlZcylOqpt +xdsqIuddY4wgEtLbujbklbaSN0aTJE6iqrhVnjQXUWvIukbYynhuBABuizXCfp6H6UUYttf9XTDd +ht719b7ZWR3+SRcXySXC/cP8DL/N12kaf8wQSBkjjLKkAPBDnLyL32YFQur67qtbXtxcd/23w375 +8vepr3/5vAL41/9/5syZ58uPAAAA///ViF8vAAgAAA==` + + defer tests.PrepareTestEnv(t)() + + user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ + Name: "dancheg97", + ID: 2, + }) + + session := loginUser(t, "dancheg97") + token := getTokenForLoggedInUser(t, session, auth_model.AccessTokenScopeWriteRepository) + + req := NewRequestWithJSON(t, "POST", "/api/v1/user/gpg_keys?token="+token, api.CreateGPGKeyOption{ + ArmoredKey: gpgkey, + }) + MakeRequest(t, req, http.StatusOK) + + pkgData, err := base64.StdEncoding.DecodeString(pkg) + assert.NoError(t, err) + + sigData, err := base64.StdEncoding.DecodeString(signature) + assert.NoError(t, err) + + mdSigData, err := base64.StdEncoding.DecodeString(mdsign) + assert.NoError(t, err) + + dbData, err := base64.StdEncoding.DecodeString(db) + assert.NoError(t, err) + + rootURL := fmt.Sprintf("/api/packages/%s/arch", user.Name) + + // Set test time to 2023-07-04T19:57:09+03:00 before test execution. + // Add package gpg key to user in tests. + + t.Run("push", func(t *testing.T) { + req := NewRequest(t, "PUT", path.Join(rootURL, "/push")) + + req.Header.Set("filename", "randpkg-1-1-x86_64.pkg.tar.zst") + req.Header.Set("email", "dancheg97@fmnx.su") + req.Header.Set("time", "2023-07-04T19:57:09+03:00") + req.Header.Set("distro", "archlinux") + req.Header.Set("metasign", hex.EncodeToString(mdSigData)) + req.Header.Set("pkgsign", hex.EncodeToString(sigData)) + req.Header.Set("Content-Type", "application/octet-stream") + req.Body = io.NopCloser(bytes.NewReader(pkgData)) + + MakeRequest(t, req, http.StatusOK) + }) + + t.Run("get", func(t *testing.T) { + t.Run("package", func(t *testing.T) { + req := NewRequest(t, "GET", path.Join(rootURL, "/archlinux/x86_64/randpkg-1-1-x86_64.pkg.tar.zst")) + + req.Body = io.NopCloser(bytes.NewReader(pkgData)) + + resp := MakeRequest(t, req, http.StatusOK) + + assert.Equal(t, resp.Body.Bytes(), pkgData) + }) + + t.Run("database", func(t *testing.T) { + dbname := fmt.Sprintf("%s.%s.db", user.Name, setting.Domain) + + req := NewRequest(t, "GET", path.Join(rootURL, "/archlinux/x86_64/"+dbname)) + + req.Body = io.NopCloser(bytes.NewReader(pkgData)) + + resp := MakeRequest(t, req, http.StatusOK) + + assert.Equal(t, resp.Body.Bytes(), dbData) + }) + }) + + t.Run("remove", func(t *testing.T) { + req := NewRequest(t, "PUT", path.Join(rootURL, "/push")) + + req.Header.Set("username", "dancheg97") + req.Header.Set("email", "dancheg97@fmnx.su") + req.Header.Set("target", "randpkg") + req.Header.Set("time", "2023-07-04T19:57:09+03:00") + req.Header.Set("version", "1-1") + req.Header.Set("Content-Type", "application/octet-stream") + req.Body = io.NopCloser(bytes.NewReader(mdSigData)) + + MakeRequest(t, req, http.StatusOK) + }) +} From d8623beb56f71afbc942edf22d1ad8ddcbd37d17 Mon Sep 17 00:00:00 2001 From: dancheg97 Date: Thu, 6 Jul 2023 23:44:09 +0300 Subject: [PATCH 027/124] added test printing after execution --- docs/content/doc/usage/packages/arch.en-us.md | 12 ------------ tests/integration/api_packages_arch_test.go | 9 +++++++++ 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/docs/content/doc/usage/packages/arch.en-us.md b/docs/content/doc/usage/packages/arch.en-us.md index b67266338271..d3e70ec439e1 100644 --- a/docs/content/doc/usage/packages/arch.en-us.md +++ b/docs/content/doc/usage/packages/arch.en-us.md @@ -119,12 +119,6 @@ curl -X PUT \ --data-binary @$package ``` -Alternatively, you can install [pack](https://fmnx.su/core/pack) and execute push command. Pack is automatically handling all gpg/http related operations: - -```sh -pack -P {domain}/{owner}/{package} -``` - ## Delete packages 1. Prepare signature for delete message. @@ -172,9 +166,3 @@ curl -X DELETE \ --header 'Content-Type: application/octet-stream' \ --data-binary @md.sig ``` - -Alternatively, you can use [pack](https://fmnx.su/core/pack) to execute remote delete operations: - -```sh -pack -R {domain}/{owner}/{package}@{version-release} -``` diff --git a/tests/integration/api_packages_arch_test.go b/tests/integration/api_packages_arch_test.go index 88a9fc10c26b..a63fa032a4c3 100644 --- a/tests/integration/api_packages_arch_test.go +++ b/tests/integration/api_packages_arch_test.go @@ -278,6 +278,8 @@ ht719b7ZWR3+SRcXySXC/cP8DL/N12kaf8wQSBkjjLKkAPBDnLyL32YFQur67qtbXtxcd/23w375 // Add package gpg key to user in tests. t.Run("push", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + req := NewRequest(t, "PUT", path.Join(rootURL, "/push")) req.Header.Set("filename", "randpkg-1-1-x86_64.pkg.tar.zst") @@ -293,7 +295,10 @@ ht719b7ZWR3+SRcXySXC/cP8DL/N12kaf8wQSBkjjLKkAPBDnLyL32YFQur67qtbXtxcd/23w375 }) t.Run("get", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() t.Run("package", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + req := NewRequest(t, "GET", path.Join(rootURL, "/archlinux/x86_64/randpkg-1-1-x86_64.pkg.tar.zst")) req.Body = io.NopCloser(bytes.NewReader(pkgData)) @@ -304,6 +309,8 @@ ht719b7ZWR3+SRcXySXC/cP8DL/N12kaf8wQSBkjjLKkAPBDnLyL32YFQur67qtbXtxcd/23w375 }) t.Run("database", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + dbname := fmt.Sprintf("%s.%s.db", user.Name, setting.Domain) req := NewRequest(t, "GET", path.Join(rootURL, "/archlinux/x86_64/"+dbname)) @@ -317,6 +324,8 @@ ht719b7ZWR3+SRcXySXC/cP8DL/N12kaf8wQSBkjjLKkAPBDnLyL32YFQur67qtbXtxcd/23w375 }) t.Run("remove", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + req := NewRequest(t, "PUT", path.Join(rootURL, "/push")) req.Header.Set("username", "dancheg97") From 364c33b0adac811fbbf5b98be15a8d2d79269031 Mon Sep 17 00:00:00 2001 From: dancheg97 Date: Sun, 9 Jul 2023 11:48:46 +0300 Subject: [PATCH 028/124] fixed pacman database creation error for case with multiple architectures and distributions --- modules/packages/arch/metadata.go | 27 +++++++++++++-- routers/api/packages/arch/arch.go | 10 ++++++ services/packages/arch/db_manager.go | 49 +++++++++++++++++++++++----- 3 files changed, 75 insertions(+), 11 deletions(-) diff --git a/modules/packages/arch/metadata.go b/modules/packages/arch/metadata.go index 629d94b22ba1..ab7148788dde 100644 --- a/modules/packages/arch/metadata.go +++ b/modules/packages/arch/metadata.go @@ -34,7 +34,7 @@ type Metadata struct { BuildDate int64 `json:"build-date"` BaseDomain string `json:"base-domain"` Packager string `json:"packager"` - Distribution string `json:"distribution"` + Distribution []string `json:"distribution"` Provides []string `json:"provides"` License []string `json:"license"` Arch []string `json:"arch"` @@ -43,6 +43,9 @@ type Metadata struct { MakeDepends []string `json:"make-depends"` CheckDepends []string `json:"check-depends"` Backup []string `json:"backup"` + // This list is created to ensure the consistency of pacman database file + // for specific combination of distribution and architecture. + DistroArch []string `json:"distro-arch"` } // Function that recieves arch package archive data and returns it's metadata. @@ -57,7 +60,7 @@ func EjectMetadata(filename, distribution, domain string, pkg []byte) (*Metadata CompressedSize: int64(len(pkg)), MD5: md5sum(pkg), SHA256: sha256sum(pkg), - Distribution: distribution, + Distribution: []string{distribution}, } for _, line := range strings.Split(pkginfo, "\n") { splt := strings.Split(line, " = ") @@ -95,6 +98,7 @@ func EjectMetadata(filename, distribution, domain string, pkg []byte) (*Metadata md.License = append(md.License, splt[1]) case "arch": md.Arch = append(md.Arch, splt[1]) + md.DistroArch = append(md.DistroArch, distribution+"-"+splt[1]) case "depend": md.Depends = append(md.Depends, splt[1]) case "optdepend": @@ -111,7 +115,7 @@ func EjectMetadata(filename, distribution, domain string, pkg []byte) (*Metadata } func getPkginfo(data []byte) (string, error) { - pkgreader := io.LimitReader(bytes.NewReader(data), 250000) + pkgreader := bytes.NewReader(data) zstd := archiver.NewTarZstd() err := zstd.Open(pkgreader, int64(250000)) if err != nil { @@ -258,3 +262,20 @@ func writeToArchive(files map[string][]byte, buf io.Writer) error { } return nil } + +// This function can be used to create a list containing unique values from 2 +// passed arguements. The first is +func UnifiedList(first, second []string) []string { + unique := map[string]struct{}{} + for _, v := range first { + unique[v] = struct{}{} + } + for _, v := range second { + unique[v] = struct{}{} + } + var archs []string + for k := range unique { + archs = append(archs, k) + } + return archs +} diff --git a/routers/api/packages/arch/arch.go b/routers/api/packages/arch/arch.go index b214c35261b6..6d489d84d7ca 100644 --- a/routers/api/packages/arch/arch.go +++ b/routers/api/packages/arch/arch.go @@ -122,6 +122,16 @@ func Push(ctx *context.Context) { return } + // Add existing architectures and distros to current metadata. + err = arch_service.UpdateMetadata(ctx, &arch_service.UpdateMetadataParameters{ + User: user, + Md: md, + }) + if err != nil { + apiError(ctx, http.StatusUnauthorized, err) + return + } + // Automatically connect repository for provided package if name matched. err = arch_service.RepositoryAutoconnect(ctx, owner, md.Name, pkgid) if err != nil { diff --git a/services/packages/arch/db_manager.go b/services/packages/arch/db_manager.go index 361f5e74776e..270519cc0130 100644 --- a/services/packages/arch/db_manager.go +++ b/services/packages/arch/db_manager.go @@ -21,6 +21,39 @@ import ( pkg_service "code.gitea.io/gitea/services/packages" ) +type UpdateMetadataParameters struct { + User *user.User + Md *arch.Metadata +} + +// This function parses incoming metadata, gets existing if present, combines +// architectures and creates new one with base parameters of new meta. +func UpdateMetadata(ctx *context.Context, p *UpdateMetadataParameters) error { + ver, err := pkg_model.GetVersionByNameAndVersion(ctx, p.User.ID, pkg_model.TypeArch, p.Md.Name, p.Md.Version) + if err != nil { + return err + } + + var currmd arch.Metadata + err = json.Unmarshal([]byte(ver.MetadataJSON), &currmd) + if err != nil { + return err + } + + currmd.Arch = arch.UnifiedList(currmd.Arch, p.Md.Arch) + currmd.Distribution = arch.UnifiedList(currmd.Distribution, p.Md.Distribution) + currmd.DistroArch = arch.UnifiedList(currmd.DistroArch, p.Md.DistroArch) + + b, err := json.Marshal(&currmd) + if err != nil { + return err + } + + ver.MetadataJSON = string(b) + + return pkg_model.UpdateVersion(ctx, ver) +} + // Parameters required to save new arch package. type SaveFileParams struct { *org_model.Organization @@ -32,8 +65,8 @@ type SaveFileParams struct { IsLead bool } -// This function create new package, version and package file properties in -// database, and write blob to file storage. If package/version/blob exists it +// This function creates new package, version and package_file properties in +// database, and writes blob to file storage. If package/version/blob exists it // will overwrite existing data. Package id and error will be returned. func SaveFile(ctx *context.Context, p *SaveFileParams) (int64, error) { buf, err := packages.CreateHashedBufferFromReader(bytes.NewReader(p.Data)) @@ -143,7 +176,7 @@ func CreatePacmanDb(ctx *context.Context, owner, architecture, distro string) ([ if err != nil { return nil, err } - if checkArchitecture(architecture, md.Arch) && md.Distribution == distro { + if checkPackageCompatability(distro, architecture, md.DistroArch) { mds = append(mds, &md) break } @@ -163,13 +196,13 @@ func RemovePackage(ctx *context.Context, u *user.User, name, version string) err return pkg_service.RemovePackageVersion(u, ver) } -// Check wether package architecture is relevant for requestsed database. -func checkArchitecture(architecture string, list []string) bool { - for _, v := range list { - if v == "any" { +// This function will check, wether package should be added to resulting database. +func checkPackageCompatability(distro, arch string, distroarchs []string) bool { + for _, distroarch := range distroarchs { + if distroarch == distro+"-any" { return true } - if v == architecture { + if distroarch == distro+"-"+arch { return true } } From 38024b50deca239e1e05672f8a1962b0cc1b6ba3 Mon Sep 17 00:00:00 2001 From: dancheg97 Date: Sun, 9 Jul 2023 13:28:05 +0300 Subject: [PATCH 029/124] time mocked for arch package unit tests --- go.mod | 1 + go.sum | 2 ++ tests/integration/api_packages_arch_test.go | 9 ++++++++- 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 885bb3422046..d6840529a106 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module code.gitea.io/gitea go 1.20 require ( + bou.ke/monkey v1.0.2 code.gitea.io/actions-proto-go v0.3.0 code.gitea.io/gitea-vet v0.2.2 code.gitea.io/sdk/gitea v0.15.1 diff --git a/go.sum b/go.sum index 9b4538bc65bc..ca9901118f16 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +bou.ke/monkey v1.0.2 h1:kWcnsrCNUatbxncxR/ThdYqbytgOIArtYWqcQLQzKLI= +bou.ke/monkey v1.0.2/go.mod h1:OqickVX3tNx6t33n1xvtTtu85YN5s6cKwVug+oHMaIA= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= diff --git a/tests/integration/api_packages_arch_test.go b/tests/integration/api_packages_arch_test.go index a63fa032a4c3..3d8c27ad353c 100644 --- a/tests/integration/api_packages_arch_test.go +++ b/tests/integration/api_packages_arch_test.go @@ -12,7 +12,9 @@ import ( "net/http" "path" "testing" + "time" + "bou.ke/monkey" auth_model "code.gitea.io/gitea/models/auth" "code.gitea.io/gitea/models/unittest" user_model "code.gitea.io/gitea/models/user" @@ -274,9 +276,14 @@ ht719b7ZWR3+SRcXySXC/cP8DL/N12kaf8wQSBkjjLKkAPBDnLyL32YFQur67qtbXtxcd/23w375 rootURL := fmt.Sprintf("/api/packages/%s/arch", user.Name) - // Set test time to 2023-07-04T19:57:09+03:00 before test execution. // Add package gpg key to user in tests. + wayback, err := time.Parse(time.RFC3339, "2023-07-04T19:57:09+03:00") + assert.NoError(t, err) + + patch := monkey.Patch(time.Now, func() time.Time { return wayback }) + defer patch.Unpatch() + t.Run("push", func(t *testing.T) { defer tests.PrintCurrentTest(t)() From da5c7c252ba69c651261b5778b0579831326ff38 Mon Sep 17 00:00:00 2001 From: dancheg97 Date: Sun, 9 Jul 2023 14:47:41 +0300 Subject: [PATCH 030/124] created user with gpg token in arch package tests --- tests/integration/api_packages_arch_test.go | 33 ++++++++++++--------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/tests/integration/api_packages_arch_test.go b/tests/integration/api_packages_arch_test.go index 3d8c27ad353c..2d1c77ec2ef3 100644 --- a/tests/integration/api_packages_arch_test.go +++ b/tests/integration/api_packages_arch_test.go @@ -247,15 +247,22 @@ xdsqIuddY4wgEtLbujbklbaSN0aTJE6iqrhVnjQXUWvIukbYynhuBABuizXCfp6H6UUYttf9XTDd ht719b7ZWR3+SRcXySXC/cP8DL/N12kaf8wQSBkjjLKkAPBDnLyL32YFQur67qtbXtxcd/23w375 8vepr3/5vAL41/9/5syZ58uPAAAA///ViF8vAAgAAA==` - defer tests.PrepareTestEnv(t)() - - user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ - Name: "dancheg97", - ID: 2, - }) + assert.NoError(t, unittest.PrepareTestDatabase()) + + user := &user_model.User{ + Name: "dancheg97", + Email: "dancheg97@fmnx.su", + Passwd: "password", + IsAdmin: false, + Theme: setting.UI.DefaultTheme, + MustChangePassword: false, + } + + err := user_model.CreateUser(user) + assert.NoError(t, err) session := loginUser(t, "dancheg97") - token := getTokenForLoggedInUser(t, session, auth_model.AccessTokenScopeWriteRepository) + token := getTokenForLoggedInUser(t, session, auth_model.AccessTokenScopeAll) req := NewRequestWithJSON(t, "POST", "/api/v1/user/gpg_keys?token="+token, api.CreateGPGKeyOption{ ArmoredKey: gpgkey, @@ -276,15 +283,13 @@ ht719b7ZWR3+SRcXySXC/cP8DL/N12kaf8wQSBkjjLKkAPBDnLyL32YFQur67qtbXtxcd/23w375 rootURL := fmt.Sprintf("/api/packages/%s/arch", user.Name) - // Add package gpg key to user in tests. - wayback, err := time.Parse(time.RFC3339, "2023-07-04T19:57:09+03:00") assert.NoError(t, err) patch := monkey.Patch(time.Now, func() time.Time { return wayback }) defer patch.Unpatch() - t.Run("push", func(t *testing.T) { + t.Run("Push", func(t *testing.T) { defer tests.PrintCurrentTest(t)() req := NewRequest(t, "PUT", path.Join(rootURL, "/push")) @@ -301,9 +306,9 @@ ht719b7ZWR3+SRcXySXC/cP8DL/N12kaf8wQSBkjjLKkAPBDnLyL32YFQur67qtbXtxcd/23w375 MakeRequest(t, req, http.StatusOK) }) - t.Run("get", func(t *testing.T) { + t.Run("Get", func(t *testing.T) { defer tests.PrintCurrentTest(t)() - t.Run("package", func(t *testing.T) { + t.Run("Package", func(t *testing.T) { defer tests.PrintCurrentTest(t)() req := NewRequest(t, "GET", path.Join(rootURL, "/archlinux/x86_64/randpkg-1-1-x86_64.pkg.tar.zst")) @@ -315,7 +320,7 @@ ht719b7ZWR3+SRcXySXC/cP8DL/N12kaf8wQSBkjjLKkAPBDnLyL32YFQur67qtbXtxcd/23w375 assert.Equal(t, resp.Body.Bytes(), pkgData) }) - t.Run("database", func(t *testing.T) { + t.Run("Database", func(t *testing.T) { defer tests.PrintCurrentTest(t)() dbname := fmt.Sprintf("%s.%s.db", user.Name, setting.Domain) @@ -330,7 +335,7 @@ ht719b7ZWR3+SRcXySXC/cP8DL/N12kaf8wQSBkjjLKkAPBDnLyL32YFQur67qtbXtxcd/23w375 }) }) - t.Run("remove", func(t *testing.T) { + t.Run("Remove", func(t *testing.T) { defer tests.PrintCurrentTest(t)() req := NewRequest(t, "PUT", path.Join(rootURL, "/push")) From 1df0a1d252f424c33e74455eee48222d7ef65b93 Mon Sep 17 00:00:00 2001 From: dancheg97 Date: Sun, 9 Jul 2023 18:26:01 +0300 Subject: [PATCH 031/124] fixed file size for mock and scope of time patch --- modules/setting/packages.go | 1 + routers/api/packages/arch/arch.go | 1 + services/packages/packages.go | 2 ++ tests/integration/api_packages_arch_test.go | 15 +++++++++++---- 4 files changed, 15 insertions(+), 4 deletions(-) diff --git a/modules/setting/packages.go b/modules/setting/packages.go index dc8d98d29fbc..0eb5a586c6fc 100644 --- a/modules/setting/packages.go +++ b/modules/setting/packages.go @@ -24,6 +24,7 @@ var ( LimitTotalOwnerCount int64 LimitTotalOwnerSize int64 LimitSizeAlpine int64 + LimitSizeArch int64 LimitSizeCargo int64 LimitSizeChef int64 LimitSizeComposer int64 diff --git a/routers/api/packages/arch/arch.go b/routers/api/packages/arch/arch.go index 6d489d84d7ca..f52fcd27aaa7 100644 --- a/routers/api/packages/arch/arch.go +++ b/routers/api/packages/arch/arch.go @@ -59,6 +59,7 @@ func Push(ctx *context.Context) { return } + // Check if message is outdated. if time.Since(t) > time.Hour { apiError(ctx, http.StatusUnauthorized, "outdated message") return diff --git a/services/packages/packages.go b/services/packages/packages.go index bdc56efeefa8..b7dd75d33d6c 100644 --- a/services/packages/packages.go +++ b/services/packages/packages.go @@ -355,6 +355,8 @@ func CheckSizeQuotaExceeded(ctx context.Context, doer, owner *user_model.User, p switch packageType { case packages_model.TypeAlpine: typeSpecificSize = setting.Packages.LimitSizeAlpine + case packages_model.TypeArch: + typeSpecificSize = setting.Packages.LimitSizeArch case packages_model.TypeCargo: typeSpecificSize = setting.Packages.LimitSizeCargo case packages_model.TypeChef: diff --git a/tests/integration/api_packages_arch_test.go b/tests/integration/api_packages_arch_test.go index 2d1c77ec2ef3..bba6896b1c42 100644 --- a/tests/integration/api_packages_arch_test.go +++ b/tests/integration/api_packages_arch_test.go @@ -249,6 +249,8 @@ ht719b7ZWR3+SRcXySXC/cP8DL/N12kaf8wQSBkjjLKkAPBDnLyL32YFQur67qtbXtxcd/23w375 assert.NoError(t, unittest.PrepareTestDatabase()) + setting.Packages.LimitSizeArch = 99999999999999 + user := &user_model.User{ Name: "dancheg97", Email: "dancheg97@fmnx.su", @@ -267,7 +269,7 @@ ht719b7ZWR3+SRcXySXC/cP8DL/N12kaf8wQSBkjjLKkAPBDnLyL32YFQur67qtbXtxcd/23w375 req := NewRequestWithJSON(t, "POST", "/api/v1/user/gpg_keys?token="+token, api.CreateGPGKeyOption{ ArmoredKey: gpgkey, }) - MakeRequest(t, req, http.StatusOK) + MakeRequest(t, req, http.StatusCreated) pkgData, err := base64.StdEncoding.DecodeString(pkg) assert.NoError(t, err) @@ -286,12 +288,11 @@ ht719b7ZWR3+SRcXySXC/cP8DL/N12kaf8wQSBkjjLKkAPBDnLyL32YFQur67qtbXtxcd/23w375 wayback, err := time.Parse(time.RFC3339, "2023-07-04T19:57:09+03:00") assert.NoError(t, err) - patch := monkey.Patch(time.Now, func() time.Time { return wayback }) - defer patch.Unpatch() - t.Run("Push", func(t *testing.T) { defer tests.PrintCurrentTest(t)() + patch := monkey.Patch(time.Now, func() time.Time { return wayback }) + req := NewRequest(t, "PUT", path.Join(rootURL, "/push")) req.Header.Set("filename", "randpkg-1-1-x86_64.pkg.tar.zst") @@ -304,6 +305,8 @@ ht719b7ZWR3+SRcXySXC/cP8DL/N12kaf8wQSBkjjLKkAPBDnLyL32YFQur67qtbXtxcd/23w375 req.Body = io.NopCloser(bytes.NewReader(pkgData)) MakeRequest(t, req, http.StatusOK) + + patch.Unpatch() }) t.Run("Get", func(t *testing.T) { @@ -338,6 +341,8 @@ ht719b7ZWR3+SRcXySXC/cP8DL/N12kaf8wQSBkjjLKkAPBDnLyL32YFQur67qtbXtxcd/23w375 t.Run("Remove", func(t *testing.T) { defer tests.PrintCurrentTest(t)() + patch := monkey.Patch(time.Now, func() time.Time { return wayback }) + req := NewRequest(t, "PUT", path.Join(rootURL, "/push")) req.Header.Set("username", "dancheg97") @@ -349,5 +354,7 @@ ht719b7ZWR3+SRcXySXC/cP8DL/N12kaf8wQSBkjjLKkAPBDnLyL32YFQur67qtbXtxcd/23w375 req.Body = io.NopCloser(bytes.NewReader(mdSigData)) MakeRequest(t, req, http.StatusOK) + + patch.Unpatch() }) } From f912dd6431a86c071cdc0d83770bb44991e6c751 Mon Sep 17 00:00:00 2001 From: dancheg97 Date: Mon, 10 Jul 2023 20:23:50 +0300 Subject: [PATCH 032/124] corrected tests, added test cases for databases with different architectures for one package --- docs/content/doc/usage/packages/arch.en-us.md | 8 +- tests/integration/api_packages_arch_test.go | 669 ++++++++++++------ 2 files changed, 451 insertions(+), 226 deletions(-) diff --git a/docs/content/doc/usage/packages/arch.en-us.md b/docs/content/doc/usage/packages/arch.en-us.md index d3e70ec439e1..4d7343f7dcde 100644 --- a/docs/content/doc/usage/packages/arch.en-us.md +++ b/docs/content/doc/usage/packages/arch.en-us.md @@ -41,7 +41,7 @@ pacman -Sy package ## GPG Verification -Upload and remove operation are validated with [GnuPG](https://gnupg.org/). First, you need to export and upload your public gpg key to `SSH/GPG Keys` in account settings. This works similarly with SSH key. You can export gpg key with command: +Upload and remove operation are validated with [GnuPG](https://gnupg.org/). First, you need to export and upload your public gpg key to `SSH/GPG Keys` in account settings. This works similarly to SSH keys. You can export gpg key with command: ```sh gpg --armor --export @@ -166,3 +166,9 @@ curl -X DELETE \ --header 'Content-Type: application/octet-stream' \ --data-binary @md.sig ``` + +## Clients + +You can generate client code with tools like [thunder client](https://marketplace.visualstudio.com/items?itemName=rangav.vscode-thunder-client), [postman](https://blog.postman.com/curl-and-postman-work-wonderfully-together/) or other code generators to write your own client. + +Also you can take a look at [pack](https://fmnx.su/core/pack) which provides `pacman` functionality with additional commands to build, sign and push your arch packages to gitea. diff --git a/tests/integration/api_packages_arch_test.go b/tests/integration/api_packages_arch_test.go index bba6896b1c42..c031c3c83ce1 100644 --- a/tests/integration/api_packages_arch_test.go +++ b/tests/integration/api_packages_arch_test.go @@ -25,178 +25,6 @@ import ( ) func TestPackageArch(t *testing.T) { - const pkg = `KLUv/QRYPQ8BWoM5Q0aQriRpAwIANx0x6IHGcK3rM6A/BEBTh6DDFs4wGLaRRiRyf3f3Jj8RQily -i6KqPmMmDr1trRFCJHLRBHKMnQsHagvJ4UABCgTsA+QDD6VE+myp6RlHFEUtjl6KsWjtuJ1NkyjW -udbrkGuys5M7mlrdpJ/eTp/coaikr0vXcbvsYtP6OZJVclavK0o7VcnUPZD6olZL6Y6kr5O+cdbX -mbnW9403U6rdoOgNRTHcNI/khy8t09qty1AUF8wOfYi7j69LtMJ2+aKqlMay1owbXGNrx9XOClmz -YH0dejQUa4kfvSil5ZwaqvXBGh+Nwbc+kLqGjjFlKmmabypZqvV1a910pqklv2lbM4Q9KkrbKLZD -4bFlGbnG41JL54+eKH1r/uhLUSs2xtTq99KJXmwqUUsnannJi5FJLjLlaHkj2LhupT8N2cY2gklr -G2XH2UYuUw/V+r2EoLvw4bfChtN3mdaWb1+Xb6eyVKJW0zpnhQ1lR1Gs462yMk0lj26opRbWDnUS -LtVo/sHUkjQWtZTCWM/oZ71Po5taSmmnNWProEqWlH5dpopSmGI9Z73voEM1mGJd7z9U0k7Kv6hz -WY/V2mqNPTv4zefrsB18MMUa+3UNHc3+hUOttu5CXL4XPZUoprFTr23c3NxYP9RSrF/mehuKuRP/ -LuqhVmdZ8XvpYNTSFx7JphwqaVrxM70U1RrLzxH9x7T0C6UZthRTSz/YvZYueUzasXvPvepewzWH -EhunorVb4fjHSYtiIDdrjppqSyVtNQvWLinNGqrVHDGVdP4Y9c3Z0VTrOaGcKIeiWvutsNtb3Hnp -thrrwVbrRqtzHfo34lBPEX3JGl0xoxbVHnhb42z/Qtz9wMuSnpVRS0N5xrYo7OGulsZEHedjwqgd -lktDK+yIU2W1NbfDsinW72VrqEQtC9bY5Tf12o7XwXZYMtXU6ozp0wp/KUVpfCu+EWyX3zmU0klr -plj79Cc3lJ2GxAh1zbqRmLYUq/jGLMUqoqYs1RpnWpuu17bHCtvhjSpufzUmzVSiWsuHSphKWjt8 -6TrGmUYVwmadqKWxlbTV2lrY8h0/xzDBiC6ly5i+ETVLJJzoeUTCnRtZN+1f3I+RbKe/l/Czy1jW -mt9LJDCs0f7NmGGCgWn9bKsR1HmMBAT9gf5AByESt8oKIcz4IxIMEwyJelNLMQ65yTDBkLXmfF3G -SSThsz5JMEww7jPlkTwSh4NhDBMMzy4a4YEIq4nOBgSNh+MPpC52gPFAEfqzloDh5FOnhOMg6KwR -LTqlYc0xTGPwwRwN5hyuPH/+gIxb2mOt2FAai8OhrMdzNJLm3qyEJQ2zHIwOhgnGjDCzNPfgE1F+ -pDAnGMfEMMF44HUOZI1gzRgmGNcl7J4DWSMcyEbqLsMEw6ejDmSNaAwTjOzmrQ0j13rXKuK6jKPF -KKLRSBpbjKUlyrKsRVEYg1GUpbGKYCgKs9YqggUlWMESSqRgCRmIQAXOIc57yTycRM57yWzNt97r -7VjnvXTVeS/jg95LKknnvYzcmh9X6/ed99JZ7ryX72Vz3svVU7bO/TaKNaSeaW3eS+Wb5r1sDZ1r -KDbvZVPM5I1l6qwlj5K/l5GpWndIfy9PrGSx1lZ7L5WPXHbRmWI6Or9RiowpRbWeN76XSz8wO4s/ -Jsz/ZqbWe4lt5P5ozSVOheNW4OvxpXptjKqE7+VSraKpxdp9pe46m7SuH+H71Q711ZgFnYp2K5a7 -1x+1r1a767M5+F629ovHniOPNb1RI76G33tp33uZ3Ei+95JHCFb53kvHQ61ur3lFLWq18kgYlB5M -rc7XTexy45YIHYxTkWAKaYS6CLv3siXvJav/XiaMBVOsMY2yESWeUPp0clEPdhO3XN6Fko51QhGH -HtRYWwtpxSgUoxitmKnVsGnFKBpTbbFui2G/KB6RYmqp1Rpjl9H54xeFkgLFYsEaVinddBSRerCG -IgrqtjZjUFw6U5+03peot5MciusWOxRMtaYVY+pQtI6QZsE6/8KU0rjiZ8w6FHsu22e6eY7qtc9q -zTLTV54+c/PbsmD2dtS9Q7Wm/tjIk6gFeWj0PPB1tm3Oj635iTjI4/zD1Oq3Lv7TYGp9BL/RFxfR -a657mGJt/2JRrHEjrei8pTayMkra5UdLL0ppBeIvQbmTORUoJ0ppKrFgnUB5MaVYsPbsoo0aykmi -WMMaG8rXp/PruKnoXmO/D3H7tuk7kbSdsWJtI8ZMWZxEGlXcJnbTH4gd5qtxjcXhKIZZrmLWi25T -r5O09ctUopgFa8+/qD/W3SZbxrHYHO4oR3JVQy27mmOT5rpNurluP9PoQre43ky/M3QUr65WscyR -oppq7diIOpBpxEdsToXCboXL/prSGHIOydLRn9fz0zqXdvnZURwWrCFYY9fcD3bxute25s7Q2Vkh -9Tx5k9IfNt3Fxg79nB1jx+30Ja3YqbIpFswh/lJnWptIfgi/YsGuZ+qwZb2ZVgdjC3Mo6+FUNOVW -LMUkjqNu9q2yQrBKz9Bd98DrInXFyQhKB5vSSyX95FqzR/Kp1dfVHLn2WemjUdZyi3I4VlOrqZ1O -mz8fHS0Hw7yXTiNRrSGGNSbd3kunoZoFdCskO+33kmkoZtEao3BwIzu/olrnd97L5h2m2OQ5u5Mk -rbrDIgmrN6UoiQXryHXuzOgw1RTVujXRm942FJ2mWM8XHaaUVvpChykGncW68ViO5Wgsp4KxCkY1 -lXRDK4QOawidphwNnaaWhiDOSRxHzUM9cNQwlTiOGhb//W0Splo3t5HsfhuVdI6k434bhalm0dr5 -hlaYaW0WplZjSNZGmVw2CXSyCU+OtIb4J7atm+jVXK9NHNrYwnrtg9lqWeeHZLUWsopaTd2cUNSP -UaZO3yqqTU1GN9dXb0MgEKaW/hutjtDPHbk5SVTz+bkpprNT1uS7aSPZffTZFiIZxTQEAnkvlbUU -q8+afa6OU0VRTLvjMPa5LMUo5qFaYducdHbsuOxQiZz5Ydqhn/HLR1sNptZivSwHq6Te8rqiVjut -Eral5XXr+SXE0CpqsW5+oVWUo/1C+9AyqqFa0tHzZQtH962zylqqQ52dzqrFOm699yd1EzoLQWeT -W22VNlGPfjCmbutqjts7OXVbmcauRuq2TkfdViYddVsfimk3oxs5T2uE3xLWR6JWT4lm6GZEaV+L -6ZMtfTDl6Gnp41HU6bnSRmdbKQqPlapH24jTHitlKOjTnJ22zDDStpxQ8I0g7UodbTma/WRnP8tI -UUspFmzmmTTm2GnVUk0pfRvLlNrQb+pedPW+nuqL4OglXd/C/PShmTCLpz8Z9tkzujgdUImzqRXJ -EIxix+ukdFMpTJkXUrmQDqZYN2lpF8kXZ8MiSxzFlKuWYlRiwajjEzOZkpRRUr9QjPrFA7kOMu83 -19+6niLalNLUakm+bAxTh/HBNM9dUtFa0m5LUaRxmiPTRabEOJoJTkmaMs4LS+rNfPD0Xkbqd0pu -pO5O0ZyWnJNa+vpLTkLTg6a7Tp/91Ewt09pM7xXT2Kdvpjz5BGd03HRxkuA0LYlSlDL9xBTrB5la -E8PYpFiIopa0fh9in0OYUjpTfwkf5L2cFTrImqz0XuJ/t0r5usQ6cjZK16W/sR322U5q7aT3UlGs -pSjb6TlqcYrmw9F7+XAUvZcRvNFDaMQyfho1FLNgDt3NrnFAZ06FUtFQimr3s9JN89nd6u3dk99L -x3SeH0wpzSh22W+UrYPd1GtxBLc5PWOmWLcHY6ZWPxgnd6otsVHF76VaKlYcKX4v30ultL0v8Xu5 -VNIVFdB5/F4uv9CxnRVCxwvGDjOHIY3g+14yi4Tvw+97mTSIu+8Vc1EKiGWKYe1YRva9tJGp3pYu -36ooxr6XC7+076XjzYg6rY99LxubjmWNnVuH0D2076WD9qFmzL6Xq7Od1G1VD8dJ6a/lT7M6Tl0S -00jzuTpFsZ70vVRJy+bw6Xu55Pw4tOWGLo0biL7MThv6XrZFrc4QBt3NlFZ44nqW32Sn7yV9LxUF -R/Jz5KhrKo5MO9/LRyRtrJPO99Khk7WemrniVDk7dbNlKN1UUb6XLIEycQ9DckKxvBvldtyiqPgr -/veSSfw5svSXfp7+e/lgLv/V+e/lShwOxkD/9zJpND7893KBv3wkUve8l82fHEYshgmGXzxOIgkL -bHN+GCYY8kZG0PquBMMEA2KPLFGMwXDo44hRgchjRDcBowJNYY2lrGUCFLRAAWDWMliAgAVg8AN0 -gyKo4QUXuIAJInwhB2ZQf6AAD5QIhBU6oIcdbCADDAiSI9DQhScAgggiSCIJRGBBCRgoQQhd+BGE -4Ygj0IFJAiMGVCCCEQxRBRD8YA6HQMQeklCDA6AQgYsoVCihAQLsYQpIEMERUojDQ/tAgQpIgFGA -g4+YgxTYcHEFGdhBBVwwxAaOiLgg7GEBMOCAF86wRAxqEIACdGELjBAuMIANJBQsujkCCEAYgTNE -GxLQgyGkcDTBC0UwbrAwgyCSqBDAAR4cP/hAFhJxhz5g4AEz8AHOwwoaQIQlviCEJchBD3UYgxiy -QAc0EEFywwkdSD4M4gKFGIQF0hAHLdDgogcIyAEaGECANyzBBcYEHuCCETBBiVmEFKzBCUqgiSDD -i0Ii9FCABIRgACfQAhx6IIADGGECC2uwZClrkYAEL4iBCtQSM75Y0GLSJZfrcV4+zj89uUwjuB98 -0RZ2x4nwgy8urlN3erp7cWH/Yv6FonSKWa9fSDw1Kz3SmoskF5OypqnFlybn1qnha+u1K9Zpn/40 -HzRlxWKF1Ll1SlTH6+qLOFRjrWw57FKKRlsacDnK8WhctWa9T8OlqNKmwTyl8ZyKpD0qGuyhHg/P -9JPVksfXROaMYrNg/TlpHVknTtfV40vtK2vHUZ1TwvCzdTSCmWYczcS05c7hVCytIlEsOx+d6MRp -QhQ8Izji0FoPTg9OSS9Mm5bi7p5aHiu5SmGYWhZJK0ybmo7UIeieW4jDAr60NsInnJUYN7JCmqZC -/Fq1khMYu/T7kAf+A3EHsrDDei3M9zlkWSg7impQdlp+6Dlk0erR3aWpJLoqO8JfGnWQjBLPUmuo -lSWGJU019WhALDl0wmcka+OmxHAjnTtfYth86bmMVXwpiVKTipSo1vJztPrhaD0g1UbM/gWNFPWA -8McIJm21i7yYw03O19/LbnJusMUaW63j1xu7yRnnlnRynWGcu03r184KI/RzY/K8smLBWlrP013H -pecXc6JYX5eTlrOEIzhZWkcmuc6SVQmrMcKNqdXw20b5upzvrNld3Lh5sItSKCpJgWWFi8VxcrfL -/TYMq/gRsdZRPDGts+H2mqY5pyJhFU35/NnRr3uw0pAa3YMTtTpex9el6dEx++ccn2gy4hYxhhg7 -HF38ldL7YPfctVRTjuj8rmQ5k9yN1MmIgzSTUziJqbvZkTDD7rj5TfrsrpPQYzM5xcI0k6/jSLLu -o48+yo50vD/yjxjHdO1Dvx/9ZlyffjiCQDitR4KdIznSQ+tE1un6+b40peiIkxZ/bMgjPRg31Jlx -tcO3MmKLK45edHQ0PQXD3kt5I89O2un5b4RPOPRtJC9Mivz4oZR2+Doa9iiF34vs0MfX2YiNWAkd -jdiF+XQaazDiUE2tWKeDUawWlTTLOiIZ9NNdZxPVnhy52jhsI+i6mVBIjdB1E3LzIWn81h74rVtP -jp4cORyqNZdOrY6ikhU11DN2DVM6UrON3OzxuuebBi61LJU0taxHa4aoqJNFmlyip0nnmEJDQpMU -JB1DEUAgGHg4Go8LxpNq+xOAQBBHoTgQCAOBQCAQCAqKI4Gkh0mIzMzIyEiSpNAcs1q6QNNKdO0V -6C4jRKBDbQd0z9CAjuuUAzrlsc8dnGYEunQypynOcG6GDNCxvAaoq/gM7hh0gdR1QKA7pw0QaDok -5OibuXZ4XrD68t6NgS4GnQHfe5kIXzhcijLa5Hf/VV0lQni2biDXHr3eGEBmHiXKJXc5zWMjIO4k -fFUvLrLnRprRrNYzMC0SC4wlLtl6mJOKeexFFMks5ez8sRoVjX2Z9803JRqOj8NAjDiIGzZtJEFa -zLpEHpszYuSdT7VappDg/O0phV0K4LmXvEA/8LhBFZAQmNjBYOKl/iUVhsbEH9CyFimWn5m0+nSK -cLRVm0eQrioSNNHjmtCEUNmYT3KK/vlmwC9xZGwlpGX5CB3JCtKFJhUMRYs0QQXHafEqwdQ4ibYh -O0gs39VmjojW2JMWUPsuIwASABVNhWVUBmi4ojD8xY7qbMAr0aqNZeG+inR3fZHwn05MIdn1MrAQ -TyGfxKY3eciClcScRFwSGvE40U/PNA2nnELw2Iwjz2djO6C/lqF8WoEmYHnwxaAK960JyEu4aNpr -UPq7j8JjSg/hHmf1ySBbcqXxfr1odYRpe9BmblpASW+BNWT9ml3tuCMl94+b7e3tt1QNjY3y+yEF -HBJrji0lHOWJ8sgZ02HqEJL8m/pbzqL/siUayjR8D1TFoM0VsHSkcjD1MzDtsf87kaNZ0HuKA5m9 -StwYlxZgnHS7kIl/nustI9sFt4TlT+bBP86sf/Ly23bboAeddYAxDjd9L2mAxHJ9x7JM8olacOzj -rKb2GPLTW4aGhq1nSz7OLqZeZWhAGmdkRZPuNhj7W5OgJcU49ahcnRIWJntjACpGBId4MbJjjtrX -pJTTYcjJB5oRSyHRZ2lkbhbrV5jUS6JRnOchofXpJ7sHH9f8yyjdUXWpc6gkzZAtKWMva6QSe7Yf -01Zoq3FstZ0M1MymPulFVK2I+ORuLD8jpez5usPDWixyYmockuX8pRqFud5TWRBhEjfItmPGeb1J -s0jrf3C8+TMzwyodrsAC+URiNaOEnMMz4tRLgraoS8Cd86cdJcVAshPx7AqqiE5lEuc6c2thpSJV -898cAgjtjbTNAmPQ0ZMgTdseXkDWRZH663ZE1CcYP+onRHf5/5zpcVk8UJiyZ3agT6eTMZP0hZP1 -DJBL/NGJVvwQJIZpoJFwFlDMw5JMexEGcERxGFyQMozuE7DEsflKfEgHxq1OkulcJIe2ROj6CdeV -eOFDWB8qWfnqhyKyV3IyJj5FT4I1w7acqmyJn5yNrmIkcFQJlGgpY3lA/LKmjjUIulvQcIjtp487 -aRqDDbo3K4J257r1xnySQvnbV3tm7TLyDOr373xVmBdDBmYkQdEqpIq0JUfuFkoMNYAbyhryobMs -vEYg/WRAb3UjAiOSohRrKYcbDgMd6+6yhGbTLHpcuLfWctciDam8I+VwkJ7viftFSxcLYt+8gTKh -yEBXhIugsqChV0vZ4BlhM6Lmu7/SYP6tLhoycfXhESvb0Enn4qCPS5W1GDN3kaQLGsJDXXtoF0cM -28HAwBItnwPMR1avGEOAgBaMv5u1uDrgHpIlRkmlJ4Vs+m9W+kOs/7Idkjvjf4cl08kiEkq8JxBp -tFd62eDlTy8zzuoQ5q1gGKsLmSMDWI2AvCDP2UFGsh19Io1FlS8OVsWy/q98sF+Pa+Z1SWOOLmTW -cF5NhRRXq3jWG+zkVVOGZnxToexPwewJ6KrDMrhy+/iC9YhVL7dvqMvfOoVBi2X5hFYdwPnDQHqL -CSbCEyUm14SHTUH5bCAtewKhlbDogflLEquYqVcp7gShHZmhV2m1Qi/A+jQHGWI3uNS7+OhmlSpL -qjdaqtisRULhlOPj4dzpYXh4IUdGuHL+JBnygS6+2DtK6NqYRWaru4ek6e23BYjMWAWhZptLaufU -QLvSie7c86f1j8aQOJu2in2TARwQcsvsjvXBDJAHjzEr1ZBFKoafkVy4a7MAWVw4zamGWMnF95r3 -mQrvjduOmZ+bQDgxZg6cv2scLEZAyaqTihukzHJGetU+j4bf8MS5GvJgCAxVHEwkFT5wuchsOGgt -Vkg+p8I9Jv4Q9I8DvW4bDAgmXK/o89kNHnegFJJLG/IkkpHyfJ+uVFqBUf6n/S0LWXNyYz879L8S -tYzLWpWX3pBnQYkaB1Lz4qwdVx6HddS6/1Q4vrNOVUIEFGYcbhYKt3Hkcr3713sE7CIrk3741lta -fOhW2kVOSPWDCGgbFDFYIkHHDuzETAnMuzrOK38nvTSHg3XzcRNGizl9LrNQ5HS73xsV2HNmsPME -kGqquX1upDtkwqORcUgkCZcXjGOByVg/KDF/7ACiSdEWWbQcAWdnwDwbZEQrNMYKHG6idvfZdKkf -SFGkLkyrPw4qHsx3PGQ4r8Qg86VfWq8jELChbU0+WZ802EZ2s63U8W2vgCC5D1oDnrtIdyptGVmg -a7ScPlmYFTHLMyRcUDwmkhrCwsnpjvEo92PtZ2706XUJU61AkBh6guNHkuU4KiPSYq/ldiya9dBf -4iNCPKovP+duoIebNcwihVY/ajVxPRzQlqWyB2UC6lQpiYs6TqL+pGriKL8SANQAddkqw1uZRZca -9YtRDLqIQIgIGBMTQJm9YlRk/JppEnJnXg0iqk3YayCEaHX0Q4vR6ynZdF2RwJshSyAQ9hG8hnc4 -nKsG9+fr63OdwEH51tF5Jm2AzPwKesEmh2izqcC+GLwWFFlvbwioavZyUrY9Lv2aB5dvlS+AAazY -fFpefnpZD9JKNNH3gisS+Gy481c+dkih03vcHzvajk6KHFhyDtzoMZBUdui6L2necoBR/ztieQmC -IKMh9i9ZaTY5y4x0VIqFcuAmcuWvzYjXakmxR8Lv+HZ0mjnQQadtXGj1WFFgAdn2whaGe0LgivQS -GLBVasJoFm0037ylI93QcB5hdWeil6AyuIuFrFl7Ga3V1sPT2Mxb9vTK+kA75YU7nbMXCddl6V4f -GLvzjQzFbAVFLhO8m3KvNtnj4+UtN1AgJYaq3uJyKRshSD2OKHZTzrACCHATvb4fmAuMMAYc5/kK -J7W03jgWM4aYEI84KX0YfU2DyZ7rhmhjzIka6XzVK5vQTNQeNvSKuc6/k6dWyq9j6QidvqMRdhe0 -6e8lf3r409UR4To2FiBVAs35Dq96xZHxcejDZWoz6gXw7B7W6mW1mOaNGIujR+OWKOFnRjQXEHhf -ZyIse/zizc3MhUYx9uMvvfNn2lFsGOPcal7lKyMFg6EXgup63tuZDFVN1+o+0fqniDGRUOpPrTox -AW+COhx01z6oqC+ezJqGBVaQVxcP7Z4NH2MRMps1qONWRA90aZFDAQAuigUiWrVdVVtihZvmT9cy -m7kqbjmn+g3s20T0GstQnQiSPQfyBapoy5OKVNCg8FpLXx28R6WPgNb84E6Z1xfWz1rjRIT1hSTV -VE3d3Ip6P9QUT/ZEbD4gdN15MXwtavtDstNyC3KxM97gvv11aYk7Vfv+gGRraqXaduvSvvZISBvf -Z6Y+JAhq0kJyDg/jyF8kdZu5ekg6e6sQpegqfZyFHPFzn5b7I82IWM4pJDSOaQoTWwBHrVcgHCLj -nFsbpsv/IyqO7iFDq5PSi6O/+jtDjik5Q+jzBQwJWzDmf4n7OWChlriDG3vMV77g/do+zQSrFcJs -v4PdZT2yj0fGP9uRg5rP3QMXCPmBm8CTTgARPFRsCGQm96JpDoXsE4lC7d3JALoIwOciH8OrrHvH -oFahjl4xTaUcQGlagt2n/rSdlwPWvWT1EZ6qvR50hI60Ces5ASeUK95/k9u5MpSAWHc5vR3MH6aY -jDo2YK9YRZTWOPa1kq0/hFrZz0oh7DqppUSNe816GWXKolShkYsG15isKZbAUG7kO4pMbKkEwUk4 -OWH3kOaY+ILgsPhNCZQ6zl+DXOetnQEohuij9hYK2uZSe27M6mQ/bp9YZRIG9Jn0TyBGclNHil4/ -mAV0SN9JiXC1rVwuFs7hW4S3TKc8WMT+GPwyipjTXJ1ZCs+0k5k9AQX9NkhgMyqRyuhlTvZKQjRA -/U7ixMjDgcLfpZbNR6tRZAnIAo2ajV8JQy1hyilwoLWOSxdItDooWbIO4QcbuxNJPgxScxl/YWmN -onjDyr8ZkJApZe1ccv0evuAcPSEDtL8PjQZbTsBN7bgvIzgdmxEsS8cNdjzS+5pnLYtDj05SBPIV -KrE0HrQUZaawJ4phPKlwvqz9APiRTCEDumLkChL7XM+YImUvfunEUxhFIZar4gEEH9bgDF0DGnOV -NwmrzcDs94UO3cSuSAnrzkeTPhX8e3OcdW9yYd/FHN0AKrgGsQR11gTcqJh8Q281gD/a2r4UlWOu -8olrS3LM5Wq2+UpnwTOnPVhsppr1+hxUdjNP+Yw/EfG+2hW8ATzbnnXlbkMzO10MSiZHlF3EcTdS -EQILYngHjVCHUigFFVGTh32qfV2kuR6p0ZRE6/xu3BpVudNTKWHggA/32j9b6i4InmAY9iKAKKd0 -OaP2/q1L/BPCBk3C4PvzkV62k4KEzVTLlrId5WpSprheGFGBVoBNeK2vbf5VqA9UYCkILAos+uI7 -KQtz3GgE0WYGSSsdv4EepaKjbVebOCIMleKwjPqtJRMQNevBClT44kph9auwddj0Y1dkrTlGNL28 -q1zZqGpqF62H3kPqbaGyLRxrXBPcdbhYyqTMseWQo3sI+zzAZzCq+vUgcGqpZoOWh+xSMfHZiBfQ -V7oLSRmiOLflBl4V2+AYOEJzQ9IFVokUdfQGnlhsJqr4Qi6sqPn2XB9q7pYt5A7uKV+L+AcviJvO -ELzSY3vD8Q0r/E4vrUU/equw+bLeitPWh9ZqU5+Cw3AraSY5iAuGSDFh7BjFLdWVbyKGPSluradQ -n7q9h8fDK/ImJBBk7T0Pz4CM1fQXtnJFzVNLJd7w9QvVZGWkme+R/HZTfmh3y/iyokT0sgXw0ln9 -qTt1UvMYd10AMmoCY3i1T+4LgOQHSAzKOl839oDJ5FsDcZ6fgFH+nqOi2uuFVLFkyGEPl48ZOkJu -5tHLY0emfnz3eSuP/quXDSME9ofKRLozMLkc3GW1yw5XbWbWgcQqIYxBc9dRQgZGnv4b2ZIt1fJQ -mC30pfq2ztjpLcoUYFu5ebJ8/JTKTF6Z1LBA+fHxo27lTE73/wnx1NaFNAW6j+AaPIHEAX3kC4Zp -yZSoIXuvCKZpeyVy2CYeNN0edJfrSsiTVDS4Z3hAA9Ub5H9ZfxtHpYf0wYlFFa8JkVAgvaVyN5eW -5GCDGiLto6zdwDB6ZzqmdCHPrMfj5bbGG8ut2eUidRVasj6xZmhi1fCg6kEXX6rr4KE3K4H2uRAx -lLdjO3+zp7yTMIE2sRmvPEPVkfARrDuB+psg84sRMiNguXTrCOAwzL11NnU0Mh+gRYrECd1KbBb2 -6Am48uG79aTdB8eKnON+DqSDZOrQQWidRwB8plDbdhZZNoZZRpZqYQejAIz9K9ltwiKP+UUvgTWC -OAJ8Cdjbme7u0v5dByy7MQ7hhgLptxFslsdCBa119odRyGifx02Z3hPByM/iUPjmxWioiFNc2Sut -pdxaFAxcy9iX4aDaSvXchsOm6uVQq+roGxWUs25F5uN1hRsHPn9HVCMvwCKCywY0wvroPUUdtolq -E0QugmLyESwQCaLIBFElQfRqirLVHbZs5IQg2c4ohjxe8dVZ1SeXhWA0lhNd54zUZSWgU6W8kLDd -sLkw6hyju9t3QVfqi0JGZ5xa1ZTRwVQEIQH6LQ==` - - const signature = `iQGzBAABCAAdFiEEQ6WNuVKuWHQJqhdSETT46/mKoG8FAmSi2NYACgkQETT46/mKoG8ccgwAiyWE -lWTZ11Ah5AzQruNwVrC8FjW5EVP4BmWnpppuTF4pNlCaBNKh1saj2WMY3UYjwqsRGFuxJXrREU0O -l8m376ilOzoHMd4T5WBCOoJnuLu17HNElwe0YtnKbImwM2oYy/QdQP26GC6kiZ0Rmwia+tK9CvgI -epmK9MOacjs/TakA0z1+1itD+nVYpb3YEDTyHu0eMMtKeBwVIco26fmFa4ilyhIU2PzwxQnUd6sB -csrS4nvqPAS/a6Ubp9N1g3DYlz/Tzxz1F0zpBuSToQ1k/XEONI8HW4QkUpc3XFnAXNgw/ipAyC1j -3dmAfEV5lT93cCdjoxQ4vrygoWbuAfAyCcgs8iiDsXl+4tRhc0aMM+/CyYh8cUiVemldgCqEcFLF -Ta5/MiibmkmseToY8VFmNJiMDaey7Pu2smpbQXpOVhHUBQBru1VGm7G9N1QP6tp57z+xw7ombXzz -7R/6iYoqN7CdRK5IstoWoNV1qYdxmlpf1zZBYU7NFm/9OyRnHfhy` - - const mdsign = `iQGzBAABCAAdFiEEQ6WNuVKuWHQJqhdSETT46/mKoG8FAmSkT2UACgkQETT46/mKoG8s+wv/URHr -D7IvXVzcXzaaQhRKRDdywUPey4Yb+QSHl4eJ93qnaRslTdDc1wR4PGpZBahXGhXKGIBf9C+MSsmB -bUWIL05Bia3myBC1/oivm8or29vL6elt0Q+rNaOACMmVpGIRt6MxnI5KPhCPstv14p2y09ROf6/r -dQvFc1jeoNeS4JbcuA4bzABCMjZm822Z3ywYDtIJzGGMqbsGNSChYrY1NKdf60e24cJH9enRJUNX -k/qv2tUToChe+trVjx2EAOlbaenGHd1RjpRQk+CmEwNdhk8OKR6SEayynVYvU04QoTc/w170YXQp -G9oBJsMHOjlSIiZSByLe1kj1qsyGHfgZSg+C/igfGl2PXA4smgbEem4Y+P0j/i5Db6nQ81vZKZH0 -sNBESpX3kvN8rAR0cmHnGlelCklILv93SWZHL6+4yZaXVcm78Dfc905B1HPONb/zY5hqQ/CDuMK8 -ej9Z9qp6STZxDfr6HUNNMWCKgel/li9ptNlp3dOONEJHpJvGSGCk` - const gpgkey = `-----BEGIN PGP PUBLIC KEY BLOCK----- mQGNBGR6EBQBDACseiWT4OrE4FW7RuKmVN+ETK+yXUdViB2lWJRJrdm/XsjYKRvw @@ -239,28 +67,395 @@ ZmaX8KGlqBgJ/6s6/8ERh7IbNvNu+PolTI5+PXDTofZ8ssKIcALQgF9zjaVIORfF =5NFR -----END PGP PUBLIC KEY BLOCK-----` - const db = `H4sIAAAAAAAA/+yST2/TQBDFc55PkcscY++s9y9CCGO71CIpyCYcuKC1106sUNeyXanw6REpgjZX -VCHU/C6zs3rat293Rtf74bBb0YpC30z14glgjDGt9bEyxk4rUw/Wx30iraPFkj3FZU65nWY3Lthf -e52G+0/Ai3ydXcWbDOHBKKzujPqiRDAcdsHsxuD7NAPgIxkAvonLR/2nrCjz91cItCIATLMyQUib -qR67Ye5uegBMyvxzhmB0JAEwv+8YAG5SWW43CIaT9555LmsvIm+0p4i3kVMtk5o5B4DlZcylOqpt -xdsqIuddY4wgEtLbujbklbaSN0aTJE6iqrhVnjQXUWvIukbYynhuBABuizXCfp6H6UUYttf9XTDd -ht719b7ZWR3+SRcXySXC/cP8DL/N12kaf8wQSBkjjLKkAPBDnLyL32YFQur67qtbXtxcd/23w375 -8vepr3/5vAL41/9/5syZ58uPAAAA///ViF8vAAgAAA==` + const firstPackage = `KLUv/QRY1QwBGoVZQ0SgriRpAxihjQU4nU1abf1BN6nGKR/bld+8hZmisY00IpH7u7t3+4kQSpHb +p1+3roM678K21giRyF40KUz2hNLQrAXR9A4E6QPnA/X6wzHZ2cgbLKlm0Edvp0eNJLW+Dl3H7bKL +TevnR1bIGyutpv/6ugOzr3o9SbqpSabOgdSVpFpKn7cy2UkrHb2eI5eNr7TSsmAdqZMRB2nZeKP1 +ddAzvvo2M9f6nnFmQrXZIzojSQszzCP54Uv7RNZuXX48UcH8+DpEK2yX76mRpaSxrDVjBtfI2nG1 +s8LFLFZff8AaHWzBtzqQuoaOMY0qtcbzsVJjmGepqcT6urVu+sKSks+0rfmBPSZJ2ye2M+GRZfk4 +xqNKSueP3pK+NX90lSQVnxdZaknplpTXEktLvxgX5OJKjZb3gYzrVrrTD9vYPrCxtk92nG3kruRI +rN/LB3QXOvxWyHD6LtPK8u3r8u1MWlJN65wV9iM3SVodb5V1Yak5mpFUUlY31EmoEoN5x0pKYyhJ +pWStns/Peh3IkkpJO60ZWwdTU1L6dZkmKVlp9Zz1voONxFhpdb3vSK2dlH9Rp6IcijHFGnt28JnP +12E76FhpNfbrGjaY/Yu3rLsQl+89LLW0NG7qtY2Zm99qR1Jp9ctc72Ok5U38y2iVViOtzrhWJTmS +6iwrfi8bS1L6wh/JUiO1phUvrZLEGsvPD/2kl6XSSko72L2WDnls7di95151j+FYI4lMU8HYrWi8 +4yAlLQ83a36SXozEan5Wap0/Pn1zbrDEes5HbqmRJNZ+K2z2FndWmimGcizFmtHaJIrR0L8Rf3T0 +0JeL0RUvSUnMgZc1zvYvxN0OvKv1rEtS+pFnZEqyHO5JYcvTcTomfNphqTC0wo40VVZbczssWVr9 +XrbDkpFaUharsctn6rUdr4PtsFyJJdUZ06cVvkpJ0vhWfB/YLr9rpKQb65VW+3Qn9yM3DYkR6no1 +Ay1MpVXxbFFpVTwsUYk1zrQyXa9tjxW2w/tU3P5qbL1SS6ylI7WVGmuHD13HONOnQri6JYUt1Zhi +bS1s+Y5fs1apkVRDj0xxFCQookvpMqZPRM2yiOZ5Hovw5j7WTfsX92Mk2+nvJfzsMpa15veyCAxr +tH8zpiBBgWn9bCsRtHlcxAO6A92BDsIibpUVQpjxxyIoSFBA1FlSafGHmxQkKGStOV+XsRHNw0bq +rhFG+KyOERQkKO4vrSkOR6OxLAoSFJ5dJOIDD04RnUwIGI7GHUhd9ACF84jQn/UICiefuhE07gGd +JUK1xpzH9hgGI0LmrzWl0RK2HgN6czgYj3mxMYyjKSthrME06HAQemvOtaY0KEhQzAcvCnMOLhHl +P8lSYmlEFCQoHHhd87BErF4UJCiuQ9i95mGJaB42Y88UJCh8Oto8LBGMggRFdvNWhiLXepUCwHUZ +P4oCAEvS2FIUaqmmVFOSZFGotSQMBQCKJFmUUgBgwRErOMEIUnACGIggBU/uxq2Jir/ify+XxJ8f +S19p5+m/l47l8l+d/162RmMt0P+9bIzGh/9eKvCVj0TqnPeSuZOdQ5r3cnn4ad7LZWu+9V7vxjbv +pavNexnf472kkGzey8it+XG1fr95LxvlzXv5XrLmvVQdZevcL5NWQ+qZVua9TJ5h3kvW0Ll+ROa9 +ZGmJnK1MfTXkEfL3MhJV6w3p72UJ1dRiTLH3MvnHZRd9paWj8xuhuFZKEut543uptAOzs/hjW/43 +r6TeS2wfdwdjrjUVjVuBr8eH6rXxqRK+lyqxCpbUaveVuutsY10/wveqG+qKLQo2FexWKHevP2o/ +qXbXJ3PwvWTtF489R95qeh9GfAy/99K+97LdR773kMcHrPK9lw1HUs3ee7mSauaTpCTViaMtCLn3 +0rGS6nzdxC4zTomwsTQVDVNII9Q92L136L1c9d/LtlisxthlWo1plIwo8YTSp5NKcrCbuKXy7qN1 +rPMR8Ud8D7baWkgrNpGWtFZSDZtWbEKp1UwtbCI5YUKxWA2rlG66CWWxM7ESa1oxps5ER8grKyWN +K/61bCOt57L9pZnnp177q9YsM/2kr+Pr7NNfbn7rSFSv7Vsxddz1nykKZm9H3TcSa+qOjTiMlhTE +oY9ja3YiDuI076yk+q2L7zBWUv/AZ/TFxfOY6xzW9i+UtBozkIrNW2ofK5/G5T9KKynpBOIPPXLT +bc7kkVtKWGoWq9sjq5USi9WeXbRPP/L16fxGZPM4ju419vsQt2+bfvNI2xmn1TZivJLFLdKn4ond +dAfiCNmCCkuWJi2HctOi3qOYepvW1u9KLS2L1Z5fUXesu0y2zeJWZA1vUqPdxEjKKtbYxly3DeMz +fS50i+vN9DtDN/GqYhXL/CSJJdaNfWgDF0Z0RNZUJOtWuOyPJY0fziFZOvrzen5aZ9KwWP2ANXbN +7WAXr3tta+4MnZ0VUs+RNyndWdNdbNzQz9kxdqza6UtacVMlSyyWQ/yhzrQykfwP/InFup6pw5b1 +ZlrhUlYjUY6mgiW3QqWlNY662bfK+oBVeobuOgdeF6knTj5QOsiSVqm1k2vN/siXVF9X8+PaZ6V2 +Nmz+bJqjH8kSw6qxLO9lw2iJNUyavZcNIzGL1Q39x62Q7DQtDGyiwYzs/Elinb95L5mPPGdvWmuq +N+uRsDpLSdIsVkeu82bGZiWWJNaMeV70th+xYWn1fLFZKemkL2xWWhA2qyFsWGo0bFhS+gFxTXP8 +MCs1xw+z4r+/TFuJNXMbye6XSa3zIxv3yyQrMQvWzTO0wkwro6ykGj9kZZLIJdOgk4wjJZH+Uml6 +xpEkSUqjVVosWDduJ8O0tDrX2eN1zjMMVElRKSnq0erkCGuIf2LLmnlezfXa1tDGFtZrHctWuzr/ +Q1bbPGySVFM3IxP145Op07dJYhGTz8311dsQCGQlpf8+qiP0c0dubi0xn59zdjo7XUy+mzaS3Uef +bSGSsaSlIRDIe5kolVb9ltdXr8+1cZqW9sZh7FNRaUnLP2qFbfNqxVLr7LhxuZFay5mdBb90MMVY +SanVinKwSuotrydJtdMqIVNaXreeX0IMbZLUauYX2iQ12i+0D+2SGIm1jp7vUvi5b51dEqXUhjo7 +nU2KVZisjrO33vuTugmdfUBn2622StuSox2MqdvKmAnqtt7Iqds6HXVbl3TUbWU4fIapEUmrIy3t +5nOf5mmN8FvC6mhJ9VSa0u7jQzRDNx9KW0r6WkydbB0rNXpa5ngTdXqujM62UsUam/BYaXK0jTTt +sdJlgj7N2WnLDCNlSokE3wfSJGks0OGaxG5iiqlu6Geneek073TkMo3grtTR/Mx2srOf5ZMklRKL +NXNjq7HTJpVYSvoydiWVod/UvejqfR3VF8HPa13fwvz0H7Mti6c/K/bpsM+e0cXZgEmcKVnJvHAk +F5JBSrtHvsisuFqTVnKTSgsuMWWbkKR+kRj1CwdyHWTeZ9Li3Om3rqOHspSwpBqSL9nC1OFLx8I8 +d62CscZuQ0+EcZoj00UmtDSYCE4IirLCkjozHRy9l5H6jdqN1N1nRkvOKCl9/SEnH9F7RNNdp78+ +YiKWaWWi92lp7NEzUY48gjM2broYQTCKlpaSlEQfrbT6wSha1ooSC3kvU5KUtH4fYp9D3sukM/WX +8EHey1mhg6jIQu8l/ncrlK9Dqx9nI/RersTaQe9lklZDT7bT88Ni9MyHn/fy4ed5Lx94n4fQZ2X8 +9GGkZbEcupvfS2ZxPWN3O1O/t2bHNKCvpiKpYKQksfs5aYb57G71d05+LxvTeXaslPSi2GW/T7YO +NlOvxRHM5vSMV1rNHIxXUu1g3G5UWbNPxe9lUilWHCl+L9/LlLS9L/F7qVJbiXVFBWwev5fKL3Rs +Z4XQsYKxw8thSCP4vpeLAuHr8PteNgZx931a7jkBsSstqx3Lx76X9r1MqqXLtzbUvpcKv7TvZePN +hzqtjn0v2ZqNXY2dW4fQPbTvpYP2H72WfS9VZzup25ocjZPSX8uPZm2aqiSmEeZzbecmZUlaPel7 +mVpLupjDp++lkrPTUJYZqjRmGH2ZnTL0vWRKUp0hTt/LBd3NlFZY4nqWz2Sn9L1MEvzIz4+jzDEV +R+ZijdsJ7XwvHY+0sU4638uGTtYRM1WcKWenbrIMpZspyvdyNSjfy3YPP2RkQnk3RCgKEhR+8XjN +CpNLO13kRhhhgW3ODgUJCnkfIrxxGd8/goIEBcQeV0uLgqKhjx+KC0QeRfQSUFwgWYutVQIUtCAF +YGYBDA5YQxSWHlyBBlXAgx5a4DIXjDAHCpyBBCx4QRLw0AWHANwBywQRJoACGDCAB3VIgx1K0KEJ +amigAogQZcAEelCCDGYAAR8cIAhDIIQJbgACPRDiEH6IgB6AcAQUbOC4MYjDACSwgRMQQYU3FAEQ +BvjAFI6QQgJsIAE+tOABLYczUAEBOKAAtoIhkgagAQ0JIFEGMTgUEIMjoEAIChNoBL4QgwsOPOFE +GHJAiC9ggQxM+MESKuCGxk8MwMAJW9CAAwqQhxnQPwjiEGZogyGCAAg6QGQdHhLQ0CihA3cgwwgK +sfRhDDvQQQoiMAg4UOB1YAs3kAEcatCBNlwgYcMb3vCAELCgRABKYAEC4GEFE4B4IQB9eMMCLNCD +AyhmgAMhArEHLeigASdKfOANBFjAGsAKsuAGHQzgAEOoQGMsi6lEKRKQwAUvUEFSYsYX6qGWdMil +cvAFU9ib5sEOvnBcXKfe9HT34mL+RZL0iVmvXzDmIu0iSlQUsfhQ1Bi+tl6rYp326UfzPaKcFuuk +0ooV0ubWqKWO19UX8Ue01BprXdAJBlMYUDWp4WDcpGa9DsOhp1KGsbyE8ZqKxhwVjOVIDof0TL+p +hjw+JjLnE5nF6s+N9WObOF1Xjw+xr9Bqx0+dE8Lws3U0ghvrpcFEC1PuGk2FwipaWrn52MQmRhOS +4BnBEX8o9WD0YNRaWVikEnf3kuJQ7aZkWUlZtE4WFjEdqT+ge24hDQv40toHl3BWYszICmGYCvFr +k2olMHbp9yEOfAfiDURhh/VamO9ziLKP3CSJPXLD8j+eQxSWWnRVdoSvMPeQT/MMMUZSnhursaSS +akAM+aOEz0dWxk1oYUY2dz60rPnQcxmn+BD919RvlQ61CDGZQC2xlp8f1Q8/Dki1z7J/QZ8kOR74 +4wMbU+wiq+Wuv5fN5MxYarGlWMevl2FvReU5morGqnggZ8JjMzkrRdX88jrYTM44s9bt+sI4N5s2 +M1KSGZFntRKL1dJ6ZtPdxqXnF3NLq6/LjeUM4QhuCuu4INcZsqmtGh/MVlINv+2Tr8v5zprdxY0Z +B7vnRJLaCSwrVCyOkbtV7pdZVsWOiLWO4olpnYw9hmGuqWirgiWfPzv6dQ5OGlKjc3BLquN1fB2a +jvH65xy3EkxGzCLGEGOH7ZPxc5+Lv1J6Heycq1RiqRGdNwtfkLsKIyVpSUnf66K7Sz7hJKbuNhBe +2Bs3n0Gf3XUSeuwln1CWXvJ1/EDW/fMPdVv7n9zIxvsf/2dpRNc+9PvRb8bV6YcfCISRcjTsGqiB +HtrmsU3Xz/eh6ERHnLT4Y0McyMGYoc2Mqh2+lRFbXPHzYqOf6VGlSXRInUM9/zuxrPdS3sezk3Z6 +/vvgEg19G8nKciI/dqSkHb4Nhv054fciN/TxdTZiIxI6GrHMob9R6cUSg4xuJJZUrLOxKE5Kasyu +jki7Hj/dddbRLTEnP67S+Iw58FunnPw08tNoJOayqUlqjZXUw1DP2DEr6UjNNnIDhomog1WSWCSo +SeecQkMykySFDkMRQCAYeDQYEEvms3L9E4CAT2ehMBAIBIFAMBAICIsDgaSnSafMjMjISJIkaQ4C +XLq8/bNe3F9AF32MALrc/TNySvNnOU+w8N0JzdEMoFFLMff80mUN12eqxz59Y5tqx0wXcdEAl9C+ +AYCuaPFyTd8HbQ6THRtd7JcDPBeuU5wnwhqiBy0RJhErdeUD41vcFZo208nBsLSPhs1DA8h/HWZK +FENQPoNp0/hzbxrcDnLTO6JUDRsSC7hf8tlFgbNrhpz3o94COuVslXlubzn6kat89igykIKM4031 +U45jOtlfchCJKuf5KuhEUAAkYqvx6EVMGI2lHk7nL+JeE9h3tYdrQ+pW47/DlP7IEBxTbpJ1n/7H +v4wrARE7egHFS2a2hoHWo8IKS/6UHXwkN7B5aFOv3tXqcSupg8XH2UVx3MU68TcvPPT3Ttuo44qO +uZaiRQrjQZBG8A+2L9lg1OAoUIsp2JpktXx9tOMfK7Iwh+0nh/uzwxOn9PQ5nEhyspUoVLolQx26 +dssH5Y+R7z+kxJtmd0L2GkCSFH0WzLDAojBleIv7HFCj7I1C9vdk1sNMH7wd+dNKm7JEM65GbogJ +hNXKKqlRQAUiP91fY+GT5qpyh/w4KlkQ6eu/DLRfHwuUMYE4NxS96m7xNIfHnLI+B8YVaId4ldrC +523sQiLO8w12hoeLzsRIUjO6/E49e8rCelkhi09sfl+JWCZumQ8/TLkaa/32Z1ro/3wEhLCjXphO +OmjbKxvkqkyWpdVci+ZZ2CV7TGcDXER1SkeErM1u9nRp9ljgqXdtrTsPGnoLz+mquIc6HmJwIpV7 +pAhJh/zUZrx2u+nq3NfOMp9rOABVYEhAYGHsg29/qfE5KyHMKsFank+CeHH0iHjVB03dkKbLnAJR +PGH3riGzCr3INZityhc8XTUqH06xftHJU8CGPIroek4mIVA5/K6BleACKWw7r4Vm5saPSwqL1vA7 +J/KVT5AzSmjcYer/BuhGD3JJudm19NvNw6jYTqqcEYesw19VtK20lIubcQzF2RKrlrcas6NDRY21 +K9FARId+qvRM6xTI7pfeiWO29/le2ZcsgoDnEllSjVamnN1bEDHUGiKFMOotmSD+IKMqWOPBVm72 +a97kjZEEr19zDboNGfRuKtCyQoaCMmv/SWI6oerpOkz3Bg5aeFDtgrWirFwYWD1kOctQcr9VcTWf +5QSxZIZ1iGRJDui0xlQYPeTpIW+UxFDmR7KtsCUG1q6DVoTFctnxJExfi/g/8aH0dBDP0fRAMbX7 +vVa9m1OETK1rL1VtQ6yu83ioMk2kMSOL6zYG2X8dyNPrIBNVnKJMp9TGiMBpTrcKE+NN1nAVIheM +FkGIkp2hEu1cljZRSQa//M2jqniS1rodjVuG5YetrTYxTrGgKwXKDVZH0YQatAAUokTKMaJMvNaK +IeJvEaIbGXOi8TGjjp4PpWQ9YvwoB1foZ23EmpGAdllrFs8NzfVl3Nq2K2rl0ReNrRWLaG+jS9uU +zpNDmVvi9lLRmb9jrMIjfiuE5KbAXVDPpcalvXjgLB2T/vObHKYXWla0APm4/IARUtYtpDs6o9Pi +Kg89Mx+dBJEN4VpXkSWIiINFgF+aOG3p9OOr0wuBJSC0DFZyQy3tCgfgx1SiheEnBW+aMwvnNtZc +T2ewDRq3YyHNqaihYZmhZ33CHlSQycrOeyrXv3kwNEM1rA/YERYYWxWcPEMcwDl2gWRGA0yw2amb +PGasItrwCSlvPkQsVzzg9d6Oa+a6PGINOTHILUDFqkmpYgR6qulUj3DwI0msidJ9S2L0Sht2+VAf +0SOZZ+Ci87XbIDeRgoTT3LSvg85kSjh0TVL9FOHv+ucwKSOqH3dd9yXSNuNuD87eR01Fv0nas7Bt +SMbCcOX6rMRaMUWAh5EuFZ2ck2wOsGhN+0Y33EtoGgbqrF8zx0r9+rGY3KxigdbY+ZEGt0fULofF +UrReRlXh7qjS+Wv59thWl7q797LlgRM7QQFHLPktAdS2N9xcF6F7SV2uLxmRe89bun3Lc5u+5CkX +s/RYVHIsjm8NXaEUjTSVNOPcxloPHQ7wnhkA0G8W8JWEExhODtJDy1EIv7h6RFXw0jIoSi3gHUSA +cO30g6/8bFAt1aDz95LbPmP+G4YVFJFeOZ9nsUt+/9suqZ3LXfGvxrZ2iX1pSrVApgSA6X8ByI55 +EDiYmUhoKotD4Ss17IjMMlrCGx0uK2IPNbb61BfAWVkEfzUmWCrSQssQZVO1UA42lYUUls1NiXs2 +xzgMRnJ8p3yJjAHpjzKTjFi9QLbGaBJz0KMEOjLehGfnNgzX+O5y+880Hh1DBkEy7rWkoMlZOSLN +EWw/221YRkDyqsLy1bh1omiTbb7tmBkbnpCWknsilyDcCvbjL3KaQNBsZvyn1Bg3ERJgWTP/eFE/ +Kk7R+MsoUMhsCj570o/A1pEA8PkCh+GIZCVJtOZ0+Feov2CYgMlCuhygFsSsxI76EQyaVWnDv+fC +bULYXNqCJLCFWskicdnd9Q5efqy8bPiiYssAFyY796lx+8czpBfX+Kp2b4B1OgCuLZbeXFgFo7Tf +ePihzuK/8hopN0l41CE3jP9sG27XQoIWwvdDMcTU7heKtC9vx3TQT+TPsBKVctrG4hgesgItmKkW +nMbPHWpfmlMZER5BKwCW2Mse/6EzwqJJvc/aEi5q7l2BFgJoEhFA0A/LXgOa16Rj8jG89k7mGJfd +YVcMlcIwuI+CAdwFtIhnxtGk1udtOshzgBpbrOv20N3X1SDM1q1cQOv4M9xck+oxfNxhEzx4UUGl +neZuwBgAWEmBy2jDGxvV5T4TbRfly2WzL5e4UkjhslZx8gFfBYbZ4/JDA9pcon5/3ulTn9HRzj8v +/lwI/KaT+DedCC4v4AkBIGVE4MTTZv5rKeTfgV6/rL+eLv41gWSa2lBbuxMtpEzmNs8WqliwlblM +hr4V9OXjhyXgXJtVUBj9lp+Q7occfNHrd9rN6noO4HucH2QqwlrJdW2g08jgUjWthsaHs53Xdeze +4UZZM24JV7l6AvtiQNlcOxp/GJ9QnoLC7JUpeZvE9Jv4pBZKq8ggCK6hUcjsirSw0RkNx7sR/e8A +jN/meeCXW2FUWQNM0vruVWLNZXekV7hwIRfFDv9lKKrG8UH3BNny5Em7+x8zOZrvWbca4uVfqCmH +mapQTOnklpi8jmXXvUebt1T05LUucTsyb+iexy87wn/v4VYFCMIHC7gqPJxUgHK4R3taFnnT8xFz +56hp2bISjAS6ozTT9EIkwlJOvli/yOhyo7Be8YOR8xTtQGQfHK7qk9und4jBOK32SPot8NdWWjon +3TKZGFI6TUvp6s9LndsAbok6JlzcPvSzLz3NLsECK5B3XLxDKzE4RmALqjvfwJVMD1DVIg6g/Oiw +AEmIdbtVo09Kd4sEkCWY0YHjS2h6E5GCV/7kHJM38gdZn6tATaX9kELKhONLAPqGFG3/g91UzdcD +yGqX/PusP0iAM8oNbKcqes2t4tuYOI1iDwPgQ0NX3eup4/nfvqnsQU0Ki+M/Wa5NRLhXbbU5GUOV +BysGnLR207W5xg5B7ebxqz9mybJ0gwlhlj42aIDHDkq97jHehFNjo9V/36Y1CTGKvUKjTFNq/oHc +h3yBxiCjSlMGqyXT5S99l4cGqzOgF+mOz3f4joEFhtbPIxz9vmCGWh3J4h4o8BEcdI9xzJeYr8XK +TA1zb/P7HWY8ZImQPXLIuZcXqviP7VbzogN9sZMUFABg7IV7RJMj/JC7jfp0fXG8c3K3C2Hq8yOP +QX/20CkZG1NcdhGZ9SZKO84rPoSpraWarkbiw2JIfpF4n52kEdlMPhwJcgR6b/2VV1mlgM7rG8pW +Bfp4wQF48oxWwyIK1Gdmzl7vVt4PimUlULqq0KKjJiOOcUP3b1F50SBZA2Sxw64qPl4dJfVGv9GV +LDgJWKUrS9mcDJ5vjuUcOF0S6J32jxpqvAeA6LMSarNQ0Kr/w/YNhgPnB+CJGSUNBty9ugLkXhO4 +0qi4xwzbYTMAG8Im0PFE7CWnD+2vZCe/Qo5FDC08I89XswOgR6mBDGfRIQpINQe67xXxRI4GU7L8 +H2Bw0hP+GNih8Bogw5iBzyihPJMLvabmNTRMUoWqoO4nsrJWOxKpSa7gus0ZxxRAgiEYL0LVboIc +y+CldfK8f4YE43KZkqjBWBNQsL3P73G5/+h0lNKgv8w5X8w8OE0QVCbwag97F2znojl/dXJjOGXg +ggqVxK7M8/v3l38TJFuXxO3OSpf8XfWc9gwy2RPV+/ywLFdV3WGhroJopQIVCmInbseg4f7TDTzn +KYkjTPdEQtbNPFnxx79SinVxkkdHGxTcREvjwvTbPxWpWguyc23B63AZ/oEOMjN74TGu5pFKCKZ9 +DjI3FDItF+b/I382ohFpi69z6gQt0fl6D4FU2BQlxulAZrwB67uQ9CQ4FHPu14qiK8p9+oR13Kpj +hVuXqOpLGpyJMBa+9ZIexHQoAiGJTHPL9gzEYW9lpo5R79qxCFodxBt54r3QF+hH0MFCoEppC053 +nNE+OzewlSkRDheXfq5KYN0PILFv55SCvoIcBU6r+kWSQ6z/hxBJGmdi1Ff41mGcVPk/ru3OmKtw +co0TQ0Nvz+ewk0vfq6d79zteLmAXY29dl/I2ETM3Y80ijVpRM4kfvMl0IqiN8rJJA0pb23lCGtIs +ZwtJQCyFI9DAuKuZeBPaVkKjc8oivdT4MjCUZUXjXvPmqU23icbq/0eUrbJqVbXdFCaUChPbVUBf +vFJ2XudBrYuVwAMYfps+L/xSzycsWpjP27owmjJVQBoTBs4kAAVlOd92Ish/ADfzM5/n5amNMAtA +KoK2d3acwRbvoPThgaC6iAdxSsyiwNABkbrCc0s+7mb+Xd92bnURx5EI7yvCxNdW5mCQLHAz0Wqx +47iPpcezPcaW3JYLiptqJ/DSPFQz1ZfywpoZqLJdJgO/BvxKWOlwHAaq7xnrwgk/Dco30jHHe+c5 +4q8FuElSF1pMIf7CtpaoF5stvMKWHkT4KbkseBzhSDyKAMKmP24EQHagROionvNu8ebJIoK/PSQM +S8KqUIlCcp6sHr5wlNxmHDL3MV9G3XlK516DS+di6GIF04ZQlxQzaLXMkHZQzMfxUzRC6dLuSrLH +7YliFis4XUHFvrmhdktnXOvcUUqk6UV/SDQpI1CNfSn6nJ+lf/d+lsScDfem0J/AQrAzMq9UAa8+ +tnjOv6bocDTh8pI6fy05nmQ4rP9Km/CM4QJ68fbgubbfjfa0QiMpmOA6FcoeLoXktlDh0RkhEcKl +AS+o+uKsL0tNKtVdC4DgdqVDQsSke7NtqRtlf7Xk2NzUbmY4WpLgohOd+E5tINpJ6ae43NKAB4Ab +sQdFI55QUjA21EH9nqb6QZZYvTC1tkZ1AjPMulkCO81Bx3xToeImwKAw8dt1+3cfsKxjHMKwourZ +CSunY1KDDxy4Y/0arUDJ9I4hxXryLzr8a7tkGnTjRDX7SL/KEougwLw4zvxwqP7lYXAbeC3yyumP +qkc2GG5nOIsMDPoK5jiQZ5Fb5mVxAzY0IbJ/k2LAnOWf0I9tXNQEC4rgbzuCIiLBEk2ADglcP9Oa +7bV8bKiThPdl3TVOb5nym1OpAhcLCx2WO7pOgpHLSuCSSnlREDphTTd2Ztpt9pcRDeyiSEaN44ud +smiYikcA2Mk=` + firstPackageData, err := base64.StdEncoding.DecodeString(firstPackage) + assert.NoError(t, err) + + const firstPackageSignature = `iQGzBAABCAAdFiEEQ6WNuVKuWHQJqhdSETT46/mKoG8FAmSsBgkACgkQETT46/mKoG+Qsgv/WcrY +nMQ0BH6n+MAc99puw/VEj1o/qwwwrYV1jNIIWfWaMnlZHiTgShxfcfDgnjhMDnJGslia2mIBZoBy +JRUf/KuWye22MH3uzBIK2gz4RFt6qA57ZaU7y3YYrJ6pbwNvwayjuUa9Z8PEDs0t04mEiPhJYJVU +cM0mtWZUuYQt7kdnqm8Oxx0RSMPamQ/c/wC+h6p/DWMngoWgmDKbKwNA9/UjDrQujuwwVU1bUiUk +cX/cACNaTVcsIjSe9wD7xLqvBblCdjvl8fumYRYH2t/oj86HTLiCswEMYqsgf+NQ9ZFBSxd82Bay +vhhZgJPEjDBykHKdhqD/WZdOE97TwlbySg8l5oY9yUBrV0iim+FuqX+bD/X82IzE11597SG2V50z +K1/Xdvx76yOUHSO0t/731d/LyGxEVoD2ITwM5GgJgTYnOByaI4mkr9iz6OeIwHVgcTFYcQ+lMP5c +jMF9gK0ZWKyVAYH5FFhDJAm+lxWpcBu0Fnpj56LGqcaDORsvIVkA` + firstPackageSignatureData, err := base64.StdEncoding.DecodeString(firstPackageSignature) + assert.NoError(t, err) + + const firstPackageMetadataSignature = `iQGzBAABCAAdFiEEQ6WNuVKuWHQJqhdSETT46/mKoG8FAmSsMPgACgkQETT46/mKoG948wv+IkNg +3zLDySwDxcBXmuhAtMMgUs4D+rypUp6qZI+N/WAWtV7niWEmZI3RQRzSmHn7NpmXPiQyGIa6ViET +bjfmbVhU/iaFLIQa3UY7dJCT8x0QVQNCvmLEWzAkqQMiY4QKaitetw+nA6gw/MAL9UNWLiRlCj4I +K8ySQdZr1dtSTTw/KK2t6kgFbEVpwLauKZrT6Vw3LVU5biuq76918JG7k49po4wUQ62a8Er7WqOV +4DyhNMKqcbfKd+9/WRVqTMHep7dC2fzbq1/MFx3TNL/vGKl5uMrpePyTN74AmxqfQWNwLDXYPxsD +/y4vPYer/KQdIs8omDpNaxkTrAvsFizKfxsFlPRt6uknu0WKY15SUXRI3fIG+vFXigW1JNVtuynH +KyH0USuoUzrfn7hKPp5ubusdeDTwr5GXDBM7qEpBxGce8YMVNYQUTDdK9D+D25fMMb5DkQa8zKjy +GXcSu4lwxHmarGlD45/DQrYH7tSpVqj2cBaAxtkn/ocpoPawta67` + firstPackageMetadataSignatureData, err := base64.StdEncoding.DecodeString(firstPackageMetadataSignature) + assert.NoError(t, err) + + const firstPackageDatabase = `H4sIAAAAAAAA/+ySwY7TMBCGe56nyGWOTcd2PE4QQoQmZSPaBTWUAxfkON626rYbNV6p8PSoXYTY +XlGF0Pa7jMf6JfsbTfB96DbLoRiKUet7N7gARETGmFMlovNKTPL3+XQvJCkaRHSJz5zz2Ae7H9Bf +v3Uu95+Ak2pa3uazEuGPVRgeUv7GSdxtlnGw+/hHHwDwWQwA3+X1s/5LOa+rj7cIYigAsCjrMcJk +ve9DdAxFnXUbu/QAOK6rryVCykIDYPXUEQDOCl0vZgjsRdtmaaqMTlLyJGzD1EonrNOcOAeA9U0u +NZ/SjZAti8TIVElrWSestffGi4aYnWq0UVLphpWV5IVsMs+Zc+auJXbS2hYAF/MpwiqErn81GvmD +3Xb3PnYP29EvvaNQPh/fIDyN5qi/qKZFkX8uEQSnaZZppRgAP+XjD/n7co5Q2N363kaTh+16932z +il63dudWfpmZt3fb3SHuH98A/OsNuHLlykvlZwAAAP//LUzfXwAIAAA=` + firstPackageDatabaseData, err := base64.StdEncoding.DecodeString(firstPackageDatabase) + assert.NoError(t, err) + + const secondPackage = `KLUv/QRYPQ0BGoadQ0aQriRpA3QL6NcPUx9kgUekBUgLXzPAM0Aznbpi2NaaRPb+7u7dfiKEUuS2 +Pqxqd9d+9La1RgiRyP2TztWrbVgeLHxZWAYKEATtA+0DzcGtStzBkmoWveSrdIljSUpbj6zfbrjB +nvRxjjRyx0qr57u2Dj3si1pfln7ogmeD3nQmqY7RpaUObrHGNq1jicfHV1ppWbCGs8VJi3R83KG0 +bdE7vto+ElPq3nFHRrQZBHpjSes6zCf438eaIWuvDUMyVA//tR5N+tqjyxSytPSNlOLrXApZ+6VV +0reYxWrrkEfhgzXXUujN1s/vnVAlpa+ErKQ4zrOkqMTaeq1NesOSiu90pRhyHS5L1wy7oXDIcMzN +cagS0/ihK0tbih+6WpKC2UGWlMS0khinJJaadhCuqMGVHB1tfg3rNTrUkNq35qewrhn7xXViV3ok +1u4j5DX7Hm7pa/j8hidl2Lb1+FUuSlI9qZT0NQQ/S1oNLY20YUl5dCOpxKx+ZotPpYbzj5UYpcEk +lZa1WuaX1EKPJZWW9knxre0lRcXotuG5pGWl1VJS++05UmOl1dT+IyndYvwLKhn1UIwp1tdxe+88 +tv66vcdKq6/b1s/B6l+4VZuNeHQus6Skpu9Dbe3rWOxWP5JKqz9ial0jLX/g20artBxpNb6UMumR +VONIr/voaBLTF54jS46k9KS3abUk1jc+zvOXbpZKKzEN3eY6euRQab/NueZoc43GHAt0ngrGbIXj +oRYxqYE0SXFeejUSozivpDR+mNti7GCJtZQQrCTHkli7pa/Z19uYaaYa6rEUazbps6jGM99OHNKS +PD+uRtPbJCYx6FnWF9e373ZDzy6lJW0S0xAsIWOW9XBPDGtyQ/mQL3e7kXEmfT15aKSV4m43srTa +fex2YyMpSVmsvh7fobY2tO11u3ElllTjO3/S9yotS19Lr82vG/7mSEsrrFda7dIh3BD8dAROZuvV +TtQwlVbFs0alVZHZohLriyd1mtraDulrfzbT2+4oVHolJbGOj6SspLD295H1e/HM9L3VSmLYUgpT +rGt9Hd+vW5oTjUPSrFVyJNXPIWMawwQDeowN3/kkKI5GPNk5NMIfm2uT9S/swwmu0t3H97jhGynF +7qMR91FY3+LLMMG4kz6ulMR8/hoBeQ49h157zwhLI33v4fvQCIYJRjSdJZWaB1Id0iTDBCNSirFt ++JJgJR5IhbPZEiVcUqgEwwTDvrIozOPhcDSLYYLhuEESGMg3SaZ0NNB4PA692eAFGBBkMl/SDzBa +/Nk68DTIa5WEwtyhajiKQ60FKoxrjXOoph7NowEfD9Y0rvFoMI2pB2swjwbZsjQc6CiqgWxxHgwT +DJlvw7AG3RMwfp7MxjDBgJ5tD6SSWL0YJhjWo9vcA6kkHkjF1zHDBMNlmw+kkmAMEwzcpKUOA1Nq +lRrCNnyzYgzRYBS2FKOxlKKUwizLYrDVLA01BGNZGqXUECf4wBEi6EAJRMCCDaQZ9+MVBb1P77uP +K97Huc5nGvr57uNjeXxH5buPisOxmufvPipswn/vPjLvmZ/A2SD3kTmEm0ce97FxNz/u46oUW2qt +91Mf97HRx32EDuI+zig+7uPEUvyXVrf/uI+P8sd9dB/Z4z6qluDa3K2TVr/peFLHfVzecdxH1q+1 +hkDHfWSpkThbePrqyGHk7uNEQqs7oruPJ5SiFmOKuY/L54Yb9JWahs0tjOBaaUmspYXuI9PQw63e +h8ryt3gl5T7emt0fjDXlqXDYimsdfkRthZnG5z6qxCpYUqvb09lsqwpr+vC5V/1MV6xR76lgtkI1 +a93N+kl1sy5Ze+4ja7/463jiVk+bG9Cx69zH6txHxebo3EcOIY9G5z46Hkk1c+7jSqqdXxKTVC8P +Zb2ouY+PlVRj2+Rt2GkMfI7mqVDufHMyW77NvUfu46LvPiqNxep7PabVd8LYgPHKF122yCToNnk7 +Ms5ClIZUQiYOgQ7CVtf6Jr0oUpPWSqpfT3pRKLWaqXVRLClQMBarH42xSUeh6m0oVmI96b2zoWgY +eWalpS+9v1Z1pOUart80c5yprb8oxRHPX9r6ta3+/NXkKz2RUFvb0jv9Nv1njHrYu832jsR6OlQn +UENJKgLNDFWKoUmLQM9DK6n+2uBDjZXU5/eNvrjIzrENYl3/gkmrrxMp+HydNdeYH4+fmWbS0kvE +P4LgpxUpFwhW0sKSYrFagWC10mKx2nGDNTcEW5fN7SQ+2S9srq/bj3j92vOfHGvjm1ZXeO9K9Spw +ZnrlbdKhd+Fj6zFsaZ60HoxNjXJZOdQ+Sle3KylpWax2/Go6VJt1cH2atiBz+JMcik2NpKpiT1Uc +6/VpPJ7ZPq+XWjy/8XMUjipGb8R5SSyxfmqez2sa8AHZU7EsW9GwO7b0hTSP4NjmS+v4J5WLw2I1 +5FHYFDd0G7TNdaW48WtV0jcdS1yJ0aElvcG+z3yM/V6/qn1+nPQ+NLLFYnnEP2o8qTPBD7m/WCzr +ePrrSC2e9C1mORb1eCrYYitUapSnzSbb0kghj0bHr9kGPdvg9KXF/GJ7bGmVlIYwpdhzdEm1bRTn +1i7prPJh8uWjPDpHlthVjqZxH5+Gkli/pZn7+DQSs1j9zIdYGsE+U9O4KJ7rxMa/JNb4H/fReYlj +7I+iPNSfleOjztKyKBarJ7b5I+GzElsSa+ZkJ/naEPiwtFo6+Ky09NL2PSut957V7z0sOfo9LDEN +ibRH8ZudlRS/2Vnw3VtHWYk18zrB7tZJSuMcH3frLCsxC9bPO5M+PKnDrKT6QiJ1FolHR3ktOo+0 +wPlLJen4TpYlMY5WqbFg/XiVjqOk1ZhS65C4Tix2aBv0jvNUYlRKjHKTQnjC+t2Xt7J2sqOY2qo8 +s2991NbHqrSuxg+JtD6QuiTVs0kJCvow4+nz65KYhMVsMXXUdiQSWYnpt1n1ZD7uicWKkpjLxxhD +jX0uFr/JOsHtocuukdhoUtORSMR9XJRKi35H66vXY/r4TE370+51yajUpOYhlL6ueLViSWns92nY +kZSOEkPN/fhgqrGSUqsZ1R6N0ztaX5JqnzQ+xnS0Xh1/fPfVJanVzO2rS3K021f/1SY1ElMaOrZL +3Wy/ttosiqHPbFW2mpjKsEihVi219uVs8rUKea0qllYaq5IeDd07vVLGUEyv1Ep8eqWyTa90xTa9 +0oa/dxw6iZM+UtNNZpufnxS+7/joQ0mqJaMw3R5+NPFrMs+ppKVtvRPC9bGSo2Vlj0dBpWPaaFzp +VKwvCod0pkfXydMO6Vwo5k+MfXbED07GnFiuzW8ui6N5j/YkZhNTTPUzH0PPR+h5qCUNz8ltOtvE +WTaEW30c85JUWiyWxApbT5U1qcTS0tapa6HO/J7NwUatawl1k5ud0vTrwz8fIpVl8fMlvS7bddkS +Nigdb4EyLWuRF4/lIrKI6Zajg86CS3nSWmxSqbknZFRkFKdfJDb9AorYFpH2ndQ0h/pra0meLC0s +qY7iR7bu9Pfx0TjOm1LBmMJsR3ni+MQT2SYyahyO5MkokmB143RHQlfiPsLpVqJYOJvNUrKilCSm +rX/UIkTiIBLZrM9fL3EkDE/qSNynpq9L3pFgiUuehE+TDUqiJ5E0SlrSInnJSqvdk0iatSRLjbiP +aUkqVrcfqc4j7uPSePrH5yLuo6SvRZSkRu7jfWdphK1Hq3OrMHIfV2LdIvdxSaujjKt0nBmUZOlu +dh/dzdl9zM/mj8y88P2ZG2lZLH/NYveRWVjHt9nG062luDmO5+upWCoaaUnMPl7acR43S/0bhN1H +h2yOHystveZt2G3Gtb12qK13cpmUju9Kqxl070qqoXsVK6FMqZle9zGpBOmF87qP7mNautqP131U +SVmJdUXFe/66j8rt81slfc8vc2+7q903J9e6jw0TXftw6z4qLNKs+7RaThGpKzWr/cZc3cfqPibV +sWFLn1ndR3U/VvfR4WSePilU3Ue25FNX3+bV32v/qvvYXn1Ir1XdR9W4yumVpsfTYnTX8SWSPg9V +8c6J85h2a3KyJa2W031MSse5mL+f7iODMfRMhp3J9HXY/Ih9OtN9ZExSjSM+3cf1msVz0nfCOo7v +YJ/TfVyWm+Pj3CZrDr0TudjjVb4q3cdHjhVSOaX7+EzJWuJIBWXC2GeTDL/YZILRfVzKi+5jN3ch +UYKCcTYkFMMEwy/+Lklf8lhlk3iJEha3YgwxTDCizST8afg6ZsBJn6uAYYLxrsOlpMVgPPNvZjxg +4hjQQcB4gFIWa4CABChIAJBLaMIVDJCAPeChB1BgeCHKArQGKQMWFOAHRVQAV4EJcBjDCgSQb9CA +J7BACUN0wT4koIRJDeRQQgEcqGJQwhcmUCGEJiQ5cAANJWACHAhRhHKAH2hQA4kP+mCGMSyhCB0E +oAU3QEEF3EAGH6jgBk8wQQQOUiCELsQuLGAKOHCCEqZgAxkYQAovcEEgOsAEoQLqDM2DBeTgDhBY +wBBgUCIGU6AAEFxghBbgocQQmmCG6IUw8CAEgKAB88IBvgcIIMKDAkGwwhFUaMMc9BAHHxwBDAeo +wx8cwAYtdCEIR8jA4w1HMOADYsDCDsRghz9goAQfpGABCCrsYRBRwMMYkkAHIRShCRBEgmGFNHAh +BXnQgBRGAAYLOA3MYCGCAGUwYgBqWAISIgtYQBARcFsoRBmIKMAi4ogqkIEGASjCBw5QVGMpalHK +Ax5IwQpMkBiI74WCqBU98qige8HU9efJF7oXjwvr05+WzV5cyL9Ylk4hqfULxtpEuZAsSiJh8COJ +wq6t1FYFqaw/XyIdRILTYp1UWpC++VgqUVJD26ibOETSUAopXVGKBmMaTz3J8WjYpCS1UMOjTKfT +aNzScE+Fwh4VjfVIj0d0PF9RHTl0zkTKDJnF6scK61wfKFtThx+xp9Fqv5lKGd33uLY5uQrrxuFI +Gqa8OZ4KhlUoaeHn4QMfKJGR5crJnThEKfck7kmUZhomUYt7c4l5KMWmpVlJWSi9NEziNJwOec15 +jTgs3sda8z3RarzXiTTiODTitialnLjXo9uPQO+hiD8Rdf1RWx+2ziOqQvCzJAbBD8MPcR5hWFJg +o7Hhe4Y1SMyK44g1knLsWApLKinHu5FDTrjMkTpNRus68bHyo2bJj1zDN8GP5ruebmn0SIERi0uk +JNbxcVbtbn5EaM2r/sXMS3pA7sP8FKbYBVarWXcf28G4sdRiS7GGT63D3IKMezwVjlWRo4bCYTsY +K0ZR/NG21w7GFzOlFevrXtxMVtxIC25IHCu8erFYHatjJpt9PDp2ECtptW1YYRhHd3IVhjVcUWsc +1aQsCvNlK6l+3zVj2zC2kuJmb18Hui2nWJKS4kb6mHqhxL0qd+s0i97HpNY2r7yTSsac47D2VCir +giWXLxu6bdBdOkJhg66SVEPr13ok/d71rvlVTrAIL4P3vnvbrRnfbLO9T+e00G2QVSqx5IDNFQtf +UbNMIy2pSUtb22CzDU7R4p3NPtFd158mn0WPm23xOewGp2CabrD1m6PaPn+eXml/xk98vD/758Yh +sfWf24du8aVQu5sj70nUQ7ntiZ7oX31yfZo+th9JUvSkxXofdgSKoHud+Uio2t/XCG+99GYHHZ2l +S+hcoL/ZoOn4XYpmuY/RZsctVun4bb4nnvl1gpkmBf77SEu3ax3sek7h9gI/869tdVIn8bU5qcyj +t5DpxRJ7jXYklhSk0rHmTUxSWF09iXVBXjbb6qOVxCCcG53wGXvcr01BODtwdjgS8/jQJSmOlVR2 +puPbnLX0hAaGh6hjZZJYJKhJ55xCUzKTJIUOQxFAIBh4NBgQS+azcv0TgIBPZ6EwEAgEgUAwEAgI +iwOBpKdJp8yMyMhIkiRpDgJcusT9s14aGND1FyeAroT/jHLy/FkU9ISDEc0AGo0p5v5juqzk+kwP +2acRtqnemOkKFw5wCd0NAHRFNy9XDD9tcsju2OhivxzguXCL4j+ezhE9aEg8iViorSAY3+Je0bSd +TiOGpS01bJ4aR/4TMlPQGILyGUymxp9/0xB4kFvhEaVq2JBYm/slzDMKHHEz5Mof9RQEKmer+HP7 +1dEPXeqzI5KBZGQcbyqfchzSyfaSQ0h0c55VQSeCGiARYY1fL6KEUSw1cjrvIs6aQMGr3bg2Umw1 +fh2k9EyGUJlyJFn+9Kn8y1yJEbF/GFB8ZKY1DHQ90qywJKjswwd+A/uMNnUAXq2PW8maWHy8vRfH +LdZxf/PNQ39z2pI62OiYbyllpNAQgrQEfxt+qQ2jGkeBWkzxLTyr5eqjef9YqYU5bJ/5658dnzih +p0/gxJ2TrUXhkS4Z6tCF+0Eof4h8GyMl7jS7E9qwAURJ6bNgGgsshimEG7nPjRrlhBSyBSizEmby +we3Il620aUv0cYbkJp9AmFaWSo3aVCD46a4aCz7pVxV3SMdRkQWRvvDLoP0qskAlE4hzK9KrGhdP +c3jgKatzYHCBdser1C38emMX432ez7Azv110Jp5JzYTmd/rsyQ7rJQxZ7GPz+ytieX/LTPthOtbY +A7i/1kJb6COIhB2uYboKQttx2SBXZVIureYtcs/CLrkxncLgIsop7SJkbbvZp4u0RwVPPW5rafSg +z1t4nK6ReyjyEPMWqdwmxWI65H5txrXb7VznfncWFl2DAaQCwwD+C2Nn+PZOjSdaCXpWCdLyNBPE +M6VH36s+6NSNYl3mRIhiFbuHEJlV60UuzWxVvWDrqlF5fYr1Ip28GGyIVES3MzMJ4crhFRv4DC7Q +ZNsVLjTnN36BUlichl8+kW+LgmxTQvsdpv4XgW7Qkcubzq5Lv7E9jErupEobcYc7/GlFG1FLublK +xxCdLaHqv9XgPHpa1Oi8Ej8iOhCoSrBpJf/IOReKBaAf/Axq1pIsdtvGJVJkiVaqOLtYiBxwDTm1 +AeoxnyDEKOMSWPOBlmz2c9bkiZEIXO8mDdrNGfxjKtTqQhZZGXf/pON1wOq5Fcx/DQhY+FPJBXOd +rH4YqD5kPGeok363cFWv5QhjSQ57AsQSH5BqxTQWPfj1Q9ooiS7mR5mtkAWG1KhjeQTFatnhSJha +R6xzIoPpiRDPeZ6gNJ377YZ7N9JFptRrD5W2gUxdx/KQM6OmMeaLa+uV7LcD5Kl1mI0qXKGMl9TG +WOC0pluFE+MZa7gqkQuKFsFEyU5XiXZolpaoJINfvv1R1TxJ67odjluGn4etrTYxTrGrKwW6G6xO +0ZgatGAUokTKMSJMhNZ6IOJ/EaAbkS3R+DKjjrIPpbIeOX4pBxf0MzJijZGA9llrlssNzd1l3MK2 +K2pV0BeNrRULam+jS9uUzpNDmVvC9lLJmT9irMAjfiOk5abAVVDPo8alvXjgLCmT/vmbXKUXula0 +YPJx+YARU9YtSnf0odPiKw+dZT6aBBEf4VqvIksQEReLDL80EW3J6ccXJxcCQ0BoHazkBlvaHy7g +x6hEC2NPCt40ZxbOVay5n85gmzEux0KaU1VDczJDT/qkPZAgk52d93Cuf/FgqIZqWB+gKyygtio4 +eYY4gBnsAokZzZhg81A3edpYfbThA1LefAhYrvjD660dv8x1d8QaYmJQtgCVVbNSxXz0VOOpHuHg +V5JYJ0r/lkTZK23s8nF9RA8yz8VFJ9dulzwiBRNOctNcB52SqeDQP0l1KcKn65/CpERU/9x12ZeI +2Iy7Pjiwj5oXbZO0p2DbABmL4cr1WRFrXYoATyNdVnQ+J5k5wKKa9o1ueC/BaRioWL9qjjX162cx +ebNKBVpjJ0ca1B5RudwWS9F6GRWFa6NK7K9l22NTXdTdHcuWB37sLAUcMcmPBFCmveF+XQjdS+py +vWS07r3e0u1bnm/6Nk+5mqVvUclhcb81RIVSZKRU0oRzG5t76BDAe+bgQ7/ZxlcSTGBxcnAPbY9C +SOLq0aqCS8ugIGoBbxAhwrUTH7zys4VqqQad/0vu9hn7bxiuoID0ynye8S757W+7ZOxc+or/G9u1 +S+ZLU8oCWS8ATPMLAHbMQWDAzE9CU2kcal6pYSIyktFC3Oh2WSl7qNXqk74AYmVJ+KuRYHGRFl6G +KEvVgnKwryykWH5uSt+zOcbhMZLjdcpLZBiQ9igz6YjhBbJojCAxhx6l0ZHBTXg6t2G4xnSX7/9M +I+lYZHDS4F4rxZqciRFpPsH2027DaQQkVRWWXI1bEUWbb/vbDpqxoQppl9wTchXCLWU//iOnCASt +ZsafSo27iZAAy/L5x0v1o+IULb+MwoXMpuCZJ/0AbJ0IAJ8vcJhHxFaSSGu+h3/F+guGCTgspMsx +tSBlJfbcj2CgWYc2fHku3CzUzaUhSEIo1EoQicveXY/g5QfkZcMvIrYMdGGwc54aR388U3pxm69q +8wbYpwOw2mKJzYVVMor6jcce6iz+L6+R8pOERR1Sw5DPtuF2LSRqIUQ/FCOm9n+hiH15e0wH+kT+ +BStROad9LA7iIQvQglm14FR/LlD7jJ3KiPAgWgGQxV62+A9ZEZZO6vis7XBRQ+8KbCFIkwgMgn5Y +9hrQfCYd84/htXcyxzjmDrt6qHCGwX0UC+AuoEU8M44utT5r08GeA/TYYl23ju7+rgZhti7lAk7H +H3BzkVQP4uMHm6DBiwIq7Tx3A8IAwEsKPKNVbwxUl9OZaKOIL5e5fbnESiGFy1uFyAd6FRjWHt8f +GmBz+fr9+U6f4oyOMv55qeZCUDedhN90Aru8gCcEkJQRgRNnm/lvpJA/B3r9sv56nvjXCCRpakNr +9U60kLKa2zxbqGLJVuYyEfrW15fPH5a0c21aXWH0X35Aui9ysKPXf7ebFes5wPc4fxCoCGsl07WB +SiMDS9W2GlofzHaq6xi8wy1lHdwS8uXqie6LgbL57Wj8YvxCeQIKM1empG2S6Tcxl1AorZFBOLjG +RyHzL9Jio2M0HO9GxJMDMGabB4FfWoVRvQaY+PXdH8Say9ORXnP1hVyIHR6XoVCNA4PuKWHLGyft +dn/M5Gg+bN1qiIt/oaYcaqpCMKWbW3LyOpZd9xFt3lHRnde6lNvBvIF77r/sH/+9h1t1EYQPc3BV +eDipAOV7j/a7LPKmpyDmrqPQskUnGAGER2mm6YUAhaWcfLF6kdHtRmG94gcj5wnagcQuOBzXJxdP +75CDcVTtkfxbIK+tdHJOustkYqb0h5bS159nOrcBuB3qmEgJ+9BPX7qaXRIL7CLveHkbrcToGOEs +qO49gStJPUBFiziG8gPFAiQh1m1XjT4p3S0SwJdgRgeOKaFpTSQKXvFTckzekD/IfS4DNZ/2Q3Ip +E5wvAdg3SrT9GewONV8XQNZd8s+zviEBTkduYDVVMTK3ij/HxDEaexiEDw296lZP3eff9s1lD1JS +WCz8yXJtItq9KqrNxRiqOFgx7KC1W6zNNfYSajePX+1xS5alG0w4s/SxQQM8dlDqdYvxHpyaGa0+ ++/asSYpR7AiNsptScwngPqQKNAYZFZoyWS2ZKn+pez00UJ0BvEhnvL/D/xhYM7R6HuHoNwtm2KpO +FndAgQ/BQXuMQ75Edy1WZ2qde5vX7zDjLUuE5JELzr28UMV9breaHB3oiJ00owAAsRvuEZsc8Yfc +eezTncXx7pM7XwhTzw91DLqyi07JxJ7iskXUrPdT2rGe+RDmbX1rupoXH9Yh+ZHEW+0kjctm8sRI +kCPQe+iv/MqqCOh83lC2qqCPCQ7Aw2e0GhZRQP4zc1bzbiV/UBgrgd5VmRYNagJxjDd0fxeVPw1S +NUCQHfZT8fHRUbJv9BtayVKRgFW6spTNyeD55ljBgdMkgd5p/1FDjWcAiD6rtzYLBa36P2zfYCTI +fgCfmHHScMDNqytArjSBK42Ke8WwFWYDsCFsBR1P1F5y+tD+CnTyG+RY1NDAM/J/NWsAFJQagEEW +HUABg+ZAr14Rp8jRkEiW3AEG93vCGQNbFJ4BMhgz8I0ilOdyoddqfkPDdKpQLdRdIivbal8iNeQK +Lrw5Y5oCyBmC0SJU2U3YYxleWl/P+7dIcC+XXRI5GOsHBZv3+QWX+x2dRikN8Mic8/XMg9NEghoE +Xp15d8G2zjTn70JuDKfMLqh4SWzMPL/9/rI2QVLrkvjdWQmSv6ueE51BJniie5+fluU6qTss1NUS +rVStQiF24u0YENzfuYGHPCWRhemCSMi4macv/vhXarEuQvL4aMOEm2hpXJh++7ciVWdB5lwBeB0m +/z/QHWZmWjDGzTxSCYG0z0XmxoRM68J8f+RvjWhEbWHpnIqgJXa+fkOAFbZDiRE6kB1v4PguJC4J +DsSc37Wi6Ypxnz5hCbfqWOHiJar6kgZnIoyFb72kBzAdFYFAIpPesj0DcdhbGaxj1Lt2LIJUB/lG +nnov9Bz6UXRYCHRQ2gLpDme0h84NVWVKhMNFpJ+PEvjbA0jm23mloKsgB4fTqn6R5BDr+yFAksaZ +GPUVvnUYJ9X/j2u7M+YqnFrjzNDQ2/N58uTS9+rp7v2OlwvYxZha0qU8pnbm5lgzSKNU1Ejwg/eY +TgRqM7xs1oBCazudkIaI5WwxEnBL4Qg0EO5qdt6EtpXW6JCySC81ZQaGmq9o3GvePLXpNtFYVf+I +olU2raqmm8DkUmFiew/oi1cqz+t81OqxElgAA27T5+ov9fKEZQvzuZULA1NmCohiwkBLApCRl7Nm +J4L/B/IyP/PjvDypEaYApCNo+87CGWz3HJQ+HggKF1Egp8ReFDA6ILJrPbeU447Mv+VF51Y/4jgP +4X0LTLzcylwNkgVuNlo5dhzwWLp4tidsyZVySXFT9QRem4c8U/06L6yZgSrbZTLwa8CvhL0OR+Kj ++p6xbpzw0yB4oxlzfO88IP5agCsmddtiCuUvZmuJt9hsw2u39MDNT8llweMIR6JRBBA2/X0jALID +JUJH9Zx3izdPFhG88yFhWBJphUoUkvNk9fCF7+Q245O5j/ly685TOl8aXDqPocsUTBtCnSTPoBVk +huxBMR/HT8MIJUu7R5JtbE8Us/jB6RMq9soNtS2dsa1zp1EioRc9Q6JpGUk1zqXo83SWDrv3aRLz +Z7iHQnMCC/WdkfhSBbz6zeI5fE1xw9Ggy0t1/llyJMnWsH6UtuEZxAV08ebBU22/He1phYZRMOs6 +Fcce/orktqjwOBmhCOGqgBdsfXHWl6Um9equDUBwu9InIWLSvdm21F3Zn1q2bG5SNjMcLcm66OQm +vlMbEG2m9FNcbmnwA8CPWIMiEk8oKRgb6lq/p6l+mCU2L0yprQmdwLBZb0tgoDnomG8qVPwFGDgm +frhu9+7jlXWMQxhWVD07YeV0TFLwmQMnrN+iFWi73im8WI/8aw7/TpecBn04UWkfOVdZYhEsMBeN +Mxwc6lZ5GNwGTos8crqp6pENRsEZTlHzgb6COQ7kWeSWuegyOhEVd/wBL13/tHxsg1ITuFAEYtsR +/IgErmiCZUiA/kxgRclZEIdxYnkl791s3+1QfqOVKnCxsNFhtq8bEqQXVwKXVMoLgtABa3pj59Pu +vr+IaHkXRTJq3F/slEXDVATl77ij` + secondPackageData, err := base64.StdEncoding.DecodeString(secondPackage) + assert.NoError(t, err) + _ = secondPackageData + + const secondPackageSignature = `iQGzBAABCAAdFiEEQ6WNuVKuWHQJqhdSETT46/mKoG8FAmSsBg8ACgkQETT46/mKoG/t4wv8D6jQ +uNLj1vvVUlt3cXJ8dulp1TuR0KQrGxqPm/fhCNoEncffR1+1ehMjd69XlsA5XcGxY55QGjKK/W5z +LOZDXatNLTi6PRx1ApbbA7gCCzx/Tx0gF0sCe75Wu286j9ud8xFx8/ray8yS9iV6K6XpsOZUnVEw +yWC6DKsGb2Tv2VvRkj3yNo9ZdTlJOpmLX/OHjAOlFCTxJpa3Nf4YDc5V2d4xYzG0eeCasJueNHYz +d29ZdB58jZQj4ZUuv2Ln/fc2kmmBA4PIZaoTkc9kFwdYFIQV+c1OM6kTPnt5GJffNg7EgCVyv6ON +TJM9ud9iHaxwz+keBa/yQqEq4PFLNT0+IiCmidTpZI9iFVByqbXNuoKti0bdJ2pjnKRcSYWVV1AO +V8m4OwfXaTgsNr41SW9hZJnPUknJFACgc2LVJtJNVFwfb9iyRMdPCUD0NZ/6z28XL11vcNFbJldG +V2DfzGUB63cabGvZKvYMynHjfaxo8MwKVjUzgvWqMfNL2P5QpQ6S` + secondPackageSignatureData, err := base64.StdEncoding.DecodeString(secondPackageSignature) + assert.NoError(t, err) + _ = secondPackageSignatureData + + const secondPackageMetadataSignature = `iQGzBAABCAAdFiEEQ6WNuVKuWHQJqhdSETT46/mKoG8FAmSsMPoACgkQETT46/mKoG/GCQv9Hr4T +DYBG2KpgzEeC3vwy4YvHbrYc+TwZWeRRS5xgpk+mG6xJqRitHhi8K8quP0Z4/PChr8yZE2ZHgKOv +2Oul8fYA9z6xAmGP9vNSVjUJM4T0zWMB9uO6g7ZWh62BTTVoWWA19PRQxlcRg6uonso5ZfXpYbit +RpfNSKRXqo3KgrLNpx1neXMoFZb3fuUW4Nc69qdjpCcJ/UrEVfyN/SD719JrIn9WQiidud0sBtA0 +5F/DF456UXgpDSb+LDSkGMlWcHBz8VoTs3XMW+w50dC03IJI4Yji+TLMrjrUNVNeR50D9siAjeRr +ihGI33HB0kTQv7kw6iG+H/me1bI/pI20slA4KabMEmoiyO6a4XCwTObl6/mpcTvpDQqLgvmLHgEQ ++oKANt0BiOrtDlLhGBdIM9NXkVfWQGF0MwZVTn/agH/qB95JNNT8DH1+beEDOna9HjImyOtp9Z13 +HkZaMSkzwZmNAopA9pYpm0dIBquFCxaX+hKkoeLma2JMtCOHaV4Y` + secondPackageMetadataSignatureData, err := base64.StdEncoding.DecodeString(secondPackageMetadataSignature) + assert.NoError(t, err) + _ = secondPackageMetadataSignatureData + + const secondPackageDatabase = `H4sIAAAAAAAA/+ySQY/TMBCFe55fkcscm47teJwghAhNyka0C2ooBy7Icbxt1U02arJSl1+P2kWI +7RVVCNHvMh7rSfZ7eoPvh263HouxmNS+d6MLQERkjDlNIjqfxMS/zqd7IUlFo4Au8ZlzHvvB7kf0 +x2+dm/tHwFkxz2/TRY7wWxXGh5i/cRR2u3U42H34vR8A8IUMAN+l5Yv9S74si4+3CGIsADDLyynC +bLvvh+AoCjrrdnbtAXBaFl9zhJiFBsDieSMAXGS6XC0Q2Iu6TuJYGR3F5EnYiqmWTlinOXIOAMub +VGo+qSshaxaRkbGS1rKOWGvvjRcVMTtVaaOk0hUrK8kLWSWeE+fMXU3spLU1AK6Wc4TNMHT9q8nE +H2zT3fvQPTSTn/aOhtLl9AbhORqw7dMxglUxz7L0c44gOI6TRCvFAPgpnX5I3+dLhMy223sbzB6a +bfu02wSva9u6jV8n5u1d0x7C/vENwN9uwZUrV/5HfgQAAP//SZpudwAIAAA=` + secondPackageDatabaseData, err := base64.StdEncoding.DecodeString(secondPackageDatabase) + assert.NoError(t, err) + _ = secondPackageDatabaseData assert.NoError(t, unittest.PrepareTestDatabase()) setting.Packages.LimitSizeArch = 99999999999999 user := &user_model.User{ - Name: "dancheg97", - Email: "dancheg97@fmnx.su", - Passwd: "password", - IsAdmin: false, - Theme: setting.UI.DefaultTheme, - MustChangePassword: false, + Name: "dancheg97", + Email: "dancheg97@fmnx.su", + Passwd: "password", } - err := user_model.CreateUser(user) + err = user_model.CreateUser(user) assert.NoError(t, err) session := loginUser(t, "dancheg97") @@ -271,87 +466,111 @@ ht719b7ZWR3+SRcXySXC/cP8DL/N12kaf8wQSBkjjLKkAPBDnLyL32YFQur67qtbXtxcd/23w375 }) MakeRequest(t, req, http.StatusCreated) - pkgData, err := base64.StdEncoding.DecodeString(pkg) - assert.NoError(t, err) + rootURL := fmt.Sprintf("/api/packages/%s/arch", user.Name) - sigData, err := base64.StdEncoding.DecodeString(signature) + wayback, err := time.Parse(time.RFC3339, "2023-07-10T19:25:28+03:00") assert.NoError(t, err) - mdSigData, err := base64.StdEncoding.DecodeString(mdsign) - assert.NoError(t, err) + t.Run("PushFirstPackage", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() - dbData, err := base64.StdEncoding.DecodeString(db) - assert.NoError(t, err) + patch := monkey.Patch(time.Now, func() time.Time { return wayback }) - rootURL := fmt.Sprintf("/api/packages/%s/arch", user.Name) + req := NewRequest(t, "PUT", path.Join(rootURL, "/push")) - wayback, err := time.Parse(time.RFC3339, "2023-07-04T19:57:09+03:00") - assert.NoError(t, err) + req.Header.Set("filename", "testpkg-1-1-x86_64.pkg.tar.zst") + req.Header.Set("email", "dancheg97@fmnx.su") + req.Header.Set("time", "2023-07-10T19:25:28+03:00") + req.Header.Set("distro", "archlinux") + req.Header.Set("metasign", hex.EncodeToString(firstPackageMetadataSignatureData)) + req.Header.Set("pkgsign", hex.EncodeToString(firstPackageSignatureData)) + req.Header.Set("Content-Type", "application/octet-stream") + req.Body = io.NopCloser(bytes.NewReader(firstPackageData)) + + MakeRequest(t, req, http.StatusOK) + + patch.Unpatch() + }) + + t.Run("GetFirstPackage", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + req := NewRequest(t, "GET", path.Join(rootURL, "/archlinux/x86_64/testpkg-1-1-x86_64.pkg.tar.zst")) + + resp := MakeRequest(t, req, http.StatusOK) + + assert.Equal(t, resp.Body.Bytes(), firstPackageData) + }) - t.Run("Push", func(t *testing.T) { + t.Run("GetFirstDatabase", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + dbname := fmt.Sprintf("%s.%s.db", user.Name, setting.Domain) + + req := NewRequest(t, "GET", path.Join(rootURL, "/archlinux/x86_64/"+dbname)) + + resp := MakeRequest(t, req, http.StatusOK) + + assert.Equal(t, resp.Body.Bytes(), firstPackageDatabaseData) + }) + + t.Run("PushSecond", func(t *testing.T) { defer tests.PrintCurrentTest(t)() patch := monkey.Patch(time.Now, func() time.Time { return wayback }) req := NewRequest(t, "PUT", path.Join(rootURL, "/push")) - req.Header.Set("filename", "randpkg-1-1-x86_64.pkg.tar.zst") + req.Header.Set("filename", "testpkg-1-1-any.pkg.tar.zst") req.Header.Set("email", "dancheg97@fmnx.su") - req.Header.Set("time", "2023-07-04T19:57:09+03:00") + req.Header.Set("time", "2023-07-10T19:25:30+03:00") req.Header.Set("distro", "archlinux") - req.Header.Set("metasign", hex.EncodeToString(mdSigData)) - req.Header.Set("pkgsign", hex.EncodeToString(sigData)) + req.Header.Set("metasign", hex.EncodeToString(secondPackageMetadataSignatureData)) + req.Header.Set("pkgsign", hex.EncodeToString(secondPackageSignatureData)) req.Header.Set("Content-Type", "application/octet-stream") - req.Body = io.NopCloser(bytes.NewReader(pkgData)) + req.Body = io.NopCloser(bytes.NewReader(secondPackageData)) MakeRequest(t, req, http.StatusOK) patch.Unpatch() }) - t.Run("Get", func(t *testing.T) { + t.Run("GetSecondPackage", func(t *testing.T) { defer tests.PrintCurrentTest(t)() - t.Run("Package", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - - req := NewRequest(t, "GET", path.Join(rootURL, "/archlinux/x86_64/randpkg-1-1-x86_64.pkg.tar.zst")) - - req.Body = io.NopCloser(bytes.NewReader(pkgData)) - resp := MakeRequest(t, req, http.StatusOK) + req := NewRequest(t, "GET", path.Join(rootURL, "/archlinux/any/testpkg-1-1-any.pkg.tar.zst")) - assert.Equal(t, resp.Body.Bytes(), pkgData) - }) + resp := MakeRequest(t, req, http.StatusOK) - t.Run("Database", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + assert.Equal(t, resp.Body.Bytes(), secondPackageData) + }) - dbname := fmt.Sprintf("%s.%s.db", user.Name, setting.Domain) + t.Run("GetSecondDatabase", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() - req := NewRequest(t, "GET", path.Join(rootURL, "/archlinux/x86_64/"+dbname)) + dbname := fmt.Sprintf("%s.%s.db.tar.gz", user.Name, setting.Domain) - req.Body = io.NopCloser(bytes.NewReader(pkgData)) + req := NewRequest(t, "GET", path.Join(rootURL, "/archlinux/x86_64/"+dbname)) - resp := MakeRequest(t, req, http.StatusOK) + resp := MakeRequest(t, req, http.StatusOK) - assert.Equal(t, resp.Body.Bytes(), dbData) - }) + assert.Equal(t, resp.Body.Bytes(), secondPackageDatabaseData) }) - t.Run("Remove", func(t *testing.T) { + t.Run("RemoveFirst", func(t *testing.T) { defer tests.PrintCurrentTest(t)() patch := monkey.Patch(time.Now, func() time.Time { return wayback }) - req := NewRequest(t, "PUT", path.Join(rootURL, "/push")) + req := NewRequest(t, "DELETE", path.Join(rootURL, "/remove")) req.Header.Set("username", "dancheg97") req.Header.Set("email", "dancheg97@fmnx.su") - req.Header.Set("target", "randpkg") - req.Header.Set("time", "2023-07-04T19:57:09+03:00") + req.Header.Set("target", "testpkg") + req.Header.Set("time", "2023-07-10T19:25:28+03:00") req.Header.Set("version", "1-1") req.Header.Set("Content-Type", "application/octet-stream") - req.Body = io.NopCloser(bytes.NewReader(mdSigData)) + req.Body = io.NopCloser(bytes.NewReader(firstPackageMetadataSignatureData)) MakeRequest(t, req, http.StatusOK) From 819518560707bbb932129f77451759f9fe592e05 Mon Sep 17 00:00:00 2001 From: dancheg97 Date: Mon, 10 Jul 2023 20:50:44 +0300 Subject: [PATCH 033/124] lint corrections --- modules/packages/arch/metadata.go | 4 ++-- services/packages/arch/verificator.go | 1 + templates/package/content/arch.tmpl | 2 +- tests/integration/api_packages_arch_test.go | 3 ++- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/modules/packages/arch/metadata.go b/modules/packages/arch/metadata.go index ab7148788dde..4e3606bf663a 100644 --- a/modules/packages/arch/metadata.go +++ b/modules/packages/arch/metadata.go @@ -54,7 +54,7 @@ func EjectMetadata(filename, distribution, domain string, pkg []byte) (*Metadata if err != nil { return nil, err } - var md = Metadata{ + md := Metadata{ Filename: filename, BaseDomain: domain, CompressedSize: int64(len(pkg)), @@ -108,7 +108,7 @@ func EjectMetadata(filename, distribution, domain string, pkg []byte) (*Metadata case "checkdepend": md.CheckDepends = append(md.CheckDepends, splt[1]) case "backup": - md.Depends = append(md.Backup, splt[1]) + md.Backup = append(md.Backup, splt[1]) } } return &md, nil diff --git a/services/packages/arch/verificator.go b/services/packages/arch/verificator.go index 31e072a644cd..dfea40ae7a0c 100644 --- a/services/packages/arch/verificator.go +++ b/services/packages/arch/verificator.go @@ -14,6 +14,7 @@ import ( org "code.gitea.io/gitea/models/organization" "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/context" + "github.com/keybase/go-crypto/openpgp" ) diff --git a/templates/package/content/arch.tmpl b/templates/package/content/arch.tmpl index 8483898d1e40..3d108f26b4ff 100644 --- a/templates/package/content/arch.tmpl +++ b/templates/package/content/arch.tmpl @@ -48,7 +48,7 @@ Server = https://{{.PackageDescriptor.Metadata.BaseDomain}}/api/packages/{{.Pack - + {{range $key := .PackageDescriptor.Metadata.Provides}} diff --git a/tests/integration/api_packages_arch_test.go b/tests/integration/api_packages_arch_test.go index c031c3c83ce1..717e86ed8feb 100644 --- a/tests/integration/api_packages_arch_test.go +++ b/tests/integration/api_packages_arch_test.go @@ -14,13 +14,14 @@ import ( "testing" "time" - "bou.ke/monkey" auth_model "code.gitea.io/gitea/models/auth" "code.gitea.io/gitea/models/unittest" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/setting" api "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/tests" + + "bou.ke/monkey" "github.com/stretchr/testify/assert" ) From f3841f0777ebe2a29984319d5c5f9362d85209d2 Mon Sep 17 00:00:00 2001 From: dancheg97 Date: Wed, 12 Jul 2023 11:41:20 +0300 Subject: [PATCH 034/124] fixed response status code for metadata update operation --- routers/api/packages/arch/arch.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routers/api/packages/arch/arch.go b/routers/api/packages/arch/arch.go index f52fcd27aaa7..ad051d0396e0 100644 --- a/routers/api/packages/arch/arch.go +++ b/routers/api/packages/arch/arch.go @@ -129,7 +129,7 @@ func Push(ctx *context.Context) { Md: md, }) if err != nil { - apiError(ctx, http.StatusUnauthorized, err) + apiError(ctx, http.StatusInternalServerError, err) return } From c4f3fb302bf7949e9de932e58e6b7851773d6200 Mon Sep 17 00:00:00 2001 From: dancheg97 Date: Wed, 12 Jul 2023 13:32:38 +0300 Subject: [PATCH 035/124] markdown and code documentation lint corrections --- docs/content/doc/usage/packages/arch.en-us.md | 10 +++++----- modules/packages/arch/metadata.go | 6 +++--- services/packages/arch/verificator.go | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/content/doc/usage/packages/arch.en-us.md b/docs/content/doc/usage/packages/arch.en-us.md index 4d7343f7dcde..b7853562f1da 100644 --- a/docs/content/doc/usage/packages/arch.en-us.md +++ b/docs/content/doc/usage/packages/arch.en-us.md @@ -58,27 +58,27 @@ mQENBGSYoJUBCADSJ6v8Egst/gNJVC2206o8JqTzRBxTULKm/DH5J7AzrhJBxC2/ ## Upload packages -1. Ensure, that your package have been signed with your gpg key (more about arch package signing)[https://wiki.archlinux.org/title/DeveloperWiki:Package_signing]. You can do that by running following command: +1. Ensure, that your package have been signed with your gpg key (more about arch [package signing](https://wiki.archlinux.org/title/DeveloperWiki:Package_signing)). You can do that by running following command: ```sh gpg --verify package-ver-1-x86_64.pkg.tar.zst.sig ``` -2. Sign message metadata, which consists of package owner (namespace in gitea), package file name and send time. You can do that by running following command: +1. Sign message metadata, which consists of package owner (namespace in gitea), package file name and send time. You can do that by running following command: ```sh echo -n {owner}{package}$(date --rfc-3339=seconds | tr " " T) >> md gpg --detach-sign md ``` -3. Decode message and metadata signatures to hex, by running following commands, save output somewhere. +1. Decode message and metadata signatures to hex, by running following commands, save output somewhere. ```sh xxd -p md.sig >> md.sig.hex xxd -p package-1-1-x86_64.pkg.tar.zst.sig >> pkg.sig.hex ``` -4. Paste your parameters and push package with [curl](https://curl.se/). Important, that time should be the same with metadata (signed md file), since this value is verified with GnuPG. +1. Paste your parameters and push package with [curl](https://curl.se/). Important, that time should be the same with metadata (signed md file), since this value is verified with GnuPG. ```sh curl -X PUT \ @@ -128,7 +128,7 @@ echo -n {owner}{package}$(date --rfc-3339=seconds | tr " " T) >> md gpg --detach-sign md ``` -2. Send delete message with [curl](https://curl.se/). Time should be the same with saved in `md` file. +1. Send delete message with [curl](https://curl.se/). Time should be the same with saved in `md` file. ```sh curl -X DELETE \ diff --git a/modules/packages/arch/metadata.go b/modules/packages/arch/metadata.go index 4e3606bf663a..6f553b8634c1 100644 --- a/modules/packages/arch/metadata.go +++ b/modules/packages/arch/metadata.go @@ -48,7 +48,7 @@ type Metadata struct { DistroArch []string `json:"distro-arch"` } -// Function that recieves arch package archive data and returns it's metadata. +// Function that receives arch package archive data and returns it's metadata. func EjectMetadata(filename, distribution, domain string, pkg []byte) (*Metadata, error) { pkginfo, err := getPkginfo(pkg) if err != nil { @@ -263,8 +263,8 @@ func writeToArchive(files map[string][]byte, buf io.Writer) error { return nil } -// This function can be used to create a list containing unique values from 2 -// passed arguements. The first is +// This function creates a list containing unique values formed of 2 passed +// slices. func UnifiedList(first, second []string) []string { unique := map[string]struct{}{} for _, v := range first { diff --git a/services/packages/arch/verificator.go b/services/packages/arch/verificator.go index dfea40ae7a0c..f9ac67fb8955 100644 --- a/services/packages/arch/verificator.go +++ b/services/packages/arch/verificator.go @@ -24,7 +24,7 @@ type IdentidyOwnerParameters struct { Email string } -// This function will find user related to provided email adress and check if +// This function will find user related to provided email address and check if // he is able to push packages to provided namespace (user/organization/or // empty namespace allowed for admin users). Function will return user making // operation, organization or user owning the package. From c20e1e0b99adf3d370f466a2a957410ad6ea3441 Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Sun, 16 Jul 2023 13:09:21 +0300 Subject: [PATCH 036/124] temporarily removed dependency for time mock in tests --- go.mod | 1 - 1 file changed, 1 deletion(-) diff --git a/go.mod b/go.mod index d6840529a106..885bb3422046 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,6 @@ module code.gitea.io/gitea go 1.20 require ( - bou.ke/monkey v1.0.2 code.gitea.io/actions-proto-go v0.3.0 code.gitea.io/gitea-vet v0.2.2 code.gitea.io/sdk/gitea v0.15.1 From baf3a03c2d1e3df1991128b5c99e7ce42adcf48f Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Sun, 16 Jul 2023 13:28:38 +0300 Subject: [PATCH 037/124] fixed dependency for time mocks in integration tests for arch linux packages --- go.mod | 1 + 1 file changed, 1 insertion(+) diff --git a/go.mod b/go.mod index 99b25a261943..70b20cec62af 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module code.gitea.io/gitea go 1.20 require ( + bou.ke/monkey v1.0.2 code.gitea.io/actions-proto-go v0.3.1 code.gitea.io/gitea-vet v0.2.2 code.gitea.io/sdk/gitea v0.15.1 From e24db369629ad345386ba8fba1bd44bad1110d3f Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Sun, 16 Jul 2023 21:15:06 +0300 Subject: [PATCH 038/124] fixed version order when creating pacman database for package with multiple versions --- services/packages/arch/db_manager.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/services/packages/arch/db_manager.go b/services/packages/arch/db_manager.go index 270519cc0130..c96dbaadd31e 100644 --- a/services/packages/arch/db_manager.go +++ b/services/packages/arch/db_manager.go @@ -7,6 +7,7 @@ import ( "bytes" "fmt" "io" + "sort" "strings" "code.gitea.io/gitea/models/db" @@ -166,13 +167,18 @@ func CreatePacmanDb(ctx *context.Context, owner, architecture, distro string) ([ var mds []*arch.Metadata for _, pkg := range pkgs { - vers, err := pkg_model.GetVersionsByPackageName(ctx, u.ID, pkg_model.TypeArch, pkg.Name) + versions, err := pkg_model.GetVersionsByPackageName(ctx, u.ID, pkg_model.TypeArch, pkg.Name) if err != nil { return nil, err } - for i := len(vers) - 1; i >= 0; i-- { + + sort.Slice(versions, func(i, j int) bool { + return versions[i].CreatedUnix > versions[j].CreatedUnix + }) + + for _, version := range versions { var md arch.Metadata - err = json.Unmarshal([]byte(vers[i].MetadataJSON), &md) + err = json.Unmarshal([]byte(version.MetadataJSON), &md) if err != nil { return nil, err } From 9e7c6ef5e9cc831f8c3045d261d4c3a2e0ae333e Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Mon, 17 Jul 2023 00:03:24 +0300 Subject: [PATCH 039/124] updated documentation about metadata update function --- routers/api/packages/arch/arch.go | 2 +- services/packages/arch/db_manager.go | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/routers/api/packages/arch/arch.go b/routers/api/packages/arch/arch.go index ad051d0396e0..2533c937acde 100644 --- a/routers/api/packages/arch/arch.go +++ b/routers/api/packages/arch/arch.go @@ -123,7 +123,7 @@ func Push(ctx *context.Context) { return } - // Add existing architectures and distros to current metadata. + // Add new architectures and distribution info to package version metadata. err = arch_service.UpdateMetadata(ctx, &arch_service.UpdateMetadataParameters{ User: user, Md: md, diff --git a/services/packages/arch/db_manager.go b/services/packages/arch/db_manager.go index c96dbaadd31e..f5c3360d87f6 100644 --- a/services/packages/arch/db_manager.go +++ b/services/packages/arch/db_manager.go @@ -27,8 +27,9 @@ type UpdateMetadataParameters struct { Md *arch.Metadata } -// This function parses incoming metadata, gets existing if present, combines -// architectures and creates new one with base parameters of new meta. +// This function gets existing package metadata for provided version present, +// combines architecture and distribution info and creates new metadata with +// combined set of parameters. func UpdateMetadata(ctx *context.Context, p *UpdateMetadataParameters) error { ver, err := pkg_model.GetVersionByNameAndVersion(ctx, p.User.ID, pkg_model.TypeArch, p.Md.Name, p.Md.Version) if err != nil { From 1ea4d48b39deb9d9aadb2c36871e902aa8ce3cc3 Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Mon, 17 Jul 2023 14:19:54 +0300 Subject: [PATCH 040/124] fixed package version owner in metadata update operation --- routers/api/packages/arch/arch.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routers/api/packages/arch/arch.go b/routers/api/packages/arch/arch.go index 2533c937acde..1596fe7061e4 100644 --- a/routers/api/packages/arch/arch.go +++ b/routers/api/packages/arch/arch.go @@ -125,7 +125,7 @@ func Push(ctx *context.Context) { // Add new architectures and distribution info to package version metadata. err = arch_service.UpdateMetadata(ctx, &arch_service.UpdateMetadataParameters{ - User: user, + User: org.AsUser(), Md: md, }) if err != nil { From faff9d9c57a0418e885daf7d47d59081c22937f1 Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Mon, 17 Jul 2023 20:30:36 +0300 Subject: [PATCH 041/124] getting storage object instead of reading bytes --- routers/api/packages/arch/arch.go | 4 ++-- services/packages/arch/db_manager.go | 11 +++-------- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/routers/api/packages/arch/arch.go b/routers/api/packages/arch/arch.go index 1596fe7061e4..3f0aa58691c3 100644 --- a/routers/api/packages/arch/arch.go +++ b/routers/api/packages/arch/arch.go @@ -155,13 +155,13 @@ func Get(ctx *context.Context) { // Packages are stored in different way from pacman databases, and loaded // with LoadPackageFile function. if strings.HasSuffix(file, "tar.zst") || strings.HasSuffix(file, "zst.sig") { - pkgdata, err := arch_service.LoadFile(ctx, distro, file) + pkg, err := arch_service.GetFileObject(ctx, distro, file) if err != nil { apiError(ctx, http.StatusNotFound, err) return } - ctx.ServeContent(bytes.NewReader(pkgdata), &context.ServeHeaderOptions{ + ctx.ServeContent(pkg, &context.ServeHeaderOptions{ Filename: file, CacheDuration: time.Minute * 5, }) diff --git a/services/packages/arch/db_manager.go b/services/packages/arch/db_manager.go index f5c3360d87f6..89ec8691e6cf 100644 --- a/services/packages/arch/db_manager.go +++ b/services/packages/arch/db_manager.go @@ -6,7 +6,6 @@ package arch import ( "bytes" "fmt" - "io" "sort" "strings" @@ -19,6 +18,7 @@ import ( "code.gitea.io/gitea/modules/json" "code.gitea.io/gitea/modules/packages" "code.gitea.io/gitea/modules/packages/arch" + "code.gitea.io/gitea/modules/storage" pkg_service "code.gitea.io/gitea/services/packages" ) @@ -107,7 +107,7 @@ func SaveFile(ctx *context.Context, p *SaveFileParams) (int64, error) { // Get data related to provided file name and distribution, and update download // counter if actual package file is retrieved from database. -func LoadFile(ctx *context.Context, distro, file string) ([]byte, error) { +func GetFileObject(ctx *context.Context, distro, file string) (storage.Object, error) { db := db.GetEngine(ctx) pkgfile := &pkg_model.PackageFile{CompositeKey: distro + "-" + file} @@ -131,12 +131,7 @@ func LoadFile(ctx *context.Context, distro, file string) ([]byte, error) { cs := packages.NewContentStore() - obj, err := cs.Get(packages.BlobHash256Key(blob.HashSHA256)) - if err != nil { - return nil, err - } - - return io.ReadAll(obj) + return cs.Get(packages.BlobHash256Key(blob.HashSHA256)) } // Automatically connect repository to pushed package, if package with provided From 597a948e33a192dc7664e4a57b6dc22300de8675 Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Sun, 23 Jul 2023 15:42:52 +0300 Subject: [PATCH 042/124] replaced custom auth with existing methods --- docs/content/doc/usage/packages/arch.en-us.md | 117 +-------------- routers/api/packages/api.go | 4 +- routers/api/packages/arch/arch.go | 140 ++++-------------- services/packages/arch/db_manager.go | 13 +- services/packages/arch/verificator.go | 99 ------------- 5 files changed, 41 insertions(+), 332 deletions(-) delete mode 100644 services/packages/arch/verificator.go diff --git a/docs/content/doc/usage/packages/arch.en-us.md b/docs/content/doc/usage/packages/arch.en-us.md index b7853562f1da..a22535270b0e 100644 --- a/docs/content/doc/usage/packages/arch.en-us.md +++ b/docs/content/doc/usage/packages/arch.en-us.md @@ -14,16 +14,12 @@ menu: # Arch package registry -Gitea has arch package registry, which can act as a fully working [arch linux mirror](https://wiki.archlinux.org/title/mirrors) and connected directly in `/etc/pacman.conf`. Gitea automatically creates pacman database for packages in user space when new arch package is uploaded. +Gitea has arch package registry, which can act as a fully working [arch linux mirror](https://wiki.archlinux.org/title/mirrors) and connected directly in `/etc/pacman.conf`. Gitea automatically creates pacman database for packages in user/organization space when new arch package is uploaded. **Table of Contents** {{< toc >}} -## Requirements - -You can install packages in any environment with [pacman](https://wiki.archlinux.org/title/Pacman). Alternatively you can use [pack](https://fmnx.su/core/pack) which connects specified registries automatically and provides simple interface for package uploads and deletions. - ## Install packages First, you need to update your pacman configuration, adding following lines: @@ -39,136 +35,37 @@ Then, you can run pacman sync command (with -y flag to load connected database f pacman -Sy package ``` -## GPG Verification - -Upload and remove operation are validated with [GnuPG](https://gnupg.org/). First, you need to export and upload your public gpg key to `SSH/GPG Keys` in account settings. This works similarly to SSH keys. You can export gpg key with command: - -```sh -gpg --armor --export -``` - -``` ------BEGIN PGP PUBLIC KEY BLOCK----- - -mQENBGSYoJUBCADSJ6v8Egst/gNJVC2206o8JqTzRBxTULKm/DH5J7AzrhJBxC2/ -... - ------END PGP PUBLIC KEY BLOCK----- -``` - ## Upload packages -1. Ensure, that your package have been signed with your gpg key (more about arch [package signing](https://wiki.archlinux.org/title/DeveloperWiki:Package_signing)). You can do that by running following command: - -```sh -gpg --verify package-ver-1-x86_64.pkg.tar.zst.sig -``` - -1. Sign message metadata, which consists of package owner (namespace in gitea), package file name and send time. You can do that by running following command: - -```sh -echo -n {owner}{package}$(date --rfc-3339=seconds | tr " " T) >> md -gpg --detach-sign md -``` - 1. Decode message and metadata signatures to hex, by running following commands, save output somewhere. ```sh -xxd -p md.sig >> md.sig.hex -xxd -p package-1-1-x86_64.pkg.tar.zst.sig >> pkg.sig.hex +xxd -p package-1-1-x86_64.pkg.tar.zst.sig >> package-signature-hex ``` -1. Paste your parameters and push package with [curl](https://curl.se/). Important, that time should be the same with metadata (signed md file), since this value is verified with GnuPG. +2. Paste your parameters and push package with [curl](https://curl.se/). Important, that time should be the same with metadata (signed md file), since this value is verified with GnuPG. ```sh curl -X PUT \ 'https://{domain}/api/packages/{owner}/arch/push' \ - --header 'filename: {package}-1-1-x86_64.pkg.tar.zst' \ - --header 'email: dancheg97@fmnx.su' \ + --header 'filename: package-1-1-x86_64.pkg.tar.zst' \ --header 'distro: archlinux' \ - --header 'time: {metadata-time}' \ --header 'pkgsign: {package-signature-hex}' \ - --header 'metasign: {metadata-signature-hex}' \ --header 'Content-Type: application/octet-stream' \ - --data-binary '@/path/to/package/file/{package}-1-1-x86_64.pkg.tar.zst' -``` - -Full script for package upload: - -```sh -owner=user -package=package-0.1.0-1-x86_64.pkg.tar.zst -email=user@example.com - -time=`date --rfc-3339=seconds | tr " " T` -pkgsignhex=`xxd -p $package.sig | tr -d "\n"` - -echo -n $owner$package$time >> mddata -gpg --detach-sign mddata -mdsignhex=`xxd -p mddata.sig | tr -d "\n"` - -curl -X PUT \ - http://{domain}/api/packages/$owner/arch/push \ - --header "filename: $package" \ - --header "email: $email" \ - --header "time: $time" \ - --header "distro: archlinux" \ - --header "metasign: $mdsignhex" \ - --header "pkgsign: $pkgsignhex" \ - --header 'Content-Type: application/octet-stream' \ - --data-binary @$package + --data-binary '@/path/to/package/file/package-1-1-x86_64.pkg.tar.zst' ``` ## Delete packages -1. Prepare signature for delete message. - -```sh -echo -n {owner}{package}$(date --rfc-3339=seconds | tr " " T) >> md -gpg --detach-sign md -``` - 1. Send delete message with [curl](https://curl.se/). Time should be the same with saved in `md` file. ```sh curl -X DELETE \ http://localhost:3000/api/packages/{user}/arch/remove \ - --header "username: {user}" \ - --header "email: user@email.com" \ --header "target: package" \ - --header "time: {rmtime}" \ - --header "version: {version-release}" \ - --header 'Content-Type: application/octet-stream' \ - --data-binary @md.sig -``` - -Full script for package deletion: - -```sh -owner=user -package=package -version=0.1.0-1 -email=user@example.com -arch=x86_64 -time=`date --rfc-3339=seconds | tr " " T` - -sudo rm -rf md md.sig -echo -n $owner$package$time >> md -gpg --detach-sign md - -curl -X DELETE \ - http://{domain}/api/packages/$owner/arch/remove \ - --header "username: $owner" \ - --header "email: $email" \ - --header "target: $package" \ - --header "time: $time" \ - --header "version: $version" \ - --header 'Content-Type: application/octet-stream' \ - --data-binary @md.sig + --header "version: {version-release}" ``` ## Clients -You can generate client code with tools like [thunder client](https://marketplace.visualstudio.com/items?itemName=rangav.vscode-thunder-client), [postman](https://blog.postman.com/curl-and-postman-work-wonderfully-together/) or other code generators to write your own client. - -Also you can take a look at [pack](https://fmnx.su/core/pack) which provides `pacman` functionality with additional commands to build, sign and push your arch packages to gitea. +You can use gitea CLI tool to - [tea](https://gitea.com/gitea/tea) to push/remove arch packages from gitea. Alternatively, you can try [pack](https://fmnx.su/core/pack). diff --git a/routers/api/packages/api.go b/routers/api/packages/api.go index a4221e848c4f..1930d726f28a 100644 --- a/routers/api/packages/api.go +++ b/routers/api/packages/api.go @@ -124,8 +124,8 @@ func CommonRoutes() *web.Route { }) }, reqPackageAccess(perm.AccessModeRead)) r.Group("/arch", func() { - r.Put("/push", arch.Push) - r.Delete("/remove", arch.Remove) + r.Put("/push", arch.Push, reqPackageAccess(perm.AccessModeWrite)) + r.Delete("/remove", arch.Remove, reqPackageAccess(perm.AccessModeWrite)) r.Get("/{distro}/{arch}/{file}", arch.Get) }) r.Group("/cargo", func() { diff --git a/routers/api/packages/arch/arch.go b/routers/api/packages/arch/arch.go index 3f0aa58691c3..564acfdfa505 100644 --- a/routers/api/packages/arch/arch.go +++ b/routers/api/packages/arch/arch.go @@ -23,20 +23,10 @@ func Push(ctx *context.Context) { var ( owner = ctx.Params("username") filename = ctx.Req.Header.Get("filename") - email = ctx.Req.Header.Get("email") distro = ctx.Req.Header.Get("distro") - sendtime = ctx.Req.Header.Get("time") - pkgsign = ctx.Req.Header.Get("pkgsign") - metasign = ctx.Req.Header.Get("metasign") + sign = ctx.Req.Header.Get("sign") ) - // Decoding package signature. - sigdata, err := hex.DecodeString(pkgsign) - if err != nil { - apiError(ctx, http.StatusBadRequest, err) - return - } - // Read package to memory for signature validation. pkgdata, err := io.ReadAll(ctx.Req.Body) if err != nil { @@ -45,33 +35,6 @@ func Push(ctx *context.Context) { } defer ctx.Req.Body.Close() - // Get user and organization owning arch package. - user, org, err := arch_service.IdentifyOwner(ctx, owner, email) - if err != nil { - apiError(ctx, http.StatusUnauthorized, err) - return - } - - // Decoding time when message was created. - t, err := time.Parse(time.RFC3339, sendtime) - if err != nil { - apiError(ctx, http.StatusBadRequest, err) - return - } - - // Check if message is outdated. - if time.Since(t) > time.Hour { - apiError(ctx, http.StatusUnauthorized, "outdated message") - return - } - - // Decoding signature related to metadata. - msigdata, err := hex.DecodeString(metasign) - if err != nil { - apiError(ctx, http.StatusBadRequest, err) - return - } - // Parse metadata contained in arch package archive. md, err := arch_module.EjectMetadata(filename, distro, setting.Domain, pkgdata) if err != nil { @@ -79,53 +42,40 @@ func Push(ctx *context.Context) { return } - // Validating metadata signature, to ensure that operation push operation - // is initiated by original package owner. - sendmetadata := []byte(owner + md.Name + sendtime) - err = arch_service.ValidateSignature(ctx, sendmetadata, msigdata, user) - if err != nil { - apiError(ctx, http.StatusUnauthorized, err) - return - } - - // Validate package signature with any of user's GnuPG keys. - err = arch_service.ValidateSignature(ctx, pkgdata, sigdata, user) - if err != nil { - apiError(ctx, http.StatusUnauthorized, err) - return - } - // Save file related to arch package. pkgid, err := arch_service.SaveFile(ctx, &arch_service.SaveFileParams{ - Organization: org, - User: user, - Metadata: md, - Filename: filename, - Data: pkgdata, - Distro: distro, + Creator: ctx.Doer, + Owner: ctx.Package.Owner, + Metadata: md, + Filename: filename, + Data: pkgdata, + Distro: distro, }) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return } - // Save file related to arch package signature. - _, err = arch_service.SaveFile(ctx, &arch_service.SaveFileParams{ - Organization: org, - User: user, - Metadata: md, - Data: sigdata, - Filename: filename + ".sig", - Distro: distro, - }) - if err != nil { - apiError(ctx, http.StatusInternalServerError, err) - return + // Decoding package signature, if present saving with package as file. + sigdata, err := hex.DecodeString(sign) + if err == nil { + _, err = arch_service.SaveFile(ctx, &arch_service.SaveFileParams{ + Creator: ctx.Doer, + Owner: ctx.Package.Owner, + Metadata: md, + Data: sigdata, + Filename: filename + ".sig", + Distro: distro, + }) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } } // Add new architectures and distribution info to package version metadata. err = arch_service.UpdateMetadata(ctx, &arch_service.UpdateMetadataParameters{ - User: org.AsUser(), + User: ctx.Package.Owner, Md: md, }) if err != nil { @@ -190,50 +140,12 @@ func Get(ctx *context.Context) { // Remove specific package version, related files and pacman database entry. func Remove(ctx *context.Context) { var ( - owner = ctx.Params("username") - email = ctx.Req.Header.Get("email") - target = ctx.Req.Header.Get("target") - stime = ctx.Req.Header.Get("time") - version = ctx.Req.Header.Get("version") + pkg = ctx.Req.Header.Get("package") + ver = ctx.Req.Header.Get("version") ) - // Parse sent time and check if it is within last minute. - t, err := time.Parse(time.RFC3339, stime) - if err != nil { - apiError(ctx, http.StatusBadRequest, err) - return - } - - if time.Since(t) > time.Minute { - apiError(ctx, http.StatusUnauthorized, "outdated message") - return - } - - // Get user owning the package. - user, org, err := arch_service.IdentifyOwner(ctx, owner, email) - if err != nil { - apiError(ctx, http.StatusUnauthorized, err) - return - } - - // Read signature data from request body. - sigdata, err := io.ReadAll(ctx.Req.Body) - if err != nil { - apiError(ctx, http.StatusInternalServerError, err) - return - } - defer ctx.Req.Body.Close() - - // Validate package signature with any of user's GnuPG keys. - mesdata := []byte(owner + target + stime) - err = arch_service.ValidateSignature(ctx, mesdata, sigdata, user) - if err != nil { - apiError(ctx, http.StatusUnauthorized, err) - return - } - // Remove package files and pacman database entry. - err = arch_service.RemovePackage(ctx, org.AsUser(), target, version) + err := arch_service.RemovePackage(ctx, ctx.Package.Owner, pkg, ver) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return diff --git a/services/packages/arch/db_manager.go b/services/packages/arch/db_manager.go index 89ec8691e6cf..97964e50663d 100644 --- a/services/packages/arch/db_manager.go +++ b/services/packages/arch/db_manager.go @@ -10,7 +10,6 @@ import ( "strings" "code.gitea.io/gitea/models/db" - org_model "code.gitea.io/gitea/models/organization" pkg_model "code.gitea.io/gitea/models/packages" "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/models/user" @@ -58,9 +57,9 @@ func UpdateMetadata(ctx *context.Context, p *UpdateMetadataParameters) error { // Parameters required to save new arch package. type SaveFileParams struct { - *org_model.Organization - *user.User - *arch.Metadata + Creator *user.User + Owner *user.User + Metadata *arch.Metadata Data []byte Filename string Distro string @@ -80,12 +79,12 @@ func SaveFile(ctx *context.Context, p *SaveFileParams) (int64, error) { pv, _, err := pkg_service.CreatePackageOrAddFileToExisting( &pkg_service.PackageCreationInfo{ PackageInfo: pkg_service.PackageInfo{ - Owner: p.Organization.AsUser(), + Owner: p.Owner, PackageType: pkg_model.TypeArch, Name: p.Metadata.Name, Version: p.Metadata.Version, }, - Creator: p.User, + Creator: p.Creator, Metadata: p.Metadata, }, &pkg_service.PackageFileCreationInfo{ @@ -93,7 +92,7 @@ func SaveFile(ctx *context.Context, p *SaveFileParams) (int64, error) { Filename: p.Filename, CompositeKey: p.Distro + "-" + p.Filename, }, - Creator: p.User, + Creator: p.Creator, Data: buf, OverwriteExisting: true, IsLead: p.IsLead, diff --git a/services/packages/arch/verificator.go b/services/packages/arch/verificator.go deleted file mode 100644 index f9ac67fb8955..000000000000 --- a/services/packages/arch/verificator.go +++ /dev/null @@ -1,99 +0,0 @@ -// Copyright 2023 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package arch - -import ( - "bytes" - "errors" - "fmt" - "strings" - - "code.gitea.io/gitea/models/asymkey" - "code.gitea.io/gitea/models/db" - org "code.gitea.io/gitea/models/organization" - "code.gitea.io/gitea/models/user" - "code.gitea.io/gitea/modules/context" - - "github.com/keybase/go-crypto/openpgp" -) - -type IdentidyOwnerParameters struct { - *context.Context - Owner string - Email string -} - -// This function will find user related to provided email address and check if -// he is able to push packages to provided namespace (user/organization/or -// empty namespace allowed for admin users). Function will return user making -// operation, organization or user owning the package. -func IdentifyOwner(ctx *context.Context, owner, email string) (*user.User, *org.Organization, error) { - u, err := user.GetUserByEmail(ctx, email) - if err != nil { - return nil, nil, fmt.Errorf("unable to find user with email %s, %v", email, err) - } - - if owner == "" && u.IsAdmin { - return u, (*org.Organization)(u), nil - } - - if owner == u.Name { - return u, (*org.Organization)(u), nil - } - - if u.Name != owner { - org, err := org.GetOrgByName(ctx, owner) - if err != nil { - return nil, nil, fmt.Errorf("unable to get organization: %s, %v", owner, err) - } - ismember, err := org.IsOrgMember(u.ID) - if err != nil { - return nil, nil, fmt.Errorf("unable to check if user %s belongs to organization %s: %v", u.Name, org.Name, err) - } - if !ismember { - return nil, nil, fmt.Errorf("user %s is not member of organization %s", u.Name, org.Name) - } - return u, org, nil - } - return nil, nil, fmt.Errorf("unknown package owner") -} - -// Validate package signature with owner's GnuPG keys stored in gitea's database. -func ValidateSignature(ctx *context.Context, pkg, sign []byte, u *user.User) error { - keys, err := asymkey.ListGPGKeys(ctx, u.ID, db.ListOptions{ - ListAll: true, - }) - if err != nil { - return errors.New("unable to get public keys") - } - if len(keys) == 0 { - return errors.New("no keys for user with email: " + u.Email) - } - - var keyarmors []string - for _, key := range keys { - k, err := asymkey.GetGPGImportByKeyID(key.KeyID) - if err != nil { - return errors.New("unable to import GPG key armor") - } - keyarmors = append(keyarmors, k.Content) - } - - var trace []error - for _, armor := range keyarmors { - kr, err := openpgp.ReadArmoredKeyRing(strings.NewReader(armor)) - if err != nil { - trace = append(trace, fmt.Errorf("unable to get keys for %s: %v", u.Name, err)) - continue - } - _, err = openpgp.CheckDetachedSignature(kr, bytes.NewReader(pkg), bytes.NewReader(sign)) - if err != nil { - trace = append(trace, err) - continue - } - return nil - } - - return errors.Join(trace...) -} From 542882d97345902210852c7d22b0b30b62bedc81 Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Sun, 23 Jul 2023 20:04:25 +0300 Subject: [PATCH 043/124] fixed tests, corrected documentation --- docs/content/doc/usage/packages/arch.en-us.md | 14 ++- go.mod | 1 - go.sum | 2 - tests/integration/api_packages_arch_test.go | 117 +----------------- 4 files changed, 13 insertions(+), 121 deletions(-) diff --git a/docs/content/doc/usage/packages/arch.en-us.md b/docs/content/doc/usage/packages/arch.en-us.md index a22535270b0e..eadc0b402328 100644 --- a/docs/content/doc/usage/packages/arch.en-us.md +++ b/docs/content/doc/usage/packages/arch.en-us.md @@ -48,10 +48,11 @@ xxd -p package-1-1-x86_64.pkg.tar.zst.sig >> package-signature-hex ```sh curl -X PUT \ 'https://{domain}/api/packages/{owner}/arch/push' \ - --header 'filename: package-1-1-x86_64.pkg.tar.zst' \ - --header 'distro: archlinux' \ - --header 'pkgsign: {package-signature-hex}' \ - --header 'Content-Type: application/octet-stream' \ + -H 'Authorization: {your-authorization-token}' \ + -H 'filename: package-1-1-x86_64.pkg.tar.zst' \ + -H 'distro: archlinux' \ + -H 'sign: {package-signature-hex}' \ + -H 'Content-Type: application/octet-stream' \ --data-binary '@/path/to/package/file/package-1-1-x86_64.pkg.tar.zst' ``` @@ -62,8 +63,9 @@ curl -X PUT \ ```sh curl -X DELETE \ http://localhost:3000/api/packages/{user}/arch/remove \ - --header "target: package" \ - --header "version: {version-release}" + -H 'Authorization: {your-authorization-token}' \ + -H "target: package" \ + -H "version: {version-release}" ``` ## Clients diff --git a/go.mod b/go.mod index 3671162658ec..29a17b16abf5 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,6 @@ module code.gitea.io/gitea go 1.20 require ( - bou.ke/monkey v1.0.2 code.gitea.io/actions-proto-go v0.3.1 code.gitea.io/gitea-vet v0.2.2 code.gitea.io/sdk/gitea v0.15.1 diff --git a/go.sum b/go.sum index a14d547bd6c5..9c6250c1016c 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,3 @@ -bou.ke/monkey v1.0.2 h1:kWcnsrCNUatbxncxR/ThdYqbytgOIArtYWqcQLQzKLI= -bou.ke/monkey v1.0.2/go.mod h1:OqickVX3tNx6t33n1xvtTtu85YN5s6cKwVug+oHMaIA= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= diff --git a/tests/integration/api_packages_arch_test.go b/tests/integration/api_packages_arch_test.go index 717e86ed8feb..c18530033d5a 100644 --- a/tests/integration/api_packages_arch_test.go +++ b/tests/integration/api_packages_arch_test.go @@ -12,62 +12,16 @@ import ( "net/http" "path" "testing" - "time" - auth_model "code.gitea.io/gitea/models/auth" "code.gitea.io/gitea/models/unittest" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/setting" - api "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/tests" - "bou.ke/monkey" "github.com/stretchr/testify/assert" ) func TestPackageArch(t *testing.T) { - const gpgkey = `-----BEGIN PGP PUBLIC KEY BLOCK----- - -mQGNBGR6EBQBDACseiWT4OrE4FW7RuKmVN+ETK+yXUdViB2lWJRJrdm/XsjYKRvw -uFqDHDqZDOTV8cy93JzVa2xAIfa+aWdNrXyoZFOVgDdtuW00RrbNwztDDIDS5yy4 -bOX0pIv0UY3KgPX0N8zr+mn6wdpRqYvSx3tZMr+kAeXOnWQB1dsStixL8lO+iq3j -oPbOkMVxR+3ydQUz8f5xycKWSu8zU4L/OdCsrc7VN4UpeQcaOL2g+X09Y+1d96eG -u4knTGvUbix07WLNM8X6KtCtmAZW6qPfOM7pHWtqN3U4GrKQHHLh4xWYdT6yPqKA -WgngqqHMXinqC9+HPLKkWjHnbri0w3t1pS6WraHmcuYOZJepfEjacWxBpjgoP/PD -vrb9uUU72o6jyYFkoH/ji1dw5b/uaNR3FW+fuLHY0ymGFMr7BgWE7XK0X5ikLVjC -waOPowl7B3wbR7nIz7cWwDcC1W6rBaqltuaya5yWU231rC7YahOABJM0unBDyCpX -u4VfHNYeUMzOad8AEQEAAbQjRGFuaWxhIEZvbWlueWtoIDxkYW5jaGVnOTdAZm1u -eC5zdT6JAdQEEwEIAD4WIQRDpY25Uq5YdAmqF1IRNPjr+YqgbwUCZHoQFAIbAwUJ -A8JnAAULCQgHAgYVCgkICwIEFgIDAQIeAQIXgAAKCRARNPjr+Yqgby4pC/9Valyu -VmoWYbqFLkQhGdpsgyfDnIZ4VpZGZ4nWMnaOhQAzgcRnDYCvPryQYhTf87NzoCif -QhP4RMD7n3EmZNXxEINIKLgvWkp6ts5YquR6e+PhiQBoVhNOg1o7L/g4WCnyXqQM -68mla9jEBD+qKVQD9EIGS0HN/645ch39S3mMUFEGg4NDoRvoQcas3/3sbl1tyqTj -Q+oSHp5nRQVw+BtGBU4mu81gM75isf5fdRkYJdcAufQJbiTkEy1qhBTB2WpqBJe6 -a4KzhcpDDv8LXhanRUioRVrgUphlAUf9PCV3z6TLirDqm2tt4op1Sd/HnBQYOzs4 -ubBryS7bw48uAVEeJ4Ri3N0aHJDmtK4R56fhUBUMOeZb3Rg+erTsJ/V1U2aJxoF8 -i+w9aZcuA3CAMOyg5r8ILs+qD5ZpWDKUyWWa2toJK4+OXpzEUo6Mr9c4lsHLitvQ -zXHfgFSu7xZvi7jwBV09GM07C9Zv70f/4L2d+Qhj6udfFM44DFE+3Tzi9025AY0E -ZHoQFAEMAMY4zcCR8OR6mkcvVKsuDOdTy4LzSUPNL5WnoIICBbWFBVpU4T0AY+5E -E+3JxNDpgzzQp7YNNyO02Kkwo662bCYIMC0xa3siIeZZb82l/8+L68w/4wXDUXFF -qP5cWyw5J0zPLt7nRpEFlG3RbHJn6ndSL7uDTxMaJtREqyDvfuHhLcRqjAtnTj7W -SHumvIAacP1IWrcjfJVv6aBZzg3y31wx7KF0cx2CLAGBKC6DXhZdyLSzqhtgy0OY -ROf7x7sP/xu/UqStWVsQIj0/mxoNYoNH5g8Paa/80zIXZZlqvN1FycDH+U761hUd -1L2Gl5TBJy1gCoUGtbwog2LBdc76tY/IcHd6+8cUdxpKV7sCbTmLXI0sQm8imjez -JbG6dC/B5U2AdTdpVd1KIqC1SCZHG9dsnXvJDB6pKwDCcPC0Dl59lwHaD9b08YMP -pAA1J9rW6T682cGMsLHO3ipUaGnf1z4gA2IGhIsqgl00V4W2Es3gHzkAD6YJ9+H6 -hMRJCRaSZwARAQABiQG8BBgBCAAmFiEEQ6WNuVKuWHQJqhdSETT46/mKoG8FAmR6 -EBQCGwwFCQPCZwAACgkQETT46/mKoG8XxgwAqKYu0/irkZzhbLUiZPljE1dZSozX -bd0HtgizWt7iOiUdJzJrM9WGaXJ9hYykYp0IlDID5HNG+ZMgqS18XRAbUqjTalav -0ftjQdx25mUplCBOJmP7bciKjGUS2BLImjhMAHgwyf0MAQJApiBvH3kGLOEsL3Lh -uNwB04+iPFKYjIv6A4abRo4LnMy3we7o/IOwFqsSMZKKVWjY9q+au4h1q8h4YI70 -kDp+wItT92I90OMRHnSs88MrP8aMvVzI6Arph/5tVkthm0olO/TtSFC6ki4H3LdI -3Yzrx42A0MGeZQrQ49gydFFDreISYwU99AgIk33j85Ut3hkCriW5LbgOqHY21Fbs -LzlPDJpA9VZIQMcLZW0pFISTyWkGsSf0P1J8Z03RHzU8L5ROxonIyMriw7thMK/s -ZmaX8KGlqBgJ/6s6/8ERh7IbNvNu+PolTI5+PXDTofZ8ssKIcALQgF9zjaVIORfF -8Iu2wss37Re8PI2n7enU+iNe1UN81ZMojf/V -=5NFR ------END PGP PUBLIC KEY BLOCK-----` - const firstPackage = `KLUv/QRY1QwBGoVZQ0SgriRpAxihjQU4nU1abf1BN6nGKR/bld+8hZmisY00IpH7u7t3+4kQSpHb p1+3roM678K21giRyF40KUz2hNLQrAXR9A4E6QPnA/X6wzHZ2cgbLKlm0Edvp0eNJLW+Dl3H7bKL TevnR1bIGyutpv/6ugOzr3o9SbqpSabOgdSVpFpKn7cy2UkrHb2eI5eNr7TSsmAdqZMRB2nZeKP1 @@ -234,17 +188,6 @@ jMF9gK0ZWKyVAYH5FFhDJAm+lxWpcBu0Fnpj56LGqcaDORsvIVkA` firstPackageSignatureData, err := base64.StdEncoding.DecodeString(firstPackageSignature) assert.NoError(t, err) - const firstPackageMetadataSignature = `iQGzBAABCAAdFiEEQ6WNuVKuWHQJqhdSETT46/mKoG8FAmSsMPgACgkQETT46/mKoG948wv+IkNg -3zLDySwDxcBXmuhAtMMgUs4D+rypUp6qZI+N/WAWtV7niWEmZI3RQRzSmHn7NpmXPiQyGIa6ViET -bjfmbVhU/iaFLIQa3UY7dJCT8x0QVQNCvmLEWzAkqQMiY4QKaitetw+nA6gw/MAL9UNWLiRlCj4I -K8ySQdZr1dtSTTw/KK2t6kgFbEVpwLauKZrT6Vw3LVU5biuq76918JG7k49po4wUQ62a8Er7WqOV -4DyhNMKqcbfKd+9/WRVqTMHep7dC2fzbq1/MFx3TNL/vGKl5uMrpePyTN74AmxqfQWNwLDXYPxsD -/y4vPYer/KQdIs8omDpNaxkTrAvsFizKfxsFlPRt6uknu0WKY15SUXRI3fIG+vFXigW1JNVtuynH -KyH0USuoUzrfn7hKPp5ubusdeDTwr5GXDBM7qEpBxGce8YMVNYQUTDdK9D+D25fMMb5DkQa8zKjy -GXcSu4lwxHmarGlD45/DQrYH7tSpVqj2cBaAxtkn/ocpoPawta67` - firstPackageMetadataSignatureData, err := base64.StdEncoding.DecodeString(firstPackageMetadataSignature) - assert.NoError(t, err) - const firstPackageDatabase = `H4sIAAAAAAAA/+ySwY7TMBCGe56nyGWOTcd2PE4QQoQmZSPaBTWUAxfkON626rYbNV6p8PSoXYTY XlGF0Pa7jMf6JfsbTfB96DbLoRiKUet7N7gARETGmFMlovNKTPL3+XQvJCkaRHSJz5zz2Ae7H9Bf v3Uu95+Ak2pa3uazEuGPVRgeUv7GSdxtlnGw+/hHHwDwWQwA3+X1s/5LOa+rj7cIYigAsCjrMcJk @@ -423,18 +366,6 @@ V2DfzGUB63cabGvZKvYMynHjfaxo8MwKVjUzgvWqMfNL2P5QpQ6S` assert.NoError(t, err) _ = secondPackageSignatureData - const secondPackageMetadataSignature = `iQGzBAABCAAdFiEEQ6WNuVKuWHQJqhdSETT46/mKoG8FAmSsMPoACgkQETT46/mKoG/GCQv9Hr4T -DYBG2KpgzEeC3vwy4YvHbrYc+TwZWeRRS5xgpk+mG6xJqRitHhi8K8quP0Z4/PChr8yZE2ZHgKOv -2Oul8fYA9z6xAmGP9vNSVjUJM4T0zWMB9uO6g7ZWh62BTTVoWWA19PRQxlcRg6uonso5ZfXpYbit -RpfNSKRXqo3KgrLNpx1neXMoFZb3fuUW4Nc69qdjpCcJ/UrEVfyN/SD719JrIn9WQiidud0sBtA0 -5F/DF456UXgpDSb+LDSkGMlWcHBz8VoTs3XMW+w50dC03IJI4Yji+TLMrjrUNVNeR50D9siAjeRr -ihGI33HB0kTQv7kw6iG+H/me1bI/pI20slA4KabMEmoiyO6a4XCwTObl6/mpcTvpDQqLgvmLHgEQ -+oKANt0BiOrtDlLhGBdIM9NXkVfWQGF0MwZVTn/agH/qB95JNNT8DH1+beEDOna9HjImyOtp9Z13 -HkZaMSkzwZmNAopA9pYpm0dIBquFCxaX+hKkoeLma2JMtCOHaV4Y` - secondPackageMetadataSignatureData, err := base64.StdEncoding.DecodeString(secondPackageMetadataSignature) - assert.NoError(t, err) - _ = secondPackageMetadataSignatureData - const secondPackageDatabase = `H4sIAAAAAAAA/+ySQY/TMBCFe55fkcscm47teJwghAhNyka0C2ooBy7Icbxt1U02arJSl1+P2kWI 7RVVCNHvMh7rSfZ7eoPvh263HouxmNS+d6MLQERkjDlNIjqfxMS/zqd7IUlFo4Au8ZlzHvvB7kf0 x2+dm/tHwFkxz2/TRY7wWxXGh5i/cRR2u3U42H34vR8A8IUMAN+l5Yv9S74si4+3CGIsADDLyynC @@ -444,53 +375,28 @@ H2zT3fvQPTSTn/aOhtLl9AbhORqw7dMxglUxz7L0c44gOI6TRCvFAPgpnX5I3+dLhMy223sbzB6a bfu02wSva9u6jV8n5u1d0x7C/vENwN9uwZUrV/5HfgQAAP//SZpudwAIAAA=` secondPackageDatabaseData, err := base64.StdEncoding.DecodeString(secondPackageDatabase) assert.NoError(t, err) - _ = secondPackageDatabaseData assert.NoError(t, unittest.PrepareTestDatabase()) setting.Packages.LimitSizeArch = 99999999999999 - user := &user_model.User{ - Name: "dancheg97", - Email: "dancheg97@fmnx.su", - Passwd: "password", - } - - err = user_model.CreateUser(user) - assert.NoError(t, err) - - session := loginUser(t, "dancheg97") - token := getTokenForLoggedInUser(t, session, auth_model.AccessTokenScopeAll) - - req := NewRequestWithJSON(t, "POST", "/api/v1/user/gpg_keys?token="+token, api.CreateGPGKeyOption{ - ArmoredKey: gpgkey, - }) - MakeRequest(t, req, http.StatusCreated) + user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) rootURL := fmt.Sprintf("/api/packages/%s/arch", user.Name) - wayback, err := time.Parse(time.RFC3339, "2023-07-10T19:25:28+03:00") - assert.NoError(t, err) - t.Run("PushFirstPackage", func(t *testing.T) { defer tests.PrintCurrentTest(t)() - patch := monkey.Patch(time.Now, func() time.Time { return wayback }) - req := NewRequest(t, "PUT", path.Join(rootURL, "/push")) req.Header.Set("filename", "testpkg-1-1-x86_64.pkg.tar.zst") - req.Header.Set("email", "dancheg97@fmnx.su") - req.Header.Set("time", "2023-07-10T19:25:28+03:00") req.Header.Set("distro", "archlinux") - req.Header.Set("metasign", hex.EncodeToString(firstPackageMetadataSignatureData)) req.Header.Set("pkgsign", hex.EncodeToString(firstPackageSignatureData)) req.Header.Set("Content-Type", "application/octet-stream") req.Body = io.NopCloser(bytes.NewReader(firstPackageData)) + req = AddBasicAuthHeader(req, user.Name) MakeRequest(t, req, http.StatusOK) - - patch.Unpatch() }) t.Run("GetFirstPackage", func(t *testing.T) { @@ -518,22 +424,16 @@ bfu02wSva9u6jV8n5u1d0x7C/vENwN9uwZUrV/5HfgQAAP//SZpudwAIAAA=` t.Run("PushSecond", func(t *testing.T) { defer tests.PrintCurrentTest(t)() - patch := monkey.Patch(time.Now, func() time.Time { return wayback }) - req := NewRequest(t, "PUT", path.Join(rootURL, "/push")) req.Header.Set("filename", "testpkg-1-1-any.pkg.tar.zst") - req.Header.Set("email", "dancheg97@fmnx.su") - req.Header.Set("time", "2023-07-10T19:25:30+03:00") req.Header.Set("distro", "archlinux") - req.Header.Set("metasign", hex.EncodeToString(secondPackageMetadataSignatureData)) req.Header.Set("pkgsign", hex.EncodeToString(secondPackageSignatureData)) req.Header.Set("Content-Type", "application/octet-stream") req.Body = io.NopCloser(bytes.NewReader(secondPackageData)) + req = AddBasicAuthHeader(req, user.Name) MakeRequest(t, req, http.StatusOK) - - patch.Unpatch() }) t.Run("GetSecondPackage", func(t *testing.T) { @@ -561,20 +461,13 @@ bfu02wSva9u6jV8n5u1d0x7C/vENwN9uwZUrV/5HfgQAAP//SZpudwAIAAA=` t.Run("RemoveFirst", func(t *testing.T) { defer tests.PrintCurrentTest(t)() - patch := monkey.Patch(time.Now, func() time.Time { return wayback }) - req := NewRequest(t, "DELETE", path.Join(rootURL, "/remove")) - req.Header.Set("username", "dancheg97") - req.Header.Set("email", "dancheg97@fmnx.su") - req.Header.Set("target", "testpkg") - req.Header.Set("time", "2023-07-10T19:25:28+03:00") + req.Header.Set("package", "testpkg") req.Header.Set("version", "1-1") - req.Header.Set("Content-Type", "application/octet-stream") - req.Body = io.NopCloser(bytes.NewReader(firstPackageMetadataSignatureData)) + req = AddBasicAuthHeader(req, user.Name) MakeRequest(t, req, http.StatusOK) - patch.Unpatch() }) } From cdc5107b5d8275f73a76dc324fbccdfe285ae690 Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Sun, 23 Jul 2023 20:06:42 +0300 Subject: [PATCH 044/124] corrected SigLevel value for pacman database connection in documentation --- docs/content/doc/usage/packages/arch.en-us.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/content/doc/usage/packages/arch.en-us.md b/docs/content/doc/usage/packages/arch.en-us.md index eadc0b402328..74ffb9f7895a 100644 --- a/docs/content/doc/usage/packages/arch.en-us.md +++ b/docs/content/doc/usage/packages/arch.en-us.md @@ -26,6 +26,7 @@ First, you need to update your pacman configuration, adding following lines: ```conf [{owner}.{domain}] +SigLevel = Optional TrustAll Server = https://{domain}/api/packages/{owner}/arch/{distribution}/{architecture} ``` From 0093004d0fa875e91300915922ecc6a58da1e077 Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Sun, 23 Jul 2023 20:18:26 +0300 Subject: [PATCH 045/124] removed distro field from arch package metadata, corrected related UI --- modules/packages/arch/metadata.go | 7 ++++--- services/packages/arch/db_manager.go | 1 - templates/package/content/arch.tmpl | 7 +++++++ 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/modules/packages/arch/metadata.go b/modules/packages/arch/metadata.go index 6f553b8634c1..1adb5f93c575 100644 --- a/modules/packages/arch/metadata.go +++ b/modules/packages/arch/metadata.go @@ -34,7 +34,6 @@ type Metadata struct { BuildDate int64 `json:"build-date"` BaseDomain string `json:"base-domain"` Packager string `json:"packager"` - Distribution []string `json:"distribution"` Provides []string `json:"provides"` License []string `json:"license"` Arch []string `json:"arch"` @@ -44,7 +43,10 @@ type Metadata struct { CheckDepends []string `json:"check-depends"` Backup []string `json:"backup"` // This list is created to ensure the consistency of pacman database file - // for specific combination of distribution and architecture. + // for specific combination of distribution and architecture. This value + // is used when creating pacman database, to ensure that only packages + // with specific combination of distribution and architecture are present + // in pacman database file. DistroArch []string `json:"distro-arch"` } @@ -60,7 +62,6 @@ func EjectMetadata(filename, distribution, domain string, pkg []byte) (*Metadata CompressedSize: int64(len(pkg)), MD5: md5sum(pkg), SHA256: sha256sum(pkg), - Distribution: []string{distribution}, } for _, line := range strings.Split(pkginfo, "\n") { splt := strings.Split(line, " = ") diff --git a/services/packages/arch/db_manager.go b/services/packages/arch/db_manager.go index 97964e50663d..7ac72874da50 100644 --- a/services/packages/arch/db_manager.go +++ b/services/packages/arch/db_manager.go @@ -42,7 +42,6 @@ func UpdateMetadata(ctx *context.Context, p *UpdateMetadataParameters) error { } currmd.Arch = arch.UnifiedList(currmd.Arch, p.Md.Arch) - currmd.Distribution = arch.UnifiedList(currmd.Distribution, p.Md.Distribution) currmd.DistroArch = arch.UnifiedList(currmd.DistroArch, p.Md.DistroArch) b, err := json.Marshal(&currmd) diff --git a/templates/package/content/arch.tmpl b/templates/package/content/arch.tmpl index 3d108f26b4ff..5f5c8cd49791 100644 --- a/templates/package/content/arch.tmpl +++ b/templates/package/content/arch.tmpl @@ -97,6 +97,13 @@ Server = https://{{.PackageDescriptor.Metadata.BaseDomain}}/api/packages/{{.Pack {{end}} + + {{range $key := .PackageDescriptor.Metadata.DistroArch}} + + + + + {{end}}
Description
 {{.PackageDescriptor.Metadata.Description}}
Compressed size
{{.PackageDescriptor.Metadata.CompressedSizeMib}}
Installed size
{{.PackageDescriptor.Metadata.InstalledSizeMib}}
Official URL
 {{.PackageDescriptor.Metadata.URL}}
Build date
{{.PackageDescriptor.Metadata.BuildDateStr}}
Packager
 {{.PackageDescriptor.Metadata.Packager}}
Provides
Packager
 {{.PackageDescriptor.Metadata.Packager}}
Compressed size
{{FileSize .PackageDescriptor.Metadata.CompressedSize}}
Installed size
{{FileSize .PackageDescriptor.Metadata.InstalledSize}}
Build date
{{DateTime "short" .PackageDescriptor.Metadata.BuildDate}}
Build date
 {{DateTime "short" .PackageDescriptor.Metadata.BuildDate}}
Provides
{{$key}}
Available
{{$key}}
From 0632f578e0e12045a57216fa5ba714fa325e2ce6 Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Sun, 23 Jul 2023 20:26:24 +0300 Subject: [PATCH 046/124] lint corrections --- docs/content/doc/usage/packages/arch.en-us.md | 6 +++--- tests/integration/api_packages_arch_test.go | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/content/doc/usage/packages/arch.en-us.md b/docs/content/doc/usage/packages/arch.en-us.md index 74ffb9f7895a..18bf3e89ccf7 100644 --- a/docs/content/doc/usage/packages/arch.en-us.md +++ b/docs/content/doc/usage/packages/arch.en-us.md @@ -38,13 +38,13 @@ pacman -Sy package ## Upload packages -1. Decode message and metadata signatures to hex, by running following commands, save output somewhere. +1. Decode message and metadata signatures to hex, by running following commands, save output. ```sh xxd -p package-1-1-x86_64.pkg.tar.zst.sig >> package-signature-hex ``` -2. Paste your parameters and push package with [curl](https://curl.se/). Important, that time should be the same with metadata (signed md file), since this value is verified with GnuPG. +2. Paste your parameters and push package with [curl](https://curl.se/). ```sh curl -X PUT \ @@ -59,7 +59,7 @@ curl -X PUT \ ## Delete packages -1. Send delete message with [curl](https://curl.se/). Time should be the same with saved in `md` file. +Send delete message with [curl](https://curl.se/). ```sh curl -X DELETE \ diff --git a/tests/integration/api_packages_arch_test.go b/tests/integration/api_packages_arch_test.go index c18530033d5a..93eadececb96 100644 --- a/tests/integration/api_packages_arch_test.go +++ b/tests/integration/api_packages_arch_test.go @@ -468,6 +468,5 @@ bfu02wSva9u6jV8n5u1d0x7C/vENwN9uwZUrV/5HfgQAAP//SZpudwAIAAA=` req = AddBasicAuthHeader(req, user.Name) MakeRequest(t, req, http.StatusOK) - }) } From 67bd2bc58b2c7a6c51ad225a14e55421717a0568 Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Mon, 24 Jul 2023 12:07:53 +0300 Subject: [PATCH 047/124] simplified push in docs, corrected pkg UI --- docs/content/doc/usage/packages/arch.en-us.md | 20 +++++++------------ templates/package/content/arch.tmpl | 3 ++- 2 files changed, 9 insertions(+), 14 deletions(-) diff --git a/docs/content/doc/usage/packages/arch.en-us.md b/docs/content/doc/usage/packages/arch.en-us.md index 18bf3e89ccf7..8986548d812d 100644 --- a/docs/content/doc/usage/packages/arch.en-us.md +++ b/docs/content/doc/usage/packages/arch.en-us.md @@ -38,22 +38,16 @@ pacman -Sy package ## Upload packages -1. Decode message and metadata signatures to hex, by running following commands, save output. - -```sh -xxd -p package-1-1-x86_64.pkg.tar.zst.sig >> package-signature-hex -``` - -2. Paste your parameters and push package with [curl](https://curl.se/). +Get into folder with package and signature, push package with [curl](https://curl.se/). ```sh curl -X PUT \ 'https://{domain}/api/packages/{owner}/arch/push' \ - -H 'Authorization: {your-authorization-token}' \ - -H 'filename: package-1-1-x86_64.pkg.tar.zst' \ - -H 'distro: archlinux' \ - -H 'sign: {package-signature-hex}' \ - -H 'Content-Type: application/octet-stream' \ + -H "Authorization: {your-authorization-token}" \ + -H "filename: package-1-1-x86_64.pkg.tar.zst" \ + -H "distro: archlinux" \ + -H "sign: $(xxd -p package-1-1-x86_64.pkg.tar.zst.sig | tr -d '\n')" + -H "Content-Type: application/octet-stream" \ --data-binary '@/path/to/package/file/package-1-1-x86_64.pkg.tar.zst' ``` @@ -64,7 +58,7 @@ Send delete message with [curl](https://curl.se/). ```sh curl -X DELETE \ http://localhost:3000/api/packages/{user}/arch/remove \ - -H 'Authorization: {your-authorization-token}' \ + -H "Authorization: {your-authorization-token}" \ -H "target: package" \ -H "version: {version-release}" ``` diff --git a/templates/package/content/arch.tmpl b/templates/package/content/arch.tmpl index 5f5c8cd49791..2320633e5bcd 100644 --- a/templates/package/content/arch.tmpl +++ b/templates/package/content/arch.tmpl @@ -6,7 +6,8 @@ 
[{{.PackageDescriptor.Owner.LowerName}}.{{.PackageDescriptor.Metadata.BaseDomain}}]
-Server = https://{{.PackageDescriptor.Metadata.BaseDomain}}/api/packages/{{.PackageDescriptor.Owner.LowerName}}/arch/archlinux/x86_64
+SigLevel = Optional TrustAll +Server = {{AppUrl}}api/packages/{{.PackageDescriptor.Owner.LowerName}}/arch/archlinux/x86_64
From 23f1ba193d909318ded2fd4f3e9e73a52ea24f87 Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Mon, 24 Jul 2023 13:33:45 +0300 Subject: [PATCH 048/124] replaced base domain variable in metadata with registry host variable in similarly with container registry --- modules/packages/arch/metadata.go | 4 +--- routers/api/packages/arch/arch.go | 5 ++--- routers/web/user/package.go | 2 +- templates/package/content/arch.tmpl | 2 +- 4 files changed, 5 insertions(+), 8 deletions(-) diff --git a/modules/packages/arch/metadata.go b/modules/packages/arch/metadata.go index 1adb5f93c575..a7bb639f7ee2 100644 --- a/modules/packages/arch/metadata.go +++ b/modules/packages/arch/metadata.go @@ -32,7 +32,6 @@ type Metadata struct { SHA256 string `json:"sha256"` URL string `json:"url"` BuildDate int64 `json:"build-date"` - BaseDomain string `json:"base-domain"` Packager string `json:"packager"` Provides []string `json:"provides"` License []string `json:"license"` @@ -51,14 +50,13 @@ type Metadata struct { } // Function that receives arch package archive data and returns it's metadata. -func EjectMetadata(filename, distribution, domain string, pkg []byte) (*Metadata, error) { +func EjectMetadata(filename, distribution string, pkg []byte) (*Metadata, error) { pkginfo, err := getPkginfo(pkg) if err != nil { return nil, err } md := Metadata{ Filename: filename, - BaseDomain: domain, CompressedSize: int64(len(pkg)), MD5: md5sum(pkg), SHA256: sha256sum(pkg), diff --git a/routers/api/packages/arch/arch.go b/routers/api/packages/arch/arch.go index 564acfdfa505..6b1ddd62f544 100644 --- a/routers/api/packages/arch/arch.go +++ b/routers/api/packages/arch/arch.go @@ -13,7 +13,6 @@ import ( "code.gitea.io/gitea/modules/context" arch_module "code.gitea.io/gitea/modules/packages/arch" - "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/routers/api/packages/helper" arch_service "code.gitea.io/gitea/services/packages/arch" ) @@ -27,7 +26,7 @@ func Push(ctx *context.Context) { sign = ctx.Req.Header.Get("sign") ) - // Read package to memory for signature validation. + // Read package to memory for package validation. pkgdata, err := io.ReadAll(ctx.Req.Body) if err != nil { apiError(ctx, http.StatusInternalServerError, err) @@ -36,7 +35,7 @@ func Push(ctx *context.Context) { defer ctx.Req.Body.Close() // Parse metadata contained in arch package archive. - md, err := arch_module.EjectMetadata(filename, distro, setting.Domain, pkgdata) + md, err := arch_module.EjectMetadata(filename, distro, pkgdata) if err != nil { apiError(ctx, http.StatusBadRequest, err) return diff --git a/routers/web/user/package.go b/routers/web/user/package.go index 2e2c2a6e1f9d..e1620c0fd4a9 100644 --- a/routers/web/user/package.go +++ b/routers/web/user/package.go @@ -169,7 +169,7 @@ func ViewPackageVersion(ctx *context.Context) { ctx.Data["PackageDescriptor"] = pd switch pd.Package.Type { - case packages_model.TypeContainer: + case packages_model.TypeContainer, packages_model.TypeArch: ctx.Data["RegistryHost"] = setting.Packages.RegistryHost case packages_model.TypeAlpine: branches := make(container.Set[string]) diff --git a/templates/package/content/arch.tmpl b/templates/package/content/arch.tmpl index 2320633e5bcd..17fb163254d5 100644 --- a/templates/package/content/arch.tmpl +++ b/templates/package/content/arch.tmpl @@ -5,7 +5,7 @@
- 
[{{.PackageDescriptor.Owner.LowerName}}.{{.PackageDescriptor.Metadata.BaseDomain}}]
+				
[{{.PackageDescriptor.Owner.LowerName}}.{{.RegistryHost}}]
 SigLevel = Optional TrustAll
 Server = {{AppUrl}}api/packages/{{.PackageDescriptor.Owner.LowerName}}/arch/archlinux/x86_64
From aec178011d835ba6438ba71f090e296a37f6142c Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Mon, 24 Jul 2023 15:45:19 +0300 Subject: [PATCH 049/124] arch icon lint fix --- public/{ => assets}/img/svg/gitea-arch.svg | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename public/{ => assets}/img/svg/gitea-arch.svg (100%) diff --git a/public/img/svg/gitea-arch.svg b/public/assets/img/svg/gitea-arch.svg similarity index 100% rename from public/img/svg/gitea-arch.svg rename to public/assets/img/svg/gitea-arch.svg From 59bf7a7fcf015e62b841b7d354226d718afaed37 Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Wed, 26 Jul 2023 16:31:00 +0300 Subject: [PATCH 050/124] changed mirrors, documentation location --- .../{doc => }/usage/packages/arch.en-us.md | 16 ++++++++-------- templates/package/content/arch.tmpl | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) rename docs/content/{doc => }/usage/packages/arch.en-us.md (79%) diff --git a/docs/content/doc/usage/packages/arch.en-us.md b/docs/content/usage/packages/arch.en-us.md similarity index 79% rename from docs/content/doc/usage/packages/arch.en-us.md rename to docs/content/usage/packages/arch.en-us.md index 8986548d812d..e611853b9657 100644 --- a/docs/content/doc/usage/packages/arch.en-us.md +++ b/docs/content/usage/packages/arch.en-us.md @@ -43,11 +43,11 @@ Get into folder with package and signature, push package with [curl](https://cur ```sh curl -X PUT \ 'https://{domain}/api/packages/{owner}/arch/push' \ - -H "Authorization: {your-authorization-token}" \ - -H "filename: package-1-1-x86_64.pkg.tar.zst" \ - -H "distro: archlinux" \ - -H "sign: $(xxd -p package-1-1-x86_64.pkg.tar.zst.sig | tr -d '\n')" - -H "Content-Type: application/octet-stream" \ + --header "Authorization: {your-authorization-token}" \ + --header "filename: package-1-1-x86_64.pkg.tar.zst" \ + --header "distro: archlinux" \ + --header "sign: $(xxd -p package-1-1-x86_64.pkg.tar.zst.sig | tr -d '\n')" + --header "Content-Type: application/octet-stream" \ --data-binary '@/path/to/package/file/package-1-1-x86_64.pkg.tar.zst' ``` @@ -58,9 +58,9 @@ Send delete message with [curl](https://curl.se/). ```sh curl -X DELETE \ http://localhost:3000/api/packages/{user}/arch/remove \ - -H "Authorization: {your-authorization-token}" \ - -H "target: package" \ - -H "version: {version-release}" + --header "Authorization: {your-authorization-token}" \ + --header "target: package" \ + --header "version: {version-release}" ``` ## Clients diff --git a/templates/package/content/arch.tmpl b/templates/package/content/arch.tmpl index 17fb163254d5..d638f3daa6ea 100644 --- a/templates/package/content/arch.tmpl +++ b/templates/package/content/arch.tmpl @@ -16,7 +16,7 @@ Server = {{AppUrl}}api/packages/{{.PackageDescriptor.Owner.LowerName}}/arch/arch
- +
From ab1a9a1587da7bcd5f16e2140a67b8766da992a6 Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Wed, 26 Jul 2023 17:26:16 +0300 Subject: [PATCH 051/124] changed context doer to context user, fixed package config setting --- modules/setting/packages.go | 1 + routers/api/packages/arch/arch.go | 4 ++-- templates/package/content/arch.tmpl | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/modules/setting/packages.go b/modules/setting/packages.go index 5fbaaa0571d9..c4a4b9695951 100644 --- a/modules/setting/packages.go +++ b/modules/setting/packages.go @@ -83,6 +83,7 @@ func loadPackagesFrom(rootCfg ConfigProvider) (err error) { Packages.LimitTotalOwnerSize = mustBytes(sec, "LIMIT_TOTAL_OWNER_SIZE") Packages.LimitSizeAlpine = mustBytes(sec, "LIMIT_SIZE_ALPINE") + Packages.LimitSizeArch = mustBytes(sec, "LIMIT_SIZE_ARCH") Packages.LimitSizeCargo = mustBytes(sec, "LIMIT_SIZE_CARGO") Packages.LimitSizeChef = mustBytes(sec, "LIMIT_SIZE_CHEF") Packages.LimitSizeComposer = mustBytes(sec, "LIMIT_SIZE_COMPOSER") diff --git a/routers/api/packages/arch/arch.go b/routers/api/packages/arch/arch.go index 6b1ddd62f544..66e4d92cadf5 100644 --- a/routers/api/packages/arch/arch.go +++ b/routers/api/packages/arch/arch.go @@ -43,7 +43,7 @@ func Push(ctx *context.Context) { // Save file related to arch package. pkgid, err := arch_service.SaveFile(ctx, &arch_service.SaveFileParams{ - Creator: ctx.Doer, + Creator: ctx.ContextUser, Owner: ctx.Package.Owner, Metadata: md, Filename: filename, @@ -59,7 +59,7 @@ func Push(ctx *context.Context) { sigdata, err := hex.DecodeString(sign) if err == nil { _, err = arch_service.SaveFile(ctx, &arch_service.SaveFileParams{ - Creator: ctx.Doer, + Creator: ctx.ContextUser, Owner: ctx.Package.Owner, Metadata: md, Data: sigdata, diff --git a/templates/package/content/arch.tmpl b/templates/package/content/arch.tmpl index d638f3daa6ea..07f967b5ac7a 100644 --- a/templates/package/content/arch.tmpl +++ b/templates/package/content/arch.tmpl @@ -7,7 +7,7 @@ 
[{{.PackageDescriptor.Owner.LowerName}}.{{.RegistryHost}}]
 SigLevel = Optional TrustAll
-Server = {{AppUrl}}api/packages/{{.PackageDescriptor.Owner.LowerName}}/arch/archlinux/x86_64
+Server =
From ce0d3935bfc423da4b5c903b9cf2dd59b880f7c4 Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Wed, 26 Jul 2023 18:42:00 +0300 Subject: [PATCH 052/124] removed unnecessary headers in arch package template --- modules/packages/arch/metadata.go | 18 -------------- templates/package/content/arch.tmpl | 38 ++++++++++++++--------------- 2 files changed, 19 insertions(+), 37 deletions(-) diff --git a/modules/packages/arch/metadata.go b/modules/packages/arch/metadata.go index a7bb639f7ee2..d510ddfa645c 100644 --- a/modules/packages/arch/metadata.go +++ b/modules/packages/arch/metadata.go @@ -198,24 +198,6 @@ func rmEmptyStrings(s []string) []string { return r } -// Join database or package names to prevent collisions with same packages in -// different user spaces. Skips empty strings and returns name joined with -// dots. -func Join(s ...string) string { - rez := "" - for i, v := range s { - if v == "" { - continue - } - if i+1 == len(s) { - rez += v - continue - } - rez += v + "." - } - return rez -} - // Create pacman database archive based on provided package metadata structs. func CreatePacmanDb(mds []*Metadata) ([]byte, error) { entries := make(map[string][]byte) diff --git a/templates/package/content/arch.tmpl b/templates/package/content/arch.tmpl index 07f967b5ac7a..7169e64b6944 100644 --- a/templates/package/content/arch.tmpl +++ b/templates/package/content/arch.tmpl @@ -50,59 +50,59 @@ Server = - {{range $key := .PackageDescriptor.Metadata.Provides}} + {{if .PackageDescriptor.Metadata.Provides}}
Provides
- {{$key}} + {{StringUtils.Join $.PackageDescriptor.Metadata.Provides " "}} {{end}} - {{range $key := .PackageDescriptor.Metadata.Arch}} + {{if .PackageDescriptor.Metadata.Arch}}
Architecture
- {{$key}} + {{StringUtils.Join $.PackageDescriptor.Metadata.Arch " "}} {{end}} - {{range $key := .PackageDescriptor.Metadata.Depends}} + {{if .PackageDescriptor.Metadata.Depends}}
Depends
- {{$key}} + {{StringUtils.Join $.PackageDescriptor.Metadata.Depends " "}} {{end}} - {{range $key := .PackageDescriptor.Metadata.OptDepends}} + {{if .PackageDescriptor.Metadata.OptDepends}}
Optional depends
- {{$key}} + {{StringUtils.Join $.PackageDescriptor.Metadata.OptDepends " "}} {{end}} - {{range $key := .PackageDescriptor.Metadata.MakeDepends}} + {{if .PackageDescriptor.Metadata.MakeDepends}}
Make depends
- {{$key}} + {{StringUtils.Join $.PackageDescriptor.Metadata.MakeDepends " "}} {{end}} - - {{range $key := .PackageDescriptor.Metadata.CheckDepends}} + + {{if .PackageDescriptor.Metadata.CheckDepends}}
Check depends
- {{$key}} + {{StringUtils.Join $.PackageDescriptor.Metadata.CheckDepends " "}} {{end}} - - {{range $key := .PackageDescriptor.Metadata.Backup}} + + {{if .PackageDescriptor.Metadata.Backup}}
Backup file
- {{$key}} + {{StringUtils.Join $.PackageDescriptor.Metadata.Backup " "}} {{end}} - {{range $key := .PackageDescriptor.Metadata.DistroArch}} + {{if .PackageDescriptor.Metadata.DistroArch}} -
Available
- {{$key}} +
Available for
+ {{StringUtils.Join $.PackageDescriptor.Metadata.DistroArch " "}} {{end}} From 9543262c901977ffde791456bf215edd8511731b Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Wed, 26 Jul 2023 18:46:50 +0300 Subject: [PATCH 053/124] removed unused package size limit --- tests/integration/api_packages_arch_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/integration/api_packages_arch_test.go b/tests/integration/api_packages_arch_test.go index 93eadececb96..9dad5cc702bb 100644 --- a/tests/integration/api_packages_arch_test.go +++ b/tests/integration/api_packages_arch_test.go @@ -378,8 +378,6 @@ bfu02wSva9u6jV8n5u1d0x7C/vENwN9uwZUrV/5HfgQAAP//SZpudwAIAAA=` assert.NoError(t, unittest.PrepareTestDatabase()) - setting.Packages.LimitSizeArch = 99999999999999 - user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) rootURL := fmt.Sprintf("/api/packages/%s/arch", user.Name) From 3cef8c6ebe8a9402fa1f1bca3b9c72610b0638c8 Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Wed, 26 Jul 2023 20:25:54 +0300 Subject: [PATCH 054/124] corrected title in documentation --- docs/content/usage/packages/arch.en-us.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/content/usage/packages/arch.en-us.md b/docs/content/usage/packages/arch.en-us.md index e611853b9657..63a417d4aaa1 100644 --- a/docs/content/usage/packages/arch.en-us.md +++ b/docs/content/usage/packages/arch.en-us.md @@ -1,6 +1,6 @@ --- date: "2016-11-08T16:00:00+02:00" -title: "Title" +title: "Arch Package Registry" weight: 10 toc: true draft: false From 1abff64ccda450eb89ac0dbbc1ce3cb11ee56cae Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Wed, 26 Jul 2023 20:32:40 +0300 Subject: [PATCH 055/124] refactoring function to find unique values --- modules/packages/arch/metadata.go | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/modules/packages/arch/metadata.go b/modules/packages/arch/metadata.go index d510ddfa645c..c2608003a1bf 100644 --- a/modules/packages/arch/metadata.go +++ b/modules/packages/arch/metadata.go @@ -16,6 +16,7 @@ import ( "strconv" "strings" + "code.gitea.io/gitea/modules/container" "github.com/mholt/archiver/v3" ) @@ -247,16 +248,8 @@ func writeToArchive(files map[string][]byte, buf io.Writer) error { // This function creates a list containing unique values formed of 2 passed // slices. func UnifiedList(first, second []string) []string { - unique := map[string]struct{}{} - for _, v := range first { - unique[v] = struct{}{} - } - for _, v := range second { - unique[v] = struct{}{} - } - var archs []string - for k := range unique { - archs = append(archs, k) - } - return archs + set := make(container.Set[string], len(first)+len(second)) + set.AddMultiple(first...) + set.AddMultiple(second...) + return set.Values() } From 7b92529bc4bf52c4918c14c91bdd517cc7e6267d Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Thu, 27 Jul 2023 12:35:17 +0300 Subject: [PATCH 056/124] Update routers/api/packages/arch/arch.go Co-authored-by: KN4CK3R --- routers/api/packages/arch/arch.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routers/api/packages/arch/arch.go b/routers/api/packages/arch/arch.go index 66e4d92cadf5..7fd92082a306 100644 --- a/routers/api/packages/arch/arch.go +++ b/routers/api/packages/arch/arch.go @@ -150,7 +150,7 @@ func Remove(ctx *context.Context) { return } - ctx.Resp.WriteHeader(http.StatusOK) + ctx.Status(http.StatusOK) } func apiError(ctx *context.Context, status int, obj interface{}) { From 524a2a01c186ba8059dc7dbdc0a2393ba417c407 Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Thu, 27 Jul 2023 12:35:35 +0300 Subject: [PATCH 057/124] Update routers/api/packages/arch/arch.go Co-authored-by: KN4CK3R --- routers/api/packages/arch/arch.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routers/api/packages/arch/arch.go b/routers/api/packages/arch/arch.go index 7fd92082a306..ccd0d31a2f75 100644 --- a/routers/api/packages/arch/arch.go +++ b/routers/api/packages/arch/arch.go @@ -133,7 +133,7 @@ func Get(ctx *context.Context) { return } - ctx.Resp.WriteHeader(http.StatusNotFound) + ctx.Status(http.StatusNotFound) } // Remove specific package version, related files and pacman database entry. From 1e33f61375bca1c8acd0e779ab2afcaee2a314d0 Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Thu, 27 Jul 2023 12:50:44 +0300 Subject: [PATCH 058/124] Update routers/api/packages/arch/arch.go Co-authored-by: KN4CK3R --- routers/api/packages/arch/arch.go | 1 - 1 file changed, 1 deletion(-) diff --git a/routers/api/packages/arch/arch.go b/routers/api/packages/arch/arch.go index ccd0d31a2f75..a9210ca4b346 100644 --- a/routers/api/packages/arch/arch.go +++ b/routers/api/packages/arch/arch.go @@ -112,7 +112,6 @@ func Get(ctx *context.Context) { ctx.ServeContent(pkg, &context.ServeHeaderOptions{ Filename: file, - CacheDuration: time.Minute * 5, }) return } From 8b06f76360c562613a3cbd9ee482d3b706f16786 Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Thu, 27 Jul 2023 13:21:11 +0300 Subject: [PATCH 059/124] replaced HTTP headers with url params --- routers/api/packages/api.go | 4 +-- routers/api/packages/arch/arch.go | 16 ++++++------ tests/integration/api_packages_arch_test.go | 28 +++++++++++++-------- 3 files changed, 26 insertions(+), 22 deletions(-) diff --git a/routers/api/packages/api.go b/routers/api/packages/api.go index 1930d726f28a..0a5e9158bdf0 100644 --- a/routers/api/packages/api.go +++ b/routers/api/packages/api.go @@ -124,8 +124,8 @@ func CommonRoutes() *web.Route { }) }, reqPackageAccess(perm.AccessModeRead)) r.Group("/arch", func() { - r.Put("/push", arch.Push, reqPackageAccess(perm.AccessModeWrite)) - r.Delete("/remove", arch.Remove, reqPackageAccess(perm.AccessModeWrite)) + r.Put("/push/{filename}/{distro}/{sign}", arch.Push, reqPackageAccess(perm.AccessModeWrite)) + r.Delete("/remove/{package}/{version}", arch.Remove, reqPackageAccess(perm.AccessModeWrite)) r.Get("/{distro}/{arch}/{file}", arch.Get) }) r.Group("/cargo", func() { diff --git a/routers/api/packages/arch/arch.go b/routers/api/packages/arch/arch.go index a9210ca4b346..b36e4da44885 100644 --- a/routers/api/packages/arch/arch.go +++ b/routers/api/packages/arch/arch.go @@ -9,7 +9,6 @@ import ( "io" "net/http" "strings" - "time" "code.gitea.io/gitea/modules/context" arch_module "code.gitea.io/gitea/modules/packages/arch" @@ -21,9 +20,9 @@ import ( func Push(ctx *context.Context) { var ( owner = ctx.Params("username") - filename = ctx.Req.Header.Get("filename") - distro = ctx.Req.Header.Get("distro") - sign = ctx.Req.Header.Get("sign") + filename = ctx.Params("filename") + distro = ctx.Params("distro") + sign = ctx.Params("sign") ) // Read package to memory for package validation. @@ -111,7 +110,7 @@ func Get(ctx *context.Context) { } ctx.ServeContent(pkg, &context.ServeHeaderOptions{ - Filename: file, + Filename: file, }) return } @@ -126,8 +125,7 @@ func Get(ctx *context.Context) { } ctx.ServeContent(bytes.NewReader(db), &context.ServeHeaderOptions{ - Filename: file, - CacheDuration: time.Minute * 5, + Filename: file, }) return } @@ -138,8 +136,8 @@ func Get(ctx *context.Context) { // Remove specific package version, related files and pacman database entry. func Remove(ctx *context.Context) { var ( - pkg = ctx.Req.Header.Get("package") - ver = ctx.Req.Header.Get("version") + pkg = ctx.Params("package") + ver = ctx.Params("version") ) // Remove package files and pacman database entry. diff --git a/tests/integration/api_packages_arch_test.go b/tests/integration/api_packages_arch_test.go index 9dad5cc702bb..817ebc66d574 100644 --- a/tests/integration/api_packages_arch_test.go +++ b/tests/integration/api_packages_arch_test.go @@ -385,11 +385,13 @@ bfu02wSva9u6jV8n5u1d0x7C/vENwN9uwZUrV/5HfgQAAP//SZpudwAIAAA=` t.Run("PushFirstPackage", func(t *testing.T) { defer tests.PrintCurrentTest(t)() - req := NewRequest(t, "PUT", path.Join(rootURL, "/push")) + req := NewRequest(t, "PUT", path.Join( + rootURL, "/push", + "testpkg-1-1-x86_64.pkg.tar.zst", + "archlinux", + hex.EncodeToString(firstPackageSignatureData), + )) - req.Header.Set("filename", "testpkg-1-1-x86_64.pkg.tar.zst") - req.Header.Set("distro", "archlinux") - req.Header.Set("pkgsign", hex.EncodeToString(firstPackageSignatureData)) req.Header.Set("Content-Type", "application/octet-stream") req.Body = io.NopCloser(bytes.NewReader(firstPackageData)) req = AddBasicAuthHeader(req, user.Name) @@ -422,11 +424,13 @@ bfu02wSva9u6jV8n5u1d0x7C/vENwN9uwZUrV/5HfgQAAP//SZpudwAIAAA=` t.Run("PushSecond", func(t *testing.T) { defer tests.PrintCurrentTest(t)() - req := NewRequest(t, "PUT", path.Join(rootURL, "/push")) + req := NewRequest(t, "PUT", path.Join( + rootURL, "/push", + "testpkg-1-1-any.pkg.tar.zst", + "archlinux", + hex.EncodeToString(secondPackageSignatureData), + )) - req.Header.Set("filename", "testpkg-1-1-any.pkg.tar.zst") - req.Header.Set("distro", "archlinux") - req.Header.Set("pkgsign", hex.EncodeToString(secondPackageSignatureData)) req.Header.Set("Content-Type", "application/octet-stream") req.Body = io.NopCloser(bytes.NewReader(secondPackageData)) req = AddBasicAuthHeader(req, user.Name) @@ -459,10 +463,12 @@ bfu02wSva9u6jV8n5u1d0x7C/vENwN9uwZUrV/5HfgQAAP//SZpudwAIAAA=` t.Run("RemoveFirst", func(t *testing.T) { defer tests.PrintCurrentTest(t)() - req := NewRequest(t, "DELETE", path.Join(rootURL, "/remove")) + req := NewRequest(t, "DELETE", path.Join( + rootURL, "/remove", + "testpkg", + "1-1", + )) - req.Header.Set("package", "testpkg") - req.Header.Set("version", "1-1") req = AddBasicAuthHeader(req, user.Name) MakeRequest(t, req, http.StatusOK) From 730cbb126a08a9d8e53caa8d6884792c8b1ccdb7 Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Thu, 27 Jul 2023 13:55:09 +0300 Subject: [PATCH 060/124] dynamic architecture and distribution in arch repo link --- templates/package/content/arch.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/package/content/arch.tmpl b/templates/package/content/arch.tmpl index 7169e64b6944..0831ce8fce2d 100644 --- a/templates/package/content/arch.tmpl +++ b/templates/package/content/arch.tmpl @@ -7,7 +7,7 @@ 
[{{.PackageDescriptor.Owner.LowerName}}.{{.RegistryHost}}]
 SigLevel = Optional TrustAll
-Server =
+Server =
From 3df8d52fb8fcccb051f1e8bae2ec7cbfe1127490 Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Fri, 28 Jul 2023 16:32:01 +0300 Subject: [PATCH 061/124] changed pacman database creation mechanics, storing package desc as file in object storage, metadata structure update --- modules/packages/arch/metadata.go | 89 ++++++++++++++-------------- routers/api/packages/arch/arch.go | 82 +++++++++++++++++++++---- services/packages/arch/db_manager.go | 54 ++++++++--------- templates/package/content/arch.tmpl | 34 ----------- 4 files changed, 140 insertions(+), 119 deletions(-) diff --git a/modules/packages/arch/metadata.go b/modules/packages/arch/metadata.go index c2608003a1bf..2806e69ce804 100644 --- a/modules/packages/arch/metadata.go +++ b/modules/packages/arch/metadata.go @@ -5,10 +5,9 @@ package arch import ( "archive/tar" + "bufio" "bytes" "compress/gzip" - "crypto/md5" - "crypto/sha256" "encoding/hex" "fmt" "io" @@ -17,11 +16,30 @@ import ( "strings" "code.gitea.io/gitea/modules/container" + pkg_module "code.gitea.io/gitea/modules/packages" "github.com/mholt/archiver/v3" ) -// Metadata for arch package. +// JSON with pacakage parameters that are not related to specific +// architecture/distribution that will be stored in sql database. type Metadata struct { + Name string + Version string + URL string `json:"url"` + Description string `json:"description"` + Provides []string `json:"provides,omitempty"` + License []string `json:"license,omitempty"` + Depends []string `json:"depends,omitempty"` + OptDepends []string `json:"opt-depends,omitempty"` + MakeDepends []string `json:"make-depends,omitempty"` + CheckDepends []string `json:"check-depends,omitempty"` + Backup []string `json:"backup,omitempty"` + DistroArch []string `json:"distro-arch,omitempty"` +} + +// Package description file that will be saved as .desc file in object storage. +// Resulting file will be used to create pacman database. +type DbDesc struct { Filename string `json:"filename"` Name string `json:"name"` Base string `json:"base"` @@ -34,33 +52,31 @@ type Metadata struct { URL string `json:"url"` BuildDate int64 `json:"build-date"` Packager string `json:"packager"` - Provides []string `json:"provides"` - License []string `json:"license"` - Arch []string `json:"arch"` - Depends []string `json:"depends"` - OptDepends []string `json:"opt-depends"` - MakeDepends []string `json:"make-depends"` - CheckDepends []string `json:"check-depends"` - Backup []string `json:"backup"` - // This list is created to ensure the consistency of pacman database file - // for specific combination of distribution and architecture. This value - // is used when creating pacman database, to ensure that only packages - // with specific combination of distribution and architecture are present - // in pacman database file. - DistroArch []string `json:"distro-arch"` + Provides []string `json:"provides,omitempty"` + License []string `json:"license,omitempty"` + Arch []string `json:"arch,omitempty"` + Depends []string `json:"depends,omitempty"` + OptDepends []string `json:"opt-depends,omitempty"` + MakeDepends []string `json:"make-depends,omitempty"` + CheckDepends []string `json:"check-depends,omitempty"` + Backup []string `json:"backup,omitempty"` } // Function that receives arch package archive data and returns it's metadata. -func EjectMetadata(filename, distribution string, pkg []byte) (*Metadata, error) { - pkginfo, err := getPkginfo(pkg) +func EjectMetadata(filename, distribution string, buf *pkg_module.HashedBuffer) (*DbDesc, error) { + pkginfo, err := getPkginfo(buf) if err != nil { return nil, err } - md := Metadata{ + + // Add package blob parameters to arch related desc. + hashMD5, _, hashSHA256, _ := buf.Sums() + md := DbDesc{ Filename: filename, - CompressedSize: int64(len(pkg)), - MD5: md5sum(pkg), - SHA256: sha256sum(pkg), + Name: filename, + CompressedSize: buf.Size(), + MD5: hex.EncodeToString(hashMD5), + SHA256: hex.EncodeToString(hashSHA256), } for _, line := range strings.Split(pkginfo, "\n") { splt := strings.Split(line, " = ") @@ -98,7 +114,6 @@ func EjectMetadata(filename, distribution string, pkg []byte) (*Metadata, error) md.License = append(md.License, splt[1]) case "arch": md.Arch = append(md.Arch, splt[1]) - md.DistroArch = append(md.DistroArch, distribution+"-"+splt[1]) case "depend": md.Depends = append(md.Depends, splt[1]) case "optdepend": @@ -114,10 +129,10 @@ func EjectMetadata(filename, distribution string, pkg []byte) (*Metadata, error) return &md, nil } -func getPkginfo(data []byte) (string, error) { - pkgreader := bytes.NewReader(data) +func getPkginfo(data io.Reader) (string, error) { + br := bufio.NewReader(data) zstd := archiver.NewTarZstd() - err := zstd.Open(pkgreader, int64(250000)) + err := zstd.Open(br, int64(250000)) if err != nil { return ``, err } @@ -137,19 +152,9 @@ func getPkginfo(data []byte) (string, error) { } } -func md5sum(data []byte) string { - sum := md5.Sum(data) - return hex.EncodeToString(sum[:]) -} - -func sha256sum(data []byte) string { - sum := sha256.Sum256(data) - return hex.EncodeToString(sum[:]) -} - // This function returns pacman package description in unarchived raw database // format. -func (m *Metadata) GetDbDesc() string { +func (m *DbDesc) GetDbDesc() string { return strings.Join(rmEmptyStrings([]string{ formatField("FILENAME", m.Filename), formatField("NAME", m.Name), @@ -200,13 +205,7 @@ func rmEmptyStrings(s []string) []string { } // Create pacman database archive based on provided package metadata structs. -func CreatePacmanDb(mds []*Metadata) ([]byte, error) { - entries := make(map[string][]byte) - - for _, md := range mds { - entries[md.Name+"-"+md.Version+"/desc"] = []byte(md.GetDbDesc()) - } - +func CreatePacmanDb(entries map[string][]byte) ([]byte, error) { var out bytes.Buffer // Write entries to new buffer and return it. diff --git a/routers/api/packages/arch/arch.go b/routers/api/packages/arch/arch.go index b36e4da44885..21ac89858264 100644 --- a/routers/api/packages/arch/arch.go +++ b/routers/api/packages/arch/arch.go @@ -11,6 +11,7 @@ import ( "strings" "code.gitea.io/gitea/modules/context" + pkg_module "code.gitea.io/gitea/modules/packages" arch_module "code.gitea.io/gitea/modules/packages/arch" "code.gitea.io/gitea/routers/api/packages/helper" arch_service "code.gitea.io/gitea/services/packages/arch" @@ -25,28 +26,79 @@ func Push(ctx *context.Context) { sign = ctx.Params("sign") ) - // Read package to memory for package validation. - pkgdata, err := io.ReadAll(ctx.Req.Body) + upload, close, err := ctx.UploadStream() if err != nil { apiError(ctx, http.StatusInternalServerError, err) return } - defer ctx.Req.Body.Close() + if close { + defer upload.Close() + } + + buf, err := pkg_module.CreateHashedBufferFromReader(upload) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + defer buf.Close() - // Parse metadata contained in arch package archive. - md, err := arch_module.EjectMetadata(filename, distro, pkgdata) + // Parse metadata related to package contained in arch package archive. + pkgmd, err := arch_module.EjectMetadata(filename, distro, buf) if err != nil { apiError(ctx, http.StatusBadRequest, err) return } + if _, err := buf.Seek(0, io.SeekStart); err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + // Metadata related to SQL database. + dbmd := &arch_module.Metadata{ + Name: pkgmd.Name, + Version: pkgmd.Version, + URL: pkgmd.URL, + Description: pkgmd.Description, + Provides: pkgmd.Provides, + License: pkgmd.License, + Depends: pkgmd.Depends, + OptDepends: pkgmd.OptDepends, + MakeDepends: pkgmd.MakeDepends, + CheckDepends: pkgmd.CheckDepends, + Backup: pkgmd.Backup, + DistroArch: []string{distro + "-" + pkgmd.Arch[0]}, + } + // Save file related to arch package. pkgid, err := arch_service.SaveFile(ctx, &arch_service.SaveFileParams{ Creator: ctx.ContextUser, Owner: ctx.Package.Owner, - Metadata: md, Filename: filename, - Data: pkgdata, + Buf: buf, + Metadata: dbmd, + Distro: distro, + }) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + r := io.NopCloser(bytes.NewReader([]byte(pkgmd.GetDbDesc()))) + buf, err = pkg_module.CreateHashedBufferFromReader(r) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + defer buf.Close() + + // Save file related to arch package description for pacman database. + _, err = arch_service.SaveFile(ctx, &arch_service.SaveFileParams{ + Creator: ctx.ContextUser, + Owner: ctx.Package.Owner, + Filename: pkgmd.Name + "-" + pkgmd.Version + "-" + pkgmd.Arch[0] + ".desc", + Buf: buf, + Metadata: dbmd, Distro: distro, }) if err != nil { @@ -57,12 +109,20 @@ func Push(ctx *context.Context) { // Decoding package signature, if present saving with package as file. sigdata, err := hex.DecodeString(sign) if err == nil { + r := io.NopCloser(bytes.NewReader(sigdata)) + buf, err := pkg_module.CreateHashedBufferFromReader(r) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + defer buf.Close() + _, err = arch_service.SaveFile(ctx, &arch_service.SaveFileParams{ Creator: ctx.ContextUser, Owner: ctx.Package.Owner, - Metadata: md, - Data: sigdata, + Buf: buf, Filename: filename + ".sig", + Metadata: dbmd, Distro: distro, }) if err != nil { @@ -74,7 +134,7 @@ func Push(ctx *context.Context) { // Add new architectures and distribution info to package version metadata. err = arch_service.UpdateMetadata(ctx, &arch_service.UpdateMetadataParameters{ User: ctx.Package.Owner, - Md: md, + Md: dbmd, }) if err != nil { apiError(ctx, http.StatusInternalServerError, err) @@ -82,7 +142,7 @@ func Push(ctx *context.Context) { } // Automatically connect repository for provided package if name matched. - err = arch_service.RepositoryAutoconnect(ctx, owner, md.Name, pkgid) + err = arch_service.RepositoryAutoconnect(ctx, owner, dbmd.Name, pkgid) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return diff --git a/services/packages/arch/db_manager.go b/services/packages/arch/db_manager.go index 7ac72874da50..d01927bd31d1 100644 --- a/services/packages/arch/db_manager.go +++ b/services/packages/arch/db_manager.go @@ -4,8 +4,8 @@ package arch import ( - "bytes" "fmt" + "io" "sort" "strings" @@ -41,7 +41,6 @@ func UpdateMetadata(ctx *context.Context, p *UpdateMetadataParameters) error { return err } - currmd.Arch = arch.UnifiedList(currmd.Arch, p.Md.Arch) currmd.DistroArch = arch.UnifiedList(currmd.DistroArch, p.Md.DistroArch) b, err := json.Marshal(&currmd) @@ -59,7 +58,7 @@ type SaveFileParams struct { Creator *user.User Owner *user.User Metadata *arch.Metadata - Data []byte + Buf packages.HashedSizeReader Filename string Distro string IsLead bool @@ -69,13 +68,7 @@ type SaveFileParams struct { // database, and writes blob to file storage. If package/version/blob exists it // will overwrite existing data. Package id and error will be returned. func SaveFile(ctx *context.Context, p *SaveFileParams) (int64, error) { - buf, err := packages.CreateHashedBufferFromReader(bytes.NewReader(p.Data)) - if err != nil { - return 0, err - } - defer buf.Close() - - pv, _, err := pkg_service.CreatePackageOrAddFileToExisting( + ver, _, err := pkg_service.CreatePackageOrAddFileToExisting( &pkg_service.PackageCreationInfo{ PackageInfo: pkg_service.PackageInfo{ Owner: p.Owner, @@ -92,7 +85,7 @@ func SaveFile(ctx *context.Context, p *SaveFileParams) (int64, error) { CompositeKey: p.Distro + "-" + p.Filename, }, Creator: p.Creator, - Data: buf, + Data: p.Buf, OverwriteExisting: true, IsLead: p.IsLead, }, @@ -100,7 +93,7 @@ func SaveFile(ctx *context.Context, p *SaveFileParams) (int64, error) { if err != nil { return 0, err } - return pv.PackageID, nil + return ver.ID, nil } // Get data related to provided file name and distribution, and update download @@ -158,7 +151,7 @@ func CreatePacmanDb(ctx *context.Context, owner, architecture, distro string) ([ return nil, err } - var mds []*arch.Metadata + var entries = make(map[string][]byte) for _, pkg := range pkgs { versions, err := pkg_model.GetVersionsByPackageName(ctx, u.ID, pkg_model.TypeArch, pkg.Name) @@ -176,14 +169,30 @@ func CreatePacmanDb(ctx *context.Context, owner, architecture, distro string) ([ if err != nil { return nil, err } - if checkPackageCompatability(distro, architecture, md.DistroArch) { - mds = append(mds, &md) + var found bool + for _, da := range md.DistroArch { + if da == distro+"-"+architecture { + desckey := pkg.Name + "-" + version.Version + "-" + architecture + ".desc" + descfile, err := GetFileObject(ctx, distro, desckey) + if err != nil { + return nil, err + } + descbytes, err := io.ReadAll(descfile) + if err != nil { + return nil, err + } + entries[pkg.Name+"-"+version.Version+"/desc"] = descbytes + found = true + break + } + } + if found { break } } } - return arch.CreatePacmanDb(mds) + return arch.CreatePacmanDb(entries) } // Remove specific package version related to provided user or organization. @@ -195,16 +204,3 @@ func RemovePackage(ctx *context.Context, u *user.User, name, version string) err return pkg_service.RemovePackageVersion(u, ver) } - -// This function will check, wether package should be added to resulting database. -func checkPackageCompatability(distro, arch string, distroarchs []string) bool { - for _, distroarch := range distroarchs { - if distroarch == distro+"-any" { - return true - } - if distroarch == distro+"-"+arch { - return true - } - } - return false -} diff --git a/templates/package/content/arch.tmpl b/templates/package/content/arch.tmpl index 0831ce8fce2d..e58ddd669a19 100644 --- a/templates/package/content/arch.tmpl +++ b/templates/package/content/arch.tmpl @@ -30,26 +30,6 @@ Server =
Packager
- {{.PackageDescriptor.Metadata.Packager}} - - - -
Compressed size
- {{FileSize .PackageDescriptor.Metadata.CompressedSize}} - - - -
Installed size
- {{FileSize .PackageDescriptor.Metadata.InstalledSize}} - - - -
Build date
- {{DateTime "short" .PackageDescriptor.Metadata.BuildDate}} - - {{if .PackageDescriptor.Metadata.Provides}}
Provides
@@ -57,13 +37,6 @@ Server =
Architecture
- {{StringUtils.Join $.PackageDescriptor.Metadata.Arch " "}} - - {{end}} - {{if .PackageDescriptor.Metadata.Depends}}
Depends
@@ -98,13 +71,6 @@ Server = {{end}} - - {{if .PackageDescriptor.Metadata.DistroArch}} - -
Available for
- {{StringUtils.Join $.PackageDescriptor.Metadata.DistroArch " "}} - - {{end}}
From f422942a7a59584ad505d3164f0bde1f538c8ef3 Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Mon, 31 Jul 2023 12:18:18 +0300 Subject: [PATCH 062/124] fixed mechanics of pacman database creation --- services/packages/arch/db_manager.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/packages/arch/db_manager.go b/services/packages/arch/db_manager.go index d01927bd31d1..6f75bb66ab26 100644 --- a/services/packages/arch/db_manager.go +++ b/services/packages/arch/db_manager.go @@ -171,7 +171,7 @@ func CreatePacmanDb(ctx *context.Context, owner, architecture, distro string) ([ } var found bool for _, da := range md.DistroArch { - if da == distro+"-"+architecture { + if da == distro+"-"+architecture || da == distro+"-any" { desckey := pkg.Name + "-" + version.Version + "-" + architecture + ".desc" descfile, err := GetFileObject(ctx, distro, desckey) if err != nil { From d06930382506e063d4ace5a78556bfd5668735cc Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Wed, 2 Aug 2023 00:23:12 +0300 Subject: [PATCH 063/124] fixed pacman database creation algorithm, arch UI template --- modules/packages/arch/metadata.go | 2 +- services/packages/arch/db_manager.go | 69 +++++++++++++++++++--------- templates/package/content/arch.tmpl | 16 +++++-- 3 files changed, 60 insertions(+), 27 deletions(-) diff --git a/modules/packages/arch/metadata.go b/modules/packages/arch/metadata.go index 2806e69ce804..a51b8455b046 100644 --- a/modules/packages/arch/metadata.go +++ b/modules/packages/arch/metadata.go @@ -38,7 +38,7 @@ type Metadata struct { } // Package description file that will be saved as .desc file in object storage. -// Resulting file will be used to create pacman database. +// This file will be used to create pacman database. type DbDesc struct { Filename string `json:"filename"` Name string `json:"name"` diff --git a/services/packages/arch/db_manager.go b/services/packages/arch/db_manager.go index 6f75bb66ab26..7eea1612120a 100644 --- a/services/packages/arch/db_manager.go +++ b/services/packages/arch/db_manager.go @@ -164,37 +164,64 @@ func CreatePacmanDb(ctx *context.Context, owner, architecture, distro string) ([ }) for _, version := range versions { - var md arch.Metadata - err = json.Unmarshal([]byte(version.MetadataJSON), &md) + desc, err := GetPacmanDbDesc(ctx, &DescParams{ + Version: version, + Arch: architecture, + Distro: distro, + }) if err != nil { return nil, err } - var found bool - for _, da := range md.DistroArch { - if da == distro+"-"+architecture || da == distro+"-any" { - desckey := pkg.Name + "-" + version.Version + "-" + architecture + ".desc" - descfile, err := GetFileObject(ctx, distro, desckey) - if err != nil { - return nil, err - } - descbytes, err := io.ReadAll(descfile) - if err != nil { - return nil, err - } - entries[pkg.Name+"-"+version.Version+"/desc"] = descbytes - found = true - break - } - } - if found { - break + if desc == nil { + continue } + entries[pkg.Name+"-"+version.Version+"/desc"] = desc + break } } return arch.CreatePacmanDb(entries) } +type DescParams struct { + Version *pkg_model.PackageVersion + Arch string + Distro string +} + +// Checks if desc file exists for required architecture or any and returns it +// in form of byte slice. +func GetPacmanDbDesc(ctx *context.Context, p *DescParams) ([]byte, error) { + var md arch.Metadata + err := json.Unmarshal([]byte(p.Version.MetadataJSON), &md) + if err != nil { + return nil, err + } + + for _, distroarch := range md.DistroArch { + var storagekey string + + if distroarch == p.Distro+"-"+p.Arch { + storagekey = md.Name + "-" + md.Version + "-" + p.Arch + ".desc" + } + if distroarch == p.Distro+"-any" { + storagekey = md.Name + "-" + md.Version + "-any.desc" + } + + if storagekey == "" { + continue + } + + descfile, err := GetFileObject(ctx, p.Distro, storagekey) + if err != nil { + return nil, err + } + + return io.ReadAll(descfile) + } + return nil, nil +} + // Remove specific package version related to provided user or organization. func RemovePackage(ctx *context.Context, u *user.User, name, version string) error { ver, err := pkg_model.GetVersionByNameAndVersion(ctx, u.ID, pkg_model.TypeArch, name, version) diff --git a/templates/package/content/arch.tmpl b/templates/package/content/arch.tmpl index e58ddd669a19..8cbac8943819 100644 --- a/templates/package/content/arch.tmpl +++ b/templates/package/content/arch.tmpl @@ -23,7 +23,7 @@ Server = {{.locale.Tr "packages.arch.properties"}}
- +
@@ -47,7 +47,7 @@ Server =
Optional depends
- + {{end}} @@ -57,22 +57,28 @@ Server = {{end}} - + {{if .PackageDescriptor.Metadata.CheckDepends}} {{end}} - + {{if .PackageDescriptor.Metadata.Backup}} {{end}} + + {{if .PackageDescriptor.Metadata.DistroArch}} + + + + + {{end}}
Description
{{StringUtils.Join $.PackageDescriptor.Metadata.OptDepends " "}}{{StringUtils.Join $.PackageDescriptor.Metadata.OptDepends ", "}}
Check depends
 {{StringUtils.Join $.PackageDescriptor.Metadata.CheckDepends " "}}
Backup file
 {{StringUtils.Join $.PackageDescriptor.Metadata.Backup " "}}
Available for
{{StringUtils.Join $.PackageDescriptor.Metadata.DistroArch ", "}}
- {{end}} From d2d8aa700fe5fe22fe8da1668408b25012cb5de5 Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Wed, 2 Aug 2023 13:35:41 +0300 Subject: [PATCH 064/124] documentation fixes --- docs/content/usage/packages/arch.en-us.md | 21 ++++++++----------- docs/content/usage/packages/overview.en-us.md | 1 + 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/docs/content/usage/packages/arch.en-us.md b/docs/content/usage/packages/arch.en-us.md index 63a417d4aaa1..7f1cd79a854e 100644 --- a/docs/content/usage/packages/arch.en-us.md +++ b/docs/content/usage/packages/arch.en-us.md @@ -38,31 +38,28 @@ pacman -Sy package ## Upload packages -Get into folder with package and signature, push package with [curl](https://curl.se/). +When uploading package to gitea, you have to prepare package file with `.pkg.tar.zst` extension and it's `.pkg.tar.zst.sig` signature. You can use [curl](https://curl.se/) or any other HTTP client, gitea supports multiple [authentication schemes](https://docs.gitea.com/usage/authentication). Upload command will create 3 files: package, signature and desc file for pacman database (which will be created automatically on request). + +Following command will upload arch package and related signature to gitea. Example with basic auth: ```sh curl -X PUT \ - 'https://{domain}/api/packages/{owner}/arch/push' \ - --header "Authorization: {your-authorization-token}" \ - --header "filename: package-1-1-x86_64.pkg.tar.zst" \ - --header "distro: archlinux" \ - --header "sign: $(xxd -p package-1-1-x86_64.pkg.tar.zst.sig | tr -d '\n')" + https://{domain}/api/packages/{owner}/arch/push/{package-1-1-x86_64.pkg.tar.zst}/{archlinux}/$(xxd -p package-1-1-x86_64.pkg.tar.zst.sig | tr -d '\n') \ + --header "{username}:{password}" \ --header "Content-Type: application/octet-stream" \ --data-binary '@/path/to/package/file/package-1-1-x86_64.pkg.tar.zst' ``` ## Delete packages -Send delete message with [curl](https://curl.se/). +Delete operation will remove specific package version, and all package files related to that version. ```sh curl -X DELETE \ - http://localhost:3000/api/packages/{user}/arch/remove \ - --header "Authorization: {your-authorization-token}" \ - --header "target: package" \ - --header "version: {version-release}" + https://{domain}/api/packages/{user}/arch/remove{package}/{version} \ + --header "{username}:{password}" ``` ## Clients -You can use gitea CLI tool to - [tea](https://gitea.com/gitea/tea) to push/remove arch packages from gitea. Alternatively, you can try [pack](https://fmnx.su/core/pack). +Any `pacman` compatible package manager/AUR-helper can be used to install packages from gitea ([yay](https://github.com/Jguer/yay), [paru](https://github.com/Morganamilo/paru), [pikaur](https://github.com/actionless/pikaur), [aura](https://github.com/fosskers/aura)). Alternatively, you can try [pack](https://fmnx.su/core/pack) which supports full gitea API (install/push/remove). Also, any HTTP client can be used to execute push/remove operations ([curl](https://curl.se/), [postman](https://www.postman.com/), [thunder-client](https://www.thunderclient.com/)). diff --git a/docs/content/usage/packages/overview.en-us.md b/docs/content/usage/packages/overview.en-us.md index 44d18ff48230..6cd82b548234 100644 --- a/docs/content/usage/packages/overview.en-us.md +++ b/docs/content/usage/packages/overview.en-us.md @@ -24,6 +24,7 @@ The following package managers are currently supported: | Name | Language | Package client | | ---- | -------- | -------------- | | [Alpine](usage/packages/alpine.md) | - | `apk` | +| [Arch](usage/packages/arch.md) | - | `pacman`,`yay`,`paru`,`pikaur`,`aura`,`pack` | | [Cargo](usage/packages/cargo.md) | Rust | `cargo` | | [Chef](usage/packages/chef.md) | - | `knife` | | [Composer](usage/packages/composer.md) | PHP | `composer` | From bfe6b37ecf9ae668c3deb02bc9502efeb225fb44 Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Thu, 3 Aug 2023 14:37:09 +0300 Subject: [PATCH 065/124] reduced package size in tests, additional tests to verify package signatures --- tests/integration/api_packages_arch_test.go | 506 ++++++-------------- 1 file changed, 140 insertions(+), 366 deletions(-) diff --git a/tests/integration/api_packages_arch_test.go b/tests/integration/api_packages_arch_test.go index 817ebc66d574..8a90a75df7ab 100644 --- a/tests/integration/api_packages_arch_test.go +++ b/tests/integration/api_packages_arch_test.go @@ -22,357 +22,129 @@ import ( ) func TestPackageArch(t *testing.T) { - const firstPackage = `KLUv/QRY1QwBGoVZQ0SgriRpAxihjQU4nU1abf1BN6nGKR/bld+8hZmisY00IpH7u7t3+4kQSpHb -p1+3roM678K21giRyF40KUz2hNLQrAXR9A4E6QPnA/X6wzHZ2cgbLKlm0Edvp0eNJLW+Dl3H7bKL -TevnR1bIGyutpv/6ugOzr3o9SbqpSabOgdSVpFpKn7cy2UkrHb2eI5eNr7TSsmAdqZMRB2nZeKP1 -ddAzvvo2M9f6nnFmQrXZIzojSQszzCP54Uv7RNZuXX48UcH8+DpEK2yX76mRpaSxrDVjBtfI2nG1 -s8LFLFZff8AaHWzBtzqQuoaOMY0qtcbzsVJjmGepqcT6urVu+sKSks+0rfmBPSZJ2ye2M+GRZfk4 -xqNKSueP3pK+NX90lSQVnxdZaknplpTXEktLvxgX5OJKjZb3gYzrVrrTD9vYPrCxtk92nG3kruRI -rN/LB3QXOvxWyHD6LtPK8u3r8u1MWlJN65wV9iM3SVodb5V1Yak5mpFUUlY31EmoEoN5x0pKYyhJ -pWStns/Peh3IkkpJO60ZWwdTU1L6dZkmKVlp9Zz1voONxFhpdb3vSK2dlH9Rp6IcijHFGnt28JnP -12E76FhpNfbrGjaY/Yu3rLsQl+89LLW0NG7qtY2Zm99qR1Jp9ctc72Ok5U38y2iVViOtzrhWJTmS -6iwrfi8bS1L6wh/JUiO1phUvrZLEGsvPD/2kl6XSSko72L2WDnls7di95151j+FYI4lMU8HYrWi8 -4yAlLQ83a36SXozEan5Wap0/Pn1zbrDEes5HbqmRJNZ+K2z2FndWmimGcizFmtHaJIrR0L8Rf3T0 -0JeL0RUvSUnMgZc1zvYvxN0OvKv1rEtS+pFnZEqyHO5JYcvTcTomfNphqTC0wo40VVZbczssWVr9 -XrbDkpFaUharsctn6rUdr4PtsFyJJdUZ06cVvkpJ0vhWfB/YLr9rpKQb65VW+3Qn9yM3DYkR6no1 -Ay1MpVXxbFFpVTwsUYk1zrQyXa9tjxW2w/tU3P5qbL1SS6ylI7WVGmuHD13HONOnQri6JYUt1Zhi -bS1s+Y5fs1apkVRDj0xxFCQookvpMqZPRM2yiOZ5Hovw5j7WTfsX92Mk2+nvJfzsMpa15veyCAxr -tH8zpiBBgWn9bCsRtHlcxAO6A92BDsIibpUVQpjxxyIoSFBA1FlSafGHmxQkKGStOV+XsRHNw0bq -rhFG+KyOERQkKO4vrSkOR6OxLAoSFJ5dJOIDD04RnUwIGI7GHUhd9ACF84jQn/UICiefuhE07gGd -JUK1xpzH9hgGI0LmrzWl0RK2HgN6czgYj3mxMYyjKSthrME06HAQemvOtaY0KEhQzAcvCnMOLhHl -P8lSYmlEFCQoHHhd87BErF4UJCiuQ9i95mGJaB42Y88UJCh8Oto8LBGMggRFdvNWhiLXepUCwHUZ -P4oCAEvS2FIUaqmmVFOSZFGotSQMBQCKJFmUUgBgwRErOMEIUnACGIggBU/uxq2Jir/ify+XxJ8f -S19p5+m/l47l8l+d/162RmMt0P+9bIzGh/9eKvCVj0TqnPeSuZOdQ5r3cnn4ad7LZWu+9V7vxjbv -pavNexnf472kkGzey8it+XG1fr95LxvlzXv5XrLmvVQdZevcL5NWQ+qZVua9TJ5h3kvW0Ll+ROa9 -ZGmJnK1MfTXkEfL3MhJV6w3p72UJ1dRiTLH3MvnHZRd9paWj8xuhuFZKEut543uptAOzs/hjW/43 -r6TeS2wfdwdjrjUVjVuBr8eH6rXxqRK+lyqxCpbUaveVuutsY10/wveqG+qKLQo2FexWKHevP2o/ -qXbXJ3PwvWTtF489R95qeh9GfAy/99K+97LdR773kMcHrPK9lw1HUs3ee7mSauaTpCTViaMtCLn3 -0rGS6nzdxC4zTomwsTQVDVNII9Q92L136L1c9d/LtlisxthlWo1plIwo8YTSp5NKcrCbuKXy7qN1 -rPMR8Ud8D7baWkgrNpGWtFZSDZtWbEKp1UwtbCI5YUKxWA2rlG66CWWxM7ESa1oxps5ER8grKyWN -K/61bCOt57L9pZnnp177q9YsM/2kr+Pr7NNfbn7rSFSv7Vsxddz1nykKZm9H3TcSa+qOjTiMlhTE -oY9ja3YiDuI076yk+q2L7zBWUv/AZ/TFxfOY6xzW9i+UtBozkIrNW2ofK5/G5T9KKynpBOIPPXLT -bc7kkVtKWGoWq9sjq5USi9WeXbRPP/L16fxGZPM4ju419vsQt2+bfvNI2xmn1TZivJLFLdKn4ond -dAfiCNmCCkuWJi2HctOi3qOYepvW1u9KLS2L1Z5fUXesu0y2zeJWZA1vUqPdxEjKKtbYxly3DeMz -fS50i+vN9DtDN/GqYhXL/CSJJdaNfWgDF0Z0RNZUJOtWuOyPJY0fziFZOvrzen5aZ9KwWP2ANXbN -7WAXr3tta+4MnZ0VUs+RNyndWdNdbNzQz9kxdqza6UtacVMlSyyWQ/yhzrQykfwP/InFup6pw5b1 -ZlrhUlYjUY6mgiW3QqWlNY662bfK+oBVeobuOgdeF6knTj5QOsiSVqm1k2vN/siXVF9X8+PaZ6V2 -Nmz+bJqjH8kSw6qxLO9lw2iJNUyavZcNIzGL1Q39x62Q7DQtDGyiwYzs/Elinb95L5mPPGdvWmuq -N+uRsDpLSdIsVkeu82bGZiWWJNaMeV70th+xYWn1fLFZKemkL2xWWhA2qyFsWGo0bFhS+gFxTXP8 -MCs1xw+z4r+/TFuJNXMbye6XSa3zIxv3yyQrMQvWzTO0wkwro6ykGj9kZZLIJdOgk4wjJZH+Uml6 -xpEkSUqjVVosWDduJ8O0tDrX2eN1zjMMVElRKSnq0erkCGuIf2LLmnlezfXa1tDGFtZrHctWuzr/ -Q1bbPGySVFM3IxP145Op07dJYhGTz8311dsQCGQlpf8+qiP0c0dubi0xn59zdjo7XUy+mzaS3Uef -bSGSsaSlIRDIe5kolVb9ltdXr8+1cZqW9sZh7FNRaUnLP2qFbfNqxVLr7LhxuZFay5mdBb90MMVY -SanVinKwSuotrydJtdMqIVNaXreeX0IMbZLUauYX2iQ12i+0D+2SGIm1jp7vUvi5b51dEqXUhjo7 -nU2KVZisjrO33vuTugmdfUBn2622StuSox2MqdvKmAnqtt7Iqds6HXVbl3TUbWU4fIapEUmrIy3t -5nOf5mmN8FvC6mhJ9VSa0u7jQzRDNx9KW0r6WkydbB0rNXpa5ngTdXqujM62UsUam/BYaXK0jTTt -sdJlgj7N2WnLDCNlSokE3wfSJGks0OGaxG5iiqlu6Geneek073TkMo3grtTR/Mx2srOf5ZMklRKL -NXNjq7HTJpVYSvoydiWVod/UvejqfR3VF8HPa13fwvz0H7Mti6c/K/bpsM+e0cXZgEmcKVnJvHAk -F5JBSrtHvsisuFqTVnKTSgsuMWWbkKR+kRj1CwdyHWTeZ9Li3Om3rqOHspSwpBqSL9nC1OFLx8I8 -d62CscZuQ0+EcZoj00UmtDSYCE4IirLCkjozHRy9l5H6jdqN1N1nRkvOKCl9/SEnH9F7RNNdp78+ -YiKWaWWi92lp7NEzUY48gjM2broYQTCKlpaSlEQfrbT6wSha1ooSC3kvU5KUtH4fYp9D3sukM/WX -8EHey1mhg6jIQu8l/ncrlK9Dqx9nI/RersTaQe9lklZDT7bT88Ni9MyHn/fy4ed5Lx94n4fQZ2X8 -9GGkZbEcupvfS2ZxPWN3O1O/t2bHNKCvpiKpYKQksfs5aYb57G71d05+LxvTeXaslPSi2GW/T7YO -NlOvxRHM5vSMV1rNHIxXUu1g3G5UWbNPxe9lUilWHCl+L9/LlLS9L/F7qVJbiXVFBWwev5fKL3Rs -Z4XQsYKxw8thSCP4vpeLAuHr8PteNgZx931a7jkBsSstqx3Lx76X9r1MqqXLtzbUvpcKv7TvZePN -hzqtjn0v2ZqNXY2dW4fQPbTvpYP2H72WfS9VZzup25ocjZPSX8uPZm2aqiSmEeZzbecmZUlaPel7 -mVpLupjDp++lkrPTUJYZqjRmGH2ZnTL0vWRKUp0hTt/LBd3NlFZY4nqWz2Sn9L1MEvzIz4+jzDEV -R+ZijdsJ7XwvHY+0sU4638uGTtYRM1WcKWenbrIMpZspyvdyNSjfy3YPP2RkQnk3RCgKEhR+8XjN -CpNLO13kRhhhgW3ODgUJCnkfIrxxGd8/goIEBcQeV0uLgqKhjx+KC0QeRfQSUFwgWYutVQIUtCAF -YGYBDA5YQxSWHlyBBlXAgx5a4DIXjDAHCpyBBCx4QRLw0AWHANwBywQRJoACGDCAB3VIgx1K0KEJ -amigAogQZcAEelCCDGYAAR8cIAhDIIQJbgACPRDiEH6IgB6AcAQUbOC4MYjDACSwgRMQQYU3FAEQ -BvjAFI6QQgJsIAE+tOABLYczUAEBOKAAtoIhkgagAQ0JIFEGMTgUEIMjoEAIChNoBL4QgwsOPOFE -GHJAiC9ggQxM+MESKuCGxk8MwMAJW9CAAwqQhxnQPwjiEGZogyGCAAg6QGQdHhLQ0CihA3cgwwgK -sfRhDDvQQQoiMAg4UOB1YAs3kAEcatCBNlwgYcMb3vCAELCgRABKYAEC4GEFE4B4IQB9eMMCLNCD -AyhmgAMhArEHLeigASdKfOANBFjAGsAKsuAGHQzgAEOoQGMsi6lEKRKQwAUvUEFSYsYX6qGWdMil -cvAFU9ib5sEOvnBcXKfe9HT34mL+RZL0iVmvXzDmIu0iSlQUsfhQ1Bi+tl6rYp326UfzPaKcFuuk -0ooV0ubWqKWO19UX8Ue01BprXdAJBlMYUDWp4WDcpGa9DsOhp1KGsbyE8ZqKxhwVjOVIDof0TL+p -hjw+JjLnE5nF6s+N9WObOF1Xjw+xr9Bqx0+dE8Lws3U0ghvrpcFEC1PuGk2FwipaWrn52MQmRhOS -4BnBEX8o9WD0YNRaWVikEnf3kuJQ7aZkWUlZtE4WFjEdqT+ge24hDQv40toHl3BWYszICmGYCvFr -k2olMHbp9yEOfAfiDURhh/VamO9ziLKP3CSJPXLD8j+eQxSWWnRVdoSvMPeQT/MMMUZSnhursaSS -akAM+aOEz0dWxk1oYUY2dz60rPnQcxmn+BD919RvlQ61CDGZQC2xlp8f1Q8/Dki1z7J/QZ8kOR74 -4wMbU+wiq+Wuv5fN5MxYarGlWMevl2FvReU5morGqnggZ8JjMzkrRdX88jrYTM44s9bt+sI4N5s2 -M1KSGZFntRKL1dJ6ZtPdxqXnF3NLq6/LjeUM4QhuCuu4INcZsqmtGh/MVlINv+2Tr8v5zprdxY0Z -B7vnRJLaCSwrVCyOkbtV7pdZVsWOiLWO4olpnYw9hmGuqWirgiWfPzv6dQ5OGlKjc3BLquN1fB2a -jvH65xy3EkxGzCLGEGOH7ZPxc5+Lv1J6Heycq1RiqRGdNwtfkLsKIyVpSUnf66K7Sz7hJKbuNhBe -2Bs3n0Gf3XUSeuwln1CWXvJ1/EDW/fMPdVv7n9zIxvsf/2dpRNc+9PvRb8bV6YcfCISRcjTsGqiB -HtrmsU3Xz/eh6ERHnLT4Y0McyMGYoc2Mqh2+lRFbXPHzYqOf6VGlSXRInUM9/zuxrPdS3sezk3Z6 -/vvgEg19G8nKciI/dqSkHb4Nhv054fciN/TxdTZiIxI6GrHMob9R6cUSg4xuJJZUrLOxKE5Kasyu -jki7Hj/dddbRLTEnP67S+Iw58FunnPw08tNoJOayqUlqjZXUw1DP2DEr6UjNNnIDhomog1WSWCSo -SeecQkMykySFDkMRQCAYeDQYEEvms3L9E4CAT2ehMBAIBIFAMBAICIsDgaSnSafMjMjISJIkaQ4C -XLq8/bNe3F9AF32MALrc/TNySvNnOU+w8N0JzdEMoFFLMff80mUN12eqxz59Y5tqx0wXcdEAl9C+ -AYCuaPFyTd8HbQ6THRtd7JcDPBeuU5wnwhqiBy0RJhErdeUD41vcFZo208nBsLSPhs1DA8h/HWZK -FENQPoNp0/hzbxrcDnLTO6JUDRsSC7hf8tlFgbNrhpz3o94COuVslXlubzn6kat89igykIKM4031 -U45jOtlfchCJKuf5KuhEUAAkYqvx6EVMGI2lHk7nL+JeE9h3tYdrQ+pW47/DlP7IEBxTbpJ1n/7H -v4wrARE7egHFS2a2hoHWo8IKS/6UHXwkN7B5aFOv3tXqcSupg8XH2UVx3MU68TcvPPT3Ttuo44qO -uZaiRQrjQZBG8A+2L9lg1OAoUIsp2JpktXx9tOMfK7Iwh+0nh/uzwxOn9PQ5nEhyspUoVLolQx26 -dssH5Y+R7z+kxJtmd0L2GkCSFH0WzLDAojBleIv7HFCj7I1C9vdk1sNMH7wd+dNKm7JEM65GbogJ -hNXKKqlRQAUiP91fY+GT5qpyh/w4KlkQ6eu/DLRfHwuUMYE4NxS96m7xNIfHnLI+B8YVaId4ldrC -523sQiLO8w12hoeLzsRIUjO6/E49e8rCelkhi09sfl+JWCZumQ8/TLkaa/32Z1ro/3wEhLCjXphO -OmjbKxvkqkyWpdVci+ZZ2CV7TGcDXER1SkeErM1u9nRp9ljgqXdtrTsPGnoLz+mquIc6HmJwIpV7 -pAhJh/zUZrx2u+nq3NfOMp9rOABVYEhAYGHsg29/qfE5KyHMKsFank+CeHH0iHjVB03dkKbLnAJR -PGH3riGzCr3INZityhc8XTUqH06xftHJU8CGPIroek4mIVA5/K6BleACKWw7r4Vm5saPSwqL1vA7 -J/KVT5AzSmjcYer/BuhGD3JJudm19NvNw6jYTqqcEYesw19VtK20lIubcQzF2RKrlrcas6NDRY21 -K9FARId+qvRM6xTI7pfeiWO29/le2ZcsgoDnEllSjVamnN1bEDHUGiKFMOotmSD+IKMqWOPBVm72 -a97kjZEEr19zDboNGfRuKtCyQoaCMmv/SWI6oerpOkz3Bg5aeFDtgrWirFwYWD1kOctQcr9VcTWf -5QSxZIZ1iGRJDui0xlQYPeTpIW+UxFDmR7KtsCUG1q6DVoTFctnxJExfi/g/8aH0dBDP0fRAMbX7 -vVa9m1OETK1rL1VtQ6yu83ioMk2kMSOL6zYG2X8dyNPrIBNVnKJMp9TGiMBpTrcKE+NN1nAVIheM -FkGIkp2hEu1cljZRSQa//M2jqniS1rodjVuG5YetrTYxTrGgKwXKDVZH0YQatAAUokTKMaJMvNaK -IeJvEaIbGXOi8TGjjp4PpWQ9YvwoB1foZ23EmpGAdllrFs8NzfVl3Nq2K2rl0ReNrRWLaG+jS9uU -zpNDmVvi9lLRmb9jrMIjfiuE5KbAXVDPpcalvXjgLB2T/vObHKYXWla0APm4/IARUtYtpDs6o9Pi -Kg89Mx+dBJEN4VpXkSWIiINFgF+aOG3p9OOr0wuBJSC0DFZyQy3tCgfgx1SiheEnBW+aMwvnNtZc -T2ewDRq3YyHNqaihYZmhZ33CHlSQycrOeyrXv3kwNEM1rA/YERYYWxWcPEMcwDl2gWRGA0yw2amb -PGasItrwCSlvPkQsVzzg9d6Oa+a6PGINOTHILUDFqkmpYgR6qulUj3DwI0msidJ9S2L0Sht2+VAf -0SOZZ+Ci87XbIDeRgoTT3LSvg85kSjh0TVL9FOHv+ucwKSOqH3dd9yXSNuNuD87eR01Fv0nas7Bt -SMbCcOX6rMRaMUWAh5EuFZ2ck2wOsGhN+0Y33EtoGgbqrF8zx0r9+rGY3KxigdbY+ZEGt0fULofF -UrReRlXh7qjS+Wv59thWl7q797LlgRM7QQFHLPktAdS2N9xcF6F7SV2uLxmRe89bun3Lc5u+5CkX -s/RYVHIsjm8NXaEUjTSVNOPcxloPHQ7wnhkA0G8W8JWEExhODtJDy1EIv7h6RFXw0jIoSi3gHUSA -cO30g6/8bFAt1aDz95LbPmP+G4YVFJFeOZ9nsUt+/9suqZ3LXfGvxrZ2iX1pSrVApgSA6X8ByI55 -EDiYmUhoKotD4Ss17IjMMlrCGx0uK2IPNbb61BfAWVkEfzUmWCrSQssQZVO1UA42lYUUls1NiXs2 -xzgMRnJ8p3yJjAHpjzKTjFi9QLbGaBJz0KMEOjLehGfnNgzX+O5y+880Hh1DBkEy7rWkoMlZOSLN -EWw/221YRkDyqsLy1bh1omiTbb7tmBkbnpCWknsilyDcCvbjL3KaQNBsZvyn1Bg3ERJgWTP/eFE/ -Kk7R+MsoUMhsCj570o/A1pEA8PkCh+GIZCVJtOZ0+Feov2CYgMlCuhygFsSsxI76EQyaVWnDv+fC -bULYXNqCJLCFWskicdnd9Q5efqy8bPiiYssAFyY796lx+8czpBfX+Kp2b4B1OgCuLZbeXFgFo7Tf -ePihzuK/8hopN0l41CE3jP9sG27XQoIWwvdDMcTU7heKtC9vx3TQT+TPsBKVctrG4hgesgItmKkW -nMbPHWpfmlMZER5BKwCW2Mse/6EzwqJJvc/aEi5q7l2BFgJoEhFA0A/LXgOa16Rj8jG89k7mGJfd -YVcMlcIwuI+CAdwFtIhnxtGk1udtOshzgBpbrOv20N3X1SDM1q1cQOv4M9xck+oxfNxhEzx4UUGl -neZuwBgAWEmBy2jDGxvV5T4TbRfly2WzL5e4UkjhslZx8gFfBYbZ4/JDA9pcon5/3ulTn9HRzj8v -/lwI/KaT+DedCC4v4AkBIGVE4MTTZv5rKeTfgV6/rL+eLv41gWSa2lBbuxMtpEzmNs8WqliwlblM -hr4V9OXjhyXgXJtVUBj9lp+Q7occfNHrd9rN6noO4HucH2QqwlrJdW2g08jgUjWthsaHs53Xdeze -4UZZM24JV7l6AvtiQNlcOxp/GJ9QnoLC7JUpeZvE9Jv4pBZKq8ggCK6hUcjsirSw0RkNx7sR/e8A -jN/meeCXW2FUWQNM0vruVWLNZXekV7hwIRfFDv9lKKrG8UH3BNny5Em7+x8zOZrvWbca4uVfqCmH -mapQTOnklpi8jmXXvUebt1T05LUucTsyb+iexy87wn/v4VYFCMIHC7gqPJxUgHK4R3taFnnT8xFz -56hp2bISjAS6ozTT9EIkwlJOvli/yOhyo7Be8YOR8xTtQGQfHK7qk9und4jBOK32SPot8NdWWjon -3TKZGFI6TUvp6s9LndsAbok6JlzcPvSzLz3NLsECK5B3XLxDKzE4RmALqjvfwJVMD1DVIg6g/Oiw -AEmIdbtVo09Kd4sEkCWY0YHjS2h6E5GCV/7kHJM38gdZn6tATaX9kELKhONLAPqGFG3/g91UzdcD -yGqX/PusP0iAM8oNbKcqes2t4tuYOI1iDwPgQ0NX3eup4/nfvqnsQU0Ki+M/Wa5NRLhXbbU5GUOV -BysGnLR207W5xg5B7ebxqz9mybJ0gwlhlj42aIDHDkq97jHehFNjo9V/36Y1CTGKvUKjTFNq/oHc -h3yBxiCjSlMGqyXT5S99l4cGqzOgF+mOz3f4joEFhtbPIxz9vmCGWh3J4h4o8BEcdI9xzJeYr8XK -TA1zb/P7HWY8ZImQPXLIuZcXqviP7VbzogN9sZMUFABg7IV7RJMj/JC7jfp0fXG8c3K3C2Hq8yOP -QX/20CkZG1NcdhGZ9SZKO84rPoSpraWarkbiw2JIfpF4n52kEdlMPhwJcgR6b/2VV1mlgM7rG8pW -Bfp4wQF48oxWwyIK1Gdmzl7vVt4PimUlULqq0KKjJiOOcUP3b1F50SBZA2Sxw64qPl4dJfVGv9GV -LDgJWKUrS9mcDJ5vjuUcOF0S6J32jxpqvAeA6LMSarNQ0Kr/w/YNhgPnB+CJGSUNBty9ugLkXhO4 -0qi4xwzbYTMAG8Im0PFE7CWnD+2vZCe/Qo5FDC08I89XswOgR6mBDGfRIQpINQe67xXxRI4GU7L8 -H2Bw0hP+GNih8Bogw5iBzyihPJMLvabmNTRMUoWqoO4nsrJWOxKpSa7gus0ZxxRAgiEYL0LVboIc -y+CldfK8f4YE43KZkqjBWBNQsL3P73G5/+h0lNKgv8w5X8w8OE0QVCbwag97F2znojl/dXJjOGXg -ggqVxK7M8/v3l38TJFuXxO3OSpf8XfWc9gwy2RPV+/ywLFdV3WGhroJopQIVCmInbseg4f7TDTzn -KYkjTPdEQtbNPFnxx79SinVxkkdHGxTcREvjwvTbPxWpWguyc23B63AZ/oEOMjN74TGu5pFKCKZ9 -DjI3FDItF+b/I382ohFpi69z6gQt0fl6D4FU2BQlxulAZrwB67uQ9CQ4FHPu14qiK8p9+oR13Kpj -hVuXqOpLGpyJMBa+9ZIexHQoAiGJTHPL9gzEYW9lpo5R79qxCFodxBt54r3QF+hH0MFCoEppC053 -nNE+OzewlSkRDheXfq5KYN0PILFv55SCvoIcBU6r+kWSQ6z/hxBJGmdi1Ff41mGcVPk/ru3OmKtw -co0TQ0Nvz+ewk0vfq6d79zteLmAXY29dl/I2ETM3Y80ijVpRM4kfvMl0IqiN8rJJA0pb23lCGtIs -ZwtJQCyFI9DAuKuZeBPaVkKjc8oivdT4MjCUZUXjXvPmqU23icbq/0eUrbJqVbXdFCaUChPbVUBf -vFJ2XudBrYuVwAMYfps+L/xSzycsWpjP27owmjJVQBoTBs4kAAVlOd92Ish/ADfzM5/n5amNMAtA -KoK2d3acwRbvoPThgaC6iAdxSsyiwNABkbrCc0s+7mb+Xd92bnURx5EI7yvCxNdW5mCQLHAz0Wqx -47iPpcezPcaW3JYLiptqJ/DSPFQz1ZfywpoZqLJdJgO/BvxKWOlwHAaq7xnrwgk/Dco30jHHe+c5 -4q8FuElSF1pMIf7CtpaoF5stvMKWHkT4KbkseBzhSDyKAMKmP24EQHagROionvNu8ebJIoK/PSQM -S8KqUIlCcp6sHr5wlNxmHDL3MV9G3XlK516DS+di6GIF04ZQlxQzaLXMkHZQzMfxUzRC6dLuSrLH -7YliFis4XUHFvrmhdktnXOvcUUqk6UV/SDQpI1CNfSn6nJ+lf/d+lsScDfem0J/AQrAzMq9UAa8+ -tnjOv6bocDTh8pI6fy05nmQ4rP9Km/CM4QJ68fbgubbfjfa0QiMpmOA6FcoeLoXktlDh0RkhEcKl -AS+o+uKsL0tNKtVdC4DgdqVDQsSke7NtqRtlf7Xk2NzUbmY4WpLgohOd+E5tINpJ6ae43NKAB4Ab -sQdFI55QUjA21EH9nqb6QZZYvTC1tkZ1AjPMulkCO81Bx3xToeImwKAw8dt1+3cfsKxjHMKwourZ -CSunY1KDDxy4Y/0arUDJ9I4hxXryLzr8a7tkGnTjRDX7SL/KEougwLw4zvxwqP7lYXAbeC3yyumP -qkc2GG5nOIsMDPoK5jiQZ5Fb5mVxAzY0IbJ/k2LAnOWf0I9tXNQEC4rgbzuCIiLBEk2ADglcP9Oa -7bV8bKiThPdl3TVOb5nym1OpAhcLCx2WO7pOgpHLSuCSSnlREDphTTd2Ztpt9pcRDeyiSEaN44ud -smiYikcA2Mk=` + const firstPackage = `KLUv/QRYlUcA6nxwGD6gsLQNvOd6axb347rXM33zsZKpE9vgsf0Ld/n//99rY7sJi2vbpBtD3se3 +qPGsw5p1E4nc33ohr0LLfwiK+m8BUgFTAdxyVWq/0afytiwo9jGT9PRVrkKxE/tIJX36mOzYG7Wv +cveuyTpPM0VnlTUUv80UWkllNk8Tq7PKL8Sb6bbk0hjizOq2xCR97KvMnb1Ul3dztp2HuEjs2egb +uesZ3W+P2oIaN23sMXUNXg7nDnfbeRbFzuFsxDvR03h3fqkuHoq4a7Eo/DA3anWX1V1WZ7vOaItZ +7Lquc+wZXaorKbz152+3ZcFLdWVR7H3bzou+lj7al49HFxRZR5fq2piiNtQpNVMbvVTX7dYr2jGu +UEkMWyGNS3XtIovSVprGddxJsxidXmo1VxlGM5vR1Mmzp0yrp9XqrjQ7T/duKXuyM5t281IrLts8 +eqkucOSbVi2L5/AyHrMtx4ZaulQXq9v2a+kiKumluvZN4pb7WrrofrbVWX2prgva8Gvponqprm7B +La6uLop7TgDbdj7OC0B2wiqdFz1pWZbVLcvopWlbpRPAlWXbnBMAUgoYLE64YEGjxAfbByo5v3nW +p1LjdguTngi2fnvKdFse19roT9nJUOQZ/SpDkfPQ7tgqpRqjnn6gkt/M2a5x9pust4/i1vixkxy7 +VN1m16+0ocX1hP2UtltPiVXpU6lxEyZvxuO3WombPib13R+tT+UFP/3x3Mfwy75+lZ8eQw1+J+yr +7Dict9NXWTmc60/Zl27v98PqUwlyHudwll33VSq7rPu+viZbSvbLlm6en/rMC7Gn88HbORO/Ga+y +7DV5M15V7Knsorhvy6661Ut/yp5K/olb81W9kz62WmGvPja/yn21tqC8HU663nxNduzdb+Qu6jP6 +VPIRVR97TWKvcV7Fvv4mOU/Or4+SeY/d9lXqsV/eNdPGPnW3mHpM2pxxkwhqMzVrnVXy222HV7jv +20nboarGTZXK1M+nEucu7vqox2Qo8k3O7KvkGfcxxFwmNzqfviZZVOabdD1M3h3HINfzM/mFkutO +3FrUqKdfSzPtHt4Q6w17TeZ9dfaTynp3p3E3qdz+9FWyum71pLXOClqNvfZUglajOsd5i3tj8CRl +aMHO6sx5eLX4VLK6DlO937BqObLfJL98j11I9c+vn8m90mD2WYp+9lRmYCpryr9Jvq9nL0tibSvl +4e0o/SprBpc27LPPlh77KcW+bxKH875dg7ePHM6d43C0K1R36ech83D67KvMotg37in6maQhtx2V +PSbvHru2nVyJW3MmzuFcWZzPu7vLNSh23eWomrnlVE/baQ5n3JKUaixrWu+sL6y9hjj3G67CVmd3 +D0Oxy7Fvz3jVb8jhTKnsNKj9fCpzCHI4v5ctx1v1p8wcczhnNfTwFve+8tPfpE/lx+GcPSbxasXh +nH2VPbxJF0V/ylW+HM6on/ILbcfhrDkO5y20I+eWw/n0mLxJmNVty+GMBUXzUl23CXNR3320b6gk +HoFi9zdr0C7do6MPXPfOulRX3mMR3h2/3pXH1eU0LtXV8etdoYj5vknXF2I+XheYuMu73e4bXhXX +hVNGaz1hKlrE+AB5hBg/RgQQAsPFQRQFf26EsFG04gwPaqAAlSeClrQACD4s2bkwg9uzJgTSDBxE +FtQ41OHTIlDYgLMqcdA8UEMDwRsrbA7NJXgOzZkjYgKI+WWRoiYICyk45FRxM8NEkTxZBBBMAkDB +oKYNBuFCgA4zQxYlE5DOHLiyxaQw02JKmRNDJuCw00NGi+4ETQMCSLQM+gAmYOI1aACo0BUygUeY +OgxWZEmyBQYWMYEQIbLixRDSEi584GARQAZNjid9hiCgWnzQEdTkI4kVMnoUgJFwwNwp9CfCKQ2g +HU2iQOHwAwkpgAoeAnhs0CDoi44KEgQQiGFUoIuX1ptEZgCxGBPCEBAEJAbA+JLExwADRyzQeqLY +zOZEgULGjBfvdd5Ev3I4N2WvyaYlWjFLCfsqM+5jh9VZmzrlvMsK6n6S4irES7djT+W2HRS7JM7E +T49pVXr6M/qpp1L0Rl47zjw4/UwqiZ5l8c+kd/mcN8TZs3c/VmWBFaiRHKSYQggxNDMjBUmyHHEE +hCDonKSQDxJAUFGMQRAIQxAEQhCGolCgZsokSJCknBQWWgMMyBbo9PMBsFwQnPDPtvRb/dr0T6ca +rtWEfRFesPpr1z8Hfi1LxlZ7GHuAqLAB/NNkT2fx9grmc8euvTXOAFacJCHIHICRPX0dBGk3RXcV +O9Vbi1MpQyibg34fKNSG4gIAImaNUbIu0zfT9FobmWjMOOItiiByrLEAB6d8YsQZbHhLrsELnaDN +YwLqgJZWL5Vift1uHpg7gRyXkl2anG/zeyw1mIGTPwyZqFfredyRUiNx6NtKPFgwNYHNOPVomgJM +BaZp2FQUdJDBjT42M7a1kJEekgrdLcA1m0WTzMdEe+7KcX/e8MttkzdAsRKuH4mndiUf8jnMxYj8 +J80ggDEVFkFGIIOGiFOY0GDf8XoZ0rarlrzAoXFcSQfwQzQE6hBdPKxtrb3F5CUsCvwdwovq+VMV +5/sQEYCRaRsDohAoCw3tOMLg9pbOS+5ARZR9JDx5ScgyXOo5O6pbsBi1V7aja81BSEGZQFzHzpIv +NvXb/U5ZNldpCgWmyJfNGgiJJwIzzEgG5DChlpsh2W07AybfEiJWG8YZTMRChZk1dCnsTGxXdAKb +yEkYN9mks01OZ1gyVQaSakNg88h7upWYM40YIx4EJuyAwSP1yhfZPk6ZxNU3TrQygytGzwUxS2QO +IkJ4YHo8TXvkN5vq2aWjzeEJBFWQHV5rkrC1HkgK9Zk0B3RoSs2/mBmuI0iGSroNA/xAictf1DqT +0HVqqjQg1ACXRABLTDRjhUSa9kjdMcX9YypiMXY4iECBSl4oQeIQ112zLCppoz+HULQL+csQyVJc +CLajh0i2JJvzlFnmg2XIndtEIyTU7MyAiE8KO7SOxFc9mlYAyXKFFBFJkas615qu3PEr6bRA27AE +weALS7CuDOK6KF6YdYvOWdiCAoHI6xI=` firstPackageData, err := base64.StdEncoding.DecodeString(firstPackage) assert.NoError(t, err) - const firstPackageSignature = `iQGzBAABCAAdFiEEQ6WNuVKuWHQJqhdSETT46/mKoG8FAmSsBgkACgkQETT46/mKoG+Qsgv/WcrY -nMQ0BH6n+MAc99puw/VEj1o/qwwwrYV1jNIIWfWaMnlZHiTgShxfcfDgnjhMDnJGslia2mIBZoBy -JRUf/KuWye22MH3uzBIK2gz4RFt6qA57ZaU7y3YYrJ6pbwNvwayjuUa9Z8PEDs0t04mEiPhJYJVU -cM0mtWZUuYQt7kdnqm8Oxx0RSMPamQ/c/wC+h6p/DWMngoWgmDKbKwNA9/UjDrQujuwwVU1bUiUk -cX/cACNaTVcsIjSe9wD7xLqvBblCdjvl8fumYRYH2t/oj86HTLiCswEMYqsgf+NQ9ZFBSxd82Bay -vhhZgJPEjDBykHKdhqD/WZdOE97TwlbySg8l5oY9yUBrV0iim+FuqX+bD/X82IzE11597SG2V50z -K1/Xdvx76yOUHSO0t/731d/LyGxEVoD2ITwM5GgJgTYnOByaI4mkr9iz6OeIwHVgcTFYcQ+lMP5c -jMF9gK0ZWKyVAYH5FFhDJAm+lxWpcBu0Fnpj56LGqcaDORsvIVkA` + const firstPackageSignature = `iQEzBAABCAAdFiEEdez1wBMjH7w+s6HCOjpq1/IBK5QFAmTKScUACgkQOjpq1/IBK5RDjwf/YZQS +QM9JgxtNqp9jxT7eqyNtYY5Jwte6Fpq6RpOd2qbkJodJVrAp0HAWPS71W9k0lhvOSeq4hL7jufUR +y5gmbvmN6CqOjoMAnSxe51OKgZuPb8fbWrpt58BqtR7iCtPav1tMG9lpIPWSLS2/jxoTIxjcgVQJ +05s2bqUtpoDy5fCB2Y5tdIPQbMjSr6/TkmWg4ulwactJg46bWgowwKxnWzUx7IYPjC3lwknU6Mll +DArW/X0zMaFT3zuMBJFlbSzv59tcH0yICa1yMRtWCnbufZo6Q/BUvZ3P3Wr/APokAEt5U1U/u0EK +Qck04tbpECPL0eABIygaLmwqii6wX6NIYA==` firstPackageSignatureData, err := base64.StdEncoding.DecodeString(firstPackageSignature) assert.NoError(t, err) - const firstPackageDatabase = `H4sIAAAAAAAA/+ySwY7TMBCGe56nyGWOTcd2PE4QQoQmZSPaBTWUAxfkON626rYbNV6p8PSoXYTY -XlGF0Pa7jMf6JfsbTfB96DbLoRiKUet7N7gARETGmFMlovNKTPL3+XQvJCkaRHSJz5zz2Ae7H9Bf -v3Uu95+Ak2pa3uazEuGPVRgeUv7GSdxtlnGw+/hHHwDwWQwA3+X1s/5LOa+rj7cIYigAsCjrMcJk -ve9DdAxFnXUbu/QAOK6rryVCykIDYPXUEQDOCl0vZgjsRdtmaaqMTlLyJGzD1EonrNOcOAeA9U0u -NZ/SjZAti8TIVElrWSestffGi4aYnWq0UVLphpWV5IVsMs+Zc+auJXbS2hYAF/MpwiqErn81GvmD -3Xb3PnYP29EvvaNQPh/fIDyN5qi/qKZFkX8uEQSnaZZppRgAP+XjD/n7co5Q2N363kaTh+16932z -il63dudWfpmZt3fb3SHuH98A/OsNuHLlykvlZwAAAP//LUzfXwAIAAA=` + const firstPackageDatabase = `H4sIAAAAAAAA/+ySQW+cMBCF9zy/Yi9zBMZgbOiNAm1Qd9MISg+9VLbXTqNNAAFRov76it2qavZa +RVUkvst4rJH9/PwGZY7q1nrMY8HBTmbzChARSSlPlYguK8WC/Vmf9hmTXGy29BpiLnmcZjVu6J/v +unzcGwE/VLvyOtuXCH9FwXtOxHfB/eF4689q9H9OMwC+GAPA91nzov9a1k31+RqBeQwAi7LJEdrJ +usf77ZKt8W6Y7/oOAPOm+lYihBFFAFidOwLAfRE37R5BHwxLnYoY06lUWnCKJFfGakYsTEQIgM1V +FsbiNB0ro0MTkTah0TzRsSXhbHhwziklVWStIKYkiSTlSlspuBBSac0Sbo1zi9i23iH8mOdhehcE +9lk9DPfWN/1D0D91dgxGO/QAmNX5FcLZnMWAttoVRfalRGAipVQmIlkOu8nyT9nHskZou2PXP3Xb +m7NJI8D//vCVlZWV3/wKAAD//xmJdqQACAAA` firstPackageDatabaseData, err := base64.StdEncoding.DecodeString(firstPackageDatabase) assert.NoError(t, err) - const secondPackage = `KLUv/QRYPQ0BGoadQ0aQriRpA3QL6NcPUx9kgUekBUgLXzPAM0Aznbpi2NaaRPb+7u7dfiKEUuS2 -Pqxqd9d+9La1RgiRyP2TztWrbVgeLHxZWAYKEATtA+0DzcGtStzBkmoWveSrdIljSUpbj6zfbrjB -nvRxjjRyx0qr57u2Dj3si1pfln7ogmeD3nQmqY7RpaUObrHGNq1jicfHV1ppWbCGs8VJi3R83KG0 -bdE7vto+ElPq3nFHRrQZBHpjSes6zCf438eaIWuvDUMyVA//tR5N+tqjyxSytPSNlOLrXApZ+6VV -0reYxWrrkEfhgzXXUujN1s/vnVAlpa+ErKQ4zrOkqMTaeq1NesOSiu90pRhyHS5L1wy7oXDIcMzN -cagS0/ihK0tbih+6WpKC2UGWlMS0khinJJaadhCuqMGVHB1tfg3rNTrUkNq35qewrhn7xXViV3ok -1u4j5DX7Hm7pa/j8hidl2Lb1+FUuSlI9qZT0NQQ/S1oNLY20YUl5dCOpxKx+ZotPpYbzj5UYpcEk -lZa1WuaX1EKPJZWW9knxre0lRcXotuG5pGWl1VJS++05UmOl1dT+IyndYvwLKhn1UIwp1tdxe+88 -tv66vcdKq6/b1s/B6l+4VZuNeHQus6Skpu9Dbe3rWOxWP5JKqz9ial0jLX/g20artBxpNb6UMumR -VONIr/voaBLTF54jS46k9KS3abUk1jc+zvOXbpZKKzEN3eY6euRQab/NueZoc43GHAt0ngrGbIXj -oRYxqYE0SXFeejUSozivpDR+mNti7GCJtZQQrCTHkli7pa/Z19uYaaYa6rEUazbps6jGM99OHNKS -PD+uRtPbJCYx6FnWF9e373ZDzy6lJW0S0xAsIWOW9XBPDGtyQ/mQL3e7kXEmfT15aKSV4m43srTa -fex2YyMpSVmsvh7fobY2tO11u3ElllTjO3/S9yotS19Lr82vG/7mSEsrrFda7dIh3BD8dAROZuvV -TtQwlVbFs0alVZHZohLriyd1mtraDulrfzbT2+4oVHolJbGOj6SspLD295H1e/HM9L3VSmLYUgpT -rGt9Hd+vW5oTjUPSrFVyJNXPIWMawwQDeowN3/kkKI5GPNk5NMIfm2uT9S/swwmu0t3H97jhGynF -7qMR91FY3+LLMMG4kz6ulMR8/hoBeQ49h157zwhLI33v4fvQCIYJRjSdJZWaB1Id0iTDBCNSirFt -+JJgJR5IhbPZEiVcUqgEwwTDvrIozOPhcDSLYYLhuEESGMg3SaZ0NNB4PA692eAFGBBkMl/SDzBa -/Nk68DTIa5WEwtyhajiKQ60FKoxrjXOoph7NowEfD9Y0rvFoMI2pB2swjwbZsjQc6CiqgWxxHgwT -DJlvw7AG3RMwfp7MxjDBgJ5tD6SSWL0YJhjWo9vcA6kkHkjF1zHDBMNlmw+kkmAMEwzcpKUOA1Nq -lRrCNnyzYgzRYBS2FKOxlKKUwizLYrDVLA01BGNZGqXUECf4wBEi6EAJRMCCDaQZ9+MVBb1P77uP -K97Huc5nGvr57uNjeXxH5buPisOxmufvPipswn/vPjLvmZ/A2SD3kTmEm0ce97FxNz/u46oUW2qt -91Mf97HRx32EDuI+zig+7uPEUvyXVrf/uI+P8sd9dB/Z4z6qluDa3K2TVr/peFLHfVzecdxH1q+1 -hkDHfWSpkThbePrqyGHk7uNEQqs7oruPJ5SiFmOKuY/L54Yb9JWahs0tjOBaaUmspYXuI9PQw63e -h8ryt3gl5T7emt0fjDXlqXDYimsdfkRthZnG5z6qxCpYUqvb09lsqwpr+vC5V/1MV6xR76lgtkI1 -a93N+kl1sy5Ze+4ja7/463jiVk+bG9Cx69zH6txHxebo3EcOIY9G5z46Hkk1c+7jSqqdXxKTVC8P -Zb2ouY+PlVRj2+Rt2GkMfI7mqVDufHMyW77NvUfu46LvPiqNxep7PabVd8LYgPHKF122yCToNnk7 -Ms5ClIZUQiYOgQ7CVtf6Jr0oUpPWSqpfT3pRKLWaqXVRLClQMBarH42xSUeh6m0oVmI96b2zoWgY -eWalpS+9v1Z1pOUart80c5yprb8oxRHPX9r6ta3+/NXkKz2RUFvb0jv9Nv1njHrYu832jsR6OlQn -UENJKgLNDFWKoUmLQM9DK6n+2uBDjZXU5/eNvrjIzrENYl3/gkmrrxMp+HydNdeYH4+fmWbS0kvE -P4LgpxUpFwhW0sKSYrFagWC10mKx2nGDNTcEW5fN7SQ+2S9srq/bj3j92vOfHGvjm1ZXeO9K9Spw -ZnrlbdKhd+Fj6zFsaZ60HoxNjXJZOdQ+Sle3KylpWax2/Go6VJt1cH2atiBz+JMcik2NpKpiT1Uc -6/VpPJ7ZPq+XWjy/8XMUjipGb8R5SSyxfmqez2sa8AHZU7EsW9GwO7b0hTSP4NjmS+v4J5WLw2I1 -5FHYFDd0G7TNdaW48WtV0jcdS1yJ0aElvcG+z3yM/V6/qn1+nPQ+NLLFYnnEP2o8qTPBD7m/WCzr -ePrrSC2e9C1mORb1eCrYYitUapSnzSbb0kghj0bHr9kGPdvg9KXF/GJ7bGmVlIYwpdhzdEm1bRTn -1i7prPJh8uWjPDpHlthVjqZxH5+Gkli/pZn7+DQSs1j9zIdYGsE+U9O4KJ7rxMa/JNb4H/fReYlj -7I+iPNSfleOjztKyKBarJ7b5I+GzElsSa+ZkJ/naEPiwtFo6+Ky09NL2PSut957V7z0sOfo9LDEN -ibRH8ZudlRS/2Vnw3VtHWYk18zrB7tZJSuMcH3frLCsxC9bPO5M+PKnDrKT6QiJ1FolHR3ktOo+0 -wPlLJen4TpYlMY5WqbFg/XiVjqOk1ZhS65C4Tix2aBv0jvNUYlRKjHKTQnjC+t2Xt7J2sqOY2qo8 -s2991NbHqrSuxg+JtD6QuiTVs0kJCvow4+nz65KYhMVsMXXUdiQSWYnpt1n1ZD7uicWKkpjLxxhD -jX0uFr/JOsHtocuukdhoUtORSMR9XJRKi35H66vXY/r4TE370+51yajUpOYhlL6ueLViSWns92nY -kZSOEkPN/fhgqrGSUqsZ1R6N0ztaX5JqnzQ+xnS0Xh1/fPfVJanVzO2rS3K021f/1SY1ElMaOrZL -3Wy/ttosiqHPbFW2mpjKsEihVi219uVs8rUKea0qllYaq5IeDd07vVLGUEyv1Ep8eqWyTa90xTa9 -0oa/dxw6iZM+UtNNZpufnxS+7/joQ0mqJaMw3R5+NPFrMs+ppKVtvRPC9bGSo2Vlj0dBpWPaaFzp -VKwvCod0pkfXydMO6Vwo5k+MfXbED07GnFiuzW8ui6N5j/YkZhNTTPUzH0PPR+h5qCUNz8ltOtvE -WTaEW30c85JUWiyWxApbT5U1qcTS0tapa6HO/J7NwUatawl1k5ud0vTrwz8fIpVl8fMlvS7bddkS -Nigdb4EyLWuRF4/lIrKI6Zajg86CS3nSWmxSqbknZFRkFKdfJDb9AorYFpH2ndQ0h/pra0meLC0s -qY7iR7bu9Pfx0TjOm1LBmMJsR3ni+MQT2SYyahyO5MkokmB143RHQlfiPsLpVqJYOJvNUrKilCSm -rX/UIkTiIBLZrM9fL3EkDE/qSNynpq9L3pFgiUuehE+TDUqiJ5E0SlrSInnJSqvdk0iatSRLjbiP -aUkqVrcfqc4j7uPSePrH5yLuo6SvRZSkRu7jfWdphK1Hq3OrMHIfV2LdIvdxSaujjKt0nBmUZOlu -dh/dzdl9zM/mj8y88P2ZG2lZLH/NYveRWVjHt9nG062luDmO5+upWCoaaUnMPl7acR43S/0bhN1H -h2yOHystveZt2G3Gtb12qK13cpmUju9Kqxl070qqoXsVK6FMqZle9zGpBOmF87qP7mNautqP131U -SVmJdUXFe/66j8rt81slfc8vc2+7q903J9e6jw0TXftw6z4qLNKs+7RaThGpKzWr/cZc3cfqPibV -sWFLn1ndR3U/VvfR4WSePilU3Ue25FNX3+bV32v/qvvYXn1Ir1XdR9W4yumVpsfTYnTX8SWSPg9V -8c6J85h2a3KyJa2W031MSse5mL+f7iODMfRMhp3J9HXY/Ih9OtN9ZExSjSM+3cf1msVz0nfCOo7v -YJ/TfVyWm+Pj3CZrDr0TudjjVb4q3cdHjhVSOaX7+EzJWuJIBWXC2GeTDL/YZILRfVzKi+5jN3ch -UYKCcTYkFMMEwy/+Lklf8lhlk3iJEha3YgwxTDCizST8afg6ZsBJn6uAYYLxrsOlpMVgPPNvZjxg -4hjQQcB4gFIWa4CABChIAJBLaMIVDJCAPeChB1BgeCHKArQGKQMWFOAHRVQAV4EJcBjDCgSQb9CA -J7BACUN0wT4koIRJDeRQQgEcqGJQwhcmUCGEJiQ5cAANJWACHAhRhHKAH2hQA4kP+mCGMSyhCB0E -oAU3QEEF3EAGH6jgBk8wQQQOUiCELsQuLGAKOHCCEqZgAxkYQAovcEEgOsAEoQLqDM2DBeTgDhBY -wBBgUCIGU6AAEFxghBbgocQQmmCG6IUw8CAEgKAB88IBvgcIIMKDAkGwwhFUaMMc9BAHHxwBDAeo -wx8cwAYtdCEIR8jA4w1HMOADYsDCDsRghz9goAQfpGABCCrsYRBRwMMYkkAHIRShCRBEgmGFNHAh -BXnQgBRGAAYLOA3MYCGCAGUwYgBqWAISIgtYQBARcFsoRBmIKMAi4ogqkIEGASjCBw5QVGMpalHK -Ax5IwQpMkBiI74WCqBU98qige8HU9efJF7oXjwvr05+WzV5cyL9Ylk4hqfULxtpEuZAsSiJh8COJ -wq6t1FYFqaw/XyIdRILTYp1UWpC++VgqUVJD26ibOETSUAopXVGKBmMaTz3J8WjYpCS1UMOjTKfT -aNzScE+Fwh4VjfVIj0d0PF9RHTl0zkTKDJnF6scK61wfKFtThx+xp9Fqv5lKGd33uLY5uQrrxuFI -Gqa8OZ4KhlUoaeHn4QMfKJGR5crJnThEKfck7kmUZhomUYt7c4l5KMWmpVlJWSi9NEziNJwOec15 -jTgs3sda8z3RarzXiTTiODTitialnLjXo9uPQO+hiD8Rdf1RWx+2ziOqQvCzJAbBD8MPcR5hWFJg -o7Hhe4Y1SMyK44g1knLsWApLKinHu5FDTrjMkTpNRus68bHyo2bJj1zDN8GP5ruebmn0SIERi0uk -JNbxcVbtbn5EaM2r/sXMS3pA7sP8FKbYBVarWXcf28G4sdRiS7GGT63D3IKMezwVjlWRo4bCYTsY -K0ZR/NG21w7GFzOlFevrXtxMVtxIC25IHCu8erFYHatjJpt9PDp2ECtptW1YYRhHd3IVhjVcUWsc -1aQsCvNlK6l+3zVj2zC2kuJmb18Hui2nWJKS4kb6mHqhxL0qd+s0i97HpNY2r7yTSsac47D2VCir -giWXLxu6bdBdOkJhg66SVEPr13ok/d71rvlVTrAIL4P3vnvbrRnfbLO9T+e00G2QVSqx5IDNFQtf -UbNMIy2pSUtb22CzDU7R4p3NPtFd158mn0WPm23xOewGp2CabrD1m6PaPn+eXml/xk98vD/758Yh -sfWf24du8aVQu5sj70nUQ7ntiZ7oX31yfZo+th9JUvSkxXofdgSKoHud+Uio2t/XCG+99GYHHZ2l -S+hcoL/ZoOn4XYpmuY/RZsctVun4bb4nnvl1gpkmBf77SEu3ax3sek7h9gI/869tdVIn8bU5qcyj -t5DpxRJ7jXYklhSk0rHmTUxSWF09iXVBXjbb6qOVxCCcG53wGXvcr01BODtwdjgS8/jQJSmOlVR2 -puPbnLX0hAaGh6hjZZJYJKhJ55xCUzKTJIUOQxFAIBh4NBgQS+azcv0TgIBPZ6EwEAgEgUAwEAgI -iwOBpKdJp8yMyMhIkiRpDgJcusT9s14aGND1FyeAroT/jHLy/FkU9ISDEc0AGo0p5v5juqzk+kwP -2acRtqnemOkKFw5wCd0NAHRFNy9XDD9tcsju2OhivxzguXCL4j+ezhE9aEg8iViorSAY3+Je0bSd -TiOGpS01bJ4aR/4TMlPQGILyGUymxp9/0xB4kFvhEaVq2JBYm/slzDMKHHEz5Mof9RQEKmer+HP7 -1dEPXeqzI5KBZGQcbyqfchzSyfaSQ0h0c55VQSeCGiARYY1fL6KEUSw1cjrvIs6aQMGr3bg2Umw1 -fh2k9EyGUJlyJFn+9Kn8y1yJEbF/GFB8ZKY1DHQ90qywJKjswwd+A/uMNnUAXq2PW8maWHy8vRfH -LdZxf/PNQ39z2pI62OiYbyllpNAQgrQEfxt+qQ2jGkeBWkzxLTyr5eqjef9YqYU5bJ/5658dnzih -p0/gxJ2TrUXhkS4Z6tCF+0Eof4h8GyMl7jS7E9qwAURJ6bNgGgsshimEG7nPjRrlhBSyBSizEmby -we3Il620aUv0cYbkJp9AmFaWSo3aVCD46a4aCz7pVxV3SMdRkQWRvvDLoP0qskAlE4hzK9KrGhdP -c3jgKatzYHCBdser1C38emMX432ez7Azv110Jp5JzYTmd/rsyQ7rJQxZ7GPz+ytieX/LTPthOtbY -A7i/1kJb6COIhB2uYboKQttx2SBXZVIureYtcs/CLrkxncLgIsop7SJkbbvZp4u0RwVPPW5rafSg -z1t4nK6ReyjyEPMWqdwmxWI65H5txrXb7VznfncWFl2DAaQCwwD+C2Nn+PZOjSdaCXpWCdLyNBPE -M6VH36s+6NSNYl3mRIhiFbuHEJlV60UuzWxVvWDrqlF5fYr1Ip28GGyIVES3MzMJ4crhFRv4DC7Q -ZNsVLjTnN36BUlichl8+kW+LgmxTQvsdpv4XgW7Qkcubzq5Lv7E9jErupEobcYc7/GlFG1FLublK -xxCdLaHqv9XgPHpa1Oi8Ej8iOhCoSrBpJf/IOReKBaAf/Axq1pIsdtvGJVJkiVaqOLtYiBxwDTm1 -AeoxnyDEKOMSWPOBlmz2c9bkiZEIXO8mDdrNGfxjKtTqQhZZGXf/pON1wOq5Fcx/DQhY+FPJBXOd -rH4YqD5kPGeok363cFWv5QhjSQ57AsQSH5BqxTQWPfj1Q9ooiS7mR5mtkAWG1KhjeQTFatnhSJha -R6xzIoPpiRDPeZ6gNJ377YZ7N9JFptRrD5W2gUxdx/KQM6OmMeaLa+uV7LcD5Kl1mI0qXKGMl9TG -WOC0pluFE+MZa7gqkQuKFsFEyU5XiXZolpaoJINfvv1R1TxJ67odjluGn4etrTYxTrGrKwW6G6xO -0ZgatGAUokTKMSJMhNZ6IOJ/EaAbkS3R+DKjjrIPpbIeOX4pBxf0MzJijZGA9llrlssNzd1l3MK2 -K2pV0BeNrRULam+jS9uUzpNDmVvC9lLJmT9irMAjfiOk5abAVVDPo8alvXjgLCmT/vmbXKUXula0 -YPJx+YARU9YtSnf0odPiKw+dZT6aBBEf4VqvIksQEReLDL80EW3J6ccXJxcCQ0BoHazkBlvaHy7g -x6hEC2NPCt40ZxbOVay5n85gmzEux0KaU1VDczJDT/qkPZAgk52d93Cuf/FgqIZqWB+gKyygtio4 -eYY4gBnsAokZzZhg81A3edpYfbThA1LefAhYrvjD660dv8x1d8QaYmJQtgCVVbNSxXz0VOOpHuHg -V5JYJ0r/lkTZK23s8nF9RA8yz8VFJ9dulzwiBRNOctNcB52SqeDQP0l1KcKn65/CpERU/9x12ZeI -2Iy7Pjiwj5oXbZO0p2DbABmL4cr1WRFrXYoATyNdVnQ+J5k5wKKa9o1ueC/BaRioWL9qjjX162cx -ebNKBVpjJ0ca1B5RudwWS9F6GRWFa6NK7K9l22NTXdTdHcuWB37sLAUcMcmPBFCmveF+XQjdS+py -vWS07r3e0u1bnm/6Nk+5mqVvUclhcb81RIVSZKRU0oRzG5t76BDAe+bgQ7/ZxlcSTGBxcnAPbY9C -SOLq0aqCS8ugIGoBbxAhwrUTH7zys4VqqQad/0vu9hn7bxiuoID0ynye8S757W+7ZOxc+or/G9u1 -S+ZLU8oCWS8ATPMLAHbMQWDAzE9CU2kcal6pYSIyktFC3Oh2WSl7qNXqk74AYmVJ+KuRYHGRFl6G -KEvVgnKwryykWH5uSt+zOcbhMZLjdcpLZBiQ9igz6YjhBbJojCAxhx6l0ZHBTXg6t2G4xnSX7/9M -I+lYZHDS4F4rxZqciRFpPsH2027DaQQkVRWWXI1bEUWbb/vbDpqxoQppl9wTchXCLWU//iOnCASt -ZsafSo27iZAAy/L5x0v1o+IULb+MwoXMpuCZJ/0AbJ0IAJ8vcJhHxFaSSGu+h3/F+guGCTgspMsx -tSBlJfbcj2CgWYc2fHku3CzUzaUhSEIo1EoQicveXY/g5QfkZcMvIrYMdGGwc54aR388U3pxm69q -8wbYpwOw2mKJzYVVMor6jcce6iz+L6+R8pOERR1Sw5DPtuF2LSRqIUQ/FCOm9n+hiH15e0wH+kT+ -BStROad9LA7iIQvQglm14FR/LlD7jJ3KiPAgWgGQxV62+A9ZEZZO6vis7XBRQ+8KbCFIkwgMgn5Y -9hrQfCYd84/htXcyxzjmDrt6qHCGwX0UC+AuoEU8M44utT5r08GeA/TYYl23ju7+rgZhti7lAk7H -H3BzkVQP4uMHm6DBiwIq7Tx3A8IAwEsKPKNVbwxUl9OZaKOIL5e5fbnESiGFy1uFyAd6FRjWHt8f -GmBz+fr9+U6f4oyOMv55qeZCUDedhN90Aru8gCcEkJQRgRNnm/lvpJA/B3r9sv56nvjXCCRpakNr -9U60kLKa2zxbqGLJVuYyEfrW15fPH5a0c21aXWH0X35Aui9ysKPXf7ebFes5wPc4fxCoCGsl07WB -SiMDS9W2GlofzHaq6xi8wy1lHdwS8uXqie6LgbL57Wj8YvxCeQIKM1empG2S6Tcxl1AorZFBOLjG -RyHzL9Jio2M0HO9GxJMDMGabB4FfWoVRvQaY+PXdH8Say9ORXnP1hVyIHR6XoVCNA4PuKWHLGyft -dn/M5Gg+bN1qiIt/oaYcaqpCMKWbW3LyOpZd9xFt3lHRnde6lNvBvIF77r/sH/+9h1t1EYQPc3BV -eDipAOV7j/a7LPKmpyDmrqPQskUnGAGER2mm6YUAhaWcfLF6kdHtRmG94gcj5wnagcQuOBzXJxdP -75CDcVTtkfxbIK+tdHJOustkYqb0h5bS159nOrcBuB3qmEgJ+9BPX7qaXRIL7CLveHkbrcToGOEs -qO49gStJPUBFiziG8gPFAiQh1m1XjT4p3S0SwJdgRgeOKaFpTSQKXvFTckzekD/IfS4DNZ/2Q3Ip -E5wvAdg3SrT9GewONV8XQNZd8s+zviEBTkduYDVVMTK3ij/HxDEaexiEDw296lZP3eff9s1lD1JS -WCz8yXJtItq9KqrNxRiqOFgx7KC1W6zNNfYSajePX+1xS5alG0w4s/SxQQM8dlDqdYvxHpyaGa0+ -+/asSYpR7AiNsptScwngPqQKNAYZFZoyWS2ZKn+pez00UJ0BvEhnvL/D/xhYM7R6HuHoNwtm2KpO -FndAgQ/BQXuMQ75Edy1WZ2qde5vX7zDjLUuE5JELzr28UMV9breaHB3oiJ00owAAsRvuEZsc8Yfc -eezTncXx7pM7XwhTzw91DLqyi07JxJ7iskXUrPdT2rGe+RDmbX1rupoXH9Yh+ZHEW+0kjctm8sRI -kCPQe+iv/MqqCOh83lC2qqCPCQ7Aw2e0GhZRQP4zc1bzbiV/UBgrgd5VmRYNagJxjDd0fxeVPw1S -NUCQHfZT8fHRUbJv9BtayVKRgFW6spTNyeD55ljBgdMkgd5p/1FDjWcAiD6rtzYLBa36P2zfYCTI -fgCfmHHScMDNqytArjSBK42Ke8WwFWYDsCFsBR1P1F5y+tD+CnTyG+RY1NDAM/J/NWsAFJQagEEW -HUABg+ZAr14Rp8jRkEiW3AEG93vCGQNbFJ4BMhgz8I0ilOdyoddqfkPDdKpQLdRdIivbal8iNeQK -Lrw5Y5oCyBmC0SJU2U3YYxleWl/P+7dIcC+XXRI5GOsHBZv3+QWX+x2dRikN8Mic8/XMg9NEghoE -Xp15d8G2zjTn70JuDKfMLqh4SWzMPL/9/rI2QVLrkvjdWQmSv6ueE51BJniie5+fluU6qTss1NUS -rVStQiF24u0YENzfuYGHPCWRhemCSMi4macv/vhXarEuQvL4aMOEm2hpXJh++7ciVWdB5lwBeB0m -/z/QHWZmWjDGzTxSCYG0z0XmxoRM68J8f+RvjWhEbWHpnIqgJXa+fkOAFbZDiRE6kB1v4PguJC4J -DsSc37Wi6Ypxnz5hCbfqWOHiJar6kgZnIoyFb72kBzAdFYFAIpPesj0DcdhbGaxj1Lt2LIJUB/lG -nnov9Bz6UXRYCHRQ2gLpDme0h84NVWVKhMNFpJ+PEvjbA0jm23mloKsgB4fTqn6R5BDr+yFAksaZ -GPUVvnUYJ9X/j2u7M+YqnFrjzNDQ2/N58uTS9+rp7v2OlwvYxZha0qU8pnbm5lgzSKNU1Ejwg/eY -TgRqM7xs1oBCazudkIaI5WwxEnBL4Qg0EO5qdt6EtpXW6JCySC81ZQaGmq9o3GvePLXpNtFYVf+I -olU2raqmm8DkUmFiew/oi1cqz+t81OqxElgAA27T5+ov9fKEZQvzuZULA1NmCohiwkBLApCRl7Nm -J4L/B/IyP/PjvDypEaYApCNo+87CGWz3HJQ+HggKF1Egp8ReFDA6ILJrPbeU447Mv+VF51Y/4jgP -4X0LTLzcylwNkgVuNlo5dhzwWLp4tidsyZVySXFT9QRem4c8U/06L6yZgSrbZTLwa8CvhL0OR+Kj -+p6xbpzw0yB4oxlzfO88IP5agCsmddtiCuUvZmuJt9hsw2u39MDNT8llweMIR6JRBBA2/X0jALID -JUJH9Zx3izdPFhG88yFhWBJphUoUkvNk9fCF7+Q245O5j/ly685TOl8aXDqPocsUTBtCnSTPoBVk -huxBMR/HT8MIJUu7R5JtbE8Us/jB6RMq9soNtS2dsa1zp1EioRc9Q6JpGUk1zqXo83SWDrv3aRLz -Z7iHQnMCC/WdkfhSBbz6zeI5fE1xw9Ggy0t1/llyJMnWsH6UtuEZxAV08ebBU22/He1phYZRMOs6 -Fcce/orktqjwOBmhCOGqgBdsfXHWl6Um9equDUBwu9InIWLSvdm21F3Zn1q2bG5SNjMcLcm66OQm -vlMbEG2m9FNcbmnwA8CPWIMiEk8oKRgb6lq/p6l+mCU2L0yprQmdwLBZb0tgoDnomG8qVPwFGDgm -frhu9+7jlXWMQxhWVD07YeV0TFLwmQMnrN+iFWi73im8WI/8aw7/TpecBn04UWkfOVdZYhEsMBeN -Mxwc6lZ5GNwGTos8crqp6pENRsEZTlHzgb6COQ7kWeSWuegyOhEVd/wBL13/tHxsg1ITuFAEYtsR -/IgErmiCZUiA/kxgRclZEIdxYnkl791s3+1QfqOVKnCxsNFhtq8bEqQXVwKXVMoLgtABa3pj59Pu -vr+IaHkXRTJq3F/slEXDVATl77ij` + const secondPackage = `KLUv/QRYPUcA+nxYGD6gsLQN5PnPkjy6fXl2KEtLb63ASYEmmw08uPr///fa2G7C4lq3UGMDqi4V +ALcz1mwbtcj9A2UA4JPvxl6MF28BUgFTAVuVp9foU3lbFhT7mEmK+iq9UOTEPlJJnz4mO+7G01e5 +ez/JOlFTRWeVNRQ/ptBKKrOJmlidVX4h3ky3JZfGEGcWtzMNh5Zakpikj32VmbPX6upuzrbzEBuJ +PUf4xu12Ee63R21BjZs27pi6Bu+GM4e77TyLYt9wjsA5sdN4d36tLh6KuGvRKPwwj9DiLou7LM5y +XIQtZpHjuM5xF+FaXUnhrT9fuy0LXqsri2Lv23Zu9LX00b58PLqgyDq6VtfGKhQqhVJNlUav1XW7 +7Yz2iitUEsNWCONaXdtIqx52utxmOxQKq/NyqDlPk1NdT5upelNdVzN7563b9LTZ0aqqt6ObpZvK +qyltXqsLHLl2qpbFcXQZj0yhvVYXi9v2a2kjKum1uvZN4nb7WtrofrbFWX2trgva8Gtpo3qtrm7B +LXpXF8U9J4BtOx/nBYCisErnhWGoLMuqlmX0qiet0gngyjJtzglAxYgUJUIglYDx4QNUcn7zrE/l +adstTHYi2HrtKdNtdVtLoz8lJ0ORZ/SrDEXOQ6tDq5RqnHq6gUp+82a7xtlrst4+ilvjx1By5FRV +m1y/0oYWVxT2U9puOyVWpU+lxk2YvBmPn+eJmz4m9d0frU/lBT/98dzH8Mu+fpWfHkMNfijsq+Q2 +nDXUV1k3nOtP2Zdu7/fD6lMJbh3fcJYc91UquYz7vv5JtpTsly3dPF/1WRdiT+eDt3MmfjP2suxP +8mbsVeyp7KK4b8t63eqlR2VPJf/ErblX76SPeR7u6mPzq9xXawvKy+Gk280/yY67+43bTX1Gn0o+ +pupjf5LYn7auYl9fk1sn59dPybxHTvsq9dgv75ppY6+6W1Q9Jm3OuEkET1M1a51V8tsthz3c9+Wk +5VJV46ZKpernU4lzF3f91GMyFLkmZ/ZV8oz7GOItkxqdT/8kWVTmm3Q7TF5uAxjken4mv1ByzYlb +izr19Gtppt3DG2KtYX+SeV+d/aSy3s1pzE0qtUd9lSyuWz1prbOCVmN/eipBq1Od47zFvTGIkjK0 +IGd15jy8WnwqWV2Hqt5vWE84stckv3yPXEj1z6+fye1pMPtMRT97KjNQlTXl1yTft7OXJbG2lfLw +cpR+lTWDSxr22WdLj/2UYt83acN5367B28cN585xOFovxV36ZcgyoD77KrMo9o27in4machtT2WP +ybvHri0nPXFrzsQ3nCuL83k3d7kGRY67PFUzt5zqaTm94YxbklKN5dOpd9YX1l5DnPsNvbDF2d3D +UORw7Nsz9voNN5wplZwGTz+fyhyCG87fZbvxVv0pM8cbzlkNO7zFva/89DfpU/ltOGePSex5G87Z +V9nDm3RT9Kf08t1wTv2UX2i5DWe9bThroR05txvOqMfkTcIsbtsNZ0zbEEbzWl23CXNR3320b6gk +HoEi9zdr0C7do6MNXPfOulZX3qMR3hy/3ZVH73IY1+ri+O2uUMR836TrCzEfLwlM29XdbvcNL4lL +AnaiFaVhJE6s+AA5TI4VOACRlwlwBHRJZLDZFE5sMDLFEJgsTOio4BW2DUh4ggBFXPxMudLSYsTk +A0icCxiUSJAyxsqXITdYFLikk4ElMkhm+CjCgQOPgkS2KYD8kxTgBYiKBMlLnDcubCBJ05oRINWA +BtLop45XosUKC2AUOFlY6AmiQYQDzwmLPE5ScPCg0aEFyBkuIR7cQX0OliUtI0g4QEDKEkRMCB05 +7gbBe2KVlHwh8qKDDlHm+AgJNQmyJkoUpTc/BHR1zKjpU4jDEaArJ0H2jqEYcpyoeiQIEqgIEniA +gVIVN13ssNA19pTYEkBQlwWqECPmUElz6MgvKrNBWRSIAAIPA0yMacNDQAEhECqKYjObU4SIFi9Q +fNd5E/264dyU/Uk2LdGKWUrYV5lxHzmsztrUKeddVlB3lBS9EC/djj2V23ZQ5JI4E0c9dqqy05/R +Vz2VYjfy2nHewelnUknsLIt/JrvL57whzp29+7EqOR6BFKiRHKSYQggxNDMjBUmyHHEEhCDonKSQ +DxJAcFGKQRAIQxAGQhCGolCgZsoUFBSknNJCaxLIFuLu8wEs1xIq/L/t/VY/OP3dqYbLsrBfxw5W +cvD6s/Cv/cvYagebBwgUG8B/WPN0YPGWImZwya7tsnkAK6GnndwczJA9fR0EafdFdxk7lVqLXylD +l81lvycUSkMRAwBBs8Y2WZfpW2l6rR1MNGYc8RZFEDnFWLCDUwoxkgw2uiU3wUtO0PoxwXRAS6uf +SjG6bucH5mYgx0nJPk1O2/zuyxjMwMlfhkzEqzU17siikQD0bSkeNJiaYjP6/Z6mwFcBaBqWFQWF +ZXDvjs28bS1upEdSobsFeM1mwSQzmWjKXXncnxk+c9ukDVCEhOtU4ilNyQd2DqMwk/+sGQR0TAVE +kDHIoCPi1CQ0CHe8V4Y029VBXuBsHLfcAfwQDUUdlotXa9trfzGZCYsOf0d4UT9/CsX5HiKCsD1t +k0UUDmVhooUjEDzfknjJEKiIvo+EJ4cJeYZLNmfXdAsWo+XKdvSsOVApMBMI4dgh+YKmfufflOV+ +lV6hgBS5s1mDkPhE4MKMeHAOk2y5GZLdujOA+ZYnYsUwXsEELFSYWUuX8s5cdUVX2ASdhGmT7Tpz +cnrHkqk1wFTbAptj3tOthM9UxRjhINCwAwweiSv/yHY5ZR5XxzhRywxiGb0Q3CwROUgI4Yfp8TTt +kV/aVGSXqjaHJxCpIH14LUhiaz0jKdBnbA7UoSmYf+FmGI7gG6rlNgzwAyUuf1HrTCbXKVOlAKEe +XF4BLD3RTCsc0nRH2o5p6R9rEdvYiyBiDyqpogShQxzsmnGpBOiHtFETHT+pV6+EY8/bCavohoSI +88iZNYTdzZ3RRGogZlPW6YbdqMPzqp1J64BkWgkRjSRHVMMs/JVreyWdFvgalsYYrLAH1s+Ir4vi +hVm78J3RLCgQ9mBp` secondPackageData, err := base64.StdEncoding.DecodeString(secondPackage) assert.NoError(t, err) - _ = secondPackageData - - const secondPackageSignature = `iQGzBAABCAAdFiEEQ6WNuVKuWHQJqhdSETT46/mKoG8FAmSsBg8ACgkQETT46/mKoG/t4wv8D6jQ -uNLj1vvVUlt3cXJ8dulp1TuR0KQrGxqPm/fhCNoEncffR1+1ehMjd69XlsA5XcGxY55QGjKK/W5z -LOZDXatNLTi6PRx1ApbbA7gCCzx/Tx0gF0sCe75Wu286j9ud8xFx8/ray8yS9iV6K6XpsOZUnVEw -yWC6DKsGb2Tv2VvRkj3yNo9ZdTlJOpmLX/OHjAOlFCTxJpa3Nf4YDc5V2d4xYzG0eeCasJueNHYz -d29ZdB58jZQj4ZUuv2Ln/fc2kmmBA4PIZaoTkc9kFwdYFIQV+c1OM6kTPnt5GJffNg7EgCVyv6ON -TJM9ud9iHaxwz+keBa/yQqEq4PFLNT0+IiCmidTpZI9iFVByqbXNuoKti0bdJ2pjnKRcSYWVV1AO -V8m4OwfXaTgsNr41SW9hZJnPUknJFACgc2LVJtJNVFwfb9iyRMdPCUD0NZ/6z28XL11vcNFbJldG -V2DfzGUB63cabGvZKvYMynHjfaxo8MwKVjUzgvWqMfNL2P5QpQ6S` + + const secondPackageSignature = `iQEzBAABCAAdFiEEdez1wBMjH7w+s6HCOjpq1/IBK5QFAmTKSigACgkQOjpq1/IBK5TnkQgAotU2 +kdi275k87qzRLp4SgOX4QTpwCyjmXK4ZEm/FGBiF84mYT/sQmKbSsxPbxd4lumHhbll5SMVdM3C+ +1pB4kWT1fewQi1YukGEe+Na6SAa33yQqQThf30WPYJhuOxSNDX0DNCrR7Ei98hNq0FuvZkZWzepF +ylhUP+OSGPWFsVOnXCaANxW6457LtnNPeQFDwQL2y9qv0Hgpnn3KS09n0SPXT2Kr02iYu9rICoOB +KsqYqMGxHoPypwT24o6oDElGt9/Z0pqZcwRJc7C0npv3q5a17S7nAdo8/NWgfDri7w224s8odtWy +qTAeEmTTQ8awXFYounZaHg+455+8u5npkw==` secondPackageSignatureData, err := base64.StdEncoding.DecodeString(secondPackageSignature) assert.NoError(t, err) - _ = secondPackageSignatureData - - const secondPackageDatabase = `H4sIAAAAAAAA/+ySQY/TMBCFe55fkcscm47teJwghAhNyka0C2ooBy7Icbxt1U02arJSl1+P2kWI -7RVVCNHvMh7rSfZ7eoPvh263HouxmNS+d6MLQERkjDlNIjqfxMS/zqd7IUlFo4Au8ZlzHvvB7kf0 -x2+dm/tHwFkxz2/TRY7wWxXGh5i/cRR2u3U42H34vR8A8IUMAN+l5Yv9S74si4+3CGIsADDLyynC -bLvvh+AoCjrrdnbtAXBaFl9zhJiFBsDieSMAXGS6XC0Q2Iu6TuJYGR3F5EnYiqmWTlinOXIOAMub -VGo+qSshaxaRkbGS1rKOWGvvjRcVMTtVaaOk0hUrK8kLWSWeE+fMXU3spLU1AK6Wc4TNMHT9q8nE -H2zT3fvQPTSTn/aOhtLl9AbhORqw7dMxglUxz7L0c44gOI6TRCvFAPgpnX5I3+dLhMy223sbzB6a -bfu02wSva9u6jV8n5u1d0x7C/vENwN9uwZUrV/5HfgQAAP//SZpudwAIAAA=` + + const secondPackageDatabase = `H4sIAAAAAAAA/+zSzW7bMAwA4Jz5FLnwmJhyLMnazUu81VjSFfG8w276oboirW3YLrru6YemwLDk +OhTDgHwXigIBiRJ76w/2lhdiIZLAo5+9ASIirfUxEtF5JClXv9fHfSF0pmdzeovLnHscJzvM6K/P +Om/uP4Efqm15XexKhD9GYWHb52V/uF1Odlj+HCcAPKkBwPdFfZJ/Lfd19fkaQSwEAG7Keo3QjBwf +7+cvgzXc9dNd1wLguq6+lQhpalIArF4zAsDdRtbNDoFXPmfjNbsQg/UyzRWFXLAwdsUx8wBYXxWp +VMfqVfQZp9GYYJ0yLhB7zYG0IymEiuykCqlWlHpnQ5Y5qzgXLroQZRZUFgGw2W8Rvk9TP75LEv5h +H/p7XvruIemeWh6SgfsOAIv9+grBts8v3TfVdrMpvpQIQhkyOtfaAOBNsf5UfCz3CE17aLundn7z ++kIDwL/+6ouLi4sTvwIAAP//Stn2gwAIAAA=` secondPackageDatabaseData, err := base64.StdEncoding.DecodeString(secondPackageDatabase) assert.NoError(t, err) @@ -385,12 +157,7 @@ bfu02wSva9u6jV8n5u1d0x7C/vENwN9uwZUrV/5HfgQAAP//SZpudwAIAAA=` t.Run("PushFirstPackage", func(t *testing.T) { defer tests.PrintCurrentTest(t)() - req := NewRequest(t, "PUT", path.Join( - rootURL, "/push", - "testpkg-1-1-x86_64.pkg.tar.zst", - "archlinux", - hex.EncodeToString(firstPackageSignatureData), - )) + req := NewRequest(t, "PUT", path.Join(rootURL, "/push", "package-1-1-x86_64.pkg.tar.zst", "archlinux", hex.EncodeToString(firstPackageSignatureData))) req.Header.Set("Content-Type", "application/octet-stream") req.Body = io.NopCloser(bytes.NewReader(firstPackageData)) @@ -402,34 +169,37 @@ bfu02wSva9u6jV8n5u1d0x7C/vENwN9uwZUrV/5HfgQAAP//SZpudwAIAAA=` t.Run("GetFirstPackage", func(t *testing.T) { defer tests.PrintCurrentTest(t)() - req := NewRequest(t, "GET", path.Join(rootURL, "/archlinux/x86_64/testpkg-1-1-x86_64.pkg.tar.zst")) + req := NewRequest(t, "GET", path.Join(rootURL, "/archlinux/x86_64/package-1-1-x86_64.pkg.tar.zst")) resp := MakeRequest(t, req, http.StatusOK) - assert.Equal(t, resp.Body.Bytes(), firstPackageData) + assert.Equal(t, firstPackageData, resp.Body.Bytes()) }) t.Run("GetFirstDatabase", func(t *testing.T) { defer tests.PrintCurrentTest(t)() - dbname := fmt.Sprintf("%s.%s.db", user.Name, setting.Domain) + req := NewRequest(t, "GET", path.Join(rootURL, "/archlinux/x86_64/"+fmt.Sprintf("%s.%s.db", user.Name, setting.Domain))) + + resp := MakeRequest(t, req, http.StatusOK) + + assert.Equal(t, firstPackageDatabaseData, resp.Body.Bytes()) + }) + + t.Run("GetFirstSignature", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() - req := NewRequest(t, "GET", path.Join(rootURL, "/archlinux/x86_64/"+dbname)) + req := NewRequest(t, "GET", path.Join(rootURL, "/archlinux/x86_64/package-1-1-x86_64.pkg.tar.zst.sig")) resp := MakeRequest(t, req, http.StatusOK) - assert.Equal(t, resp.Body.Bytes(), firstPackageDatabaseData) + assert.Equal(t, firstPackageSignatureData, resp.Body.Bytes()) }) t.Run("PushSecond", func(t *testing.T) { defer tests.PrintCurrentTest(t)() - req := NewRequest(t, "PUT", path.Join( - rootURL, "/push", - "testpkg-1-1-any.pkg.tar.zst", - "archlinux", - hex.EncodeToString(secondPackageSignatureData), - )) + req := NewRequest(t, "PUT", path.Join(rootURL, "/push", "package-1-1-any.pkg.tar.zst", "archlinux", hex.EncodeToString(secondPackageSignatureData))) req.Header.Set("Content-Type", "application/octet-stream") req.Body = io.NopCloser(bytes.NewReader(secondPackageData)) @@ -441,33 +211,37 @@ bfu02wSva9u6jV8n5u1d0x7C/vENwN9uwZUrV/5HfgQAAP//SZpudwAIAAA=` t.Run("GetSecondPackage", func(t *testing.T) { defer tests.PrintCurrentTest(t)() - req := NewRequest(t, "GET", path.Join(rootURL, "/archlinux/any/testpkg-1-1-any.pkg.tar.zst")) + req := NewRequest(t, "GET", path.Join(rootURL, "/archlinux/any/package-1-1-any.pkg.tar.zst")) resp := MakeRequest(t, req, http.StatusOK) - assert.Equal(t, resp.Body.Bytes(), secondPackageData) + assert.Equal(t, secondPackageData, resp.Body.Bytes()) }) t.Run("GetSecondDatabase", func(t *testing.T) { defer tests.PrintCurrentTest(t)() - dbname := fmt.Sprintf("%s.%s.db.tar.gz", user.Name, setting.Domain) + req := NewRequest(t, "GET", path.Join(rootURL, "/archlinux/any/"+fmt.Sprintf("%s.%s.db.tar.gz", user.Name, setting.Domain))) + + resp := MakeRequest(t, req, http.StatusOK) + + assert.Equal(t, secondPackageDatabaseData, resp.Body.Bytes()) + }) + + t.Run("GetSecondSignature", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() - req := NewRequest(t, "GET", path.Join(rootURL, "/archlinux/x86_64/"+dbname)) + req := NewRequest(t, "GET", path.Join(rootURL, "/archlinux/any/package-1-1-any.pkg.tar.zst.sig")) resp := MakeRequest(t, req, http.StatusOK) - assert.Equal(t, resp.Body.Bytes(), secondPackageDatabaseData) + assert.Equal(t, secondPackageSignatureData, resp.Body.Bytes()) }) - t.Run("RemoveFirst", func(t *testing.T) { + t.Run("Remove", func(t *testing.T) { defer tests.PrintCurrentTest(t)() - req := NewRequest(t, "DELETE", path.Join( - rootURL, "/remove", - "testpkg", - "1-1", - )) + req := NewRequest(t, "DELETE", path.Join(rootURL, "/remove", "package", "1-1")) req = AddBasicAuthHeader(req, user.Name) From e01ac83ff9112b8f9187b7be08d69c08562acbf7 Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Thu, 3 Aug 2023 17:31:21 +0300 Subject: [PATCH 066/124] documentation corrections --- docs/content/usage/packages/arch.en-us.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/content/usage/packages/arch.en-us.md b/docs/content/usage/packages/arch.en-us.md index 7f1cd79a854e..de7dc78db34c 100644 --- a/docs/content/usage/packages/arch.en-us.md +++ b/docs/content/usage/packages/arch.en-us.md @@ -45,7 +45,7 @@ Following command will upload arch package and related signature to gitea. Examp ```sh curl -X PUT \ https://{domain}/api/packages/{owner}/arch/push/{package-1-1-x86_64.pkg.tar.zst}/{archlinux}/$(xxd -p package-1-1-x86_64.pkg.tar.zst.sig | tr -d '\n') \ - --header "{username}:{password}" \ + --user your_username:your_token_or_password \ --header "Content-Type: application/octet-stream" \ --data-binary '@/path/to/package/file/package-1-1-x86_64.pkg.tar.zst' ``` @@ -56,10 +56,10 @@ Delete operation will remove specific package version, and all package files rel ```sh curl -X DELETE \ - https://{domain}/api/packages/{user}/arch/remove{package}/{version} \ - --header "{username}:{password}" + https://{domain}/api/packages/{user}/arch/remove/{package}/{version} \ + --user your_username:your_token_or_password ``` ## Clients -Any `pacman` compatible package manager/AUR-helper can be used to install packages from gitea ([yay](https://github.com/Jguer/yay), [paru](https://github.com/Morganamilo/paru), [pikaur](https://github.com/actionless/pikaur), [aura](https://github.com/fosskers/aura)). Alternatively, you can try [pack](https://fmnx.su/core/pack) which supports full gitea API (install/push/remove). Also, any HTTP client can be used to execute push/remove operations ([curl](https://curl.se/), [postman](https://www.postman.com/), [thunder-client](https://www.thunderclient.com/)). +Any `pacman` compatible package manager/AUR-helper can be used to install packages from gitea ([yay](https://github.com/Jguer/yay), [paru](https://github.com/Morganamilo/paru), [pikaur](https://github.com/actionless/pikaur), [aura](https://github.com/fosskers/aura)). Alternatively, you can try [pack](https://fmnx.su/core/pack) which supports full gitea API (install/push/remove). Also, any HTTP client can be used to execute get/push/remove operations ([curl](https://curl.se/), [postman](https://www.postman.com/), [thunder-client](https://www.thunderclient.com/)). From ebaa5436ad476f9aeeddb672e6f1202c7c399f65 Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Thu, 3 Aug 2023 17:36:48 +0300 Subject: [PATCH 067/124] lint corrections --- modules/packages/arch/metadata.go | 1 + services/packages/arch/db_manager.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/packages/arch/metadata.go b/modules/packages/arch/metadata.go index a51b8455b046..40dd4ee6cdb4 100644 --- a/modules/packages/arch/metadata.go +++ b/modules/packages/arch/metadata.go @@ -17,6 +17,7 @@ import ( "code.gitea.io/gitea/modules/container" pkg_module "code.gitea.io/gitea/modules/packages" + "github.com/mholt/archiver/v3" ) diff --git a/services/packages/arch/db_manager.go b/services/packages/arch/db_manager.go index 7eea1612120a..eb8c83533a0d 100644 --- a/services/packages/arch/db_manager.go +++ b/services/packages/arch/db_manager.go @@ -151,7 +151,7 @@ func CreatePacmanDb(ctx *context.Context, owner, architecture, distro string) ([ return nil, err } - var entries = make(map[string][]byte) + entries := make(map[string][]byte) for _, pkg := range pkgs { versions, err := pkg_model.GetVersionsByPackageName(ctx, u.ID, pkg_model.TypeArch, pkg.Name) From 0c7aa7f63ed71f66e0f6ff8eea40254902e0b1f2 Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Thu, 3 Aug 2023 20:43:13 +0300 Subject: [PATCH 068/124] refactoring --- modules/packages/arch/metadata.go | 57 +++++++++----------- routers/api/packages/arch/arch.go | 38 +++++++++----- services/packages/arch/db_manager.go | 78 ++++++++++++++++------------ 3 files changed, 95 insertions(+), 78 deletions(-) diff --git a/modules/packages/arch/metadata.go b/modules/packages/arch/metadata.go index 40dd4ee6cdb4..a3d0ca97f19f 100644 --- a/modules/packages/arch/metadata.go +++ b/modules/packages/arch/metadata.go @@ -15,7 +15,6 @@ import ( "strconv" "strings" - "code.gitea.io/gitea/modules/container" pkg_module "code.gitea.io/gitea/modules/packages" "github.com/mholt/archiver/v3" @@ -24,8 +23,6 @@ import ( // JSON with pacakage parameters that are not related to specific // architecture/distribution that will be stored in sql database. type Metadata struct { - Name string - Version string URL string `json:"url"` Description string `json:"description"` Provides []string `json:"provides,omitempty"` @@ -64,7 +61,7 @@ type DbDesc struct { } // Function that receives arch package archive data and returns it's metadata. -func EjectMetadata(filename, distribution string, buf *pkg_module.HashedBuffer) (*DbDesc, error) { +func EjectMetadata(filename, distro string, buf *pkg_module.HashedBuffer) (*DbDesc, error) { pkginfo, err := getPkginfo(buf) if err != nil { return nil, err @@ -84,52 +81,58 @@ func EjectMetadata(filename, distribution string, buf *pkg_module.HashedBuffer) if len(splt) != 2 { continue } - switch splt[0] { + var ( + parameter = splt[0] + value = splt[1] + ) + + switch parameter { case "pkgname": - md.Name = splt[1] + md.Name = value case "pkgbase": - md.Base = splt[1] + md.Base = value case "pkgver": - md.Version = splt[1] + md.Version = value case "pkgdesc": - md.Description = splt[1] + md.Description = value case "url": - md.URL = splt[1] + md.URL = value case "packager": - md.Packager = splt[1] + md.Packager = value case "builddate": - num, err := strconv.ParseInt(splt[1], 10, 64) + num, err := strconv.ParseInt(value, 10, 64) if err != nil { return nil, err } md.BuildDate = num case "size": - num, err := strconv.ParseInt(splt[1], 10, 64) + num, err := strconv.ParseInt(value, 10, 64) if err != nil { return nil, err } md.InstalledSize = num case "provides": - md.Provides = append(md.Provides, splt[1]) + md.Provides = append(md.Provides, value) case "license": - md.License = append(md.License, splt[1]) + md.License = append(md.License, value) case "arch": - md.Arch = append(md.Arch, splt[1]) + md.Arch = append(md.Arch, value) case "depend": - md.Depends = append(md.Depends, splt[1]) + md.Depends = append(md.Depends, value) case "optdepend": - md.OptDepends = append(md.OptDepends, splt[1]) + md.OptDepends = append(md.OptDepends, value) case "makedepend": - md.MakeDepends = append(md.MakeDepends, splt[1]) + md.MakeDepends = append(md.MakeDepends, value) case "checkdepend": - md.CheckDepends = append(md.CheckDepends, splt[1]) + md.CheckDepends = append(md.CheckDepends, value) case "backup": - md.Backup = append(md.Backup, splt[1]) + md.Backup = append(md.Backup, value) } } return &md, nil } +// Eject .PKGINFO file as string from package archive. func getPkginfo(data io.Reader) (string, error) { br := bufio.NewReader(data) zstd := archiver.NewTarZstd() @@ -153,8 +156,7 @@ func getPkginfo(data io.Reader) (string, error) { } } -// This function returns pacman package description in unarchived raw database -// format. +// Create pacman package description file. func (m *DbDesc) GetDbDesc() string { return strings.Join(rmEmptyStrings([]string{ formatField("FILENAME", m.Filename), @@ -244,12 +246,3 @@ func writeToArchive(files map[string][]byte, buf io.Writer) error { } return nil } - -// This function creates a list containing unique values formed of 2 passed -// slices. -func UnifiedList(first, second []string) []string { - set := make(container.Set[string], len(first)+len(second)) - set.AddMultiple(first...) - set.AddMultiple(second...) - return set.Values() -} diff --git a/routers/api/packages/arch/arch.go b/routers/api/packages/arch/arch.go index 21ac89858264..919202664334 100644 --- a/routers/api/packages/arch/arch.go +++ b/routers/api/packages/arch/arch.go @@ -10,10 +10,12 @@ import ( "net/http" "strings" + pkg_model "code.gitea.io/gitea/models/packages" "code.gitea.io/gitea/modules/context" pkg_module "code.gitea.io/gitea/modules/packages" arch_module "code.gitea.io/gitea/modules/packages/arch" "code.gitea.io/gitea/routers/api/packages/helper" + pkg_service "code.gitea.io/gitea/services/packages" arch_service "code.gitea.io/gitea/services/packages/arch" ) @@ -56,8 +58,6 @@ func Push(ctx *context.Context) { // Metadata related to SQL database. dbmd := &arch_module.Metadata{ - Name: pkgmd.Name, - Version: pkgmd.Version, URL: pkgmd.URL, Description: pkgmd.Description, Provides: pkgmd.Provides, @@ -74,10 +74,12 @@ func Push(ctx *context.Context) { pkgid, err := arch_service.SaveFile(ctx, &arch_service.SaveFileParams{ Creator: ctx.ContextUser, Owner: ctx.Package.Owner, - Filename: filename, - Buf: buf, Metadata: dbmd, + Buf: buf, + Filename: filename, Distro: distro, + PkgName: pkgmd.Name, + PkgVer: pkgmd.Version, }) if err != nil { apiError(ctx, http.StatusInternalServerError, err) @@ -100,6 +102,8 @@ func Push(ctx *context.Context) { Buf: buf, Metadata: dbmd, Distro: distro, + PkgName: pkgmd.Name, + PkgVer: pkgmd.Version, }) if err != nil { apiError(ctx, http.StatusInternalServerError, err) @@ -124,6 +128,8 @@ func Push(ctx *context.Context) { Filename: filename + ".sig", Metadata: dbmd, Distro: distro, + PkgName: pkgmd.Name, + PkgVer: pkgmd.Version, }) if err != nil { apiError(ctx, http.StatusInternalServerError, err) @@ -133,8 +139,9 @@ func Push(ctx *context.Context) { // Add new architectures and distribution info to package version metadata. err = arch_service.UpdateMetadata(ctx, &arch_service.UpdateMetadataParameters{ - User: ctx.Package.Owner, - Md: dbmd, + User: ctx.Package.Owner, + Metadata: dbmd, + DbDesc: pkgmd, }) if err != nil { apiError(ctx, http.StatusInternalServerError, err) @@ -142,7 +149,7 @@ func Push(ctx *context.Context) { } // Automatically connect repository for provided package if name matched. - err = arch_service.RepositoryAutoconnect(ctx, owner, dbmd.Name, pkgid) + err = arch_service.RepositoryAutoconnect(ctx, owner, pkgmd.Name, pkgid) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return @@ -160,8 +167,7 @@ func Get(ctx *context.Context) { arch = ctx.Params("arch") ) - // Packages are stored in different way from pacman databases, and loaded - // with LoadPackageFile function. + // Packages and signatures are loaded directly from object storage. if strings.HasSuffix(file, "tar.zst") || strings.HasSuffix(file, "zst.sig") { pkg, err := arch_service.GetFileObject(ctx, distro, file) if err != nil { @@ -175,8 +181,9 @@ func Get(ctx *context.Context) { return } - // Pacman databases is not stored in gitea's storage, it is created for - // incoming request and cached. + // Pacman databases is not stored in giteas storage and created 'on-request' + // for user/organization scope with accordance to requested architecture + // and distribution. if strings.HasSuffix(file, ".db.tar.gz") || strings.HasSuffix(file, ".db") { db, err := arch_service.CreatePacmanDb(ctx, owner, arch, distro) if err != nil { @@ -200,8 +207,13 @@ func Remove(ctx *context.Context) { ver = ctx.Params("version") ) - // Remove package files and pacman database entry. - err := arch_service.RemovePackage(ctx, ctx.Package.Owner, pkg, ver) + version, err := pkg_model.GetVersionByNameAndVersion(ctx, ctx.Package.Owner.ID, pkg_model.TypeArch, pkg, ver) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + err = pkg_service.RemovePackageVersion(ctx.Package.Owner, version) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return diff --git a/services/packages/arch/db_manager.go b/services/packages/arch/db_manager.go index eb8c83533a0d..b1864268dd45 100644 --- a/services/packages/arch/db_manager.go +++ b/services/packages/arch/db_manager.go @@ -13,6 +13,7 @@ import ( pkg_model "code.gitea.io/gitea/models/packages" "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/container" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/json" "code.gitea.io/gitea/modules/packages" @@ -22,15 +23,21 @@ import ( ) type UpdateMetadataParameters struct { - User *user.User - Md *arch.Metadata + User *user.User + Metadata *arch.Metadata + DbDesc *arch.DbDesc } -// This function gets existing package metadata for provided version present, -// combines architecture and distribution info and creates new metadata with -// combined set of parameters. +// Update package metadata stored in SQL database with new combination of +// distribution and architecture. func UpdateMetadata(ctx *context.Context, p *UpdateMetadataParameters) error { - ver, err := pkg_model.GetVersionByNameAndVersion(ctx, p.User.ID, pkg_model.TypeArch, p.Md.Name, p.Md.Version) + ver, err := pkg_model.GetVersionByNameAndVersion( + ctx, + p.User.ID, + pkg_model.TypeArch, + p.DbDesc.Name, + p.DbDesc.Version, + ) if err != nil { return err } @@ -41,7 +48,7 @@ func UpdateMetadata(ctx *context.Context, p *UpdateMetadataParameters) error { return err } - currmd.DistroArch = arch.UnifiedList(currmd.DistroArch, p.Md.DistroArch) + currmd.DistroArch = uniqueSlice(currmd.DistroArch, p.Metadata.DistroArch) b, err := json.Marshal(&currmd) if err != nil { @@ -53,6 +60,14 @@ func UpdateMetadata(ctx *context.Context, p *UpdateMetadataParameters) error { return pkg_model.UpdateVersion(ctx, ver) } +// Creates a list containing unique values formed of 2 passed slices. +func uniqueSlice(first, second []string) []string { + set := make(container.Set[string], len(first)+len(second)) + set.AddMultiple(first...) + set.AddMultiple(second...) + return set.Values() +} + // Parameters required to save new arch package. type SaveFileParams struct { Creator *user.User @@ -62,10 +77,12 @@ type SaveFileParams struct { Filename string Distro string IsLead bool + PkgName string + PkgVer string } -// This function creates new package, version and package_file properties in -// database, and writes blob to file storage. If package/version/blob exists it +// Creates new package, version and package_file properties in database, +// and writes blob to file storage. If package/version/blob exists it // will overwrite existing data. Package id and error will be returned. func SaveFile(ctx *context.Context, p *SaveFileParams) (int64, error) { ver, _, err := pkg_service.CreatePackageOrAddFileToExisting( @@ -73,8 +90,8 @@ func SaveFile(ctx *context.Context, p *SaveFileParams) (int64, error) { PackageInfo: pkg_service.PackageInfo{ Owner: p.Owner, PackageType: pkg_model.TypeArch, - Name: p.Metadata.Name, - Version: p.Metadata.Version, + Name: p.PkgName, + Version: p.PkgVer, }, Creator: p.Creator, Metadata: p.Metadata, @@ -96,8 +113,8 @@ func SaveFile(ctx *context.Context, p *SaveFileParams) (int64, error) { return ver.ID, nil } -// Get data related to provided file name and distribution, and update download -// counter if actual package file is retrieved from database. +// Get data related to provided filename and distribution, for package files +// update download counter. func GetFileObject(ctx *context.Context, distro, file string) (storage.Object, error) { db := db.GetEngine(ctx) @@ -125,8 +142,8 @@ func GetFileObject(ctx *context.Context, distro, file string) (storage.Object, e return cs.Get(packages.BlobHash256Key(blob.HashSHA256)) } -// Automatically connect repository to pushed package, if package with provided -// with provided name exists in namespace scope. +// Automatically connect repository with source code to published package, if +// repository with the same name exists in user/organization scope. func RepositoryAutoconnect(ctx *context.Context, owner, repository string, pkgid int64) error { repo, err := repo.GetRepositoryByOwnerAndName(ctx, owner, repository) if err == nil { @@ -138,8 +155,11 @@ func RepositoryAutoconnect(ctx *context.Context, owner, repository string, pkgid return nil } -// This function is collecting information about packages in some organization/ -// user space, and created pacman database archive based on package metadata. +// Finds all arch packages in user/organization scope, each package version +// starting from latest in descending order is checked to be compatible with +// requested combination of architecture and distribution. When/If the first +// compatible version is found, related desc file will be loaded from object +// storage and added to database archive. func CreatePacmanDb(ctx *context.Context, owner, architecture, distro string) ([]byte, error) { u, err := user.GetUserByName(ctx, owner) if err != nil { @@ -164,10 +184,11 @@ func CreatePacmanDb(ctx *context.Context, owner, architecture, distro string) ([ }) for _, version := range versions { - desc, err := GetPacmanDbDesc(ctx, &DescParams{ + desc, err := LoadDbDescFile(ctx, &DescParams{ Version: version, Arch: architecture, Distro: distro, + PkgName: pkg.Name, }) if err != nil { return nil, err @@ -187,11 +208,12 @@ type DescParams struct { Version *pkg_model.PackageVersion Arch string Distro string + PkgName string } -// Checks if desc file exists for required architecture or any and returns it -// in form of byte slice. -func GetPacmanDbDesc(ctx *context.Context, p *DescParams) ([]byte, error) { +// Get pacman desc file from object storage if combination of distribution and +// architecture is supported (checked in metadata). +func LoadDbDescFile(ctx *context.Context, p *DescParams) ([]byte, error) { var md arch.Metadata err := json.Unmarshal([]byte(p.Version.MetadataJSON), &md) if err != nil { @@ -202,10 +224,10 @@ func GetPacmanDbDesc(ctx *context.Context, p *DescParams) ([]byte, error) { var storagekey string if distroarch == p.Distro+"-"+p.Arch { - storagekey = md.Name + "-" + md.Version + "-" + p.Arch + ".desc" + storagekey = p.PkgName + "-" + p.Version.Version + "-" + p.Arch + ".desc" } if distroarch == p.Distro+"-any" { - storagekey = md.Name + "-" + md.Version + "-any.desc" + storagekey = p.PkgName + "-" + p.Version.Version + "-any.desc" } if storagekey == "" { @@ -221,13 +243,3 @@ func GetPacmanDbDesc(ctx *context.Context, p *DescParams) ([]byte, error) { } return nil, nil } - -// Remove specific package version related to provided user or organization. -func RemovePackage(ctx *context.Context, u *user.User, name, version string) error { - ver, err := pkg_model.GetVersionByNameAndVersion(ctx, u.ID, pkg_model.TypeArch, name, version) - if err != nil { - return err - } - - return pkg_service.RemovePackageVersion(u, ver) -} From 28712bc91eef58cec0a9f49abd78463b8d23cd4f Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Fri, 4 Aug 2023 14:52:36 +0300 Subject: [PATCH 069/124] refactoring --- modules/packages/arch/metadata.go | 19 ++++-- routers/api/packages/arch/arch.go | 68 ++++++++++++--------- services/packages/arch/db_manager.go | 36 ++++++----- tests/integration/api_packages_arch_test.go | 44 ++++++++----- 4 files changed, 103 insertions(+), 64 deletions(-) diff --git a/modules/packages/arch/metadata.go b/modules/packages/arch/metadata.go index a3d0ca97f19f..94a879cc2e95 100644 --- a/modules/packages/arch/metadata.go +++ b/modules/packages/arch/metadata.go @@ -60,22 +60,29 @@ type DbDesc struct { Backup []string `json:"backup,omitempty"` } +type EjectParams struct { + Filename string + Distribution string + Buffer *pkg_module.HashedBuffer +} + // Function that receives arch package archive data and returns it's metadata. -func EjectMetadata(filename, distro string, buf *pkg_module.HashedBuffer) (*DbDesc, error) { - pkginfo, err := getPkginfo(buf) +func EjectMetadata(p *EjectParams) (*DbDesc, error) { + pkginfo, err := getPkginfo(p.Buffer) if err != nil { return nil, err } // Add package blob parameters to arch related desc. - hashMD5, _, hashSHA256, _ := buf.Sums() + hashMD5, _, hashSHA256, _ := p.Buffer.Sums() md := DbDesc{ - Filename: filename, - Name: filename, - CompressedSize: buf.Size(), + Filename: p.Filename, + Name: p.Filename, + CompressedSize: p.Buffer.Size(), MD5: hex.EncodeToString(hashMD5), SHA256: hex.EncodeToString(hashSHA256), } + for _, line := range strings.Split(pkginfo, "\n") { splt := strings.Split(line, " = ") if len(splt) != 2 { diff --git a/routers/api/packages/arch/arch.go b/routers/api/packages/arch/arch.go index 919202664334..dbc2331979e1 100644 --- a/routers/api/packages/arch/arch.go +++ b/routers/api/packages/arch/arch.go @@ -45,7 +45,11 @@ func Push(ctx *context.Context) { defer buf.Close() // Parse metadata related to package contained in arch package archive. - pkgmd, err := arch_module.EjectMetadata(filename, distro, buf) + desc, err := arch_module.EjectMetadata(&arch_module.EjectParams{ + Filename: filename, + Distribution: distro, + Buffer: buf, + }) if err != nil { apiError(ctx, http.StatusBadRequest, err) return @@ -57,36 +61,36 @@ func Push(ctx *context.Context) { } // Metadata related to SQL database. - dbmd := &arch_module.Metadata{ - URL: pkgmd.URL, - Description: pkgmd.Description, - Provides: pkgmd.Provides, - License: pkgmd.License, - Depends: pkgmd.Depends, - OptDepends: pkgmd.OptDepends, - MakeDepends: pkgmd.MakeDepends, - CheckDepends: pkgmd.CheckDepends, - Backup: pkgmd.Backup, - DistroArch: []string{distro + "-" + pkgmd.Arch[0]}, + md := &arch_module.Metadata{ + URL: desc.URL, + Description: desc.Description, + Provides: desc.Provides, + License: desc.License, + Depends: desc.Depends, + OptDepends: desc.OptDepends, + MakeDepends: desc.MakeDepends, + CheckDepends: desc.CheckDepends, + Backup: desc.Backup, + DistroArch: []string{distro + "-" + desc.Arch[0]}, } // Save file related to arch package. pkgid, err := arch_service.SaveFile(ctx, &arch_service.SaveFileParams{ Creator: ctx.ContextUser, Owner: ctx.Package.Owner, - Metadata: dbmd, + Metadata: md, Buf: buf, Filename: filename, Distro: distro, - PkgName: pkgmd.Name, - PkgVer: pkgmd.Version, + PkgName: desc.Name, + PkgVer: desc.Version, }) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return } - r := io.NopCloser(bytes.NewReader([]byte(pkgmd.GetDbDesc()))) + r := io.NopCloser(bytes.NewReader([]byte(desc.GetDbDesc()))) buf, err = pkg_module.CreateHashedBufferFromReader(r) if err != nil { apiError(ctx, http.StatusInternalServerError, err) @@ -98,12 +102,12 @@ func Push(ctx *context.Context) { _, err = arch_service.SaveFile(ctx, &arch_service.SaveFileParams{ Creator: ctx.ContextUser, Owner: ctx.Package.Owner, - Filename: pkgmd.Name + "-" + pkgmd.Version + "-" + pkgmd.Arch[0] + ".desc", + Filename: desc.Name + "-" + desc.Version + "-" + desc.Arch[0] + ".desc", Buf: buf, - Metadata: dbmd, + Metadata: md, Distro: distro, - PkgName: pkgmd.Name, - PkgVer: pkgmd.Version, + PkgName: desc.Name, + PkgVer: desc.Version, }) if err != nil { apiError(ctx, http.StatusInternalServerError, err) @@ -126,10 +130,10 @@ func Push(ctx *context.Context) { Owner: ctx.Package.Owner, Buf: buf, Filename: filename + ".sig", - Metadata: dbmd, + Metadata: md, Distro: distro, - PkgName: pkgmd.Name, - PkgVer: pkgmd.Version, + PkgName: desc.Name, + PkgVer: desc.Version, }) if err != nil { apiError(ctx, http.StatusInternalServerError, err) @@ -140,16 +144,16 @@ func Push(ctx *context.Context) { // Add new architectures and distribution info to package version metadata. err = arch_service.UpdateMetadata(ctx, &arch_service.UpdateMetadataParameters{ User: ctx.Package.Owner, - Metadata: dbmd, - DbDesc: pkgmd, + Metadata: md, + DbDesc: desc, }) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return } - // Automatically connect repository for provided package if name matched. - err = arch_service.RepositoryAutoconnect(ctx, owner, pkgmd.Name, pkgid) + // Automatically connect repository with souce code if name matched. + err = arch_service.RepoConnect(ctx, owner, desc.Name, pkgid) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return @@ -185,7 +189,11 @@ func Get(ctx *context.Context) { // for user/organization scope with accordance to requested architecture // and distribution. if strings.HasSuffix(file, ".db.tar.gz") || strings.HasSuffix(file, ".db") { - db, err := arch_service.CreatePacmanDb(ctx, owner, arch, distro) + db, err := arch_service.CreatePacmanDb(ctx, &arch_service.PacmanDbParams{ + Owner: owner, + Architecture: arch, + Distribution: distro, + }) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return @@ -207,7 +215,9 @@ func Remove(ctx *context.Context) { ver = ctx.Params("version") ) - version, err := pkg_model.GetVersionByNameAndVersion(ctx, ctx.Package.Owner.ID, pkg_model.TypeArch, pkg, ver) + version, err := pkg_model.GetVersionByNameAndVersion( + ctx, ctx.Package.Owner.ID, pkg_model.TypeArch, pkg, ver, + ) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return diff --git a/services/packages/arch/db_manager.go b/services/packages/arch/db_manager.go index b1864268dd45..75870ee36d17 100644 --- a/services/packages/arch/db_manager.go +++ b/services/packages/arch/db_manager.go @@ -11,7 +11,7 @@ import ( "code.gitea.io/gitea/models/db" pkg_model "code.gitea.io/gitea/models/packages" - "code.gitea.io/gitea/models/repo" + repository "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/container" "code.gitea.io/gitea/modules/context" @@ -144,10 +144,10 @@ func GetFileObject(ctx *context.Context, distro, file string) (storage.Object, e // Automatically connect repository with source code to published package, if // repository with the same name exists in user/organization scope. -func RepositoryAutoconnect(ctx *context.Context, owner, repository string, pkgid int64) error { - repo, err := repo.GetRepositoryByOwnerAndName(ctx, owner, repository) +func RepoConnect(ctx *context.Context, owner, repo string, pkgid int64) error { + r, err := repository.GetRepositoryByOwnerAndName(ctx, owner, repo) if err == nil { - err = pkg_model.SetRepositoryLink(ctx, pkgid, repo.ID) + err = pkg_model.SetRepositoryLink(ctx, pkgid, r.ID) if err != nil { return err } @@ -155,13 +155,19 @@ func RepositoryAutoconnect(ctx *context.Context, owner, repository string, pkgid return nil } +type PacmanDbParams struct { + Owner string + Architecture string + Distribution string +} + // Finds all arch packages in user/organization scope, each package version // starting from latest in descending order is checked to be compatible with // requested combination of architecture and distribution. When/If the first // compatible version is found, related desc file will be loaded from object // storage and added to database archive. -func CreatePacmanDb(ctx *context.Context, owner, architecture, distro string) ([]byte, error) { - u, err := user.GetUserByName(ctx, owner) +func CreatePacmanDb(ctx *context.Context, p *PacmanDbParams) ([]byte, error) { + u, err := user.GetUserByName(ctx, p.Owner) if err != nil { return nil, err } @@ -174,7 +180,9 @@ func CreatePacmanDb(ctx *context.Context, owner, architecture, distro string) ([ entries := make(map[string][]byte) for _, pkg := range pkgs { - versions, err := pkg_model.GetVersionsByPackageName(ctx, u.ID, pkg_model.TypeArch, pkg.Name) + versions, err := pkg_model.GetVersionsByPackageName( + ctx, u.ID, pkg_model.TypeArch, pkg.Name, + ) if err != nil { return nil, err } @@ -186,8 +194,8 @@ func CreatePacmanDb(ctx *context.Context, owner, architecture, distro string) ([ for _, version := range versions { desc, err := LoadDbDescFile(ctx, &DescParams{ Version: version, - Arch: architecture, - Distro: distro, + Arch: p.Architecture, + Distro: p.Distribution, PkgName: pkg.Name, }) if err != nil { @@ -221,20 +229,20 @@ func LoadDbDescFile(ctx *context.Context, p *DescParams) ([]byte, error) { } for _, distroarch := range md.DistroArch { - var storagekey string + var file string if distroarch == p.Distro+"-"+p.Arch { - storagekey = p.PkgName + "-" + p.Version.Version + "-" + p.Arch + ".desc" + file = p.PkgName + "-" + p.Version.Version + "-" + p.Arch + ".desc" } if distroarch == p.Distro+"-any" { - storagekey = p.PkgName + "-" + p.Version.Version + "-any.desc" + file = p.PkgName + "-" + p.Version.Version + "-any.desc" } - if storagekey == "" { + if file == "" { continue } - descfile, err := GetFileObject(ctx, p.Distro, storagekey) + descfile, err := GetFileObject(ctx, p.Distro, file) if err != nil { return nil, err } diff --git a/tests/integration/api_packages_arch_test.go b/tests/integration/api_packages_arch_test.go index 8a90a75df7ab..dc7073e8369c 100644 --- a/tests/integration/api_packages_arch_test.go +++ b/tests/integration/api_packages_arch_test.go @@ -8,14 +8,12 @@ import ( "encoding/base64" "encoding/hex" "fmt" - "io" "net/http" "path" "testing" "code.gitea.io/gitea/models/unittest" user_model "code.gitea.io/gitea/models/user" - "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/tests" "github.com/stretchr/testify/assert" @@ -157,10 +155,14 @@ H/p7XvruIemeWh6SgfsOAIv9+grBts8v3TfVdrMpvpQIQhkyOtfaAOBNsf5UfCz3CE17aLundn7z t.Run("PushFirstPackage", func(t *testing.T) { defer tests.PrintCurrentTest(t)() - req := NewRequest(t, "PUT", path.Join(rootURL, "/push", "package-1-1-x86_64.pkg.tar.zst", "archlinux", hex.EncodeToString(firstPackageSignatureData))) + req := NewRequestWithBody(t, "PUT", + path.Join( + rootURL, "push", "package-1-1-x86_64.pkg.tar.zst", + "archlinux", hex.EncodeToString(firstPackageSignatureData), + ), + bytes.NewReader(firstPackageData), + ) - req.Header.Set("Content-Type", "application/octet-stream") - req.Body = io.NopCloser(bytes.NewReader(firstPackageData)) req = AddBasicAuthHeader(req, user.Name) MakeRequest(t, req, http.StatusOK) @@ -169,7 +171,9 @@ H/p7XvruIemeWh6SgfsOAIv9+grBts8v3TfVdrMpvpQIQhkyOtfaAOBNsf5UfCz3CE17aLundn7z t.Run("GetFirstPackage", func(t *testing.T) { defer tests.PrintCurrentTest(t)() - req := NewRequest(t, "GET", path.Join(rootURL, "/archlinux/x86_64/package-1-1-x86_64.pkg.tar.zst")) + req := NewRequest(t, "GET", + rootURL+"/archlinux/x86_64/package-1-1-x86_64.pkg.tar.zst", + ) resp := MakeRequest(t, req, http.StatusOK) @@ -179,7 +183,7 @@ H/p7XvruIemeWh6SgfsOAIv9+grBts8v3TfVdrMpvpQIQhkyOtfaAOBNsf5UfCz3CE17aLundn7z t.Run("GetFirstDatabase", func(t *testing.T) { defer tests.PrintCurrentTest(t)() - req := NewRequest(t, "GET", path.Join(rootURL, "/archlinux/x86_64/"+fmt.Sprintf("%s.%s.db", user.Name, setting.Domain))) + req := NewRequest(t, "GET", rootURL+"/archlinux/x86_64/test.db") resp := MakeRequest(t, req, http.StatusOK) @@ -189,7 +193,9 @@ H/p7XvruIemeWh6SgfsOAIv9+grBts8v3TfVdrMpvpQIQhkyOtfaAOBNsf5UfCz3CE17aLundn7z t.Run("GetFirstSignature", func(t *testing.T) { defer tests.PrintCurrentTest(t)() - req := NewRequest(t, "GET", path.Join(rootURL, "/archlinux/x86_64/package-1-1-x86_64.pkg.tar.zst.sig")) + req := NewRequest(t, "GET", + rootURL+"/archlinux/x86_64/package-1-1-x86_64.pkg.tar.zst.sig", + ) resp := MakeRequest(t, req, http.StatusOK) @@ -199,10 +205,14 @@ H/p7XvruIemeWh6SgfsOAIv9+grBts8v3TfVdrMpvpQIQhkyOtfaAOBNsf5UfCz3CE17aLundn7z t.Run("PushSecond", func(t *testing.T) { defer tests.PrintCurrentTest(t)() - req := NewRequest(t, "PUT", path.Join(rootURL, "/push", "package-1-1-any.pkg.tar.zst", "archlinux", hex.EncodeToString(secondPackageSignatureData))) + req := NewRequestWithBody(t, "PUT", + path.Join( + rootURL, "push", "package-1-1-any.pkg.tar.zst", + "archlinux", hex.EncodeToString(secondPackageSignatureData), + ), + bytes.NewReader(secondPackageData), + ) - req.Header.Set("Content-Type", "application/octet-stream") - req.Body = io.NopCloser(bytes.NewReader(secondPackageData)) req = AddBasicAuthHeader(req, user.Name) MakeRequest(t, req, http.StatusOK) @@ -211,7 +221,9 @@ H/p7XvruIemeWh6SgfsOAIv9+grBts8v3TfVdrMpvpQIQhkyOtfaAOBNsf5UfCz3CE17aLundn7z t.Run("GetSecondPackage", func(t *testing.T) { defer tests.PrintCurrentTest(t)() - req := NewRequest(t, "GET", path.Join(rootURL, "/archlinux/any/package-1-1-any.pkg.tar.zst")) + req := NewRequest(t, "GET", + rootURL+"/archlinux/any/package-1-1-any.pkg.tar.zst", + ) resp := MakeRequest(t, req, http.StatusOK) @@ -221,7 +233,7 @@ H/p7XvruIemeWh6SgfsOAIv9+grBts8v3TfVdrMpvpQIQhkyOtfaAOBNsf5UfCz3CE17aLundn7z t.Run("GetSecondDatabase", func(t *testing.T) { defer tests.PrintCurrentTest(t)() - req := NewRequest(t, "GET", path.Join(rootURL, "/archlinux/any/"+fmt.Sprintf("%s.%s.db.tar.gz", user.Name, setting.Domain))) + req := NewRequest(t, "GET", rootURL+"/archlinux/any/test2.db") resp := MakeRequest(t, req, http.StatusOK) @@ -231,7 +243,9 @@ H/p7XvruIemeWh6SgfsOAIv9+grBts8v3TfVdrMpvpQIQhkyOtfaAOBNsf5UfCz3CE17aLundn7z t.Run("GetSecondSignature", func(t *testing.T) { defer tests.PrintCurrentTest(t)() - req := NewRequest(t, "GET", path.Join(rootURL, "/archlinux/any/package-1-1-any.pkg.tar.zst.sig")) + req := NewRequest(t, "GET", + rootURL+"/archlinux/any/package-1-1-any.pkg.tar.zst.sig", + ) resp := MakeRequest(t, req, http.StatusOK) @@ -241,7 +255,7 @@ H/p7XvruIemeWh6SgfsOAIv9+grBts8v3TfVdrMpvpQIQhkyOtfaAOBNsf5UfCz3CE17aLundn7z t.Run("Remove", func(t *testing.T) { defer tests.PrintCurrentTest(t)() - req := NewRequest(t, "DELETE", path.Join(rootURL, "/remove", "package", "1-1")) + req := NewRequest(t, "DELETE", rootURL+"/remove/package/1-1") req = AddBasicAuthHeader(req, user.Name) From e86bdfcb0e965c8b7faf8003cc44f0f8d1697283 Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Tue, 8 Aug 2023 16:21:17 +0300 Subject: [PATCH 070/124] metadata and signature are stored in package properties instead of object storage --- modules/packages/arch/metadata.go | 3 +- routers/api/packages/arch/arch.go | 110 +++++++--------- .../arch/{db_manager.go => service.go} | 118 ++++++++---------- templates/package/content/arch.tmpl | 10 +- 4 files changed, 105 insertions(+), 136 deletions(-) rename services/packages/arch/{db_manager.go => service.go} (71%) diff --git a/modules/packages/arch/metadata.go b/modules/packages/arch/metadata.go index 94a879cc2e95..510c03e2715f 100644 --- a/modules/packages/arch/metadata.go +++ b/modules/packages/arch/metadata.go @@ -136,6 +136,7 @@ func EjectMetadata(p *EjectParams) (*DbDesc, error) { md.Backup = append(md.Backup, value) } } + return &md, nil } @@ -164,7 +165,7 @@ func getPkginfo(data io.Reader) (string, error) { } // Create pacman package description file. -func (m *DbDesc) GetDbDesc() string { +func (m *DbDesc) String() string { return strings.Join(rmEmptyStrings([]string{ formatField("FILENAME", m.Filename), formatField("NAME", m.Name), diff --git a/routers/api/packages/arch/arch.go b/routers/api/packages/arch/arch.go index dbc2331979e1..0b6ba83c924b 100644 --- a/routers/api/packages/arch/arch.go +++ b/routers/api/packages/arch/arch.go @@ -5,7 +5,6 @@ package arch import ( "bytes" - "encoding/hex" "io" "net/http" "strings" @@ -44,7 +43,6 @@ func Push(ctx *context.Context) { } defer buf.Close() - // Parse metadata related to package contained in arch package archive. desc, err := arch_module.EjectMetadata(&arch_module.EjectParams{ Filename: filename, Distribution: distro, @@ -55,12 +53,12 @@ func Push(ctx *context.Context) { return } - if _, err := buf.Seek(0, io.SeekStart); err != nil { + _, err = buf.Seek(0, io.SeekStart) + if err != nil { apiError(ctx, http.StatusInternalServerError, err) return } - // Metadata related to SQL database. md := &arch_module.Metadata{ URL: desc.URL, Description: desc.Description, @@ -74,75 +72,46 @@ func Push(ctx *context.Context) { DistroArch: []string{distro + "-" + desc.Arch[0]}, } - // Save file related to arch package. - pkgid, err := arch_service.SaveFile(ctx, &arch_service.SaveFileParams{ - Creator: ctx.ContextUser, - Owner: ctx.Package.Owner, - Metadata: md, - Buf: buf, - Filename: filename, - Distro: distro, - PkgName: desc.Name, - PkgVer: desc.Version, - }) + ver, _, err := pkg_service.CreatePackageOrAddFileToExisting( + &pkg_service.PackageCreationInfo{ + PackageInfo: pkg_service.PackageInfo{ + Owner: ctx.Package.Owner, + PackageType: pkg_model.TypeArch, + Name: desc.Name, + Version: desc.Version, + }, + Creator: ctx.ContextUser, + Metadata: md, + }, + &pkg_service.PackageFileCreationInfo{ + PackageFileInfo: pkg_service.PackageFileInfo{ + Filename: filename, + CompositeKey: distro + "-" + filename, + }, + OverwriteExisting: true, + IsLead: true, + Creator: ctx.ContextUser, + Data: buf, + }, + ) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return } - r := io.NopCloser(bytes.NewReader([]byte(desc.GetDbDesc()))) - buf, err = pkg_module.CreateHashedBufferFromReader(r) + _, err = pkg_model.InsertProperty(ctx, 0, ver.ID, distro+"-"+filename+".desc", desc.String()) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return } - defer buf.Close() - // Save file related to arch package description for pacman database. - _, err = arch_service.SaveFile(ctx, &arch_service.SaveFileParams{ - Creator: ctx.ContextUser, - Owner: ctx.Package.Owner, - Filename: desc.Name + "-" + desc.Version + "-" + desc.Arch[0] + ".desc", - Buf: buf, - Metadata: md, - Distro: distro, - PkgName: desc.Name, - PkgVer: desc.Version, - }) + _, err = pkg_model.InsertProperty(ctx, 0, ver.ID, distro+"-"+filename+".sig", sign) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return } - // Decoding package signature, if present saving with package as file. - sigdata, err := hex.DecodeString(sign) - if err == nil { - r := io.NopCloser(bytes.NewReader(sigdata)) - buf, err := pkg_module.CreateHashedBufferFromReader(r) - if err != nil { - apiError(ctx, http.StatusInternalServerError, err) - return - } - defer buf.Close() - - _, err = arch_service.SaveFile(ctx, &arch_service.SaveFileParams{ - Creator: ctx.ContextUser, - Owner: ctx.Package.Owner, - Buf: buf, - Filename: filename + ".sig", - Metadata: md, - Distro: distro, - PkgName: desc.Name, - PkgVer: desc.Version, - }) - if err != nil { - apiError(ctx, http.StatusInternalServerError, err) - return - } - } - - // Add new architectures and distribution info to package version metadata. - err = arch_service.UpdateMetadata(ctx, &arch_service.UpdateMetadataParameters{ + err = arch_service.UpdateMetadata(ctx, &arch_service.UpdateMetadataParams{ User: ctx.Package.Owner, Metadata: md, DbDesc: desc, @@ -152,8 +121,7 @@ func Push(ctx *context.Context) { return } - // Automatically connect repository with souce code if name matched. - err = arch_service.RepoConnect(ctx, owner, desc.Name, pkgid) + err = arch_service.RepoConnect(ctx, owner, desc.Name, ver.ID) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return @@ -171,8 +139,8 @@ func Get(ctx *context.Context) { arch = ctx.Params("arch") ) - // Packages and signatures are loaded directly from object storage. - if strings.HasSuffix(file, "tar.zst") || strings.HasSuffix(file, "zst.sig") { + // Packages are loaded directly from object storage. + if strings.HasSuffix(file, ".pkg.tar.zst") { pkg, err := arch_service.GetFileObject(ctx, distro, file) if err != nil { apiError(ctx, http.StatusNotFound, err) @@ -185,11 +153,25 @@ func Get(ctx *context.Context) { return } - // Pacman databases is not stored in giteas storage and created 'on-request' + // Signatures are loaded from package properties in SQL db. + if strings.HasSuffix(file, ".pkg.tar.zst.sig") { + sign, err := arch_service.GetProperty(ctx, owner, distro+"-"+file) + if err != nil { + apiError(ctx, http.StatusNotFound, err) + return + } + + ctx.ServeContent(bytes.NewReader(sign), &context.ServeHeaderOptions{ + Filename: file, + }) + return + } + + // Pacman databases is not stored in gitea storage and created 'on-request' // for user/organization scope with accordance to requested architecture // and distribution. if strings.HasSuffix(file, ".db.tar.gz") || strings.HasSuffix(file, ".db") { - db, err := arch_service.CreatePacmanDb(ctx, &arch_service.PacmanDbParams{ + db, err := arch_service.CreatePacmanDb(ctx, &arch_service.DbParams{ Owner: owner, Architecture: arch, Distribution: distro, diff --git a/services/packages/arch/db_manager.go b/services/packages/arch/service.go similarity index 71% rename from services/packages/arch/db_manager.go rename to services/packages/arch/service.go index 75870ee36d17..893d00665442 100644 --- a/services/packages/arch/db_manager.go +++ b/services/packages/arch/service.go @@ -4,8 +4,9 @@ package arch import ( + "encoding/hex" + "errors" "fmt" - "io" "sort" "strings" @@ -19,10 +20,9 @@ import ( "code.gitea.io/gitea/modules/packages" "code.gitea.io/gitea/modules/packages/arch" "code.gitea.io/gitea/modules/storage" - pkg_service "code.gitea.io/gitea/services/packages" ) -type UpdateMetadataParameters struct { +type UpdateMetadataParams struct { User *user.User Metadata *arch.Metadata DbDesc *arch.DbDesc @@ -30,13 +30,10 @@ type UpdateMetadataParameters struct { // Update package metadata stored in SQL database with new combination of // distribution and architecture. -func UpdateMetadata(ctx *context.Context, p *UpdateMetadataParameters) error { +func UpdateMetadata(ctx *context.Context, p *UpdateMetadataParams) error { ver, err := pkg_model.GetVersionByNameAndVersion( - ctx, - p.User.ID, - pkg_model.TypeArch, - p.DbDesc.Name, - p.DbDesc.Version, + ctx, p.User.ID, pkg_model.TypeArch, + p.DbDesc.Name, p.DbDesc.Version, ) if err != nil { return err @@ -68,51 +65,6 @@ func uniqueSlice(first, second []string) []string { return set.Values() } -// Parameters required to save new arch package. -type SaveFileParams struct { - Creator *user.User - Owner *user.User - Metadata *arch.Metadata - Buf packages.HashedSizeReader - Filename string - Distro string - IsLead bool - PkgName string - PkgVer string -} - -// Creates new package, version and package_file properties in database, -// and writes blob to file storage. If package/version/blob exists it -// will overwrite existing data. Package id and error will be returned. -func SaveFile(ctx *context.Context, p *SaveFileParams) (int64, error) { - ver, _, err := pkg_service.CreatePackageOrAddFileToExisting( - &pkg_service.PackageCreationInfo{ - PackageInfo: pkg_service.PackageInfo{ - Owner: p.Owner, - PackageType: pkg_model.TypeArch, - Name: p.PkgName, - Version: p.PkgVer, - }, - Creator: p.Creator, - Metadata: p.Metadata, - }, - &pkg_service.PackageFileCreationInfo{ - PackageFileInfo: pkg_service.PackageFileInfo{ - Filename: p.Filename, - CompositeKey: p.Distro + "-" + p.Filename, - }, - Creator: p.Creator, - Data: p.Buf, - OverwriteExisting: true, - IsLead: p.IsLead, - }, - ) - if err != nil { - return 0, err - } - return ver.ID, nil -} - // Get data related to provided filename and distribution, for package files // update download counter. func GetFileObject(ctx *context.Context, distro, file string) (storage.Object, error) { @@ -142,6 +94,40 @@ func GetFileObject(ctx *context.Context, distro, file string) (storage.Object, e return cs.Get(packages.BlobHash256Key(blob.HashSHA256)) } +// Get package property and transform it to string. +func GetProperty(ctx *context.Context, owner, key string) ([]byte, error) { + u, err := user.GetUserByName(ctx, owner) + if err != nil { + return nil, err + } + + k := strings.Split(key, "-") + + ver, err := pkg_model.GetVersionByNameAndVersion( + ctx, u.ID, pkg_model.TypeArch, strings.Join(k[1:len(k)-3], "-"), + strings.Join(k[len(k)-3:len(k)-1], "-"), + ) + if err != nil { + return nil, err + } + + pp, err := pkg_model.GetPropertiesByName(ctx, 0, ver.ID, key) + if err != nil { + return nil, err + } + + for _, property := range pp { + switch { + case strings.HasSuffix(key, ".desc"): + return []byte(property.Value), nil + case strings.HasSuffix(key, ".sig"): + return hex.DecodeString(property.Value) + } + } + + return nil, errors.New("unable to find package signature") +} + // Automatically connect repository with source code to published package, if // repository with the same name exists in user/organization scope. func RepoConnect(ctx *context.Context, owner, repo string, pkgid int64) error { @@ -155,7 +141,7 @@ func RepoConnect(ctx *context.Context, owner, repo string, pkgid int64) error { return nil } -type PacmanDbParams struct { +type DbParams struct { Owner string Architecture string Distribution string @@ -166,7 +152,7 @@ type PacmanDbParams struct { // requested combination of architecture and distribution. When/If the first // compatible version is found, related desc file will be loaded from object // storage and added to database archive. -func CreatePacmanDb(ctx *context.Context, p *PacmanDbParams) ([]byte, error) { +func CreatePacmanDb(ctx *context.Context, p *DbParams) ([]byte, error) { u, err := user.GetUserByName(ctx, p.Owner) if err != nil { return nil, err @@ -197,6 +183,7 @@ func CreatePacmanDb(ctx *context.Context, p *PacmanDbParams) ([]byte, error) { Arch: p.Architecture, Distro: p.Distribution, PkgName: pkg.Name, + Owner: p.Owner, }) if err != nil { return nil, err @@ -217,6 +204,7 @@ type DescParams struct { Arch string Distro string PkgName string + Owner string } // Get pacman desc file from object storage if combination of distribution and @@ -229,25 +217,23 @@ func LoadDbDescFile(ctx *context.Context, p *DescParams) ([]byte, error) { } for _, distroarch := range md.DistroArch { - var file string + var arch string if distroarch == p.Distro+"-"+p.Arch { - file = p.PkgName + "-" + p.Version.Version + "-" + p.Arch + ".desc" + arch = p.Arch } if distroarch == p.Distro+"-any" { - file = p.PkgName + "-" + p.Version.Version + "-any.desc" + arch = "any" } - if file == "" { + if arch == "" { continue } - descfile, err := GetFileObject(ctx, p.Distro, file) - if err != nil { - return nil, err - } - - return io.ReadAll(descfile) + return GetProperty(ctx, p.Owner, fmt.Sprintf( + "%s-%s-%s-%s.pkg.tar.zst.desc", + p.Distro, p.PkgName, p.Version.Version, arch, + )) } return nil, nil } diff --git a/templates/package/content/arch.tmpl b/templates/package/content/arch.tmpl index 8cbac8943819..74e32c482969 100644 --- a/templates/package/content/arch.tmpl +++ b/templates/package/content/arch.tmpl @@ -33,14 +33,14 @@ Server =
Provides
- {{StringUtils.Join $.PackageDescriptor.Metadata.Provides " "}} + {{StringUtils.Join $.PackageDescriptor.Metadata.Provides ", "}} {{end}} {{if .PackageDescriptor.Metadata.Depends}}
Depends
- {{StringUtils.Join $.PackageDescriptor.Metadata.Depends " "}} + {{StringUtils.Join $.PackageDescriptor.Metadata.Depends ", "}} {{end}} @@ -54,21 +54,21 @@ Server =
Make depends
- {{StringUtils.Join $.PackageDescriptor.Metadata.MakeDepends " "}} + {{StringUtils.Join $.PackageDescriptor.Metadata.MakeDepends ", "}} {{end}} {{if .PackageDescriptor.Metadata.CheckDepends}}
Check depends
- {{StringUtils.Join $.PackageDescriptor.Metadata.CheckDepends " "}} + {{StringUtils.Join $.PackageDescriptor.Metadata.CheckDepends ", "}} {{end}} {{if .PackageDescriptor.Metadata.Backup}}
Backup file
- {{StringUtils.Join $.PackageDescriptor.Metadata.Backup " "}} + {{StringUtils.Join $.PackageDescriptor.Metadata.Backup ", "}} {{end}} From 0d08eaa55600992639ab1b938489a113fa4ebe95 Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Tue, 8 Aug 2023 19:40:54 +0300 Subject: [PATCH 071/124] reduced amount of logic, removed unnecessary fields --- models/packages/package_file.go | 2 + models/packages/package_property.go | 13 +++ modules/packages/arch/metadata.go | 1 - routers/api/packages/arch/arch.go | 70 +++++++------- services/packages/arch/service.go | 137 ++-------------------------- templates/package/content/arch.tmpl | 7 -- 6 files changed, 53 insertions(+), 177 deletions(-) diff --git a/models/packages/package_file.go b/models/packages/package_file.go index 1c2c9ac072f4..0200a354cb69 100644 --- a/models/packages/package_file.go +++ b/models/packages/package_file.go @@ -25,6 +25,8 @@ var ( ErrDuplicatePackageFile = util.NewAlreadyExistErrorf("package file already exists") // ErrPackageFileNotExist indicates a package file not exist error ErrPackageFileNotExist = util.NewNotExistErrorf("package file does not exist") + // ErrPackagePropertyNotExist indicates a package property not exist error + ErrPackagePropertyNotExist = util.NewNotExistErrorf("package property does not exist") ) // EmptyFileKey is a named constant for an empty file key diff --git a/models/packages/package_property.go b/models/packages/package_property.go index e0170016cfc9..728b3135da59 100644 --- a/models/packages/package_property.go +++ b/models/packages/package_property.go @@ -60,6 +60,19 @@ func GetPropertiesByName(ctx context.Context, refType PropertyType, refID int64, return pps, db.GetEngine(ctx).Where("ref_type = ? AND ref_id = ? AND name = ?", refType, refID, name).Find(&pps) } +// GetPropertieWithUniqueName gets propertie with unique name +func GetPropertieWithUniqueName(ctx context.Context, name string) (*PackageProperty, error) { + p := &PackageProperty{Name: name} + has, err := db.GetEngine(ctx).Get(p) + if err != nil { + return nil, err + } + if !has { + return nil, ErrPackagePropertyNotExist + } + return p, nil +} + // UpdateProperty updates a property func UpdateProperty(ctx context.Context, pp *PackageProperty) error { _, err := db.GetEngine(ctx).ID(pp.ID).Update(pp) diff --git a/modules/packages/arch/metadata.go b/modules/packages/arch/metadata.go index 510c03e2715f..946233d2d7c4 100644 --- a/modules/packages/arch/metadata.go +++ b/modules/packages/arch/metadata.go @@ -32,7 +32,6 @@ type Metadata struct { MakeDepends []string `json:"make-depends,omitempty"` CheckDepends []string `json:"check-depends,omitempty"` Backup []string `json:"backup,omitempty"` - DistroArch []string `json:"distro-arch,omitempty"` } // Package description file that will be saved as .desc file in object storage. diff --git a/routers/api/packages/arch/arch.go b/routers/api/packages/arch/arch.go index 0b6ba83c924b..7ad7b40d6fb3 100644 --- a/routers/api/packages/arch/arch.go +++ b/routers/api/packages/arch/arch.go @@ -5,6 +5,7 @@ package arch import ( "bytes" + "encoding/hex" "io" "net/http" "strings" @@ -59,17 +60,15 @@ func Push(ctx *context.Context) { return } - md := &arch_module.Metadata{ - URL: desc.URL, - Description: desc.Description, - Provides: desc.Provides, - License: desc.License, - Depends: desc.Depends, - OptDepends: desc.OptDepends, - MakeDepends: desc.MakeDepends, - CheckDepends: desc.CheckDepends, - Backup: desc.Backup, - DistroArch: []string{distro + "-" + desc.Arch[0]}, + props := map[string]string{ + distro + "-" + filename + ".desc": desc.String(), + } + if sign != "" { + _, err := hex.DecodeString(sign) + if err != nil { + apiError(ctx, http.StatusBadRequest, err) + } + props[distro+"-"+filename+".sig"] = sign } ver, _, err := pkg_service.CreatePackageOrAddFileToExisting( @@ -80,8 +79,18 @@ func Push(ctx *context.Context) { Name: desc.Name, Version: desc.Version, }, - Creator: ctx.ContextUser, - Metadata: md, + Creator: ctx.ContextUser, + Metadata: &arch_module.Metadata{ + URL: desc.URL, + Description: desc.Description, + Provides: desc.Provides, + License: desc.License, + Depends: desc.Depends, + OptDepends: desc.OptDepends, + MakeDepends: desc.MakeDepends, + CheckDepends: desc.CheckDepends, + Backup: desc.Backup, + }, }, &pkg_service.PackageFileCreationInfo{ PackageFileInfo: pkg_service.PackageFileInfo{ @@ -92,6 +101,7 @@ func Push(ctx *context.Context) { IsLead: true, Creator: ctx.ContextUser, Data: buf, + Properties: props, }, ) if err != nil { @@ -99,28 +109,6 @@ func Push(ctx *context.Context) { return } - _, err = pkg_model.InsertProperty(ctx, 0, ver.ID, distro+"-"+filename+".desc", desc.String()) - if err != nil { - apiError(ctx, http.StatusInternalServerError, err) - return - } - - _, err = pkg_model.InsertProperty(ctx, 0, ver.ID, distro+"-"+filename+".sig", sign) - if err != nil { - apiError(ctx, http.StatusInternalServerError, err) - return - } - - err = arch_service.UpdateMetadata(ctx, &arch_service.UpdateMetadataParams{ - User: ctx.Package.Owner, - Metadata: md, - DbDesc: desc, - }) - if err != nil { - apiError(ctx, http.StatusInternalServerError, err) - return - } - err = arch_service.RepoConnect(ctx, owner, desc.Name, ver.ID) if err != nil { apiError(ctx, http.StatusInternalServerError, err) @@ -153,15 +141,21 @@ func Get(ctx *context.Context) { return } - // Signatures are loaded from package properties in SQL db. + // Signatures are loaded from package file properties in SQL db. if strings.HasSuffix(file, ".pkg.tar.zst.sig") { - sign, err := arch_service.GetProperty(ctx, owner, distro+"-"+file) + p, err := pkg_model.GetPropertieWithUniqueName(ctx, distro+"-"+file) if err != nil { apiError(ctx, http.StatusNotFound, err) return } - ctx.ServeContent(bytes.NewReader(sign), &context.ServeHeaderOptions{ + b, err := hex.DecodeString(p.Value) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + ctx.ServeContent(bytes.NewReader(b), &context.ServeHeaderOptions{ Filename: file, }) return diff --git a/services/packages/arch/service.go b/services/packages/arch/service.go index 893d00665442..bbe8e8762641 100644 --- a/services/packages/arch/service.go +++ b/services/packages/arch/service.go @@ -4,8 +4,6 @@ package arch import ( - "encoding/hex" - "errors" "fmt" "sort" "strings" @@ -14,57 +12,12 @@ import ( pkg_model "code.gitea.io/gitea/models/packages" repository "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/models/user" - "code.gitea.io/gitea/modules/container" "code.gitea.io/gitea/modules/context" - "code.gitea.io/gitea/modules/json" "code.gitea.io/gitea/modules/packages" "code.gitea.io/gitea/modules/packages/arch" "code.gitea.io/gitea/modules/storage" ) -type UpdateMetadataParams struct { - User *user.User - Metadata *arch.Metadata - DbDesc *arch.DbDesc -} - -// Update package metadata stored in SQL database with new combination of -// distribution and architecture. -func UpdateMetadata(ctx *context.Context, p *UpdateMetadataParams) error { - ver, err := pkg_model.GetVersionByNameAndVersion( - ctx, p.User.ID, pkg_model.TypeArch, - p.DbDesc.Name, p.DbDesc.Version, - ) - if err != nil { - return err - } - - var currmd arch.Metadata - err = json.Unmarshal([]byte(ver.MetadataJSON), &currmd) - if err != nil { - return err - } - - currmd.DistroArch = uniqueSlice(currmd.DistroArch, p.Metadata.DistroArch) - - b, err := json.Marshal(&currmd) - if err != nil { - return err - } - - ver.MetadataJSON = string(b) - - return pkg_model.UpdateVersion(ctx, ver) -} - -// Creates a list containing unique values formed of 2 passed slices. -func uniqueSlice(first, second []string) []string { - set := make(container.Set[string], len(first)+len(second)) - set.AddMultiple(first...) - set.AddMultiple(second...) - return set.Values() -} - // Get data related to provided filename and distribution, for package files // update download counter. func GetFileObject(ctx *context.Context, distro, file string) (storage.Object, error) { @@ -94,40 +47,6 @@ func GetFileObject(ctx *context.Context, distro, file string) (storage.Object, e return cs.Get(packages.BlobHash256Key(blob.HashSHA256)) } -// Get package property and transform it to string. -func GetProperty(ctx *context.Context, owner, key string) ([]byte, error) { - u, err := user.GetUserByName(ctx, owner) - if err != nil { - return nil, err - } - - k := strings.Split(key, "-") - - ver, err := pkg_model.GetVersionByNameAndVersion( - ctx, u.ID, pkg_model.TypeArch, strings.Join(k[1:len(k)-3], "-"), - strings.Join(k[len(k)-3:len(k)-1], "-"), - ) - if err != nil { - return nil, err - } - - pp, err := pkg_model.GetPropertiesByName(ctx, 0, ver.ID, key) - if err != nil { - return nil, err - } - - for _, property := range pp { - switch { - case strings.HasSuffix(key, ".desc"): - return []byte(property.Value), nil - case strings.HasSuffix(key, ".sig"): - return hex.DecodeString(property.Value) - } - } - - return nil, errors.New("unable to find package signature") -} - // Automatically connect repository with source code to published package, if // repository with the same name exists in user/organization scope. func RepoConnect(ctx *context.Context, owner, repo string, pkgid int64) error { @@ -178,62 +97,18 @@ func CreatePacmanDb(ctx *context.Context, p *DbParams) ([]byte, error) { }) for _, version := range versions { - desc, err := LoadDbDescFile(ctx, &DescParams{ - Version: version, - Arch: p.Architecture, - Distro: p.Distribution, - PkgName: pkg.Name, - Owner: p.Owner, - }) + p, err := pkg_model.GetPropertieWithUniqueName(ctx, fmt.Sprintf( + "%s-%s-%s-%s.pkg.tar.zst.desc", + p.Distribution, pkg.Name, version.Version, p.Architecture, + )) if err != nil { return nil, err } - if desc == nil { - continue - } - entries[pkg.Name+"-"+version.Version+"/desc"] = desc + + entries[pkg.Name+"-"+version.Version+"/desc"] = []byte(p.Value) break } } return arch.CreatePacmanDb(entries) } - -type DescParams struct { - Version *pkg_model.PackageVersion - Arch string - Distro string - PkgName string - Owner string -} - -// Get pacman desc file from object storage if combination of distribution and -// architecture is supported (checked in metadata). -func LoadDbDescFile(ctx *context.Context, p *DescParams) ([]byte, error) { - var md arch.Metadata - err := json.Unmarshal([]byte(p.Version.MetadataJSON), &md) - if err != nil { - return nil, err - } - - for _, distroarch := range md.DistroArch { - var arch string - - if distroarch == p.Distro+"-"+p.Arch { - arch = p.Arch - } - if distroarch == p.Distro+"-any" { - arch = "any" - } - - if arch == "" { - continue - } - - return GetProperty(ctx, p.Owner, fmt.Sprintf( - "%s-%s-%s-%s.pkg.tar.zst.desc", - p.Distro, p.PkgName, p.Version.Version, arch, - )) - } - return nil, nil -} diff --git a/templates/package/content/arch.tmpl b/templates/package/content/arch.tmpl index 74e32c482969..17df1fb7368d 100644 --- a/templates/package/content/arch.tmpl +++ b/templates/package/content/arch.tmpl @@ -71,13 +71,6 @@ Server = {{end}} - - {{if .PackageDescriptor.Metadata.DistroArch}} - -
Available for
- {{StringUtils.Join $.PackageDescriptor.Metadata.DistroArch ", "}} - - {{end}} From 27fa8158ef4e2bd827e6a6768e4cb4ac0b5dab82 Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Tue, 8 Aug 2023 19:46:58 +0300 Subject: [PATCH 072/124] fixed database creation procedure for packages with any architecture --- services/packages/arch/service.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/services/packages/arch/service.go b/services/packages/arch/service.go index bbe8e8762641..9b10a0dace26 100644 --- a/services/packages/arch/service.go +++ b/services/packages/arch/service.go @@ -97,15 +97,21 @@ func CreatePacmanDb(ctx *context.Context, p *DbParams) ([]byte, error) { }) for _, version := range versions { - p, err := pkg_model.GetPropertieWithUniqueName(ctx, fmt.Sprintf( + pp, err := pkg_model.GetPropertieWithUniqueName(ctx, fmt.Sprintf( "%s-%s-%s-%s.pkg.tar.zst.desc", p.Distribution, pkg.Name, version.Version, p.Architecture, )) if err != nil { - return nil, err + pp, err = pkg_model.GetPropertieWithUniqueName(ctx, fmt.Sprintf( + "%s-%s-%s-any.pkg.tar.zst.desc", + p.Distribution, pkg.Name, version.Version, + )) + if err != nil { + return nil, err + } } - entries[pkg.Name+"-"+version.Version+"/desc"] = []byte(p.Value) + entries[pkg.Name+"-"+version.Version+"/desc"] = []byte(pp.Value) break } } From 9273dc055da296527a24de338cca36e965533ec8 Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Wed, 16 Aug 2023 17:26:26 +0300 Subject: [PATCH 073/124] removed AUR helpers from arch package manager list --- docs/content/usage/packages/overview.en-us.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/content/usage/packages/overview.en-us.md b/docs/content/usage/packages/overview.en-us.md index 6cd82b548234..04a8b546c4ca 100644 --- a/docs/content/usage/packages/overview.en-us.md +++ b/docs/content/usage/packages/overview.en-us.md @@ -24,7 +24,7 @@ The following package managers are currently supported: | Name | Language | Package client | | ---- | -------- | -------------- | | [Alpine](usage/packages/alpine.md) | - | `apk` | -| [Arch](usage/packages/arch.md) | - | `pacman`,`yay`,`paru`,`pikaur`,`aura`,`pack` | +| [Arch](usage/packages/arch.md) | - | `pacman` | | [Cargo](usage/packages/cargo.md) | Rust | `cargo` | | [Chef](usage/packages/chef.md) | - | `knife` | | [Composer](usage/packages/composer.md) | PHP | `composer` | From ce01499d8f7a892f1363ed90cc7f8fa8a926e22c Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Fri, 18 Aug 2023 18:09:08 +0300 Subject: [PATCH 074/124] removed/corrected some documentation --- routers/api/packages/arch/arch.go | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/routers/api/packages/arch/arch.go b/routers/api/packages/arch/arch.go index 7ad7b40d6fb3..edd8d9691ca5 100644 --- a/routers/api/packages/arch/arch.go +++ b/routers/api/packages/arch/arch.go @@ -127,7 +127,6 @@ func Get(ctx *context.Context) { arch = ctx.Params("arch") ) - // Packages are loaded directly from object storage. if strings.HasSuffix(file, ".pkg.tar.zst") { pkg, err := arch_service.GetFileObject(ctx, distro, file) if err != nil { @@ -141,7 +140,6 @@ func Get(ctx *context.Context) { return } - // Signatures are loaded from package file properties in SQL db. if strings.HasSuffix(file, ".pkg.tar.zst.sig") { p, err := pkg_model.GetPropertieWithUniqueName(ctx, distro+"-"+file) if err != nil { @@ -161,9 +159,6 @@ func Get(ctx *context.Context) { return } - // Pacman databases is not stored in gitea storage and created 'on-request' - // for user/organization scope with accordance to requested architecture - // and distribution. if strings.HasSuffix(file, ".db.tar.gz") || strings.HasSuffix(file, ".db") { db, err := arch_service.CreatePacmanDb(ctx, &arch_service.DbParams{ Owner: owner, @@ -184,7 +179,7 @@ func Get(ctx *context.Context) { ctx.Status(http.StatusNotFound) } -// Remove specific package version, related files and pacman database entry. +// Remove specific package version, related files with properties. func Remove(ctx *context.Context) { var ( pkg = ctx.Params("package") From 431dabfc5ca922ce31b87f92ff137cafe09cfe49 Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Tue, 5 Sep 2023 14:51:29 -0300 Subject: [PATCH 075/124] fixed procedure of retrieving package properties for signatures and pacman database file --- models/packages/package_property.go | 13 --- routers/api/packages/arch/arch.go | 12 +-- services/packages/arch/service.go | 114 +++++++++++++++----- tests/integration/api_packages_arch_test.go | 2 +- 4 files changed, 92 insertions(+), 49 deletions(-) diff --git a/models/packages/package_property.go b/models/packages/package_property.go index 728b3135da59..e0170016cfc9 100644 --- a/models/packages/package_property.go +++ b/models/packages/package_property.go @@ -60,19 +60,6 @@ func GetPropertiesByName(ctx context.Context, refType PropertyType, refID int64, return pps, db.GetEngine(ctx).Where("ref_type = ? AND ref_id = ? AND name = ?", refType, refID, name).Find(&pps) } -// GetPropertieWithUniqueName gets propertie with unique name -func GetPropertieWithUniqueName(ctx context.Context, name string) (*PackageProperty, error) { - p := &PackageProperty{Name: name} - has, err := db.GetEngine(ctx).Get(p) - if err != nil { - return nil, err - } - if !has { - return nil, ErrPackagePropertyNotExist - } - return p, nil -} - // UpdateProperty updates a property func UpdateProperty(ctx context.Context, pp *PackageProperty) error { _, err := db.GetEngine(ctx).ID(pp.ID).Update(pp) diff --git a/routers/api/packages/arch/arch.go b/routers/api/packages/arch/arch.go index edd8d9691ca5..9903d610343e 100644 --- a/routers/api/packages/arch/arch.go +++ b/routers/api/packages/arch/arch.go @@ -128,7 +128,7 @@ func Get(ctx *context.Context) { ) if strings.HasSuffix(file, ".pkg.tar.zst") { - pkg, err := arch_service.GetFileObject(ctx, distro, file) + pkg, err := arch_service.GetPackageFile(ctx, distro, file) if err != nil { apiError(ctx, http.StatusNotFound, err) return @@ -141,19 +141,13 @@ func Get(ctx *context.Context) { } if strings.HasSuffix(file, ".pkg.tar.zst.sig") { - p, err := pkg_model.GetPropertieWithUniqueName(ctx, distro+"-"+file) + sig, err := arch_service.GetPackageSignature(ctx, distro, file) if err != nil { apiError(ctx, http.StatusNotFound, err) return } - b, err := hex.DecodeString(p.Value) - if err != nil { - apiError(ctx, http.StatusInternalServerError, err) - return - } - - ctx.ServeContent(bytes.NewReader(b), &context.ServeHeaderOptions{ + ctx.ServeContent(sig, &context.ServeHeaderOptions{ Filename: file, }) return diff --git a/services/packages/arch/service.go b/services/packages/arch/service.go index 9b10a0dace26..e20ce39202ce 100644 --- a/services/packages/arch/service.go +++ b/services/packages/arch/service.go @@ -4,6 +4,9 @@ package arch import ( + "bytes" + "encoding/hex" + "errors" "fmt" "sort" "strings" @@ -11,7 +14,6 @@ import ( "code.gitea.io/gitea/models/db" pkg_model "code.gitea.io/gitea/models/packages" repository "code.gitea.io/gitea/models/repo" - "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/packages" "code.gitea.io/gitea/modules/packages/arch" @@ -20,17 +22,15 @@ import ( // Get data related to provided filename and distribution, for package files // update download counter. -func GetFileObject(ctx *context.Context, distro, file string) (storage.Object, error) { - db := db.GetEngine(ctx) - +func GetPackageFile(ctx *context.Context, distro, file string) (storage.Object, error) { pkgfile := &pkg_model.PackageFile{CompositeKey: distro + "-" + file} - ok, err := db.Get(pkgfile) + ok, err := db.GetEngine(ctx).Get(pkgfile) if err != nil || !ok { return nil, fmt.Errorf("%+v %t", err, ok) } - blob, err := pkg_model.GetBlobByID(ctx, pkgfile.BlobID) + b, err := pkg_model.GetBlobByID(ctx, pkgfile.BlobID) if err != nil { return nil, err } @@ -42,9 +42,53 @@ func GetFileObject(ctx *context.Context, distro, file string) (storage.Object, e } } - cs := packages.NewContentStore() + return packages.NewContentStore().Get(packages.BlobHash256Key(b.HashSHA256)) +} + +// This function will search for package signature and if present, will load it +// from package file properties, and return its byte reader. +func GetPackageSignature(ctx *context.Context, distro, file string) (*bytes.Reader, error) { + var ( + splt = strings.Split(file, "-") + packagename = strings.Join(splt[0:len(splt)-3], "-") + versionname = splt[len(splt)-3] + "-" + splt[len(splt)-2] + pkgfilename = strings.TrimSuffix(file, ".sig") + filekey = distro + "-" + pkgfilename + signkey = distro + "-" + file + ) + + version, err := pkg_model.GetVersionByNameAndVersion( + ctx, ctx.Package.Owner.ID, pkg_model.TypeArch, packagename, versionname, + ) + if err != nil { + return nil, err + } + + pkgfile, err := pkg_model.GetFileForVersionByName( + ctx, version.ID, pkgfilename, filekey, + ) + if err != nil { + return nil, err + } + + proprs, err := pkg_model.GetProperties( + ctx, pkg_model.PropertyTypeFile, pkgfile.ID, + ) + if err != nil { + return nil, err + } - return cs.Get(packages.BlobHash256Key(blob.HashSHA256)) + for _, pp := range proprs { + if pp.Name == signkey { + b, err := hex.DecodeString(pp.Value) + if err != nil { + return nil, err + } + return bytes.NewReader(b), nil + } + } + + return nil, errors.New("signature for requested package not found") } // Automatically connect repository with source code to published package, if @@ -69,15 +113,10 @@ type DbParams struct { // Finds all arch packages in user/organization scope, each package version // starting from latest in descending order is checked to be compatible with // requested combination of architecture and distribution. When/If the first -// compatible version is found, related desc file will be loaded from object -// storage and added to database archive. +// compatible version is found, related desc file will be loaded from database +// and added to resulting .db.tar.gz archive. func CreatePacmanDb(ctx *context.Context, p *DbParams) ([]byte, error) { - u, err := user.GetUserByName(ctx, p.Owner) - if err != nil { - return nil, err - } - - pkgs, err := pkg_model.GetPackagesByType(ctx, u.ID, pkg_model.TypeArch) + pkgs, err := pkg_model.GetPackagesByType(ctx, ctx.Package.Owner.ID, pkg_model.TypeArch) if err != nil { return nil, err } @@ -86,7 +125,7 @@ func CreatePacmanDb(ctx *context.Context, p *DbParams) ([]byte, error) { for _, pkg := range pkgs { versions, err := pkg_model.GetVersionsByPackageName( - ctx, u.ID, pkg_model.TypeArch, pkg.Name, + ctx, ctx.Package.Owner.ID, pkg_model.TypeArch, pkg.Name, ) if err != nil { return nil, err @@ -97,21 +136,44 @@ func CreatePacmanDb(ctx *context.Context, p *DbParams) ([]byte, error) { }) for _, version := range versions { - pp, err := pkg_model.GetPropertieWithUniqueName(ctx, fmt.Sprintf( - "%s-%s-%s-%s.pkg.tar.zst.desc", - p.Distribution, pkg.Name, version.Version, p.Architecture, - )) + filename := fmt.Sprintf( + "%s-%s-%s.pkg.tar.zst", + pkg.Name, version.Version, p.Architecture, + ) + + file, err := pkg_model.GetFileForVersionByName( + ctx, version.ID, filename, p.Distribution+"-"+filename, + ) if err != nil { - pp, err = pkg_model.GetPropertieWithUniqueName(ctx, fmt.Sprintf( - "%s-%s-%s-any.pkg.tar.zst.desc", - p.Distribution, pkg.Name, version.Version, - )) + filename := fmt.Sprintf( + "%s-%s-any.pkg.tar.zst", + pkg.Name, version.Version, + ) + file, err = pkg_model.GetFileForVersionByName( + ctx, version.ID, filename, p.Distribution+filename, + ) if err != nil { return nil, err } } - entries[pkg.Name+"-"+version.Version+"/desc"] = []byte(pp.Value) + pps, err := pkg_model.GetProperties(ctx, pkg_model.PropertyTypeFile, file.ID) + if err != nil { + return nil, err + } + + var descvalue string + for _, pp := range pps { + if strings.HasSuffix(pp.Name, ".desc") { + descvalue = pp.Value + } + } + + if descvalue == "" { + continue + } + + entries[pkg.Name+"-"+version.Version+"/desc"] = []byte(descvalue) break } } diff --git a/tests/integration/api_packages_arch_test.go b/tests/integration/api_packages_arch_test.go index dc7073e8369c..22c2597a92e2 100644 --- a/tests/integration/api_packages_arch_test.go +++ b/tests/integration/api_packages_arch_test.go @@ -233,7 +233,7 @@ H/p7XvruIemeWh6SgfsOAIv9+grBts8v3TfVdrMpvpQIQhkyOtfaAOBNsf5UfCz3CE17aLundn7z t.Run("GetSecondDatabase", func(t *testing.T) { defer tests.PrintCurrentTest(t)() - req := NewRequest(t, "GET", rootURL+"/archlinux/any/test2.db") + req := NewRequest(t, "GET", rootURL+"/archlinux/any/localhost.db") resp := MakeRequest(t, req, http.StatusOK) From 7c2d1b3ca3557523488a88c63c42a694906d4ddb Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Thu, 7 Sep 2023 15:23:13 -0300 Subject: [PATCH 076/124] fixed package key when getting any architecture --- services/packages/arch/service.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/packages/arch/service.go b/services/packages/arch/service.go index e20ce39202ce..2890f6161cab 100644 --- a/services/packages/arch/service.go +++ b/services/packages/arch/service.go @@ -150,7 +150,7 @@ func CreatePacmanDb(ctx *context.Context, p *DbParams) ([]byte, error) { pkg.Name, version.Version, ) file, err = pkg_model.GetFileForVersionByName( - ctx, version.ID, filename, p.Distribution+filename, + ctx, version.ID, filename, p.Distribution+"-"+filename, ) if err != nil { return nil, err From a2aaae3dc7eb3d06b56db7fb151ca97f95471cb6 Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Tue, 12 Sep 2023 12:05:25 -0300 Subject: [PATCH 077/124] refactoring funciton creating arch package database archive --- modules/packages/arch/metadata.go | 25 +++++++++---------------- 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/modules/packages/arch/metadata.go b/modules/packages/arch/metadata.go index 946233d2d7c4..cf2db7debd1a 100644 --- a/modules/packages/arch/metadata.go +++ b/modules/packages/arch/metadata.go @@ -218,23 +218,12 @@ func rmEmptyStrings(s []string) []string { func CreatePacmanDb(entries map[string][]byte) ([]byte, error) { var out bytes.Buffer - // Write entries to new buffer and return it. - err := writeToArchive(entries, &out) - if err != nil { - return nil, err - } - - return out.Bytes(), nil -} - -// Write pacman package entries to tarball. -func writeToArchive(files map[string][]byte, buf io.Writer) error { - gw := gzip.NewWriter(buf) + gw := gzip.NewWriter(&out) defer gw.Close() tw := tar.NewWriter(gw) defer tw.Close() - for name, content := range files { + for name, content := range entries { hdr := &tar.Header{ Name: name, Size: int64(len(content)), @@ -243,13 +232,17 @@ func writeToArchive(files map[string][]byte, buf io.Writer) error { err := tw.WriteHeader(hdr) if err != nil { - return err + return nil, err } _, err = io.Copy(tw, bytes.NewReader(content)) if err != nil { - return err + return nil, err } } - return nil + + tw.Close() + gw.Close() + + return out.Bytes(), nil } From d7bd108358ed5295a98ec228a0bc79de16f506da Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Sat, 23 Sep 2023 18:01:03 -0300 Subject: [PATCH 078/124] Update models/packages/package_file.go Co-authored-by: KN4CK3R --- models/packages/package_file.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/models/packages/package_file.go b/models/packages/package_file.go index 0200a354cb69..1c2c9ac072f4 100644 --- a/models/packages/package_file.go +++ b/models/packages/package_file.go @@ -25,8 +25,6 @@ var ( ErrDuplicatePackageFile = util.NewAlreadyExistErrorf("package file already exists") // ErrPackageFileNotExist indicates a package file not exist error ErrPackageFileNotExist = util.NewNotExistErrorf("package file does not exist") - // ErrPackagePropertyNotExist indicates a package property not exist error - ErrPackagePropertyNotExist = util.NewNotExistErrorf("package property does not exist") ) // EmptyFileKey is a named constant for an empty file key From 60052811ad721f46f1b334da3abfefe1190e2761 Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Sat, 23 Sep 2023 18:05:36 -0300 Subject: [PATCH 079/124] fix request permission access position for push/remove handlers --- routers/api/packages/api.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/routers/api/packages/api.go b/routers/api/packages/api.go index 0a5e9158bdf0..5a830a39a929 100644 --- a/routers/api/packages/api.go +++ b/routers/api/packages/api.go @@ -124,8 +124,8 @@ func CommonRoutes() *web.Route { }) }, reqPackageAccess(perm.AccessModeRead)) r.Group("/arch", func() { - r.Put("/push/{filename}/{distro}/{sign}", arch.Push, reqPackageAccess(perm.AccessModeWrite)) - r.Delete("/remove/{package}/{version}", arch.Remove, reqPackageAccess(perm.AccessModeWrite)) + r.Put("/push/{filename}/{distro}/{sign}", reqPackageAccess(perm.AccessModeWrite), arch.Push) + r.Delete("/remove/{package}/{version}", reqPackageAccess(perm.AccessModeWrite), arch.Remove) r.Get("/{distro}/{arch}/{file}", arch.Get) }) r.Group("/cargo", func() { From a9a5b7a2cf63568cb517479ed4b6bd02dacb6d11 Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Sat, 23 Sep 2023 20:26:51 -0300 Subject: [PATCH 080/124] service method to get package file stream --- services/packages/arch/service.go | 21 +++++---------------- 1 file changed, 5 insertions(+), 16 deletions(-) diff --git a/services/packages/arch/service.go b/services/packages/arch/service.go index 2890f6161cab..3c7d603f063c 100644 --- a/services/packages/arch/service.go +++ b/services/packages/arch/service.go @@ -8,6 +8,7 @@ import ( "encoding/hex" "errors" "fmt" + "io" "sort" "strings" @@ -15,14 +16,13 @@ import ( pkg_model "code.gitea.io/gitea/models/packages" repository "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/modules/context" - "code.gitea.io/gitea/modules/packages" "code.gitea.io/gitea/modules/packages/arch" - "code.gitea.io/gitea/modules/storage" + pkg_service "code.gitea.io/gitea/services/packages" ) // Get data related to provided filename and distribution, for package files // update download counter. -func GetPackageFile(ctx *context.Context, distro, file string) (storage.Object, error) { +func GetPackageFile(ctx *context.Context, distro, file string) (io.ReadSeekCloser, error) { pkgfile := &pkg_model.PackageFile{CompositeKey: distro + "-" + file} ok, err := db.GetEngine(ctx).Get(pkgfile) @@ -30,19 +30,8 @@ func GetPackageFile(ctx *context.Context, distro, file string) (storage.Object, return nil, fmt.Errorf("%+v %t", err, ok) } - b, err := pkg_model.GetBlobByID(ctx, pkgfile.BlobID) - if err != nil { - return nil, err - } - - if strings.HasSuffix(file, ".pkg.tar.zst") { - err = pkg_model.IncrementDownloadCounter(ctx, pkgfile.VersionID) - if err != nil { - return nil, err - } - } - - return packages.NewContentStore().Get(packages.BlobHash256Key(b.HashSHA256)) + filestream, _, _, err := pkg_service.GetPackageFileStream(ctx, pkgfile) + return filestream, err } // This function will search for package signature and if present, will load it From 54fd84d83f0c95021c98b9d7736e313c790fcd70 Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Sun, 24 Sep 2023 13:32:53 -0300 Subject: [PATCH 081/124] function to get package file property by composite key, refactoring function creating pacman database --- models/packages/package_file.go | 14 ++++++++ modules/packages/arch/metadata.go | 6 ++-- routers/api/packages/arch/arch.go | 6 +--- services/packages/arch/service.go | 55 ++++++++----------------------- 4 files changed, 33 insertions(+), 48 deletions(-) diff --git a/models/packages/package_file.go b/models/packages/package_file.go index 1c2c9ac072f4..b9d753829067 100644 --- a/models/packages/package_file.go +++ b/models/packages/package_file.go @@ -65,6 +65,20 @@ func TryInsertFile(ctx context.Context, pf *PackageFile) (*PackageFile, error) { return pf, nil } +// GetFileByCompositeKey gets unique file by composite key if exists +func GetFileByCompositeKey(ctx context.Context, key string) (*PackageFile, error) { + pf := &PackageFile{CompositeKey: key} + + has, err := db.GetEngine(ctx).Get(pf) + if err != nil { + return nil, err + } + if !has { + return nil, ErrPackageFileNotExist + } + return pf, nil +} + // GetFilesByVersionID gets all files of a version func GetFilesByVersionID(ctx context.Context, versionID int64) ([]*PackageFile, error) { pfs := make([]*PackageFile, 0, 10) diff --git a/modules/packages/arch/metadata.go b/modules/packages/arch/metadata.go index cf2db7debd1a..ecd96ddfcc13 100644 --- a/modules/packages/arch/metadata.go +++ b/modules/packages/arch/metadata.go @@ -219,9 +219,7 @@ func CreatePacmanDb(entries map[string][]byte) ([]byte, error) { var out bytes.Buffer gw := gzip.NewWriter(&out) - defer gw.Close() tw := tar.NewWriter(gw) - defer tw.Close() for name, content := range entries { hdr := &tar.Header{ @@ -232,11 +230,15 @@ func CreatePacmanDb(entries map[string][]byte) ([]byte, error) { err := tw.WriteHeader(hdr) if err != nil { + tw.Close() + gw.Close() return nil, err } _, err = io.Copy(tw, bytes.NewReader(content)) if err != nil { + tw.Close() + gw.Close() return nil, err } } diff --git a/routers/api/packages/arch/arch.go b/routers/api/packages/arch/arch.go index 9903d610343e..d0a4357bbd79 100644 --- a/routers/api/packages/arch/arch.go +++ b/routers/api/packages/arch/arch.go @@ -154,11 +154,7 @@ func Get(ctx *context.Context) { } if strings.HasSuffix(file, ".db.tar.gz") || strings.HasSuffix(file, ".db") { - db, err := arch_service.CreatePacmanDb(ctx, &arch_service.DbParams{ - Owner: owner, - Architecture: arch, - Distribution: distro, - }) + db, err := arch_service.CreatePacmanDb(ctx, owner, arch, distro) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return diff --git a/services/packages/arch/service.go b/services/packages/arch/service.go index 3c7d603f063c..5e01e49b3773 100644 --- a/services/packages/arch/service.go +++ b/services/packages/arch/service.go @@ -12,63 +12,42 @@ import ( "sort" "strings" - "code.gitea.io/gitea/models/db" pkg_model "code.gitea.io/gitea/models/packages" repository "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/modules/context" - "code.gitea.io/gitea/modules/packages/arch" + arch_module "code.gitea.io/gitea/modules/packages/arch" pkg_service "code.gitea.io/gitea/services/packages" ) // Get data related to provided filename and distribution, for package files // update download counter. func GetPackageFile(ctx *context.Context, distro, file string) (io.ReadSeekCloser, error) { - pkgfile := &pkg_model.PackageFile{CompositeKey: distro + "-" + file} - - ok, err := db.GetEngine(ctx).Get(pkgfile) - if err != nil || !ok { - return nil, fmt.Errorf("%+v %t", err, ok) + pf, err := pkg_model.GetFileByCompositeKey(ctx, distro+"-"+file) + if err != nil { + return nil, err } - filestream, _, _, err := pkg_service.GetPackageFileStream(ctx, pkgfile) + filestream, _, _, err := pkg_service.GetPackageFileStream(ctx, pf) return filestream, err } // This function will search for package signature and if present, will load it // from package file properties, and return its byte reader. func GetPackageSignature(ctx *context.Context, distro, file string) (*bytes.Reader, error) { - var ( - splt = strings.Split(file, "-") - packagename = strings.Join(splt[0:len(splt)-3], "-") - versionname = splt[len(splt)-3] + "-" + splt[len(splt)-2] - pkgfilename = strings.TrimSuffix(file, ".sig") - filekey = distro + "-" + pkgfilename - signkey = distro + "-" + file - ) - - version, err := pkg_model.GetVersionByNameAndVersion( - ctx, ctx.Package.Owner.ID, pkg_model.TypeArch, packagename, versionname, - ) - if err != nil { - return nil, err - } + pkgfile := strings.TrimSuffix(distro+"-"+file, ".sig") - pkgfile, err := pkg_model.GetFileForVersionByName( - ctx, version.ID, pkgfilename, filekey, - ) + pf, err := pkg_model.GetFileByCompositeKey(ctx, pkgfile) if err != nil { return nil, err } - proprs, err := pkg_model.GetProperties( - ctx, pkg_model.PropertyTypeFile, pkgfile.ID, - ) + proprs, err := pkg_model.GetProperties(ctx, pkg_model.PropertyTypeFile, pf.ID) if err != nil { return nil, err } for _, pp := range proprs { - if pp.Name == signkey { + if pp.Name == distro+"-"+file { b, err := hex.DecodeString(pp.Value) if err != nil { return nil, err @@ -93,18 +72,12 @@ func RepoConnect(ctx *context.Context, owner, repo string, pkgid int64) error { return nil } -type DbParams struct { - Owner string - Architecture string - Distribution string -} - // Finds all arch packages in user/organization scope, each package version // starting from latest in descending order is checked to be compatible with // requested combination of architecture and distribution. When/If the first // compatible version is found, related desc file will be loaded from database // and added to resulting .db.tar.gz archive. -func CreatePacmanDb(ctx *context.Context, p *DbParams) ([]byte, error) { +func CreatePacmanDb(ctx *context.Context, owner, arch, distro string) ([]byte, error) { pkgs, err := pkg_model.GetPackagesByType(ctx, ctx.Package.Owner.ID, pkg_model.TypeArch) if err != nil { return nil, err @@ -127,11 +100,11 @@ func CreatePacmanDb(ctx *context.Context, p *DbParams) ([]byte, error) { for _, version := range versions { filename := fmt.Sprintf( "%s-%s-%s.pkg.tar.zst", - pkg.Name, version.Version, p.Architecture, + pkg.Name, version.Version, arch, ) file, err := pkg_model.GetFileForVersionByName( - ctx, version.ID, filename, p.Distribution+"-"+filename, + ctx, version.ID, filename, distro+"-"+filename, ) if err != nil { filename := fmt.Sprintf( @@ -139,7 +112,7 @@ func CreatePacmanDb(ctx *context.Context, p *DbParams) ([]byte, error) { pkg.Name, version.Version, ) file, err = pkg_model.GetFileForVersionByName( - ctx, version.ID, filename, p.Distribution+"-"+filename, + ctx, version.ID, filename, distro+"-"+filename, ) if err != nil { return nil, err @@ -167,5 +140,5 @@ func CreatePacmanDb(ctx *context.Context, p *DbParams) ([]byte, error) { } } - return arch.CreatePacmanDb(entries) + return arch_module.CreatePacmanDb(entries) } From 907aa021a11962555c2dfb28ba3735664828eadc Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Sun, 24 Sep 2023 14:12:07 -0300 Subject: [PATCH 082/124] refactoring create pacman database function --- modules/packages/arch/metadata.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/modules/packages/arch/metadata.go b/modules/packages/arch/metadata.go index ecd96ddfcc13..d8ca5ed1b706 100644 --- a/modules/packages/arch/metadata.go +++ b/modules/packages/arch/metadata.go @@ -222,21 +222,19 @@ func CreatePacmanDb(entries map[string][]byte) ([]byte, error) { tw := tar.NewWriter(gw) for name, content := range entries { - hdr := &tar.Header{ + header := &tar.Header{ Name: name, Size: int64(len(content)), Mode: int64(os.ModePerm), } - err := tw.WriteHeader(hdr) - if err != nil { + if err := tw.WriteHeader(header); err != nil { tw.Close() gw.Close() return nil, err } - _, err = io.Copy(tw, bytes.NewReader(content)) - if err != nil { + if _, err := tw.Write(content); err != nil { tw.Close() gw.Close() return nil, err From 4656b3b9588a922bdec2093368feb3af3573af11 Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Mon, 25 Sep 2023 18:42:48 -0300 Subject: [PATCH 083/124] replaced bytes.Buffer with packages.HashedBuffer to return working io.ReadSeeker implementation --- modules/packages/arch/metadata.go | 12 +++++++----- routers/api/packages/arch/arch.go | 3 +-- services/packages/arch/service.go | 2 +- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/modules/packages/arch/metadata.go b/modules/packages/arch/metadata.go index d8ca5ed1b706..4149e87bfad4 100644 --- a/modules/packages/arch/metadata.go +++ b/modules/packages/arch/metadata.go @@ -6,7 +6,6 @@ package arch import ( "archive/tar" "bufio" - "bytes" "compress/gzip" "encoding/hex" "fmt" @@ -215,10 +214,13 @@ func rmEmptyStrings(s []string) []string { } // Create pacman database archive based on provided package metadata structs. -func CreatePacmanDb(entries map[string][]byte) ([]byte, error) { - var out bytes.Buffer +func CreatePacmanDb(entries map[string][]byte) (io.ReadSeeker, error) { + out, err := pkg_module.NewHashedBuffer() + if err != nil { + return nil, err + } - gw := gzip.NewWriter(&out) + gw := gzip.NewWriter(out) tw := tar.NewWriter(gw) for name, content := range entries { @@ -244,5 +246,5 @@ func CreatePacmanDb(entries map[string][]byte) ([]byte, error) { tw.Close() gw.Close() - return out.Bytes(), nil + return out, nil } diff --git a/routers/api/packages/arch/arch.go b/routers/api/packages/arch/arch.go index d0a4357bbd79..e7499b4a4732 100644 --- a/routers/api/packages/arch/arch.go +++ b/routers/api/packages/arch/arch.go @@ -4,7 +4,6 @@ package arch import ( - "bytes" "encoding/hex" "io" "net/http" @@ -160,7 +159,7 @@ func Get(ctx *context.Context) { return } - ctx.ServeContent(bytes.NewReader(db), &context.ServeHeaderOptions{ + ctx.ServeContent(db, &context.ServeHeaderOptions{ Filename: file, }) return diff --git a/services/packages/arch/service.go b/services/packages/arch/service.go index 5e01e49b3773..bddc9a78e19a 100644 --- a/services/packages/arch/service.go +++ b/services/packages/arch/service.go @@ -77,7 +77,7 @@ func RepoConnect(ctx *context.Context, owner, repo string, pkgid int64) error { // requested combination of architecture and distribution. When/If the first // compatible version is found, related desc file will be loaded from database // and added to resulting .db.tar.gz archive. -func CreatePacmanDb(ctx *context.Context, owner, arch, distro string) ([]byte, error) { +func CreatePacmanDb(ctx *context.Context, owner, arch, distro string) (io.ReadSeeker, error) { pkgs, err := pkg_model.GetPackagesByType(ctx, ctx.Package.Owner.ID, pkg_model.TypeArch) if err != nil { return nil, err From 0dd3db22806f1045942ae68616bf26008ae828cf Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Mon, 25 Sep 2023 18:54:27 -0300 Subject: [PATCH 084/124] fixed user in package upload operation --- routers/api/packages/arch/arch.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routers/api/packages/arch/arch.go b/routers/api/packages/arch/arch.go index e7499b4a4732..6b3705731119 100644 --- a/routers/api/packages/arch/arch.go +++ b/routers/api/packages/arch/arch.go @@ -78,7 +78,7 @@ func Push(ctx *context.Context) { Name: desc.Name, Version: desc.Version, }, - Creator: ctx.ContextUser, + Creator: ctx.Doer, Metadata: &arch_module.Metadata{ URL: desc.URL, Description: desc.Description, From c09da2a9e16969aeec35cf2cb4495358a59cc157 Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Thu, 28 Sep 2023 17:13:40 -0300 Subject: [PATCH 085/124] removed functionality related to automatic source code repository connection --- routers/api/packages/arch/arch.go | 9 +-------- services/packages/arch/service.go | 14 -------------- 2 files changed, 1 insertion(+), 22 deletions(-) diff --git a/routers/api/packages/arch/arch.go b/routers/api/packages/arch/arch.go index 6b3705731119..4023c6fa1e58 100644 --- a/routers/api/packages/arch/arch.go +++ b/routers/api/packages/arch/arch.go @@ -21,7 +21,6 @@ import ( // Push new package to arch package registry. func Push(ctx *context.Context) { var ( - owner = ctx.Params("username") filename = ctx.Params("filename") distro = ctx.Params("distro") sign = ctx.Params("sign") @@ -70,7 +69,7 @@ func Push(ctx *context.Context) { props[distro+"-"+filename+".sig"] = sign } - ver, _, err := pkg_service.CreatePackageOrAddFileToExisting( + _, _, err = pkg_service.CreatePackageOrAddFileToExisting( &pkg_service.PackageCreationInfo{ PackageInfo: pkg_service.PackageInfo{ Owner: ctx.Package.Owner, @@ -108,12 +107,6 @@ func Push(ctx *context.Context) { return } - err = arch_service.RepoConnect(ctx, owner, desc.Name, ver.ID) - if err != nil { - apiError(ctx, http.StatusInternalServerError, err) - return - } - ctx.Status(http.StatusOK) } diff --git a/services/packages/arch/service.go b/services/packages/arch/service.go index bddc9a78e19a..d493d2212809 100644 --- a/services/packages/arch/service.go +++ b/services/packages/arch/service.go @@ -13,7 +13,6 @@ import ( "strings" pkg_model "code.gitea.io/gitea/models/packages" - repository "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/modules/context" arch_module "code.gitea.io/gitea/modules/packages/arch" pkg_service "code.gitea.io/gitea/services/packages" @@ -59,19 +58,6 @@ func GetPackageSignature(ctx *context.Context, distro, file string) (*bytes.Read return nil, errors.New("signature for requested package not found") } -// Automatically connect repository with source code to published package, if -// repository with the same name exists in user/organization scope. -func RepoConnect(ctx *context.Context, owner, repo string, pkgid int64) error { - r, err := repository.GetRepositoryByOwnerAndName(ctx, owner, repo) - if err == nil { - err = pkg_model.SetRepositoryLink(ctx, pkgid, r.ID) - if err != nil { - return err - } - } - return nil -} - // Finds all arch packages in user/organization scope, each package version // starting from latest in descending order is checked to be compatible with // requested combination of architecture and distribution. When/If the first From ba92fc54b75f5c87ee724e29ea86085aa2ae0e0f Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Thu, 28 Sep 2023 17:25:22 -0300 Subject: [PATCH 086/124] pass context to package service methods --- routers/api/packages/arch/arch.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/routers/api/packages/arch/arch.go b/routers/api/packages/arch/arch.go index 4023c6fa1e58..789eb9966c8c 100644 --- a/routers/api/packages/arch/arch.go +++ b/routers/api/packages/arch/arch.go @@ -70,7 +70,7 @@ func Push(ctx *context.Context) { } _, _, err = pkg_service.CreatePackageOrAddFileToExisting( - &pkg_service.PackageCreationInfo{ + ctx, &pkg_service.PackageCreationInfo{ PackageInfo: pkg_service.PackageInfo{ Owner: ctx.Package.Owner, PackageType: pkg_model.TypeArch, @@ -176,7 +176,7 @@ func Remove(ctx *context.Context) { return } - err = pkg_service.RemovePackageVersion(ctx.Package.Owner, version) + err = pkg_service.RemovePackageVersion(ctx, ctx.Package.Owner, version) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return From fcb3fd11debcb01f6b03eff8447fdd081b3b2ee8 Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Mon, 2 Oct 2023 13:19:57 -0300 Subject: [PATCH 087/124] corrected procedure of retrieving package file property to package owner scope --- models/packages/package_file.go | 14 -------------- services/packages/arch/service.go | 31 +++++++++++++++++++++++++++---- 2 files changed, 27 insertions(+), 18 deletions(-) diff --git a/models/packages/package_file.go b/models/packages/package_file.go index b9d753829067..1c2c9ac072f4 100644 --- a/models/packages/package_file.go +++ b/models/packages/package_file.go @@ -65,20 +65,6 @@ func TryInsertFile(ctx context.Context, pf *PackageFile) (*PackageFile, error) { return pf, nil } -// GetFileByCompositeKey gets unique file by composite key if exists -func GetFileByCompositeKey(ctx context.Context, key string) (*PackageFile, error) { - pf := &PackageFile{CompositeKey: key} - - has, err := db.GetEngine(ctx).Get(pf) - if err != nil { - return nil, err - } - if !has { - return nil, ErrPackageFileNotExist - } - return pf, nil -} - // GetFilesByVersionID gets all files of a version func GetFilesByVersionID(ctx context.Context, versionID int64) ([]*PackageFile, error) { pfs := make([]*PackageFile, 0, 10) diff --git a/services/packages/arch/service.go b/services/packages/arch/service.go index d493d2212809..c82e7cc720ae 100644 --- a/services/packages/arch/service.go +++ b/services/packages/arch/service.go @@ -21,7 +21,7 @@ import ( // Get data related to provided filename and distribution, for package files // update download counter. func GetPackageFile(ctx *context.Context, distro, file string) (io.ReadSeekCloser, error) { - pf, err := pkg_model.GetFileByCompositeKey(ctx, distro+"-"+file) + pf, err := getPackageFile(ctx, distro, file) if err != nil { return nil, err } @@ -33,9 +33,7 @@ func GetPackageFile(ctx *context.Context, distro, file string) (io.ReadSeekClose // This function will search for package signature and if present, will load it // from package file properties, and return its byte reader. func GetPackageSignature(ctx *context.Context, distro, file string) (*bytes.Reader, error) { - pkgfile := strings.TrimSuffix(distro+"-"+file, ".sig") - - pf, err := pkg_model.GetFileByCompositeKey(ctx, pkgfile) + pf, err := getPackageFile(ctx, distro, strings.TrimSuffix(file, ".sig")) if err != nil { return nil, err } @@ -58,6 +56,31 @@ func GetPackageSignature(ctx *context.Context, distro, file string) (*bytes.Read return nil, errors.New("signature for requested package not found") } +// Ejects parameters required to get package file property from file name. +func getPackageFile(ctx *context.Context, distro, file string) (*pkg_model.PackageFile, error) { + var ( + splt = strings.Split(file, "-") + pkgname = strings.Join(splt[0:len(splt)-3], "-") + vername = splt[len(splt)-3] + "-" + splt[len(splt)-2] + compkey = distro + "-" + file + ) + + version, err := pkg_model.GetVersionByNameAndVersion( + ctx, ctx.Package.Owner.ID, pkg_model.TypeArch, pkgname, vername, + ) + if err != nil { + return nil, err + } + + pkgfile, err := pkg_model.GetFileForVersionByName( + ctx, version.ID, file, compkey, + ) + if err != nil { + return nil, err + } + return pkgfile, nil +} + // Finds all arch packages in user/organization scope, each package version // starting from latest in descending order is checked to be compatible with // requested combination of architecture and distribution. When/If the first From f515f6ee7cfded9e2166da5c9ce0562948764643 Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Mon, 2 Oct 2023 14:33:37 -0300 Subject: [PATCH 088/124] removed unnecessary filename from composite key --- routers/api/packages/arch/arch.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routers/api/packages/arch/arch.go b/routers/api/packages/arch/arch.go index 789eb9966c8c..1622c0887673 100644 --- a/routers/api/packages/arch/arch.go +++ b/routers/api/packages/arch/arch.go @@ -93,7 +93,7 @@ func Push(ctx *context.Context) { &pkg_service.PackageFileCreationInfo{ PackageFileInfo: pkg_service.PackageFileInfo{ Filename: filename, - CompositeKey: distro + "-" + filename, + CompositeKey: distro, }, OverwriteExisting: true, IsLead: true, From 4bf84c1fd65df1e0bd9094dc201f75e662755f83 Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Mon, 2 Oct 2023 18:04:58 -0300 Subject: [PATCH 089/124] corrected arch template and related locale options --- options/locale/locale_en-US.ini | 2 +- templates/package/content/arch.tmpl | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 8d2331ffab29..3ab1920c511d 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -3331,7 +3331,7 @@ alpine.repository = Repository Info alpine.repository.branches = Branches alpine.repository.repositories = Repositories alpine.repository.architectures = Architectures -arch.pacmanconf = Add server to pacman.conf with related distribution and architecture: +arch.pacmanconf = Add server with related distribution and architecture to /etc/pacman.conf: arch.pacmansync = Sync package with pacman: arch.documentation = For more information on the arch mirrors, see the documentation. arch.properties = Package properties diff --git a/templates/package/content/arch.tmpl b/templates/package/content/arch.tmpl index 17df1fb7368d..bfb4eb906f36 100644 --- a/templates/package/content/arch.tmpl +++ b/templates/package/content/arch.tmpl @@ -4,14 +4,14 @@
- + 
[{{.PackageDescriptor.Owner.LowerName}}.{{.RegistryHost}}]
 SigLevel = Optional TrustAll
 Server =
- + 
pacman -Sy {{.PackageDescriptor.Package.LowerName}}
From 8974d27ba3f976e54923d84a7266e76d21606ac0 Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Mon, 2 Oct 2023 19:12:06 -0300 Subject: [PATCH 090/124] refactoring in metadata module, better readability shorter function calls in arch service fix - request now ends if unable to decode package signature from hex fix - now package search continues to previous versions in package file for required architecture not found for current version --- modules/packages/arch/metadata.go | 93 +++++++++++++------------------ routers/api/packages/arch/arch.go | 5 +- services/packages/arch/service.go | 35 ++++-------- 3 files changed, 52 insertions(+), 81 deletions(-) diff --git a/modules/packages/arch/metadata.go b/modules/packages/arch/metadata.go index 4149e87bfad4..970008686bf3 100644 --- a/modules/packages/arch/metadata.go +++ b/modules/packages/arch/metadata.go @@ -104,18 +104,6 @@ func EjectMetadata(p *EjectParams) (*DbDesc, error) { md.URL = value case "packager": md.Packager = value - case "builddate": - num, err := strconv.ParseInt(value, 10, 64) - if err != nil { - return nil, err - } - md.BuildDate = num - case "size": - num, err := strconv.ParseInt(value, 10, 64) - if err != nil { - return nil, err - } - md.InstalledSize = num case "provides": md.Provides = append(md.Provides, value) case "license": @@ -132,6 +120,16 @@ func EjectMetadata(p *EjectParams) (*DbDesc, error) { md.CheckDepends = append(md.CheckDepends, value) case "backup": md.Backup = append(md.Backup, value) + case "builddate": + md.BuildDate, err = strconv.ParseInt(value, 10, 64) + if err != nil { + return nil, err + } + case "size": + md.InstalledSize, err = strconv.ParseInt(value, 10, 64) + if err != nil { + return nil, err + } } } @@ -164,53 +162,38 @@ func getPkginfo(data io.Reader) (string, error) { // Create pacman package description file. func (m *DbDesc) String() string { - return strings.Join(rmEmptyStrings([]string{ - formatField("FILENAME", m.Filename), - formatField("NAME", m.Name), - formatField("BASE", m.Base), - formatField("VERSION", m.Version), - formatField("DESC", m.Description), - formatField("CSIZE", m.CompressedSize), - formatField("ISIZE", m.InstalledSize), - formatField("MD5SUM", m.MD5), - formatField("SHA256SUM", m.SHA256), - formatField("URL", m.URL), - formatField("LICENSE", m.License), - formatField("ARCH", m.Arch), - formatField("BUILDDATE", m.BuildDate), - formatField("PACKAGER", m.Packager), - formatField("PROVIDES", m.Provides), - formatField("DEPENDS", m.Depends), - formatField("OPTDEPENDS", m.OptDepends), - formatField("MAKEDEPENDS", m.MakeDepends), - formatField("CHECKDEPENDS", m.CheckDepends), - }), "\n\n") + "\n\n" -} - -func formatField(field string, value any) string { - switch value := value.(type) { - case []string: - if value == nil { - return `` - } - val := strings.Join(value, "\n") - return fmt.Sprintf("%%%s%%\n%s", field, val) - case string: - return fmt.Sprintf("%%%s%%\n%s", field, value) - case int64: - return fmt.Sprintf("%%%s%%\n%d", field, value) + var entries = []struct { + Key string + Value string + }{ + {Key: "FILENAME", Value: m.Filename}, + {Key: "NAME", Value: m.Name}, + {Key: "BASE", Value: m.Base}, + {Key: "VERSION", Value: m.Version}, + {Key: "DESC", Value: m.Description}, + {Key: "CSIZE", Value: fmt.Sprintf("%d", m.CompressedSize)}, + {Key: "ISIZE", Value: fmt.Sprintf("%d", m.InstalledSize)}, + {Key: "MD5SUM", Value: m.MD5}, + {Key: "SHA256SUM", Value: m.SHA256}, + {Key: "URL", Value: m.URL}, + {Key: "LICENSE", Value: strings.Join(m.License, "\n")}, + {Key: "ARCH", Value: strings.Join(m.Arch, "\n")}, + {Key: "BUILDDATE", Value: fmt.Sprintf("%d", m.BuildDate)}, + {Key: "PACKAGER", Value: m.Packager}, + {Key: "PROVIDES", Value: strings.Join(m.Provides, "\n")}, + {Key: "DEPENDS", Value: strings.Join(m.Depends, "\n")}, + {Key: "OPTDEPENDS", Value: strings.Join(m.OptDepends, "\n")}, + {Key: "MAKEDEPENDS", Value: strings.Join(m.MakeDepends, "\n")}, + {Key: "CHECKDEPENDS", Value: strings.Join(m.CheckDepends, "\n")}, } - return `` -} -func rmEmptyStrings(s []string) []string { - var r []string - for _, str := range s { - if str != "" { - r = append(r, str) + var result string + for _, v := range entries { + if v.Value != "" { + result += fmt.Sprintf("%%%s%%\n%s\n\n", v.Key, v.Value) } } - return r + return result } // Create pacman database archive based on provided package metadata structs. diff --git a/routers/api/packages/arch/arch.go b/routers/api/packages/arch/arch.go index 1622c0887673..2a0f23557f94 100644 --- a/routers/api/packages/arch/arch.go +++ b/routers/api/packages/arch/arch.go @@ -59,14 +59,15 @@ func Push(ctx *context.Context) { } props := map[string]string{ - distro + "-" + filename + ".desc": desc.String(), + "desc": desc.String(), } if sign != "" { _, err := hex.DecodeString(sign) if err != nil { apiError(ctx, http.StatusBadRequest, err) + return } - props[distro+"-"+filename+".sig"] = sign + props["sign"] = sign } _, _, err = pkg_service.CreatePackageOrAddFileToExisting( diff --git a/services/packages/arch/service.go b/services/packages/arch/service.go index c82e7cc720ae..327b12f9c9cd 100644 --- a/services/packages/arch/service.go +++ b/services/packages/arch/service.go @@ -44,7 +44,7 @@ func GetPackageSignature(ctx *context.Context, distro, file string) (*bytes.Read } for _, pp := range proprs { - if pp.Name == distro+"-"+file { + if pp.Name == "sign" { b, err := hex.DecodeString(pp.Value) if err != nil { return nil, err @@ -62,7 +62,6 @@ func getPackageFile(ctx *context.Context, distro, file string) (*pkg_model.Packa splt = strings.Split(file, "-") pkgname = strings.Join(splt[0:len(splt)-3], "-") vername = splt[len(splt)-3] + "-" + splt[len(splt)-2] - compkey = distro + "-" + file ) version, err := pkg_model.GetVersionByNameAndVersion( @@ -72,9 +71,7 @@ func getPackageFile(ctx *context.Context, distro, file string) (*pkg_model.Packa return nil, err } - pkgfile, err := pkg_model.GetFileForVersionByName( - ctx, version.ID, file, compkey, - ) + pkgfile, err := pkg_model.GetFileForVersionByName(ctx, version.ID, file, distro) if err != nil { return nil, err } @@ -106,36 +103,26 @@ func CreatePacmanDb(ctx *context.Context, owner, arch, distro string) (io.ReadSe return versions[i].CreatedUnix > versions[j].CreatedUnix }) - for _, version := range versions { - filename := fmt.Sprintf( - "%s-%s-%s.pkg.tar.zst", - pkg.Name, version.Version, arch, - ) + for _, ver := range versions { + file := fmt.Sprintf("%s-%s-%s.pkg.tar.zst", pkg.Name, ver.Version, arch) - file, err := pkg_model.GetFileForVersionByName( - ctx, version.ID, filename, distro+"-"+filename, - ) + pf, err := pkg_model.GetFileForVersionByName(ctx, ver.ID, file, distro) if err != nil { - filename := fmt.Sprintf( - "%s-%s-any.pkg.tar.zst", - pkg.Name, version.Version, - ) - file, err = pkg_model.GetFileForVersionByName( - ctx, version.ID, filename, distro+"-"+filename, - ) + file = fmt.Sprintf("%s-%s-any.pkg.tar.zst", pkg.Name, ver.Version) + pf, err = pkg_model.GetFileForVersionByName(ctx, ver.ID, file, distro) if err != nil { - return nil, err + continue } } - pps, err := pkg_model.GetProperties(ctx, pkg_model.PropertyTypeFile, file.ID) + pps, err := pkg_model.GetProperties(ctx, pkg_model.PropertyTypeFile, pf.ID) if err != nil { return nil, err } var descvalue string for _, pp := range pps { - if strings.HasSuffix(pp.Name, ".desc") { + if pp.Name == "desc" { descvalue = pp.Value } } @@ -144,7 +131,7 @@ func CreatePacmanDb(ctx *context.Context, owner, arch, distro string) (io.ReadSe continue } - entries[pkg.Name+"-"+version.Version+"/desc"] = []byte(descvalue) + entries[pkg.Name+"-"+ver.Version+"/desc"] = []byte(descvalue) break } } From 784b4f9304692c0c02469b1e6b11c690f68cb62e Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Mon, 2 Oct 2023 22:00:44 -0300 Subject: [PATCH 091/124] refactoring in metadata module, pacman database creation function --- modules/packages/arch/metadata.go | 38 +++++++++++++++---------------- services/packages/arch/service.go | 15 ++++-------- 2 files changed, 23 insertions(+), 30 deletions(-) diff --git a/modules/packages/arch/metadata.go b/modules/packages/arch/metadata.go index 970008686bf3..ce873f867b1d 100644 --- a/modules/packages/arch/metadata.go +++ b/modules/packages/arch/metadata.go @@ -166,25 +166,25 @@ func (m *DbDesc) String() string { Key string Value string }{ - {Key: "FILENAME", Value: m.Filename}, - {Key: "NAME", Value: m.Name}, - {Key: "BASE", Value: m.Base}, - {Key: "VERSION", Value: m.Version}, - {Key: "DESC", Value: m.Description}, - {Key: "CSIZE", Value: fmt.Sprintf("%d", m.CompressedSize)}, - {Key: "ISIZE", Value: fmt.Sprintf("%d", m.InstalledSize)}, - {Key: "MD5SUM", Value: m.MD5}, - {Key: "SHA256SUM", Value: m.SHA256}, - {Key: "URL", Value: m.URL}, - {Key: "LICENSE", Value: strings.Join(m.License, "\n")}, - {Key: "ARCH", Value: strings.Join(m.Arch, "\n")}, - {Key: "BUILDDATE", Value: fmt.Sprintf("%d", m.BuildDate)}, - {Key: "PACKAGER", Value: m.Packager}, - {Key: "PROVIDES", Value: strings.Join(m.Provides, "\n")}, - {Key: "DEPENDS", Value: strings.Join(m.Depends, "\n")}, - {Key: "OPTDEPENDS", Value: strings.Join(m.OptDepends, "\n")}, - {Key: "MAKEDEPENDS", Value: strings.Join(m.MakeDepends, "\n")}, - {Key: "CHECKDEPENDS", Value: strings.Join(m.CheckDepends, "\n")}, + {"FILENAME", m.Filename}, + {"NAME", m.Name}, + {"BASE", m.Base}, + {"VERSION", m.Version}, + {"DESC", m.Description}, + {"CSIZE", fmt.Sprintf("%d", m.CompressedSize)}, + {"ISIZE", fmt.Sprintf("%d", m.InstalledSize)}, + {"MD5SUM", m.MD5}, + {"SHA256SUM", m.SHA256}, + {"URL", m.URL}, + {"LICENSE", strings.Join(m.License, "\n")}, + {"ARCH", strings.Join(m.Arch, "\n")}, + {"BUILDDATE", fmt.Sprintf("%d", m.BuildDate)}, + {"PACKAGER", m.Packager}, + {"PROVIDES", strings.Join(m.Provides, "\n")}, + {"DEPENDS", strings.Join(m.Depends, "\n")}, + {"OPTDEPENDS", strings.Join(m.OptDepends, "\n")}, + {"MAKEDEPENDS", strings.Join(m.MakeDepends, "\n")}, + {"CHECKDEPENDS", strings.Join(m.CheckDepends, "\n")}, } var result string diff --git a/services/packages/arch/service.go b/services/packages/arch/service.go index 327b12f9c9cd..7f8f9c07cc23 100644 --- a/services/packages/arch/service.go +++ b/services/packages/arch/service.go @@ -81,8 +81,8 @@ func getPackageFile(ctx *context.Context, distro, file string) (*pkg_model.Packa // Finds all arch packages in user/organization scope, each package version // starting from latest in descending order is checked to be compatible with // requested combination of architecture and distribution. When/If the first -// compatible version is found, related desc file will be loaded from database -// and added to resulting .db.tar.gz archive. +// compatible version is found, related desc file will be loaded from package +// properties and added to resulting .db.tar.gz archive. func CreatePacmanDb(ctx *context.Context, owner, arch, distro string) (io.ReadSeeker, error) { pkgs, err := pkg_model.GetPackagesByType(ctx, ctx.Package.Owner.ID, pkg_model.TypeArch) if err != nil { @@ -120,19 +120,12 @@ func CreatePacmanDb(ctx *context.Context, owner, arch, distro string) (io.ReadSe return nil, err } - var descvalue string for _, pp := range pps { if pp.Name == "desc" { - descvalue = pp.Value + entries[pkg.Name+"-"+ver.Version+"/desc"] = []byte(pp.Value) + break } } - - if descvalue == "" { - continue - } - - entries[pkg.Name+"-"+ver.Version+"/desc"] = []byte(descvalue) - break } } From 94f45f402fc4901d81e899c2ff06ab51b70c7a23 Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Mon, 2 Oct 2023 23:47:32 -0300 Subject: [PATCH 092/124] change interface{} to any, move apiError function to bottom --- routers/api/packages/arch/arch.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/routers/api/packages/arch/arch.go b/routers/api/packages/arch/arch.go index 2a0f23557f94..535bb394711d 100644 --- a/routers/api/packages/arch/arch.go +++ b/routers/api/packages/arch/arch.go @@ -18,6 +18,12 @@ import ( arch_service "code.gitea.io/gitea/services/packages/arch" ) +func apiError(ctx *context.Context, status int, obj any) { + helper.LogAndProcessError(ctx, status, obj, func(message string) { + ctx.PlainText(status, message) + }) +} + // Push new package to arch package registry. func Push(ctx *context.Context) { var ( @@ -185,9 +191,3 @@ func Remove(ctx *context.Context) { ctx.Status(http.StatusOK) } - -func apiError(ctx *context.Context, status int, obj interface{}) { - helper.LogAndProcessError(ctx, status, obj, func(message string) { - ctx.PlainText(status, message) - }) -} From d473e01d7e7b5512cbca0ab7fbe7115d19ce493a Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Tue, 3 Oct 2023 14:31:24 -0300 Subject: [PATCH 093/124] fixed pacman database creation process, corrected switch to next entry when current package desc is found --- services/packages/arch/service.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/services/packages/arch/service.go b/services/packages/arch/service.go index 7f8f9c07cc23..a79246d91a2d 100644 --- a/services/packages/arch/service.go +++ b/services/packages/arch/service.go @@ -115,16 +115,16 @@ func CreatePacmanDb(ctx *context.Context, owner, arch, distro string) (io.ReadSe } } - pps, err := pkg_model.GetProperties(ctx, pkg_model.PropertyTypeFile, pf.ID) + pps, err := pkg_model.GetPropertiesByName( + ctx, pkg_model.PropertyTypeFile, pf.ID, "desc", + ) if err != nil { return nil, err } - for _, pp := range pps { - if pp.Name == "desc" { - entries[pkg.Name+"-"+ver.Version+"/desc"] = []byte(pp.Value) - break - } + if len(pps) >= 1 { + entries[pkg.Name+"-"+ver.Version+"/desc"] = []byte(pps[0].Value) + break } } } From b7725c6b53023afda3a5a9012f41ea43e52aff5c Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Tue, 3 Oct 2023 14:58:58 -0300 Subject: [PATCH 094/124] formatting corrections --- modules/packages/arch/metadata.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/packages/arch/metadata.go b/modules/packages/arch/metadata.go index ce873f867b1d..5cbf07b223ab 100644 --- a/modules/packages/arch/metadata.go +++ b/modules/packages/arch/metadata.go @@ -162,7 +162,7 @@ func getPkginfo(data io.Reader) (string, error) { // Create pacman package description file. func (m *DbDesc) String() string { - var entries = []struct { + entries := []struct { Key string Value string }{ From 54c0816377a7a7cc020593c77f34abb58cbe7c8c Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Thu, 5 Oct 2023 16:16:02 -0300 Subject: [PATCH 095/124] refactoring, better readability in arch package metadata module --- modules/packages/arch/metadata.go | 29 ++++++++++------------------- routers/api/packages/arch/arch.go | 12 ++++-------- 2 files changed, 14 insertions(+), 27 deletions(-) diff --git a/modules/packages/arch/metadata.go b/modules/packages/arch/metadata.go index 5cbf07b223ab..c9e224142f8f 100644 --- a/modules/packages/arch/metadata.go +++ b/modules/packages/arch/metadata.go @@ -58,25 +58,19 @@ type DbDesc struct { Backup []string `json:"backup,omitempty"` } -type EjectParams struct { - Filename string - Distribution string - Buffer *pkg_module.HashedBuffer -} - // Function that receives arch package archive data and returns it's metadata. -func EjectMetadata(p *EjectParams) (*DbDesc, error) { - pkginfo, err := getPkginfo(p.Buffer) +func EjectMetadata(file, distro string, b *pkg_module.HashedBuffer) (*DbDesc, error) { + pkginfo, err := getPkginfo(b) if err != nil { return nil, err } // Add package blob parameters to arch related desc. - hashMD5, _, hashSHA256, _ := p.Buffer.Sums() + hashMD5, _, hashSHA256, _ := b.Sums() md := DbDesc{ - Filename: p.Filename, - Name: p.Filename, - CompressedSize: p.Buffer.Size(), + Filename: file, + Name: file, + CompressedSize: b.Size(), MD5: hex.EncodeToString(hashMD5), SHA256: hex.EncodeToString(hashSHA256), } @@ -162,10 +156,7 @@ func getPkginfo(data io.Reader) (string, error) { // Create pacman package description file. func (m *DbDesc) String() string { - entries := []struct { - Key string - Value string - }{ + entries := []struct{ key, value string }{ {"FILENAME", m.Filename}, {"NAME", m.Name}, {"BASE", m.Base}, @@ -188,9 +179,9 @@ func (m *DbDesc) String() string { } var result string - for _, v := range entries { - if v.Value != "" { - result += fmt.Sprintf("%%%s%%\n%s\n\n", v.Key, v.Value) + for _, e := range entries { + if e.value != "" { + result += fmt.Sprintf("%%%s%%\n%s\n\n", e.key, e.value) } } return result diff --git a/routers/api/packages/arch/arch.go b/routers/api/packages/arch/arch.go index 535bb394711d..9823d749343d 100644 --- a/routers/api/packages/arch/arch.go +++ b/routers/api/packages/arch/arch.go @@ -48,11 +48,7 @@ func Push(ctx *context.Context) { } defer buf.Close() - desc, err := arch_module.EjectMetadata(&arch_module.EjectParams{ - Filename: filename, - Distribution: distro, - Buffer: buf, - }) + desc, err := arch_module.EjectMetadata(filename, distro, buf) if err != nil { apiError(ctx, http.StatusBadRequest, err) return @@ -64,7 +60,7 @@ func Push(ctx *context.Context) { return } - props := map[string]string{ + properties := map[string]string{ "desc": desc.String(), } if sign != "" { @@ -73,7 +69,7 @@ func Push(ctx *context.Context) { apiError(ctx, http.StatusBadRequest, err) return } - props["sign"] = sign + properties["sign"] = sign } _, _, err = pkg_service.CreatePackageOrAddFileToExisting( @@ -106,7 +102,7 @@ func Push(ctx *context.Context) { IsLead: true, Creator: ctx.ContextUser, Data: buf, - Properties: props, + Properties: properties, }, ) if err != nil { From b63af7e277220026d19e2c506253c2a54dc00c11 Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Mon, 9 Oct 2023 03:13:53 -0300 Subject: [PATCH 096/124] metadata tests, json struct tag unification with other linux packages, removed package backup files from metadata and arch package UI --- modules/packages/arch/metadata.go | 25 ++-- modules/packages/arch/metadata_test.go | 176 +++++++++++++++++++++++++ routers/api/packages/arch/arch.go | 3 +- templates/package/content/arch.tmpl | 7 - 4 files changed, 189 insertions(+), 22 deletions(-) create mode 100644 modules/packages/arch/metadata_test.go diff --git a/modules/packages/arch/metadata.go b/modules/packages/arch/metadata.go index c9e224142f8f..930cedc6c4c9 100644 --- a/modules/packages/arch/metadata.go +++ b/modules/packages/arch/metadata.go @@ -27,10 +27,9 @@ type Metadata struct { Provides []string `json:"provides,omitempty"` License []string `json:"license,omitempty"` Depends []string `json:"depends,omitempty"` - OptDepends []string `json:"opt-depends,omitempty"` - MakeDepends []string `json:"make-depends,omitempty"` - CheckDepends []string `json:"check-depends,omitempty"` - Backup []string `json:"backup,omitempty"` + OptDepends []string `json:"opt_depends,omitempty"` + MakeDepends []string `json:"make_depends,omitempty"` + CheckDepends []string `json:"check_depends,omitempty"` } // Package description file that will be saved as .desc file in object storage. @@ -41,20 +40,20 @@ type DbDesc struct { Base string `json:"base"` Version string `json:"version"` Description string `json:"description"` - CompressedSize int64 `json:"compressed-size"` - InstalledSize int64 `json:"installed-size"` + CompressedSize int64 `json:"compressed_size"` + InstalledSize int64 `json:"installed_size"` MD5 string `json:"md5"` SHA256 string `json:"sha256"` - URL string `json:"url"` - BuildDate int64 `json:"build-date"` + ProjectURL string `json:"project_url"` + BuildDate int64 `json:"build_date"` Packager string `json:"packager"` Provides []string `json:"provides,omitempty"` License []string `json:"license,omitempty"` Arch []string `json:"arch,omitempty"` Depends []string `json:"depends,omitempty"` - OptDepends []string `json:"opt-depends,omitempty"` - MakeDepends []string `json:"make-depends,omitempty"` - CheckDepends []string `json:"check-depends,omitempty"` + OptDepends []string `json:"opt_depends,omitempty"` + MakeDepends []string `json:"make_depends,omitempty"` + CheckDepends []string `json:"check_depends,omitempty"` Backup []string `json:"backup,omitempty"` } @@ -95,7 +94,7 @@ func EjectMetadata(file, distro string, b *pkg_module.HashedBuffer) (*DbDesc, er case "pkgdesc": md.Description = value case "url": - md.URL = value + md.ProjectURL = value case "packager": md.Packager = value case "provides": @@ -166,7 +165,7 @@ func (m *DbDesc) String() string { {"ISIZE", fmt.Sprintf("%d", m.InstalledSize)}, {"MD5SUM", m.MD5}, {"SHA256SUM", m.SHA256}, - {"URL", m.URL}, + {"URL", m.ProjectURL}, {"LICENSE", strings.Join(m.License, "\n")}, {"ARCH", strings.Join(m.Arch, "\n")}, {"BUILDDATE", fmt.Sprintf("%d", m.BuildDate)}, diff --git a/modules/packages/arch/metadata_test.go b/modules/packages/arch/metadata_test.go new file mode 100644 index 000000000000..66239d927d5b --- /dev/null +++ b/modules/packages/arch/metadata_test.go @@ -0,0 +1,176 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package arch + +import ( + "encoding/base64" + "io" + "os" + "testing" + "testing/fstest" + "time" + + pkg_module "code.gitea.io/gitea/modules/packages" + + "github.com/mholt/archiver/v3" + "github.com/stretchr/testify/assert" +) + +const pkginfo = `# Generated by makepkg 6.0.2 +# using fakeroot version 1.31 +pkgname = zstd +pkgbase = zstd +pkgver = 1.5.5-1 +pkgdesc = Zstandard - Fast real-time compression algorithm +url = https://facebook.github.io/zstd/ +builddate = 1681646714 +packager = Jelle van der Waa +size = 1500453 +arch = x86_64 +license = BSD +license = GPL2 +provides = libzstd.so=1-64 +depend = glibc +depend = gcc-libs +depend = zlib +depend = xz +depend = lz4 +makedepend = cmake +makedepend = gtest +makedepend = ninja +` + +const pkgdesc = `%%FILENAME% +zstd-1.5.5-1-x86_64.pkg.tar.zst + +%NAME% +zstd + +%BASE% +zstd + +%VERSION% +1.5.5-1 + +%DESC% +Zstandard - Fast real-time compression algorithm + +%CSIZE% +401 + +%ISIZE% +1500453 + +%MD5SUM% +5016660ef3d9aa148a7b72a08d3df1b2 + +%SHA256SUM% +9fa4ede47e35f5971e4f26ecadcbfb66ab79f1d638317ac80334a3362dedbabd + +%%URL% +https://facebook.github.io/zstd/ + +%LICENSE% +BSD +GPL2 + +%ARCH% +x86_64 + +%BUILDDATE% +1681646714 + +%PACKAGER% +Jelle van der Waa + +%PROVIDES% +libzstd.so=1-64 + +%DEPENDS% +glibc +gcc-libs +zlib +xz +lz4 + +%MAKEDEPENDS% +cmake +gtest +ninja + +` + +const dbarchive = "H4sIAAAAAAAA/0rLzEnVS60oYaAhMDAwMDA3NwfTBgYG6LSBgYEpEtuAwcDQwMzUgEHBgJaOgoHS4pLEIgYDiu1C99wQASmlubmVA+2IUTAKRsEoGAV0B4AAAAD//2VF3KIACAAA" + +func TestMetadata(t *testing.T) { + fs := fstest.MapFS{ + "pkginfo": &fstest.MapFile{ + Data: []byte(pkginfo), + Mode: os.ModePerm, + ModTime: time.Now(), + }, + } + + info, err := fs.Stat("pkginfo") + assert.NoError(t, err) + + file, err := fs.Open("pkginfo") + assert.NoError(t, err) + + buf, err := pkg_module.NewHashedBuffer() + assert.NoError(t, err) + + archive := archiver.NewTarZstd() + archive.Create(buf) + + n, err := archiver.NameInArchive(info, ".PKGINFO", ".PKGINFO") + assert.NoError(t, err) + + err = archive.Write(archiver.File{ + FileInfo: archiver.FileInfo{ + FileInfo: info, + CustomName: n, + }, + ReadCloser: file, + }) + file.Close() + archive.Close() + assert.NoError(t, err) + + md, err := EjectMetadata("zstd-1.5.5-1-x86_64.pkg.tar.zst", "archlinux", buf) + assert.NoError(t, err) + + assert.Equal(t, md.Name, "zstd") + assert.Equal(t, md.Base, "zstd") + assert.Equal(t, md.Version, "1.5.5-1") + assert.Equal(t, md.Description, "Zstandard - Fast real-time compression algorithm") + assert.Equal(t, md.ProjectURL, "https://facebook.github.io/zstd/") + assert.Equal(t, md.BuildDate, int64(1681646714)) + assert.Equal(t, md.Packager, "Jelle van der Waa ") + assert.Equal(t, md.InstalledSize, int64(1500453)) + assert.Equal(t, md.Arch, []string{"x86_64"}) + assert.Equal(t, md.License, []string{"BSD", "GPL2"}) + assert.Equal(t, md.Provides, []string{"libzstd.so=1-64"}) + assert.Equal(t, md.Provides, []string{"libzstd.so=1-64"}) + assert.Equal(t, md.Depends, []string{"glibc", "gcc-libs", "zlib", "xz", "lz4"}) + assert.Equal(t, md.MakeDepends, []string{"cmake", "gtest", "ninja"}) + + desc := md.String() + assert.Equal(t, desc, pkgdesc) +} + +func TestDatabase(t *testing.T) { + db, err := CreatePacmanDb(map[string][]byte{ + "file.ext": []byte("dummy"), + }) + assert.NoError(t, err) + + actual, err := io.ReadAll(db) + assert.NoError(t, err) + + expected, err := base64.RawStdEncoding.DecodeString(dbarchive) + assert.NoError(t, err) + + assert.Equal(t, expected, actual) +} diff --git a/routers/api/packages/arch/arch.go b/routers/api/packages/arch/arch.go index 9823d749343d..4d84f43fbb74 100644 --- a/routers/api/packages/arch/arch.go +++ b/routers/api/packages/arch/arch.go @@ -82,7 +82,7 @@ func Push(ctx *context.Context) { }, Creator: ctx.Doer, Metadata: &arch_module.Metadata{ - URL: desc.URL, + URL: desc.ProjectURL, Description: desc.Description, Provides: desc.Provides, License: desc.License, @@ -90,7 +90,6 @@ func Push(ctx *context.Context) { OptDepends: desc.OptDepends, MakeDepends: desc.MakeDepends, CheckDepends: desc.CheckDepends, - Backup: desc.Backup, }, }, &pkg_service.PackageFileCreationInfo{ diff --git a/templates/package/content/arch.tmpl b/templates/package/content/arch.tmpl index bfb4eb906f36..83c5baba32ee 100644 --- a/templates/package/content/arch.tmpl +++ b/templates/package/content/arch.tmpl @@ -64,13 +64,6 @@ Server = {{end}} - - {{if .PackageDescriptor.Metadata.Backup}} - -
Backup file
- {{StringUtils.Join $.PackageDescriptor.Metadata.Backup ", "}} - - {{end}}
From 54a9c8bdffc00ff55ed44c46e0f9d3729de079d9 Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Thu, 12 Oct 2023 09:15:49 -0300 Subject: [PATCH 097/124] corrected response codes in push and remove operations --- routers/api/packages/arch/arch.go | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/routers/api/packages/arch/arch.go b/routers/api/packages/arch/arch.go index 4d84f43fbb74..cc92317eeeab 100644 --- a/routers/api/packages/arch/arch.go +++ b/routers/api/packages/arch/arch.go @@ -105,7 +105,14 @@ func Push(ctx *context.Context) { }, ) if err != nil { - apiError(ctx, http.StatusInternalServerError, err) + switch err { + case pkg_model.ErrDuplicatePackageVersion, pkg_model.ErrDuplicatePackageFile: + apiError(ctx, http.StatusConflict, err) + case pkg_service.ErrQuotaTotalCount, pkg_service.ErrQuotaTypeSize, pkg_service.ErrQuotaTotalSize: + apiError(ctx, http.StatusForbidden, err) + default: + apiError(ctx, http.StatusInternalServerError, err) + } return } @@ -174,7 +181,12 @@ func Remove(ctx *context.Context) { ctx, ctx.Package.Owner.ID, pkg_model.TypeArch, pkg, ver, ) if err != nil { - apiError(ctx, http.StatusInternalServerError, err) + switch err { + case pkg_model.ErrPackageNotExist: + apiError(ctx, http.StatusNotFound, err) + default: + apiError(ctx, http.StatusInternalServerError, err) + } return } From 6052aea5f28d6964ac4561474bcae922bc3753d7 Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Thu, 12 Oct 2023 11:43:33 -0300 Subject: [PATCH 098/124] moved upload function to arch service --- modules/packages/arch/metadata.go | 2 +- modules/packages/arch/metadata_test.go | 2 +- routers/api/packages/arch/arch.go | 68 +------------------ services/packages/arch/upload.go | 90 ++++++++++++++++++++++++++ 4 files changed, 93 insertions(+), 69 deletions(-) create mode 100644 services/packages/arch/upload.go diff --git a/modules/packages/arch/metadata.go b/modules/packages/arch/metadata.go index 930cedc6c4c9..fba2e2073a9a 100644 --- a/modules/packages/arch/metadata.go +++ b/modules/packages/arch/metadata.go @@ -58,7 +58,7 @@ type DbDesc struct { } // Function that receives arch package archive data and returns it's metadata. -func EjectMetadata(file, distro string, b *pkg_module.HashedBuffer) (*DbDesc, error) { +func ParseMetadata(file, distro string, b *pkg_module.HashedBuffer) (*DbDesc, error) { pkginfo, err := getPkginfo(b) if err != nil { return nil, err diff --git a/modules/packages/arch/metadata_test.go b/modules/packages/arch/metadata_test.go index 66239d927d5b..da13f0415b5b 100644 --- a/modules/packages/arch/metadata_test.go +++ b/modules/packages/arch/metadata_test.go @@ -138,7 +138,7 @@ func TestMetadata(t *testing.T) { archive.Close() assert.NoError(t, err) - md, err := EjectMetadata("zstd-1.5.5-1-x86_64.pkg.tar.zst", "archlinux", buf) + md, err := ParseMetadata("zstd-1.5.5-1-x86_64.pkg.tar.zst", "archlinux", buf) assert.NoError(t, err) assert.Equal(t, md.Name, "zstd") diff --git a/routers/api/packages/arch/arch.go b/routers/api/packages/arch/arch.go index cc92317eeeab..3690d9d058a8 100644 --- a/routers/api/packages/arch/arch.go +++ b/routers/api/packages/arch/arch.go @@ -4,15 +4,11 @@ package arch import ( - "encoding/hex" - "io" "net/http" "strings" pkg_model "code.gitea.io/gitea/models/packages" "code.gitea.io/gitea/modules/context" - pkg_module "code.gitea.io/gitea/modules/packages" - arch_module "code.gitea.io/gitea/modules/packages/arch" "code.gitea.io/gitea/routers/api/packages/helper" pkg_service "code.gitea.io/gitea/services/packages" arch_service "code.gitea.io/gitea/services/packages/arch" @@ -41,69 +37,7 @@ func Push(ctx *context.Context) { defer upload.Close() } - buf, err := pkg_module.CreateHashedBufferFromReader(upload) - if err != nil { - apiError(ctx, http.StatusInternalServerError, err) - return - } - defer buf.Close() - - desc, err := arch_module.EjectMetadata(filename, distro, buf) - if err != nil { - apiError(ctx, http.StatusBadRequest, err) - return - } - - _, err = buf.Seek(0, io.SeekStart) - if err != nil { - apiError(ctx, http.StatusInternalServerError, err) - return - } - - properties := map[string]string{ - "desc": desc.String(), - } - if sign != "" { - _, err := hex.DecodeString(sign) - if err != nil { - apiError(ctx, http.StatusBadRequest, err) - return - } - properties["sign"] = sign - } - - _, _, err = pkg_service.CreatePackageOrAddFileToExisting( - ctx, &pkg_service.PackageCreationInfo{ - PackageInfo: pkg_service.PackageInfo{ - Owner: ctx.Package.Owner, - PackageType: pkg_model.TypeArch, - Name: desc.Name, - Version: desc.Version, - }, - Creator: ctx.Doer, - Metadata: &arch_module.Metadata{ - URL: desc.ProjectURL, - Description: desc.Description, - Provides: desc.Provides, - License: desc.License, - Depends: desc.Depends, - OptDepends: desc.OptDepends, - MakeDepends: desc.MakeDepends, - CheckDepends: desc.CheckDepends, - }, - }, - &pkg_service.PackageFileCreationInfo{ - PackageFileInfo: pkg_service.PackageFileInfo{ - Filename: filename, - CompositeKey: distro, - }, - OverwriteExisting: true, - IsLead: true, - Creator: ctx.ContextUser, - Data: buf, - Properties: properties, - }, - ) + _, _, err = arch_service.UploadArchPackage(ctx, upload, filename, distro, sign) if err != nil { switch err { case pkg_model.ErrDuplicatePackageVersion, pkg_model.ErrDuplicatePackageFile: diff --git a/services/packages/arch/upload.go b/services/packages/arch/upload.go new file mode 100644 index 000000000000..1bb38e967cf4 --- /dev/null +++ b/services/packages/arch/upload.go @@ -0,0 +1,90 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package arch + +import ( + "encoding/hex" + "errors" + "io" + + packages_model "code.gitea.io/gitea/models/packages" + "code.gitea.io/gitea/modules/context" + packages_module "code.gitea.io/gitea/modules/packages" + arch_module "code.gitea.io/gitea/modules/packages/arch" + packages_service "code.gitea.io/gitea/services/packages" +) + +// UploadArchPackage adds an Arch Package to the registry. +// The first return value indictaes if the error is a user error. +func UploadArchPackage(ctx *context.Context, upload io.Reader, filename, distro, sign string) (bool, *packages_model.PackageVersion, error) { + buf, err := packages_module.CreateHashedBufferFromReader(upload) + if err != nil { + return false, nil, err + } + defer buf.Close() + + desc, err := arch_module.ParseMetadata(filename, distro, buf) + if err != nil { + return false, nil, err + } + + _, err = buf.Seek(0, io.SeekStart) + if err != nil { + return false, nil, err + } + + properties := map[string]string{ + "desc": desc.String(), + } + if sign != "" { + _, err := hex.DecodeString(sign) + if err != nil { + return true, nil, errors.New("unable to decode package signature") + } + properties["sign"] = sign + } + + ver, _, err := packages_service.CreatePackageOrAddFileToExisting( + ctx, &packages_service.PackageCreationInfo{ + PackageInfo: packages_service.PackageInfo{ + Owner: ctx.Package.Owner, + PackageType: packages_model.TypeArch, + Name: desc.Name, + Version: desc.Version, + }, + Creator: ctx.Doer, + Metadata: &arch_module.Metadata{ + URL: desc.ProjectURL, + Description: desc.Description, + Provides: desc.Provides, + License: desc.License, + Depends: desc.Depends, + OptDepends: desc.OptDepends, + MakeDepends: desc.MakeDepends, + CheckDepends: desc.CheckDepends, + }, + }, + &packages_service.PackageFileCreationInfo{ + PackageFileInfo: packages_service.PackageFileInfo{ + Filename: filename, + CompositeKey: distro, + }, + OverwriteExisting: true, + IsLead: true, + Creator: ctx.ContextUser, + Data: buf, + Properties: properties, + }, + ) + if err != nil { + switch err { + case packages_model.ErrDuplicatePackageVersion, packages_model.ErrDuplicatePackageFile, packages_service.ErrQuotaTotalCount, packages_service.ErrQuotaTypeSize, packages_service.ErrQuotaTotalSize: + return true, nil, err + default: + return false, nil, err + } + } + + return false, ver, nil +} From f812aa7cf71840003f6e3a29052430cd2b793aa6 Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Thu, 12 Oct 2023 11:47:17 -0300 Subject: [PATCH 099/124] fixes in arch package metadata tests --- modules/packages/arch/metadata_test.go | 28 ++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/modules/packages/arch/metadata_test.go b/modules/packages/arch/metadata_test.go index da13f0415b5b..3ef361e60286 100644 --- a/modules/packages/arch/metadata_test.go +++ b/modules/packages/arch/metadata_test.go @@ -41,7 +41,7 @@ makedepend = gtest makedepend = ninja ` -const pkgdesc = `%%FILENAME% +const pkgdesc = `%FILENAME% zstd-1.5.5-1-x86_64.pkg.tar.zst %NAME% @@ -68,7 +68,7 @@ Zstandard - Fast real-time compression algorithm %SHA256SUM% 9fa4ede47e35f5971e4f26ecadcbfb66ab79f1d638317ac80334a3362dedbabd -%%URL% +%URL% https://facebook.github.io/zstd/ %LICENSE% @@ -152,12 +152,32 @@ func TestMetadata(t *testing.T) { assert.Equal(t, md.Arch, []string{"x86_64"}) assert.Equal(t, md.License, []string{"BSD", "GPL2"}) assert.Equal(t, md.Provides, []string{"libzstd.so=1-64"}) - assert.Equal(t, md.Provides, []string{"libzstd.so=1-64"}) assert.Equal(t, md.Depends, []string{"glibc", "gcc-libs", "zlib", "xz", "lz4"}) assert.Equal(t, md.MakeDepends, []string{"cmake", "gtest", "ninja"}) +} +func TestDescString(t *testing.T) { + md := &DbDesc{ + Filename: "zstd-1.5.5-1-x86_64.pkg.tar.zst", + Name: "zstd", + Base: "zstd", + Version: "1.5.5-1", + Description: "Zstandard - Fast real-time compression algorithm", + CompressedSize: 401, + InstalledSize: 1500453, + MD5: "5016660ef3d9aa148a7b72a08d3df1b2", + SHA256: "9fa4ede47e35f5971e4f26ecadcbfb66ab79f1d638317ac80334a3362dedbabd", + ProjectURL: "https://facebook.github.io/zstd/", + BuildDate: 1681646714, + Packager: "Jelle van der Waa ", + Provides: []string{"libzstd.so=1-64"}, + License: []string{"BSD", "GPL2"}, + Arch: []string{"x86_64"}, + Depends: []string{"glibc", "gcc-libs", "zlib", "xz", "lz4"}, + MakeDepends: []string{"cmake", "gtest", "ninja"}, + } desc := md.String() - assert.Equal(t, desc, pkgdesc) + assert.Equal(t, pkgdesc, desc) } func TestDatabase(t *testing.T) { From 95a1df6dfc7be5db857979efc066b6c73a7bb78a Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Thu, 12 Oct 2023 11:56:54 -0300 Subject: [PATCH 100/124] import naming unification with other package types --- modules/packages/arch/metadata.go | 6 +++--- modules/packages/arch/metadata_test.go | 4 ++-- routers/api/packages/arch/arch.go | 16 +++++++------- services/packages/arch/service.go | 30 +++++++++++++------------- 4 files changed, 28 insertions(+), 28 deletions(-) diff --git a/modules/packages/arch/metadata.go b/modules/packages/arch/metadata.go index fba2e2073a9a..59d4b0c085be 100644 --- a/modules/packages/arch/metadata.go +++ b/modules/packages/arch/metadata.go @@ -14,7 +14,7 @@ import ( "strconv" "strings" - pkg_module "code.gitea.io/gitea/modules/packages" + packages_module "code.gitea.io/gitea/modules/packages" "github.com/mholt/archiver/v3" ) @@ -58,7 +58,7 @@ type DbDesc struct { } // Function that receives arch package archive data and returns it's metadata. -func ParseMetadata(file, distro string, b *pkg_module.HashedBuffer) (*DbDesc, error) { +func ParseMetadata(file, distro string, b *packages_module.HashedBuffer) (*DbDesc, error) { pkginfo, err := getPkginfo(b) if err != nil { return nil, err @@ -188,7 +188,7 @@ func (m *DbDesc) String() string { // Create pacman database archive based on provided package metadata structs. func CreatePacmanDb(entries map[string][]byte) (io.ReadSeeker, error) { - out, err := pkg_module.NewHashedBuffer() + out, err := packages_module.NewHashedBuffer() if err != nil { return nil, err } diff --git a/modules/packages/arch/metadata_test.go b/modules/packages/arch/metadata_test.go index 3ef361e60286..fb5897dd9c7c 100644 --- a/modules/packages/arch/metadata_test.go +++ b/modules/packages/arch/metadata_test.go @@ -11,7 +11,7 @@ import ( "testing/fstest" "time" - pkg_module "code.gitea.io/gitea/modules/packages" + packages_module "code.gitea.io/gitea/modules/packages" "github.com/mholt/archiver/v3" "github.com/stretchr/testify/assert" @@ -118,7 +118,7 @@ func TestMetadata(t *testing.T) { file, err := fs.Open("pkginfo") assert.NoError(t, err) - buf, err := pkg_module.NewHashedBuffer() + buf, err := packages_module.NewHashedBuffer() assert.NoError(t, err) archive := archiver.NewTarZstd() diff --git a/routers/api/packages/arch/arch.go b/routers/api/packages/arch/arch.go index 3690d9d058a8..c52ce2f88bba 100644 --- a/routers/api/packages/arch/arch.go +++ b/routers/api/packages/arch/arch.go @@ -7,10 +7,10 @@ import ( "net/http" "strings" - pkg_model "code.gitea.io/gitea/models/packages" + packages_model "code.gitea.io/gitea/models/packages" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/routers/api/packages/helper" - pkg_service "code.gitea.io/gitea/services/packages" + packages_service "code.gitea.io/gitea/services/packages" arch_service "code.gitea.io/gitea/services/packages/arch" ) @@ -40,9 +40,9 @@ func Push(ctx *context.Context) { _, _, err = arch_service.UploadArchPackage(ctx, upload, filename, distro, sign) if err != nil { switch err { - case pkg_model.ErrDuplicatePackageVersion, pkg_model.ErrDuplicatePackageFile: + case packages_model.ErrDuplicatePackageVersion, packages_model.ErrDuplicatePackageFile: apiError(ctx, http.StatusConflict, err) - case pkg_service.ErrQuotaTotalCount, pkg_service.ErrQuotaTypeSize, pkg_service.ErrQuotaTotalSize: + case packages_service.ErrQuotaTotalCount, packages_service.ErrQuotaTypeSize, packages_service.ErrQuotaTotalSize: apiError(ctx, http.StatusForbidden, err) default: apiError(ctx, http.StatusInternalServerError, err) @@ -111,12 +111,12 @@ func Remove(ctx *context.Context) { ver = ctx.Params("version") ) - version, err := pkg_model.GetVersionByNameAndVersion( - ctx, ctx.Package.Owner.ID, pkg_model.TypeArch, pkg, ver, + version, err := packages_model.GetVersionByNameAndVersion( + ctx, ctx.Package.Owner.ID, packages_model.TypeArch, pkg, ver, ) if err != nil { switch err { - case pkg_model.ErrPackageNotExist: + case packages_model.ErrPackageNotExist: apiError(ctx, http.StatusNotFound, err) default: apiError(ctx, http.StatusInternalServerError, err) @@ -124,7 +124,7 @@ func Remove(ctx *context.Context) { return } - err = pkg_service.RemovePackageVersion(ctx, ctx.Package.Owner, version) + err = packages_service.RemovePackageVersion(ctx, ctx.Package.Owner, version) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return diff --git a/services/packages/arch/service.go b/services/packages/arch/service.go index a79246d91a2d..c78689212c8e 100644 --- a/services/packages/arch/service.go +++ b/services/packages/arch/service.go @@ -12,10 +12,10 @@ import ( "sort" "strings" - pkg_model "code.gitea.io/gitea/models/packages" + packages_model "code.gitea.io/gitea/models/packages" "code.gitea.io/gitea/modules/context" arch_module "code.gitea.io/gitea/modules/packages/arch" - pkg_service "code.gitea.io/gitea/services/packages" + packages_service "code.gitea.io/gitea/services/packages" ) // Get data related to provided filename and distribution, for package files @@ -26,7 +26,7 @@ func GetPackageFile(ctx *context.Context, distro, file string) (io.ReadSeekClose return nil, err } - filestream, _, _, err := pkg_service.GetPackageFileStream(ctx, pf) + filestream, _, _, err := packages_service.GetPackageFileStream(ctx, pf) return filestream, err } @@ -38,7 +38,7 @@ func GetPackageSignature(ctx *context.Context, distro, file string) (*bytes.Read return nil, err } - proprs, err := pkg_model.GetProperties(ctx, pkg_model.PropertyTypeFile, pf.ID) + proprs, err := packages_model.GetProperties(ctx, packages_model.PropertyTypeFile, pf.ID) if err != nil { return nil, err } @@ -57,21 +57,21 @@ func GetPackageSignature(ctx *context.Context, distro, file string) (*bytes.Read } // Ejects parameters required to get package file property from file name. -func getPackageFile(ctx *context.Context, distro, file string) (*pkg_model.PackageFile, error) { +func getPackageFile(ctx *context.Context, distro, file string) (*packages_model.PackageFile, error) { var ( splt = strings.Split(file, "-") pkgname = strings.Join(splt[0:len(splt)-3], "-") vername = splt[len(splt)-3] + "-" + splt[len(splt)-2] ) - version, err := pkg_model.GetVersionByNameAndVersion( - ctx, ctx.Package.Owner.ID, pkg_model.TypeArch, pkgname, vername, + version, err := packages_model.GetVersionByNameAndVersion( + ctx, ctx.Package.Owner.ID, packages_model.TypeArch, pkgname, vername, ) if err != nil { return nil, err } - pkgfile, err := pkg_model.GetFileForVersionByName(ctx, version.ID, file, distro) + pkgfile, err := packages_model.GetFileForVersionByName(ctx, version.ID, file, distro) if err != nil { return nil, err } @@ -84,7 +84,7 @@ func getPackageFile(ctx *context.Context, distro, file string) (*pkg_model.Packa // compatible version is found, related desc file will be loaded from package // properties and added to resulting .db.tar.gz archive. func CreatePacmanDb(ctx *context.Context, owner, arch, distro string) (io.ReadSeeker, error) { - pkgs, err := pkg_model.GetPackagesByType(ctx, ctx.Package.Owner.ID, pkg_model.TypeArch) + pkgs, err := packages_model.GetPackagesByType(ctx, ctx.Package.Owner.ID, packages_model.TypeArch) if err != nil { return nil, err } @@ -92,8 +92,8 @@ func CreatePacmanDb(ctx *context.Context, owner, arch, distro string) (io.ReadSe entries := make(map[string][]byte) for _, pkg := range pkgs { - versions, err := pkg_model.GetVersionsByPackageName( - ctx, ctx.Package.Owner.ID, pkg_model.TypeArch, pkg.Name, + versions, err := packages_model.GetVersionsByPackageName( + ctx, ctx.Package.Owner.ID, packages_model.TypeArch, pkg.Name, ) if err != nil { return nil, err @@ -106,17 +106,17 @@ func CreatePacmanDb(ctx *context.Context, owner, arch, distro string) (io.ReadSe for _, ver := range versions { file := fmt.Sprintf("%s-%s-%s.pkg.tar.zst", pkg.Name, ver.Version, arch) - pf, err := pkg_model.GetFileForVersionByName(ctx, ver.ID, file, distro) + pf, err := packages_model.GetFileForVersionByName(ctx, ver.ID, file, distro) if err != nil { file = fmt.Sprintf("%s-%s-any.pkg.tar.zst", pkg.Name, ver.Version) - pf, err = pkg_model.GetFileForVersionByName(ctx, ver.ID, file, distro) + pf, err = packages_model.GetFileForVersionByName(ctx, ver.ID, file, distro) if err != nil { continue } } - pps, err := pkg_model.GetPropertiesByName( - ctx, pkg_model.PropertyTypeFile, pf.ID, "desc", + pps, err := packages_model.GetPropertiesByName( + ctx, packages_model.PropertyTypeFile, pf.ID, "desc", ) if err != nil { return nil, err From 8aeada8c307da90d889a6e0ce8c58df814dcb015 Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Mon, 30 Oct 2023 21:53:14 -0300 Subject: [PATCH 101/124] additional validations for package metadata fields, split package metadata into file and version related structures, better test coverage and refactoring in metadata module --- models/packages/descriptor.go | 2 +- modules/packages/arch/metadata.go | 317 ++++++++++------- modules/packages/arch/metadata_test.go | 448 +++++++++++++++++++------ routers/api/packages/arch/arch.go | 3 +- services/packages/arch/service.go | 2 +- services/packages/arch/upload.go | 23 +- 6 files changed, 559 insertions(+), 236 deletions(-) diff --git a/models/packages/descriptor.go b/models/packages/descriptor.go index 97f6022d2842..01505d5577ed 100644 --- a/models/packages/descriptor.go +++ b/models/packages/descriptor.go @@ -142,7 +142,7 @@ func GetPackageDescriptor(ctx context.Context, pv *PackageVersion) (*PackageDesc case TypeAlpine: metadata = &alpine.VersionMetadata{} case TypeArch: - metadata = &arch.Metadata{} + metadata = &arch.VersionMetadata{} case TypeCargo: metadata = &cargo.Metadata{} case TypeChef: diff --git a/modules/packages/arch/metadata.go b/modules/packages/arch/metadata.go index 59d4b0c085be..daef0af43354 100644 --- a/modules/packages/arch/metadata.go +++ b/modules/packages/arch/metadata.go @@ -6,194 +6,274 @@ package arch import ( "archive/tar" "bufio" + "bytes" "compress/gzip" "encoding/hex" + "errors" "fmt" "io" "os" + "regexp" "strconv" "strings" - packages_module "code.gitea.io/gitea/modules/packages" - + "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/modules/validation" "github.com/mholt/archiver/v3" ) -// JSON with pacakage parameters that are not related to specific -// architecture/distribution that will be stored in sql database. -type Metadata struct { - URL string `json:"url"` +type Package struct { + Name string `json:"name"` + Version string `json:"version"` + VersionMetadata VersionMetadata + FileMetadata FileMetadata +} + +// Arch package metadata related to specific version. +// Version metadata the same across different architectures and distributions. +type VersionMetadata struct { + Base string `json:"base"` Description string `json:"description"` + ProjectURL string `json:"project_url"` + Groups []string `json:"groups,omitempty"` Provides []string `json:"provides,omitempty"` License []string `json:"license,omitempty"` Depends []string `json:"depends,omitempty"` OptDepends []string `json:"opt_depends,omitempty"` MakeDepends []string `json:"make_depends,omitempty"` CheckDepends []string `json:"check_depends,omitempty"` + Backup []string `json:"backup,omitempty"` } -// Package description file that will be saved as .desc file in object storage. -// This file will be used to create pacman database. -type DbDesc struct { - Filename string `json:"filename"` - Name string `json:"name"` - Base string `json:"base"` - Version string `json:"version"` - Description string `json:"description"` - CompressedSize int64 `json:"compressed_size"` - InstalledSize int64 `json:"installed_size"` - MD5 string `json:"md5"` - SHA256 string `json:"sha256"` - ProjectURL string `json:"project_url"` - BuildDate int64 `json:"build_date"` - Packager string `json:"packager"` - Provides []string `json:"provides,omitempty"` - License []string `json:"license,omitempty"` - Arch []string `json:"arch,omitempty"` - Depends []string `json:"depends,omitempty"` - OptDepends []string `json:"opt_depends,omitempty"` - MakeDepends []string `json:"make_depends,omitempty"` - CheckDepends []string `json:"check_depends,omitempty"` - Backup []string `json:"backup,omitempty"` +// Metadata related to specific pakcage file. +// This metadata might vary for different architecture and distribution. +type FileMetadata struct { + CompressedSize int64 `json:"compressed_size"` + InstalledSize int64 `json:"installed_size"` + MD5 string `json:"md5"` + SHA256 string `json:"sha256"` + BuildDate int64 `json:"build_date"` + Packager string `json:"packager"` + Arch string `json:"arch"` } // Function that receives arch package archive data and returns it's metadata. -func ParseMetadata(file, distro string, b *packages_module.HashedBuffer) (*DbDesc, error) { - pkginfo, err := getPkginfo(b) +func ParsePackage(r io.Reader, md5, sha256 []byte, size int64) (*Package, error) { + zstd := archiver.NewTarZstd() + err := zstd.Open(r, 0) if err != nil { return nil, err } + defer zstd.Close() - // Add package blob parameters to arch related desc. - hashMD5, _, hashSHA256, _ := b.Sums() - md := DbDesc{ - Filename: file, - Name: file, - CompressedSize: b.Size(), - MD5: hex.EncodeToString(hashMD5), - SHA256: hex.EncodeToString(hashSHA256), + var pkg *Package + var mtree bool + + for { + f, err := zstd.Read() + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + defer f.Close() + + switch f.Name() { + case ".PKGINFO": + pkg, err = ParsePackageInfo(f) + if err != nil { + return nil, err + } + case ".MTREE": + mtree = true + } + } + + if pkg == nil { + return nil, util.NewInvalidArgumentErrorf(".PKGINFO file not found") } - for _, line := range strings.Split(pkginfo, "\n") { - splt := strings.Split(line, " = ") - if len(splt) != 2 { + if !mtree { + return nil, util.NewInvalidArgumentErrorf(".MTREE file not found") + } + + pkg.FileMetadata.CompressedSize = size + pkg.FileMetadata.SHA256 = hex.EncodeToString(sha256) + pkg.FileMetadata.MD5 = hex.EncodeToString(md5) + + return pkg, nil +} + +// Function that accepts reader for .PKGINFO file from package archive, +// validates all field according to PKGBUILD spec and returns package. +func ParsePackageInfo(r io.Reader) (*Package, error) { + p := &Package{} + + scanner := bufio.NewScanner(r) + for scanner.Scan() { + line := scanner.Text() + + if strings.HasPrefix(line, "#") { continue } - var ( - parameter = splt[0] - value = splt[1] - ) - switch parameter { + i := strings.IndexRune(line, '=') + if i == -1 { + continue + } + + key := strings.TrimSpace(line[:i]) + value := strings.TrimSpace(line[i+1:]) + + switch key { case "pkgname": - md.Name = value + p.Name = value case "pkgbase": - md.Base = value + p.VersionMetadata.Base = value case "pkgver": - md.Version = value + p.Version = value case "pkgdesc": - md.Description = value + p.VersionMetadata.Description = value case "url": - md.ProjectURL = value + p.VersionMetadata.ProjectURL = value case "packager": - md.Packager = value + p.FileMetadata.Packager = value + case "arch": + p.FileMetadata.Arch = value case "provides": - md.Provides = append(md.Provides, value) + p.VersionMetadata.Provides = append(p.VersionMetadata.Provides, value) case "license": - md.License = append(md.License, value) - case "arch": - md.Arch = append(md.Arch, value) + p.VersionMetadata.License = append(p.VersionMetadata.License, value) case "depend": - md.Depends = append(md.Depends, value) + p.VersionMetadata.Depends = append(p.VersionMetadata.Depends, value) case "optdepend": - md.OptDepends = append(md.OptDepends, value) + p.VersionMetadata.OptDepends = append(p.VersionMetadata.OptDepends, value) case "makedepend": - md.MakeDepends = append(md.MakeDepends, value) + p.VersionMetadata.MakeDepends = append(p.VersionMetadata.MakeDepends, value) case "checkdepend": - md.CheckDepends = append(md.CheckDepends, value) + p.VersionMetadata.CheckDepends = append(p.VersionMetadata.CheckDepends, value) case "backup": - md.Backup = append(md.Backup, value) + p.VersionMetadata.Backup = append(p.VersionMetadata.Backup, value) + case "group": + p.VersionMetadata.Groups = append(p.VersionMetadata.Groups, value) case "builddate": - md.BuildDate, err = strconv.ParseInt(value, 10, 64) + bd, err := strconv.ParseInt(value, 10, 64) if err != nil { return nil, err } + p.FileMetadata.BuildDate = bd case "size": - md.InstalledSize, err = strconv.ParseInt(value, 10, 64) + is, err := strconv.ParseInt(value, 10, 64) if err != nil { return nil, err } + p.FileMetadata.InstalledSize = is } } - return &md, nil + return p, errors.Join(scanner.Err(), ValidatePackageSpec(p)) } -// Eject .PKGINFO file as string from package archive. -func getPkginfo(data io.Reader) (string, error) { - br := bufio.NewReader(data) - zstd := archiver.NewTarZstd() - err := zstd.Open(br, int64(250000)) - if err != nil { - return ``, err +// Arch package validation according to PKGBUILD specification: +// https://man.archlinux.org/man/PKGBUILD.5 +func ValidatePackageSpec(p *Package) error { + var ( + reName = regexp.MustCompile(`^[a-zA-Z0-9@._+-]+$`) + reVer = regexp.MustCompile(`^[a-zA-Z0-9:_.+]+-+[0-9]+$`) + reOptDep = regexp.MustCompile(`^[a-zA-Z0-9@._+-]+$|^[a-zA-Z0-9@._+-]+(:.*)`) + rePkgVer = regexp.MustCompile(`^[a-zA-Z0-9@._+-]+$|^[a-zA-Z0-9@._+-]+(>.*)|^[a-zA-Z0-9@._+-]+(<.*)|^[a-zA-Z0-9@._+-]+(=.*)`) + ) + + if !reName.MatchString(p.Name) { + return util.NewInvalidArgumentErrorf("invalid package name") } - for { - f, err := zstd.Read() - if err != nil { - return ``, err + if !reName.MatchString(p.VersionMetadata.Base) { + return util.NewInvalidArgumentErrorf("invalid package base") + } + if !reVer.MatchString(p.Version) { + return util.NewInvalidArgumentErrorf("invalid package version") + } + if p.FileMetadata.Arch == "" { + return util.NewInvalidArgumentErrorf("architecture should be specified") + } + if p.VersionMetadata.ProjectURL != "" { + if !validation.IsValidURL(p.VersionMetadata.ProjectURL) { + return util.NewInvalidArgumentErrorf("invalid project URL") } - if f.Name() != ".PKGINFO" { - continue + } + for _, cd := range p.VersionMetadata.CheckDepends { + if !rePkgVer.MatchString(cd) { + return util.NewInvalidArgumentErrorf("invalid check dependency: " + cd) } - b, err := io.ReadAll(f) - if err != nil { - return ``, err + } + for _, d := range p.VersionMetadata.Depends { + if !rePkgVer.MatchString(d) { + return util.NewInvalidArgumentErrorf("invalid dependency: " + d) } - return string(b), nil } + for _, md := range p.VersionMetadata.MakeDepends { + if !rePkgVer.MatchString(md) { + return util.NewInvalidArgumentErrorf("invalid make dependency: " + md) + } + } + for _, p := range p.VersionMetadata.Provides { + if !rePkgVer.MatchString(p) { + return util.NewInvalidArgumentErrorf("invalid provides: " + p) + } + } + for _, od := range p.VersionMetadata.OptDepends { + if !reOptDep.MatchString(od) { + return util.NewInvalidArgumentErrorf("invalid optional dependency: " + od) + } + } + for _, bf := range p.VersionMetadata.Backup { + if strings.HasPrefix(bf, "/") { + return util.NewInvalidArgumentErrorf("backup file contains leading forward slash") + } + } + return nil } // Create pacman package description file. -func (m *DbDesc) String() string { - entries := []struct{ key, value string }{ - {"FILENAME", m.Filename}, - {"NAME", m.Name}, - {"BASE", m.Base}, - {"VERSION", m.Version}, - {"DESC", m.Description}, - {"CSIZE", fmt.Sprintf("%d", m.CompressedSize)}, - {"ISIZE", fmt.Sprintf("%d", m.InstalledSize)}, - {"MD5SUM", m.MD5}, - {"SHA256SUM", m.SHA256}, - {"URL", m.ProjectURL}, - {"LICENSE", strings.Join(m.License, "\n")}, - {"ARCH", strings.Join(m.Arch, "\n")}, - {"BUILDDATE", fmt.Sprintf("%d", m.BuildDate)}, - {"PACKAGER", m.Packager}, - {"PROVIDES", strings.Join(m.Provides, "\n")}, - {"DEPENDS", strings.Join(m.Depends, "\n")}, - {"OPTDEPENDS", strings.Join(m.OptDepends, "\n")}, - {"MAKEDEPENDS", strings.Join(m.MakeDepends, "\n")}, - {"CHECKDEPENDS", strings.Join(m.CheckDepends, "\n")}, +func (p *Package) Desc() string { + entries := [40]string{ + "FILENAME", fmt.Sprintf("%s-%s-%s.pkg.tar.zst", p.Name, p.Version, p.FileMetadata.Arch), + "NAME", p.Name, + "BASE", p.VersionMetadata.Base, + "VERSION", p.Version, + "DESC", p.VersionMetadata.Description, + "GROUPS", strings.Join(p.VersionMetadata.Groups, "\n"), + "CSIZE", fmt.Sprintf("%d", p.FileMetadata.CompressedSize), + "ISIZE", fmt.Sprintf("%d", p.FileMetadata.InstalledSize), + "MD5SUM", p.FileMetadata.MD5, + "SHA256SUM", p.FileMetadata.SHA256, + "URL", p.VersionMetadata.ProjectURL, + "LICENSE", strings.Join(p.VersionMetadata.License, "\n"), + "ARCH", p.FileMetadata.Arch, + "BUILDDATE", fmt.Sprintf("%d", p.FileMetadata.BuildDate), + "PACKAGER", p.FileMetadata.Packager, + "PROVIDES", strings.Join(p.VersionMetadata.Provides, "\n"), + "DEPENDS", strings.Join(p.VersionMetadata.Depends, "\n"), + "OPTDEPENDS", strings.Join(p.VersionMetadata.OptDepends, "\n"), + "MAKEDEPENDS", strings.Join(p.VersionMetadata.MakeDepends, "\n"), + "CHECKDEPENDS", strings.Join(p.VersionMetadata.CheckDepends, "\n"), } var result string - for _, e := range entries { - if e.value != "" { - result += fmt.Sprintf("%%%s%%\n%s\n\n", e.key, e.value) + for i := 0; i < 40; i += 2 { + if entries[i+1] != "" { + result += fmt.Sprintf("%%%s%%\n%s\n\n", entries[i], entries[i+1]) } } return result } // Create pacman database archive based on provided package metadata structs. -func CreatePacmanDb(entries map[string][]byte) (io.ReadSeeker, error) { - out, err := packages_module.NewHashedBuffer() - if err != nil { - return nil, err - } +func CreatePacmanDb(entries map[string][]byte) (*bytes.Buffer, error) { + var b bytes.Buffer - gw := gzip.NewWriter(out) + gw := gzip.NewWriter(&b) tw := tar.NewWriter(gw) for name, content := range entries { @@ -204,20 +284,13 @@ func CreatePacmanDb(entries map[string][]byte) (io.ReadSeeker, error) { } if err := tw.WriteHeader(header); err != nil { - tw.Close() - gw.Close() - return nil, err + return nil, errors.Join(err, tw.Close(), gw.Close()) } if _, err := tw.Write(content); err != nil { - tw.Close() - gw.Close() - return nil, err + return nil, errors.Join(err, tw.Close(), gw.Close()) } } - tw.Close() - gw.Close() - - return out, nil + return &b, errors.Join(tw.Close(), gw.Close()) } diff --git a/modules/packages/arch/metadata_test.go b/modules/packages/arch/metadata_test.go index fb5897dd9c7c..b08c4b416394 100644 --- a/modules/packages/arch/metadata_test.go +++ b/modules/packages/arch/metadata_test.go @@ -4,44 +4,337 @@ package arch import ( + "bytes" "encoding/base64" + "errors" "io" "os" + "strings" "testing" "testing/fstest" "time" - packages_module "code.gitea.io/gitea/modules/packages" - "github.com/mholt/archiver/v3" "github.com/stretchr/testify/assert" ) -const pkginfo = `# Generated by makepkg 6.0.2 +func TestParsePackage(t *testing.T) { + // Minimal PKGINFO contents and test FS + const PKGINFO = `pkgname = a +pkgbase = b +pkgver = 1-2 +arch = x86_64 +` + fs := fstest.MapFS{ + "pkginfo": &fstest.MapFile{ + Data: []byte(PKGINFO), + Mode: os.ModePerm, + ModTime: time.Now(), + }, + "mtree": &fstest.MapFile{ + Data: []byte("data"), + Mode: os.ModePerm, + ModTime: time.Now(), + }, + } + + // Test .PKGINFO file + pinf, err := fs.Stat("pkginfo") + assert.NoError(t, err) + + pfile, err := fs.Open("pkginfo") + assert.NoError(t, err) + + parcname, err := archiver.NameInArchive(pinf, ".PKGINFO", ".PKGINFO") + assert.NoError(t, err) + + // Test .MTREE file + minf, err := fs.Stat("mtree") + assert.NoError(t, err) + + mfile, err := fs.Open("mtree") + assert.NoError(t, err) + + marcname, err := archiver.NameInArchive(minf, ".MTREE", ".MTREE") + assert.NoError(t, err) + + t.Run("normal archive", func(t *testing.T) { + var buf bytes.Buffer + + archive := archiver.NewTarZstd() + archive.Create(&buf) + + err = archive.Write(archiver.File{ + FileInfo: archiver.FileInfo{ + FileInfo: pinf, + CustomName: parcname, + }, + ReadCloser: pfile, + }) + assert.NoError(t, errors.Join(pfile.Close(), err)) + + err = archive.Write(archiver.File{ + FileInfo: archiver.FileInfo{ + FileInfo: minf, + CustomName: marcname, + }, + ReadCloser: mfile, + }) + assert.NoError(t, errors.Join(mfile.Close(), archive.Close(), err)) + + _, err = ParsePackage(&buf, []byte{}, []byte{}, 0) + + assert.NoError(t, err) + }) + + t.Run("missing .PKGINFO", func(t *testing.T) { + var buf bytes.Buffer + + archive := archiver.NewTarZstd() + archive.Create(&buf) + + assert.NoError(t, archive.Close()) + + _, err = ParsePackage(&buf, []byte{}, []byte{}, 0) + + assert.Error(t, err) + assert.Contains(t, err.Error(), ".PKGINFO file not found") + }) + + t.Run("missing .MTREE", func(t *testing.T) { + var buf bytes.Buffer + + pfile, err := fs.Open("pkginfo") + assert.NoError(t, err) + + archive := archiver.NewTarZstd() + archive.Create(&buf) + + err = archive.Write(archiver.File{ + FileInfo: archiver.FileInfo{ + FileInfo: pinf, + CustomName: parcname, + }, + ReadCloser: pfile, + }) + assert.NoError(t, errors.Join(pfile.Close(), archive.Close(), err)) + + _, err = ParsePackage(&buf, []byte{}, []byte{}, 0) + + assert.Error(t, err) + assert.Contains(t, err.Error(), ".MTREE file not found") + }) +} + +func TestParsePackageInfo(t *testing.T) { + const PKGINFO = `# Generated by makepkg 6.0.2 # using fakeroot version 1.31 -pkgname = zstd -pkgbase = zstd -pkgver = 1.5.5-1 -pkgdesc = Zstandard - Fast real-time compression algorithm -url = https://facebook.github.io/zstd/ -builddate = 1681646714 -packager = Jelle van der Waa -size = 1500453 +pkgname = a +pkgbase = b +pkgver = 1-2 +pkgdesc = comment +url = https://example.com/ +group = group +builddate = 3 +packager = Name Surname +size = 5 arch = x86_64 license = BSD -license = GPL2 -provides = libzstd.so=1-64 -depend = glibc -depend = gcc-libs -depend = zlib -depend = xz -depend = lz4 +provides = pvd +depend = smth +optdepend = hex +checkdepend = ola makedepend = cmake -makedepend = gtest -makedepend = ninja +backup = usr/bin/paket1 ` + p, err := ParsePackageInfo(strings.NewReader(PKGINFO)) + assert.NoError(t, err) + assert.Equal(t, Package{ + Name: "a", + Version: "1-2", + VersionMetadata: VersionMetadata{ + Base: "b", + Description: "comment", + ProjectURL: "https://example.com/", + Groups: []string{"group"}, + Provides: []string{"pvd"}, + License: []string{"BSD"}, + Depends: []string{"smth"}, + OptDepends: []string{"hex"}, + MakeDepends: []string{"cmake"}, + CheckDepends: []string{"ola"}, + Backup: []string{"usr/bin/paket1"}, + }, + FileMetadata: FileMetadata{ + InstalledSize: 5, + BuildDate: 3, + Packager: "Name Surname ", + Arch: "x86_64", + }, + }, *p) +} + +func TestValidatePackageSpec(t *testing.T) { + var newpkg = func() Package { + return Package{ + Name: "abc", + Version: "1-1", + VersionMetadata: VersionMetadata{ + Base: "ghx", + Description: "whoami", + ProjectURL: "https://example.com/", + Groups: []string{"gnome"}, + Provides: []string{"abc", "def"}, + License: []string{"GPL"}, + Depends: []string{"go", "gpg=1", "curl>=3", "git<=7"}, + OptDepends: []string{"git: something", "make"}, + MakeDepends: []string{"chrom"}, + CheckDepends: []string{"bariy"}, + Backup: []string{"etc/pacman.d/filo"}, + }, + FileMetadata: FileMetadata{ + CompressedSize: 1, + InstalledSize: 2, + MD5: "abc", + SHA256: "def", + BuildDate: 3, + Packager: "smon", + Arch: "x86_64", + }, + } + } + + t.Run("valid package", func(t *testing.T) { + p := newpkg() + + err := ValidatePackageSpec(&p) + + assert.NoError(t, err) + }) + + t.Run("invalid package name", func(t *testing.T) { + p := newpkg() + p.Name = "!$%@^!*&()" + + err := ValidatePackageSpec(&p) + + assert.Error(t, err) + assert.Contains(t, err.Error(), "invalid package name") + }) + + t.Run("invalid package base", func(t *testing.T) { + p := newpkg() + p.VersionMetadata.Base = "!$%@^!*&()" + + err := ValidatePackageSpec(&p) + + assert.Error(t, err) + assert.Contains(t, err.Error(), "invalid package base") + }) + + t.Run("invalid package version", func(t *testing.T) { + p := newpkg() + p.VersionMetadata.Base = "una-luna?" + + err := ValidatePackageSpec(&p) + + assert.Error(t, err) + assert.Contains(t, err.Error(), "invalid package base") + }) + + t.Run("invalid package version", func(t *testing.T) { + p := newpkg() + p.Version = "una-luna" + + err := ValidatePackageSpec(&p) -const pkgdesc = `%FILENAME% + assert.Error(t, err) + assert.Contains(t, err.Error(), "invalid package version") + }) + + t.Run("missing architecture", func(t *testing.T) { + p := newpkg() + p.FileMetadata.Arch = "" + + err := ValidatePackageSpec(&p) + + assert.Error(t, err) + assert.Contains(t, err.Error(), "architecture should be specified") + }) + + t.Run("invalid URL", func(t *testing.T) { + p := newpkg() + p.VersionMetadata.ProjectURL = "http%%$#" + + err := ValidatePackageSpec(&p) + + assert.Error(t, err) + assert.Contains(t, err.Error(), "invalid project URL") + }) + + t.Run("invalid check dependency", func(t *testing.T) { + p := newpkg() + p.VersionMetadata.CheckDepends = []string{"Err^_^"} + + err := ValidatePackageSpec(&p) + + assert.Error(t, err) + assert.Contains(t, err.Error(), "invalid check dependency") + }) + + t.Run("invalid dependency", func(t *testing.T) { + p := newpkg() + p.VersionMetadata.Depends = []string{"^^abc"} + + err := ValidatePackageSpec(&p) + + assert.Error(t, err) + assert.Contains(t, err.Error(), "invalid dependency") + }) + + t.Run("invalid make dependency", func(t *testing.T) { + p := newpkg() + p.VersionMetadata.MakeDepends = []string{"^m^"} + + err := ValidatePackageSpec(&p) + + assert.Error(t, err) + assert.Contains(t, err.Error(), "invalid make dependency") + }) + + t.Run("invalid provides", func(t *testing.T) { + p := newpkg() + p.VersionMetadata.Provides = []string{"^m^"} + + err := ValidatePackageSpec(&p) + + assert.Error(t, err) + assert.Contains(t, err.Error(), "invalid provides") + }) + + t.Run("invalid optional dependency", func(t *testing.T) { + p := newpkg() + p.VersionMetadata.OptDepends = []string{"^m^:MM"} + + err := ValidatePackageSpec(&p) + + assert.Error(t, err) + assert.Contains(t, err.Error(), "invalid optional dependency") + }) + + t.Run("invalid optional dependency", func(t *testing.T) { + p := newpkg() + p.VersionMetadata.Backup = []string{"/ola/cola"} + + err := ValidatePackageSpec(&p) + + assert.Error(t, err) + assert.Contains(t, err.Error(), "backup file contains leading forward slash") + }) +} + +func TestDescString(t *testing.T) { + const pkgdesc = `%FILENAME% zstd-1.5.5-1-x86_64.pkg.tar.zst %NAME% @@ -56,6 +349,10 @@ zstd %DESC% Zstandard - Fast real-time compression algorithm +%GROUPS% +dummy1 +dummy2 + %CSIZE% 401 @@ -94,93 +391,52 @@ zlib xz lz4 +%OPTDEPENDS% +dummy3 +dummy4 + %MAKEDEPENDS% cmake gtest ninja -` +%CHECKDEPENDS% +dummy5 +dummy6 -const dbarchive = "H4sIAAAAAAAA/0rLzEnVS60oYaAhMDAwMDA3NwfTBgYG6LSBgYEpEtuAwcDQwMzUgEHBgJaOgoHS4pLEIgYDiu1C99wQASmlubmVA+2IUTAKRsEoGAV0B4AAAAD//2VF3KIACAAA" +` -func TestMetadata(t *testing.T) { - fs := fstest.MapFS{ - "pkginfo": &fstest.MapFile{ - Data: []byte(pkginfo), - Mode: os.ModePerm, - ModTime: time.Now(), + md := &Package{ + Name: "zstd", + Version: "1.5.5-1", + VersionMetadata: VersionMetadata{ + Base: "zstd", + Description: "Zstandard - Fast real-time compression algorithm", + ProjectURL: "https://facebook.github.io/zstd/", + Groups: []string{"dummy1", "dummy2"}, + Provides: []string{"libzstd.so=1-64"}, + License: []string{"BSD", "GPL2"}, + Depends: []string{"glibc", "gcc-libs", "zlib", "xz", "lz4"}, + OptDepends: []string{"dummy3", "dummy4"}, + MakeDepends: []string{"cmake", "gtest", "ninja"}, + CheckDepends: []string{"dummy5", "dummy6"}, }, - } - - info, err := fs.Stat("pkginfo") - assert.NoError(t, err) - - file, err := fs.Open("pkginfo") - assert.NoError(t, err) - - buf, err := packages_module.NewHashedBuffer() - assert.NoError(t, err) - - archive := archiver.NewTarZstd() - archive.Create(buf) - - n, err := archiver.NameInArchive(info, ".PKGINFO", ".PKGINFO") - assert.NoError(t, err) - - err = archive.Write(archiver.File{ - FileInfo: archiver.FileInfo{ - FileInfo: info, - CustomName: n, + FileMetadata: FileMetadata{ + CompressedSize: 401, + InstalledSize: 1500453, + MD5: "5016660ef3d9aa148a7b72a08d3df1b2", + SHA256: "9fa4ede47e35f5971e4f26ecadcbfb66ab79f1d638317ac80334a3362dedbabd", + BuildDate: 1681646714, + Packager: "Jelle van der Waa ", + Arch: "x86_64", }, - ReadCloser: file, - }) - file.Close() - archive.Close() - assert.NoError(t, err) - - md, err := ParseMetadata("zstd-1.5.5-1-x86_64.pkg.tar.zst", "archlinux", buf) - assert.NoError(t, err) - - assert.Equal(t, md.Name, "zstd") - assert.Equal(t, md.Base, "zstd") - assert.Equal(t, md.Version, "1.5.5-1") - assert.Equal(t, md.Description, "Zstandard - Fast real-time compression algorithm") - assert.Equal(t, md.ProjectURL, "https://facebook.github.io/zstd/") - assert.Equal(t, md.BuildDate, int64(1681646714)) - assert.Equal(t, md.Packager, "Jelle van der Waa ") - assert.Equal(t, md.InstalledSize, int64(1500453)) - assert.Equal(t, md.Arch, []string{"x86_64"}) - assert.Equal(t, md.License, []string{"BSD", "GPL2"}) - assert.Equal(t, md.Provides, []string{"libzstd.so=1-64"}) - assert.Equal(t, md.Depends, []string{"glibc", "gcc-libs", "zlib", "xz", "lz4"}) - assert.Equal(t, md.MakeDepends, []string{"cmake", "gtest", "ninja"}) -} - -func TestDescString(t *testing.T) { - md := &DbDesc{ - Filename: "zstd-1.5.5-1-x86_64.pkg.tar.zst", - Name: "zstd", - Base: "zstd", - Version: "1.5.5-1", - Description: "Zstandard - Fast real-time compression algorithm", - CompressedSize: 401, - InstalledSize: 1500453, - MD5: "5016660ef3d9aa148a7b72a08d3df1b2", - SHA256: "9fa4ede47e35f5971e4f26ecadcbfb66ab79f1d638317ac80334a3362dedbabd", - ProjectURL: "https://facebook.github.io/zstd/", - BuildDate: 1681646714, - Packager: "Jelle van der Waa ", - Provides: []string{"libzstd.so=1-64"}, - License: []string{"BSD", "GPL2"}, - Arch: []string{"x86_64"}, - Depends: []string{"glibc", "gcc-libs", "zlib", "xz", "lz4"}, - MakeDepends: []string{"cmake", "gtest", "ninja"}, } - desc := md.String() - assert.Equal(t, pkgdesc, desc) + assert.Equal(t, pkgdesc, md.Desc()) } -func TestDatabase(t *testing.T) { +func TestCreatePacmanDb(t *testing.T) { + const dbarchive = "H4sIAAAAAAAA/0rLzEnVS60oYaAhMDAwMDA3NwfTBgYG6LSBgYEpEtuAwcDQwMzUgEHBgJaOgoHS4pLEIgYDiu1C99wQASmlubmVA+2IUTAKRsEoGAV0B4AAAAD//2VF3KIACAAA" + db, err := CreatePacmanDb(map[string][]byte{ "file.ext": []byte("dummy"), }) diff --git a/routers/api/packages/arch/arch.go b/routers/api/packages/arch/arch.go index c52ce2f88bba..358ca81c6e73 100644 --- a/routers/api/packages/arch/arch.go +++ b/routers/api/packages/arch/arch.go @@ -4,6 +4,7 @@ package arch import ( + "bytes" "net/http" "strings" @@ -95,7 +96,7 @@ func Get(ctx *context.Context) { return } - ctx.ServeContent(db, &context.ServeHeaderOptions{ + ctx.ServeContent(bytes.NewReader(db.Bytes()), &context.ServeHeaderOptions{ Filename: file, }) return diff --git a/services/packages/arch/service.go b/services/packages/arch/service.go index c78689212c8e..aa11a73048ad 100644 --- a/services/packages/arch/service.go +++ b/services/packages/arch/service.go @@ -83,7 +83,7 @@ func getPackageFile(ctx *context.Context, distro, file string) (*packages_model. // requested combination of architecture and distribution. When/If the first // compatible version is found, related desc file will be loaded from package // properties and added to resulting .db.tar.gz archive. -func CreatePacmanDb(ctx *context.Context, owner, arch, distro string) (io.ReadSeeker, error) { +func CreatePacmanDb(ctx *context.Context, owner, arch, distro string) (*bytes.Buffer, error) { pkgs, err := packages_model.GetPackagesByType(ctx, ctx.Package.Owner.ID, packages_model.TypeArch) if err != nil { return nil, err diff --git a/services/packages/arch/upload.go b/services/packages/arch/upload.go index 1bb38e967cf4..654bbebf9b37 100644 --- a/services/packages/arch/upload.go +++ b/services/packages/arch/upload.go @@ -24,7 +24,9 @@ func UploadArchPackage(ctx *context.Context, upload io.Reader, filename, distro, } defer buf.Close() - desc, err := arch_module.ParseMetadata(filename, distro, buf) + md5, _, sha256, _ := buf.Sums() + + p, err := arch_module.ParsePackage(buf, md5, sha256, buf.Size()) if err != nil { return false, nil, err } @@ -35,7 +37,7 @@ func UploadArchPackage(ctx *context.Context, upload io.Reader, filename, distro, } properties := map[string]string{ - "desc": desc.String(), + "desc": p.Desc(), } if sign != "" { _, err := hex.DecodeString(sign) @@ -50,20 +52,11 @@ func UploadArchPackage(ctx *context.Context, upload io.Reader, filename, distro, PackageInfo: packages_service.PackageInfo{ Owner: ctx.Package.Owner, PackageType: packages_model.TypeArch, - Name: desc.Name, - Version: desc.Version, - }, - Creator: ctx.Doer, - Metadata: &arch_module.Metadata{ - URL: desc.ProjectURL, - Description: desc.Description, - Provides: desc.Provides, - License: desc.License, - Depends: desc.Depends, - OptDepends: desc.OptDepends, - MakeDepends: desc.MakeDepends, - CheckDepends: desc.CheckDepends, + Name: p.Name, + Version: p.Version, }, + Creator: ctx.Doer, + Metadata: p.VersionMetadata, }, &packages_service.PackageFileCreationInfo{ PackageFileInfo: packages_service.PackageFileInfo{ From 5becd3c4e01ec92c0bb6ec2aff8318d1accec8e6 Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Tue, 31 Oct 2023 14:51:12 -0300 Subject: [PATCH 102/124] initialize regular expressions for arch package validation once --- modules/packages/arch/metadata.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/modules/packages/arch/metadata.go b/modules/packages/arch/metadata.go index daef0af43354..c64b0c5a2b2a 100644 --- a/modules/packages/arch/metadata.go +++ b/modules/packages/arch/metadata.go @@ -22,6 +22,14 @@ import ( "github.com/mholt/archiver/v3" ) +var ( + // https://man.archlinux.org/man/PKGBUILD.5 + reName = regexp.MustCompile(`^[a-zA-Z0-9@._+-]+$`) + reVer = regexp.MustCompile(`^[a-zA-Z0-9:_.+]+-+[0-9]+$`) + reOptDep = regexp.MustCompile(`^[a-zA-Z0-9@._+-]+$|^[a-zA-Z0-9@._+-]+(:.*)`) + rePkgVer = regexp.MustCompile(`^[a-zA-Z0-9@._+-]+$|^[a-zA-Z0-9@._+-]+(>.*)|^[a-zA-Z0-9@._+-]+(<.*)|^[a-zA-Z0-9@._+-]+(=.*)`) +) + type Package struct { Name string `json:"name"` Version string `json:"version"` @@ -175,16 +183,8 @@ func ParsePackageInfo(r io.Reader) (*Package, error) { return p, errors.Join(scanner.Err(), ValidatePackageSpec(p)) } -// Arch package validation according to PKGBUILD specification: -// https://man.archlinux.org/man/PKGBUILD.5 +// Arch package validation according to PKGBUILD specification. func ValidatePackageSpec(p *Package) error { - var ( - reName = regexp.MustCompile(`^[a-zA-Z0-9@._+-]+$`) - reVer = regexp.MustCompile(`^[a-zA-Z0-9:_.+]+-+[0-9]+$`) - reOptDep = regexp.MustCompile(`^[a-zA-Z0-9@._+-]+$|^[a-zA-Z0-9@._+-]+(:.*)`) - rePkgVer = regexp.MustCompile(`^[a-zA-Z0-9@._+-]+$|^[a-zA-Z0-9@._+-]+(>.*)|^[a-zA-Z0-9@._+-]+(<.*)|^[a-zA-Z0-9@._+-]+(=.*)`) - ) - if !reName.MatchString(p.Name) { return util.NewInvalidArgumentErrorf("invalid package name") } From 73aef3a2c7d4b2469881044a553199be12f197dd Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Tue, 31 Oct 2023 19:16:31 -0300 Subject: [PATCH 103/124] integration tests for databases with different architectures, versions and any packages --- routers/api/packages/api.go | 1 + tests/integration/api_packages_arch_test.go | 591 +++++++++++++------- 2 files changed, 397 insertions(+), 195 deletions(-) diff --git a/routers/api/packages/api.go b/routers/api/packages/api.go index 5a830a39a929..5fc6751ca5e3 100644 --- a/routers/api/packages/api.go +++ b/routers/api/packages/api.go @@ -124,6 +124,7 @@ func CommonRoutes() *web.Route { }) }, reqPackageAccess(perm.AccessModeRead)) r.Group("/arch", func() { + r.Put("/push/{filename}/{distro}", reqPackageAccess(perm.AccessModeWrite), arch.Push) r.Put("/push/{filename}/{distro}/{sign}", reqPackageAccess(perm.AccessModeWrite), arch.Push) r.Delete("/remove/{package}/{version}", reqPackageAccess(perm.AccessModeWrite), arch.Remove) r.Get("/{distro}/{arch}/{file}", arch.Get) diff --git a/tests/integration/api_packages_arch_test.go b/tests/integration/api_packages_arch_test.go index 22c2597a92e2..6488d7ecdbbb 100644 --- a/tests/integration/api_packages_arch_test.go +++ b/tests/integration/api_packages_arch_test.go @@ -4,261 +4,462 @@ package integration import ( + "bufio" "bytes" - "encoding/base64" + "crypto/md5" "encoding/hex" + "errors" "fmt" + "io" "net/http" + "os" "path" "testing" + "testing/fstest" + "time" "code.gitea.io/gitea/models/unittest" user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/packages/arch" "code.gitea.io/gitea/tests" + "github.com/mholt/archiver/v3" + "github.com/minio/sha256-simd" "github.com/stretchr/testify/assert" ) func TestPackageArch(t *testing.T) { - const firstPackage = `KLUv/QRYlUcA6nxwGD6gsLQNvOd6axb347rXM33zsZKpE9vgsf0Ld/n//99rY7sJi2vbpBtD3se3 -qPGsw5p1E4nc33ohr0LLfwiK+m8BUgFTAdxyVWq/0afytiwo9jGT9PRVrkKxE/tIJX36mOzYG7Wv -cveuyTpPM0VnlTUUv80UWkllNk8Tq7PKL8Sb6bbk0hjizOq2xCR97KvMnb1Ul3dztp2HuEjs2egb -uesZ3W+P2oIaN23sMXUNXg7nDnfbeRbFzuFsxDvR03h3fqkuHoq4a7Eo/DA3anWX1V1WZ7vOaItZ -7Lquc+wZXaorKbz152+3ZcFLdWVR7H3bzou+lj7al49HFxRZR5fq2piiNtQpNVMbvVTX7dYr2jGu -UEkMWyGNS3XtIovSVprGddxJsxidXmo1VxlGM5vR1Mmzp0yrp9XqrjQ7T/duKXuyM5t281IrLts8 -eqkucOSbVi2L5/AyHrMtx4ZaulQXq9v2a+kiKumluvZN4pb7WrrofrbVWX2prgva8Gvponqprm7B -La6uLop7TgDbdj7OC0B2wiqdFz1pWZbVLcvopWlbpRPAlWXbnBMAUgoYLE64YEGjxAfbByo5v3nW -p1LjdguTngi2fnvKdFse19roT9nJUOQZ/SpDkfPQ7tgqpRqjnn6gkt/M2a5x9pust4/i1vixkxy7 -VN1m16+0ocX1hP2UtltPiVXpU6lxEyZvxuO3WombPib13R+tT+UFP/3x3Mfwy75+lZ8eQw1+J+yr -7Dict9NXWTmc60/Zl27v98PqUwlyHudwll33VSq7rPu+viZbSvbLlm6en/rMC7Gn88HbORO/Ga+y -7DV5M15V7Knsorhvy6661Ut/yp5K/olb81W9kz62WmGvPja/yn21tqC8HU663nxNduzdb+Qu6jP6 -VPIRVR97TWKvcV7Fvv4mOU/Or4+SeY/d9lXqsV/eNdPGPnW3mHpM2pxxkwhqMzVrnVXy222HV7jv -20nboarGTZXK1M+nEucu7vqox2Qo8k3O7KvkGfcxxFwmNzqfviZZVOabdD1M3h3HINfzM/mFkutO -3FrUqKdfSzPtHt4Q6w17TeZ9dfaTynp3p3E3qdz+9FWyum71pLXOClqNvfZUglajOsd5i3tj8CRl -aMHO6sx5eLX4VLK6DlO937BqObLfJL98j11I9c+vn8m90mD2WYp+9lRmYCpryr9Jvq9nL0tibSvl -4e0o/SprBpc27LPPlh77KcW+bxKH875dg7ePHM6d43C0K1R36ech83D67KvMotg37in6maQhtx2V -PSbvHru2nVyJW3MmzuFcWZzPu7vLNSh23eWomrnlVE/baQ5n3JKUaixrWu+sL6y9hjj3G67CVmd3 -D0Oxy7Fvz3jVb8jhTKnsNKj9fCpzCHI4v5ctx1v1p8wcczhnNfTwFve+8tPfpE/lx+GcPSbxasXh -nH2VPbxJF0V/ylW+HM6on/ILbcfhrDkO5y20I+eWw/n0mLxJmNVty+GMBUXzUl23CXNR3320b6gk -HoFi9zdr0C7do6MPXPfOulRX3mMR3h2/3pXH1eU0LtXV8etdoYj5vknXF2I+XheYuMu73e4bXhXX -hVNGaz1hKlrE+AB5hBg/RgQQAsPFQRQFf26EsFG04gwPaqAAlSeClrQACD4s2bkwg9uzJgTSDBxE -FtQ41OHTIlDYgLMqcdA8UEMDwRsrbA7NJXgOzZkjYgKI+WWRoiYICyk45FRxM8NEkTxZBBBMAkDB -oKYNBuFCgA4zQxYlE5DOHLiyxaQw02JKmRNDJuCw00NGi+4ETQMCSLQM+gAmYOI1aACo0BUygUeY -OgxWZEmyBQYWMYEQIbLixRDSEi584GARQAZNjid9hiCgWnzQEdTkI4kVMnoUgJFwwNwp9CfCKQ2g -HU2iQOHwAwkpgAoeAnhs0CDoi44KEgQQiGFUoIuX1ptEZgCxGBPCEBAEJAbA+JLExwADRyzQeqLY -zOZEgULGjBfvdd5Ev3I4N2WvyaYlWjFLCfsqM+5jh9VZmzrlvMsK6n6S4irES7djT+W2HRS7JM7E -T49pVXr6M/qpp1L0Rl47zjw4/UwqiZ5l8c+kd/mcN8TZs3c/VmWBFaiRHKSYQggxNDMjBUmyHHEE -hCDonKSQDxJAUFGMQRAIQxAEQhCGolCgZsokSJCknBQWWgMMyBbo9PMBsFwQnPDPtvRb/dr0T6ca -rtWEfRFesPpr1z8Hfi1LxlZ7GHuAqLAB/NNkT2fx9grmc8euvTXOAFacJCHIHICRPX0dBGk3RXcV -O9Vbi1MpQyibg34fKNSG4gIAImaNUbIu0zfT9FobmWjMOOItiiByrLEAB6d8YsQZbHhLrsELnaDN -YwLqgJZWL5Vift1uHpg7gRyXkl2anG/zeyw1mIGTPwyZqFfredyRUiNx6NtKPFgwNYHNOPVomgJM -BaZp2FQUdJDBjT42M7a1kJEekgrdLcA1m0WTzMdEe+7KcX/e8MttkzdAsRKuH4mndiUf8jnMxYj8 -J80ggDEVFkFGIIOGiFOY0GDf8XoZ0rarlrzAoXFcSQfwQzQE6hBdPKxtrb3F5CUsCvwdwovq+VMV -5/sQEYCRaRsDohAoCw3tOMLg9pbOS+5ARZR9JDx5ScgyXOo5O6pbsBi1V7aja81BSEGZQFzHzpIv -NvXb/U5ZNldpCgWmyJfNGgiJJwIzzEgG5DChlpsh2W07AybfEiJWG8YZTMRChZk1dCnsTGxXdAKb -yEkYN9mks01OZ1gyVQaSakNg88h7upWYM40YIx4EJuyAwSP1yhfZPk6ZxNU3TrQygytGzwUxS2QO -IkJ4YHo8TXvkN5vq2aWjzeEJBFWQHV5rkrC1HkgK9Zk0B3RoSs2/mBmuI0iGSroNA/xAictf1DqT -0HVqqjQg1ACXRABLTDRjhUSa9kjdMcX9YypiMXY4iECBSl4oQeIQ112zLCppoz+HULQL+csQyVJc -CLajh0i2JJvzlFnmg2XIndtEIyTU7MyAiE8KO7SOxFc9mlYAyXKFFBFJkas615qu3PEr6bRA27AE -weALS7CuDOK6KF6YdYvOWdiCAoHI6xI=` - firstPackageData, err := base64.StdEncoding.DecodeString(firstPackage) - assert.NoError(t, err) + assert.NoError(t, unittest.PrepareTestDatabase()) - const firstPackageSignature = `iQEzBAABCAAdFiEEdez1wBMjH7w+s6HCOjpq1/IBK5QFAmTKScUACgkQOjpq1/IBK5RDjwf/YZQS -QM9JgxtNqp9jxT7eqyNtYY5Jwte6Fpq6RpOd2qbkJodJVrAp0HAWPS71W9k0lhvOSeq4hL7jufUR -y5gmbvmN6CqOjoMAnSxe51OKgZuPb8fbWrpt58BqtR7iCtPav1tMG9lpIPWSLS2/jxoTIxjcgVQJ -05s2bqUtpoDy5fCB2Y5tdIPQbMjSr6/TkmWg4ulwactJg46bWgowwKxnWzUx7IYPjC3lwknU6Mll -DArW/X0zMaFT3zuMBJFlbSzv59tcH0yICa1yMRtWCnbufZo6Q/BUvZ3P3Wr/APokAEt5U1U/u0EK -Qck04tbpECPL0eABIygaLmwqii6wX6NIYA==` - firstPackageSignatureData, err := base64.StdEncoding.DecodeString(firstPackageSignature) - assert.NoError(t, err) + var ( + firstV1x86_64 = BuildArchPackage(t, "first", "1-1", "x86_64") + firstV1i686 = BuildArchPackage(t, "first", "1-1", "i686") + secondV1any = BuildArchPackage(t, "second", "1-1", "any") + firstV2x86_64 = BuildArchPackage(t, "first", "2-1", "x86_64") + secondV2any = BuildArchPackage(t, "second", "2-1", "any") + + firstSign = []byte{1, 2, 3, 4} + secondSign = []byte{4, 3, 2, 1} + + V1x86_64database = BuildArchDatabase([]arch.Package{ + { + Name: "first", + Version: "1-1", + VersionMetadata: arch.VersionMetadata{ + Base: "first", + }, + FileMetadata: arch.FileMetadata{ + CompressedSize: firstV1x86_64.size, + MD5: firstV1x86_64.md5, + SHA256: firstV1x86_64.sha256, + Arch: "x86_64", + }, + }, + { + Name: "second", + Version: "1-1", + VersionMetadata: arch.VersionMetadata{ + Base: "second", + }, + FileMetadata: arch.FileMetadata{ + CompressedSize: secondV1any.size, + MD5: secondV1any.md5, + SHA256: secondV1any.sha256, + Arch: "any", + }, + }, + }) + V1i686database = BuildArchDatabase([]arch.Package{ + { + Name: "first", + Version: "1-1", + VersionMetadata: arch.VersionMetadata{ + Base: "first", + }, + FileMetadata: arch.FileMetadata{ + CompressedSize: firstV1i686.size, + MD5: firstV1i686.md5, + SHA256: firstV1i686.sha256, + Arch: "i686", + }, + }, + { + Name: "second", + Version: "1-1", + VersionMetadata: arch.VersionMetadata{ + Base: "second", + }, + FileMetadata: arch.FileMetadata{ + CompressedSize: secondV1any.size, + MD5: secondV1any.md5, + SHA256: secondV1any.sha256, + Arch: "any", + }, + }, + }) + + V2x86_64database = BuildArchDatabase([]arch.Package{ + { + Name: "first", + Version: "2-1", + VersionMetadata: arch.VersionMetadata{ + Base: "first", + }, + FileMetadata: arch.FileMetadata{ + CompressedSize: firstV2x86_64.size, + MD5: firstV2x86_64.md5, + SHA256: firstV2x86_64.sha256, + Arch: "i686", + }, + }, + { + Name: "second", + Version: "2-1", + VersionMetadata: arch.VersionMetadata{ + Base: "second", + }, + FileMetadata: arch.FileMetadata{ + CompressedSize: secondV2any.size, + MD5: secondV2any.md5, + SHA256: secondV2any.sha256, + Arch: "any", + }, + }, + }) + V2i686database = BuildArchDatabase([]arch.Package{ + { + Name: "first", + Version: "1-1", + VersionMetadata: arch.VersionMetadata{ + Base: "first", + }, + FileMetadata: arch.FileMetadata{ + CompressedSize: firstV1i686.size, + MD5: firstV1i686.md5, + SHA256: firstV1i686.sha256, + Arch: "i686", + }, + }, + { + Name: "second", + Version: "2-1", + VersionMetadata: arch.VersionMetadata{ + Base: "second", + }, + FileMetadata: arch.FileMetadata{ + CompressedSize: secondV2any.size, + MD5: secondV2any.md5, + SHA256: secondV2any.sha256, + Arch: "any", + }, + }, + }) + + user = unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) + rootURL = fmt.Sprintf("/api/packages/%s/arch", user.Name) + ) + + t.Run("Version_1", func(t *testing.T) { + t.Run("Push_first_x86_64", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + req := NewRequestWithBody(t, "PUT", + path.Join( + rootURL, "push", "first-1-1-x86_64.pkg.tar.zst", + "archlinux", hex.EncodeToString(firstSign), + ), + bytes.NewReader(firstV1x86_64.data), + ) + + req = AddBasicAuthHeader(req, user.Name) + + MakeRequest(t, req, http.StatusOK) + }) + + t.Run("Push_first_i686", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + req := NewRequestWithBody(t, "PUT", + path.Join( + rootURL, "push", "first-1-1-i686.pkg.tar.zst", + "archlinux", hex.EncodeToString(secondSign), + ), + bytes.NewReader(firstV1i686.data), + ) + + req = AddBasicAuthHeader(req, user.Name) + + MakeRequest(t, req, http.StatusOK) + }) + + t.Run("Push_second_any", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + req := NewRequestWithBody(t, "PUT", + path.Join(rootURL, "push", "second-1-1-any.pkg.tar.zst", "archlinux"), + bytes.NewReader(secondV1any.data), + ) + + req = AddBasicAuthHeader(req, user.Name) + + MakeRequest(t, req, http.StatusOK) + }) + + t.Run("Get_first_x86_64_package", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + req := NewRequest(t, "GET", + rootURL+"/archlinux/x86_64/first-1-1-x86_64.pkg.tar.zst", + ) + + resp := MakeRequest(t, req, http.StatusOK) + + assert.Equal(t, firstV1x86_64.data, resp.Body.Bytes()) + }) + + t.Run("Get_first_i686_package", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() - const firstPackageDatabase = `H4sIAAAAAAAA/+ySQW+cMBCF9zy/Yi9zBMZgbOiNAm1Qd9MISg+9VLbXTqNNAAFRov76it2qavZa -RVUkvst4rJH9/PwGZY7q1nrMY8HBTmbzChARSSlPlYguK8WC/Vmf9hmTXGy29BpiLnmcZjVu6J/v -unzcGwE/VLvyOtuXCH9FwXtOxHfB/eF4689q9H9OMwC+GAPA91nzov9a1k31+RqBeQwAi7LJEdrJ -usf77ZKt8W6Y7/oOAPOm+lYihBFFAFidOwLAfRE37R5BHwxLnYoY06lUWnCKJFfGakYsTEQIgM1V -FsbiNB0ro0MTkTah0TzRsSXhbHhwziklVWStIKYkiSTlSlspuBBSac0Sbo1zi9i23iH8mOdhehcE -9lk9DPfWN/1D0D91dgxGO/QAmNX5FcLZnMWAttoVRfalRGAipVQmIlkOu8nyT9nHskZou2PXP3Xb -m7NJI8D//vCVlZWV3/wKAAD//xmJdqQACAAA` - firstPackageDatabaseData, err := base64.StdEncoding.DecodeString(firstPackageDatabase) - assert.NoError(t, err) + req := NewRequest(t, "GET", + rootURL+"/archlinux/x86_64/first-1-1-i686.pkg.tar.zst", + ) - const secondPackage = `KLUv/QRYPUcA+nxYGD6gsLQN5PnPkjy6fXl2KEtLb63ASYEmmw08uPr///fa2G7C4lq3UGMDqi4V -ALcz1mwbtcj9A2UA4JPvxl6MF28BUgFTAVuVp9foU3lbFhT7mEmK+iq9UOTEPlJJnz4mO+7G01e5 -ez/JOlFTRWeVNRQ/ptBKKrOJmlidVX4h3ky3JZfGEGcWtzMNh5Zakpikj32VmbPX6upuzrbzEBuJ -PUf4xu12Ee63R21BjZs27pi6Bu+GM4e77TyLYt9wjsA5sdN4d36tLh6KuGvRKPwwj9DiLou7LM5y -XIQtZpHjuM5xF+FaXUnhrT9fuy0LXqsri2Lv23Zu9LX00b58PLqgyDq6VtfGKhQqhVJNlUav1XW7 -7Yz2iitUEsNWCONaXdtIqx52utxmOxQKq/NyqDlPk1NdT5upelNdVzN7563b9LTZ0aqqt6ObpZvK -qyltXqsLHLl2qpbFcXQZj0yhvVYXi9v2a2kjKum1uvZN4nb7WtrofrbFWX2trgva8Gtpo3qtrm7B -LXpXF8U9J4BtOx/nBYCisErnhWGoLMuqlmX0qiet0gngyjJtzglAxYgUJUIglYDx4QNUcn7zrE/l -adstTHYi2HrtKdNtdVtLoz8lJ0ORZ/SrDEXOQ6tDq5RqnHq6gUp+82a7xtlrst4+ilvjx1By5FRV -m1y/0oYWVxT2U9puOyVWpU+lxk2YvBmPn+eJmz4m9d0frU/lBT/98dzH8Mu+fpWfHkMNfijsq+Q2 -nDXUV1k3nOtP2Zdu7/fD6lMJbh3fcJYc91UquYz7vv5JtpTsly3dPF/1WRdiT+eDt3MmfjP2suxP -8mbsVeyp7KK4b8t63eqlR2VPJf/ErblX76SPeR7u6mPzq9xXawvKy+Gk280/yY67+43bTX1Gn0o+ -pupjf5LYn7auYl9fk1sn59dPybxHTvsq9dgv75ppY6+6W1Q9Jm3OuEkET1M1a51V8tsthz3c9+Wk -5VJV46ZKpernU4lzF3f91GMyFLkmZ/ZV8oz7GOItkxqdT/8kWVTmm3Q7TF5uAxjken4mv1ByzYlb -izr19Gtppt3DG2KtYX+SeV+d/aSy3s1pzE0qtUd9lSyuWz1prbOCVmN/eipBq1Od47zFvTGIkjK0 -IGd15jy8WnwqWV2Hqt5vWE84stckv3yPXEj1z6+fye1pMPtMRT97KjNQlTXl1yTft7OXJbG2lfLw -cpR+lTWDSxr22WdLj/2UYt83acN5367B28cN585xOFovxV36ZcgyoD77KrMo9o27in4machtT2WP -ybvHri0nPXFrzsQ3nCuL83k3d7kGRY67PFUzt5zqaTm94YxbklKN5dOpd9YX1l5DnPsNvbDF2d3D -UORw7Nsz9voNN5wplZwGTz+fyhyCG87fZbvxVv0pM8cbzlkNO7zFva/89DfpU/ltOGePSex5G87Z -V9nDm3RT9Kf08t1wTv2UX2i5DWe9bThroR05txvOqMfkTcIsbtsNZ0zbEEbzWl23CXNR3320b6gk -HoEi9zdr0C7do6MNXPfOulZX3qMR3hy/3ZVH73IY1+ri+O2uUMR836TrCzEfLwlM29XdbvcNL4lL -AnaiFaVhJE6s+AA5TI4VOACRlwlwBHRJZLDZFE5sMDLFEJgsTOio4BW2DUh4ggBFXPxMudLSYsTk -A0icCxiUSJAyxsqXITdYFLikk4ElMkhm+CjCgQOPgkS2KYD8kxTgBYiKBMlLnDcubCBJ05oRINWA -BtLop45XosUKC2AUOFlY6AmiQYQDzwmLPE5ScPCg0aEFyBkuIR7cQX0OliUtI0g4QEDKEkRMCB05 -7gbBe2KVlHwh8qKDDlHm+AgJNQmyJkoUpTc/BHR1zKjpU4jDEaArJ0H2jqEYcpyoeiQIEqgIEniA -gVIVN13ssNA19pTYEkBQlwWqECPmUElz6MgvKrNBWRSIAAIPA0yMacNDQAEhECqKYjObU4SIFi9Q -fNd5E/264dyU/Uk2LdGKWUrYV5lxHzmsztrUKeddVlB3lBS9EC/djj2V23ZQ5JI4E0c9dqqy05/R -Vz2VYjfy2nHewelnUknsLIt/JrvL57whzp29+7EqOR6BFKiRHKSYQggxNDMjBUmyHHEEhCDonKSQ -DxJAcFGKQRAIQxAGQhCGolCgZsoUFBSknNJCaxLIFuLu8wEs1xIq/L/t/VY/OP3dqYbLsrBfxw5W -cvD6s/Cv/cvYagebBwgUG8B/WPN0YPGWImZwya7tsnkAK6GnndwczJA9fR0EafdFdxk7lVqLXylD -l81lvycUSkMRAwBBs8Y2WZfpW2l6rR1MNGYc8RZFEDnFWLCDUwoxkgw2uiU3wUtO0PoxwXRAS6uf -SjG6bucH5mYgx0nJPk1O2/zuyxjMwMlfhkzEqzU17siikQD0bSkeNJiaYjP6/Z6mwFcBaBqWFQWF -ZXDvjs28bS1upEdSobsFeM1mwSQzmWjKXXncnxk+c9ukDVCEhOtU4ilNyQd2DqMwk/+sGQR0TAVE -kDHIoCPi1CQ0CHe8V4Y029VBXuBsHLfcAfwQDUUdlotXa9trfzGZCYsOf0d4UT9/CsX5HiKCsD1t -k0UUDmVhooUjEDzfknjJEKiIvo+EJ4cJeYZLNmfXdAsWo+XKdvSsOVApMBMI4dgh+YKmfufflOV+ -lV6hgBS5s1mDkPhE4MKMeHAOk2y5GZLdujOA+ZYnYsUwXsEELFSYWUuX8s5cdUVX2ASdhGmT7Tpz -cnrHkqk1wFTbAptj3tOthM9UxRjhINCwAwweiSv/yHY5ZR5XxzhRywxiGb0Q3CwROUgI4Yfp8TTt -kV/aVGSXqjaHJxCpIH14LUhiaz0jKdBnbA7UoSmYf+FmGI7gG6rlNgzwAyUuf1HrTCbXKVOlAKEe -XF4BLD3RTCsc0nRH2o5p6R9rEdvYiyBiDyqpogShQxzsmnGpBOiHtFETHT+pV6+EY8/bCavohoSI -88iZNYTdzZ3RRGogZlPW6YbdqMPzqp1J64BkWgkRjSRHVMMs/JVreyWdFvgalsYYrLAH1s+Ir4vi -hVm78J3RLCgQ9mBp` - secondPackageData, err := base64.StdEncoding.DecodeString(secondPackage) - assert.NoError(t, err) + resp := MakeRequest(t, req, http.StatusOK) - const secondPackageSignature = `iQEzBAABCAAdFiEEdez1wBMjH7w+s6HCOjpq1/IBK5QFAmTKSigACgkQOjpq1/IBK5TnkQgAotU2 -kdi275k87qzRLp4SgOX4QTpwCyjmXK4ZEm/FGBiF84mYT/sQmKbSsxPbxd4lumHhbll5SMVdM3C+ -1pB4kWT1fewQi1YukGEe+Na6SAa33yQqQThf30WPYJhuOxSNDX0DNCrR7Ei98hNq0FuvZkZWzepF -ylhUP+OSGPWFsVOnXCaANxW6457LtnNPeQFDwQL2y9qv0Hgpnn3KS09n0SPXT2Kr02iYu9rICoOB -KsqYqMGxHoPypwT24o6oDElGt9/Z0pqZcwRJc7C0npv3q5a17S7nAdo8/NWgfDri7w224s8odtWy -qTAeEmTTQ8awXFYounZaHg+455+8u5npkw==` - secondPackageSignatureData, err := base64.StdEncoding.DecodeString(secondPackageSignature) - assert.NoError(t, err) + assert.Equal(t, firstV1i686.data, resp.Body.Bytes()) + }) - const secondPackageDatabase = `H4sIAAAAAAAA/+zSzW7bMAwA4Jz5FLnwmJhyLMnazUu81VjSFfG8w276oboirW3YLrru6YemwLDk -OhTDgHwXigIBiRJ76w/2lhdiIZLAo5+9ASIirfUxEtF5JClXv9fHfSF0pmdzeovLnHscJzvM6K/P -Om/uP4Efqm15XexKhD9GYWHb52V/uF1Odlj+HCcAPKkBwPdFfZJ/Lfd19fkaQSwEAG7Keo3QjBwf -7+cvgzXc9dNd1wLguq6+lQhpalIArF4zAsDdRtbNDoFXPmfjNbsQg/UyzRWFXLAwdsUx8wBYXxWp -VMfqVfQZp9GYYJ0yLhB7zYG0IymEiuykCqlWlHpnQ5Y5qzgXLroQZRZUFgGw2W8Rvk9TP75LEv5h -H/p7XvruIemeWh6SgfsOAIv9+grBts8v3TfVdrMpvpQIQhkyOtfaAOBNsf5UfCz3CE17aLundn7z -+kIDwL/+6ouLi4sTvwIAAP//Stn2gwAIAAA=` - secondPackageDatabaseData, err := base64.StdEncoding.DecodeString(secondPackageDatabase) - assert.NoError(t, err) + t.Run("Get_first_x86_64_package_signature", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() - assert.NoError(t, unittest.PrepareTestDatabase()) + req := NewRequest(t, "GET", + rootURL+"/archlinux/x86_64/first-1-1-x86_64.pkg.tar.zst.sig", + ) - user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) + resp := MakeRequest(t, req, http.StatusOK) - rootURL := fmt.Sprintf("/api/packages/%s/arch", user.Name) + assert.Equal(t, firstSign, resp.Body.Bytes()) + }) - t.Run("PushFirstPackage", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("Get_first_i686_package_signature", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() - req := NewRequestWithBody(t, "PUT", - path.Join( - rootURL, "push", "package-1-1-x86_64.pkg.tar.zst", - "archlinux", hex.EncodeToString(firstPackageSignatureData), - ), - bytes.NewReader(firstPackageData), - ) + req := NewRequest(t, "GET", + rootURL+"/archlinux/x86_64/first-1-1-i686.pkg.tar.zst.sig", + ) - req = AddBasicAuthHeader(req, user.Name) + resp := MakeRequest(t, req, http.StatusOK) - MakeRequest(t, req, http.StatusOK) - }) + assert.Equal(t, secondSign, resp.Body.Bytes()) + }) - t.Run("GetFirstPackage", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("Get_any_package_from_x86_64_group", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() - req := NewRequest(t, "GET", - rootURL+"/archlinux/x86_64/package-1-1-x86_64.pkg.tar.zst", - ) + req := NewRequest(t, "GET", + rootURL+"/archlinux/x86_64/second-1-1-any.pkg.tar.zst", + ) - resp := MakeRequest(t, req, http.StatusOK) + resp := MakeRequest(t, req, http.StatusOK) - assert.Equal(t, firstPackageData, resp.Body.Bytes()) - }) + assert.Equal(t, secondV1any.data, resp.Body.Bytes()) + }) - t.Run("GetFirstDatabase", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("Get_any_package_from_i686_group", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() - req := NewRequest(t, "GET", rootURL+"/archlinux/x86_64/test.db") + req := NewRequest(t, "GET", + rootURL+"/archlinux/i686/second-1-1-any.pkg.tar.zst", + ) - resp := MakeRequest(t, req, http.StatusOK) + resp := MakeRequest(t, req, http.StatusOK) - assert.Equal(t, firstPackageDatabaseData, resp.Body.Bytes()) - }) + assert.Equal(t, secondV1any.data, resp.Body.Bytes()) + }) - t.Run("GetFirstSignature", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("Get_x86_64_pacman_database", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() - req := NewRequest(t, "GET", - rootURL+"/archlinux/x86_64/package-1-1-x86_64.pkg.tar.zst.sig", - ) + req := NewRequest(t, "GET", + rootURL+"/archlinux/x86_64/user.db.tar.gz", + ) - resp := MakeRequest(t, req, http.StatusOK) + resp := MakeRequest(t, req, http.StatusOK) - assert.Equal(t, firstPackageSignatureData, resp.Body.Bytes()) - }) + assert.Equal(t, V1x86_64database, resp.Body.Bytes()) + }) - t.Run("PushSecond", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("Get_i686_pacman_database", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() - req := NewRequestWithBody(t, "PUT", - path.Join( - rootURL, "push", "package-1-1-any.pkg.tar.zst", - "archlinux", hex.EncodeToString(secondPackageSignatureData), - ), - bytes.NewReader(secondPackageData), - ) + req := NewRequest(t, "GET", + rootURL+"/archlinux/i686/user.db.tar.gz", + ) - req = AddBasicAuthHeader(req, user.Name) + resp := MakeRequest(t, req, http.StatusOK) - MakeRequest(t, req, http.StatusOK) + assert.Equal(t, V1i686database, resp.Body.Bytes()) + }) }) - t.Run("GetSecondPackage", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("Version_2", func(t *testing.T) { + t.Run("Push_first_x86_64", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() - req := NewRequest(t, "GET", - rootURL+"/archlinux/any/package-1-1-any.pkg.tar.zst", - ) + req := NewRequestWithBody(t, "PUT", + path.Join(rootURL, "push", "first-2-1-x86_64.pkg.tar.zst", "archlinux"), + bytes.NewReader(firstV2x86_64.data), + ) - resp := MakeRequest(t, req, http.StatusOK) + req = AddBasicAuthHeader(req, user.Name) - assert.Equal(t, secondPackageData, resp.Body.Bytes()) - }) + MakeRequest(t, req, http.StatusOK) + }) - t.Run("GetSecondDatabase", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("Push_second_any", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() - req := NewRequest(t, "GET", rootURL+"/archlinux/any/localhost.db") + req := NewRequestWithBody(t, "PUT", + path.Join(rootURL, "push", "first-2-1-x86_64.pkg.tar.zst", "archlinux"), + bytes.NewReader(firstV1x86_64.data), + ) - resp := MakeRequest(t, req, http.StatusOK) + req = AddBasicAuthHeader(req, user.Name) - assert.Equal(t, secondPackageDatabaseData, resp.Body.Bytes()) - }) + MakeRequest(t, req, http.StatusOK) + }) + + t.Run("Get_x86_64_pacman_database", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + req := NewRequest(t, "GET", + rootURL+"/archlinux/x86_64/user2.db.tar.gz", + ) + + resp := MakeRequest(t, req, http.StatusOK) + + assert.Equal(t, V2x86_64database, resp.Body.Bytes()) + }) - t.Run("GetSecondSignature", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("Get_i686_pacman_database", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() - req := NewRequest(t, "GET", - rootURL+"/archlinux/any/package-1-1-any.pkg.tar.zst.sig", - ) + req := NewRequest(t, "GET", + rootURL+"/archlinux/i686/user2.db.tar.gz", + ) - resp := MakeRequest(t, req, http.StatusOK) + resp := MakeRequest(t, req, http.StatusOK) - assert.Equal(t, secondPackageSignatureData, resp.Body.Bytes()) + assert.Equal(t, V2i686database, resp.Body.Bytes()) + }) }) +} - t.Run("Remove", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() +type testArchPackage struct { + data []byte + md5 string + sha256 string + size int64 +} - req := NewRequest(t, "DELETE", rootURL+"/remove/package/1-1") +func BuildArchPackage(t *testing.T, name, ver, arch string) testArchPackage { + fs := fstest.MapFS{ + "pkginfo": &fstest.MapFile{ + Data: []byte(fmt.Sprintf( + "pkgname = %s\npkgbase = %s\npkgver = %s\narch = %s\n", + name, name, ver, arch, + )), + Mode: os.ModePerm, + ModTime: time.Now(), + }, + "mtree": &fstest.MapFile{ + Data: []byte("test"), + Mode: os.ModePerm, + ModTime: time.Now(), + }, + } + + pinf, err := fs.Stat("pkginfo") + assert.NoError(t, err) - req = AddBasicAuthHeader(req, user.Name) + pfile, err := fs.Open("pkginfo") + assert.NoError(t, err) - MakeRequest(t, req, http.StatusOK) + parcname, err := archiver.NameInArchive(pinf, ".PKGINFO", ".PKGINFO") + assert.NoError(t, err) + + minf, err := fs.Stat("mtree") + assert.NoError(t, err) + + mfile, err := fs.Open("mtree") + assert.NoError(t, err) + + marcname, err := archiver.NameInArchive(minf, ".MTREE", ".MTREE") + assert.NoError(t, err) + + var buf bytes.Buffer + + archive := archiver.NewTarZstd() + archive.Create(&buf) + + err = archive.Write(archiver.File{ + FileInfo: archiver.FileInfo{ + FileInfo: pinf, + CustomName: parcname, + }, + ReadCloser: pfile, }) + assert.NoError(t, errors.Join(pfile.Close(), err)) + + err = archive.Write(archiver.File{ + FileInfo: archiver.FileInfo{ + FileInfo: minf, + CustomName: marcname, + }, + ReadCloser: mfile, + }) + assert.NoError(t, errors.Join(mfile.Close(), archive.Close(), err)) + + md5, sha256, size := archPkgParams(buf.Bytes()) + + return testArchPackage{ + data: buf.Bytes(), + md5: hex.EncodeToString(md5), + sha256: hex.EncodeToString(sha256), + size: size, + } +} + +func archPkgParams(b []byte) ([]byte, []byte, int64) { + md5 := md5.New() + sha256 := sha256.New() + c := counter{bytes.NewReader(b), 0} + + br := bufio.NewReader(io.TeeReader(&c, io.MultiWriter(md5, sha256))) + + io.ReadAll(br) + return md5.Sum(nil), sha256.Sum(nil), int64(c.n) +} + +type counter struct { + io.Reader + n int +} + +func (w *counter) Read(p []byte) (int, error) { + n, err := w.Reader.Read(p) + w.n += n + return n, err +} + +func BuildArchDatabase(pkgs []arch.Package) []byte { + entries := map[string][]byte{} + for _, p := range pkgs { + entries[fmt.Sprintf("%s-%s/desc", p.Name, p.Version)] = []byte(p.Desc()) + } + b, err := arch.CreatePacmanDb(entries) + if err != nil { + panic(err) + } + return b.Bytes() } From 184020775f86a7894a2a02dfb701b19b69b6f5ae Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Tue, 31 Oct 2023 19:53:43 -0300 Subject: [PATCH 104/124] refactoring arch package integration tests --- tests/integration/api_packages_arch_test.go | 208 ++++++-------------- 1 file changed, 60 insertions(+), 148 deletions(-) diff --git a/tests/integration/api_packages_arch_test.go b/tests/integration/api_packages_arch_test.go index 6488d7ecdbbb..59962040c239 100644 --- a/tests/integration/api_packages_arch_test.go +++ b/tests/integration/api_packages_arch_test.go @@ -32,127 +32,31 @@ func TestPackageArch(t *testing.T) { assert.NoError(t, unittest.PrepareTestDatabase()) var ( - firstV1x86_64 = BuildArchPackage(t, "first", "1-1", "x86_64") - firstV1i686 = BuildArchPackage(t, "first", "1-1", "i686") - secondV1any = BuildArchPackage(t, "second", "1-1", "any") - firstV2x86_64 = BuildArchPackage(t, "first", "2-1", "x86_64") - secondV2any = BuildArchPackage(t, "second", "2-1", "any") + gitV1x86_64 = BuildArchPackage(t, "git", "1-1", "x86_64") + gitV1i686 = BuildArchPackage(t, "git", "1-1", "i686") + iconsV1any = BuildArchPackage(t, "icons", "1-1", "any") + gitV2x86_64 = BuildArchPackage(t, "git", "2-1", "x86_64") + iconsV2any = BuildArchPackage(t, "icons", "2-1", "any") firstSign = []byte{1, 2, 3, 4} secondSign = []byte{4, 3, 2, 1} V1x86_64database = BuildArchDatabase([]arch.Package{ - { - Name: "first", - Version: "1-1", - VersionMetadata: arch.VersionMetadata{ - Base: "first", - }, - FileMetadata: arch.FileMetadata{ - CompressedSize: firstV1x86_64.size, - MD5: firstV1x86_64.md5, - SHA256: firstV1x86_64.sha256, - Arch: "x86_64", - }, - }, - { - Name: "second", - Version: "1-1", - VersionMetadata: arch.VersionMetadata{ - Base: "second", - }, - FileMetadata: arch.FileMetadata{ - CompressedSize: secondV1any.size, - MD5: secondV1any.md5, - SHA256: secondV1any.sha256, - Arch: "any", - }, - }, + gitV1x86_64.pkg, + iconsV1any.pkg, }) V1i686database = BuildArchDatabase([]arch.Package{ - { - Name: "first", - Version: "1-1", - VersionMetadata: arch.VersionMetadata{ - Base: "first", - }, - FileMetadata: arch.FileMetadata{ - CompressedSize: firstV1i686.size, - MD5: firstV1i686.md5, - SHA256: firstV1i686.sha256, - Arch: "i686", - }, - }, - { - Name: "second", - Version: "1-1", - VersionMetadata: arch.VersionMetadata{ - Base: "second", - }, - FileMetadata: arch.FileMetadata{ - CompressedSize: secondV1any.size, - MD5: secondV1any.md5, - SHA256: secondV1any.sha256, - Arch: "any", - }, - }, + gitV1i686.pkg, + iconsV1any.pkg, }) V2x86_64database = BuildArchDatabase([]arch.Package{ - { - Name: "first", - Version: "2-1", - VersionMetadata: arch.VersionMetadata{ - Base: "first", - }, - FileMetadata: arch.FileMetadata{ - CompressedSize: firstV2x86_64.size, - MD5: firstV2x86_64.md5, - SHA256: firstV2x86_64.sha256, - Arch: "i686", - }, - }, - { - Name: "second", - Version: "2-1", - VersionMetadata: arch.VersionMetadata{ - Base: "second", - }, - FileMetadata: arch.FileMetadata{ - CompressedSize: secondV2any.size, - MD5: secondV2any.md5, - SHA256: secondV2any.sha256, - Arch: "any", - }, - }, + gitV2x86_64.pkg, + iconsV2any.pkg, }) V2i686database = BuildArchDatabase([]arch.Package{ - { - Name: "first", - Version: "1-1", - VersionMetadata: arch.VersionMetadata{ - Base: "first", - }, - FileMetadata: arch.FileMetadata{ - CompressedSize: firstV1i686.size, - MD5: firstV1i686.md5, - SHA256: firstV1i686.sha256, - Arch: "i686", - }, - }, - { - Name: "second", - Version: "2-1", - VersionMetadata: arch.VersionMetadata{ - Base: "second", - }, - FileMetadata: arch.FileMetadata{ - CompressedSize: secondV2any.size, - MD5: secondV2any.md5, - SHA256: secondV2any.sha256, - Arch: "any", - }, - }, + gitV1i686.pkg, + iconsV2any.pkg, }) user = unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) @@ -160,15 +64,15 @@ func TestPackageArch(t *testing.T) { ) t.Run("Version_1", func(t *testing.T) { - t.Run("Push_first_x86_64", func(t *testing.T) { + t.Run("Push_git_x86_64", func(t *testing.T) { defer tests.PrintCurrentTest(t)() req := NewRequestWithBody(t, "PUT", path.Join( - rootURL, "push", "first-1-1-x86_64.pkg.tar.zst", + rootURL, "push", "git-1-1-x86_64.pkg.tar.zst", "archlinux", hex.EncodeToString(firstSign), ), - bytes.NewReader(firstV1x86_64.data), + bytes.NewReader(gitV1x86_64.data), ) req = AddBasicAuthHeader(req, user.Name) @@ -176,15 +80,15 @@ func TestPackageArch(t *testing.T) { MakeRequest(t, req, http.StatusOK) }) - t.Run("Push_first_i686", func(t *testing.T) { + t.Run("Push_git_i686", func(t *testing.T) { defer tests.PrintCurrentTest(t)() req := NewRequestWithBody(t, "PUT", path.Join( - rootURL, "push", "first-1-1-i686.pkg.tar.zst", + rootURL, "push", "git-1-1-i686.pkg.tar.zst", "archlinux", hex.EncodeToString(secondSign), ), - bytes.NewReader(firstV1i686.data), + bytes.NewReader(gitV1i686.data), ) req = AddBasicAuthHeader(req, user.Name) @@ -192,12 +96,12 @@ func TestPackageArch(t *testing.T) { MakeRequest(t, req, http.StatusOK) }) - t.Run("Push_second_any", func(t *testing.T) { + t.Run("Push_icons_any", func(t *testing.T) { defer tests.PrintCurrentTest(t)() req := NewRequestWithBody(t, "PUT", - path.Join(rootURL, "push", "second-1-1-any.pkg.tar.zst", "archlinux"), - bytes.NewReader(secondV1any.data), + path.Join(rootURL, "push", "icons-1-1-any.pkg.tar.zst", "archlinux"), + bytes.NewReader(iconsV1any.data), ) req = AddBasicAuthHeader(req, user.Name) @@ -205,35 +109,35 @@ func TestPackageArch(t *testing.T) { MakeRequest(t, req, http.StatusOK) }) - t.Run("Get_first_x86_64_package", func(t *testing.T) { + t.Run("Get_git_x86_64_package", func(t *testing.T) { defer tests.PrintCurrentTest(t)() req := NewRequest(t, "GET", - rootURL+"/archlinux/x86_64/first-1-1-x86_64.pkg.tar.zst", + rootURL+"/archlinux/x86_64/git-1-1-x86_64.pkg.tar.zst", ) resp := MakeRequest(t, req, http.StatusOK) - assert.Equal(t, firstV1x86_64.data, resp.Body.Bytes()) + assert.Equal(t, gitV1x86_64.data, resp.Body.Bytes()) }) - t.Run("Get_first_i686_package", func(t *testing.T) { + t.Run("Get_git_i686_package", func(t *testing.T) { defer tests.PrintCurrentTest(t)() req := NewRequest(t, "GET", - rootURL+"/archlinux/x86_64/first-1-1-i686.pkg.tar.zst", + rootURL+"/archlinux/i686/git-1-1-i686.pkg.tar.zst", ) resp := MakeRequest(t, req, http.StatusOK) - assert.Equal(t, firstV1i686.data, resp.Body.Bytes()) + assert.Equal(t, gitV1i686.data, resp.Body.Bytes()) }) - t.Run("Get_first_x86_64_package_signature", func(t *testing.T) { + t.Run("Get_git_x86_64_package_signature", func(t *testing.T) { defer tests.PrintCurrentTest(t)() req := NewRequest(t, "GET", - rootURL+"/archlinux/x86_64/first-1-1-x86_64.pkg.tar.zst.sig", + rootURL+"/archlinux/x86_64/git-1-1-x86_64.pkg.tar.zst.sig", ) resp := MakeRequest(t, req, http.StatusOK) @@ -241,11 +145,11 @@ func TestPackageArch(t *testing.T) { assert.Equal(t, firstSign, resp.Body.Bytes()) }) - t.Run("Get_first_i686_package_signature", func(t *testing.T) { + t.Run("Get_git_i686_package_signature", func(t *testing.T) { defer tests.PrintCurrentTest(t)() req := NewRequest(t, "GET", - rootURL+"/archlinux/x86_64/first-1-1-i686.pkg.tar.zst.sig", + rootURL+"/archlinux/i686/git-1-1-i686.pkg.tar.zst.sig", ) resp := MakeRequest(t, req, http.StatusOK) @@ -257,24 +161,24 @@ func TestPackageArch(t *testing.T) { defer tests.PrintCurrentTest(t)() req := NewRequest(t, "GET", - rootURL+"/archlinux/x86_64/second-1-1-any.pkg.tar.zst", + rootURL+"/archlinux/x86_64/icons-1-1-any.pkg.tar.zst", ) resp := MakeRequest(t, req, http.StatusOK) - assert.Equal(t, secondV1any.data, resp.Body.Bytes()) + assert.Equal(t, iconsV1any.data, resp.Body.Bytes()) }) t.Run("Get_any_package_from_i686_group", func(t *testing.T) { defer tests.PrintCurrentTest(t)() req := NewRequest(t, "GET", - rootURL+"/archlinux/i686/second-1-1-any.pkg.tar.zst", + rootURL+"/archlinux/i686/icons-1-1-any.pkg.tar.zst", ) resp := MakeRequest(t, req, http.StatusOK) - assert.Equal(t, secondV1any.data, resp.Body.Bytes()) + assert.Equal(t, iconsV1any.data, resp.Body.Bytes()) }) t.Run("Get_x86_64_pacman_database", func(t *testing.T) { @@ -303,12 +207,12 @@ func TestPackageArch(t *testing.T) { }) t.Run("Version_2", func(t *testing.T) { - t.Run("Push_first_x86_64", func(t *testing.T) { + t.Run("Push_git_x86_64", func(t *testing.T) { defer tests.PrintCurrentTest(t)() req := NewRequestWithBody(t, "PUT", - path.Join(rootURL, "push", "first-2-1-x86_64.pkg.tar.zst", "archlinux"), - bytes.NewReader(firstV2x86_64.data), + path.Join(rootURL, "push", "git-2-1-x86_64.pkg.tar.zst", "archlinux"), + bytes.NewReader(gitV2x86_64.data), ) req = AddBasicAuthHeader(req, user.Name) @@ -316,12 +220,12 @@ func TestPackageArch(t *testing.T) { MakeRequest(t, req, http.StatusOK) }) - t.Run("Push_second_any", func(t *testing.T) { + t.Run("Push_icons_any", func(t *testing.T) { defer tests.PrintCurrentTest(t)() req := NewRequestWithBody(t, "PUT", - path.Join(rootURL, "push", "first-2-1-x86_64.pkg.tar.zst", "archlinux"), - bytes.NewReader(firstV1x86_64.data), + path.Join(rootURL, "push", "icons-2-1-any.pkg.tar.zst", "archlinux"), + bytes.NewReader(iconsV2any.data), ) req = AddBasicAuthHeader(req, user.Name) @@ -356,18 +260,16 @@ func TestPackageArch(t *testing.T) { } type testArchPackage struct { - data []byte - md5 string - sha256 string - size int64 + data []byte + pkg arch.Package } -func BuildArchPackage(t *testing.T, name, ver, arch string) testArchPackage { +func BuildArchPackage(t *testing.T, name, ver, architecture string) testArchPackage { fs := fstest.MapFS{ "pkginfo": &fstest.MapFile{ Data: []byte(fmt.Sprintf( "pkgname = %s\npkgbase = %s\npkgver = %s\narch = %s\n", - name, name, ver, arch, + name, name, ver, architecture, )), Mode: os.ModePerm, ModTime: time.Now(), @@ -423,10 +325,20 @@ func BuildArchPackage(t *testing.T, name, ver, arch string) testArchPackage { md5, sha256, size := archPkgParams(buf.Bytes()) return testArchPackage{ - data: buf.Bytes(), - md5: hex.EncodeToString(md5), - sha256: hex.EncodeToString(sha256), - size: size, + data: buf.Bytes(), + pkg: arch.Package{ + Name: name, + Version: ver, + VersionMetadata: arch.VersionMetadata{ + Base: name, + }, + FileMetadata: arch.FileMetadata{ + CompressedSize: size, + MD5: hex.EncodeToString(md5), + SHA256: hex.EncodeToString(sha256), + Arch: architecture, + }, + }, } } From 6652fbaee1ccac1157481478645b2f3f55a4b758 Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Tue, 31 Oct 2023 20:23:49 -0300 Subject: [PATCH 105/124] remove endpoint tests --- tests/integration/api_packages_arch_test.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/integration/api_packages_arch_test.go b/tests/integration/api_packages_arch_test.go index 59962040c239..c4481a4561cc 100644 --- a/tests/integration/api_packages_arch_test.go +++ b/tests/integration/api_packages_arch_test.go @@ -256,6 +256,24 @@ func TestPackageArch(t *testing.T) { assert.Equal(t, V2i686database, resp.Body.Bytes()) }) + + t.Run("Remove_v2_git_x86_64", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + req := NewRequest(t, "DELETE", rootURL+"/remove/git/2-1") + req = AddBasicAuthHeader(req, user.Name) + + MakeRequest(t, req, http.StatusOK) + }) + + t.Run("Remove_v2_icons_any", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + req := NewRequest(t, "DELETE", rootURL+"/remove/icons/2-1") + req = AddBasicAuthHeader(req, user.Name) + + MakeRequest(t, req, http.StatusOK) + }) }) } From 5c22b915bb55dfa8850a6a20a60901024cdcf419 Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Tue, 31 Oct 2023 20:36:08 -0300 Subject: [PATCH 106/124] lint corrections --- modules/packages/arch/metadata.go | 1 + modules/packages/arch/metadata_test.go | 2 +- tests/integration/api_packages_arch_test.go | 12 ++++++------ 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/modules/packages/arch/metadata.go b/modules/packages/arch/metadata.go index c64b0c5a2b2a..379fd81f67bc 100644 --- a/modules/packages/arch/metadata.go +++ b/modules/packages/arch/metadata.go @@ -19,6 +19,7 @@ import ( "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/modules/validation" + "github.com/mholt/archiver/v3" ) diff --git a/modules/packages/arch/metadata_test.go b/modules/packages/arch/metadata_test.go index b08c4b416394..0f3cede9ccff 100644 --- a/modules/packages/arch/metadata_test.go +++ b/modules/packages/arch/metadata_test.go @@ -175,7 +175,7 @@ backup = usr/bin/paket1 } func TestValidatePackageSpec(t *testing.T) { - var newpkg = func() Package { + newpkg := func() Package { return Package{ Name: "abc", Version: "1-1", diff --git a/tests/integration/api_packages_arch_test.go b/tests/integration/api_packages_arch_test.go index c4481a4561cc..15809215a865 100644 --- a/tests/integration/api_packages_arch_test.go +++ b/tests/integration/api_packages_arch_test.go @@ -32,11 +32,11 @@ func TestPackageArch(t *testing.T) { assert.NoError(t, unittest.PrepareTestDatabase()) var ( - gitV1x86_64 = BuildArchPackage(t, "git", "1-1", "x86_64") - gitV1i686 = BuildArchPackage(t, "git", "1-1", "i686") - iconsV1any = BuildArchPackage(t, "icons", "1-1", "any") - gitV2x86_64 = BuildArchPackage(t, "git", "2-1", "x86_64") - iconsV2any = BuildArchPackage(t, "icons", "2-1", "any") + gitV1x86_64 = buildArchPackage(t, "git", "1-1", "x86_64") + gitV1i686 = buildArchPackage(t, "git", "1-1", "i686") + iconsV1any = buildArchPackage(t, "icons", "1-1", "any") + gitV2x86_64 = buildArchPackage(t, "git", "2-1", "x86_64") + iconsV2any = buildArchPackage(t, "icons", "2-1", "any") firstSign = []byte{1, 2, 3, 4} secondSign = []byte{4, 3, 2, 1} @@ -282,7 +282,7 @@ type testArchPackage struct { pkg arch.Package } -func BuildArchPackage(t *testing.T, name, ver, architecture string) testArchPackage { +func buildArchPackage(t *testing.T, name, ver, architecture string) testArchPackage { fs := fstest.MapFS{ "pkginfo": &fstest.MapFile{ Data: []byte(fmt.Sprintf( From 1e972d7504ac1b4b0eaa0b6167d3184b8eb5567a Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Tue, 31 Oct 2023 21:56:11 -0300 Subject: [PATCH 107/124] unified approach to compare pacman databases (independed from archiver cache) --- tests/integration/api_packages_arch_test.go | 91 ++++++++++++++------- 1 file changed, 60 insertions(+), 31 deletions(-) diff --git a/tests/integration/api_packages_arch_test.go b/tests/integration/api_packages_arch_test.go index 15809215a865..ef55b57b6c7a 100644 --- a/tests/integration/api_packages_arch_test.go +++ b/tests/integration/api_packages_arch_test.go @@ -4,8 +4,10 @@ package integration import ( + "archive/tar" "bufio" "bytes" + "compress/gzip" "crypto/md5" "encoding/hex" "errors" @@ -32,33 +34,20 @@ func TestPackageArch(t *testing.T) { assert.NoError(t, unittest.PrepareTestDatabase()) var ( - gitV1x86_64 = buildArchPackage(t, "git", "1-1", "x86_64") - gitV1i686 = buildArchPackage(t, "git", "1-1", "i686") - iconsV1any = buildArchPackage(t, "icons", "1-1", "any") - gitV2x86_64 = buildArchPackage(t, "git", "2-1", "x86_64") - iconsV2any = buildArchPackage(t, "icons", "2-1", "any") + gitV1x86_64 = BuildArchPackage(t, "git", "1-1", "x86_64") + gitV1i686 = BuildArchPackage(t, "git", "1-1", "i686") + iconsV1any = BuildArchPackage(t, "icons", "1-1", "any") + gitV2x86_64 = BuildArchPackage(t, "git", "2-1", "x86_64") + iconsV2any = BuildArchPackage(t, "icons", "2-1", "any") + + dbV1x86_64d = BuildArchDb([]arch.Package{gitV1x86_64.pkg, iconsV1any.pkg}) + dbV1i686 = BuildArchDb([]arch.Package{gitV1i686.pkg, iconsV1any.pkg}) + dbV2x86_64 = BuildArchDb([]arch.Package{gitV2x86_64.pkg, iconsV2any.pkg}) + dbV2i686 = BuildArchDb([]arch.Package{gitV1i686.pkg, iconsV2any.pkg}) firstSign = []byte{1, 2, 3, 4} secondSign = []byte{4, 3, 2, 1} - V1x86_64database = BuildArchDatabase([]arch.Package{ - gitV1x86_64.pkg, - iconsV1any.pkg, - }) - V1i686database = BuildArchDatabase([]arch.Package{ - gitV1i686.pkg, - iconsV1any.pkg, - }) - - V2x86_64database = BuildArchDatabase([]arch.Package{ - gitV2x86_64.pkg, - iconsV2any.pkg, - }) - V2i686database = BuildArchDatabase([]arch.Package{ - gitV1i686.pkg, - iconsV2any.pkg, - }) - user = unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) rootURL = fmt.Sprintf("/api/packages/%s/arch", user.Name) ) @@ -190,7 +179,7 @@ func TestPackageArch(t *testing.T) { resp := MakeRequest(t, req, http.StatusOK) - assert.Equal(t, V1x86_64database, resp.Body.Bytes()) + assert.True(t, CompareDbEntries(dbV1x86_64d, resp.Body.Bytes())) }) t.Run("Get_i686_pacman_database", func(t *testing.T) { @@ -202,7 +191,7 @@ func TestPackageArch(t *testing.T) { resp := MakeRequest(t, req, http.StatusOK) - assert.Equal(t, V1i686database, resp.Body.Bytes()) + assert.True(t, CompareDbEntries(dbV1i686, resp.Body.Bytes())) }) }) @@ -242,7 +231,7 @@ func TestPackageArch(t *testing.T) { resp := MakeRequest(t, req, http.StatusOK) - assert.Equal(t, V2x86_64database, resp.Body.Bytes()) + assert.True(t, CompareDbEntries(dbV2x86_64, resp.Body.Bytes())) }) t.Run("Get_i686_pacman_database", func(t *testing.T) { @@ -254,7 +243,7 @@ func TestPackageArch(t *testing.T) { resp := MakeRequest(t, req, http.StatusOK) - assert.Equal(t, V2i686database, resp.Body.Bytes()) + assert.True(t, CompareDbEntries(dbV2i686, resp.Body.Bytes())) }) t.Run("Remove_v2_git_x86_64", func(t *testing.T) { @@ -277,12 +266,12 @@ func TestPackageArch(t *testing.T) { }) } -type testArchPackage struct { +type TestArchPackage struct { data []byte pkg arch.Package } -func buildArchPackage(t *testing.T, name, ver, architecture string) testArchPackage { +func BuildArchPackage(t *testing.T, name, ver, architecture string) TestArchPackage { fs := fstest.MapFS{ "pkginfo": &fstest.MapFile{ Data: []byte(fmt.Sprintf( @@ -342,7 +331,7 @@ func buildArchPackage(t *testing.T, name, ver, architecture string) testArchPack md5, sha256, size := archPkgParams(buf.Bytes()) - return testArchPackage{ + return TestArchPackage{ data: buf.Bytes(), pkg: arch.Package{ Name: name, @@ -382,7 +371,7 @@ func (w *counter) Read(p []byte) (int, error) { return n, err } -func BuildArchDatabase(pkgs []arch.Package) []byte { +func BuildArchDb(pkgs []arch.Package) []byte { entries := map[string][]byte{} for _, p := range pkgs { entries[fmt.Sprintf("%s-%s/desc", p.Name, p.Version)] = []byte(p.Desc()) @@ -393,3 +382,43 @@ func BuildArchDatabase(pkgs []arch.Package) []byte { } return b.Bytes() } + +func CompareDbEntries(first, second []byte) bool { + fgz, err := gzip.NewReader(bytes.NewReader(first)) + if err != nil { + return false + } + ftar := tar.NewReader(fgz) + + validatemap := map[string]struct{}{} + + for { + h, err := ftar.Next() + if err != nil { + break + } + + validatemap[h.Name] = struct{}{} + } + + sgz, err := gzip.NewReader(bytes.NewReader(second)) + if err != nil { + return false + } + star := tar.NewReader(sgz) + + for { + h, err := star.Next() + if err != nil { + break + } + + _, ok := validatemap[h.Name] + if !ok { + return false + } + delete(validatemap, h.Name) + } + + return len(validatemap) == 0 +} From fc8f807a30a1ded5cb504dc9b729518ec5e26ee1 Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Wed, 1 Nov 2023 11:15:30 -0300 Subject: [PATCH 108/124] comparison of pacman database entries in integration tests --- tests/integration/api_packages_arch_test.go | 26 ++++++++++++++------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/tests/integration/api_packages_arch_test.go b/tests/integration/api_packages_arch_test.go index ef55b57b6c7a..57304df1b447 100644 --- a/tests/integration/api_packages_arch_test.go +++ b/tests/integration/api_packages_arch_test.go @@ -179,7 +179,7 @@ func TestPackageArch(t *testing.T) { resp := MakeRequest(t, req, http.StatusOK) - assert.True(t, CompareDbEntries(dbV1x86_64d, resp.Body.Bytes())) + CompareDbEntries(t, dbV1x86_64d, resp.Body.Bytes()) }) t.Run("Get_i686_pacman_database", func(t *testing.T) { @@ -191,7 +191,7 @@ func TestPackageArch(t *testing.T) { resp := MakeRequest(t, req, http.StatusOK) - assert.True(t, CompareDbEntries(dbV1i686, resp.Body.Bytes())) + CompareDbEntries(t, dbV1i686, resp.Body.Bytes()) }) }) @@ -231,7 +231,7 @@ func TestPackageArch(t *testing.T) { resp := MakeRequest(t, req, http.StatusOK) - assert.True(t, CompareDbEntries(dbV2x86_64, resp.Body.Bytes())) + CompareDbEntries(t, dbV2x86_64, resp.Body.Bytes()) }) t.Run("Get_i686_pacman_database", func(t *testing.T) { @@ -243,7 +243,7 @@ func TestPackageArch(t *testing.T) { resp := MakeRequest(t, req, http.StatusOK) - assert.True(t, CompareDbEntries(dbV2i686, resp.Body.Bytes())) + CompareDbEntries(t, dbV2i686, resp.Body.Bytes()) }) t.Run("Remove_v2_git_x86_64", func(t *testing.T) { @@ -383,10 +383,11 @@ func BuildArchDb(pkgs []arch.Package) []byte { return b.Bytes() } -func CompareDbEntries(first, second []byte) bool { +func CompareDbEntries(t *testing.T, first, second []byte) { fgz, err := gzip.NewReader(bytes.NewReader(first)) if err != nil { - return false + assert.NoError(t, err) + return } ftar := tar.NewReader(fgz) @@ -403,7 +404,8 @@ func CompareDbEntries(first, second []byte) bool { sgz, err := gzip.NewReader(bytes.NewReader(second)) if err != nil { - return false + assert.NoError(t, err) + return } star := tar.NewReader(sgz) @@ -415,10 +417,16 @@ func CompareDbEntries(first, second []byte) bool { _, ok := validatemap[h.Name] if !ok { - return false + assert.Fail(t, "Entry not found in first archive: "+h.Name) } delete(validatemap, h.Name) } - return len(validatemap) == 0 + if len(validatemap) == 0 { + return + } + + for e := range validatemap { + assert.Fail(t, "Entry not found in second archive: "+e) + } } From d6a05c30d274cc4f96055d01381f5aad1308fd81 Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Wed, 1 Nov 2023 15:24:40 -0300 Subject: [PATCH 109/124] moved package properties to constants in metadata module --- modules/packages/arch/metadata.go | 5 +++++ services/packages/arch/service.go | 4 ++-- services/packages/arch/upload.go | 4 ++-- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/modules/packages/arch/metadata.go b/modules/packages/arch/metadata.go index 379fd81f67bc..dc51956aef18 100644 --- a/modules/packages/arch/metadata.go +++ b/modules/packages/arch/metadata.go @@ -23,6 +23,11 @@ import ( "github.com/mholt/archiver/v3" ) +const ( + PropertyDescription = "arch.description" + PropertySignature = "arch.signature" +) + var ( // https://man.archlinux.org/man/PKGBUILD.5 reName = regexp.MustCompile(`^[a-zA-Z0-9@._+-]+$`) diff --git a/services/packages/arch/service.go b/services/packages/arch/service.go index aa11a73048ad..a5859009c05c 100644 --- a/services/packages/arch/service.go +++ b/services/packages/arch/service.go @@ -44,7 +44,7 @@ func GetPackageSignature(ctx *context.Context, distro, file string) (*bytes.Read } for _, pp := range proprs { - if pp.Name == "sign" { + if pp.Name == arch_module.PropertySignature { b, err := hex.DecodeString(pp.Value) if err != nil { return nil, err @@ -116,7 +116,7 @@ func CreatePacmanDb(ctx *context.Context, owner, arch, distro string) (*bytes.Bu } pps, err := packages_model.GetPropertiesByName( - ctx, packages_model.PropertyTypeFile, pf.ID, "desc", + ctx, packages_model.PropertyTypeFile, pf.ID, arch_module.PropertyDescription, ) if err != nil { return nil, err diff --git a/services/packages/arch/upload.go b/services/packages/arch/upload.go index 654bbebf9b37..0a5e7eb245bb 100644 --- a/services/packages/arch/upload.go +++ b/services/packages/arch/upload.go @@ -37,14 +37,14 @@ func UploadArchPackage(ctx *context.Context, upload io.Reader, filename, distro, } properties := map[string]string{ - "desc": p.Desc(), + arch_module.PropertyDescription: p.Desc(), } if sign != "" { _, err := hex.DecodeString(sign) if err != nil { return true, nil, errors.New("unable to decode package signature") } - properties["sign"] = sign + properties[arch_module.PropertySignature] = sign } ver, _, err := packages_service.CreatePackageOrAddFileToExisting( From ca66c6d3247d3384bdb092f895150abeff77613c Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Wed, 1 Nov 2023 19:19:49 -0300 Subject: [PATCH 110/124] use buffer in pacman db desc function --- modules/packages/arch/metadata.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/packages/arch/metadata.go b/modules/packages/arch/metadata.go index dc51956aef18..4175cb46e8d2 100644 --- a/modules/packages/arch/metadata.go +++ b/modules/packages/arch/metadata.go @@ -266,13 +266,13 @@ func (p *Package) Desc() string { "CHECKDEPENDS", strings.Join(p.VersionMetadata.CheckDepends, "\n"), } - var result string + var buf bytes.Buffer for i := 0; i < 40; i += 2 { if entries[i+1] != "" { - result += fmt.Sprintf("%%%s%%\n%s\n\n", entries[i], entries[i+1]) + fmt.Fprintf(&buf, "%%%s%%\n%s\n\n", entries[i], entries[i+1]) } } - return result + return buf.String() } // Create pacman database archive based on provided package metadata structs. From c9abcf904f7707eadf5db1bbda62b4844930a114 Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Wed, 1 Nov 2023 19:26:40 -0300 Subject: [PATCH 111/124] error message clarification, typo fix in integration tests --- tests/integration/api_packages_arch_test.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/integration/api_packages_arch_test.go b/tests/integration/api_packages_arch_test.go index 57304df1b447..6391a648ccb0 100644 --- a/tests/integration/api_packages_arch_test.go +++ b/tests/integration/api_packages_arch_test.go @@ -40,10 +40,10 @@ func TestPackageArch(t *testing.T) { gitV2x86_64 = BuildArchPackage(t, "git", "2-1", "x86_64") iconsV2any = BuildArchPackage(t, "icons", "2-1", "any") - dbV1x86_64d = BuildArchDb([]arch.Package{gitV1x86_64.pkg, iconsV1any.pkg}) - dbV1i686 = BuildArchDb([]arch.Package{gitV1i686.pkg, iconsV1any.pkg}) - dbV2x86_64 = BuildArchDb([]arch.Package{gitV2x86_64.pkg, iconsV2any.pkg}) - dbV2i686 = BuildArchDb([]arch.Package{gitV1i686.pkg, iconsV2any.pkg}) + dbV1x86_64 = BuildArchDb([]arch.Package{gitV1x86_64.pkg, iconsV1any.pkg}) + dbV1i686 = BuildArchDb([]arch.Package{gitV1i686.pkg, iconsV1any.pkg}) + dbV2x86_64 = BuildArchDb([]arch.Package{gitV2x86_64.pkg, iconsV2any.pkg}) + dbV2i686 = BuildArchDb([]arch.Package{gitV1i686.pkg, iconsV2any.pkg}) firstSign = []byte{1, 2, 3, 4} secondSign = []byte{4, 3, 2, 1} @@ -179,7 +179,7 @@ func TestPackageArch(t *testing.T) { resp := MakeRequest(t, req, http.StatusOK) - CompareDbEntries(t, dbV1x86_64d, resp.Body.Bytes()) + CompareDbEntries(t, dbV1x86_64, resp.Body.Bytes()) }) t.Run("Get_i686_pacman_database", func(t *testing.T) { @@ -383,8 +383,8 @@ func BuildArchDb(pkgs []arch.Package) []byte { return b.Bytes() } -func CompareDbEntries(t *testing.T, first, second []byte) { - fgz, err := gzip.NewReader(bytes.NewReader(first)) +func CompareDbEntries(t *testing.T, expected, actual []byte) { + fgz, err := gzip.NewReader(bytes.NewReader(expected)) if err != nil { assert.NoError(t, err) return @@ -402,7 +402,7 @@ func CompareDbEntries(t *testing.T, first, second []byte) { validatemap[h.Name] = struct{}{} } - sgz, err := gzip.NewReader(bytes.NewReader(second)) + sgz, err := gzip.NewReader(bytes.NewReader(actual)) if err != nil { assert.NoError(t, err) return @@ -417,7 +417,7 @@ func CompareDbEntries(t *testing.T, first, second []byte) { _, ok := validatemap[h.Name] if !ok { - assert.Fail(t, "Entry not found in first archive: "+h.Name) + assert.Fail(t, "Unexpected entry in archive: "+h.Name) } delete(validatemap, h.Name) } @@ -427,6 +427,6 @@ func CompareDbEntries(t *testing.T, first, second []byte) { } for e := range validatemap { - assert.Fail(t, "Entry not found in second archive: "+e) + assert.Fail(t, "Entry not found in archive: "+e) } } From 11bc5f633f5a543c8b4e201de8b051cc2f841bd9 Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Wed, 1 Nov 2023 19:57:10 -0300 Subject: [PATCH 112/124] Update docs/content/usage/packages/arch.en-us.md Co-authored-by: silverwind --- docs/content/usage/packages/arch.en-us.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/content/usage/packages/arch.en-us.md b/docs/content/usage/packages/arch.en-us.md index de7dc78db34c..105054ef19b0 100644 --- a/docs/content/usage/packages/arch.en-us.md +++ b/docs/content/usage/packages/arch.en-us.md @@ -14,7 +14,7 @@ menu: # Arch package registry -Gitea has arch package registry, which can act as a fully working [arch linux mirror](https://wiki.archlinux.org/title/mirrors) and connected directly in `/etc/pacman.conf`. Gitea automatically creates pacman database for packages in user/organization space when new arch package is uploaded. +Gitea has a Arch Linux package registry, which can act as a fully working [Arch linux mirror](https://wiki.archlinux.org/title/mirrors) and connected directly in `/etc/pacman.conf`. Gitea automatically creates pacman database for packages in user/organization space when a new Arch package is uploaded. **Table of Contents** From 1d168c6c7cce30db66d00a5899476abcb818ed4d Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Wed, 1 Nov 2023 19:57:19 -0300 Subject: [PATCH 113/124] Update docs/content/usage/packages/arch.en-us.md Co-authored-by: silverwind --- docs/content/usage/packages/arch.en-us.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/content/usage/packages/arch.en-us.md b/docs/content/usage/packages/arch.en-us.md index 105054ef19b0..e73ac19530ac 100644 --- a/docs/content/usage/packages/arch.en-us.md +++ b/docs/content/usage/packages/arch.en-us.md @@ -30,7 +30,7 @@ SigLevel = Optional TrustAll Server = https://{domain}/api/packages/{owner}/arch/{distribution}/{architecture} ``` -Then, you can run pacman sync command (with -y flag to load connected database file), to install your package. +Then, you can run pacman sync command (with -y flag to load connected database file), to install your package: ```sh pacman -Sy package From 8d0e3016f6e958dd98385b1732064c6df7079701 Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Wed, 1 Nov 2023 19:57:39 -0300 Subject: [PATCH 114/124] Update docs/content/usage/packages/arch.en-us.md Co-authored-by: silverwind --- docs/content/usage/packages/arch.en-us.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/content/usage/packages/arch.en-us.md b/docs/content/usage/packages/arch.en-us.md index e73ac19530ac..2e82768f0991 100644 --- a/docs/content/usage/packages/arch.en-us.md +++ b/docs/content/usage/packages/arch.en-us.md @@ -38,7 +38,7 @@ pacman -Sy package ## Upload packages -When uploading package to gitea, you have to prepare package file with `.pkg.tar.zst` extension and it's `.pkg.tar.zst.sig` signature. You can use [curl](https://curl.se/) or any other HTTP client, gitea supports multiple [authentication schemes](https://docs.gitea.com/usage/authentication). Upload command will create 3 files: package, signature and desc file for pacman database (which will be created automatically on request). +When uploading the package to gitea, you have to prepare package file with the `.pkg.tar.zst` extension and its `.pkg.tar.zst.sig` signature. You can use [curl](https://curl.se/) or any other HTTP client, Gitea supports multiple [authentication schemes](https://docs.gitea.com/usage/authentication). The upload command will create 3 files: package, signature and desc file for the pacman database (which will be created automatically on request). Following command will upload arch package and related signature to gitea. Example with basic auth: From d42e3812c95267a40b208fcd6ba76b7089e0bcca Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Wed, 1 Nov 2023 19:57:48 -0300 Subject: [PATCH 115/124] Update docs/content/usage/packages/arch.en-us.md Co-authored-by: silverwind --- docs/content/usage/packages/arch.en-us.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/content/usage/packages/arch.en-us.md b/docs/content/usage/packages/arch.en-us.md index 2e82768f0991..12da63c0ba24 100644 --- a/docs/content/usage/packages/arch.en-us.md +++ b/docs/content/usage/packages/arch.en-us.md @@ -40,7 +40,7 @@ pacman -Sy package When uploading the package to gitea, you have to prepare package file with the `.pkg.tar.zst` extension and its `.pkg.tar.zst.sig` signature. You can use [curl](https://curl.se/) or any other HTTP client, Gitea supports multiple [authentication schemes](https://docs.gitea.com/usage/authentication). The upload command will create 3 files: package, signature and desc file for the pacman database (which will be created automatically on request). -Following command will upload arch package and related signature to gitea. Example with basic auth: +The following command will upload arch package and related signature to gitea with basic authentification: ```sh curl -X PUT \ From 373d904db1acda8aa5686a12b591def539977b93 Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Wed, 1 Nov 2023 19:58:21 -0300 Subject: [PATCH 116/124] Update docs/content/usage/packages/arch.en-us.md Co-authored-by: silverwind --- docs/content/usage/packages/arch.en-us.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/content/usage/packages/arch.en-us.md b/docs/content/usage/packages/arch.en-us.md index 12da63c0ba24..a448d00a9d8c 100644 --- a/docs/content/usage/packages/arch.en-us.md +++ b/docs/content/usage/packages/arch.en-us.md @@ -52,7 +52,7 @@ curl -X PUT \ ## Delete packages -Delete operation will remove specific package version, and all package files related to that version. +The `DELETE` method will remove specific package version, and all package files related to that version: ```sh curl -X DELETE \ From 16da9f90b3c6d98b378543c9d5183686114b9a71 Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Wed, 1 Nov 2023 19:58:58 -0300 Subject: [PATCH 117/124] Update docs/content/usage/packages/arch.en-us.md Co-authored-by: silverwind --- docs/content/usage/packages/arch.en-us.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/content/usage/packages/arch.en-us.md b/docs/content/usage/packages/arch.en-us.md index a448d00a9d8c..095d024a3de3 100644 --- a/docs/content/usage/packages/arch.en-us.md +++ b/docs/content/usage/packages/arch.en-us.md @@ -62,4 +62,4 @@ curl -X DELETE \ ## Clients -Any `pacman` compatible package manager/AUR-helper can be used to install packages from gitea ([yay](https://github.com/Jguer/yay), [paru](https://github.com/Morganamilo/paru), [pikaur](https://github.com/actionless/pikaur), [aura](https://github.com/fosskers/aura)). Alternatively, you can try [pack](https://fmnx.su/core/pack) which supports full gitea API (install/push/remove). Also, any HTTP client can be used to execute get/push/remove operations ([curl](https://curl.se/), [postman](https://www.postman.com/), [thunder-client](https://www.thunderclient.com/)). +Any `pacman` compatible package manager or AUR-helper can be used to install packages from gitea ([yay](https://github.com/Jguer/yay), [paru](https://github.com/Morganamilo/paru), [pikaur](https://github.com/actionless/pikaur), [aura](https://github.com/fosskers/aura)). Alternatively, you can try [pack](https://fmnx.su/core/pack) which supports full gitea API (install/push/remove). Also, any HTTP client can be used to execute get/push/remove operations ([curl](https://curl.se/), [postman](https://www.postman.com/), [thunder-client](https://www.thunderclient.com/)). From ffca863e37dbe22e6f78ced74f5f876c3e7dad3b Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Wed, 1 Nov 2023 20:16:53 -0300 Subject: [PATCH 118/124] moved link to arch package documentation from translation to html template --- options/locale/locale_en-US.ini | 2 +- templates/package/content/arch.tmpl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index ccef6f9dab3e..d593ee517876 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -3340,7 +3340,7 @@ alpine.repository.repositories = Repositories alpine.repository.architectures = Architectures arch.pacmanconf = Add server with related distribution and architecture to /etc/pacman.conf: arch.pacmansync = Sync package with pacman: -arch.documentation = For more information on the arch mirrors, see the documentation. +arch.documentation = For more information on the arch mirrors, see %sthe documentation%s. arch.properties = Package properties cargo.registry = Setup this registry in the Cargo configuration file (for example ~/.cargo/config.toml): cargo.install = To install the package using Cargo, run the following command: diff --git a/templates/package/content/arch.tmpl b/templates/package/content/arch.tmpl index 83c5baba32ee..1c0175aa1ecb 100644 --- a/templates/package/content/arch.tmpl +++ b/templates/package/content/arch.tmpl @@ -16,7 +16,7 @@ Server = - + {{.locale.Tr "packages.arch.documentation" (printf ``) (printf ``) | Safe}} From 4b96ddd28a906a116130ebdecb5031e4eff9fb25 Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Wed, 1 Nov 2023 20:38:24 -0300 Subject: [PATCH 119/124] Update web_src/svg/gitea-arch.svg Co-authored-by: silverwind --- web_src/svg/gitea-arch.svg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web_src/svg/gitea-arch.svg b/web_src/svg/gitea-arch.svg index 264994d15067..ba8254d8049e 100644 --- a/web_src/svg/gitea-arch.svg +++ b/web_src/svg/gitea-arch.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file From a054f02dc9d47b9761444a4d3022db54a6ff5f41 Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Wed, 1 Nov 2023 20:59:36 -0300 Subject: [PATCH 120/124] fixed context references in arch package html templates, removed URL from translations --- templates/package/content/arch.tmpl | 10 +++++----- templates/package/metadata/arch.tmpl | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/templates/package/content/arch.tmpl b/templates/package/content/arch.tmpl index 1c0175aa1ecb..a8d6e5d286c2 100644 --- a/templates/package/content/arch.tmpl +++ b/templates/package/content/arch.tmpl @@ -1,27 +1,27 @@ {{if eq .PackageDescriptor.Package.Type "arch"}} -

{{.locale.Tr "packages.installation"}}

+

{{ctx.Locale.Tr "packages.installation"}}

- + 
[{{.PackageDescriptor.Owner.LowerName}}.{{.RegistryHost}}]
 SigLevel = Optional TrustAll
 Server =
- + 
pacman -Sy {{.PackageDescriptor.Package.LowerName}}
- {{.locale.Tr "packages.arch.documentation" (printf ``) (printf ``) | Safe}} + {{ctx.Locale.Tr "packages.arch.documentation" (printf ``) (printf ``) | Safe}}
-

{{.locale.Tr "packages.arch.properties"}}

+

{{ctx.Locale.Tr "packages.arch.properties"}}

diff --git a/templates/package/metadata/arch.tmpl b/templates/package/metadata/arch.tmpl index 33e5fda9bd30..822973eb7d98 100644 --- a/templates/package/metadata/arch.tmpl +++ b/templates/package/metadata/arch.tmpl @@ -1,4 +1,4 @@ {{if eq .PackageDescriptor.Package.Type "arch"}} {{range .PackageDescriptor.Metadata.License}}
{{svg "octicon-law" 16 "gt-mr-3"}} {{.}}
{{end}} - {{if .PackageDescriptor.Metadata.URL}}
{{svg "octicon-link-external" 16 "mr-3"}} {{.locale.Tr "packages.details.project_site"}}
{{end}} + {{if .PackageDescriptor.Metadata.ProjectURL}}
{{svg "octicon-link-external" 16 "mr-3"}} {{ctx.Locale.Tr "packages.details.project_site"}}
{{end}} {{end}} From d234b661f097b8ef45405b3adcb91dbb494f462d Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Wed, 1 Nov 2023 21:28:34 -0300 Subject: [PATCH 121/124] translations for arch metadata fields --- options/locale/locale_en-US.ini | 6 ++++++ templates/package/content/arch.tmpl | 12 ++++++------ 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index d593ee517876..16ec01c56e59 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -3342,6 +3342,12 @@ arch.pacmanconf = Add server with related distribution and architecture to ~/.cargo/config.toml): cargo.install = To install the package using Cargo, run the following command: chef.registry = Setup this registry in your ~/.chef/config.rb file: diff --git a/templates/package/content/arch.tmpl b/templates/package/content/arch.tmpl index a8d6e5d286c2..b457af031a53 100644 --- a/templates/package/content/arch.tmpl +++ b/templates/package/content/arch.tmpl @@ -26,41 +26,41 @@ Server = - + {{if .PackageDescriptor.Metadata.Provides}} - + {{end}} {{if .PackageDescriptor.Metadata.Depends}} - + {{end}} {{if .PackageDescriptor.Metadata.OptDepends}} - + {{end}} {{if .PackageDescriptor.Metadata.MakeDepends}} - + {{end}} {{if .PackageDescriptor.Metadata.CheckDepends}} - + {{end}} From 4ba056780bec27e5a9b2eeac6e6eeced728f6782 Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Wed, 1 Nov 2023 22:08:42 -0300 Subject: [PATCH 122/124] fix arch linux icon --- public/assets/img/svg/gitea-arch.svg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/assets/img/svg/gitea-arch.svg b/public/assets/img/svg/gitea-arch.svg index c6173d67db99..943a92c57946 100644 --- a/public/assets/img/svg/gitea-arch.svg +++ b/public/assets/img/svg/gitea-arch.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file From b12de2a0d9e5d7b6c2c74e12ad9731785e1a04cd Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Fri, 3 Nov 2023 16:48:56 -0300 Subject: [PATCH 123/124] integration tests fixes and enhancements: isolated tests for different functionality, test cases for different distributions, architectures and versions --- tests/integration/api_packages_arch_test.go | 444 ++++++++++---------- 1 file changed, 233 insertions(+), 211 deletions(-) diff --git a/tests/integration/api_packages_arch_test.go b/tests/integration/api_packages_arch_test.go index 6391a648ccb0..3a91d797dd67 100644 --- a/tests/integration/api_packages_arch_test.go +++ b/tests/integration/api_packages_arch_test.go @@ -15,11 +15,12 @@ import ( "io" "net/http" "os" - "path" "testing" "testing/fstest" "time" + "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/models/packages" "code.gitea.io/gitea/models/unittest" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/packages/arch" @@ -34,244 +35,260 @@ func TestPackageArch(t *testing.T) { assert.NoError(t, unittest.PrepareTestDatabase()) var ( - gitV1x86_64 = BuildArchPackage(t, "git", "1-1", "x86_64") - gitV1i686 = BuildArchPackage(t, "git", "1-1", "i686") - iconsV1any = BuildArchPackage(t, "icons", "1-1", "any") - gitV2x86_64 = BuildArchPackage(t, "git", "2-1", "x86_64") - iconsV2any = BuildArchPackage(t, "icons", "2-1", "any") - - dbV1x86_64 = BuildArchDb([]arch.Package{gitV1x86_64.pkg, iconsV1any.pkg}) - dbV1i686 = BuildArchDb([]arch.Package{gitV1i686.pkg, iconsV1any.pkg}) - dbV2x86_64 = BuildArchDb([]arch.Package{gitV2x86_64.pkg, iconsV2any.pkg}) - dbV2i686 = BuildArchDb([]arch.Package{gitV1i686.pkg, iconsV2any.pkg}) - - firstSign = []byte{1, 2, 3, 4} - secondSign = []byte{4, 3, 2, 1} - - user = unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) - rootURL = fmt.Sprintf("/api/packages/%s/arch", user.Name) - ) - - t.Run("Version_1", func(t *testing.T) { - t.Run("Push_git_x86_64", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - - req := NewRequestWithBody(t, "PUT", - path.Join( - rootURL, "push", "git-1-1-x86_64.pkg.tar.zst", - "archlinux", hex.EncodeToString(firstSign), - ), - bytes.NewReader(gitV1x86_64.data), - ) - - req = AddBasicAuthHeader(req, user.Name) - - MakeRequest(t, req, http.StatusOK) - }) - - t.Run("Push_git_i686", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - - req := NewRequestWithBody(t, "PUT", - path.Join( - rootURL, "push", "git-1-1-i686.pkg.tar.zst", - "archlinux", hex.EncodeToString(secondSign), - ), - bytes.NewReader(gitV1i686.data), - ) - - req = AddBasicAuthHeader(req, user.Name) - - MakeRequest(t, req, http.StatusOK) - }) - - t.Run("Push_icons_any", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - - req := NewRequestWithBody(t, "PUT", - path.Join(rootURL, "push", "icons-1-1-any.pkg.tar.zst", "archlinux"), - bytes.NewReader(iconsV1any.data), - ) - - req = AddBasicAuthHeader(req, user.Name) - - MakeRequest(t, req, http.StatusOK) - }) - - t.Run("Get_git_x86_64_package", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - - req := NewRequest(t, "GET", - rootURL+"/archlinux/x86_64/git-1-1-x86_64.pkg.tar.zst", - ) - - resp := MakeRequest(t, req, http.StatusOK) - - assert.Equal(t, gitV1x86_64.data, resp.Body.Bytes()) - }) - - t.Run("Get_git_i686_package", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - - req := NewRequest(t, "GET", - rootURL+"/archlinux/i686/git-1-1-i686.pkg.tar.zst", - ) - - resp := MakeRequest(t, req, http.StatusOK) - - assert.Equal(t, gitV1i686.data, resp.Body.Bytes()) - }) - - t.Run("Get_git_x86_64_package_signature", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - - req := NewRequest(t, "GET", - rootURL+"/archlinux/x86_64/git-1-1-x86_64.pkg.tar.zst.sig", - ) - - resp := MakeRequest(t, req, http.StatusOK) - - assert.Equal(t, firstSign, resp.Body.Bytes()) - }) - - t.Run("Get_git_i686_package_signature", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - - req := NewRequest(t, "GET", - rootURL+"/archlinux/i686/git-1-1-i686.pkg.tar.zst.sig", - ) - - resp := MakeRequest(t, req, http.StatusOK) - - assert.Equal(t, secondSign, resp.Body.Bytes()) - }) - - t.Run("Get_any_package_from_x86_64_group", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - - req := NewRequest(t, "GET", - rootURL+"/archlinux/x86_64/icons-1-1-any.pkg.tar.zst", - ) - - resp := MakeRequest(t, req, http.StatusOK) - - assert.Equal(t, iconsV1any.data, resp.Body.Bytes()) - }) - - t.Run("Get_any_package_from_i686_group", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - - req := NewRequest(t, "GET", - rootURL+"/archlinux/i686/icons-1-1-any.pkg.tar.zst", - ) - - resp := MakeRequest(t, req, http.StatusOK) - - assert.Equal(t, iconsV1any.data, resp.Body.Bytes()) - }) + user = unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1}) + + pushBatch = []*TestArchPackage{ + BuildArchPackage(t, "git", "1-1", "x86_64"), + BuildArchPackage(t, "git", "2-1", "x86_64"), + BuildArchPackage(t, "git", "1-1", "i686"), + BuildArchPackage(t, "adwaita", "1-1", "any"), + BuildArchPackage(t, "adwaita", "2-1", "any"), + } - t.Run("Get_x86_64_pacman_database", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + removeBatch = []*TestArchPackage{ + BuildArchPackage(t, "curl", "1-1", "x86_64"), + BuildArchPackage(t, "curl", "2-1", "x86_64"), + BuildArchPackage(t, "dock", "1-1", "any"), + BuildArchPackage(t, "dock", "2-1", "any"), + } - req := NewRequest(t, "GET", - rootURL+"/archlinux/x86_64/user.db.tar.gz", - ) + firstDatabaseBatch = []*TestArchPackage{ + BuildArchPackage(t, "pacman", "1-1", "x86_64"), + BuildArchPackage(t, "pacman", "1-1", "i686"), + BuildArchPackage(t, "htop", "1-1", "x86_64"), + BuildArchPackage(t, "htop", "1-1", "i686"), + BuildArchPackage(t, "dash", "1-1", "any"), + } - resp := MakeRequest(t, req, http.StatusOK) + secondDatabaseBatch = []*TestArchPackage{ + BuildArchPackage(t, "pacman", "2-1", "x86_64"), + BuildArchPackage(t, "htop", "2-1", "i686"), + BuildArchPackage(t, "dash", "2-1", "any"), + } - CompareDbEntries(t, dbV1x86_64, resp.Body.Bytes()) - }) + PacmanDBx86 = BuildPacmanDb(t, + secondDatabaseBatch[0].Pkg, + firstDatabaseBatch[2].Pkg, + secondDatabaseBatch[2].Pkg, + ) - t.Run("Get_i686_pacman_database", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + PacmanDBi686 = BuildPacmanDb(t, + firstDatabaseBatch[0].Pkg, + secondDatabaseBatch[1].Pkg, + secondDatabaseBatch[2].Pkg, + ) - req := NewRequest(t, "GET", - rootURL+"/archlinux/i686/user.db.tar.gz", - ) + signdata = []byte{1, 2, 3, 4} + ) - resp := MakeRequest(t, req, http.StatusOK) + t.Run("PushWithSignature", func(t *testing.T) { + for _, p := range pushBatch { + t.Run(p.File, func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + url := fmt.Sprintf( + "/api/packages/%s/arch/push/%s/archlinux/%s", + user.Name, p.File, hex.EncodeToString(signdata), + ) + + req := NewRequestWithBody(t, "PUT", url, bytes.NewReader(p.Data)) + req = AddBasicAuthHeader(req, user.Name) + MakeRequest(t, req, http.StatusOK) + + pv, err := packages.GetVersionByNameAndVersion( + db.DefaultContext, user.ID, packages.TypeArch, p.Name, p.Ver, + ) + assert.NoError(t, err) + + pf, err := packages.GetFileForVersionByName( + db.DefaultContext, pv.ID, p.File, "archlinux", + ) + assert.NoError(t, err) + assert.NotNil(t, pf) + + pps, err := packages.GetPropertiesByName( + db.DefaultContext, packages.PropertyTypeFile, + pf.ID, arch.PropertySignature, + ) + assert.NoError(t, err) + assert.Len(t, pps, 1) + }) + } + }) - CompareDbEntries(t, dbV1i686, resp.Body.Bytes()) - }) + t.Run("PushWithoutSignature", func(t *testing.T) { + for _, p := range pushBatch { + t.Run(p.File, func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + url := fmt.Sprintf( + "/api/packages/%s/arch/push/%s/parabola", + user.Name, p.File, + ) + + req := NewRequestWithBody(t, "PUT", url, bytes.NewReader(p.Data)) + req = AddBasicAuthHeader(req, user.Name) + MakeRequest(t, req, http.StatusOK) + + pv, err := packages.GetVersionByNameAndVersion( + db.DefaultContext, user.ID, packages.TypeArch, p.Name, p.Ver, + ) + assert.NoError(t, err) + + pf, err := packages.GetFileForVersionByName( + db.DefaultContext, pv.ID, p.File, "parabola", + ) + assert.NoError(t, err) + assert.NotNil(t, pf) + }) + } }) - t.Run("Version_2", func(t *testing.T) { - t.Run("Push_git_x86_64", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("GetPackage", func(t *testing.T) { + for _, p := range pushBatch { + t.Run(p.File, func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + url := fmt.Sprintf( + "/api/packages/%s/arch/push/%s/artix/%s", + user.Name, p.File, hex.EncodeToString(signdata), + ) + req := NewRequestWithBody(t, "PUT", url, bytes.NewReader(p.Data)) + req = AddBasicAuthHeader(req, user.Name) + MakeRequest(t, req, http.StatusOK) + + url = fmt.Sprintf( + "/api/packages/%s/arch/artix/%s/%s", + user.Name, p.Arch, p.File, + ) + req = NewRequest(t, "GET", url) + resp := MakeRequest(t, req, http.StatusOK) + assert.Equal(t, p.Data, resp.Body.Bytes()) + }) + } + }) - req := NewRequestWithBody(t, "PUT", - path.Join(rootURL, "push", "git-2-1-x86_64.pkg.tar.zst", "archlinux"), - bytes.NewReader(gitV2x86_64.data), - ) + t.Run("GetSignature", func(t *testing.T) { + for _, p := range pushBatch { + t.Run(p.File, func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + url := fmt.Sprintf( + "/api/packages/%s/arch/push/%s/arco/%s", + user.Name, p.File, hex.EncodeToString(signdata), + ) + req := NewRequestWithBody(t, "PUT", url, bytes.NewReader(p.Data)) + req = AddBasicAuthHeader(req, user.Name) + MakeRequest(t, req, http.StatusOK) + + url = fmt.Sprintf( + "/api/packages/%s/arch/arco/%s/%s.sig", + user.Name, p.Arch, p.File, + ) + req = NewRequest(t, "GET", url) + resp := MakeRequest(t, req, http.StatusOK) + assert.Equal(t, signdata, resp.Body.Bytes()) + }) + } + }) - req = AddBasicAuthHeader(req, user.Name) + t.Run("Remove", func(t *testing.T) { + for _, p := range removeBatch { + t.Run(p.File, func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + url := fmt.Sprintf( + "/api/packages/%s/arch/push/%s/manjaro/%s", + user.Name, p.File, hex.EncodeToString(signdata), + ) + req := NewRequestWithBody(t, "PUT", url, bytes.NewReader(p.Data)) + req = AddBasicAuthHeader(req, user.Name) + MakeRequest(t, req, http.StatusOK) + + url = fmt.Sprintf( + "/api/packages/%s/arch/remove/%s/%s", + user.Name, p.Name, p.Ver, + ) + req = NewRequest(t, "DELETE", url) + req = AddBasicAuthHeader(req, user.Name) + MakeRequest(t, req, http.StatusOK) + + _, err := packages.GetVersionByNameAndVersion( + db.DefaultContext, user.ID, packages.TypeArch, p.Name, p.Ver, + ) + assert.ErrorIs(t, err, packages.ErrPackageNotExist) + }) + } + }) - MakeRequest(t, req, http.StatusOK) - }) + t.Run("PacmanDatabase", func(t *testing.T) { + prepareDatabasePackages := func(t *testing.T) { + for _, p := range firstDatabaseBatch { + url := fmt.Sprintf( + "/api/packages/%s/arch/push/%s/ion/%s", + user.Name, p.File, hex.EncodeToString(signdata), + ) + req := NewRequestWithBody(t, "PUT", url, bytes.NewReader(p.Data)) + req = AddBasicAuthHeader(req, user.Name) + MakeRequest(t, req, http.StatusOK) + } + + // While creating pacman database, package versions are sorted by + // UnixTime, second delay is required to ensure that newer package + // version creation time differs from older packages. + time.Sleep(time.Second) + + for _, p := range secondDatabaseBatch { + url := fmt.Sprintf( + "/api/packages/%s/arch/push/%s/ion/%s", + user.Name, p.File, hex.EncodeToString(signdata), + ) + req := NewRequestWithBody(t, "PUT", url, bytes.NewReader(p.Data)) + req = AddBasicAuthHeader(req, user.Name) + MakeRequest(t, req, http.StatusOK) + } + } - t.Run("Push_icons_any", func(t *testing.T) { + t.Run("x86_64", func(t *testing.T) { defer tests.PrintCurrentTest(t)() - req := NewRequestWithBody(t, "PUT", - path.Join(rootURL, "push", "icons-2-1-any.pkg.tar.zst", "archlinux"), - bytes.NewReader(iconsV2any.data), - ) - - req = AddBasicAuthHeader(req, user.Name) - - MakeRequest(t, req, http.StatusOK) - }) - - t.Run("Get_x86_64_pacman_database", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + prepareDatabasePackages(t) - req := NewRequest(t, "GET", - rootURL+"/archlinux/x86_64/user2.db.tar.gz", + url := fmt.Sprintf( + "/api/packages/%s/arch/ion/x86_64/user.db.tar.gz", user.Name, ) - + req := NewRequest(t, "GET", url) resp := MakeRequest(t, req, http.StatusOK) - CompareDbEntries(t, dbV2x86_64, resp.Body.Bytes()) + CompareTarGzEntries(t, PacmanDBx86, resp.Body.Bytes()) }) - t.Run("Get_i686_pacman_database", func(t *testing.T) { + t.Run("i686", func(t *testing.T) { defer tests.PrintCurrentTest(t)() - req := NewRequest(t, "GET", - rootURL+"/archlinux/i686/user2.db.tar.gz", - ) + prepareDatabasePackages(t) + url := fmt.Sprintf( + "/api/packages/%s/arch/ion/i686/user.db", user.Name, + ) + req := NewRequest(t, "GET", url) resp := MakeRequest(t, req, http.StatusOK) - CompareDbEntries(t, dbV2i686, resp.Body.Bytes()) - }) - - t.Run("Remove_v2_git_x86_64", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - - req := NewRequest(t, "DELETE", rootURL+"/remove/git/2-1") - req = AddBasicAuthHeader(req, user.Name) - - MakeRequest(t, req, http.StatusOK) - }) - - t.Run("Remove_v2_icons_any", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - - req := NewRequest(t, "DELETE", rootURL+"/remove/icons/2-1") - req = AddBasicAuthHeader(req, user.Name) - - MakeRequest(t, req, http.StatusOK) + CompareTarGzEntries(t, PacmanDBi686, resp.Body.Bytes()) }) }) } type TestArchPackage struct { - data []byte - pkg arch.Package + Pkg arch.Package + Data []byte + File string + Name string + Ver string + Arch string } -func BuildArchPackage(t *testing.T, name, ver, architecture string) TestArchPackage { +func BuildArchPackage(t *testing.T, name, ver, architecture string) *TestArchPackage { fs := fstest.MapFS{ "pkginfo": &fstest.MapFile{ Data: []byte(fmt.Sprintf( @@ -331,9 +348,13 @@ func BuildArchPackage(t *testing.T, name, ver, architecture string) TestArchPack md5, sha256, size := archPkgParams(buf.Bytes()) - return TestArchPackage{ - data: buf.Bytes(), - pkg: arch.Package{ + return &TestArchPackage{ + Data: buf.Bytes(), + Name: name, + Ver: ver, + Arch: architecture, + File: fmt.Sprintf("%s-%s-%s.pkg.tar.zst", name, ver, architecture), + Pkg: arch.Package{ Name: name, Version: ver, VersionMetadata: arch.VersionMetadata{ @@ -371,19 +392,20 @@ func (w *counter) Read(p []byte) (int, error) { return n, err } -func BuildArchDb(pkgs []arch.Package) []byte { +func BuildPacmanDb(t *testing.T, pkgs ...arch.Package) []byte { entries := map[string][]byte{} for _, p := range pkgs { entries[fmt.Sprintf("%s-%s/desc", p.Name, p.Version)] = []byte(p.Desc()) } b, err := arch.CreatePacmanDb(entries) if err != nil { - panic(err) + assert.NoError(t, err) + return nil } return b.Bytes() } -func CompareDbEntries(t *testing.T, expected, actual []byte) { +func CompareTarGzEntries(t *testing.T, expected, actual []byte) { fgz, err := gzip.NewReader(bytes.NewReader(expected)) if err != nil { assert.NoError(t, err) From b97d4f9a4534806e349bb01348095f188d8fa13c Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Wed, 8 Nov 2023 18:55:00 -0300 Subject: [PATCH 124/124] fix: context doer as package creator --- services/packages/arch/upload.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/packages/arch/upload.go b/services/packages/arch/upload.go index 0a5e7eb245bb..221801b4f87b 100644 --- a/services/packages/arch/upload.go +++ b/services/packages/arch/upload.go @@ -65,7 +65,7 @@ func UploadArchPackage(ctx *context.Context, upload io.Reader, filename, distro, }, OverwriteExisting: true, IsLead: true, - Creator: ctx.ContextUser, + Creator: ctx.Doer, Data: buf, Properties: properties, },
Description
{{ctx.Locale.Tr "packages.arch.description"}}
 {{.PackageDescriptor.Metadata.Description}}
Provides
{{ctx.Locale.Tr "packages.arch.provides"}}
 {{StringUtils.Join $.PackageDescriptor.Metadata.Provides ", "}}
Depends
{{ctx.Locale.Tr "packages.arch.depends"}}
 {{StringUtils.Join $.PackageDescriptor.Metadata.Depends ", "}}
Optional depends
{{ctx.Locale.Tr "packages.arch.optdepends"}}
 {{StringUtils.Join $.PackageDescriptor.Metadata.OptDepends ", "}}
Make depends
{{ctx.Locale.Tr "packages.arch.makedepends"}}
 {{StringUtils.Join $.PackageDescriptor.Metadata.MakeDepends ", "}}
Check depends
{{ctx.Locale.Tr "packages.arch.checkdepends"}}
 {{StringUtils.Join $.PackageDescriptor.Metadata.CheckDepends ", "}}