diff --git a/docs/content/usage/packages/arch.en-us.md b/docs/content/usage/packages/arch.en-us.md new file mode 100644 index 000000000000..095d024a3de3 --- /dev/null +++ b/docs/content/usage/packages/arch.en-us.md @@ -0,0 +1,65 @@ +--- +date: "2016-11-08T16:00:00+02:00" +title: "Arch Package Registry" +weight: 10 +toc: true +draft: false +menu: + sidebar: + parent: "packages" + name: "Arch" + weight: 10 + identifier: "arch" +--- + +# Arch package registry + +Gitea has a Arch Linux package registry, which can act as a fully working [Arch linux mirror](https://wiki.archlinux.org/title/mirrors) and connected directly in `/etc/pacman.conf`. Gitea automatically creates pacman database for packages in user/organization space when a new Arch package is uploaded. + +**Table of Contents** + +{{< toc >}} + +## Install packages + +First, you need to update your pacman configuration, adding following lines: + +```conf +[{owner}.{domain}] +SigLevel = Optional TrustAll +Server = https://{domain}/api/packages/{owner}/arch/{distribution}/{architecture} +``` + +Then, you can run pacman sync command (with -y flag to load connected database file), to install your package: + +```sh +pacman -Sy package +``` + +## Upload packages + +When uploading the package to gitea, you have to prepare package file with the `.pkg.tar.zst` extension and its `.pkg.tar.zst.sig` signature. You can use [curl](https://curl.se/) or any other HTTP client, Gitea supports multiple [authentication schemes](https://docs.gitea.com/usage/authentication). The upload command will create 3 files: package, signature and desc file for the pacman database (which will be created automatically on request). + +The following command will upload arch package and related signature to gitea with basic authentification: + +```sh +curl -X PUT \ + https://{domain}/api/packages/{owner}/arch/push/{package-1-1-x86_64.pkg.tar.zst}/{archlinux}/$(xxd -p package-1-1-x86_64.pkg.tar.zst.sig | tr -d '\n') \ + --user your_username:your_token_or_password \ + --header "Content-Type: application/octet-stream" \ + --data-binary '@/path/to/package/file/package-1-1-x86_64.pkg.tar.zst' +``` + +## Delete packages + +The `DELETE` method will remove specific package version, and all package files related to that version: + +```sh +curl -X DELETE \ + https://{domain}/api/packages/{user}/arch/remove/{package}/{version} \ + --user your_username:your_token_or_password +``` + +## Clients + +Any `pacman` compatible package manager or AUR-helper can be used to install packages from gitea ([yay](https://github.com/Jguer/yay), [paru](https://github.com/Morganamilo/paru), [pikaur](https://github.com/actionless/pikaur), [aura](https://github.com/fosskers/aura)). Alternatively, you can try [pack](https://fmnx.su/core/pack) which supports full gitea API (install/push/remove). Also, any HTTP client can be used to execute get/push/remove operations ([curl](https://curl.se/), [postman](https://www.postman.com/), [thunder-client](https://www.thunderclient.com/)). diff --git a/docs/content/usage/packages/overview.en-us.md b/docs/content/usage/packages/overview.en-us.md index 89fc6f286e65..b01163b8f6e7 100644 --- a/docs/content/usage/packages/overview.en-us.md +++ b/docs/content/usage/packages/overview.en-us.md @@ -24,6 +24,7 @@ The following package managers are currently supported: | Name | Language | Package client | | ---- | -------- | -------------- | | [Alpine](usage/packages/alpine.md) | - | `apk` | +| [Arch](usage/packages/arch.md) | - | `pacman` | | [Cargo](usage/packages/cargo.md) | Rust | `cargo` | | [Chef](usage/packages/chef.md) | - | `knife` | | [Composer](usage/packages/composer.md) | PHP | `composer` | diff --git a/models/packages/descriptor.go b/models/packages/descriptor.go index b8ef698d3822..803b73c96899 100644 --- a/models/packages/descriptor.go +++ b/models/packages/descriptor.go @@ -13,6 +13,7 @@ import ( user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/json" "code.gitea.io/gitea/modules/packages/alpine" + "code.gitea.io/gitea/modules/packages/arch" "code.gitea.io/gitea/modules/packages/cargo" "code.gitea.io/gitea/modules/packages/chef" "code.gitea.io/gitea/modules/packages/composer" @@ -150,6 +151,8 @@ func GetPackageDescriptor(ctx context.Context, pv *PackageVersion) (*PackageDesc switch p.Type { case TypeAlpine: metadata = &alpine.VersionMetadata{} + case TypeArch: + metadata = &arch.VersionMetadata{} case TypeCargo: metadata = &cargo.Metadata{} case TypeChef: diff --git a/models/packages/package.go b/models/packages/package.go index 65a25741509e..417d62d19931 100644 --- a/models/packages/package.go +++ b/models/packages/package.go @@ -31,6 +31,7 @@ type Type string // List of supported packages const ( TypeAlpine Type = "alpine" + TypeArch Type = "arch" TypeCargo Type = "cargo" TypeChef Type = "chef" TypeComposer Type = "composer" @@ -55,6 +56,7 @@ const ( var TypeList = []Type{ TypeAlpine, + TypeArch, TypeCargo, TypeChef, TypeComposer, @@ -82,6 +84,8 @@ func (pt Type) Name() string { switch pt { case TypeAlpine: return "Alpine" + case TypeArch: + return "Arch" case TypeCargo: return "Cargo" case TypeChef: @@ -131,6 +135,8 @@ func (pt Type) SVGName() string { switch pt { case TypeAlpine: return "gitea-alpine" + case TypeArch: + return "gitea-arch" case TypeCargo: return "gitea-cargo" case TypeChef: diff --git a/modules/packages/arch/metadata.go b/modules/packages/arch/metadata.go new file mode 100644 index 000000000000..4175cb46e8d2 --- /dev/null +++ b/modules/packages/arch/metadata.go @@ -0,0 +1,302 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package arch + +import ( + "archive/tar" + "bufio" + "bytes" + "compress/gzip" + "encoding/hex" + "errors" + "fmt" + "io" + "os" + "regexp" + "strconv" + "strings" + + "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/modules/validation" + + "github.com/mholt/archiver/v3" +) + +const ( + PropertyDescription = "arch.description" + PropertySignature = "arch.signature" +) + +var ( + // https://man.archlinux.org/man/PKGBUILD.5 + reName = regexp.MustCompile(`^[a-zA-Z0-9@._+-]+$`) + reVer = regexp.MustCompile(`^[a-zA-Z0-9:_.+]+-+[0-9]+$`) + reOptDep = regexp.MustCompile(`^[a-zA-Z0-9@._+-]+$|^[a-zA-Z0-9@._+-]+(:.*)`) + rePkgVer = regexp.MustCompile(`^[a-zA-Z0-9@._+-]+$|^[a-zA-Z0-9@._+-]+(>.*)|^[a-zA-Z0-9@._+-]+(<.*)|^[a-zA-Z0-9@._+-]+(=.*)`) +) + +type Package struct { + Name string `json:"name"` + Version string `json:"version"` + VersionMetadata VersionMetadata + FileMetadata FileMetadata +} + +// Arch package metadata related to specific version. +// Version metadata the same across different architectures and distributions. +type VersionMetadata struct { + Base string `json:"base"` + Description string `json:"description"` + ProjectURL string `json:"project_url"` + Groups []string `json:"groups,omitempty"` + Provides []string `json:"provides,omitempty"` + License []string `json:"license,omitempty"` + Depends []string `json:"depends,omitempty"` + OptDepends []string `json:"opt_depends,omitempty"` + MakeDepends []string `json:"make_depends,omitempty"` + CheckDepends []string `json:"check_depends,omitempty"` + Backup []string `json:"backup,omitempty"` +} + +// Metadata related to specific pakcage file. +// This metadata might vary for different architecture and distribution. +type FileMetadata struct { + CompressedSize int64 `json:"compressed_size"` + InstalledSize int64 `json:"installed_size"` + MD5 string `json:"md5"` + SHA256 string `json:"sha256"` + BuildDate int64 `json:"build_date"` + Packager string `json:"packager"` + Arch string `json:"arch"` +} + +// Function that receives arch package archive data and returns it's metadata. +func ParsePackage(r io.Reader, md5, sha256 []byte, size int64) (*Package, error) { + zstd := archiver.NewTarZstd() + err := zstd.Open(r, 0) + if err != nil { + return nil, err + } + defer zstd.Close() + + var pkg *Package + var mtree bool + + for { + f, err := zstd.Read() + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + defer f.Close() + + switch f.Name() { + case ".PKGINFO": + pkg, err = ParsePackageInfo(f) + if err != nil { + return nil, err + } + case ".MTREE": + mtree = true + } + } + + if pkg == nil { + return nil, util.NewInvalidArgumentErrorf(".PKGINFO file not found") + } + + if !mtree { + return nil, util.NewInvalidArgumentErrorf(".MTREE file not found") + } + + pkg.FileMetadata.CompressedSize = size + pkg.FileMetadata.SHA256 = hex.EncodeToString(sha256) + pkg.FileMetadata.MD5 = hex.EncodeToString(md5) + + return pkg, nil +} + +// Function that accepts reader for .PKGINFO file from package archive, +// validates all field according to PKGBUILD spec and returns package. +func ParsePackageInfo(r io.Reader) (*Package, error) { + p := &Package{} + + scanner := bufio.NewScanner(r) + for scanner.Scan() { + line := scanner.Text() + + if strings.HasPrefix(line, "#") { + continue + } + + i := strings.IndexRune(line, '=') + if i == -1 { + continue + } + + key := strings.TrimSpace(line[:i]) + value := strings.TrimSpace(line[i+1:]) + + switch key { + case "pkgname": + p.Name = value + case "pkgbase": + p.VersionMetadata.Base = value + case "pkgver": + p.Version = value + case "pkgdesc": + p.VersionMetadata.Description = value + case "url": + p.VersionMetadata.ProjectURL = value + case "packager": + p.FileMetadata.Packager = value + case "arch": + p.FileMetadata.Arch = value + case "provides": + p.VersionMetadata.Provides = append(p.VersionMetadata.Provides, value) + case "license": + p.VersionMetadata.License = append(p.VersionMetadata.License, value) + case "depend": + p.VersionMetadata.Depends = append(p.VersionMetadata.Depends, value) + case "optdepend": + p.VersionMetadata.OptDepends = append(p.VersionMetadata.OptDepends, value) + case "makedepend": + p.VersionMetadata.MakeDepends = append(p.VersionMetadata.MakeDepends, value) + case "checkdepend": + p.VersionMetadata.CheckDepends = append(p.VersionMetadata.CheckDepends, value) + case "backup": + p.VersionMetadata.Backup = append(p.VersionMetadata.Backup, value) + case "group": + p.VersionMetadata.Groups = append(p.VersionMetadata.Groups, value) + case "builddate": + bd, err := strconv.ParseInt(value, 10, 64) + if err != nil { + return nil, err + } + p.FileMetadata.BuildDate = bd + case "size": + is, err := strconv.ParseInt(value, 10, 64) + if err != nil { + return nil, err + } + p.FileMetadata.InstalledSize = is + } + } + + return p, errors.Join(scanner.Err(), ValidatePackageSpec(p)) +} + +// Arch package validation according to PKGBUILD specification. +func ValidatePackageSpec(p *Package) error { + if !reName.MatchString(p.Name) { + return util.NewInvalidArgumentErrorf("invalid package name") + } + if !reName.MatchString(p.VersionMetadata.Base) { + return util.NewInvalidArgumentErrorf("invalid package base") + } + if !reVer.MatchString(p.Version) { + return util.NewInvalidArgumentErrorf("invalid package version") + } + if p.FileMetadata.Arch == "" { + return util.NewInvalidArgumentErrorf("architecture should be specified") + } + if p.VersionMetadata.ProjectURL != "" { + if !validation.IsValidURL(p.VersionMetadata.ProjectURL) { + return util.NewInvalidArgumentErrorf("invalid project URL") + } + } + for _, cd := range p.VersionMetadata.CheckDepends { + if !rePkgVer.MatchString(cd) { + return util.NewInvalidArgumentErrorf("invalid check dependency: " + cd) + } + } + for _, d := range p.VersionMetadata.Depends { + if !rePkgVer.MatchString(d) { + return util.NewInvalidArgumentErrorf("invalid dependency: " + d) + } + } + for _, md := range p.VersionMetadata.MakeDepends { + if !rePkgVer.MatchString(md) { + return util.NewInvalidArgumentErrorf("invalid make dependency: " + md) + } + } + for _, p := range p.VersionMetadata.Provides { + if !rePkgVer.MatchString(p) { + return util.NewInvalidArgumentErrorf("invalid provides: " + p) + } + } + for _, od := range p.VersionMetadata.OptDepends { + if !reOptDep.MatchString(od) { + return util.NewInvalidArgumentErrorf("invalid optional dependency: " + od) + } + } + for _, bf := range p.VersionMetadata.Backup { + if strings.HasPrefix(bf, "/") { + return util.NewInvalidArgumentErrorf("backup file contains leading forward slash") + } + } + return nil +} + +// Create pacman package description file. +func (p *Package) Desc() string { + entries := [40]string{ + "FILENAME", fmt.Sprintf("%s-%s-%s.pkg.tar.zst", p.Name, p.Version, p.FileMetadata.Arch), + "NAME", p.Name, + "BASE", p.VersionMetadata.Base, + "VERSION", p.Version, + "DESC", p.VersionMetadata.Description, + "GROUPS", strings.Join(p.VersionMetadata.Groups, "\n"), + "CSIZE", fmt.Sprintf("%d", p.FileMetadata.CompressedSize), + "ISIZE", fmt.Sprintf("%d", p.FileMetadata.InstalledSize), + "MD5SUM", p.FileMetadata.MD5, + "SHA256SUM", p.FileMetadata.SHA256, + "URL", p.VersionMetadata.ProjectURL, + "LICENSE", strings.Join(p.VersionMetadata.License, "\n"), + "ARCH", p.FileMetadata.Arch, + "BUILDDATE", fmt.Sprintf("%d", p.FileMetadata.BuildDate), + "PACKAGER", p.FileMetadata.Packager, + "PROVIDES", strings.Join(p.VersionMetadata.Provides, "\n"), + "DEPENDS", strings.Join(p.VersionMetadata.Depends, "\n"), + "OPTDEPENDS", strings.Join(p.VersionMetadata.OptDepends, "\n"), + "MAKEDEPENDS", strings.Join(p.VersionMetadata.MakeDepends, "\n"), + "CHECKDEPENDS", strings.Join(p.VersionMetadata.CheckDepends, "\n"), + } + + var buf bytes.Buffer + for i := 0; i < 40; i += 2 { + if entries[i+1] != "" { + fmt.Fprintf(&buf, "%%%s%%\n%s\n\n", entries[i], entries[i+1]) + } + } + return buf.String() +} + +// Create pacman database archive based on provided package metadata structs. +func CreatePacmanDb(entries map[string][]byte) (*bytes.Buffer, error) { + var b bytes.Buffer + + gw := gzip.NewWriter(&b) + tw := tar.NewWriter(gw) + + for name, content := range entries { + header := &tar.Header{ + Name: name, + Size: int64(len(content)), + Mode: int64(os.ModePerm), + } + + if err := tw.WriteHeader(header); err != nil { + return nil, errors.Join(err, tw.Close(), gw.Close()) + } + + if _, err := tw.Write(content); err != nil { + return nil, errors.Join(err, tw.Close(), gw.Close()) + } + } + + return &b, errors.Join(tw.Close(), gw.Close()) +} diff --git a/modules/packages/arch/metadata_test.go b/modules/packages/arch/metadata_test.go new file mode 100644 index 000000000000..0f3cede9ccff --- /dev/null +++ b/modules/packages/arch/metadata_test.go @@ -0,0 +1,452 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package arch + +import ( + "bytes" + "encoding/base64" + "errors" + "io" + "os" + "strings" + "testing" + "testing/fstest" + "time" + + "github.com/mholt/archiver/v3" + "github.com/stretchr/testify/assert" +) + +func TestParsePackage(t *testing.T) { + // Minimal PKGINFO contents and test FS + const PKGINFO = `pkgname = a +pkgbase = b +pkgver = 1-2 +arch = x86_64 +` + fs := fstest.MapFS{ + "pkginfo": &fstest.MapFile{ + Data: []byte(PKGINFO), + Mode: os.ModePerm, + ModTime: time.Now(), + }, + "mtree": &fstest.MapFile{ + Data: []byte("data"), + Mode: os.ModePerm, + ModTime: time.Now(), + }, + } + + // Test .PKGINFO file + pinf, err := fs.Stat("pkginfo") + assert.NoError(t, err) + + pfile, err := fs.Open("pkginfo") + assert.NoError(t, err) + + parcname, err := archiver.NameInArchive(pinf, ".PKGINFO", ".PKGINFO") + assert.NoError(t, err) + + // Test .MTREE file + minf, err := fs.Stat("mtree") + assert.NoError(t, err) + + mfile, err := fs.Open("mtree") + assert.NoError(t, err) + + marcname, err := archiver.NameInArchive(minf, ".MTREE", ".MTREE") + assert.NoError(t, err) + + t.Run("normal archive", func(t *testing.T) { + var buf bytes.Buffer + + archive := archiver.NewTarZstd() + archive.Create(&buf) + + err = archive.Write(archiver.File{ + FileInfo: archiver.FileInfo{ + FileInfo: pinf, + CustomName: parcname, + }, + ReadCloser: pfile, + }) + assert.NoError(t, errors.Join(pfile.Close(), err)) + + err = archive.Write(archiver.File{ + FileInfo: archiver.FileInfo{ + FileInfo: minf, + CustomName: marcname, + }, + ReadCloser: mfile, + }) + assert.NoError(t, errors.Join(mfile.Close(), archive.Close(), err)) + + _, err = ParsePackage(&buf, []byte{}, []byte{}, 0) + + assert.NoError(t, err) + }) + + t.Run("missing .PKGINFO", func(t *testing.T) { + var buf bytes.Buffer + + archive := archiver.NewTarZstd() + archive.Create(&buf) + + assert.NoError(t, archive.Close()) + + _, err = ParsePackage(&buf, []byte{}, []byte{}, 0) + + assert.Error(t, err) + assert.Contains(t, err.Error(), ".PKGINFO file not found") + }) + + t.Run("missing .MTREE", func(t *testing.T) { + var buf bytes.Buffer + + pfile, err := fs.Open("pkginfo") + assert.NoError(t, err) + + archive := archiver.NewTarZstd() + archive.Create(&buf) + + err = archive.Write(archiver.File{ + FileInfo: archiver.FileInfo{ + FileInfo: pinf, + CustomName: parcname, + }, + ReadCloser: pfile, + }) + assert.NoError(t, errors.Join(pfile.Close(), archive.Close(), err)) + + _, err = ParsePackage(&buf, []byte{}, []byte{}, 0) + + assert.Error(t, err) + assert.Contains(t, err.Error(), ".MTREE file not found") + }) +} + +func TestParsePackageInfo(t *testing.T) { + const PKGINFO = `# Generated by makepkg 6.0.2 +# using fakeroot version 1.31 +pkgname = a +pkgbase = b +pkgver = 1-2 +pkgdesc = comment +url = https://example.com/ +group = group +builddate = 3 +packager = Name Surname +size = 5 +arch = x86_64 +license = BSD +provides = pvd +depend = smth +optdepend = hex +checkdepend = ola +makedepend = cmake +backup = usr/bin/paket1 +` + p, err := ParsePackageInfo(strings.NewReader(PKGINFO)) + assert.NoError(t, err) + assert.Equal(t, Package{ + Name: "a", + Version: "1-2", + VersionMetadata: VersionMetadata{ + Base: "b", + Description: "comment", + ProjectURL: "https://example.com/", + Groups: []string{"group"}, + Provides: []string{"pvd"}, + License: []string{"BSD"}, + Depends: []string{"smth"}, + OptDepends: []string{"hex"}, + MakeDepends: []string{"cmake"}, + CheckDepends: []string{"ola"}, + Backup: []string{"usr/bin/paket1"}, + }, + FileMetadata: FileMetadata{ + InstalledSize: 5, + BuildDate: 3, + Packager: "Name Surname ", + Arch: "x86_64", + }, + }, *p) +} + +func TestValidatePackageSpec(t *testing.T) { + newpkg := func() Package { + return Package{ + Name: "abc", + Version: "1-1", + VersionMetadata: VersionMetadata{ + Base: "ghx", + Description: "whoami", + ProjectURL: "https://example.com/", + Groups: []string{"gnome"}, + Provides: []string{"abc", "def"}, + License: []string{"GPL"}, + Depends: []string{"go", "gpg=1", "curl>=3", "git<=7"}, + OptDepends: []string{"git: something", "make"}, + MakeDepends: []string{"chrom"}, + CheckDepends: []string{"bariy"}, + Backup: []string{"etc/pacman.d/filo"}, + }, + FileMetadata: FileMetadata{ + CompressedSize: 1, + InstalledSize: 2, + MD5: "abc", + SHA256: "def", + BuildDate: 3, + Packager: "smon", + Arch: "x86_64", + }, + } + } + + t.Run("valid package", func(t *testing.T) { + p := newpkg() + + err := ValidatePackageSpec(&p) + + assert.NoError(t, err) + }) + + t.Run("invalid package name", func(t *testing.T) { + p := newpkg() + p.Name = "!$%@^!*&()" + + err := ValidatePackageSpec(&p) + + assert.Error(t, err) + assert.Contains(t, err.Error(), "invalid package name") + }) + + t.Run("invalid package base", func(t *testing.T) { + p := newpkg() + p.VersionMetadata.Base = "!$%@^!*&()" + + err := ValidatePackageSpec(&p) + + assert.Error(t, err) + assert.Contains(t, err.Error(), "invalid package base") + }) + + t.Run("invalid package version", func(t *testing.T) { + p := newpkg() + p.VersionMetadata.Base = "una-luna?" + + err := ValidatePackageSpec(&p) + + assert.Error(t, err) + assert.Contains(t, err.Error(), "invalid package base") + }) + + t.Run("invalid package version", func(t *testing.T) { + p := newpkg() + p.Version = "una-luna" + + err := ValidatePackageSpec(&p) + + assert.Error(t, err) + assert.Contains(t, err.Error(), "invalid package version") + }) + + t.Run("missing architecture", func(t *testing.T) { + p := newpkg() + p.FileMetadata.Arch = "" + + err := ValidatePackageSpec(&p) + + assert.Error(t, err) + assert.Contains(t, err.Error(), "architecture should be specified") + }) + + t.Run("invalid URL", func(t *testing.T) { + p := newpkg() + p.VersionMetadata.ProjectURL = "http%%$#" + + err := ValidatePackageSpec(&p) + + assert.Error(t, err) + assert.Contains(t, err.Error(), "invalid project URL") + }) + + t.Run("invalid check dependency", func(t *testing.T) { + p := newpkg() + p.VersionMetadata.CheckDepends = []string{"Err^_^"} + + err := ValidatePackageSpec(&p) + + assert.Error(t, err) + assert.Contains(t, err.Error(), "invalid check dependency") + }) + + t.Run("invalid dependency", func(t *testing.T) { + p := newpkg() + p.VersionMetadata.Depends = []string{"^^abc"} + + err := ValidatePackageSpec(&p) + + assert.Error(t, err) + assert.Contains(t, err.Error(), "invalid dependency") + }) + + t.Run("invalid make dependency", func(t *testing.T) { + p := newpkg() + p.VersionMetadata.MakeDepends = []string{"^m^"} + + err := ValidatePackageSpec(&p) + + assert.Error(t, err) + assert.Contains(t, err.Error(), "invalid make dependency") + }) + + t.Run("invalid provides", func(t *testing.T) { + p := newpkg() + p.VersionMetadata.Provides = []string{"^m^"} + + err := ValidatePackageSpec(&p) + + assert.Error(t, err) + assert.Contains(t, err.Error(), "invalid provides") + }) + + t.Run("invalid optional dependency", func(t *testing.T) { + p := newpkg() + p.VersionMetadata.OptDepends = []string{"^m^:MM"} + + err := ValidatePackageSpec(&p) + + assert.Error(t, err) + assert.Contains(t, err.Error(), "invalid optional dependency") + }) + + t.Run("invalid optional dependency", func(t *testing.T) { + p := newpkg() + p.VersionMetadata.Backup = []string{"/ola/cola"} + + err := ValidatePackageSpec(&p) + + assert.Error(t, err) + assert.Contains(t, err.Error(), "backup file contains leading forward slash") + }) +} + +func TestDescString(t *testing.T) { + const pkgdesc = `%FILENAME% +zstd-1.5.5-1-x86_64.pkg.tar.zst + +%NAME% +zstd + +%BASE% +zstd + +%VERSION% +1.5.5-1 + +%DESC% +Zstandard - Fast real-time compression algorithm + +%GROUPS% +dummy1 +dummy2 + +%CSIZE% +401 + +%ISIZE% +1500453 + +%MD5SUM% +5016660ef3d9aa148a7b72a08d3df1b2 + +%SHA256SUM% +9fa4ede47e35f5971e4f26ecadcbfb66ab79f1d638317ac80334a3362dedbabd + +%URL% +https://facebook.github.io/zstd/ + +%LICENSE% +BSD +GPL2 + +%ARCH% +x86_64 + +%BUILDDATE% +1681646714 + +%PACKAGER% +Jelle van der Waa + +%PROVIDES% +libzstd.so=1-64 + +%DEPENDS% +glibc +gcc-libs +zlib +xz +lz4 + +%OPTDEPENDS% +dummy3 +dummy4 + +%MAKEDEPENDS% +cmake +gtest +ninja + +%CHECKDEPENDS% +dummy5 +dummy6 + +` + + md := &Package{ + Name: "zstd", + Version: "1.5.5-1", + VersionMetadata: VersionMetadata{ + Base: "zstd", + Description: "Zstandard - Fast real-time compression algorithm", + ProjectURL: "https://facebook.github.io/zstd/", + Groups: []string{"dummy1", "dummy2"}, + Provides: []string{"libzstd.so=1-64"}, + License: []string{"BSD", "GPL2"}, + Depends: []string{"glibc", "gcc-libs", "zlib", "xz", "lz4"}, + OptDepends: []string{"dummy3", "dummy4"}, + MakeDepends: []string{"cmake", "gtest", "ninja"}, + CheckDepends: []string{"dummy5", "dummy6"}, + }, + FileMetadata: FileMetadata{ + CompressedSize: 401, + InstalledSize: 1500453, + MD5: "5016660ef3d9aa148a7b72a08d3df1b2", + SHA256: "9fa4ede47e35f5971e4f26ecadcbfb66ab79f1d638317ac80334a3362dedbabd", + BuildDate: 1681646714, + Packager: "Jelle van der Waa ", + Arch: "x86_64", + }, + } + assert.Equal(t, pkgdesc, md.Desc()) +} + +func TestCreatePacmanDb(t *testing.T) { + const dbarchive = "H4sIAAAAAAAA/0rLzEnVS60oYaAhMDAwMDA3NwfTBgYG6LSBgYEpEtuAwcDQwMzUgEHBgJaOgoHS4pLEIgYDiu1C99wQASmlubmVA+2IUTAKRsEoGAV0B4AAAAD//2VF3KIACAAA" + + db, err := CreatePacmanDb(map[string][]byte{ + "file.ext": []byte("dummy"), + }) + assert.NoError(t, err) + + actual, err := io.ReadAll(db) + assert.NoError(t, err) + + expected, err := base64.RawStdEncoding.DecodeString(dbarchive) + assert.NoError(t, err) + + assert.Equal(t, expected, actual) +} diff --git a/modules/setting/packages.go b/modules/setting/packages.go index b225615a2401..c4a4b9695951 100644 --- a/modules/setting/packages.go +++ b/modules/setting/packages.go @@ -24,6 +24,7 @@ var ( LimitTotalOwnerCount int64 LimitTotalOwnerSize int64 LimitSizeAlpine int64 + LimitSizeArch int64 LimitSizeCargo int64 LimitSizeChef int64 LimitSizeComposer int64 @@ -82,6 +83,7 @@ func loadPackagesFrom(rootCfg ConfigProvider) (err error) { Packages.LimitTotalOwnerSize = mustBytes(sec, "LIMIT_TOTAL_OWNER_SIZE") Packages.LimitSizeAlpine = mustBytes(sec, "LIMIT_SIZE_ALPINE") + Packages.LimitSizeArch = mustBytes(sec, "LIMIT_SIZE_ARCH") Packages.LimitSizeCargo = mustBytes(sec, "LIMIT_SIZE_CARGO") Packages.LimitSizeChef = mustBytes(sec, "LIMIT_SIZE_CHEF") Packages.LimitSizeComposer = mustBytes(sec, "LIMIT_SIZE_COMPOSER") diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 6a08041a7c8b..ec328510e3f6 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -3449,6 +3449,16 @@ alpine.repository = Repository Info alpine.repository.branches = Branches alpine.repository.repositories = Repositories alpine.repository.architectures = Architectures +arch.pacmanconf = Add server with related distribution and architecture to /etc/pacman.conf: +arch.pacmansync = Sync package with pacman: +arch.documentation = For more information on the arch mirrors, see %sthe documentation%s. +arch.properties = Package properties +arch.description = Description +arch.provides = Provides +arch.depends = Depends +arch.optdepends = Optional depends +arch.makedepends = Make depends +arch.checkdepends = Check depends cargo.registry = Setup this registry in the Cargo configuration file (for example ~/.cargo/config.toml): cargo.install = To install the package using Cargo, run the following command: chef.registry = Setup this registry in your ~/.chef/config.rb file: diff --git a/public/assets/img/svg/gitea-arch.svg b/public/assets/img/svg/gitea-arch.svg new file mode 100644 index 000000000000..943a92c57946 --- /dev/null +++ b/public/assets/img/svg/gitea-arch.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/routers/api/packages/api.go b/routers/api/packages/api.go index 5e3cbac8f9cb..872e0827bc94 100644 --- a/routers/api/packages/api.go +++ b/routers/api/packages/api.go @@ -14,6 +14,7 @@ import ( "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/web" "code.gitea.io/gitea/routers/api/packages/alpine" + "code.gitea.io/gitea/routers/api/packages/arch" "code.gitea.io/gitea/routers/api/packages/cargo" "code.gitea.io/gitea/routers/api/packages/chef" "code.gitea.io/gitea/routers/api/packages/composer" @@ -121,6 +122,12 @@ func CommonRoutes() *web.Route { }) }) }, reqPackageAccess(perm.AccessModeRead)) + r.Group("/arch", func() { + r.Put("/push/{filename}/{distro}", reqPackageAccess(perm.AccessModeWrite), arch.Push) + r.Put("/push/{filename}/{distro}/{sign}", reqPackageAccess(perm.AccessModeWrite), arch.Push) + r.Delete("/remove/{package}/{version}", reqPackageAccess(perm.AccessModeWrite), arch.Remove) + r.Get("/{distro}/{arch}/{file}", arch.Get) + }) r.Group("/cargo", func() { r.Group("/api/v1/crates", func() { r.Get("", cargo.SearchPackages) diff --git a/routers/api/packages/arch/arch.go b/routers/api/packages/arch/arch.go new file mode 100644 index 000000000000..358ca81c6e73 --- /dev/null +++ b/routers/api/packages/arch/arch.go @@ -0,0 +1,135 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package arch + +import ( + "bytes" + "net/http" + "strings" + + packages_model "code.gitea.io/gitea/models/packages" + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/routers/api/packages/helper" + packages_service "code.gitea.io/gitea/services/packages" + arch_service "code.gitea.io/gitea/services/packages/arch" +) + +func apiError(ctx *context.Context, status int, obj any) { + helper.LogAndProcessError(ctx, status, obj, func(message string) { + ctx.PlainText(status, message) + }) +} + +// Push new package to arch package registry. +func Push(ctx *context.Context) { + var ( + filename = ctx.Params("filename") + distro = ctx.Params("distro") + sign = ctx.Params("sign") + ) + + upload, close, err := ctx.UploadStream() + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + if close { + defer upload.Close() + } + + _, _, err = arch_service.UploadArchPackage(ctx, upload, filename, distro, sign) + if err != nil { + switch err { + case packages_model.ErrDuplicatePackageVersion, packages_model.ErrDuplicatePackageFile: + apiError(ctx, http.StatusConflict, err) + case packages_service.ErrQuotaTotalCount, packages_service.ErrQuotaTypeSize, packages_service.ErrQuotaTotalSize: + apiError(ctx, http.StatusForbidden, err) + default: + apiError(ctx, http.StatusInternalServerError, err) + } + return + } + + ctx.Status(http.StatusOK) +} + +// Get file from arch package registry. +func Get(ctx *context.Context) { + var ( + file = ctx.Params("file") + owner = ctx.Params("username") + distro = ctx.Params("distro") + arch = ctx.Params("arch") + ) + + if strings.HasSuffix(file, ".pkg.tar.zst") { + pkg, err := arch_service.GetPackageFile(ctx, distro, file) + if err != nil { + apiError(ctx, http.StatusNotFound, err) + return + } + + ctx.ServeContent(pkg, &context.ServeHeaderOptions{ + Filename: file, + }) + return + } + + if strings.HasSuffix(file, ".pkg.tar.zst.sig") { + sig, err := arch_service.GetPackageSignature(ctx, distro, file) + if err != nil { + apiError(ctx, http.StatusNotFound, err) + return + } + + ctx.ServeContent(sig, &context.ServeHeaderOptions{ + Filename: file, + }) + return + } + + if strings.HasSuffix(file, ".db.tar.gz") || strings.HasSuffix(file, ".db") { + db, err := arch_service.CreatePacmanDb(ctx, owner, arch, distro) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + ctx.ServeContent(bytes.NewReader(db.Bytes()), &context.ServeHeaderOptions{ + Filename: file, + }) + return + } + + ctx.Status(http.StatusNotFound) +} + +// Remove specific package version, related files with properties. +func Remove(ctx *context.Context) { + var ( + pkg = ctx.Params("package") + ver = ctx.Params("version") + ) + + version, err := packages_model.GetVersionByNameAndVersion( + ctx, ctx.Package.Owner.ID, packages_model.TypeArch, pkg, ver, + ) + if err != nil { + switch err { + case packages_model.ErrPackageNotExist: + apiError(ctx, http.StatusNotFound, err) + default: + apiError(ctx, http.StatusInternalServerError, err) + } + return + } + + err = packages_service.RemovePackageVersion(ctx, ctx.Package.Owner, version) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + ctx.Status(http.StatusOK) +} diff --git a/routers/web/user/package.go b/routers/web/user/package.go index 2a1879668736..8cef1afba940 100644 --- a/routers/web/user/package.go +++ b/routers/web/user/package.go @@ -177,7 +177,7 @@ func ViewPackageVersion(ctx *context.Context) { ctx.Data["PackageDescriptor"] = pd switch pd.Package.Type { - case packages_model.TypeContainer: + case packages_model.TypeContainer, packages_model.TypeArch: ctx.Data["RegistryHost"] = setting.Packages.RegistryHost case packages_model.TypeAlpine: branches := make(container.Set[string]) diff --git a/services/packages/arch/service.go b/services/packages/arch/service.go new file mode 100644 index 000000000000..a5859009c05c --- /dev/null +++ b/services/packages/arch/service.go @@ -0,0 +1,133 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package arch + +import ( + "bytes" + "encoding/hex" + "errors" + "fmt" + "io" + "sort" + "strings" + + packages_model "code.gitea.io/gitea/models/packages" + "code.gitea.io/gitea/modules/context" + arch_module "code.gitea.io/gitea/modules/packages/arch" + packages_service "code.gitea.io/gitea/services/packages" +) + +// Get data related to provided filename and distribution, for package files +// update download counter. +func GetPackageFile(ctx *context.Context, distro, file string) (io.ReadSeekCloser, error) { + pf, err := getPackageFile(ctx, distro, file) + if err != nil { + return nil, err + } + + filestream, _, _, err := packages_service.GetPackageFileStream(ctx, pf) + return filestream, err +} + +// This function will search for package signature and if present, will load it +// from package file properties, and return its byte reader. +func GetPackageSignature(ctx *context.Context, distro, file string) (*bytes.Reader, error) { + pf, err := getPackageFile(ctx, distro, strings.TrimSuffix(file, ".sig")) + if err != nil { + return nil, err + } + + proprs, err := packages_model.GetProperties(ctx, packages_model.PropertyTypeFile, pf.ID) + if err != nil { + return nil, err + } + + for _, pp := range proprs { + if pp.Name == arch_module.PropertySignature { + b, err := hex.DecodeString(pp.Value) + if err != nil { + return nil, err + } + return bytes.NewReader(b), nil + } + } + + return nil, errors.New("signature for requested package not found") +} + +// Ejects parameters required to get package file property from file name. +func getPackageFile(ctx *context.Context, distro, file string) (*packages_model.PackageFile, error) { + var ( + splt = strings.Split(file, "-") + pkgname = strings.Join(splt[0:len(splt)-3], "-") + vername = splt[len(splt)-3] + "-" + splt[len(splt)-2] + ) + + version, err := packages_model.GetVersionByNameAndVersion( + ctx, ctx.Package.Owner.ID, packages_model.TypeArch, pkgname, vername, + ) + if err != nil { + return nil, err + } + + pkgfile, err := packages_model.GetFileForVersionByName(ctx, version.ID, file, distro) + if err != nil { + return nil, err + } + return pkgfile, nil +} + +// Finds all arch packages in user/organization scope, each package version +// starting from latest in descending order is checked to be compatible with +// requested combination of architecture and distribution. When/If the first +// compatible version is found, related desc file will be loaded from package +// properties and added to resulting .db.tar.gz archive. +func CreatePacmanDb(ctx *context.Context, owner, arch, distro string) (*bytes.Buffer, error) { + pkgs, err := packages_model.GetPackagesByType(ctx, ctx.Package.Owner.ID, packages_model.TypeArch) + if err != nil { + return nil, err + } + + entries := make(map[string][]byte) + + for _, pkg := range pkgs { + versions, err := packages_model.GetVersionsByPackageName( + ctx, ctx.Package.Owner.ID, packages_model.TypeArch, pkg.Name, + ) + if err != nil { + return nil, err + } + + sort.Slice(versions, func(i, j int) bool { + return versions[i].CreatedUnix > versions[j].CreatedUnix + }) + + for _, ver := range versions { + file := fmt.Sprintf("%s-%s-%s.pkg.tar.zst", pkg.Name, ver.Version, arch) + + pf, err := packages_model.GetFileForVersionByName(ctx, ver.ID, file, distro) + if err != nil { + file = fmt.Sprintf("%s-%s-any.pkg.tar.zst", pkg.Name, ver.Version) + pf, err = packages_model.GetFileForVersionByName(ctx, ver.ID, file, distro) + if err != nil { + continue + } + } + + pps, err := packages_model.GetPropertiesByName( + ctx, packages_model.PropertyTypeFile, pf.ID, arch_module.PropertyDescription, + ) + if err != nil { + return nil, err + } + + if len(pps) >= 1 { + entries[pkg.Name+"-"+ver.Version+"/desc"] = []byte(pps[0].Value) + break + } + } + } + + return arch_module.CreatePacmanDb(entries) +} diff --git a/services/packages/arch/upload.go b/services/packages/arch/upload.go new file mode 100644 index 000000000000..221801b4f87b --- /dev/null +++ b/services/packages/arch/upload.go @@ -0,0 +1,83 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package arch + +import ( + "encoding/hex" + "errors" + "io" + + packages_model "code.gitea.io/gitea/models/packages" + "code.gitea.io/gitea/modules/context" + packages_module "code.gitea.io/gitea/modules/packages" + arch_module "code.gitea.io/gitea/modules/packages/arch" + packages_service "code.gitea.io/gitea/services/packages" +) + +// UploadArchPackage adds an Arch Package to the registry. +// The first return value indictaes if the error is a user error. +func UploadArchPackage(ctx *context.Context, upload io.Reader, filename, distro, sign string) (bool, *packages_model.PackageVersion, error) { + buf, err := packages_module.CreateHashedBufferFromReader(upload) + if err != nil { + return false, nil, err + } + defer buf.Close() + + md5, _, sha256, _ := buf.Sums() + + p, err := arch_module.ParsePackage(buf, md5, sha256, buf.Size()) + if err != nil { + return false, nil, err + } + + _, err = buf.Seek(0, io.SeekStart) + if err != nil { + return false, nil, err + } + + properties := map[string]string{ + arch_module.PropertyDescription: p.Desc(), + } + if sign != "" { + _, err := hex.DecodeString(sign) + if err != nil { + return true, nil, errors.New("unable to decode package signature") + } + properties[arch_module.PropertySignature] = sign + } + + ver, _, err := packages_service.CreatePackageOrAddFileToExisting( + ctx, &packages_service.PackageCreationInfo{ + PackageInfo: packages_service.PackageInfo{ + Owner: ctx.Package.Owner, + PackageType: packages_model.TypeArch, + Name: p.Name, + Version: p.Version, + }, + Creator: ctx.Doer, + Metadata: p.VersionMetadata, + }, + &packages_service.PackageFileCreationInfo{ + PackageFileInfo: packages_service.PackageFileInfo{ + Filename: filename, + CompositeKey: distro, + }, + OverwriteExisting: true, + IsLead: true, + Creator: ctx.Doer, + Data: buf, + Properties: properties, + }, + ) + if err != nil { + switch err { + case packages_model.ErrDuplicatePackageVersion, packages_model.ErrDuplicatePackageFile, packages_service.ErrQuotaTotalCount, packages_service.ErrQuotaTypeSize, packages_service.ErrQuotaTotalSize: + return true, nil, err + default: + return false, nil, err + } + } + + return false, ver, nil +} diff --git a/services/packages/packages.go b/services/packages/packages.go index 64b1ddd86963..797029f0cb47 100644 --- a/services/packages/packages.go +++ b/services/packages/packages.go @@ -355,6 +355,8 @@ func CheckSizeQuotaExceeded(ctx context.Context, doer, owner *user_model.User, p switch packageType { case packages_model.TypeAlpine: typeSpecificSize = setting.Packages.LimitSizeAlpine + case packages_model.TypeArch: + typeSpecificSize = setting.Packages.LimitSizeArch case packages_model.TypeCargo: typeSpecificSize = setting.Packages.LimitSizeCargo case packages_model.TypeChef: diff --git a/templates/package/content/arch.tmpl b/templates/package/content/arch.tmpl new file mode 100644 index 000000000000..b457af031a53 --- /dev/null +++ b/templates/package/content/arch.tmpl @@ -0,0 +1,70 @@ +{{if eq .PackageDescriptor.Package.Type "arch"}} +

{{ctx.Locale.Tr "packages.installation"}}

+
+
+ +
+ + 
[{{.PackageDescriptor.Owner.LowerName}}.{{.RegistryHost}}]
+SigLevel = Optional TrustAll
+Server =
+
+ +
+ + 
pacman -Sy {{.PackageDescriptor.Package.LowerName}}
+
+ +
+ {{ctx.Locale.Tr "packages.arch.documentation" (printf ``) (printf ``) | Safe}} +
+
+
+ +

{{ctx.Locale.Tr "packages.arch.properties"}}

+
+ + + + + + + + {{if .PackageDescriptor.Metadata.Provides}} + + + + + {{end}} + + {{if .PackageDescriptor.Metadata.Depends}} + + + + + {{end}} + + {{if .PackageDescriptor.Metadata.OptDepends}} + + + + + {{end}} + + {{if .PackageDescriptor.Metadata.MakeDepends}} + + + + + {{end}} + + {{if .PackageDescriptor.Metadata.CheckDepends}} + + + + + {{end}} + +
{{ctx.Locale.Tr "packages.arch.description"}}
{{.PackageDescriptor.Metadata.Description}}
{{ctx.Locale.Tr "packages.arch.provides"}}
{{StringUtils.Join $.PackageDescriptor.Metadata.Provides ", "}}
{{ctx.Locale.Tr "packages.arch.depends"}}
{{StringUtils.Join $.PackageDescriptor.Metadata.Depends ", "}}
{{ctx.Locale.Tr "packages.arch.optdepends"}}
{{StringUtils.Join $.PackageDescriptor.Metadata.OptDepends ", "}}
{{ctx.Locale.Tr "packages.arch.makedepends"}}
{{StringUtils.Join $.PackageDescriptor.Metadata.MakeDepends ", "}}
{{ctx.Locale.Tr "packages.arch.checkdepends"}}
{{StringUtils.Join $.PackageDescriptor.Metadata.CheckDepends ", "}}
+
+{{end}} diff --git a/templates/package/metadata/arch.tmpl b/templates/package/metadata/arch.tmpl new file mode 100644 index 000000000000..822973eb7d98 --- /dev/null +++ b/templates/package/metadata/arch.tmpl @@ -0,0 +1,4 @@ +{{if eq .PackageDescriptor.Package.Type "arch"}} + {{range .PackageDescriptor.Metadata.License}}
{{svg "octicon-law" 16 "gt-mr-3"}} {{.}}
{{end}} + {{if .PackageDescriptor.Metadata.ProjectURL}}
{{svg "octicon-link-external" 16 "mr-3"}} {{ctx.Locale.Tr "packages.details.project_site"}}
{{end}} +{{end}} diff --git a/templates/package/view.tmpl b/templates/package/view.tmpl index 6beb249a7ffc..2c80dd494d47 100644 --- a/templates/package/view.tmpl +++ b/templates/package/view.tmpl @@ -19,6 +19,7 @@
{{template "package/content/alpine" .}} + {{template "package/content/arch" .}} {{template "package/content/cargo" .}} {{template "package/content/chef" .}} {{template "package/content/composer" .}} @@ -50,6 +51,7 @@
{{svg "octicon-calendar" 16 "tw-mr-2"}} {{TimeSinceUnix .PackageDescriptor.Version.CreatedUnix ctx.Locale}}
{{svg "octicon-download" 16 "tw-mr-2"}} {{.PackageDescriptor.Version.DownloadCount}}
{{template "package/metadata/alpine" .}} + {{template "package/metadata/arch" .}} {{template "package/metadata/cargo" .}} {{template "package/metadata/chef" .}} {{template "package/metadata/composer" .}} diff --git a/tests/integration/api_packages_arch_test.go b/tests/integration/api_packages_arch_test.go new file mode 100644 index 000000000000..3a91d797dd67 --- /dev/null +++ b/tests/integration/api_packages_arch_test.go @@ -0,0 +1,454 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package integration + +import ( + "archive/tar" + "bufio" + "bytes" + "compress/gzip" + "crypto/md5" + "encoding/hex" + "errors" + "fmt" + "io" + "net/http" + "os" + "testing" + "testing/fstest" + "time" + + "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/models/packages" + "code.gitea.io/gitea/models/unittest" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/packages/arch" + "code.gitea.io/gitea/tests" + + "github.com/mholt/archiver/v3" + "github.com/minio/sha256-simd" + "github.com/stretchr/testify/assert" +) + +func TestPackageArch(t *testing.T) { + assert.NoError(t, unittest.PrepareTestDatabase()) + + var ( + user = unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1}) + + pushBatch = []*TestArchPackage{ + BuildArchPackage(t, "git", "1-1", "x86_64"), + BuildArchPackage(t, "git", "2-1", "x86_64"), + BuildArchPackage(t, "git", "1-1", "i686"), + BuildArchPackage(t, "adwaita", "1-1", "any"), + BuildArchPackage(t, "adwaita", "2-1", "any"), + } + + removeBatch = []*TestArchPackage{ + BuildArchPackage(t, "curl", "1-1", "x86_64"), + BuildArchPackage(t, "curl", "2-1", "x86_64"), + BuildArchPackage(t, "dock", "1-1", "any"), + BuildArchPackage(t, "dock", "2-1", "any"), + } + + firstDatabaseBatch = []*TestArchPackage{ + BuildArchPackage(t, "pacman", "1-1", "x86_64"), + BuildArchPackage(t, "pacman", "1-1", "i686"), + BuildArchPackage(t, "htop", "1-1", "x86_64"), + BuildArchPackage(t, "htop", "1-1", "i686"), + BuildArchPackage(t, "dash", "1-1", "any"), + } + + secondDatabaseBatch = []*TestArchPackage{ + BuildArchPackage(t, "pacman", "2-1", "x86_64"), + BuildArchPackage(t, "htop", "2-1", "i686"), + BuildArchPackage(t, "dash", "2-1", "any"), + } + + PacmanDBx86 = BuildPacmanDb(t, + secondDatabaseBatch[0].Pkg, + firstDatabaseBatch[2].Pkg, + secondDatabaseBatch[2].Pkg, + ) + + PacmanDBi686 = BuildPacmanDb(t, + firstDatabaseBatch[0].Pkg, + secondDatabaseBatch[1].Pkg, + secondDatabaseBatch[2].Pkg, + ) + + signdata = []byte{1, 2, 3, 4} + ) + + t.Run("PushWithSignature", func(t *testing.T) { + for _, p := range pushBatch { + t.Run(p.File, func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + url := fmt.Sprintf( + "/api/packages/%s/arch/push/%s/archlinux/%s", + user.Name, p.File, hex.EncodeToString(signdata), + ) + + req := NewRequestWithBody(t, "PUT", url, bytes.NewReader(p.Data)) + req = AddBasicAuthHeader(req, user.Name) + MakeRequest(t, req, http.StatusOK) + + pv, err := packages.GetVersionByNameAndVersion( + db.DefaultContext, user.ID, packages.TypeArch, p.Name, p.Ver, + ) + assert.NoError(t, err) + + pf, err := packages.GetFileForVersionByName( + db.DefaultContext, pv.ID, p.File, "archlinux", + ) + assert.NoError(t, err) + assert.NotNil(t, pf) + + pps, err := packages.GetPropertiesByName( + db.DefaultContext, packages.PropertyTypeFile, + pf.ID, arch.PropertySignature, + ) + assert.NoError(t, err) + assert.Len(t, pps, 1) + }) + } + }) + + t.Run("PushWithoutSignature", func(t *testing.T) { + for _, p := range pushBatch { + t.Run(p.File, func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + url := fmt.Sprintf( + "/api/packages/%s/arch/push/%s/parabola", + user.Name, p.File, + ) + + req := NewRequestWithBody(t, "PUT", url, bytes.NewReader(p.Data)) + req = AddBasicAuthHeader(req, user.Name) + MakeRequest(t, req, http.StatusOK) + + pv, err := packages.GetVersionByNameAndVersion( + db.DefaultContext, user.ID, packages.TypeArch, p.Name, p.Ver, + ) + assert.NoError(t, err) + + pf, err := packages.GetFileForVersionByName( + db.DefaultContext, pv.ID, p.File, "parabola", + ) + assert.NoError(t, err) + assert.NotNil(t, pf) + }) + } + }) + + t.Run("GetPackage", func(t *testing.T) { + for _, p := range pushBatch { + t.Run(p.File, func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + url := fmt.Sprintf( + "/api/packages/%s/arch/push/%s/artix/%s", + user.Name, p.File, hex.EncodeToString(signdata), + ) + req := NewRequestWithBody(t, "PUT", url, bytes.NewReader(p.Data)) + req = AddBasicAuthHeader(req, user.Name) + MakeRequest(t, req, http.StatusOK) + + url = fmt.Sprintf( + "/api/packages/%s/arch/artix/%s/%s", + user.Name, p.Arch, p.File, + ) + req = NewRequest(t, "GET", url) + resp := MakeRequest(t, req, http.StatusOK) + assert.Equal(t, p.Data, resp.Body.Bytes()) + }) + } + }) + + t.Run("GetSignature", func(t *testing.T) { + for _, p := range pushBatch { + t.Run(p.File, func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + url := fmt.Sprintf( + "/api/packages/%s/arch/push/%s/arco/%s", + user.Name, p.File, hex.EncodeToString(signdata), + ) + req := NewRequestWithBody(t, "PUT", url, bytes.NewReader(p.Data)) + req = AddBasicAuthHeader(req, user.Name) + MakeRequest(t, req, http.StatusOK) + + url = fmt.Sprintf( + "/api/packages/%s/arch/arco/%s/%s.sig", + user.Name, p.Arch, p.File, + ) + req = NewRequest(t, "GET", url) + resp := MakeRequest(t, req, http.StatusOK) + assert.Equal(t, signdata, resp.Body.Bytes()) + }) + } + }) + + t.Run("Remove", func(t *testing.T) { + for _, p := range removeBatch { + t.Run(p.File, func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + url := fmt.Sprintf( + "/api/packages/%s/arch/push/%s/manjaro/%s", + user.Name, p.File, hex.EncodeToString(signdata), + ) + req := NewRequestWithBody(t, "PUT", url, bytes.NewReader(p.Data)) + req = AddBasicAuthHeader(req, user.Name) + MakeRequest(t, req, http.StatusOK) + + url = fmt.Sprintf( + "/api/packages/%s/arch/remove/%s/%s", + user.Name, p.Name, p.Ver, + ) + req = NewRequest(t, "DELETE", url) + req = AddBasicAuthHeader(req, user.Name) + MakeRequest(t, req, http.StatusOK) + + _, err := packages.GetVersionByNameAndVersion( + db.DefaultContext, user.ID, packages.TypeArch, p.Name, p.Ver, + ) + assert.ErrorIs(t, err, packages.ErrPackageNotExist) + }) + } + }) + + t.Run("PacmanDatabase", func(t *testing.T) { + prepareDatabasePackages := func(t *testing.T) { + for _, p := range firstDatabaseBatch { + url := fmt.Sprintf( + "/api/packages/%s/arch/push/%s/ion/%s", + user.Name, p.File, hex.EncodeToString(signdata), + ) + req := NewRequestWithBody(t, "PUT", url, bytes.NewReader(p.Data)) + req = AddBasicAuthHeader(req, user.Name) + MakeRequest(t, req, http.StatusOK) + } + + // While creating pacman database, package versions are sorted by + // UnixTime, second delay is required to ensure that newer package + // version creation time differs from older packages. + time.Sleep(time.Second) + + for _, p := range secondDatabaseBatch { + url := fmt.Sprintf( + "/api/packages/%s/arch/push/%s/ion/%s", + user.Name, p.File, hex.EncodeToString(signdata), + ) + req := NewRequestWithBody(t, "PUT", url, bytes.NewReader(p.Data)) + req = AddBasicAuthHeader(req, user.Name) + MakeRequest(t, req, http.StatusOK) + } + } + + t.Run("x86_64", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + prepareDatabasePackages(t) + + url := fmt.Sprintf( + "/api/packages/%s/arch/ion/x86_64/user.db.tar.gz", user.Name, + ) + req := NewRequest(t, "GET", url) + resp := MakeRequest(t, req, http.StatusOK) + + CompareTarGzEntries(t, PacmanDBx86, resp.Body.Bytes()) + }) + + t.Run("i686", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + prepareDatabasePackages(t) + + url := fmt.Sprintf( + "/api/packages/%s/arch/ion/i686/user.db", user.Name, + ) + req := NewRequest(t, "GET", url) + resp := MakeRequest(t, req, http.StatusOK) + + CompareTarGzEntries(t, PacmanDBi686, resp.Body.Bytes()) + }) + }) +} + +type TestArchPackage struct { + Pkg arch.Package + Data []byte + File string + Name string + Ver string + Arch string +} + +func BuildArchPackage(t *testing.T, name, ver, architecture string) *TestArchPackage { + fs := fstest.MapFS{ + "pkginfo": &fstest.MapFile{ + Data: []byte(fmt.Sprintf( + "pkgname = %s\npkgbase = %s\npkgver = %s\narch = %s\n", + name, name, ver, architecture, + )), + Mode: os.ModePerm, + ModTime: time.Now(), + }, + "mtree": &fstest.MapFile{ + Data: []byte("test"), + Mode: os.ModePerm, + ModTime: time.Now(), + }, + } + + pinf, err := fs.Stat("pkginfo") + assert.NoError(t, err) + + pfile, err := fs.Open("pkginfo") + assert.NoError(t, err) + + parcname, err := archiver.NameInArchive(pinf, ".PKGINFO", ".PKGINFO") + assert.NoError(t, err) + + minf, err := fs.Stat("mtree") + assert.NoError(t, err) + + mfile, err := fs.Open("mtree") + assert.NoError(t, err) + + marcname, err := archiver.NameInArchive(minf, ".MTREE", ".MTREE") + assert.NoError(t, err) + + var buf bytes.Buffer + + archive := archiver.NewTarZstd() + archive.Create(&buf) + + err = archive.Write(archiver.File{ + FileInfo: archiver.FileInfo{ + FileInfo: pinf, + CustomName: parcname, + }, + ReadCloser: pfile, + }) + assert.NoError(t, errors.Join(pfile.Close(), err)) + + err = archive.Write(archiver.File{ + FileInfo: archiver.FileInfo{ + FileInfo: minf, + CustomName: marcname, + }, + ReadCloser: mfile, + }) + assert.NoError(t, errors.Join(mfile.Close(), archive.Close(), err)) + + md5, sha256, size := archPkgParams(buf.Bytes()) + + return &TestArchPackage{ + Data: buf.Bytes(), + Name: name, + Ver: ver, + Arch: architecture, + File: fmt.Sprintf("%s-%s-%s.pkg.tar.zst", name, ver, architecture), + Pkg: arch.Package{ + Name: name, + Version: ver, + VersionMetadata: arch.VersionMetadata{ + Base: name, + }, + FileMetadata: arch.FileMetadata{ + CompressedSize: size, + MD5: hex.EncodeToString(md5), + SHA256: hex.EncodeToString(sha256), + Arch: architecture, + }, + }, + } +} + +func archPkgParams(b []byte) ([]byte, []byte, int64) { + md5 := md5.New() + sha256 := sha256.New() + c := counter{bytes.NewReader(b), 0} + + br := bufio.NewReader(io.TeeReader(&c, io.MultiWriter(md5, sha256))) + + io.ReadAll(br) + return md5.Sum(nil), sha256.Sum(nil), int64(c.n) +} + +type counter struct { + io.Reader + n int +} + +func (w *counter) Read(p []byte) (int, error) { + n, err := w.Reader.Read(p) + w.n += n + return n, err +} + +func BuildPacmanDb(t *testing.T, pkgs ...arch.Package) []byte { + entries := map[string][]byte{} + for _, p := range pkgs { + entries[fmt.Sprintf("%s-%s/desc", p.Name, p.Version)] = []byte(p.Desc()) + } + b, err := arch.CreatePacmanDb(entries) + if err != nil { + assert.NoError(t, err) + return nil + } + return b.Bytes() +} + +func CompareTarGzEntries(t *testing.T, expected, actual []byte) { + fgz, err := gzip.NewReader(bytes.NewReader(expected)) + if err != nil { + assert.NoError(t, err) + return + } + ftar := tar.NewReader(fgz) + + validatemap := map[string]struct{}{} + + for { + h, err := ftar.Next() + if err != nil { + break + } + + validatemap[h.Name] = struct{}{} + } + + sgz, err := gzip.NewReader(bytes.NewReader(actual)) + if err != nil { + assert.NoError(t, err) + return + } + star := tar.NewReader(sgz) + + for { + h, err := star.Next() + if err != nil { + break + } + + _, ok := validatemap[h.Name] + if !ok { + assert.Fail(t, "Unexpected entry in archive: "+h.Name) + } + delete(validatemap, h.Name) + } + + if len(validatemap) == 0 { + return + } + + for e := range validatemap { + assert.Fail(t, "Entry not found in archive: "+e) + } +} diff --git a/web_src/svg/gitea-arch.svg b/web_src/svg/gitea-arch.svg new file mode 100644 index 000000000000..ba8254d8049e --- /dev/null +++ b/web_src/svg/gitea-arch.svg @@ -0,0 +1 @@ + \ No newline at end of file