From ad03c6e0a36033c6f59262d8cfd6416ae3cc93d6 Mon Sep 17 00:00:00 2001 From: Maarten Becker Date: Fri, 28 Apr 2023 08:32:32 +0200 Subject: [PATCH 01/65] More detailed branch delete message (#22696) Fix #22624 . Message explains that deleted branch can only be restored for a short time period. --- options/locale/locale_en-US.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 3fff13c5a27ef..0072ac6fc36fa 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -2380,7 +2380,7 @@ branch.already_exists = A branch named "%s" already exists. branch.delete_head = Delete branch.delete = Delete Branch "%s" branch.delete_html = Delete Branch -branch.delete_desc = Deleting a branch is permanent. It CANNOT be undone. Continue? +branch.delete_desc = Deleting a branch is permanent. Although the deleted branch may exist for a short time before cleaning up, in most cases it CANNOT be undone. Continue? branch.deletion_success = Branch "%s" has been deleted. branch.deletion_failed = Failed to delete branch "%s". branch.delete_branch_has_new_commits = Branch "%s" cannot be deleted because new commits have been added after merging. From 572af214a7256fde76d0fa69fc5791b1758455ef Mon Sep 17 00:00:00 2001 From: silverwind Date: Fri, 28 Apr 2023 19:39:18 +0200 Subject: [PATCH 02/65] Ensure final newline in `assets/go-licenses.json` (#24407) This will ensure that the file always has a final newline. I'm not sure where this bug with inconsistent final newline actually comes from, it is likely Windows-related. --------- Co-authored-by: delvh --- assets/go-licenses.json | 2 +- build/generate-go-licenses.go | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/assets/go-licenses.json b/assets/go-licenses.json index 1b6ead5df654b..090f595e740ae 100644 --- a/assets/go-licenses.json +++ b/assets/go-licenses.json @@ -1104,4 +1104,4 @@ "path": "xorm.io/xorm/LICENSE", "licenseText": "Copyright (c) 2013 - 2015 The Xorm Authors\nAll rights reserved.\n\nRedistribution and use in source and binary forms, with or without\nmodification, are permitted provided that the following conditions are met:\n\n* Redistributions of source code must retain the above copyright notice, this\n list of conditions and the following disclaimer.\n\n* Redistributions in binary form must reproduce the above copyright notice,\n this list of conditions and the following disclaimer in the documentation\n and/or other materials provided with the distribution.\n\n* Neither the name of the {organization} nor the names of its\n contributors may be used to endorse or promote products derived from\n this software without specific prior written permission.\n\nTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\nAND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\nIMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE\nDISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE\nFOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL\nDAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR\nSERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER\nCAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,\nOR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\nOF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n" } -] \ No newline at end of file +] diff --git a/build/generate-go-licenses.go b/build/generate-go-licenses.go index bdac51e71c7fa..addab0762a5f7 100644 --- a/build/generate-go-licenses.go +++ b/build/generate-go-licenses.go @@ -82,6 +82,11 @@ func main() { panic(err) } + // Ensure file has a final newline + if jsonBytes[len(jsonBytes)-1] != '\n' { + jsonBytes = append(jsonBytes, '\n') + } + err = os.WriteFile(out, jsonBytes, 0o644) if err != nil { panic(err) From a6450494c3a9c05667ae8a2ebdfdea55b8206e9b Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Sat, 29 Apr 2023 02:14:26 +0800 Subject: [PATCH 03/65] Fix unclear `IsRepositoryExist` logic (#24374) There was only one `IsRepositoryExist` function, it did: `has && isDir` However it's not right, and it would cause 500 error when creating a new repository if the dir exists. Then, it was changed to `has || isDir`, it is still incorrect, it affects the "adopt repo" logic. To make the logic clear: * IsRepositoryModelOrDirExist * IsRepositoryModelExist --- models/repo/repo.go | 16 ++++++++++------ models/repo/update.go | 4 ++-- models/repo_transfer.go | 4 ++-- modules/repository/create.go | 2 +- routers/api/v1/admin/adopt.go | 4 ++-- routers/web/admin/repos.go | 2 +- routers/web/user/setting/adopt.go | 2 +- services/repository/adopt.go | 2 +- 8 files changed, 20 insertions(+), 16 deletions(-) diff --git a/models/repo/repo.go b/models/repo/repo.go index f9de6d493db45..2e8c28cbb3a87 100644 --- a/models/repo/repo.go +++ b/models/repo/repo.go @@ -726,12 +726,9 @@ func GetRepositoriesMapByIDs(ids []int64) (map[int64]*Repository, error) { return repos, db.GetEngine(db.DefaultContext).In("id", ids).Find(&repos) } -// IsRepositoryExist returns true if the repository with given name under user has already existed. -func IsRepositoryExist(ctx context.Context, u *user_model.User, repoName string) (bool, error) { - has, err := db.GetEngine(ctx).Get(&Repository{ - OwnerID: u.ID, - LowerName: strings.ToLower(repoName), - }) +// IsRepositoryModelOrDirExist returns true if the repository with given name under user has already existed. +func IsRepositoryModelOrDirExist(ctx context.Context, u *user_model.User, repoName string) (bool, error) { + has, err := IsRepositoryModelExist(ctx, u, repoName) if err != nil { return false, err } @@ -739,6 +736,13 @@ func IsRepositoryExist(ctx context.Context, u *user_model.User, repoName string) return has || isDir, err } +func IsRepositoryModelExist(ctx context.Context, u *user_model.User, repoName string) (bool, error) { + return db.GetEngine(ctx).Get(&Repository{ + OwnerID: u.ID, + LowerName: strings.ToLower(repoName), + }) +} + // GetTemplateRepo populates repo.TemplateRepo for a generated repository and // returns an error on failure (NOTE: no error is returned for // non-generated repositories, and TemplateRepo will be left untouched) diff --git a/models/repo/update.go b/models/repo/update.go index f4cb67bb8f0e5..4894e0a1b9c5f 100644 --- a/models/repo/update.go +++ b/models/repo/update.go @@ -116,7 +116,7 @@ func CheckCreateRepository(doer, u *user_model.User, name string, overwriteOrAdo return err } - has, err := IsRepositoryExist(db.DefaultContext, u, name) + has, err := IsRepositoryModelOrDirExist(db.DefaultContext, u, name) if err != nil { return fmt.Errorf("IsRepositoryExist: %w", err) } else if has { @@ -147,7 +147,7 @@ func ChangeRepositoryName(doer *user_model.User, repo *Repository, newRepoName s return err } - has, err := IsRepositoryExist(db.DefaultContext, repo.Owner, newRepoName) + has, err := IsRepositoryModelOrDirExist(db.DefaultContext, repo.Owner, newRepoName) if err != nil { return fmt.Errorf("IsRepositoryExist: %w", err) } else if has { diff --git a/models/repo_transfer.go b/models/repo_transfer.go index 27a77f9b8cc9f..1c873cec57624 100644 --- a/models/repo_transfer.go +++ b/models/repo_transfer.go @@ -172,7 +172,7 @@ func CreatePendingRepositoryTransfer(ctx context.Context, doer, newOwner *user_m } // Check if new owner has repository with same name. - if has, err := repo_model.IsRepositoryExist(ctx, newOwner, repo.Name); err != nil { + if has, err := repo_model.IsRepositoryModelExist(ctx, newOwner, repo.Name); err != nil { return fmt.Errorf("IsRepositoryExist: %w", err) } else if has { return repo_model.ErrRepoAlreadyExist{ @@ -249,7 +249,7 @@ func TransferOwnership(doer *user_model.User, newOwnerName string, repo *repo_mo newOwnerName = newOwner.Name // ensure capitalisation matches // Check if new owner has repository with same name. - if has, err := repo_model.IsRepositoryExist(ctx, newOwner, repo.Name); err != nil { + if has, err := repo_model.IsRepositoryModelOrDirExist(ctx, newOwner, repo.Name); err != nil { return fmt.Errorf("IsRepositoryExist: %w", err) } else if has { return repo_model.ErrRepoAlreadyExist{ diff --git a/modules/repository/create.go b/modules/repository/create.go index c1395242c577b..e1f0bdcdf995e 100644 --- a/modules/repository/create.go +++ b/modules/repository/create.go @@ -35,7 +35,7 @@ func CreateRepositoryByExample(ctx context.Context, doer, u *user_model.User, re return err } - has, err := repo_model.IsRepositoryExist(ctx, u, repo.Name) + has, err := repo_model.IsRepositoryModelExist(ctx, u, repo.Name) if err != nil { return fmt.Errorf("IsRepositoryExist: %w", err) } else if has { diff --git a/routers/api/v1/admin/adopt.go b/routers/api/v1/admin/adopt.go index 47fd0ef3c30cf..ccd8be9171aa0 100644 --- a/routers/api/v1/admin/adopt.go +++ b/routers/api/v1/admin/adopt.go @@ -95,7 +95,7 @@ func AdoptRepository(ctx *context.APIContext) { } // check not a repo - has, err := repo_model.IsRepositoryExist(ctx, ctxUser, repoName) + has, err := repo_model.IsRepositoryModelExist(ctx, ctxUser, repoName) if err != nil { ctx.InternalServerError(err) return @@ -157,7 +157,7 @@ func DeleteUnadoptedRepository(ctx *context.APIContext) { } // check not a repo - has, err := repo_model.IsRepositoryExist(ctx, ctxUser, repoName) + has, err := repo_model.IsRepositoryModelExist(ctx, ctxUser, repoName) if err != nil { ctx.InternalServerError(err) return diff --git a/routers/web/admin/repos.go b/routers/web/admin/repos.go index b22546f74994a..9a0e467b48712 100644 --- a/routers/web/admin/repos.go +++ b/routers/web/admin/repos.go @@ -133,7 +133,7 @@ func AdoptOrDeleteRepository(ctx *context.Context) { repoName := dirSplit[1] // check not a repo - has, err := repo_model.IsRepositoryExist(ctx, ctxUser, repoName) + has, err := repo_model.IsRepositoryModelExist(ctx, ctxUser, repoName) if err != nil { ctx.ServerError("IsRepositoryExist", err) return diff --git a/routers/web/user/setting/adopt.go b/routers/web/user/setting/adopt.go index c9995e7278c55..01668c395422f 100644 --- a/routers/web/user/setting/adopt.go +++ b/routers/web/user/setting/adopt.go @@ -31,7 +31,7 @@ func AdoptOrDeleteRepository(ctx *context.Context) { root := user_model.UserPath(ctxUser.LowerName) // check not a repo - has, err := repo_model.IsRepositoryExist(ctx, ctxUser, dir) + has, err := repo_model.IsRepositoryModelExist(ctx, ctxUser, dir) if err != nil { ctx.ServerError("IsRepositoryExist", err) return diff --git a/services/repository/adopt.go b/services/repository/adopt.go index 94b2c3f3d573f..55e77a78a78a9 100644 --- a/services/repository/adopt.go +++ b/services/repository/adopt.go @@ -206,7 +206,7 @@ func DeleteUnadoptedRepository(ctx context.Context, doer, u *user_model.User, re } } - if exist, err := repo_model.IsRepositoryExist(ctx, u, repoName); err != nil { + if exist, err := repo_model.IsRepositoryModelExist(ctx, u, repoName); err != nil { return err } else if exist { return repo_model.ErrRepoAlreadyExist{ From 5bc9f7fcf9aece92c3fa2a0ea56e5585261a7f28 Mon Sep 17 00:00:00 2001 From: yp05327 <576951401@qq.com> Date: Sat, 29 Apr 2023 03:58:59 +0900 Subject: [PATCH 04/65] Improve commit date in commit graph (#24399) The commit date time is based on server's time zone not user's local time zone. Before: ![image](https://user-images.githubusercontent.com/18380374/235074112-cc1e032f-6b43-4876-a1bc-dd48b393866b.png) After: ![image](https://user-images.githubusercontent.com/18380374/235074056-329811e8-3e81-4b55-b5a1-580ac22b6d72.png) --- templates/repo/graph/commits.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/repo/graph/commits.tmpl b/templates/repo/graph/commits.tmpl index 44f63a7d76709..f5fc27315ad03 100644 --- a/templates/repo/graph/commits.tmpl +++ b/templates/repo/graph/commits.tmpl @@ -71,7 +71,7 @@ {{$userName}} {{end}} - {{$commit.Date}} + {{DateTime "full" $commit.Date}} {{end}} {{end}} From 3843252938e55b0e769d2291afea73242e255b1b Mon Sep 17 00:00:00 2001 From: Yarden Shoham Date: Fri, 28 Apr 2023 22:20:52 +0300 Subject: [PATCH 05/65] Changelog 1.19.2 (#24365) (#24403) Frontport #24365 Add changelog for 1.19.2 Signed-off-by: jolheiser Co-authored-by: John Olheiser Co-authored-by: Giteabot --- CHANGELOG.md | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0bdd4c04f09c8..c9d1bab896cb3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,53 @@ This changelog goes through all the changes that have been made in each release without substantial changes to our git log; to see the highlights of what has been added to each release, please refer to the [blog](https://blog.gitea.io). +## [1.19.2](https://github.com/go-gitea/gitea/releases/tag/1.19.2) - 2023-04-26 + +* SECURITY + * Require repo scope for PATs for private repos and basic authentication (#24362) (#24364) + * Only delete secrets belonging to its owner (#24284) (#24286) +* API + * Fix typo in API route (#24310) (#24332) + * Fix access token issue on some public endpoints (#24194) (#24259) +* ENHANCEMENTS + * Fix broken clone script on an empty archived repo (#24339) (#24348) + * Fix Monaco IOS keyboard button (#24341) (#24347) + * Don't set meta `theme-color` by default (#24340) (#24346) + * Wrap too long push mirror addresses (#21120) (#24334) + * Add --font-weight-bold and set previous bold to 601 (#24307) (#24331) + * Unify nightly naming across binaries and docker images (#24116) (#24308) + * Fix footer display (#24251) (#24269) + * Fix label color, fix divider in dropdown (#24215) (#24244) + * Vertical widths of containers removed (#24184) (#24211) + * Use correct locale key for forks page (#24172) (#24175) + * Sort repo topic labels by name (#24123) (#24153) + * Highlight selected file in the PR file tree (#23947) (#24126) +* BUGFIXES + * Fix auth check bug (#24382) (#24387) + * Add tags list for repos whose release setting is disabled (#23465) (#24369) + * Fix wrong error info in RepoRefForAPI (#24344) (#24351) + * Fix no edit/close/delete button in org repo project view page (#24349) + * Respect the REGISTER_MANUAL_CONFIRM setting when registering via OIDC (#24035) (#24333) + * Remove org users who belong to no teams (#24247) (#24313) + * Fix bug when deleting wiki with no code write permission (#24274) (#24295) + * Handle canceled workflow as a warning instead of a fail (#24282) (#24292) + * Load reviewer for comments when dismissing a review (#24281) (#24288) + * Show commit history for closed/merged PRs (#24238) (#24261) + * Fix owner team access mode value in team_unit table (#24224) + * Fix issue attachment handling (#24202) (#24221) + * Fix incorrect CORS default values (#24206) (#24217) + * Fix template error in pull request with deleted head repo (#24192) (#24216) + * Don't list root repository on compare page if pulls not allowed (#24183) (#24210) + * Fix calReleaseNumCommitsBehind (#24148) (#24197) + * Fix Org edit page bugs: renaming detection, maxlength (#24161) (#24171) + * Update redis library to support redis v7 (#24114) (#24156) + * Use 1.18's aria role for dropdown menus (#24144) (#24155) + * Fix 2-dot direct compare to use the right base commit (#24133) (#24150) + * Fix incorrect server error content in RunnersList (#24118) (#24121) + * Fix mismatch between hook events and github event types (#24048) (#24091) +* BUILD + * Support converting varchar to nvarchar for mssql database (#24105) (#24168) + ## [1.19.1](https://github.com/go-gitea/gitea/releases/tag/v1.19.1) - 2023-04-12 * BREAKING From bc784a705bc5a49ddb3ceffe1ef1205fd15f0c08 Mon Sep 17 00:00:00 2001 From: John Olheiser Date: Fri, 28 Apr 2023 15:51:17 -0500 Subject: [PATCH 06/65] Override alias template to preserve anchor fragment (#24394) This PR fixes an annoyance where docs aliases aren't preserving their anchor fragments. The refactor included aliases to keep old links from dying, but currently they redirect without their anchor, which was used _often_ to jump to sections. This overrides the alias template with an alternative that preserves the anchor fragment. To note, this is just a copy of the [embedded template](https://github.com/gohugoio/hugo/blob/5c7b79cf7f00aa2651dd5f0364ee575af6715a31/tpl/tplimpl/embedded/templates/alias.html), but defaults to a JS redirect that preserves the anchor, and uses the meta tag as a fallback for noscript users. --------- Signed-off-by: jolheiser --- docs/layouts/alias.html | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 docs/layouts/alias.html diff --git a/docs/layouts/alias.html b/docs/layouts/alias.html new file mode 100644 index 0000000000000..b96cb342199ed --- /dev/null +++ b/docs/layouts/alias.html @@ -0,0 +1,13 @@ + + + + + {{ .Permalink }} + + + + + + From bc4e06109dae33edc0a9a918f244563f05a1a353 Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Sat, 29 Apr 2023 05:23:19 +0800 Subject: [PATCH 07/65] Make repo size style matches others (commits/branches/tags) (#24408) The "unit" part shouldn't have bold style. --- modules/templates/util_string.go | 4 ++++ modules/translation/translation.go | 8 ++++++++ modules/translation/translation_test.go | 5 +++++ templates/repo/sub_menu.tmpl | 4 +++- 4 files changed, 20 insertions(+), 1 deletion(-) diff --git a/modules/templates/util_string.go b/modules/templates/util_string.go index e86bbe9e705f6..42d11fc99068f 100644 --- a/modules/templates/util_string.go +++ b/modules/templates/util_string.go @@ -18,3 +18,7 @@ func (su *StringUtils) HasPrefix(s, prefix string) bool { func (su *StringUtils) Contains(s, substr string) bool { return strings.Contains(s, substr) } + +func (su *StringUtils) Split(s, sep string) []string { + return strings.Split(s, sep) +} diff --git a/modules/translation/translation.go b/modules/translation/translation.go index 56cf1df2d438e..49dfa84d1b0ef 100644 --- a/modules/translation/translation.go +++ b/modules/translation/translation.go @@ -13,6 +13,7 @@ import ( "code.gitea.io/gitea/modules/options" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/translation/i18n" + "code.gitea.io/gitea/modules/util" "golang.org/x/text/language" "golang.org/x/text/message" @@ -241,5 +242,12 @@ func (l *locale) TrN(cnt any, key1, keyN string, args ...any) string { func (l *locale) PrettyNumber(v any) string { // TODO: this mechanism is not good enough, the complete solution is to switch the translation system to ICU message format + if s, ok := v.(string); ok { + if num, err := util.ToInt64(s); err == nil { + v = num + } else if num, err := util.ToFloat64(s); err == nil { + v = num + } + } return l.msgPrinter.Sprintf("%v", number.Decimal(v)) } diff --git a/modules/translation/translation_test.go b/modules/translation/translation_test.go index 83a40f145815a..464aa32661eff 100644 --- a/modules/translation/translation_test.go +++ b/modules/translation/translation_test.go @@ -21,7 +21,12 @@ func TestPrettyNumber(t *testing.T) { l := NewLocale("id-ID") assert.EqualValues(t, "1.000.000", l.PrettyNumber(1000000)) + assert.EqualValues(t, "1.000.000,1", l.PrettyNumber(1000000.1)) + assert.EqualValues(t, "1.000.000", l.PrettyNumber("1000000")) + assert.EqualValues(t, "1.000.000", l.PrettyNumber("1000000.0")) + assert.EqualValues(t, "1.000.000,1", l.PrettyNumber("1000000.1")) l = NewLocale("nosuch") assert.EqualValues(t, "1,000,000", l.PrettyNumber(1000000)) + assert.EqualValues(t, "1,000,000.1", l.PrettyNumber(1000000.1)) } diff --git a/templates/repo/sub_menu.tmpl b/templates/repo/sub_menu.tmpl index 2be7d47a1c8ac..def6beccb9d95 100644 --- a/templates/repo/sub_menu.tmpl +++ b/templates/repo/sub_menu.tmpl @@ -15,7 +15,9 @@ {{end}}
- {{svg "octicon-database"}} {{FileSize .Repository.Size}} + {{$fileSizeFormatted := FileSize .Repository.Size}}{{/* the formatted string is always "{val} {unit}" */}} + {{$fileSizeFields := StringUtils.Split $fileSizeFormatted " "}} + {{svg "octicon-database"}} {{.locale.PrettyNumber (index $fileSizeFields 0)}} {{index $fileSizeFields 1}}
{{end}} From bf77e2163b670797d5bf7199da88789968e47c61 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Fri, 28 Apr 2023 23:51:36 +0200 Subject: [PATCH 08/65] Add Debian package registry (#22854) Co-authored-by: @awkwardbunny This PR adds a Debian package registry. You can follow [this tutorial](https://www.baeldung.com/linux/create-debian-package) to build a *.deb package for testing. Source packages are not supported at the moment and I did not find documentation of the architecture "all" and how these packages should be treated. --------- Co-authored-by: Brian Hong Co-authored-by: techknowlogick --- assets/go-licenses.json | 5 + cmd/migrate_storage_test.go | 2 +- custom/conf/app.example.ini | 2 + .../config-cheat-sheet.en-us.md | 1 + .../doc/usage/packages/debian.en-us.md | 134 ++++++ .../doc/usage/packages/overview.en-us.md | 1 + go.mod | 3 +- go.sum | 2 + models/migrations/migrations.go | 2 + models/migrations/v1_20/v256.go | 23 + models/packages/container/search.go | 11 +- models/packages/debian/search.go | 131 ++++++ models/packages/descriptor.go | 26 +- models/packages/package.go | 19 +- models/packages/package_file.go | 25 +- models/packages/package_version.go | 6 +- models/user/setting.go | 5 + modules/packages/debian/metadata.go | 216 +++++++++ modules/packages/debian/metadata_test.go | 171 +++++++ modules/packages/hashed_buffer.go | 22 +- modules/packages/hashed_buffer_test.go | 2 +- modules/packages/nuget/symbol_extractor.go | 2 +- modules/setting/packages.go | 2 + modules/util/filebuffer/file_backed_buffer.go | 5 +- options/locale/locale_en-US.ini | 8 + public/img/svg/gitea-debian.svg | 1 + routers/api/packages/api.go | 19 + routers/api/packages/cargo/cargo.go | 2 +- routers/api/packages/chef/chef.go | 2 +- routers/api/packages/composer/composer.go | 2 +- routers/api/packages/conan/conan.go | 13 +- routers/api/packages/conda/conda.go | 2 +- routers/api/packages/container/blob.go | 8 +- routers/api/packages/container/container.go | 4 +- routers/api/packages/debian/debian.go | 317 +++++++++++++ routers/api/packages/generic/generic.go | 2 +- routers/api/packages/helm/helm.go | 2 +- routers/api/packages/maven/maven.go | 2 +- routers/api/packages/npm/npm.go | 2 +- routers/api/packages/nuget/nuget.go | 6 +- routers/api/packages/pub/pub.go | 2 +- routers/api/packages/pypi/pypi.go | 2 +- routers/api/packages/rubygems/rubygems.go | 2 +- routers/api/packages/swift/swift.go | 2 +- routers/api/packages/vagrant/vagrant.go | 2 +- routers/api/v1/packages/package.go | 2 +- routers/web/user/package.go | 34 +- services/forms/package_form.go | 2 +- services/packages/cleanup/cleanup.go | 11 + services/packages/debian/repository.go | 443 ++++++++++++++++++ services/packages/packages.go | 46 +- templates/package/content/debian.tmpl | 65 +++ templates/package/metadata/debian.tmpl | 4 + templates/package/view.tmpl | 2 + templates/swagger/v1_json.tmpl | 1 + tests/integration/api_packages_debian_test.go | 252 ++++++++++ web_src/svg/gitea-debian.svg | 9 + 57 files changed, 1995 insertions(+), 96 deletions(-) create mode 100644 docs/content/doc/usage/packages/debian.en-us.md create mode 100644 models/migrations/v1_20/v256.go create mode 100644 models/packages/debian/search.go create mode 100644 modules/packages/debian/metadata.go create mode 100644 modules/packages/debian/metadata_test.go create mode 100644 public/img/svg/gitea-debian.svg create mode 100644 routers/api/packages/debian/debian.go create mode 100644 services/packages/debian/repository.go create mode 100644 templates/package/content/debian.tmpl create mode 100644 templates/package/metadata/debian.tmpl create mode 100644 tests/integration/api_packages_debian_test.go create mode 100644 web_src/svg/gitea-debian.svg diff --git a/assets/go-licenses.json b/assets/go-licenses.json index 090f595e740ae..57285b6a9220e 100644 --- a/assets/go-licenses.json +++ b/assets/go-licenses.json @@ -114,6 +114,11 @@ "path": "github.com/bits-and-blooms/bitset/LICENSE", "licenseText": "Copyright (c) 2014 Will Fitzgerald. All rights reserved.\n\nRedistribution and use in source and binary forms, with or without\nmodification, are permitted provided that the following conditions are\nmet:\n\n * Redistributions of source code must retain the above copyright\nnotice, this list of conditions and the following disclaimer.\n * Redistributions in binary form must reproduce the above\ncopyright notice, this list of conditions and the following disclaimer\nin the documentation and/or other materials provided with the\ndistribution.\n * Neither the name of Google Inc. nor the names of its\ncontributors may be used to endorse or promote products derived from\nthis software without specific prior written permission.\n\nTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS\n\"AS IS\" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT\nLIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR\nA PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT\nOWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,\nSPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT\nLIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\nDATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\nTHEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\nOF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n" }, + { + "name": "github.com/blakesmith/ar", + "path": "github.com/blakesmith/ar/COPYING", + "licenseText": "Copyright (c) 2013 Blake Smith \u003cblakesmith0@gmail.com\u003e\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in\nall copies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\nTHE SOFTWARE.\n\n" + }, { "name": "github.com/blevesearch/bleve/v2", "path": "github.com/blevesearch/bleve/v2/LICENSE", diff --git a/cmd/migrate_storage_test.go b/cmd/migrate_storage_test.go index e6658b6e65ef1..7f3de894ba3e2 100644 --- a/cmd/migrate_storage_test.go +++ b/cmd/migrate_storage_test.go @@ -25,7 +25,7 @@ func TestMigratePackages(t *testing.T) { creator := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1}) content := "package main\n\nfunc main() {\nfmt.Println(\"hi\")\n}\n" - buf, err := packages_module.CreateHashedBufferFromReader(strings.NewReader(content), 1024) + buf, err := packages_module.CreateHashedBufferFromReaderWithSize(strings.NewReader(content), 1024) assert.NoError(t, err) defer buf.Close() diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index 3687e0fbd4caf..f24770860fde1 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -2501,6 +2501,8 @@ ROUTER = console ;LIMIT_SIZE_CONDA = -1 ;; Maximum size of a Container upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) ;LIMIT_SIZE_CONTAINER = -1 +;; Maximum size of a Debian upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) +;LIMIT_SIZE_DEBIAN = -1 ;; Maximum size of a Generic upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) ;LIMIT_SIZE_GENERIC = -1 ;; Maximum size of a Helm upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) diff --git a/docs/content/doc/administration/config-cheat-sheet.en-us.md b/docs/content/doc/administration/config-cheat-sheet.en-us.md index 03cd93f91eba2..44ea3633abe7a 100644 --- a/docs/content/doc/administration/config-cheat-sheet.en-us.md +++ b/docs/content/doc/administration/config-cheat-sheet.en-us.md @@ -1252,6 +1252,7 @@ Task queue configuration has been moved to `queue.task`. However, the below conf - `LIMIT_SIZE_CONAN`: **-1**: Maximum size of a Conan upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) - `LIMIT_SIZE_CONDA`: **-1**: Maximum size of a Conda upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) - `LIMIT_SIZE_CONTAINER`: **-1**: Maximum size of a Container upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) +- `LIMIT_SIZE_DEBIAN`: **-1**: Maximum size of a Debian upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) - `LIMIT_SIZE_GENERIC`: **-1**: Maximum size of a Generic upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) - `LIMIT_SIZE_HELM`: **-1**: Maximum size of a Helm upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) - `LIMIT_SIZE_MAVEN`: **-1**: Maximum size of a Maven upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) diff --git a/docs/content/doc/usage/packages/debian.en-us.md b/docs/content/doc/usage/packages/debian.en-us.md new file mode 100644 index 0000000000000..ac1abed18e912 --- /dev/null +++ b/docs/content/doc/usage/packages/debian.en-us.md @@ -0,0 +1,134 @@ +--- +date: "2023-01-07T00:00:00+00:00" +title: "Debian Packages Repository" +slug: "packages/debian" +draft: false +toc: false +menu: + sidebar: + parent: "packages" + name: "Debian" + weight: 35 + identifier: "debian" +--- + +# Debian Packages Repository + +Publish [Debian](https://www.debian.org/distrib/packages) packages for your user or organization. + +**Table of Contents** + +{{< toc >}} + +## Requirements + +To work with the Debian registry, you need to use a HTTP client like `curl` to upload and a package manager like `apt` to consume packages. + +The following examples use `apt`. + +## Configuring the package registry + +To register the Debian registry add the url to the list of known apt sources: + +```shell +echo "deb https://gitea.example.com/api/packages/{owner}/debian {distribution} {component}" | sudo tee -a /etc/apt/sources.list.d/gitea.list +``` + +| Placeholder | Description | +| -------------- | ----------- | +| `owner` | The owner of the package. | +| `distribution` | The distribution to use. | +| `component` | The component to use. | + +If the registry is private, provide credentials in the url. You can use a password or a [personal access token]({{< relref "doc/development/api-usage.en-us.md#authentication" >}}): + +```shell +echo "deb https://{username}:{your_password_or_token}@gitea.example.com/api/packages/{owner}/debian {distribution} {component}" | sudo tee -a /etc/apt/sources.list.d/gitea.list +``` + +The Debian registry files are signed with a PGP key which must be known to apt: + +```shell +sudo curl https://gitea.example.com/api/packages/{owner}/debian/repository.key -o /etc/apt/trusted.gpg.d/gitea-{owner}.asc +``` + +Afterwards update the local package index: + +```shell +apt update +``` + +## Publish a package + +To publish a Debian package (`*.deb`), perform a HTTP PUT operation with the package content in the request body. + +``` +PUT https://gitea.example.com/api/packages/{owner}/debian/pool/{distribution}/{component}/upload +``` + +| Parameter | Description | +| -------------- | ----------- | +| `owner` | The owner of the package. | +| `distribution` | The distribution may match the release name of the OS, ex: `bionic`. | +| `component` | The component can be used to group packages or just `main` or similar. | + +Example request using HTTP Basic authentication: + +```shell +curl --user your_username:your_password_or_token \ + --upload-file path/to/file.deb \ + https://gitea.example.com/api/packages/testuser/debian/pool/bionic/main/upload +``` + +If you are using 2FA or OAuth use a [personal access token]({{< relref "doc/development/api-usage.en-us.md#authentication" >}}) instead of the password. +You cannot publish a file with the same name twice to a package. You must delete the existing package version first. + +The server reponds with the following HTTP Status codes. + +| HTTP Status Code | Meaning | +| ----------------- | ------- | +| `201 Created` | The package has been published. | +| `400 Bad Request` | The package name, version, distribution, component or architecture are invalid. | +| `409 Conflict` | A package file with the same combination of parameters exist already in the package. | + +## Delete a package + +To delete a Debian package perform a HTTP DELETE operation. This will delete the package version too if there is no file left. + +``` +DELETE https://gitea.example.com/api/packages/{owner}/debian/pool/{distribution}/{component}/{package_name}/{package_version}/{architecture} +``` + +| Parameter | Description | +| ----------------- | ----------- | +| `owner` | The owner of the package. | +| `package_name` | The package name. | +| `package_version` | The package version. | +| `distribution` | The package distribution. | +| `component` | The package component. | +| `architecture` | The package architecture. | + +Example request using HTTP Basic authentication: + +```shell +curl --user your_username:your_token_or_password -X DELETE \ + https://gitea.example.com/api/packages/testuser/debian/pools/bionic/main/test-package/1.0.0/amd64 +``` + +The server reponds with the following HTTP Status codes. + +| HTTP Status Code | Meaning | +| ----------------- | ------- | +| `204 No Content` | Success | +| `404 Not Found` | The package or file was not found. | + +## Install a package + +To install a package from the Debian registry, execute the following commands: + +```shell +# use latest version +apt install {package_name} +# use specific version +apt install {package_name}={package_version} +``` diff --git a/docs/content/doc/usage/packages/overview.en-us.md b/docs/content/doc/usage/packages/overview.en-us.md index fdeaf15af0edd..8e4fd87e7b7e7 100644 --- a/docs/content/doc/usage/packages/overview.en-us.md +++ b/docs/content/doc/usage/packages/overview.en-us.md @@ -33,6 +33,7 @@ The following package managers are currently supported: | [Conan]({{< relref "doc/usage/packages/conan.en-us.md" >}}) | C++ | `conan` | | [Conda]({{< relref "doc/usage/packages/conda.en-us.md" >}}) | - | `conda` | | [Container]({{< relref "doc/usage/packages/container.en-us.md" >}}) | - | any OCI compliant client | +| [Debian]({{< relref "doc/usage/packages/debian.en-us.md" >}}) | - | `apt` | | [Generic]({{< relref "doc/usage/packages/generic.en-us.md" >}}) | - | any HTTP client | | [Helm]({{< relref "doc/usage/packages/helm.en-us.md" >}}) | - | any HTTP client, `cm-push` | | [Maven]({{< relref "doc/usage/packages/maven.en-us.md" >}}) | Java | `mvn`, `gradle` | diff --git a/go.mod b/go.mod index a71b136cb1f2c..ef5f22d96eafe 100644 --- a/go.mod +++ b/go.mod @@ -17,6 +17,7 @@ require ( github.com/NYTimes/gziphandler v1.1.1 github.com/PuerkitoBio/goquery v1.8.0 github.com/alecthomas/chroma/v2 v2.5.0 + github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb github.com/blevesearch/bleve/v2 v2.3.6 github.com/bufbuild/connect-go v1.3.1 github.com/buildkite/terminal-to-html/v3 v3.7.0 @@ -96,6 +97,7 @@ require ( github.com/stretchr/testify v1.8.1 github.com/syndtr/goleveldb v1.0.0 github.com/tstranex/u2f v1.0.0 + github.com/ulikunitz/xz v0.5.11 github.com/urfave/cli v1.22.12 github.com/xanzy/go-gitlab v0.80.2 github.com/xeipuuv/gojsonschema v1.2.0 @@ -260,7 +262,6 @@ require ( github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf // indirect github.com/subosito/gotenv v1.4.1 // indirect github.com/toqueteos/webbrowser v1.2.0 // indirect - github.com/ulikunitz/xz v0.5.11 // indirect github.com/unknwon/com v1.0.1 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/fasthttp v1.44.0 // indirect diff --git a/go.sum b/go.sum index 2a3cc5967e549..ba969dddeefbd 100644 --- a/go.sum +++ b/go.sum @@ -162,6 +162,8 @@ github.com/bits-and-blooms/bitset v1.1.10/go.mod h1:w0XsmFg8qg6cmpTtJ0z3pKgjTDBM github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= github.com/bits-and-blooms/bitset v1.5.0 h1:NpE8frKRLGHIcEzkR+gZhiioW1+WbYV6fKwD6ZIpQT8= github.com/bits-and-blooms/bitset v1.5.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= +github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb h1:m935MPodAbYS46DG4pJSv7WO+VECIWUQ7OJYSoTrMh4= +github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb/go.mod h1:PkYb9DJNAwrSvRx5DYA+gUcOIgTGVMNkfSCbZM8cWpI= github.com/blevesearch/bleve/v2 v2.0.5/go.mod h1:ZjWibgnbRX33c+vBRgla9QhPb4QOjD6fdVJ+R1Bk8LM= github.com/blevesearch/bleve/v2 v2.3.6 h1:NlntUHcV5CSWIhpugx4d/BRMGCiaoI8ZZXrXlahzNq4= github.com/blevesearch/bleve/v2 v2.3.6/go.mod h1:JM2legf1cKVkdV8Ehu7msKIOKC0McSw0Q16Fmv9vsW4= diff --git a/models/migrations/migrations.go b/models/migrations/migrations.go index 1f1f43796cf4a..0e84ae9f0e29a 100644 --- a/models/migrations/migrations.go +++ b/models/migrations/migrations.go @@ -489,6 +489,8 @@ var migrations = []Migration{ NewMigration("Add ActionTaskOutput table", v1_20.AddActionTaskOutputTable), // v255 -> v256 NewMigration("Add ArchivedUnix Column", v1_20.AddArchivedUnixToRepository), + // v256 -> v257 + NewMigration("Add is_internal column to package", v1_20.AddIsInternalColumnToPackage), } // GetCurrentDBVersion returns the current db version diff --git a/models/migrations/v1_20/v256.go b/models/migrations/v1_20/v256.go new file mode 100644 index 0000000000000..ddb487f6844f4 --- /dev/null +++ b/models/migrations/v1_20/v256.go @@ -0,0 +1,23 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package v1_20 //nolint + +import ( + "xorm.io/xorm" +) + +func AddIsInternalColumnToPackage(x *xorm.Engine) error { + type Package struct { + ID int64 `xorm:"pk autoincr"` + OwnerID int64 `xorm:"UNIQUE(s) INDEX NOT NULL"` + RepoID int64 `xorm:"INDEX"` + Type string `xorm:"UNIQUE(s) INDEX NOT NULL"` + Name string `xorm:"NOT NULL"` + LowerName string `xorm:"UNIQUE(s) INDEX NOT NULL"` + SemverCompatible bool `xorm:"NOT NULL DEFAULT false"` + IsInternal bool `xorm:"INDEX NOT NULL DEFAULT false"` + } + + return x.Sync2(new(Package)) +} diff --git a/models/packages/container/search.go b/models/packages/container/search.go index b65c8634d6544..0d3664d384349 100644 --- a/models/packages/container/search.go +++ b/models/packages/container/search.go @@ -101,16 +101,7 @@ func getContainerBlobsLimit(ctx context.Context, opts *BlobSearchOptions, limit return nil, err } - pfds := make([]*packages.PackageFileDescriptor, 0, len(pfs)) - for _, pf := range pfs { - pfd, err := packages.GetPackageFileDescriptor(ctx, pf) - if err != nil { - return nil, err - } - pfds = append(pfds, pfd) - } - - return pfds, nil + return packages.GetPackageFileDescriptors(ctx, pfs) } // GetManifestVersions gets all package versions representing the matching manifest diff --git a/models/packages/debian/search.go b/models/packages/debian/search.go new file mode 100644 index 0000000000000..332a4f7040c51 --- /dev/null +++ b/models/packages/debian/search.go @@ -0,0 +1,131 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package debian + +import ( + "context" + "strconv" + + "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/models/packages" + debian_module "code.gitea.io/gitea/modules/packages/debian" + + "xorm.io/builder" +) + +type PackageSearchOptions struct { + OwnerID int64 + Distribution string + Component string + Architecture string +} + +// SearchLatestPackages gets the latest packages matching the search options +func SearchLatestPackages(ctx context.Context, opts *PackageSearchOptions) ([]*packages.PackageFileDescriptor, error) { + var cond builder.Cond = builder.Eq{ + "package_file.is_lead": true, + "package.type": packages.TypeDebian, + "package.owner_id": opts.OwnerID, + "package.is_internal": false, + "package_version.is_internal": false, + } + + props := make(map[string]string) + if opts.Distribution != "" { + props[debian_module.PropertyDistribution] = opts.Distribution + } + if opts.Component != "" { + props[debian_module.PropertyComponent] = opts.Component + } + if opts.Architecture != "" { + props[debian_module.PropertyArchitecture] = opts.Architecture + } + + if len(props) > 0 { + var propsCond builder.Cond = builder.Eq{ + "package_property.ref_type": packages.PropertyTypeFile, + } + propsCond = propsCond.And(builder.Expr("package_property.ref_id = package_file.id")) + + propsCondBlock := builder.NewCond() + for name, value := range props { + propsCondBlock = propsCondBlock.Or(builder.Eq{ + "package_property.name": name, + "package_property.value": value, + }) + } + propsCond = propsCond.And(propsCondBlock) + + cond = cond.And(builder.Eq{ + strconv.Itoa(len(props)): builder.Select("COUNT(*)").Where(propsCond).From("package_property"), + }) + } + + cond = cond. + And(builder.Expr("pv2.id IS NULL")) + + joinCond := builder. + Expr("package_version.package_id = pv2.package_id AND (package_version.created_unix < pv2.created_unix OR (package_version.created_unix = pv2.created_unix AND package_version.id < pv2.id))"). + And(builder.Eq{"pv2.is_internal": false}) + + pfs := make([]*packages.PackageFile, 0, 10) + err := db.GetEngine(ctx). + Table("package_file"). + Select("package_file.*"). + Join("INNER", "package_version", "package_version.id = package_file.version_id"). + Join("LEFT", "package_version pv2", joinCond). + Join("INNER", "package", "package.id = package_version.package_id"). + Where(cond). + Desc("package_version.created_unix"). + Find(&pfs) + if err != nil { + return nil, err + } + + return packages.GetPackageFileDescriptors(ctx, pfs) +} + +// GetDistributions gets all available distributions +func GetDistributions(ctx context.Context, ownerID int64) ([]string, error) { + return getDistinctPropertyValues(ctx, ownerID, "", debian_module.PropertyDistribution) +} + +// GetComponents gets all available components for the given distribution +func GetComponents(ctx context.Context, ownerID int64, distribution string) ([]string, error) { + return getDistinctPropertyValues(ctx, ownerID, distribution, debian_module.PropertyComponent) +} + +// GetArchitectures gets all available architectures for the given distribution +func GetArchitectures(ctx context.Context, ownerID int64, distribution string) ([]string, error) { + return getDistinctPropertyValues(ctx, ownerID, distribution, debian_module.PropertyArchitecture) +} + +func getDistinctPropertyValues(ctx context.Context, ownerID int64, distribution, propName string) ([]string, error) { + var cond builder.Cond = builder.Eq{ + "package_property.ref_type": packages.PropertyTypeFile, + "package_property.name": propName, + "package.type": packages.TypeDebian, + "package.owner_id": ownerID, + } + if distribution != "" { + innerCond := builder. + Expr("pp.ref_id = package_property.ref_id"). + And(builder.Eq{ + "pp.ref_type": packages.PropertyTypeFile, + "pp.name": debian_module.PropertyDistribution, + "pp.value": distribution, + }) + cond = cond.And(builder.Exists(builder.Select("pp.ref_id").From("package_property pp").Where(innerCond))) + } + + values := make([]string, 0, 5) + return values, db.GetEngine(ctx). + Table("package_property"). + Distinct("package_property.value"). + Join("INNER", "package_file", "package_file.id = package_property.ref_id"). + Join("INNER", "package_version", "package_version.id = package_file.version_id"). + Join("INNER", "package", "package.id = package_version.package_id"). + Where(cond). + Find(&values) +} diff --git a/models/packages/descriptor.go b/models/packages/descriptor.go index 974c5b2c362a6..9256dd563022e 100644 --- a/models/packages/descriptor.go +++ b/models/packages/descriptor.go @@ -18,6 +18,7 @@ import ( "code.gitea.io/gitea/modules/packages/conan" "code.gitea.io/gitea/modules/packages/conda" "code.gitea.io/gitea/modules/packages/container" + "code.gitea.io/gitea/modules/packages/debian" "code.gitea.io/gitea/modules/packages/helm" "code.gitea.io/gitea/modules/packages/maven" "code.gitea.io/gitea/modules/packages/npm" @@ -127,13 +128,9 @@ func GetPackageDescriptor(ctx context.Context, pv *PackageVersion) (*PackageDesc return nil, err } - pfds := make([]*PackageFileDescriptor, 0, len(pfs)) - for _, pf := range pfs { - pfd, err := GetPackageFileDescriptor(ctx, pf) - if err != nil { - return nil, err - } - pfds = append(pfds, pfd) + pfds, err := GetPackageFileDescriptors(ctx, pfs) + if err != nil { + return nil, err } var metadata interface{} @@ -150,6 +147,8 @@ func GetPackageDescriptor(ctx context.Context, pv *PackageVersion) (*PackageDesc metadata = &conda.VersionMetadata{} case TypeContainer: metadata = &container.Metadata{} + case TypeDebian: + metadata = &debian.Metadata{} case TypeGeneric: // generic packages have no metadata case TypeHelm: @@ -210,6 +209,19 @@ func GetPackageFileDescriptor(ctx context.Context, pf *PackageFile) (*PackageFil }, nil } +// GetPackageFileDescriptors gets the package file descriptors for the package files +func GetPackageFileDescriptors(ctx context.Context, pfs []*PackageFile) ([]*PackageFileDescriptor, error) { + pfds := make([]*PackageFileDescriptor, 0, len(pfs)) + for _, pf := range pfs { + pfd, err := GetPackageFileDescriptor(ctx, pf) + if err != nil { + return nil, err + } + pfds = append(pfds, pfd) + } + return pfds, nil +} + // GetPackageDescriptors gets the package descriptions for the versions func GetPackageDescriptors(ctx context.Context, pvs []*PackageVersion) ([]*PackageDescriptor, error) { pds := make([]*PackageDescriptor, 0, len(pvs)) diff --git a/models/packages/package.go b/models/packages/package.go index ccc9257c31235..579e9e4d53351 100644 --- a/models/packages/package.go +++ b/models/packages/package.go @@ -36,6 +36,7 @@ const ( TypeConan Type = "conan" TypeConda Type = "conda" TypeContainer Type = "container" + TypeDebian Type = "debian" TypeGeneric Type = "generic" TypeHelm Type = "helm" TypeMaven Type = "maven" @@ -55,6 +56,7 @@ var TypeList = []Type{ TypeConan, TypeConda, TypeContainer, + TypeDebian, TypeGeneric, TypeHelm, TypeMaven, @@ -82,6 +84,8 @@ func (pt Type) Name() string { return "Conda" case TypeContainer: return "Container" + case TypeDebian: + return "Debian" case TypeGeneric: return "Generic" case TypeHelm: @@ -121,6 +125,8 @@ func (pt Type) SVGName() string { return "gitea-conda" case TypeContainer: return "octicon-container" + case TypeDebian: + return "gitea-debian" case TypeGeneric: return "octicon-package" case TypeHelm: @@ -154,6 +160,7 @@ type Package struct { Name string `xorm:"NOT NULL"` LowerName string `xorm:"UNIQUE(s) INDEX NOT NULL"` SemverCompatible bool `xorm:"NOT NULL DEFAULT false"` + IsInternal bool `xorm:"INDEX NOT NULL DEFAULT false"` } // TryInsertPackage inserts a package. If a package exists already, ErrDuplicatePackage is returned @@ -214,9 +221,10 @@ func GetPackageByID(ctx context.Context, packageID int64) (*Package, error) { // GetPackageByName gets a package by name func GetPackageByName(ctx context.Context, ownerID int64, packageType Type, name string) (*Package, error) { var cond builder.Cond = builder.Eq{ - "package.owner_id": ownerID, - "package.type": packageType, - "package.lower_name": strings.ToLower(name), + "package.owner_id": ownerID, + "package.type": packageType, + "package.lower_name": strings.ToLower(name), + "package.is_internal": false, } p := &Package{} @@ -236,8 +244,9 @@ func GetPackageByName(ctx context.Context, ownerID int64, packageType Type, name // GetPackagesByType gets all packages of a specific type func GetPackagesByType(ctx context.Context, ownerID int64, packageType Type) ([]*Package, error) { var cond builder.Cond = builder.Eq{ - "package.owner_id": ownerID, - "package.type": packageType, + "package.owner_id": ownerID, + "package.type": packageType, + "package.is_internal": false, } ps := make([]*Package, 0, 10) diff --git a/models/packages/package_file.go b/models/packages/package_file.go index 97e7a0d4070a9..337ab1135a2f6 100644 --- a/models/packages/package_file.go +++ b/models/packages/package_file.go @@ -117,13 +117,15 @@ func DeleteFileByID(ctx context.Context, fileID int64) error { // PackageFileSearchOptions are options for SearchXXX methods type PackageFileSearchOptions struct { - OwnerID int64 - PackageType string - VersionID int64 - Query string - CompositeKey string - Properties map[string]string - OlderThan time.Duration + OwnerID int64 + PackageType string + VersionID int64 + Query string + CompositeKey string + Properties map[string]string + OlderThan time.Duration + HashAlgorithmn string + Hash string db.Paginator } @@ -182,6 +184,15 @@ func (opts *PackageFileSearchOptions) toConds() builder.Cond { cond = cond.And(builder.Lt{"package_file.created_unix": time.Now().Add(-opts.OlderThan).Unix()}) } + if opts.Hash != "" && (opts.HashAlgorithmn == "md5" || opts.HashAlgorithmn == "sha1" || opts.HashAlgorithmn == "sha256" || opts.HashAlgorithmn == "sha512") { + innerCond := builder. + Expr("package_blob.id = package_file.blob_id"). + And(builder.Eq{ + "package_blob.hash_" + opts.HashAlgorithmn: opts.Hash, + }) + cond = cond.And(builder.Exists(builder.Select("package_blob.id").From("package_blob").Where(innerCond))) + } + return cond } diff --git a/models/packages/package_version.go b/models/packages/package_version.go index 759c20abed22b..ab1bcddae5818 100644 --- a/models/packages/package_version.go +++ b/models/packages/package_version.go @@ -173,7 +173,7 @@ const ( ) // PackageSearchOptions are options for SearchXXX methods -// Besides IsInternal are all fields optional and are not used if they have their default value (nil, "", 0) +// All fields optional and are not used if they have their default value (nil, "", 0) type PackageSearchOptions struct { OwnerID int64 RepoID int64 @@ -192,7 +192,9 @@ type PackageSearchOptions struct { func (opts *PackageSearchOptions) toConds() builder.Cond { cond := builder.NewCond() if !opts.IsInternal.IsNone() { - cond = builder.Eq{"package_version.is_internal": opts.IsInternal.IsTrue()} + cond = builder.Eq{ + "package_version.is_internal": opts.IsInternal.IsTrue(), + } } if opts.OwnerID != 0 { diff --git a/models/user/setting.go b/models/user/setting.go index aec79b756bf14..a41e494db9b54 100644 --- a/models/user/setting.go +++ b/models/user/setting.go @@ -11,6 +11,7 @@ import ( "code.gitea.io/gitea/models/db" "code.gitea.io/gitea/modules/cache" setting_module "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/util" "xorm.io/builder" ) @@ -42,6 +43,10 @@ func (err ErrUserSettingIsNotExist) Error() string { return fmt.Sprintf("Setting[%s] is not exist", err.Key) } +func (err ErrUserSettingIsNotExist) Unwrap() error { + return util.ErrNotExist +} + // IsErrUserSettingIsNotExist return true if err is ErrSettingIsNotExist func IsErrUserSettingIsNotExist(err error) bool { _, ok := err.(ErrUserSettingIsNotExist) diff --git a/modules/packages/debian/metadata.go b/modules/packages/debian/metadata.go new file mode 100644 index 0000000000000..08daaf082e126 --- /dev/null +++ b/modules/packages/debian/metadata.go @@ -0,0 +1,216 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package debian + +import ( + "archive/tar" + "bufio" + "compress/gzip" + "io" + "net/mail" + "regexp" + "strings" + + "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/modules/validation" + + "github.com/blakesmith/ar" + "github.com/klauspost/compress/zstd" + "github.com/ulikunitz/xz" +) + +const ( + PropertyDistribution = "debian.distribution" + PropertyComponent = "debian.component" + PropertyArchitecture = "debian.architecture" + PropertyControl = "debian.control" + PropertyRepositoryIncludeInRelease = "debian.repository.include_in_release" + + SettingKeyPrivate = "debian.key.private" + SettingKeyPublic = "debian.key.public" + + RepositoryPackage = "_debian" + RepositoryVersion = "_repository" +) + +var ( + ErrMissingControlFile = util.NewInvalidArgumentErrorf("control file is missing") + ErrUnsupportedCompression = util.NewInvalidArgumentErrorf("unsupported compression algorithmn") + ErrInvalidName = util.NewInvalidArgumentErrorf("package name is invalid") + ErrInvalidVersion = util.NewInvalidArgumentErrorf("package version is invalid") + ErrInvalidArchitecture = util.NewInvalidArgumentErrorf("package architecture is invalid") + + // https://www.debian.org/doc/debian-policy/ch-controlfields.html#source + namePattern = regexp.MustCompile(`\A[a-z0-9][a-z0-9+-.]+\z`) + // https://www.debian.org/doc/debian-policy/ch-controlfields.html#version + versionPattern = regexp.MustCompile(`\A(?:[0-9]:)?[a-zA-Z0-9.+~]+(?:-[a-zA-Z0-9.+-~]+)?\z`) +) + +type Package struct { + Name string + Version string + Architecture string + Control string + Metadata *Metadata +} + +type Metadata struct { + Maintainer string `json:"maintainer,omitempty"` + ProjectURL string `json:"project_url,omitempty"` + Description string `json:"description,omitempty"` + Dependencies []string `json:"dependencies,omitempty"` +} + +// ParsePackage parses the Debian package file +// https://manpages.debian.org/bullseye/dpkg-dev/deb.5.en.html +func ParsePackage(r io.Reader) (*Package, error) { + arr := ar.NewReader(r) + + for { + hd, err := arr.Next() + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + + if strings.HasPrefix(hd.Name, "control.tar") { + var inner io.Reader + switch hd.Name[11:] { + case "": + inner = arr + case ".gz": + gzr, err := gzip.NewReader(arr) + if err != nil { + return nil, err + } + defer gzr.Close() + + inner = gzr + case ".xz": + xzr, err := xz.NewReader(arr) + if err != nil { + return nil, err + } + + inner = xzr + case ".zst": + zr, err := zstd.NewReader(arr) + if err != nil { + return nil, err + } + defer zr.Close() + + inner = zr + default: + return nil, ErrUnsupportedCompression + } + + tr := tar.NewReader(inner) + for { + hd, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + + if hd.Typeflag != tar.TypeReg { + continue + } + + if hd.FileInfo().Name() == "control" { + return ParseControlFile(tr) + } + } + } + } + + return nil, ErrMissingControlFile +} + +// ParseControlFile parses a Debian control file to retrieve the metadata +func ParseControlFile(r io.Reader) (*Package, error) { + p := &Package{ + Metadata: &Metadata{}, + } + + key := "" + var depends strings.Builder + var control strings.Builder + + s := bufio.NewScanner(io.TeeReader(r, &control)) + for s.Scan() { + line := s.Text() + + trimmed := strings.TrimSpace(line) + if trimmed == "" { + continue + } + + if line[0] == ' ' || line[0] == '\t' { + switch key { + case "Description": + p.Metadata.Description += line + case "Depends": + depends.WriteString(trimmed) + } + } else { + parts := strings.SplitN(trimmed, ":", 2) + if len(parts) < 2 { + continue + } + + key = parts[0] + value := strings.TrimSpace(parts[1]) + switch key { + case "Package": + if !namePattern.MatchString(value) { + return nil, ErrInvalidName + } + p.Name = value + case "Version": + if !versionPattern.MatchString(value) { + return nil, ErrInvalidVersion + } + p.Version = value + case "Architecture": + if value == "" { + return nil, ErrInvalidArchitecture + } + p.Architecture = value + case "Maintainer": + a, err := mail.ParseAddress(value) + if err != nil || a.Name == "" { + p.Metadata.Maintainer = value + } else { + p.Metadata.Maintainer = a.Name + } + case "Description": + p.Metadata.Description = value + case "Depends": + depends.WriteString(value) + case "Homepage": + if validation.IsValidURL(value) { + p.Metadata.ProjectURL = value + } + } + } + } + if err := s.Err(); err != nil { + return nil, err + } + + dependencies := strings.Split(depends.String(), ",") + for i := range dependencies { + dependencies[i] = strings.TrimSpace(dependencies[i]) + } + p.Metadata.Dependencies = dependencies + + p.Control = control.String() + + return p, nil +} diff --git a/modules/packages/debian/metadata_test.go b/modules/packages/debian/metadata_test.go new file mode 100644 index 0000000000000..69fd51ea79002 --- /dev/null +++ b/modules/packages/debian/metadata_test.go @@ -0,0 +1,171 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package debian + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "io" + "testing" + + "github.com/blakesmith/ar" + "github.com/klauspost/compress/zstd" + "github.com/stretchr/testify/assert" + "github.com/ulikunitz/xz" +) + +const ( + packageName = "gitea" + packageVersion = "0:1.0.1-te~st" + packageArchitecture = "amd64" + packageAuthor = "KN4CK3R" + description = "Description with multiple lines." + projectURL = "https://gitea.io" +) + +func TestParsePackage(t *testing.T) { + createArchive := func(files map[string][]byte) io.Reader { + var buf bytes.Buffer + aw := ar.NewWriter(&buf) + aw.WriteGlobalHeader() + for filename, content := range files { + hdr := &ar.Header{ + Name: filename, + Mode: 0o600, + Size: int64(len(content)), + } + aw.WriteHeader(hdr) + aw.Write(content) + } + return &buf + } + + t.Run("MissingControlFile", func(t *testing.T) { + data := createArchive(map[string][]byte{"dummy.txt": {}}) + + p, err := ParsePackage(data) + assert.Nil(t, p) + assert.ErrorIs(t, err, ErrMissingControlFile) + }) + + t.Run("Compression", func(t *testing.T) { + t.Run("Unsupported", func(t *testing.T) { + data := createArchive(map[string][]byte{"control.tar.foo": {}}) + + p, err := ParsePackage(data) + assert.Nil(t, p) + assert.ErrorIs(t, err, ErrUnsupportedCompression) + }) + + var buf bytes.Buffer + tw := tar.NewWriter(&buf) + tw.WriteHeader(&tar.Header{ + Name: "control", + Mode: 0o600, + Size: 50, + }) + tw.Write([]byte("Package: gitea\nVersion: 1.0.0\nArchitecture: amd64\n")) + tw.Close() + + t.Run("None", func(t *testing.T) { + data := createArchive(map[string][]byte{"control.tar": buf.Bytes()}) + + p, err := ParsePackage(data) + assert.NotNil(t, p) + assert.NoError(t, err) + assert.Equal(t, "gitea", p.Name) + }) + + t.Run("gz", func(t *testing.T) { + var zbuf bytes.Buffer + zw := gzip.NewWriter(&zbuf) + zw.Write(buf.Bytes()) + zw.Close() + + data := createArchive(map[string][]byte{"control.tar.gz": zbuf.Bytes()}) + + p, err := ParsePackage(data) + assert.NotNil(t, p) + assert.NoError(t, err) + assert.Equal(t, "gitea", p.Name) + }) + + t.Run("xz", func(t *testing.T) { + var xbuf bytes.Buffer + xw, _ := xz.NewWriter(&xbuf) + xw.Write(buf.Bytes()) + xw.Close() + + data := createArchive(map[string][]byte{"control.tar.xz": xbuf.Bytes()}) + + p, err := ParsePackage(data) + assert.NotNil(t, p) + assert.NoError(t, err) + assert.Equal(t, "gitea", p.Name) + }) + + t.Run("zst", func(t *testing.T) { + var zbuf bytes.Buffer + zw, _ := zstd.NewWriter(&zbuf) + zw.Write(buf.Bytes()) + zw.Close() + + data := createArchive(map[string][]byte{"control.tar.zst": zbuf.Bytes()}) + + p, err := ParsePackage(data) + assert.NotNil(t, p) + assert.NoError(t, err) + assert.Equal(t, "gitea", p.Name) + }) + }) +} + +func TestParseControlFile(t *testing.T) { + buildContent := func(name, version, architecture string) *bytes.Buffer { + var buf bytes.Buffer + buf.WriteString("Package: " + name + "\nVersion: " + version + "\nArchitecture: " + architecture + "\nMaintainer: " + packageAuthor + " \nHomepage: " + projectURL + "\nDepends: a,\n b\nDescription: Description\n with multiple\n lines.") + return &buf + } + + t.Run("InvalidName", func(t *testing.T) { + for _, name := range []string{"", "-cd"} { + p, err := ParseControlFile(buildContent(name, packageVersion, packageArchitecture)) + assert.Nil(t, p) + assert.ErrorIs(t, err, ErrInvalidName) + } + }) + + t.Run("InvalidVersion", func(t *testing.T) { + for _, version := range []string{"", "1-", ":1.0", "1_0"} { + p, err := ParseControlFile(buildContent(packageName, version, packageArchitecture)) + assert.Nil(t, p) + assert.ErrorIs(t, err, ErrInvalidVersion) + } + }) + + t.Run("InvalidArchitecture", func(t *testing.T) { + p, err := ParseControlFile(buildContent(packageName, packageVersion, "")) + assert.Nil(t, p) + assert.ErrorIs(t, err, ErrInvalidArchitecture) + }) + + t.Run("Valid", func(t *testing.T) { + content := buildContent(packageName, packageVersion, packageArchitecture) + full := content.String() + + p, err := ParseControlFile(content) + assert.NoError(t, err) + assert.NotNil(t, p) + + assert.Equal(t, packageName, p.Name) + assert.Equal(t, packageVersion, p.Version) + assert.Equal(t, packageArchitecture, p.Architecture) + assert.Equal(t, description, p.Metadata.Description) + assert.Equal(t, projectURL, p.Metadata.ProjectURL) + assert.Equal(t, packageAuthor, p.Metadata.Maintainer) + assert.Equal(t, []string{"a", "b"}, p.Metadata.Dependencies) + assert.Equal(t, full, p.Control) + }) +} diff --git a/modules/packages/hashed_buffer.go b/modules/packages/hashed_buffer.go index ef00a45057f7c..017ddf1c8f5a5 100644 --- a/modules/packages/hashed_buffer.go +++ b/modules/packages/hashed_buffer.go @@ -25,8 +25,15 @@ type HashedBuffer struct { combinedWriter io.Writer } -// NewHashedBuffer creates a hashed buffer with a specific maximum memory size -func NewHashedBuffer(maxMemorySize int) (*HashedBuffer, error) { +const DefaultMemorySize = 32 * 1024 * 1024 + +// NewHashedBuffer creates a hashed buffer with the default memory size +func NewHashedBuffer() (*HashedBuffer, error) { + return NewHashedBufferWithSize(DefaultMemorySize) +} + +// NewHashedBuffer creates a hashed buffer with a specific memory size +func NewHashedBufferWithSize(maxMemorySize int) (*HashedBuffer, error) { b, err := filebuffer.New(maxMemorySize) if err != nil { return nil, err @@ -43,9 +50,14 @@ func NewHashedBuffer(maxMemorySize int) (*HashedBuffer, error) { }, nil } -// CreateHashedBufferFromReader creates a hashed buffer and copies the provided reader data into it. -func CreateHashedBufferFromReader(r io.Reader, maxMemorySize int) (*HashedBuffer, error) { - b, err := NewHashedBuffer(maxMemorySize) +// CreateHashedBufferFromReader creates a hashed buffer with the default memory size and copies the provided reader data into it. +func CreateHashedBufferFromReader(r io.Reader) (*HashedBuffer, error) { + return CreateHashedBufferFromReaderWithSize(r, DefaultMemorySize) +} + +// CreateHashedBufferFromReaderWithSize creates a hashed buffer and copies the provided reader data into it. +func CreateHashedBufferFromReaderWithSize(r io.Reader, maxMemorySize int) (*HashedBuffer, error) { + b, err := NewHashedBufferWithSize(maxMemorySize) if err != nil { return nil, err } diff --git a/modules/packages/hashed_buffer_test.go b/modules/packages/hashed_buffer_test.go index e907aa060578a..564e782f18f0a 100644 --- a/modules/packages/hashed_buffer_test.go +++ b/modules/packages/hashed_buffer_test.go @@ -26,7 +26,7 @@ func TestHashedBuffer(t *testing.T) { } for _, c := range cases { - buf, err := CreateHashedBufferFromReader(strings.NewReader(c.Data), c.MaxMemorySize) + buf, err := CreateHashedBufferFromReaderWithSize(strings.NewReader(c.Data), c.MaxMemorySize) assert.NoError(t, err) assert.EqualValues(t, len(c.Data), buf.Size()) diff --git a/modules/packages/nuget/symbol_extractor.go b/modules/packages/nuget/symbol_extractor.go index b709eac4c1965..81bf0371a0b1a 100644 --- a/modules/packages/nuget/symbol_extractor.go +++ b/modules/packages/nuget/symbol_extractor.go @@ -63,7 +63,7 @@ func ExtractPortablePdb(r io.ReaderAt, size int64) (PortablePdbList, error) { return err } - buf, err := packages.CreateHashedBufferFromReader(f, 32*1024*1024) + buf, err := packages.CreateHashedBufferFromReader(f) f.Close() diff --git a/modules/setting/packages.go b/modules/setting/packages.go index 89601c3b99043..b52bbf40c7b90 100644 --- a/modules/setting/packages.go +++ b/modules/setting/packages.go @@ -30,6 +30,7 @@ var ( LimitSizeConan int64 LimitSizeConda int64 LimitSizeContainer int64 + LimitSizeDebian int64 LimitSizeGeneric int64 LimitSizeHelm int64 LimitSizeMaven int64 @@ -73,6 +74,7 @@ func loadPackagesFrom(rootCfg ConfigProvider) { Packages.LimitSizeConan = mustBytes(sec, "LIMIT_SIZE_CONAN") Packages.LimitSizeConda = mustBytes(sec, "LIMIT_SIZE_CONDA") Packages.LimitSizeContainer = mustBytes(sec, "LIMIT_SIZE_CONTAINER") + Packages.LimitSizeDebian = mustBytes(sec, "LIMIT_SIZE_DEBIAN") Packages.LimitSizeGeneric = mustBytes(sec, "LIMIT_SIZE_GENERIC") Packages.LimitSizeHelm = mustBytes(sec, "LIMIT_SIZE_HELM") Packages.LimitSizeMaven = mustBytes(sec, "LIMIT_SIZE_MAVEN") diff --git a/modules/util/filebuffer/file_backed_buffer.go b/modules/util/filebuffer/file_backed_buffer.go index bfddf90e92bcc..6b07bd0413864 100644 --- a/modules/util/filebuffer/file_backed_buffer.go +++ b/modules/util/filebuffer/file_backed_buffer.go @@ -7,11 +7,10 @@ import ( "bytes" "errors" "io" + "math" "os" ) -const maxInt = int(^uint(0) >> 1) // taken from bytes.Buffer - var ( // ErrInvalidMemorySize occurs if the memory size is not in a valid range ErrInvalidMemorySize = errors.New("Memory size must be greater 0 and lower math.MaxInt32") @@ -37,7 +36,7 @@ type FileBackedBuffer struct { // New creates a file backed buffer with a specific maximum memory size func New(maxMemorySize int) (*FileBackedBuffer, error) { - if maxMemorySize < 0 || maxMemorySize > maxInt { + if maxMemorySize < 0 || maxMemorySize > math.MaxInt32 { return nil, ErrInvalidMemorySize } diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 0072ac6fc36fa..76d8e5d4d7ced 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -3258,6 +3258,14 @@ container.layers = Image Layers container.labels = Labels container.labels.key = Key container.labels.value = Value +debian.registry = Setup this registry from the command line: +debian.registry.info = Choose <distribution> and <component> from the list below. +debian.install = To install the package, run the following command: +debian.documentation = For more information on the Debian registry, see the documentation. +debian.repository = Repository Info +debian.repository.distributions = Distributions +debian.repository.components = Components +debian.repository.architectures = Architectures generic.download = Download package from the command line: generic.documentation = For more information on the generic registry, see the documentation. helm.registry = Setup this registry from the command line: diff --git a/public/img/svg/gitea-debian.svg b/public/img/svg/gitea-debian.svg new file mode 100644 index 0000000000000..96f8f468e506a --- /dev/null +++ b/public/img/svg/gitea-debian.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/routers/api/packages/api.go b/routers/api/packages/api.go index d5acd3d261165..2ce233171c4d0 100644 --- a/routers/api/packages/api.go +++ b/routers/api/packages/api.go @@ -21,6 +21,7 @@ import ( "code.gitea.io/gitea/routers/api/packages/conan" "code.gitea.io/gitea/routers/api/packages/conda" "code.gitea.io/gitea/routers/api/packages/container" + "code.gitea.io/gitea/routers/api/packages/debian" "code.gitea.io/gitea/routers/api/packages/generic" "code.gitea.io/gitea/routers/api/packages/helm" "code.gitea.io/gitea/routers/api/packages/maven" @@ -272,6 +273,24 @@ func CommonRoutes(ctx gocontext.Context) *web.Route { conda.UploadPackageFile(ctx) }) }, reqPackageAccess(perm.AccessModeRead)) + r.Group("/debian", func() { + r.Get("/repository.key", debian.GetRepositoryKey) + r.Group("/dists/{distribution}", func() { + r.Get("/{filename}", debian.GetRepositoryFile) + r.Get("/by-hash/{algorithmn}/{hash}", debian.GetRepositoryFileByHash) + r.Group("/{component}/{architecture}", func() { + r.Get("/{filename}", debian.GetRepositoryFile) + r.Get("/by-hash/{algorithmn}/{hash}", debian.GetRepositoryFileByHash) + }) + }) + r.Group("/pool/{distribution}/{component}", func() { + r.Get("/{name}_{version}_{architecture}.deb", debian.DownloadPackageFile) + r.Group("", func() { + r.Put("/upload", debian.UploadPackageFile) + r.Delete("/{name}/{version}/{architecture}", debian.DeletePackageFile) + }, reqPackageAccess(perm.AccessModeWrite)) + }) + }, reqPackageAccess(perm.AccessModeRead)) r.Group("/generic", func() { r.Group("/{packagename}/{packageversion}", func() { r.Delete("", reqPackageAccess(perm.AccessModeWrite), generic.DeletePackage) diff --git a/routers/api/packages/cargo/cargo.go b/routers/api/packages/cargo/cargo.go index e0bf5da13adb0..18c93d328ab44 100644 --- a/routers/api/packages/cargo/cargo.go +++ b/routers/api/packages/cargo/cargo.go @@ -173,7 +173,7 @@ func UploadPackage(ctx *context.Context) { return } - buf, err := packages_module.CreateHashedBufferFromReader(cp.Content, 32*1024*1024) + buf, err := packages_module.CreateHashedBufferFromReader(cp.Content) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return diff --git a/routers/api/packages/chef/chef.go b/routers/api/packages/chef/chef.go index 28d07dea47760..b48b1778c42d1 100644 --- a/routers/api/packages/chef/chef.go +++ b/routers/api/packages/chef/chef.go @@ -263,7 +263,7 @@ func UploadPackage(ctx *context.Context) { } defer file.Close() - buf, err := packages_module.CreateHashedBufferFromReader(file, 32*1024*1024) + buf, err := packages_module.CreateHashedBufferFromReader(file) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return diff --git a/routers/api/packages/composer/composer.go b/routers/api/packages/composer/composer.go index a623952aa7387..d93b11efdf66d 100644 --- a/routers/api/packages/composer/composer.go +++ b/routers/api/packages/composer/composer.go @@ -192,7 +192,7 @@ func DownloadPackageFile(ctx *context.Context) { // UploadPackage creates a new package func UploadPackage(ctx *context.Context) { - buf, err := packages_module.CreateHashedBufferFromReader(ctx.Req.Body, 32*1024*1024) + buf, err := packages_module.CreateHashedBufferFromReader(ctx.Req.Body) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return diff --git a/routers/api/packages/conan/conan.go b/routers/api/packages/conan/conan.go index d538cc7d397db..caeb8c11bc4c8 100644 --- a/routers/api/packages/conan/conan.go +++ b/routers/api/packages/conan/conan.go @@ -318,7 +318,7 @@ func uploadFile(ctx *context.Context, fileFilter container.Set[string], fileKey defer upload.Close() } - buf, err := packages_module.CreateHashedBufferFromReader(upload, 32*1024*1024) + buf, err := packages_module.CreateHashedBufferFromReader(upload) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return @@ -648,10 +648,7 @@ func deleteRecipeOrPackage(apictx *context.Context, rref *conan_module.RecipeRef } for _, pf := range pfs { - if err := packages_model.DeleteAllProperties(ctx, packages_model.PropertyTypeFile, pf.ID); err != nil { - return err - } - if err := packages_model.DeleteFileByID(ctx, pf.ID); err != nil { + if err := packages_service.DeletePackageFile(ctx, pf); err != nil { return err } } @@ -664,11 +661,7 @@ func deleteRecipeOrPackage(apictx *context.Context, rref *conan_module.RecipeRef if !has { versionDeleted = true - if err := packages_model.DeleteAllProperties(ctx, packages_model.PropertyTypeVersion, pv.ID); err != nil { - return err - } - - if err := packages_model.DeleteVersionByID(ctx, pv.ID); err != nil { + if err := packages_service.DeletePackageVersionAndReferences(ctx, pv); err != nil { return err } } diff --git a/routers/api/packages/conda/conda.go b/routers/api/packages/conda/conda.go index 2ff619fed4e00..f7786906309b1 100644 --- a/routers/api/packages/conda/conda.go +++ b/routers/api/packages/conda/conda.go @@ -183,7 +183,7 @@ func UploadPackageFile(ctx *context.Context) { defer upload.Close() } - buf, err := packages_module.CreateHashedBufferFromReader(upload, 32*1024*1024) + buf, err := packages_module.CreateHashedBufferFromReader(upload) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return diff --git a/routers/api/packages/container/blob.go b/routers/api/packages/container/blob.go index f0457c55e19c3..c8e8dd0545290 100644 --- a/routers/api/packages/container/blob.go +++ b/routers/api/packages/container/blob.go @@ -27,10 +27,6 @@ var uploadVersionMutex sync.Mutex // saveAsPackageBlob creates a package blob from an upload // The uploaded blob gets stored in a special upload version to link them to the package/image func saveAsPackageBlob(hsr packages_module.HashedSizeReader, pci *packages_service.PackageCreationInfo) (*packages_model.PackageBlob, error) { - if err := packages_service.CheckSizeQuotaExceeded(db.DefaultContext, pci.Creator, pci.Owner, packages_model.TypeContainer, hsr.Size()); err != nil { - return nil, err - } - pb := packages_service.NewPackageBlob(hsr) exists := false @@ -43,6 +39,10 @@ func saveAsPackageBlob(hsr packages_module.HashedSizeReader, pci *packages_servi } err = db.WithTx(db.DefaultContext, func(ctx context.Context) error { + if err := packages_service.CheckSizeQuotaExceeded(ctx, pci.Creator, pci.Owner, packages_model.TypeContainer, hsr.Size()); err != nil { + return err + } + pb, exists, err = packages_model.GetOrInsertBlob(ctx, pb) if err != nil { log.Error("Error inserting package blob: %v", err) diff --git a/routers/api/packages/container/container.go b/routers/api/packages/container/container.go index 883fe73cbdf2d..63c49809a7be7 100644 --- a/routers/api/packages/container/container.go +++ b/routers/api/packages/container/container.go @@ -219,7 +219,7 @@ func InitiateUploadBlob(ctx *context.Context) { digest := ctx.FormTrim("digest") if digest != "" { - buf, err := packages_module.CreateHashedBufferFromReader(ctx.Req.Body, 32*1024*1024) + buf, err := packages_module.CreateHashedBufferFromReader(ctx.Req.Body) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return @@ -538,7 +538,7 @@ func UploadManifest(ctx *context.Context) { } maxSize := maxManifestSize + 1 - buf, err := packages_module.CreateHashedBufferFromReader(&io.LimitedReader{R: ctx.Req.Body, N: int64(maxSize)}, maxSize) + buf, err := packages_module.CreateHashedBufferFromReaderWithSize(&io.LimitedReader{R: ctx.Req.Body, N: int64(maxSize)}, maxSize) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return diff --git a/routers/api/packages/debian/debian.go b/routers/api/packages/debian/debian.go new file mode 100644 index 0000000000000..3cda04f96fe4f --- /dev/null +++ b/routers/api/packages/debian/debian.go @@ -0,0 +1,317 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package debian + +import ( + stdctx "context" + "errors" + "fmt" + "io" + "net/http" + "strings" + + "code.gitea.io/gitea/models/db" + packages_model "code.gitea.io/gitea/models/packages" + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/notification" + packages_module "code.gitea.io/gitea/modules/packages" + debian_module "code.gitea.io/gitea/modules/packages/debian" + "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/routers/api/packages/helper" + packages_service "code.gitea.io/gitea/services/packages" + debian_service "code.gitea.io/gitea/services/packages/debian" +) + +func apiError(ctx *context.Context, status int, obj interface{}) { + helper.LogAndProcessError(ctx, status, obj, func(message string) { + ctx.PlainText(status, message) + }) +} + +func GetRepositoryKey(ctx *context.Context) { + _, pub, err := debian_service.GetOrCreateKeyPair(ctx.Package.Owner.ID) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + ctx.ServeContent(strings.NewReader(pub), &context.ServeHeaderOptions{ + ContentType: "application/pgp-keys", + Filename: "repository.key", + }) +} + +// https://wiki.debian.org/DebianRepository/Format#A.22Release.22_files +// https://wiki.debian.org/DebianRepository/Format#A.22Packages.22_Indices +func GetRepositoryFile(ctx *context.Context) { + pv, err := debian_service.GetOrCreateRepositoryVersion(ctx.Package.Owner.ID) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + key := ctx.Params("distribution") + + component := ctx.Params("component") + architecture := strings.TrimPrefix(ctx.Params("architecture"), "binary-") + if component != "" && architecture != "" { + key += "|" + component + "|" + architecture + } + + s, pf, err := packages_service.GetFileStreamByPackageVersion( + ctx, + pv, + &packages_service.PackageFileInfo{ + Filename: ctx.Params("filename"), + CompositeKey: key, + }, + ) + if err != nil { + if err == packages_model.ErrPackageNotExist || err == packages_model.ErrPackageFileNotExist { + apiError(ctx, http.StatusNotFound, err) + } else { + apiError(ctx, http.StatusInternalServerError, err) + } + return + } + defer s.Close() + + ctx.ServeContent(s, &context.ServeHeaderOptions{ + Filename: pf.Name, + LastModified: pf.CreatedUnix.AsLocalTime(), + }) +} + +// https://wiki.debian.org/DebianRepository/Format#indices_acquisition_via_hashsums_.28by-hash.29 +func GetRepositoryFileByHash(ctx *context.Context) { + pv, err := debian_service.GetOrCreateRepositoryVersion(ctx.Package.Owner.ID) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + algorithmn := strings.ToLower(ctx.Params("algorithmn")) + if algorithmn == "md5sum" { + algorithmn = "md5" + } + + pfs, _, err := packages_model.SearchFiles(ctx, &packages_model.PackageFileSearchOptions{ + VersionID: pv.ID, + Hash: strings.ToLower(ctx.Params("hash")), + HashAlgorithmn: algorithmn, + }) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + if len(pfs) != 1 { + apiError(ctx, http.StatusNotFound, nil) + return + } + + s, pf, err := packages_service.GetPackageFileStream(ctx, pfs[0]) + if err != nil { + if errors.Is(err, util.ErrNotExist) { + apiError(ctx, http.StatusNotFound, err) + } else { + apiError(ctx, http.StatusInternalServerError, err) + } + return + } + defer s.Close() + + ctx.ServeContent(s, &context.ServeHeaderOptions{ + Filename: pf.Name, + LastModified: pf.CreatedUnix.AsLocalTime(), + }) +} + +func UploadPackageFile(ctx *context.Context) { + distribution := strings.TrimSpace(ctx.Params("distribution")) + component := strings.TrimSpace(ctx.Params("component")) + if distribution == "" || component == "" { + apiError(ctx, http.StatusBadRequest, "invalid distribution or component") + return + } + + upload, close, err := ctx.UploadStream() + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + if close { + defer upload.Close() + } + + buf, err := packages_module.CreateHashedBufferFromReader(upload) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + defer buf.Close() + + pck, err := debian_module.ParsePackage(buf) + if err != nil { + if errors.Is(err, util.ErrInvalidArgument) { + apiError(ctx, http.StatusBadRequest, err) + } else { + apiError(ctx, http.StatusInternalServerError, err) + } + return + } + + if _, err := buf.Seek(0, io.SeekStart); err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + _, _, err = packages_service.CreatePackageOrAddFileToExisting( + &packages_service.PackageCreationInfo{ + PackageInfo: packages_service.PackageInfo{ + Owner: ctx.Package.Owner, + PackageType: packages_model.TypeDebian, + Name: pck.Name, + Version: pck.Version, + }, + Creator: ctx.Doer, + Metadata: pck.Metadata, + }, + &packages_service.PackageFileCreationInfo{ + PackageFileInfo: packages_service.PackageFileInfo{ + Filename: fmt.Sprintf("%s_%s_%s.deb", pck.Name, pck.Version, pck.Architecture), + CompositeKey: fmt.Sprintf("%s|%s", distribution, component), + }, + Creator: ctx.Doer, + Data: buf, + IsLead: true, + Properties: map[string]string{ + debian_module.PropertyDistribution: distribution, + debian_module.PropertyComponent: component, + debian_module.PropertyArchitecture: pck.Architecture, + debian_module.PropertyControl: pck.Control, + }, + }, + ) + if err != nil { + switch err { + case packages_model.ErrDuplicatePackageVersion: + apiError(ctx, http.StatusBadRequest, err) + case packages_service.ErrQuotaTotalCount, packages_service.ErrQuotaTypeSize, packages_service.ErrQuotaTotalSize: + apiError(ctx, http.StatusForbidden, err) + default: + apiError(ctx, http.StatusInternalServerError, err) + } + return + } + + if err := debian_service.BuildSpecificRepositoryFiles(ctx, ctx.Package.Owner.ID, distribution, component, pck.Architecture); err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + ctx.Status(http.StatusCreated) +} + +func DownloadPackageFile(ctx *context.Context) { + name := ctx.Params("name") + version := ctx.Params("version") + + s, pf, err := packages_service.GetFileStreamByPackageNameAndVersion( + ctx, + &packages_service.PackageInfo{ + Owner: ctx.Package.Owner, + PackageType: packages_model.TypeDebian, + Name: name, + Version: version, + }, + &packages_service.PackageFileInfo{ + Filename: fmt.Sprintf("%s_%s_%s.deb", name, version, ctx.Params("architecture")), + CompositeKey: fmt.Sprintf("%s|%s", ctx.Params("distribution"), ctx.Params("component")), + }, + ) + if err != nil { + if err == packages_model.ErrPackageNotExist || err == packages_model.ErrPackageFileNotExist { + apiError(ctx, http.StatusNotFound, err) + } else { + apiError(ctx, http.StatusInternalServerError, err) + } + return + } + defer s.Close() + + ctx.ServeContent(s, &context.ServeHeaderOptions{ + ContentType: "application/vnd.debian.binary-package", + Filename: pf.Name, + LastModified: pf.CreatedUnix.AsLocalTime(), + }) +} + +func DeletePackageFile(ctx *context.Context) { + distribution := ctx.Params("distribution") + component := ctx.Params("component") + name := ctx.Params("name") + version := ctx.Params("version") + architecture := ctx.Params("architecture") + + owner := ctx.Package.Owner + + var pd *packages_model.PackageDescriptor + + err := db.WithTx(ctx, func(ctx stdctx.Context) error { + pv, err := packages_model.GetVersionByNameAndVersion(ctx, owner.ID, packages_model.TypeDebian, name, version) + if err != nil { + return err + } + + pf, err := packages_model.GetFileForVersionByName( + ctx, + pv.ID, + fmt.Sprintf("%s_%s_%s.deb", name, version, architecture), + fmt.Sprintf("%s|%s", distribution, component), + ) + if err != nil { + return err + } + + if err := packages_service.DeletePackageFile(ctx, pf); err != nil { + return err + } + + has, err := packages_model.HasVersionFileReferences(ctx, pv.ID) + if err != nil { + return err + } + if !has { + pd, err = packages_model.GetPackageDescriptor(ctx, pv) + if err != nil { + return err + } + + if err := packages_service.DeletePackageVersionAndReferences(ctx, pv); err != nil { + return err + } + } + + return nil + }) + if err != nil { + if errors.Is(err, util.ErrNotExist) { + apiError(ctx, http.StatusNotFound, err) + } else { + apiError(ctx, http.StatusInternalServerError, err) + } + return + } + + if pd != nil { + notification.NotifyPackageDelete(ctx, ctx.Doer, pd) + } + + if err := debian_service.BuildSpecificRepositoryFiles(ctx, ctx.Package.Owner.ID, distribution, component, architecture); err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + ctx.Status(http.StatusNoContent) +} diff --git a/routers/api/packages/generic/generic.go b/routers/api/packages/generic/generic.go index 5fba02cacd468..0c873119ef54d 100644 --- a/routers/api/packages/generic/generic.go +++ b/routers/api/packages/generic/generic.go @@ -84,7 +84,7 @@ func UploadPackage(ctx *context.Context) { defer upload.Close() } - buf, err := packages_module.CreateHashedBufferFromReader(upload, 32*1024*1024) + buf, err := packages_module.CreateHashedBufferFromReader(upload) if err != nil { log.Error("Error creating hashed buffer: %v", err) apiError(ctx, http.StatusInternalServerError, err) diff --git a/routers/api/packages/helm/helm.go b/routers/api/packages/helm/helm.go index 3bcce6bdf5855..b7edc8b7fef8a 100644 --- a/routers/api/packages/helm/helm.go +++ b/routers/api/packages/helm/helm.go @@ -155,7 +155,7 @@ func UploadPackage(ctx *context.Context) { defer upload.Close() } - buf, err := packages_module.CreateHashedBufferFromReader(upload, 32*1024*1024) + buf, err := packages_module.CreateHashedBufferFromReader(upload) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return diff --git a/routers/api/packages/maven/maven.go b/routers/api/packages/maven/maven.go index a3a23ecfa8dc9..dd270ff0edd2f 100644 --- a/routers/api/packages/maven/maven.go +++ b/routers/api/packages/maven/maven.go @@ -245,7 +245,7 @@ func UploadPackageFile(ctx *context.Context) { packageName := params.GroupID + "-" + params.ArtifactID - buf, err := packages_module.CreateHashedBufferFromReader(ctx.Req.Body, 32*1024*1024) + buf, err := packages_module.CreateHashedBufferFromReader(ctx.Req.Body) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return diff --git a/routers/api/packages/npm/npm.go b/routers/api/packages/npm/npm.go index 51b34d3e2721e..89476a776a0dd 100644 --- a/routers/api/packages/npm/npm.go +++ b/routers/api/packages/npm/npm.go @@ -189,7 +189,7 @@ func UploadPackage(ctx *context.Context) { } } - buf, err := packages_module.CreateHashedBufferFromReader(bytes.NewReader(npmPackage.Data), 32*1024*1024) + buf, err := packages_module.CreateHashedBufferFromReader(bytes.NewReader(npmPackage.Data)) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return diff --git a/routers/api/packages/nuget/nuget.go b/routers/api/packages/nuget/nuget.go index 3418bf9959a96..f6143ce291d23 100644 --- a/routers/api/packages/nuget/nuget.go +++ b/routers/api/packages/nuget/nuget.go @@ -475,7 +475,7 @@ func UploadSymbolPackage(ctx *context.Context) { Version: np.Version, } - _, _, err = packages_service.AddFileToExistingPackage( + _, err = packages_service.AddFileToExistingPackage( pi, &packages_service.PackageFileCreationInfo{ PackageFileInfo: packages_service.PackageFileInfo{ @@ -501,7 +501,7 @@ func UploadSymbolPackage(ctx *context.Context) { } for _, pdb := range pdbs { - _, _, err := packages_service.AddFileToExistingPackage( + _, err := packages_service.AddFileToExistingPackage( pi, &packages_service.PackageFileCreationInfo{ PackageFileInfo: packages_service.PackageFileInfo{ @@ -545,7 +545,7 @@ func processUploadedFile(ctx *context.Context, expectedType nuget_module.Package closables = append(closables, upload) } - buf, err := packages_module.CreateHashedBufferFromReader(upload, 32*1024*1024) + buf, err := packages_module.CreateHashedBufferFromReader(upload) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return nil, nil, closables diff --git a/routers/api/packages/pub/pub.go b/routers/api/packages/pub/pub.go index 1ece4e18ed6d4..ae0c6e7859898 100644 --- a/routers/api/packages/pub/pub.go +++ b/routers/api/packages/pub/pub.go @@ -166,7 +166,7 @@ func UploadPackageFile(ctx *context.Context) { } defer file.Close() - buf, err := packages_module.CreateHashedBufferFromReader(file, 32*1024*1024) + buf, err := packages_module.CreateHashedBufferFromReader(file) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return diff --git a/routers/api/packages/pypi/pypi.go b/routers/api/packages/pypi/pypi.go index 2f71801a8f63c..90a37ec2a8f1a 100644 --- a/routers/api/packages/pypi/pypi.go +++ b/routers/api/packages/pypi/pypi.go @@ -117,7 +117,7 @@ func UploadPackageFile(ctx *context.Context) { } defer file.Close() - buf, err := packages_module.CreateHashedBufferFromReader(file, 32*1024*1024) + buf, err := packages_module.CreateHashedBufferFromReader(file) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return diff --git a/routers/api/packages/rubygems/rubygems.go b/routers/api/packages/rubygems/rubygems.go index af358fb82fbad..740efa9baba8b 100644 --- a/routers/api/packages/rubygems/rubygems.go +++ b/routers/api/packages/rubygems/rubygems.go @@ -209,7 +209,7 @@ func UploadPackageFile(ctx *context.Context) { defer upload.Close() } - buf, err := packages_module.CreateHashedBufferFromReader(upload, 32*1024*1024) + buf, err := packages_module.CreateHashedBufferFromReader(upload) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return diff --git a/routers/api/packages/swift/swift.go b/routers/api/packages/swift/swift.go index f78f703778ba0..06f592dd648f9 100644 --- a/routers/api/packages/swift/swift.go +++ b/routers/api/packages/swift/swift.go @@ -300,7 +300,7 @@ func UploadPackageFile(ctx *context.Context) { } defer file.Close() - buf, err := packages_module.CreateHashedBufferFromReader(file, 32*1024*1024) + buf, err := packages_module.CreateHashedBufferFromReader(file) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return diff --git a/routers/api/packages/vagrant/vagrant.go b/routers/api/packages/vagrant/vagrant.go index 7b76ab79b0580..cefdc45b10ff4 100644 --- a/routers/api/packages/vagrant/vagrant.go +++ b/routers/api/packages/vagrant/vagrant.go @@ -158,7 +158,7 @@ func UploadPackageFile(ctx *context.Context) { defer upload.Close() } - buf, err := packages_module.CreateHashedBufferFromReader(upload, 32*1024*1024) + buf, err := packages_module.CreateHashedBufferFromReader(upload) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return diff --git a/routers/api/v1/packages/package.go b/routers/api/v1/packages/package.go index 200dc5aaf1401..ac48eb8a53fec 100644 --- a/routers/api/v1/packages/package.go +++ b/routers/api/v1/packages/package.go @@ -40,7 +40,7 @@ func ListPackages(ctx *context.APIContext) { // in: query // description: package type filter // type: string - // enum: [cargo, chef, composer, conan, conda, container, generic, helm, maven, npm, nuget, pub, pypi, rubygems, swift, vagrant] + // enum: [cargo, chef, composer, conan, conda, container, debian, generic, helm, maven, npm, nuget, pub, pypi, rubygems, swift, vagrant] // - name: q // in: query // description: name filter diff --git a/routers/web/user/package.go b/routers/web/user/package.go index a9acc5281feea..37ee0b86319b0 100644 --- a/routers/web/user/package.go +++ b/routers/web/user/package.go @@ -14,8 +14,10 @@ import ( access_model "code.gitea.io/gitea/models/perm/access" repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/modules/base" + "code.gitea.io/gitea/modules/container" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/log" + debian_module "code.gitea.io/gitea/modules/packages/debian" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/modules/web" @@ -163,6 +165,32 @@ func ViewPackageVersion(ctx *context.Context) { ctx.Data["IsPackagesPage"] = true ctx.Data["PackageDescriptor"] = pd + switch pd.Package.Type { + case packages_model.TypeContainer: + ctx.Data["RegistryHost"] = setting.Packages.RegistryHost + case packages_model.TypeDebian: + distributions := make(container.Set[string]) + components := make(container.Set[string]) + architectures := make(container.Set[string]) + + for _, f := range pd.Files { + for _, pp := range f.Properties { + switch pp.Name { + case debian_module.PropertyDistribution: + distributions.Add(pp.Value) + case debian_module.PropertyComponent: + components.Add(pp.Value) + case debian_module.PropertyArchitecture: + architectures.Add(pp.Value) + } + } + } + + ctx.Data["Distributions"] = distributions.Values() + ctx.Data["Components"] = components.Values() + ctx.Data["Architectures"] = architectures.Values() + } + var ( total int64 pvs []*packages_model.PackageVersion @@ -170,8 +198,6 @@ func ViewPackageVersion(ctx *context.Context) { ) switch pd.Package.Type { case packages_model.TypeContainer: - ctx.Data["RegistryHost"] = setting.Packages.RegistryHost - pvs, total, err = container_model.SearchImageTags(ctx, &container_model.ImageTagsSearchOptions{ Paginator: db.NewAbsoluteListOptions(0, 5), PackageID: pd.Package.ID, @@ -183,10 +209,6 @@ func ViewPackageVersion(ctx *context.Context) { PackageID: pd.Package.ID, IsInternal: util.OptionalBoolFalse, }) - if err != nil { - ctx.ServerError("SearchVersions", err) - return - } } if err != nil { ctx.ServerError("", err) diff --git a/services/forms/package_form.go b/services/forms/package_form.go index 699d0fe44f967..efe4a0d84a246 100644 --- a/services/forms/package_form.go +++ b/services/forms/package_form.go @@ -15,7 +15,7 @@ import ( type PackageCleanupRuleForm struct { ID int64 Enabled bool - Type string `binding:"Required;In(cargo,chef,composer,conan,conda,container,generic,helm,maven,npm,nuget,pub,pypi,rubygems,swift,vagrant)"` + Type string `binding:"Required;In(cargo,chef,composer,conan,conda,container,debian,generic,helm,maven,npm,nuget,pub,pypi,rubygems,swift,vagrant)"` KeepCount int `binding:"In(0,1,5,10,25,50,100)"` KeepPattern string `binding:"RegexPattern"` RemoveDays int `binding:"In(0,7,14,30,60,90,180)"` diff --git a/services/packages/cleanup/cleanup.go b/services/packages/cleanup/cleanup.go index 2d62a028a4c6d..43fbc1ad9b113 100644 --- a/services/packages/cleanup/cleanup.go +++ b/services/packages/cleanup/cleanup.go @@ -17,6 +17,7 @@ import ( packages_service "code.gitea.io/gitea/services/packages" cargo_service "code.gitea.io/gitea/services/packages/cargo" container_service "code.gitea.io/gitea/services/packages/container" + debian_service "code.gitea.io/gitea/services/packages/debian" ) // Cleanup removes expired package data @@ -45,6 +46,7 @@ func Cleanup(taskCtx context.Context, olderThan time.Duration) error { return fmt.Errorf("CleanupRule [%d]: GetPackagesByType failed: %w", pcr.ID, err) } + anyVersionDeleted := false for _, p := range packages { pvs, _, err := packages_model.SearchVersions(ctx, &packages_model.PackageSearchOptions{ PackageID: p.ID, @@ -91,6 +93,7 @@ func Cleanup(taskCtx context.Context, olderThan time.Duration) error { } versionDeleted = true + anyVersionDeleted = true } if versionDeleted { @@ -105,6 +108,14 @@ func Cleanup(taskCtx context.Context, olderThan time.Duration) error { } } } + + if anyVersionDeleted { + if pcr.Type == packages_model.TypeDebian { + if err := debian_service.BuildAllRepositoryFiles(ctx, pcr.OwnerID); err != nil { + return fmt.Errorf("CleanupRule [%d]: debian.BuildAllRepositoryFiles failed: %w", pcr.ID, err) + } + } + } return nil }) if err != nil { diff --git a/services/packages/debian/repository.go b/services/packages/debian/repository.go new file mode 100644 index 0000000000000..eac878256ab7b --- /dev/null +++ b/services/packages/debian/repository.go @@ -0,0 +1,443 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package debian + +import ( + "bytes" + "compress/gzip" + "context" + "errors" + "fmt" + "io" + "sort" + "strings" + "time" + + "code.gitea.io/gitea/models/db" + packages_model "code.gitea.io/gitea/models/packages" + debian_model "code.gitea.io/gitea/models/packages/debian" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/log" + packages_module "code.gitea.io/gitea/modules/packages" + debian_module "code.gitea.io/gitea/modules/packages/debian" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/util" + packages_service "code.gitea.io/gitea/services/packages" + + "github.com/keybase/go-crypto/openpgp" + "github.com/keybase/go-crypto/openpgp/armor" + "github.com/keybase/go-crypto/openpgp/clearsign" + "github.com/keybase/go-crypto/openpgp/packet" + "github.com/ulikunitz/xz" +) + +// GetOrCreateRepositoryVersion gets or creates the internal repository package +// The Debian registry needs multiple index files which are stored in this package. +func GetOrCreateRepositoryVersion(ownerID int64) (*packages_model.PackageVersion, error) { + var repositoryVersion *packages_model.PackageVersion + + return repositoryVersion, db.WithTx(db.DefaultContext, func(ctx context.Context) error { + p := &packages_model.Package{ + OwnerID: ownerID, + Type: packages_model.TypeDebian, + Name: debian_module.RepositoryPackage, + LowerName: debian_module.RepositoryPackage, + IsInternal: true, + } + var err error + if p, err = packages_model.TryInsertPackage(ctx, p); err != nil { + if err != packages_model.ErrDuplicatePackage { + log.Error("Error inserting package: %v", err) + return err + } + } + + pv := &packages_model.PackageVersion{ + PackageID: p.ID, + CreatorID: ownerID, + Version: debian_module.RepositoryVersion, + LowerVersion: debian_module.RepositoryVersion, + IsInternal: true, + MetadataJSON: "null", + } + if pv, err = packages_model.GetOrInsertVersion(ctx, pv); err != nil { + if err != packages_model.ErrDuplicatePackageVersion { + log.Error("Error inserting package version: %v", err) + return err + } + } + + repositoryVersion = pv + + return nil + }) +} + +// GetOrCreateKeyPair gets or creates the PGP keys used to sign repository files +func GetOrCreateKeyPair(ownerID int64) (string, string, error) { + priv, err := user_model.GetSetting(ownerID, debian_module.SettingKeyPrivate) + if err != nil && !errors.Is(err, util.ErrNotExist) { + return "", "", err + } + + pub, err := user_model.GetSetting(ownerID, debian_module.SettingKeyPublic) + if err != nil && !errors.Is(err, util.ErrNotExist) { + return "", "", err + } + + if priv == "" || pub == "" { + priv, pub, err = generateKeypair() + if err != nil { + return "", "", err + } + + if err := user_model.SetUserSetting(ownerID, debian_module.SettingKeyPrivate, priv); err != nil { + return "", "", err + } + + if err := user_model.SetUserSetting(ownerID, debian_module.SettingKeyPublic, pub); err != nil { + return "", "", err + } + } + + return priv, pub, nil +} + +func generateKeypair() (string, string, error) { + e, err := openpgp.NewEntity(setting.AppName, "Debian Registry", "", nil) + if err != nil { + return "", "", err + } + + var priv strings.Builder + var pub strings.Builder + + w, err := armor.Encode(&priv, openpgp.PrivateKeyType, nil) + if err != nil { + return "", "", err + } + if err := e.SerializePrivate(w, nil); err != nil { + return "", "", err + } + w.Close() + + w, err = armor.Encode(&pub, openpgp.PublicKeyType, nil) + if err != nil { + return "", "", err + } + if err := e.Serialize(w); err != nil { + return "", "", err + } + w.Close() + + return priv.String(), pub.String(), nil +} + +// BuildAllRepositoryFiles (re)builds all repository files for every available distributions, components and architectures +func BuildAllRepositoryFiles(ctx context.Context, ownerID int64) error { + pv, err := GetOrCreateRepositoryVersion(ownerID) + if err != nil { + return err + } + + // 1. Delete all existing repository files + pfs, err := packages_model.GetFilesByVersionID(ctx, pv.ID) + if err != nil { + return err + } + + for _, pf := range pfs { + if err := packages_model.DeleteAllProperties(ctx, packages_model.PropertyTypeFile, pf.ID); err != nil { + return err + } + if err := packages_model.DeleteFileByID(ctx, pf.ID); err != nil { + return err + } + } + + // 2. (Re)Build repository files for existing packages + distributions, err := debian_model.GetDistributions(ctx, ownerID) + if err != nil { + return err + } + for _, distribution := range distributions { + components, err := debian_model.GetComponents(ctx, ownerID, distribution) + if err != nil { + return err + } + architectures, err := debian_model.GetArchitectures(ctx, ownerID, distribution) + if err != nil { + return err + } + + for _, component := range components { + for _, architecture := range architectures { + if err := buildRepositoryFiles(ctx, ownerID, pv, distribution, component, architecture); err != nil { + return fmt.Errorf("failed to build repository files [%s/%s/%s]: %w", distribution, component, architecture, err) + } + } + } + } + + return nil +} + +// BuildSpecificRepositoryFiles builds index files for the repository +func BuildSpecificRepositoryFiles(ctx context.Context, ownerID int64, distribution, component, architecture string) error { + pv, err := GetOrCreateRepositoryVersion(ownerID) + if err != nil { + return err + } + + return buildRepositoryFiles(ctx, ownerID, pv, distribution, component, architecture) +} + +func buildRepositoryFiles(ctx context.Context, ownerID int64, repoVersion *packages_model.PackageVersion, distribution, component, architecture string) error { + if err := buildPackagesIndices(ctx, ownerID, repoVersion, distribution, component, architecture); err != nil { + return err + } + + return buildReleaseFiles(ctx, ownerID, repoVersion, distribution) +} + +// https://wiki.debian.org/DebianRepository/Format#A.22Packages.22_Indices +func buildPackagesIndices(ctx context.Context, ownerID int64, repoVersion *packages_model.PackageVersion, distribution, component, architecture string) error { + pfds, err := debian_model.SearchLatestPackages(ctx, &debian_model.PackageSearchOptions{ + OwnerID: ownerID, + Distribution: distribution, + Component: component, + Architecture: architecture, + }) + if err != nil { + return err + } + + // Delete the package indices if there are no packages + if len(pfds) == 0 { + key := fmt.Sprintf("%s|%s|%s", distribution, component, architecture) + for _, filename := range []string{"Packages", "Packages.gz", "Packages.xz"} { + pf, err := packages_model.GetFileForVersionByName(ctx, repoVersion.ID, filename, key) + if err != nil && !errors.Is(err, util.ErrNotExist) { + return err + } + + if err := packages_model.DeleteAllProperties(ctx, packages_model.PropertyTypeFile, pf.ID); err != nil { + return err + } + if err := packages_model.DeleteFileByID(ctx, pf.ID); err != nil { + return err + } + } + + return nil + } + + packagesContent, _ := packages_module.NewHashedBuffer() + + packagesGzipContent, _ := packages_module.NewHashedBuffer() + gzw := gzip.NewWriter(packagesGzipContent) + + packagesXzContent, _ := packages_module.NewHashedBuffer() + xzw, _ := xz.NewWriter(packagesXzContent) + + w := io.MultiWriter(packagesContent, gzw, xzw) + + addSeperator := false + for _, pfd := range pfds { + if addSeperator { + fmt.Fprintln(w) + } + addSeperator = true + + fmt.Fprint(w, pfd.Properties.GetByName(debian_module.PropertyControl)) + + fmt.Fprintf(w, "Filename: pool/%s/%s/%s\n", distribution, component, pfd.File.Name) + fmt.Fprintf(w, "Size: %d\n", pfd.Blob.Size) + fmt.Fprintf(w, "MD5sum: %s\n", pfd.Blob.HashMD5) + fmt.Fprintf(w, "SHA1: %s\n", pfd.Blob.HashSHA1) + fmt.Fprintf(w, "SHA256: %s\n", pfd.Blob.HashSHA256) + fmt.Fprintf(w, "SHA512: %s\n", pfd.Blob.HashSHA512) + } + + gzw.Close() + xzw.Close() + + for _, file := range []struct { + Name string + Data packages_module.HashedSizeReader + }{ + {"Packages", packagesContent}, + {"Packages.gz", packagesGzipContent}, + {"Packages.xz", packagesXzContent}, + } { + _, err = packages_service.AddFileToPackageVersionInternal( + repoVersion, + &packages_service.PackageFileCreationInfo{ + PackageFileInfo: packages_service.PackageFileInfo{ + Filename: file.Name, + CompositeKey: fmt.Sprintf("%s|%s|%s", distribution, component, architecture), + }, + Creator: user_model.NewGhostUser(), + Data: file.Data, + IsLead: false, + OverwriteExisting: true, + Properties: map[string]string{ + debian_module.PropertyRepositoryIncludeInRelease: "", + debian_module.PropertyDistribution: distribution, + debian_module.PropertyComponent: component, + debian_module.PropertyArchitecture: architecture, + }, + }, + ) + if err != nil { + return err + } + } + + return nil +} + +// https://wiki.debian.org/DebianRepository/Format#A.22Release.22_files +func buildReleaseFiles(ctx context.Context, ownerID int64, repoVersion *packages_model.PackageVersion, distribution string) error { + pfs, _, err := packages_model.SearchFiles(ctx, &packages_model.PackageFileSearchOptions{ + VersionID: repoVersion.ID, + Properties: map[string]string{ + debian_module.PropertyRepositoryIncludeInRelease: "", + debian_module.PropertyDistribution: distribution, + }, + }) + if err != nil { + return err + } + + // Delete the release files if there are no packages + if len(pfs) == 0 { + for _, filename := range []string{"Release", "Release.gpg", "InRelease"} { + pf, err := packages_model.GetFileForVersionByName(ctx, repoVersion.ID, filename, distribution) + if err != nil && !errors.Is(err, util.ErrNotExist) { + return err + } + + if err := packages_model.DeleteAllProperties(ctx, packages_model.PropertyTypeFile, pf.ID); err != nil { + return err + } + if err := packages_model.DeleteFileByID(ctx, pf.ID); err != nil { + return err + } + } + + return nil + } + + components, err := debian_model.GetComponents(ctx, ownerID, distribution) + if err != nil { + return err + } + + sort.Strings(components) + + architectures, err := debian_model.GetArchitectures(ctx, ownerID, distribution) + if err != nil { + return err + } + + sort.Strings(architectures) + + priv, _, err := GetOrCreateKeyPair(ownerID) + if err != nil { + return err + } + + block, err := armor.Decode(strings.NewReader(priv)) + if err != nil { + return err + } + + e, err := openpgp.ReadEntity(packet.NewReader(block.Body)) + if err != nil { + return err + } + + inReleaseContent, _ := packages_module.NewHashedBuffer() + sw, err := clearsign.Encode(inReleaseContent, e.PrivateKey, nil) + if err != nil { + return err + } + + var buf bytes.Buffer + + w := io.MultiWriter(sw, &buf) + + fmt.Fprintf(w, "Origin: %s\n", setting.AppName) + fmt.Fprintf(w, "Label: %s\n", setting.AppName) + fmt.Fprintf(w, "Suite: %s\n", distribution) + fmt.Fprintf(w, "Codename: %s\n", distribution) + fmt.Fprintf(w, "Components: %s\n", strings.Join(components, " ")) + fmt.Fprintf(w, "Architectures: %s\n", strings.Join(architectures, " ")) + fmt.Fprintf(w, "Date: %s\n", time.Now().UTC().Format(time.RFC1123)) + fmt.Fprint(w, "Acquire-By-Hash: yes") + + pfds, err := packages_model.GetPackageFileDescriptors(ctx, pfs) + if err != nil { + return err + } + + var md5, sha1, sha256, sha512 strings.Builder + for _, pfd := range pfds { + path := fmt.Sprintf("%s/binary-%s/%s", pfd.Properties.GetByName(debian_module.PropertyComponent), pfd.Properties.GetByName(debian_module.PropertyArchitecture), pfd.File.Name) + fmt.Fprintf(&md5, " %s %d %s\n", pfd.Blob.HashMD5, pfd.Blob.Size, path) + fmt.Fprintf(&sha1, " %s %d %s\n", pfd.Blob.HashSHA1, pfd.Blob.Size, path) + fmt.Fprintf(&sha256, " %s %d %s\n", pfd.Blob.HashSHA256, pfd.Blob.Size, path) + fmt.Fprintf(&sha512, " %s %d %s\n", pfd.Blob.HashSHA512, pfd.Blob.Size, path) + } + + fmt.Fprintln(w, "MD5Sum:") + fmt.Fprint(w, md5.String()) + fmt.Fprintln(w, "SHA1:") + fmt.Fprint(w, sha1.String()) + fmt.Fprintln(w, "SHA256:") + fmt.Fprint(w, sha256.String()) + fmt.Fprintln(w, "SHA512:") + fmt.Fprint(w, sha512.String()) + + sw.Close() + + releaseGpgContent, _ := packages_module.NewHashedBuffer() + if err := openpgp.ArmoredDetachSign(releaseGpgContent, e, bytes.NewReader(buf.Bytes()), nil); err != nil { + return err + } + + releaseContent, _ := packages_module.CreateHashedBufferFromReader(&buf) + + for _, file := range []struct { + Name string + Data packages_module.HashedSizeReader + }{ + {"Release", releaseContent}, + {"Release.gpg", releaseGpgContent}, + {"InRelease", inReleaseContent}, + } { + _, err = packages_service.AddFileToPackageVersionInternal( + repoVersion, + &packages_service.PackageFileCreationInfo{ + PackageFileInfo: packages_service.PackageFileInfo{ + Filename: file.Name, + CompositeKey: distribution, + }, + Creator: user_model.NewGhostUser(), + Data: file.Data, + IsLead: false, + OverwriteExisting: true, + Properties: map[string]string{ + debian_module.PropertyDistribution: distribution, + }, + }, + ) + if err != nil { + return err + } + } + + return nil +} diff --git a/services/packages/packages.go b/services/packages/packages.go index dd5c63470b8b2..735e52c854ba0 100644 --- a/services/packages/packages.go +++ b/services/packages/packages.go @@ -187,19 +187,33 @@ func createPackageAndVersion(ctx context.Context, pvci *PackageCreationInfo, all } // AddFileToExistingPackage adds a file to an existing package. If the package does not exist, ErrPackageNotExist is returned -func AddFileToExistingPackage(pvi *PackageInfo, pfci *PackageFileCreationInfo) (*packages_model.PackageVersion, *packages_model.PackageFile, error) { +func AddFileToExistingPackage(pvi *PackageInfo, pfci *PackageFileCreationInfo) (*packages_model.PackageFile, error) { + return addFileToPackageWrapper(func(ctx context.Context) (*packages_model.PackageFile, *packages_model.PackageBlob, bool, error) { + pv, err := packages_model.GetVersionByNameAndVersion(ctx, pvi.Owner.ID, pvi.PackageType, pvi.Name, pvi.Version) + if err != nil { + return nil, nil, false, err + } + + return addFileToPackageVersion(ctx, pv, pvi, pfci) + }) +} + +// AddFileToPackageVersionInternal adds a file to the package +// This method skips quota checks and should only be used for system-managed packages. +func AddFileToPackageVersionInternal(pv *packages_model.PackageVersion, pfci *PackageFileCreationInfo) (*packages_model.PackageFile, error) { + return addFileToPackageWrapper(func(ctx context.Context) (*packages_model.PackageFile, *packages_model.PackageBlob, bool, error) { + return addFileToPackageVersionUnchecked(ctx, pv, pfci) + }) +} + +func addFileToPackageWrapper(fn func(ctx context.Context) (*packages_model.PackageFile, *packages_model.PackageBlob, bool, error)) (*packages_model.PackageFile, error) { ctx, committer, err := db.TxContext(db.DefaultContext) if err != nil { - return nil, nil, err + return nil, err } defer committer.Close() - pv, err := packages_model.GetVersionByNameAndVersion(ctx, pvi.Owner.ID, pvi.PackageType, pvi.Name, pvi.Version) - if err != nil { - return nil, nil, err - } - - pf, pb, blobCreated, err := addFileToPackageVersion(ctx, pv, pvi, pfci) + pf, pb, blobCreated, err := fn(ctx) removeBlob := false defer func() { if removeBlob { @@ -211,15 +225,15 @@ func AddFileToExistingPackage(pvi *PackageInfo, pfci *PackageFileCreationInfo) ( }() if err != nil { removeBlob = blobCreated - return nil, nil, err + return nil, err } if err := committer.Commit(); err != nil { removeBlob = blobCreated - return nil, nil, err + return nil, err } - return pv, pf, nil + return pf, nil } // NewPackageBlob creates a package blob instance @@ -236,12 +250,16 @@ func NewPackageBlob(hsr packages_module.HashedSizeReader) *packages_model.Packag } func addFileToPackageVersion(ctx context.Context, pv *packages_model.PackageVersion, pvi *PackageInfo, pfci *PackageFileCreationInfo) (*packages_model.PackageFile, *packages_model.PackageBlob, bool, error) { - log.Trace("Adding package file: %v, %s", pv.ID, pfci.Filename) - if err := CheckSizeQuotaExceeded(ctx, pfci.Creator, pvi.Owner, pvi.PackageType, pfci.Data.Size()); err != nil { return nil, nil, false, err } + return addFileToPackageVersionUnchecked(ctx, pv, pfci) +} + +func addFileToPackageVersionUnchecked(ctx context.Context, pv *packages_model.PackageVersion, pfci *PackageFileCreationInfo) (*packages_model.PackageFile, *packages_model.PackageBlob, bool, error) { + log.Trace("Adding package file: %v, %s", pv.ID, pfci.Filename) + pb, exists, err := packages_model.GetOrInsertBlob(ctx, NewPackageBlob(pfci.Data)) if err != nil { log.Error("Error inserting package blob: %v", err) @@ -345,6 +363,8 @@ func CheckSizeQuotaExceeded(ctx context.Context, doer, owner *user_model.User, p typeSpecificSize = setting.Packages.LimitSizeConda case packages_model.TypeContainer: typeSpecificSize = setting.Packages.LimitSizeContainer + case packages_model.TypeDebian: + typeSpecificSize = setting.Packages.LimitSizeDebian case packages_model.TypeGeneric: typeSpecificSize = setting.Packages.LimitSizeGeneric case packages_model.TypeHelm: diff --git a/templates/package/content/debian.tmpl b/templates/package/content/debian.tmpl new file mode 100644 index 0000000000000..c10c3042b92c4 --- /dev/null +++ b/templates/package/content/debian.tmpl @@ -0,0 +1,65 @@ +{{if eq .PackageDescriptor.Package.Type "debian"}} +

{{.locale.Tr "packages.installation"}}

+
+
+
+ + 
sudo curl  -o /etc/apt/trusted.gpg.d/gitea-{{$.PackageDescriptor.Owner.Name}}.asc
+echo "deb  <distribution> <component>" | sudo tee -a /etc/apt/sources.list.d/gitea.list
+sudo apt update
+

{{.locale.Tr "packages.debian.registry.info" | Safe}}

+
+
+ +
+ 
sudo apt install {{$.PackageDescriptor.Package.Name}}={{$.PackageDescriptor.Version.Version}}
+
+
+
+ +
+
+
+ +

{{.locale.Tr "packages.debian.repository"}}

+
+ + + + + + + + + + + + + + + +
{{.locale.Tr "packages.debian.repository.distributions"}}
{{Join .Distributions ", "}}
{{.locale.Tr "packages.debian.repository.components"}}
{{Join .Components ", "}}
{{.locale.Tr "packages.debian.repository.architectures"}}
{{Join .Architectures ", "}}
+
+ + {{if .PackageDescriptor.Metadata.Description}} +

{{.locale.Tr "packages.about"}}

+
+ {{.PackageDescriptor.Metadata.Description}} +
+ {{end}} + + {{if .PackageDescriptor.Metadata.Dependencies}} +

{{.locale.Tr "packages.dependencies"}}

+
+ + + {{range .PackageDescriptor.Metadata.Dependencies}} + + + + {{end}} + +
{{.}}
+
+ {{end}} +{{end}} diff --git a/templates/package/metadata/debian.tmpl b/templates/package/metadata/debian.tmpl new file mode 100644 index 0000000000000..93b6db3bd240e --- /dev/null +++ b/templates/package/metadata/debian.tmpl @@ -0,0 +1,4 @@ +{{if eq .PackageDescriptor.Package.Type "debian"}} + {{if .PackageDescriptor.Metadata.Maintainer}}
{{svg "octicon-person" 16 "mr-3"}} {{.PackageDescriptor.Metadata.Maintainer}}
{{end}} + {{if .PackageDescriptor.Metadata.ProjectURL}}
{{svg "octicon-link-external" 16 "mr-3"}} {{.locale.Tr "packages.details.project_site"}}
{{end}} +{{end}} diff --git a/templates/package/view.tmpl b/templates/package/view.tmpl index 3cb130851fee1..30e737d09ceae 100644 --- a/templates/package/view.tmpl +++ b/templates/package/view.tmpl @@ -25,6 +25,7 @@ {{template "package/content/conan" .}} {{template "package/content/conda" .}} {{template "package/content/container" .}} + {{template "package/content/debian" .}} {{template "package/content/generic" .}} {{template "package/content/helm" .}} {{template "package/content/maven" .}} @@ -52,6 +53,7 @@ {{template "package/metadata/conan" .}} {{template "package/metadata/conda" .}} {{template "package/metadata/container" .}} + {{template "package/metadata/debian" .}} {{template "package/metadata/generic" .}} {{template "package/metadata/helm" .}} {{template "package/metadata/maven" .}} diff --git a/templates/swagger/v1_json.tmpl b/templates/swagger/v1_json.tmpl index 2db950b57a706..d656664367cdd 100644 --- a/templates/swagger/v1_json.tmpl +++ b/templates/swagger/v1_json.tmpl @@ -2415,6 +2415,7 @@ "conan", "conda", "container", + "debian", "generic", "helm", "maven", diff --git a/tests/integration/api_packages_debian_test.go b/tests/integration/api_packages_debian_test.go new file mode 100644 index 0000000000000..3e25acd8cff2c --- /dev/null +++ b/tests/integration/api_packages_debian_test.go @@ -0,0 +1,252 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package integration + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "fmt" + "io" + "net/http" + "strings" + "testing" + + "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/models/packages" + "code.gitea.io/gitea/models/unittest" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/base" + debian_module "code.gitea.io/gitea/modules/packages/debian" + "code.gitea.io/gitea/tests" + + "github.com/blakesmith/ar" + "github.com/stretchr/testify/assert" +) + +func TestPackageDebian(t *testing.T) { + defer tests.PrepareTestEnv(t)() + user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) + + packageName := "gitea" + packageVersion := "1.0.3" + packageDescription := "Package Description" + + createArchive := func(name, version, architecture string) io.Reader { + var cbuf bytes.Buffer + zw := gzip.NewWriter(&cbuf) + tw := tar.NewWriter(zw) + tw.WriteHeader(&tar.Header{ + Name: "control", + Mode: 0o600, + Size: 50, + }) + fmt.Fprintf(tw, "Package: %s\nVersion: %s\nArchitecture: %s\nDescription: %s\n", name, version, architecture, packageDescription) + tw.Close() + zw.Close() + + var buf bytes.Buffer + aw := ar.NewWriter(&buf) + aw.WriteGlobalHeader() + hdr := &ar.Header{ + Name: "control.tar.gz", + Mode: 0o600, + Size: int64(cbuf.Len()), + } + aw.WriteHeader(hdr) + aw.Write(cbuf.Bytes()) + return &buf + } + + distributions := []string{"test", "gitea"} + components := []string{"main", "stable"} + architectures := []string{"all", "amd64"} + + rootURL := fmt.Sprintf("/api/packages/%s/debian", user.Name) + + t.Run("RepositoryKey", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + req := NewRequest(t, "GET", rootURL+"/repository.key") + resp := MakeRequest(t, req, http.StatusOK) + + assert.Equal(t, "application/pgp-keys", resp.Header().Get("Content-Type")) + assert.Contains(t, resp.Body.String(), "-----BEGIN PGP PUBLIC KEY BLOCK-----") + }) + + for _, distribution := range distributions { + t.Run(fmt.Sprintf("[Distribution:%s]", distribution), func(t *testing.T) { + for _, component := range components { + for _, architecture := range architectures { + t.Run(fmt.Sprintf("[Component:%s,Architecture:%s]", component, architecture), func(t *testing.T) { + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + uploadURL := fmt.Sprintf("%s/pool/%s/%s/upload", rootURL, distribution, component) + + req := NewRequestWithBody(t, "PUT", uploadURL, bytes.NewReader([]byte{})) + MakeRequest(t, req, http.StatusUnauthorized) + + req = NewRequestWithBody(t, "PUT", uploadURL, bytes.NewReader([]byte{})) + AddBasicAuthHeader(req, user.Name) + MakeRequest(t, req, http.StatusBadRequest) + + req = NewRequestWithBody(t, "PUT", uploadURL, createArchive("", "", "")) + AddBasicAuthHeader(req, user.Name) + MakeRequest(t, req, http.StatusBadRequest) + + req = NewRequestWithBody(t, "PUT", uploadURL, createArchive(packageName, packageVersion, architecture)) + AddBasicAuthHeader(req, user.Name) + MakeRequest(t, req, http.StatusCreated) + + pvs, err := packages.GetVersionsByPackageType(db.DefaultContext, user.ID, packages.TypeDebian) + assert.NoError(t, err) + assert.Len(t, pvs, 1) + + pd, err := packages.GetPackageDescriptor(db.DefaultContext, pvs[0]) + assert.NoError(t, err) + assert.Nil(t, pd.SemVer) + assert.IsType(t, &debian_module.Metadata{}, pd.Metadata) + assert.Equal(t, packageName, pd.Package.Name) + assert.Equal(t, packageVersion, pd.Version.Version) + + pfs, err := packages.GetFilesByVersionID(db.DefaultContext, pvs[0].ID) + assert.NoError(t, err) + assert.NotEmpty(t, pfs) + assert.Condition(t, func() bool { + seen := false + expectedFilename := fmt.Sprintf("%s_%s_%s.deb", packageName, packageVersion, architecture) + expectedCompositeKey := fmt.Sprintf("%s|%s", distribution, component) + for _, pf := range pfs { + if pf.Name == expectedFilename && pf.CompositeKey == expectedCompositeKey { + if seen { + return false + } + seen = true + + assert.True(t, pf.IsLead) + + pfps, err := packages.GetProperties(db.DefaultContext, packages.PropertyTypeFile, pf.ID) + assert.NoError(t, err) + + for _, pfp := range pfps { + switch pfp.Name { + case debian_module.PropertyDistribution: + assert.Equal(t, distribution, pfp.Value) + case debian_module.PropertyComponent: + assert.Equal(t, component, pfp.Value) + case debian_module.PropertyArchitecture: + assert.Equal(t, architecture, pfp.Value) + } + } + } + } + return seen + }) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + req := NewRequest(t, "GET", fmt.Sprintf("%s/pool/%s/%s/%s_%s_%s.deb", rootURL, distribution, component, packageName, packageVersion, architecture)) + resp := MakeRequest(t, req, http.StatusOK) + + assert.Equal(t, "application/vnd.debian.binary-package", resp.Header().Get("Content-Type")) + }) + + t.Run("Packages", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + url := fmt.Sprintf("%s/dists/%s/%s/binary-%s/Packages", rootURL, distribution, component, architecture) + + req := NewRequest(t, "GET", url) + resp := MakeRequest(t, req, http.StatusOK) + + body := resp.Body.String() + + assert.Contains(t, body, "Package: "+packageName) + assert.Contains(t, body, "Version: "+packageVersion) + assert.Contains(t, body, "Architecture: "+architecture) + assert.Contains(t, body, fmt.Sprintf("Filename: pool/%s/%s/%s_%s_%s.deb", distribution, component, packageName, packageVersion, architecture)) + + req = NewRequest(t, "GET", url+".gz") + MakeRequest(t, req, http.StatusOK) + + req = NewRequest(t, "GET", url+".xz") + MakeRequest(t, req, http.StatusOK) + + url = fmt.Sprintf("%s/dists/%s/%s/%s/by-hash/SHA256/%s", rootURL, distribution, component, architecture, base.EncodeSha256(body)) + req = NewRequest(t, "GET", url) + resp = MakeRequest(t, req, http.StatusOK) + + assert.Equal(t, body, resp.Body.String()) + }) + }) + } + } + + t.Run("Release", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + req := NewRequest(t, "GET", fmt.Sprintf("%s/dists/%s/Release", rootURL, distribution)) + resp := MakeRequest(t, req, http.StatusOK) + + body := resp.Body.String() + + assert.Contains(t, body, "Components: "+strings.Join(components, " ")) + assert.Contains(t, body, "Architectures: "+strings.Join(architectures, " ")) + + for _, component := range components { + for _, architecture := range architectures { + assert.Contains(t, body, fmt.Sprintf("%s/binary-%s/Packages", component, architecture)) + assert.Contains(t, body, fmt.Sprintf("%s/binary-%s/Packages.gz", component, architecture)) + assert.Contains(t, body, fmt.Sprintf("%s/binary-%s/Packages.xz", component, architecture)) + } + } + + req = NewRequest(t, "GET", fmt.Sprintf("%s/dists/%s/by-hash/SHA256/%s", rootURL, distribution, base.EncodeSha256(body))) + resp = MakeRequest(t, req, http.StatusOK) + + assert.Equal(t, body, resp.Body.String()) + + req = NewRequest(t, "GET", fmt.Sprintf("%s/dists/%s/Release.gpg", rootURL, distribution)) + resp = MakeRequest(t, req, http.StatusOK) + + assert.Contains(t, resp.Body.String(), "-----BEGIN PGP SIGNATURE-----") + + req = NewRequest(t, "GET", fmt.Sprintf("%s/dists/%s/InRelease", rootURL, distribution)) + resp = MakeRequest(t, req, http.StatusOK) + + assert.Contains(t, resp.Body.String(), "-----BEGIN PGP SIGNED MESSAGE-----") + }) + }) + } + + t.Run("Delete", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + distribution := distributions[0] + architecture := architectures[0] + + for _, component := range components { + req := NewRequest(t, "DELETE", fmt.Sprintf("%s/pool/%s/%s/%s/%s/%s", rootURL, distribution, component, packageName, packageVersion, architecture)) + MakeRequest(t, req, http.StatusUnauthorized) + + req = NewRequest(t, "DELETE", fmt.Sprintf("%s/pool/%s/%s/%s/%s/%s", rootURL, distribution, component, packageName, packageVersion, architecture)) + AddBasicAuthHeader(req, user.Name) + MakeRequest(t, req, http.StatusNoContent) + + req = NewRequest(t, "GET", fmt.Sprintf("%s/dists/%s/%s/binary-%s/Packages", rootURL, distribution, component, architecture)) + MakeRequest(t, req, http.StatusNotFound) + } + + req := NewRequest(t, "GET", fmt.Sprintf("%s/dists/%s/Release", rootURL, distribution)) + resp := MakeRequest(t, req, http.StatusOK) + + body := resp.Body.String() + + assert.Contains(t, body, "Components: "+strings.Join(components, " ")) + assert.Contains(t, body, "Architectures: "+architectures[1]) + }) +} diff --git a/web_src/svg/gitea-debian.svg b/web_src/svg/gitea-debian.svg new file mode 100644 index 0000000000000..4046f7fc7cb17 --- /dev/null +++ b/web_src/svg/gitea-debian.svg @@ -0,0 +1,9 @@ + + + + + + + + + From c0ddec8a2a4cd348462257e0ceb253daf5692c32 Mon Sep 17 00:00:00 2001 From: Yarden Shoham Date: Sat, 29 Apr 2023 01:06:41 +0300 Subject: [PATCH 09/65] Revert "Add Debian package registry" (#24412) Reverts go-gitea/gitea#22854 --- assets/go-licenses.json | 5 - cmd/migrate_storage_test.go | 2 +- custom/conf/app.example.ini | 2 - .../config-cheat-sheet.en-us.md | 1 - .../doc/usage/packages/debian.en-us.md | 134 ------ .../doc/usage/packages/overview.en-us.md | 1 - go.mod | 3 +- go.sum | 2 - models/migrations/migrations.go | 2 - models/migrations/v1_20/v256.go | 23 - models/packages/container/search.go | 11 +- models/packages/debian/search.go | 131 ------ models/packages/descriptor.go | 26 +- models/packages/package.go | 19 +- models/packages/package_file.go | 25 +- models/packages/package_version.go | 6 +- models/user/setting.go | 5 - modules/packages/debian/metadata.go | 216 --------- modules/packages/debian/metadata_test.go | 171 ------- modules/packages/hashed_buffer.go | 22 +- modules/packages/hashed_buffer_test.go | 2 +- modules/packages/nuget/symbol_extractor.go | 2 +- modules/setting/packages.go | 2 - modules/util/filebuffer/file_backed_buffer.go | 5 +- options/locale/locale_en-US.ini | 8 - public/img/svg/gitea-debian.svg | 1 - routers/api/packages/api.go | 19 - routers/api/packages/cargo/cargo.go | 2 +- routers/api/packages/chef/chef.go | 2 +- routers/api/packages/composer/composer.go | 2 +- routers/api/packages/conan/conan.go | 13 +- routers/api/packages/conda/conda.go | 2 +- routers/api/packages/container/blob.go | 8 +- routers/api/packages/container/container.go | 4 +- routers/api/packages/debian/debian.go | 317 ------------- routers/api/packages/generic/generic.go | 2 +- routers/api/packages/helm/helm.go | 2 +- routers/api/packages/maven/maven.go | 2 +- routers/api/packages/npm/npm.go | 2 +- routers/api/packages/nuget/nuget.go | 6 +- routers/api/packages/pub/pub.go | 2 +- routers/api/packages/pypi/pypi.go | 2 +- routers/api/packages/rubygems/rubygems.go | 2 +- routers/api/packages/swift/swift.go | 2 +- routers/api/packages/vagrant/vagrant.go | 2 +- routers/api/v1/packages/package.go | 2 +- routers/web/user/package.go | 34 +- services/forms/package_form.go | 2 +- services/packages/cleanup/cleanup.go | 11 - services/packages/debian/repository.go | 443 ------------------ services/packages/packages.go | 46 +- templates/package/content/debian.tmpl | 65 --- templates/package/metadata/debian.tmpl | 4 - templates/package/view.tmpl | 2 - templates/swagger/v1_json.tmpl | 1 - tests/integration/api_packages_debian_test.go | 252 ---------- web_src/svg/gitea-debian.svg | 9 - 57 files changed, 96 insertions(+), 1995 deletions(-) delete mode 100644 docs/content/doc/usage/packages/debian.en-us.md delete mode 100644 models/migrations/v1_20/v256.go delete mode 100644 models/packages/debian/search.go delete mode 100644 modules/packages/debian/metadata.go delete mode 100644 modules/packages/debian/metadata_test.go delete mode 100644 public/img/svg/gitea-debian.svg delete mode 100644 routers/api/packages/debian/debian.go delete mode 100644 services/packages/debian/repository.go delete mode 100644 templates/package/content/debian.tmpl delete mode 100644 templates/package/metadata/debian.tmpl delete mode 100644 tests/integration/api_packages_debian_test.go delete mode 100644 web_src/svg/gitea-debian.svg diff --git a/assets/go-licenses.json b/assets/go-licenses.json index 57285b6a9220e..090f595e740ae 100644 --- a/assets/go-licenses.json +++ b/assets/go-licenses.json @@ -114,11 +114,6 @@ "path": "github.com/bits-and-blooms/bitset/LICENSE", "licenseText": "Copyright (c) 2014 Will Fitzgerald. All rights reserved.\n\nRedistribution and use in source and binary forms, with or without\nmodification, are permitted provided that the following conditions are\nmet:\n\n * Redistributions of source code must retain the above copyright\nnotice, this list of conditions and the following disclaimer.\n * Redistributions in binary form must reproduce the above\ncopyright notice, this list of conditions and the following disclaimer\nin the documentation and/or other materials provided with the\ndistribution.\n * Neither the name of Google Inc. nor the names of its\ncontributors may be used to endorse or promote products derived from\nthis software without specific prior written permission.\n\nTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS\n\"AS IS\" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT\nLIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR\nA PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT\nOWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,\nSPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT\nLIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\nDATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\nTHEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\nOF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n" }, - { - "name": "github.com/blakesmith/ar", - "path": "github.com/blakesmith/ar/COPYING", - "licenseText": "Copyright (c) 2013 Blake Smith \u003cblakesmith0@gmail.com\u003e\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in\nall copies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\nTHE SOFTWARE.\n\n" - }, { "name": "github.com/blevesearch/bleve/v2", "path": "github.com/blevesearch/bleve/v2/LICENSE", diff --git a/cmd/migrate_storage_test.go b/cmd/migrate_storage_test.go index 7f3de894ba3e2..e6658b6e65ef1 100644 --- a/cmd/migrate_storage_test.go +++ b/cmd/migrate_storage_test.go @@ -25,7 +25,7 @@ func TestMigratePackages(t *testing.T) { creator := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1}) content := "package main\n\nfunc main() {\nfmt.Println(\"hi\")\n}\n" - buf, err := packages_module.CreateHashedBufferFromReaderWithSize(strings.NewReader(content), 1024) + buf, err := packages_module.CreateHashedBufferFromReader(strings.NewReader(content), 1024) assert.NoError(t, err) defer buf.Close() diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index f24770860fde1..3687e0fbd4caf 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -2501,8 +2501,6 @@ ROUTER = console ;LIMIT_SIZE_CONDA = -1 ;; Maximum size of a Container upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) ;LIMIT_SIZE_CONTAINER = -1 -;; Maximum size of a Debian upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) -;LIMIT_SIZE_DEBIAN = -1 ;; Maximum size of a Generic upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) ;LIMIT_SIZE_GENERIC = -1 ;; Maximum size of a Helm upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) diff --git a/docs/content/doc/administration/config-cheat-sheet.en-us.md b/docs/content/doc/administration/config-cheat-sheet.en-us.md index 44ea3633abe7a..03cd93f91eba2 100644 --- a/docs/content/doc/administration/config-cheat-sheet.en-us.md +++ b/docs/content/doc/administration/config-cheat-sheet.en-us.md @@ -1252,7 +1252,6 @@ Task queue configuration has been moved to `queue.task`. However, the below conf - `LIMIT_SIZE_CONAN`: **-1**: Maximum size of a Conan upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) - `LIMIT_SIZE_CONDA`: **-1**: Maximum size of a Conda upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) - `LIMIT_SIZE_CONTAINER`: **-1**: Maximum size of a Container upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) -- `LIMIT_SIZE_DEBIAN`: **-1**: Maximum size of a Debian upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) - `LIMIT_SIZE_GENERIC`: **-1**: Maximum size of a Generic upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) - `LIMIT_SIZE_HELM`: **-1**: Maximum size of a Helm upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) - `LIMIT_SIZE_MAVEN`: **-1**: Maximum size of a Maven upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) diff --git a/docs/content/doc/usage/packages/debian.en-us.md b/docs/content/doc/usage/packages/debian.en-us.md deleted file mode 100644 index ac1abed18e912..0000000000000 --- a/docs/content/doc/usage/packages/debian.en-us.md +++ /dev/null @@ -1,134 +0,0 @@ ---- -date: "2023-01-07T00:00:00+00:00" -title: "Debian Packages Repository" -slug: "packages/debian" -draft: false -toc: false -menu: - sidebar: - parent: "packages" - name: "Debian" - weight: 35 - identifier: "debian" ---- - -# Debian Packages Repository - -Publish [Debian](https://www.debian.org/distrib/packages) packages for your user or organization. - -**Table of Contents** - -{{< toc >}} - -## Requirements - -To work with the Debian registry, you need to use a HTTP client like `curl` to upload and a package manager like `apt` to consume packages. - -The following examples use `apt`. - -## Configuring the package registry - -To register the Debian registry add the url to the list of known apt sources: - -```shell -echo "deb https://gitea.example.com/api/packages/{owner}/debian {distribution} {component}" | sudo tee -a /etc/apt/sources.list.d/gitea.list -``` - -| Placeholder | Description | -| -------------- | ----------- | -| `owner` | The owner of the package. | -| `distribution` | The distribution to use. | -| `component` | The component to use. | - -If the registry is private, provide credentials in the url. You can use a password or a [personal access token]({{< relref "doc/development/api-usage.en-us.md#authentication" >}}): - -```shell -echo "deb https://{username}:{your_password_or_token}@gitea.example.com/api/packages/{owner}/debian {distribution} {component}" | sudo tee -a /etc/apt/sources.list.d/gitea.list -``` - -The Debian registry files are signed with a PGP key which must be known to apt: - -```shell -sudo curl https://gitea.example.com/api/packages/{owner}/debian/repository.key -o /etc/apt/trusted.gpg.d/gitea-{owner}.asc -``` - -Afterwards update the local package index: - -```shell -apt update -``` - -## Publish a package - -To publish a Debian package (`*.deb`), perform a HTTP PUT operation with the package content in the request body. - -``` -PUT https://gitea.example.com/api/packages/{owner}/debian/pool/{distribution}/{component}/upload -``` - -| Parameter | Description | -| -------------- | ----------- | -| `owner` | The owner of the package. | -| `distribution` | The distribution may match the release name of the OS, ex: `bionic`. | -| `component` | The component can be used to group packages or just `main` or similar. | - -Example request using HTTP Basic authentication: - -```shell -curl --user your_username:your_password_or_token \ - --upload-file path/to/file.deb \ - https://gitea.example.com/api/packages/testuser/debian/pool/bionic/main/upload -``` - -If you are using 2FA or OAuth use a [personal access token]({{< relref "doc/development/api-usage.en-us.md#authentication" >}}) instead of the password. -You cannot publish a file with the same name twice to a package. You must delete the existing package version first. - -The server reponds with the following HTTP Status codes. - -| HTTP Status Code | Meaning | -| ----------------- | ------- | -| `201 Created` | The package has been published. | -| `400 Bad Request` | The package name, version, distribution, component or architecture are invalid. | -| `409 Conflict` | A package file with the same combination of parameters exist already in the package. | - -## Delete a package - -To delete a Debian package perform a HTTP DELETE operation. This will delete the package version too if there is no file left. - -``` -DELETE https://gitea.example.com/api/packages/{owner}/debian/pool/{distribution}/{component}/{package_name}/{package_version}/{architecture} -``` - -| Parameter | Description | -| ----------------- | ----------- | -| `owner` | The owner of the package. | -| `package_name` | The package name. | -| `package_version` | The package version. | -| `distribution` | The package distribution. | -| `component` | The package component. | -| `architecture` | The package architecture. | - -Example request using HTTP Basic authentication: - -```shell -curl --user your_username:your_token_or_password -X DELETE \ - https://gitea.example.com/api/packages/testuser/debian/pools/bionic/main/test-package/1.0.0/amd64 -``` - -The server reponds with the following HTTP Status codes. - -| HTTP Status Code | Meaning | -| ----------------- | ------- | -| `204 No Content` | Success | -| `404 Not Found` | The package or file was not found. | - -## Install a package - -To install a package from the Debian registry, execute the following commands: - -```shell -# use latest version -apt install {package_name} -# use specific version -apt install {package_name}={package_version} -``` diff --git a/docs/content/doc/usage/packages/overview.en-us.md b/docs/content/doc/usage/packages/overview.en-us.md index 8e4fd87e7b7e7..fdeaf15af0edd 100644 --- a/docs/content/doc/usage/packages/overview.en-us.md +++ b/docs/content/doc/usage/packages/overview.en-us.md @@ -33,7 +33,6 @@ The following package managers are currently supported: | [Conan]({{< relref "doc/usage/packages/conan.en-us.md" >}}) | C++ | `conan` | | [Conda]({{< relref "doc/usage/packages/conda.en-us.md" >}}) | - | `conda` | | [Container]({{< relref "doc/usage/packages/container.en-us.md" >}}) | - | any OCI compliant client | -| [Debian]({{< relref "doc/usage/packages/debian.en-us.md" >}}) | - | `apt` | | [Generic]({{< relref "doc/usage/packages/generic.en-us.md" >}}) | - | any HTTP client | | [Helm]({{< relref "doc/usage/packages/helm.en-us.md" >}}) | - | any HTTP client, `cm-push` | | [Maven]({{< relref "doc/usage/packages/maven.en-us.md" >}}) | Java | `mvn`, `gradle` | diff --git a/go.mod b/go.mod index ef5f22d96eafe..a71b136cb1f2c 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,6 @@ require ( github.com/NYTimes/gziphandler v1.1.1 github.com/PuerkitoBio/goquery v1.8.0 github.com/alecthomas/chroma/v2 v2.5.0 - github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb github.com/blevesearch/bleve/v2 v2.3.6 github.com/bufbuild/connect-go v1.3.1 github.com/buildkite/terminal-to-html/v3 v3.7.0 @@ -97,7 +96,6 @@ require ( github.com/stretchr/testify v1.8.1 github.com/syndtr/goleveldb v1.0.0 github.com/tstranex/u2f v1.0.0 - github.com/ulikunitz/xz v0.5.11 github.com/urfave/cli v1.22.12 github.com/xanzy/go-gitlab v0.80.2 github.com/xeipuuv/gojsonschema v1.2.0 @@ -262,6 +260,7 @@ require ( github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf // indirect github.com/subosito/gotenv v1.4.1 // indirect github.com/toqueteos/webbrowser v1.2.0 // indirect + github.com/ulikunitz/xz v0.5.11 // indirect github.com/unknwon/com v1.0.1 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/fasthttp v1.44.0 // indirect diff --git a/go.sum b/go.sum index ba969dddeefbd..2a3cc5967e549 100644 --- a/go.sum +++ b/go.sum @@ -162,8 +162,6 @@ github.com/bits-and-blooms/bitset v1.1.10/go.mod h1:w0XsmFg8qg6cmpTtJ0z3pKgjTDBM github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= github.com/bits-and-blooms/bitset v1.5.0 h1:NpE8frKRLGHIcEzkR+gZhiioW1+WbYV6fKwD6ZIpQT8= github.com/bits-and-blooms/bitset v1.5.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= -github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb h1:m935MPodAbYS46DG4pJSv7WO+VECIWUQ7OJYSoTrMh4= -github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb/go.mod h1:PkYb9DJNAwrSvRx5DYA+gUcOIgTGVMNkfSCbZM8cWpI= github.com/blevesearch/bleve/v2 v2.0.5/go.mod h1:ZjWibgnbRX33c+vBRgla9QhPb4QOjD6fdVJ+R1Bk8LM= github.com/blevesearch/bleve/v2 v2.3.6 h1:NlntUHcV5CSWIhpugx4d/BRMGCiaoI8ZZXrXlahzNq4= github.com/blevesearch/bleve/v2 v2.3.6/go.mod h1:JM2legf1cKVkdV8Ehu7msKIOKC0McSw0Q16Fmv9vsW4= diff --git a/models/migrations/migrations.go b/models/migrations/migrations.go index 0e84ae9f0e29a..1f1f43796cf4a 100644 --- a/models/migrations/migrations.go +++ b/models/migrations/migrations.go @@ -489,8 +489,6 @@ var migrations = []Migration{ NewMigration("Add ActionTaskOutput table", v1_20.AddActionTaskOutputTable), // v255 -> v256 NewMigration("Add ArchivedUnix Column", v1_20.AddArchivedUnixToRepository), - // v256 -> v257 - NewMigration("Add is_internal column to package", v1_20.AddIsInternalColumnToPackage), } // GetCurrentDBVersion returns the current db version diff --git a/models/migrations/v1_20/v256.go b/models/migrations/v1_20/v256.go deleted file mode 100644 index ddb487f6844f4..0000000000000 --- a/models/migrations/v1_20/v256.go +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright 2023 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package v1_20 //nolint - -import ( - "xorm.io/xorm" -) - -func AddIsInternalColumnToPackage(x *xorm.Engine) error { - type Package struct { - ID int64 `xorm:"pk autoincr"` - OwnerID int64 `xorm:"UNIQUE(s) INDEX NOT NULL"` - RepoID int64 `xorm:"INDEX"` - Type string `xorm:"UNIQUE(s) INDEX NOT NULL"` - Name string `xorm:"NOT NULL"` - LowerName string `xorm:"UNIQUE(s) INDEX NOT NULL"` - SemverCompatible bool `xorm:"NOT NULL DEFAULT false"` - IsInternal bool `xorm:"INDEX NOT NULL DEFAULT false"` - } - - return x.Sync2(new(Package)) -} diff --git a/models/packages/container/search.go b/models/packages/container/search.go index 0d3664d384349..b65c8634d6544 100644 --- a/models/packages/container/search.go +++ b/models/packages/container/search.go @@ -101,7 +101,16 @@ func getContainerBlobsLimit(ctx context.Context, opts *BlobSearchOptions, limit return nil, err } - return packages.GetPackageFileDescriptors(ctx, pfs) + pfds := make([]*packages.PackageFileDescriptor, 0, len(pfs)) + for _, pf := range pfs { + pfd, err := packages.GetPackageFileDescriptor(ctx, pf) + if err != nil { + return nil, err + } + pfds = append(pfds, pfd) + } + + return pfds, nil } // GetManifestVersions gets all package versions representing the matching manifest diff --git a/models/packages/debian/search.go b/models/packages/debian/search.go deleted file mode 100644 index 332a4f7040c51..0000000000000 --- a/models/packages/debian/search.go +++ /dev/null @@ -1,131 +0,0 @@ -// Copyright 2023 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package debian - -import ( - "context" - "strconv" - - "code.gitea.io/gitea/models/db" - "code.gitea.io/gitea/models/packages" - debian_module "code.gitea.io/gitea/modules/packages/debian" - - "xorm.io/builder" -) - -type PackageSearchOptions struct { - OwnerID int64 - Distribution string - Component string - Architecture string -} - -// SearchLatestPackages gets the latest packages matching the search options -func SearchLatestPackages(ctx context.Context, opts *PackageSearchOptions) ([]*packages.PackageFileDescriptor, error) { - var cond builder.Cond = builder.Eq{ - "package_file.is_lead": true, - "package.type": packages.TypeDebian, - "package.owner_id": opts.OwnerID, - "package.is_internal": false, - "package_version.is_internal": false, - } - - props := make(map[string]string) - if opts.Distribution != "" { - props[debian_module.PropertyDistribution] = opts.Distribution - } - if opts.Component != "" { - props[debian_module.PropertyComponent] = opts.Component - } - if opts.Architecture != "" { - props[debian_module.PropertyArchitecture] = opts.Architecture - } - - if len(props) > 0 { - var propsCond builder.Cond = builder.Eq{ - "package_property.ref_type": packages.PropertyTypeFile, - } - propsCond = propsCond.And(builder.Expr("package_property.ref_id = package_file.id")) - - propsCondBlock := builder.NewCond() - for name, value := range props { - propsCondBlock = propsCondBlock.Or(builder.Eq{ - "package_property.name": name, - "package_property.value": value, - }) - } - propsCond = propsCond.And(propsCondBlock) - - cond = cond.And(builder.Eq{ - strconv.Itoa(len(props)): builder.Select("COUNT(*)").Where(propsCond).From("package_property"), - }) - } - - cond = cond. - And(builder.Expr("pv2.id IS NULL")) - - joinCond := builder. - Expr("package_version.package_id = pv2.package_id AND (package_version.created_unix < pv2.created_unix OR (package_version.created_unix = pv2.created_unix AND package_version.id < pv2.id))"). - And(builder.Eq{"pv2.is_internal": false}) - - pfs := make([]*packages.PackageFile, 0, 10) - err := db.GetEngine(ctx). - Table("package_file"). - Select("package_file.*"). - Join("INNER", "package_version", "package_version.id = package_file.version_id"). - Join("LEFT", "package_version pv2", joinCond). - Join("INNER", "package", "package.id = package_version.package_id"). - Where(cond). - Desc("package_version.created_unix"). - Find(&pfs) - if err != nil { - return nil, err - } - - return packages.GetPackageFileDescriptors(ctx, pfs) -} - -// GetDistributions gets all available distributions -func GetDistributions(ctx context.Context, ownerID int64) ([]string, error) { - return getDistinctPropertyValues(ctx, ownerID, "", debian_module.PropertyDistribution) -} - -// GetComponents gets all available components for the given distribution -func GetComponents(ctx context.Context, ownerID int64, distribution string) ([]string, error) { - return getDistinctPropertyValues(ctx, ownerID, distribution, debian_module.PropertyComponent) -} - -// GetArchitectures gets all available architectures for the given distribution -func GetArchitectures(ctx context.Context, ownerID int64, distribution string) ([]string, error) { - return getDistinctPropertyValues(ctx, ownerID, distribution, debian_module.PropertyArchitecture) -} - -func getDistinctPropertyValues(ctx context.Context, ownerID int64, distribution, propName string) ([]string, error) { - var cond builder.Cond = builder.Eq{ - "package_property.ref_type": packages.PropertyTypeFile, - "package_property.name": propName, - "package.type": packages.TypeDebian, - "package.owner_id": ownerID, - } - if distribution != "" { - innerCond := builder. - Expr("pp.ref_id = package_property.ref_id"). - And(builder.Eq{ - "pp.ref_type": packages.PropertyTypeFile, - "pp.name": debian_module.PropertyDistribution, - "pp.value": distribution, - }) - cond = cond.And(builder.Exists(builder.Select("pp.ref_id").From("package_property pp").Where(innerCond))) - } - - values := make([]string, 0, 5) - return values, db.GetEngine(ctx). - Table("package_property"). - Distinct("package_property.value"). - Join("INNER", "package_file", "package_file.id = package_property.ref_id"). - Join("INNER", "package_version", "package_version.id = package_file.version_id"). - Join("INNER", "package", "package.id = package_version.package_id"). - Where(cond). - Find(&values) -} diff --git a/models/packages/descriptor.go b/models/packages/descriptor.go index 9256dd563022e..974c5b2c362a6 100644 --- a/models/packages/descriptor.go +++ b/models/packages/descriptor.go @@ -18,7 +18,6 @@ import ( "code.gitea.io/gitea/modules/packages/conan" "code.gitea.io/gitea/modules/packages/conda" "code.gitea.io/gitea/modules/packages/container" - "code.gitea.io/gitea/modules/packages/debian" "code.gitea.io/gitea/modules/packages/helm" "code.gitea.io/gitea/modules/packages/maven" "code.gitea.io/gitea/modules/packages/npm" @@ -128,9 +127,13 @@ func GetPackageDescriptor(ctx context.Context, pv *PackageVersion) (*PackageDesc return nil, err } - pfds, err := GetPackageFileDescriptors(ctx, pfs) - if err != nil { - return nil, err + pfds := make([]*PackageFileDescriptor, 0, len(pfs)) + for _, pf := range pfs { + pfd, err := GetPackageFileDescriptor(ctx, pf) + if err != nil { + return nil, err + } + pfds = append(pfds, pfd) } var metadata interface{} @@ -147,8 +150,6 @@ func GetPackageDescriptor(ctx context.Context, pv *PackageVersion) (*PackageDesc metadata = &conda.VersionMetadata{} case TypeContainer: metadata = &container.Metadata{} - case TypeDebian: - metadata = &debian.Metadata{} case TypeGeneric: // generic packages have no metadata case TypeHelm: @@ -209,19 +210,6 @@ func GetPackageFileDescriptor(ctx context.Context, pf *PackageFile) (*PackageFil }, nil } -// GetPackageFileDescriptors gets the package file descriptors for the package files -func GetPackageFileDescriptors(ctx context.Context, pfs []*PackageFile) ([]*PackageFileDescriptor, error) { - pfds := make([]*PackageFileDescriptor, 0, len(pfs)) - for _, pf := range pfs { - pfd, err := GetPackageFileDescriptor(ctx, pf) - if err != nil { - return nil, err - } - pfds = append(pfds, pfd) - } - return pfds, nil -} - // GetPackageDescriptors gets the package descriptions for the versions func GetPackageDescriptors(ctx context.Context, pvs []*PackageVersion) ([]*PackageDescriptor, error) { pds := make([]*PackageDescriptor, 0, len(pvs)) diff --git a/models/packages/package.go b/models/packages/package.go index 579e9e4d53351..ccc9257c31235 100644 --- a/models/packages/package.go +++ b/models/packages/package.go @@ -36,7 +36,6 @@ const ( TypeConan Type = "conan" TypeConda Type = "conda" TypeContainer Type = "container" - TypeDebian Type = "debian" TypeGeneric Type = "generic" TypeHelm Type = "helm" TypeMaven Type = "maven" @@ -56,7 +55,6 @@ var TypeList = []Type{ TypeConan, TypeConda, TypeContainer, - TypeDebian, TypeGeneric, TypeHelm, TypeMaven, @@ -84,8 +82,6 @@ func (pt Type) Name() string { return "Conda" case TypeContainer: return "Container" - case TypeDebian: - return "Debian" case TypeGeneric: return "Generic" case TypeHelm: @@ -125,8 +121,6 @@ func (pt Type) SVGName() string { return "gitea-conda" case TypeContainer: return "octicon-container" - case TypeDebian: - return "gitea-debian" case TypeGeneric: return "octicon-package" case TypeHelm: @@ -160,7 +154,6 @@ type Package struct { Name string `xorm:"NOT NULL"` LowerName string `xorm:"UNIQUE(s) INDEX NOT NULL"` SemverCompatible bool `xorm:"NOT NULL DEFAULT false"` - IsInternal bool `xorm:"INDEX NOT NULL DEFAULT false"` } // TryInsertPackage inserts a package. If a package exists already, ErrDuplicatePackage is returned @@ -221,10 +214,9 @@ func GetPackageByID(ctx context.Context, packageID int64) (*Package, error) { // GetPackageByName gets a package by name func GetPackageByName(ctx context.Context, ownerID int64, packageType Type, name string) (*Package, error) { var cond builder.Cond = builder.Eq{ - "package.owner_id": ownerID, - "package.type": packageType, - "package.lower_name": strings.ToLower(name), - "package.is_internal": false, + "package.owner_id": ownerID, + "package.type": packageType, + "package.lower_name": strings.ToLower(name), } p := &Package{} @@ -244,9 +236,8 @@ func GetPackageByName(ctx context.Context, ownerID int64, packageType Type, name // GetPackagesByType gets all packages of a specific type func GetPackagesByType(ctx context.Context, ownerID int64, packageType Type) ([]*Package, error) { var cond builder.Cond = builder.Eq{ - "package.owner_id": ownerID, - "package.type": packageType, - "package.is_internal": false, + "package.owner_id": ownerID, + "package.type": packageType, } ps := make([]*Package, 0, 10) diff --git a/models/packages/package_file.go b/models/packages/package_file.go index 337ab1135a2f6..97e7a0d4070a9 100644 --- a/models/packages/package_file.go +++ b/models/packages/package_file.go @@ -117,15 +117,13 @@ func DeleteFileByID(ctx context.Context, fileID int64) error { // PackageFileSearchOptions are options for SearchXXX methods type PackageFileSearchOptions struct { - OwnerID int64 - PackageType string - VersionID int64 - Query string - CompositeKey string - Properties map[string]string - OlderThan time.Duration - HashAlgorithmn string - Hash string + OwnerID int64 + PackageType string + VersionID int64 + Query string + CompositeKey string + Properties map[string]string + OlderThan time.Duration db.Paginator } @@ -184,15 +182,6 @@ func (opts *PackageFileSearchOptions) toConds() builder.Cond { cond = cond.And(builder.Lt{"package_file.created_unix": time.Now().Add(-opts.OlderThan).Unix()}) } - if opts.Hash != "" && (opts.HashAlgorithmn == "md5" || opts.HashAlgorithmn == "sha1" || opts.HashAlgorithmn == "sha256" || opts.HashAlgorithmn == "sha512") { - innerCond := builder. - Expr("package_blob.id = package_file.blob_id"). - And(builder.Eq{ - "package_blob.hash_" + opts.HashAlgorithmn: opts.Hash, - }) - cond = cond.And(builder.Exists(builder.Select("package_blob.id").From("package_blob").Where(innerCond))) - } - return cond } diff --git a/models/packages/package_version.go b/models/packages/package_version.go index ab1bcddae5818..759c20abed22b 100644 --- a/models/packages/package_version.go +++ b/models/packages/package_version.go @@ -173,7 +173,7 @@ const ( ) // PackageSearchOptions are options for SearchXXX methods -// All fields optional and are not used if they have their default value (nil, "", 0) +// Besides IsInternal are all fields optional and are not used if they have their default value (nil, "", 0) type PackageSearchOptions struct { OwnerID int64 RepoID int64 @@ -192,9 +192,7 @@ type PackageSearchOptions struct { func (opts *PackageSearchOptions) toConds() builder.Cond { cond := builder.NewCond() if !opts.IsInternal.IsNone() { - cond = builder.Eq{ - "package_version.is_internal": opts.IsInternal.IsTrue(), - } + cond = builder.Eq{"package_version.is_internal": opts.IsInternal.IsTrue()} } if opts.OwnerID != 0 { diff --git a/models/user/setting.go b/models/user/setting.go index a41e494db9b54..aec79b756bf14 100644 --- a/models/user/setting.go +++ b/models/user/setting.go @@ -11,7 +11,6 @@ import ( "code.gitea.io/gitea/models/db" "code.gitea.io/gitea/modules/cache" setting_module "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/modules/util" "xorm.io/builder" ) @@ -43,10 +42,6 @@ func (err ErrUserSettingIsNotExist) Error() string { return fmt.Sprintf("Setting[%s] is not exist", err.Key) } -func (err ErrUserSettingIsNotExist) Unwrap() error { - return util.ErrNotExist -} - // IsErrUserSettingIsNotExist return true if err is ErrSettingIsNotExist func IsErrUserSettingIsNotExist(err error) bool { _, ok := err.(ErrUserSettingIsNotExist) diff --git a/modules/packages/debian/metadata.go b/modules/packages/debian/metadata.go deleted file mode 100644 index 08daaf082e126..0000000000000 --- a/modules/packages/debian/metadata.go +++ /dev/null @@ -1,216 +0,0 @@ -// Copyright 2023 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package debian - -import ( - "archive/tar" - "bufio" - "compress/gzip" - "io" - "net/mail" - "regexp" - "strings" - - "code.gitea.io/gitea/modules/util" - "code.gitea.io/gitea/modules/validation" - - "github.com/blakesmith/ar" - "github.com/klauspost/compress/zstd" - "github.com/ulikunitz/xz" -) - -const ( - PropertyDistribution = "debian.distribution" - PropertyComponent = "debian.component" - PropertyArchitecture = "debian.architecture" - PropertyControl = "debian.control" - PropertyRepositoryIncludeInRelease = "debian.repository.include_in_release" - - SettingKeyPrivate = "debian.key.private" - SettingKeyPublic = "debian.key.public" - - RepositoryPackage = "_debian" - RepositoryVersion = "_repository" -) - -var ( - ErrMissingControlFile = util.NewInvalidArgumentErrorf("control file is missing") - ErrUnsupportedCompression = util.NewInvalidArgumentErrorf("unsupported compression algorithmn") - ErrInvalidName = util.NewInvalidArgumentErrorf("package name is invalid") - ErrInvalidVersion = util.NewInvalidArgumentErrorf("package version is invalid") - ErrInvalidArchitecture = util.NewInvalidArgumentErrorf("package architecture is invalid") - - // https://www.debian.org/doc/debian-policy/ch-controlfields.html#source - namePattern = regexp.MustCompile(`\A[a-z0-9][a-z0-9+-.]+\z`) - // https://www.debian.org/doc/debian-policy/ch-controlfields.html#version - versionPattern = regexp.MustCompile(`\A(?:[0-9]:)?[a-zA-Z0-9.+~]+(?:-[a-zA-Z0-9.+-~]+)?\z`) -) - -type Package struct { - Name string - Version string - Architecture string - Control string - Metadata *Metadata -} - -type Metadata struct { - Maintainer string `json:"maintainer,omitempty"` - ProjectURL string `json:"project_url,omitempty"` - Description string `json:"description,omitempty"` - Dependencies []string `json:"dependencies,omitempty"` -} - -// ParsePackage parses the Debian package file -// https://manpages.debian.org/bullseye/dpkg-dev/deb.5.en.html -func ParsePackage(r io.Reader) (*Package, error) { - arr := ar.NewReader(r) - - for { - hd, err := arr.Next() - if err == io.EOF { - break - } - if err != nil { - return nil, err - } - - if strings.HasPrefix(hd.Name, "control.tar") { - var inner io.Reader - switch hd.Name[11:] { - case "": - inner = arr - case ".gz": - gzr, err := gzip.NewReader(arr) - if err != nil { - return nil, err - } - defer gzr.Close() - - inner = gzr - case ".xz": - xzr, err := xz.NewReader(arr) - if err != nil { - return nil, err - } - - inner = xzr - case ".zst": - zr, err := zstd.NewReader(arr) - if err != nil { - return nil, err - } - defer zr.Close() - - inner = zr - default: - return nil, ErrUnsupportedCompression - } - - tr := tar.NewReader(inner) - for { - hd, err := tr.Next() - if err == io.EOF { - break - } - if err != nil { - return nil, err - } - - if hd.Typeflag != tar.TypeReg { - continue - } - - if hd.FileInfo().Name() == "control" { - return ParseControlFile(tr) - } - } - } - } - - return nil, ErrMissingControlFile -} - -// ParseControlFile parses a Debian control file to retrieve the metadata -func ParseControlFile(r io.Reader) (*Package, error) { - p := &Package{ - Metadata: &Metadata{}, - } - - key := "" - var depends strings.Builder - var control strings.Builder - - s := bufio.NewScanner(io.TeeReader(r, &control)) - for s.Scan() { - line := s.Text() - - trimmed := strings.TrimSpace(line) - if trimmed == "" { - continue - } - - if line[0] == ' ' || line[0] == '\t' { - switch key { - case "Description": - p.Metadata.Description += line - case "Depends": - depends.WriteString(trimmed) - } - } else { - parts := strings.SplitN(trimmed, ":", 2) - if len(parts) < 2 { - continue - } - - key = parts[0] - value := strings.TrimSpace(parts[1]) - switch key { - case "Package": - if !namePattern.MatchString(value) { - return nil, ErrInvalidName - } - p.Name = value - case "Version": - if !versionPattern.MatchString(value) { - return nil, ErrInvalidVersion - } - p.Version = value - case "Architecture": - if value == "" { - return nil, ErrInvalidArchitecture - } - p.Architecture = value - case "Maintainer": - a, err := mail.ParseAddress(value) - if err != nil || a.Name == "" { - p.Metadata.Maintainer = value - } else { - p.Metadata.Maintainer = a.Name - } - case "Description": - p.Metadata.Description = value - case "Depends": - depends.WriteString(value) - case "Homepage": - if validation.IsValidURL(value) { - p.Metadata.ProjectURL = value - } - } - } - } - if err := s.Err(); err != nil { - return nil, err - } - - dependencies := strings.Split(depends.String(), ",") - for i := range dependencies { - dependencies[i] = strings.TrimSpace(dependencies[i]) - } - p.Metadata.Dependencies = dependencies - - p.Control = control.String() - - return p, nil -} diff --git a/modules/packages/debian/metadata_test.go b/modules/packages/debian/metadata_test.go deleted file mode 100644 index 69fd51ea79002..0000000000000 --- a/modules/packages/debian/metadata_test.go +++ /dev/null @@ -1,171 +0,0 @@ -// Copyright 2023 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package debian - -import ( - "archive/tar" - "bytes" - "compress/gzip" - "io" - "testing" - - "github.com/blakesmith/ar" - "github.com/klauspost/compress/zstd" - "github.com/stretchr/testify/assert" - "github.com/ulikunitz/xz" -) - -const ( - packageName = "gitea" - packageVersion = "0:1.0.1-te~st" - packageArchitecture = "amd64" - packageAuthor = "KN4CK3R" - description = "Description with multiple lines." - projectURL = "https://gitea.io" -) - -func TestParsePackage(t *testing.T) { - createArchive := func(files map[string][]byte) io.Reader { - var buf bytes.Buffer - aw := ar.NewWriter(&buf) - aw.WriteGlobalHeader() - for filename, content := range files { - hdr := &ar.Header{ - Name: filename, - Mode: 0o600, - Size: int64(len(content)), - } - aw.WriteHeader(hdr) - aw.Write(content) - } - return &buf - } - - t.Run("MissingControlFile", func(t *testing.T) { - data := createArchive(map[string][]byte{"dummy.txt": {}}) - - p, err := ParsePackage(data) - assert.Nil(t, p) - assert.ErrorIs(t, err, ErrMissingControlFile) - }) - - t.Run("Compression", func(t *testing.T) { - t.Run("Unsupported", func(t *testing.T) { - data := createArchive(map[string][]byte{"control.tar.foo": {}}) - - p, err := ParsePackage(data) - assert.Nil(t, p) - assert.ErrorIs(t, err, ErrUnsupportedCompression) - }) - - var buf bytes.Buffer - tw := tar.NewWriter(&buf) - tw.WriteHeader(&tar.Header{ - Name: "control", - Mode: 0o600, - Size: 50, - }) - tw.Write([]byte("Package: gitea\nVersion: 1.0.0\nArchitecture: amd64\n")) - tw.Close() - - t.Run("None", func(t *testing.T) { - data := createArchive(map[string][]byte{"control.tar": buf.Bytes()}) - - p, err := ParsePackage(data) - assert.NotNil(t, p) - assert.NoError(t, err) - assert.Equal(t, "gitea", p.Name) - }) - - t.Run("gz", func(t *testing.T) { - var zbuf bytes.Buffer - zw := gzip.NewWriter(&zbuf) - zw.Write(buf.Bytes()) - zw.Close() - - data := createArchive(map[string][]byte{"control.tar.gz": zbuf.Bytes()}) - - p, err := ParsePackage(data) - assert.NotNil(t, p) - assert.NoError(t, err) - assert.Equal(t, "gitea", p.Name) - }) - - t.Run("xz", func(t *testing.T) { - var xbuf bytes.Buffer - xw, _ := xz.NewWriter(&xbuf) - xw.Write(buf.Bytes()) - xw.Close() - - data := createArchive(map[string][]byte{"control.tar.xz": xbuf.Bytes()}) - - p, err := ParsePackage(data) - assert.NotNil(t, p) - assert.NoError(t, err) - assert.Equal(t, "gitea", p.Name) - }) - - t.Run("zst", func(t *testing.T) { - var zbuf bytes.Buffer - zw, _ := zstd.NewWriter(&zbuf) - zw.Write(buf.Bytes()) - zw.Close() - - data := createArchive(map[string][]byte{"control.tar.zst": zbuf.Bytes()}) - - p, err := ParsePackage(data) - assert.NotNil(t, p) - assert.NoError(t, err) - assert.Equal(t, "gitea", p.Name) - }) - }) -} - -func TestParseControlFile(t *testing.T) { - buildContent := func(name, version, architecture string) *bytes.Buffer { - var buf bytes.Buffer - buf.WriteString("Package: " + name + "\nVersion: " + version + "\nArchitecture: " + architecture + "\nMaintainer: " + packageAuthor + " \nHomepage: " + projectURL + "\nDepends: a,\n b\nDescription: Description\n with multiple\n lines.") - return &buf - } - - t.Run("InvalidName", func(t *testing.T) { - for _, name := range []string{"", "-cd"} { - p, err := ParseControlFile(buildContent(name, packageVersion, packageArchitecture)) - assert.Nil(t, p) - assert.ErrorIs(t, err, ErrInvalidName) - } - }) - - t.Run("InvalidVersion", func(t *testing.T) { - for _, version := range []string{"", "1-", ":1.0", "1_0"} { - p, err := ParseControlFile(buildContent(packageName, version, packageArchitecture)) - assert.Nil(t, p) - assert.ErrorIs(t, err, ErrInvalidVersion) - } - }) - - t.Run("InvalidArchitecture", func(t *testing.T) { - p, err := ParseControlFile(buildContent(packageName, packageVersion, "")) - assert.Nil(t, p) - assert.ErrorIs(t, err, ErrInvalidArchitecture) - }) - - t.Run("Valid", func(t *testing.T) { - content := buildContent(packageName, packageVersion, packageArchitecture) - full := content.String() - - p, err := ParseControlFile(content) - assert.NoError(t, err) - assert.NotNil(t, p) - - assert.Equal(t, packageName, p.Name) - assert.Equal(t, packageVersion, p.Version) - assert.Equal(t, packageArchitecture, p.Architecture) - assert.Equal(t, description, p.Metadata.Description) - assert.Equal(t, projectURL, p.Metadata.ProjectURL) - assert.Equal(t, packageAuthor, p.Metadata.Maintainer) - assert.Equal(t, []string{"a", "b"}, p.Metadata.Dependencies) - assert.Equal(t, full, p.Control) - }) -} diff --git a/modules/packages/hashed_buffer.go b/modules/packages/hashed_buffer.go index 017ddf1c8f5a5..ef00a45057f7c 100644 --- a/modules/packages/hashed_buffer.go +++ b/modules/packages/hashed_buffer.go @@ -25,15 +25,8 @@ type HashedBuffer struct { combinedWriter io.Writer } -const DefaultMemorySize = 32 * 1024 * 1024 - -// NewHashedBuffer creates a hashed buffer with the default memory size -func NewHashedBuffer() (*HashedBuffer, error) { - return NewHashedBufferWithSize(DefaultMemorySize) -} - -// NewHashedBuffer creates a hashed buffer with a specific memory size -func NewHashedBufferWithSize(maxMemorySize int) (*HashedBuffer, error) { +// NewHashedBuffer creates a hashed buffer with a specific maximum memory size +func NewHashedBuffer(maxMemorySize int) (*HashedBuffer, error) { b, err := filebuffer.New(maxMemorySize) if err != nil { return nil, err @@ -50,14 +43,9 @@ func NewHashedBufferWithSize(maxMemorySize int) (*HashedBuffer, error) { }, nil } -// CreateHashedBufferFromReader creates a hashed buffer with the default memory size and copies the provided reader data into it. -func CreateHashedBufferFromReader(r io.Reader) (*HashedBuffer, error) { - return CreateHashedBufferFromReaderWithSize(r, DefaultMemorySize) -} - -// CreateHashedBufferFromReaderWithSize creates a hashed buffer and copies the provided reader data into it. -func CreateHashedBufferFromReaderWithSize(r io.Reader, maxMemorySize int) (*HashedBuffer, error) { - b, err := NewHashedBufferWithSize(maxMemorySize) +// CreateHashedBufferFromReader creates a hashed buffer and copies the provided reader data into it. +func CreateHashedBufferFromReader(r io.Reader, maxMemorySize int) (*HashedBuffer, error) { + b, err := NewHashedBuffer(maxMemorySize) if err != nil { return nil, err } diff --git a/modules/packages/hashed_buffer_test.go b/modules/packages/hashed_buffer_test.go index 564e782f18f0a..e907aa060578a 100644 --- a/modules/packages/hashed_buffer_test.go +++ b/modules/packages/hashed_buffer_test.go @@ -26,7 +26,7 @@ func TestHashedBuffer(t *testing.T) { } for _, c := range cases { - buf, err := CreateHashedBufferFromReaderWithSize(strings.NewReader(c.Data), c.MaxMemorySize) + buf, err := CreateHashedBufferFromReader(strings.NewReader(c.Data), c.MaxMemorySize) assert.NoError(t, err) assert.EqualValues(t, len(c.Data), buf.Size()) diff --git a/modules/packages/nuget/symbol_extractor.go b/modules/packages/nuget/symbol_extractor.go index 81bf0371a0b1a..b709eac4c1965 100644 --- a/modules/packages/nuget/symbol_extractor.go +++ b/modules/packages/nuget/symbol_extractor.go @@ -63,7 +63,7 @@ func ExtractPortablePdb(r io.ReaderAt, size int64) (PortablePdbList, error) { return err } - buf, err := packages.CreateHashedBufferFromReader(f) + buf, err := packages.CreateHashedBufferFromReader(f, 32*1024*1024) f.Close() diff --git a/modules/setting/packages.go b/modules/setting/packages.go index b52bbf40c7b90..89601c3b99043 100644 --- a/modules/setting/packages.go +++ b/modules/setting/packages.go @@ -30,7 +30,6 @@ var ( LimitSizeConan int64 LimitSizeConda int64 LimitSizeContainer int64 - LimitSizeDebian int64 LimitSizeGeneric int64 LimitSizeHelm int64 LimitSizeMaven int64 @@ -74,7 +73,6 @@ func loadPackagesFrom(rootCfg ConfigProvider) { Packages.LimitSizeConan = mustBytes(sec, "LIMIT_SIZE_CONAN") Packages.LimitSizeConda = mustBytes(sec, "LIMIT_SIZE_CONDA") Packages.LimitSizeContainer = mustBytes(sec, "LIMIT_SIZE_CONTAINER") - Packages.LimitSizeDebian = mustBytes(sec, "LIMIT_SIZE_DEBIAN") Packages.LimitSizeGeneric = mustBytes(sec, "LIMIT_SIZE_GENERIC") Packages.LimitSizeHelm = mustBytes(sec, "LIMIT_SIZE_HELM") Packages.LimitSizeMaven = mustBytes(sec, "LIMIT_SIZE_MAVEN") diff --git a/modules/util/filebuffer/file_backed_buffer.go b/modules/util/filebuffer/file_backed_buffer.go index 6b07bd0413864..bfddf90e92bcc 100644 --- a/modules/util/filebuffer/file_backed_buffer.go +++ b/modules/util/filebuffer/file_backed_buffer.go @@ -7,10 +7,11 @@ import ( "bytes" "errors" "io" - "math" "os" ) +const maxInt = int(^uint(0) >> 1) // taken from bytes.Buffer + var ( // ErrInvalidMemorySize occurs if the memory size is not in a valid range ErrInvalidMemorySize = errors.New("Memory size must be greater 0 and lower math.MaxInt32") @@ -36,7 +37,7 @@ type FileBackedBuffer struct { // New creates a file backed buffer with a specific maximum memory size func New(maxMemorySize int) (*FileBackedBuffer, error) { - if maxMemorySize < 0 || maxMemorySize > math.MaxInt32 { + if maxMemorySize < 0 || maxMemorySize > maxInt { return nil, ErrInvalidMemorySize } diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 76d8e5d4d7ced..0072ac6fc36fa 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -3258,14 +3258,6 @@ container.layers = Image Layers container.labels = Labels container.labels.key = Key container.labels.value = Value -debian.registry = Setup this registry from the command line: -debian.registry.info = Choose <distribution> and <component> from the list below. -debian.install = To install the package, run the following command: -debian.documentation = For more information on the Debian registry, see the documentation. -debian.repository = Repository Info -debian.repository.distributions = Distributions -debian.repository.components = Components -debian.repository.architectures = Architectures generic.download = Download package from the command line: generic.documentation = For more information on the generic registry, see the documentation. helm.registry = Setup this registry from the command line: diff --git a/public/img/svg/gitea-debian.svg b/public/img/svg/gitea-debian.svg deleted file mode 100644 index 96f8f468e506a..0000000000000 --- a/public/img/svg/gitea-debian.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/routers/api/packages/api.go b/routers/api/packages/api.go index 2ce233171c4d0..d5acd3d261165 100644 --- a/routers/api/packages/api.go +++ b/routers/api/packages/api.go @@ -21,7 +21,6 @@ import ( "code.gitea.io/gitea/routers/api/packages/conan" "code.gitea.io/gitea/routers/api/packages/conda" "code.gitea.io/gitea/routers/api/packages/container" - "code.gitea.io/gitea/routers/api/packages/debian" "code.gitea.io/gitea/routers/api/packages/generic" "code.gitea.io/gitea/routers/api/packages/helm" "code.gitea.io/gitea/routers/api/packages/maven" @@ -273,24 +272,6 @@ func CommonRoutes(ctx gocontext.Context) *web.Route { conda.UploadPackageFile(ctx) }) }, reqPackageAccess(perm.AccessModeRead)) - r.Group("/debian", func() { - r.Get("/repository.key", debian.GetRepositoryKey) - r.Group("/dists/{distribution}", func() { - r.Get("/{filename}", debian.GetRepositoryFile) - r.Get("/by-hash/{algorithmn}/{hash}", debian.GetRepositoryFileByHash) - r.Group("/{component}/{architecture}", func() { - r.Get("/{filename}", debian.GetRepositoryFile) - r.Get("/by-hash/{algorithmn}/{hash}", debian.GetRepositoryFileByHash) - }) - }) - r.Group("/pool/{distribution}/{component}", func() { - r.Get("/{name}_{version}_{architecture}.deb", debian.DownloadPackageFile) - r.Group("", func() { - r.Put("/upload", debian.UploadPackageFile) - r.Delete("/{name}/{version}/{architecture}", debian.DeletePackageFile) - }, reqPackageAccess(perm.AccessModeWrite)) - }) - }, reqPackageAccess(perm.AccessModeRead)) r.Group("/generic", func() { r.Group("/{packagename}/{packageversion}", func() { r.Delete("", reqPackageAccess(perm.AccessModeWrite), generic.DeletePackage) diff --git a/routers/api/packages/cargo/cargo.go b/routers/api/packages/cargo/cargo.go index 18c93d328ab44..e0bf5da13adb0 100644 --- a/routers/api/packages/cargo/cargo.go +++ b/routers/api/packages/cargo/cargo.go @@ -173,7 +173,7 @@ func UploadPackage(ctx *context.Context) { return } - buf, err := packages_module.CreateHashedBufferFromReader(cp.Content) + buf, err := packages_module.CreateHashedBufferFromReader(cp.Content, 32*1024*1024) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return diff --git a/routers/api/packages/chef/chef.go b/routers/api/packages/chef/chef.go index b48b1778c42d1..28d07dea47760 100644 --- a/routers/api/packages/chef/chef.go +++ b/routers/api/packages/chef/chef.go @@ -263,7 +263,7 @@ func UploadPackage(ctx *context.Context) { } defer file.Close() - buf, err := packages_module.CreateHashedBufferFromReader(file) + buf, err := packages_module.CreateHashedBufferFromReader(file, 32*1024*1024) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return diff --git a/routers/api/packages/composer/composer.go b/routers/api/packages/composer/composer.go index d93b11efdf66d..a623952aa7387 100644 --- a/routers/api/packages/composer/composer.go +++ b/routers/api/packages/composer/composer.go @@ -192,7 +192,7 @@ func DownloadPackageFile(ctx *context.Context) { // UploadPackage creates a new package func UploadPackage(ctx *context.Context) { - buf, err := packages_module.CreateHashedBufferFromReader(ctx.Req.Body) + buf, err := packages_module.CreateHashedBufferFromReader(ctx.Req.Body, 32*1024*1024) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return diff --git a/routers/api/packages/conan/conan.go b/routers/api/packages/conan/conan.go index caeb8c11bc4c8..d538cc7d397db 100644 --- a/routers/api/packages/conan/conan.go +++ b/routers/api/packages/conan/conan.go @@ -318,7 +318,7 @@ func uploadFile(ctx *context.Context, fileFilter container.Set[string], fileKey defer upload.Close() } - buf, err := packages_module.CreateHashedBufferFromReader(upload) + buf, err := packages_module.CreateHashedBufferFromReader(upload, 32*1024*1024) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return @@ -648,7 +648,10 @@ func deleteRecipeOrPackage(apictx *context.Context, rref *conan_module.RecipeRef } for _, pf := range pfs { - if err := packages_service.DeletePackageFile(ctx, pf); err != nil { + if err := packages_model.DeleteAllProperties(ctx, packages_model.PropertyTypeFile, pf.ID); err != nil { + return err + } + if err := packages_model.DeleteFileByID(ctx, pf.ID); err != nil { return err } } @@ -661,7 +664,11 @@ func deleteRecipeOrPackage(apictx *context.Context, rref *conan_module.RecipeRef if !has { versionDeleted = true - if err := packages_service.DeletePackageVersionAndReferences(ctx, pv); err != nil { + if err := packages_model.DeleteAllProperties(ctx, packages_model.PropertyTypeVersion, pv.ID); err != nil { + return err + } + + if err := packages_model.DeleteVersionByID(ctx, pv.ID); err != nil { return err } } diff --git a/routers/api/packages/conda/conda.go b/routers/api/packages/conda/conda.go index f7786906309b1..2ff619fed4e00 100644 --- a/routers/api/packages/conda/conda.go +++ b/routers/api/packages/conda/conda.go @@ -183,7 +183,7 @@ func UploadPackageFile(ctx *context.Context) { defer upload.Close() } - buf, err := packages_module.CreateHashedBufferFromReader(upload) + buf, err := packages_module.CreateHashedBufferFromReader(upload, 32*1024*1024) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return diff --git a/routers/api/packages/container/blob.go b/routers/api/packages/container/blob.go index c8e8dd0545290..f0457c55e19c3 100644 --- a/routers/api/packages/container/blob.go +++ b/routers/api/packages/container/blob.go @@ -27,6 +27,10 @@ var uploadVersionMutex sync.Mutex // saveAsPackageBlob creates a package blob from an upload // The uploaded blob gets stored in a special upload version to link them to the package/image func saveAsPackageBlob(hsr packages_module.HashedSizeReader, pci *packages_service.PackageCreationInfo) (*packages_model.PackageBlob, error) { + if err := packages_service.CheckSizeQuotaExceeded(db.DefaultContext, pci.Creator, pci.Owner, packages_model.TypeContainer, hsr.Size()); err != nil { + return nil, err + } + pb := packages_service.NewPackageBlob(hsr) exists := false @@ -39,10 +43,6 @@ func saveAsPackageBlob(hsr packages_module.HashedSizeReader, pci *packages_servi } err = db.WithTx(db.DefaultContext, func(ctx context.Context) error { - if err := packages_service.CheckSizeQuotaExceeded(ctx, pci.Creator, pci.Owner, packages_model.TypeContainer, hsr.Size()); err != nil { - return err - } - pb, exists, err = packages_model.GetOrInsertBlob(ctx, pb) if err != nil { log.Error("Error inserting package blob: %v", err) diff --git a/routers/api/packages/container/container.go b/routers/api/packages/container/container.go index 63c49809a7be7..883fe73cbdf2d 100644 --- a/routers/api/packages/container/container.go +++ b/routers/api/packages/container/container.go @@ -219,7 +219,7 @@ func InitiateUploadBlob(ctx *context.Context) { digest := ctx.FormTrim("digest") if digest != "" { - buf, err := packages_module.CreateHashedBufferFromReader(ctx.Req.Body) + buf, err := packages_module.CreateHashedBufferFromReader(ctx.Req.Body, 32*1024*1024) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return @@ -538,7 +538,7 @@ func UploadManifest(ctx *context.Context) { } maxSize := maxManifestSize + 1 - buf, err := packages_module.CreateHashedBufferFromReaderWithSize(&io.LimitedReader{R: ctx.Req.Body, N: int64(maxSize)}, maxSize) + buf, err := packages_module.CreateHashedBufferFromReader(&io.LimitedReader{R: ctx.Req.Body, N: int64(maxSize)}, maxSize) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return diff --git a/routers/api/packages/debian/debian.go b/routers/api/packages/debian/debian.go deleted file mode 100644 index 3cda04f96fe4f..0000000000000 --- a/routers/api/packages/debian/debian.go +++ /dev/null @@ -1,317 +0,0 @@ -// Copyright 2023 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package debian - -import ( - stdctx "context" - "errors" - "fmt" - "io" - "net/http" - "strings" - - "code.gitea.io/gitea/models/db" - packages_model "code.gitea.io/gitea/models/packages" - "code.gitea.io/gitea/modules/context" - "code.gitea.io/gitea/modules/notification" - packages_module "code.gitea.io/gitea/modules/packages" - debian_module "code.gitea.io/gitea/modules/packages/debian" - "code.gitea.io/gitea/modules/util" - "code.gitea.io/gitea/routers/api/packages/helper" - packages_service "code.gitea.io/gitea/services/packages" - debian_service "code.gitea.io/gitea/services/packages/debian" -) - -func apiError(ctx *context.Context, status int, obj interface{}) { - helper.LogAndProcessError(ctx, status, obj, func(message string) { - ctx.PlainText(status, message) - }) -} - -func GetRepositoryKey(ctx *context.Context) { - _, pub, err := debian_service.GetOrCreateKeyPair(ctx.Package.Owner.ID) - if err != nil { - apiError(ctx, http.StatusInternalServerError, err) - return - } - - ctx.ServeContent(strings.NewReader(pub), &context.ServeHeaderOptions{ - ContentType: "application/pgp-keys", - Filename: "repository.key", - }) -} - -// https://wiki.debian.org/DebianRepository/Format#A.22Release.22_files -// https://wiki.debian.org/DebianRepository/Format#A.22Packages.22_Indices -func GetRepositoryFile(ctx *context.Context) { - pv, err := debian_service.GetOrCreateRepositoryVersion(ctx.Package.Owner.ID) - if err != nil { - apiError(ctx, http.StatusInternalServerError, err) - return - } - - key := ctx.Params("distribution") - - component := ctx.Params("component") - architecture := strings.TrimPrefix(ctx.Params("architecture"), "binary-") - if component != "" && architecture != "" { - key += "|" + component + "|" + architecture - } - - s, pf, err := packages_service.GetFileStreamByPackageVersion( - ctx, - pv, - &packages_service.PackageFileInfo{ - Filename: ctx.Params("filename"), - CompositeKey: key, - }, - ) - if err != nil { - if err == packages_model.ErrPackageNotExist || err == packages_model.ErrPackageFileNotExist { - apiError(ctx, http.StatusNotFound, err) - } else { - apiError(ctx, http.StatusInternalServerError, err) - } - return - } - defer s.Close() - - ctx.ServeContent(s, &context.ServeHeaderOptions{ - Filename: pf.Name, - LastModified: pf.CreatedUnix.AsLocalTime(), - }) -} - -// https://wiki.debian.org/DebianRepository/Format#indices_acquisition_via_hashsums_.28by-hash.29 -func GetRepositoryFileByHash(ctx *context.Context) { - pv, err := debian_service.GetOrCreateRepositoryVersion(ctx.Package.Owner.ID) - if err != nil { - apiError(ctx, http.StatusInternalServerError, err) - return - } - - algorithmn := strings.ToLower(ctx.Params("algorithmn")) - if algorithmn == "md5sum" { - algorithmn = "md5" - } - - pfs, _, err := packages_model.SearchFiles(ctx, &packages_model.PackageFileSearchOptions{ - VersionID: pv.ID, - Hash: strings.ToLower(ctx.Params("hash")), - HashAlgorithmn: algorithmn, - }) - if err != nil { - apiError(ctx, http.StatusInternalServerError, err) - return - } - if len(pfs) != 1 { - apiError(ctx, http.StatusNotFound, nil) - return - } - - s, pf, err := packages_service.GetPackageFileStream(ctx, pfs[0]) - if err != nil { - if errors.Is(err, util.ErrNotExist) { - apiError(ctx, http.StatusNotFound, err) - } else { - apiError(ctx, http.StatusInternalServerError, err) - } - return - } - defer s.Close() - - ctx.ServeContent(s, &context.ServeHeaderOptions{ - Filename: pf.Name, - LastModified: pf.CreatedUnix.AsLocalTime(), - }) -} - -func UploadPackageFile(ctx *context.Context) { - distribution := strings.TrimSpace(ctx.Params("distribution")) - component := strings.TrimSpace(ctx.Params("component")) - if distribution == "" || component == "" { - apiError(ctx, http.StatusBadRequest, "invalid distribution or component") - return - } - - upload, close, err := ctx.UploadStream() - if err != nil { - apiError(ctx, http.StatusInternalServerError, err) - return - } - if close { - defer upload.Close() - } - - buf, err := packages_module.CreateHashedBufferFromReader(upload) - if err != nil { - apiError(ctx, http.StatusInternalServerError, err) - return - } - defer buf.Close() - - pck, err := debian_module.ParsePackage(buf) - if err != nil { - if errors.Is(err, util.ErrInvalidArgument) { - apiError(ctx, http.StatusBadRequest, err) - } else { - apiError(ctx, http.StatusInternalServerError, err) - } - return - } - - if _, err := buf.Seek(0, io.SeekStart); err != nil { - apiError(ctx, http.StatusInternalServerError, err) - return - } - - _, _, err = packages_service.CreatePackageOrAddFileToExisting( - &packages_service.PackageCreationInfo{ - PackageInfo: packages_service.PackageInfo{ - Owner: ctx.Package.Owner, - PackageType: packages_model.TypeDebian, - Name: pck.Name, - Version: pck.Version, - }, - Creator: ctx.Doer, - Metadata: pck.Metadata, - }, - &packages_service.PackageFileCreationInfo{ - PackageFileInfo: packages_service.PackageFileInfo{ - Filename: fmt.Sprintf("%s_%s_%s.deb", pck.Name, pck.Version, pck.Architecture), - CompositeKey: fmt.Sprintf("%s|%s", distribution, component), - }, - Creator: ctx.Doer, - Data: buf, - IsLead: true, - Properties: map[string]string{ - debian_module.PropertyDistribution: distribution, - debian_module.PropertyComponent: component, - debian_module.PropertyArchitecture: pck.Architecture, - debian_module.PropertyControl: pck.Control, - }, - }, - ) - if err != nil { - switch err { - case packages_model.ErrDuplicatePackageVersion: - apiError(ctx, http.StatusBadRequest, err) - case packages_service.ErrQuotaTotalCount, packages_service.ErrQuotaTypeSize, packages_service.ErrQuotaTotalSize: - apiError(ctx, http.StatusForbidden, err) - default: - apiError(ctx, http.StatusInternalServerError, err) - } - return - } - - if err := debian_service.BuildSpecificRepositoryFiles(ctx, ctx.Package.Owner.ID, distribution, component, pck.Architecture); err != nil { - apiError(ctx, http.StatusInternalServerError, err) - return - } - - ctx.Status(http.StatusCreated) -} - -func DownloadPackageFile(ctx *context.Context) { - name := ctx.Params("name") - version := ctx.Params("version") - - s, pf, err := packages_service.GetFileStreamByPackageNameAndVersion( - ctx, - &packages_service.PackageInfo{ - Owner: ctx.Package.Owner, - PackageType: packages_model.TypeDebian, - Name: name, - Version: version, - }, - &packages_service.PackageFileInfo{ - Filename: fmt.Sprintf("%s_%s_%s.deb", name, version, ctx.Params("architecture")), - CompositeKey: fmt.Sprintf("%s|%s", ctx.Params("distribution"), ctx.Params("component")), - }, - ) - if err != nil { - if err == packages_model.ErrPackageNotExist || err == packages_model.ErrPackageFileNotExist { - apiError(ctx, http.StatusNotFound, err) - } else { - apiError(ctx, http.StatusInternalServerError, err) - } - return - } - defer s.Close() - - ctx.ServeContent(s, &context.ServeHeaderOptions{ - ContentType: "application/vnd.debian.binary-package", - Filename: pf.Name, - LastModified: pf.CreatedUnix.AsLocalTime(), - }) -} - -func DeletePackageFile(ctx *context.Context) { - distribution := ctx.Params("distribution") - component := ctx.Params("component") - name := ctx.Params("name") - version := ctx.Params("version") - architecture := ctx.Params("architecture") - - owner := ctx.Package.Owner - - var pd *packages_model.PackageDescriptor - - err := db.WithTx(ctx, func(ctx stdctx.Context) error { - pv, err := packages_model.GetVersionByNameAndVersion(ctx, owner.ID, packages_model.TypeDebian, name, version) - if err != nil { - return err - } - - pf, err := packages_model.GetFileForVersionByName( - ctx, - pv.ID, - fmt.Sprintf("%s_%s_%s.deb", name, version, architecture), - fmt.Sprintf("%s|%s", distribution, component), - ) - if err != nil { - return err - } - - if err := packages_service.DeletePackageFile(ctx, pf); err != nil { - return err - } - - has, err := packages_model.HasVersionFileReferences(ctx, pv.ID) - if err != nil { - return err - } - if !has { - pd, err = packages_model.GetPackageDescriptor(ctx, pv) - if err != nil { - return err - } - - if err := packages_service.DeletePackageVersionAndReferences(ctx, pv); err != nil { - return err - } - } - - return nil - }) - if err != nil { - if errors.Is(err, util.ErrNotExist) { - apiError(ctx, http.StatusNotFound, err) - } else { - apiError(ctx, http.StatusInternalServerError, err) - } - return - } - - if pd != nil { - notification.NotifyPackageDelete(ctx, ctx.Doer, pd) - } - - if err := debian_service.BuildSpecificRepositoryFiles(ctx, ctx.Package.Owner.ID, distribution, component, architecture); err != nil { - apiError(ctx, http.StatusInternalServerError, err) - return - } - - ctx.Status(http.StatusNoContent) -} diff --git a/routers/api/packages/generic/generic.go b/routers/api/packages/generic/generic.go index 0c873119ef54d..5fba02cacd468 100644 --- a/routers/api/packages/generic/generic.go +++ b/routers/api/packages/generic/generic.go @@ -84,7 +84,7 @@ func UploadPackage(ctx *context.Context) { defer upload.Close() } - buf, err := packages_module.CreateHashedBufferFromReader(upload) + buf, err := packages_module.CreateHashedBufferFromReader(upload, 32*1024*1024) if err != nil { log.Error("Error creating hashed buffer: %v", err) apiError(ctx, http.StatusInternalServerError, err) diff --git a/routers/api/packages/helm/helm.go b/routers/api/packages/helm/helm.go index b7edc8b7fef8a..3bcce6bdf5855 100644 --- a/routers/api/packages/helm/helm.go +++ b/routers/api/packages/helm/helm.go @@ -155,7 +155,7 @@ func UploadPackage(ctx *context.Context) { defer upload.Close() } - buf, err := packages_module.CreateHashedBufferFromReader(upload) + buf, err := packages_module.CreateHashedBufferFromReader(upload, 32*1024*1024) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return diff --git a/routers/api/packages/maven/maven.go b/routers/api/packages/maven/maven.go index dd270ff0edd2f..a3a23ecfa8dc9 100644 --- a/routers/api/packages/maven/maven.go +++ b/routers/api/packages/maven/maven.go @@ -245,7 +245,7 @@ func UploadPackageFile(ctx *context.Context) { packageName := params.GroupID + "-" + params.ArtifactID - buf, err := packages_module.CreateHashedBufferFromReader(ctx.Req.Body) + buf, err := packages_module.CreateHashedBufferFromReader(ctx.Req.Body, 32*1024*1024) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return diff --git a/routers/api/packages/npm/npm.go b/routers/api/packages/npm/npm.go index 89476a776a0dd..51b34d3e2721e 100644 --- a/routers/api/packages/npm/npm.go +++ b/routers/api/packages/npm/npm.go @@ -189,7 +189,7 @@ func UploadPackage(ctx *context.Context) { } } - buf, err := packages_module.CreateHashedBufferFromReader(bytes.NewReader(npmPackage.Data)) + buf, err := packages_module.CreateHashedBufferFromReader(bytes.NewReader(npmPackage.Data), 32*1024*1024) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return diff --git a/routers/api/packages/nuget/nuget.go b/routers/api/packages/nuget/nuget.go index f6143ce291d23..3418bf9959a96 100644 --- a/routers/api/packages/nuget/nuget.go +++ b/routers/api/packages/nuget/nuget.go @@ -475,7 +475,7 @@ func UploadSymbolPackage(ctx *context.Context) { Version: np.Version, } - _, err = packages_service.AddFileToExistingPackage( + _, _, err = packages_service.AddFileToExistingPackage( pi, &packages_service.PackageFileCreationInfo{ PackageFileInfo: packages_service.PackageFileInfo{ @@ -501,7 +501,7 @@ func UploadSymbolPackage(ctx *context.Context) { } for _, pdb := range pdbs { - _, err := packages_service.AddFileToExistingPackage( + _, _, err := packages_service.AddFileToExistingPackage( pi, &packages_service.PackageFileCreationInfo{ PackageFileInfo: packages_service.PackageFileInfo{ @@ -545,7 +545,7 @@ func processUploadedFile(ctx *context.Context, expectedType nuget_module.Package closables = append(closables, upload) } - buf, err := packages_module.CreateHashedBufferFromReader(upload) + buf, err := packages_module.CreateHashedBufferFromReader(upload, 32*1024*1024) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return nil, nil, closables diff --git a/routers/api/packages/pub/pub.go b/routers/api/packages/pub/pub.go index ae0c6e7859898..1ece4e18ed6d4 100644 --- a/routers/api/packages/pub/pub.go +++ b/routers/api/packages/pub/pub.go @@ -166,7 +166,7 @@ func UploadPackageFile(ctx *context.Context) { } defer file.Close() - buf, err := packages_module.CreateHashedBufferFromReader(file) + buf, err := packages_module.CreateHashedBufferFromReader(file, 32*1024*1024) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return diff --git a/routers/api/packages/pypi/pypi.go b/routers/api/packages/pypi/pypi.go index 90a37ec2a8f1a..2f71801a8f63c 100644 --- a/routers/api/packages/pypi/pypi.go +++ b/routers/api/packages/pypi/pypi.go @@ -117,7 +117,7 @@ func UploadPackageFile(ctx *context.Context) { } defer file.Close() - buf, err := packages_module.CreateHashedBufferFromReader(file) + buf, err := packages_module.CreateHashedBufferFromReader(file, 32*1024*1024) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return diff --git a/routers/api/packages/rubygems/rubygems.go b/routers/api/packages/rubygems/rubygems.go index 740efa9baba8b..af358fb82fbad 100644 --- a/routers/api/packages/rubygems/rubygems.go +++ b/routers/api/packages/rubygems/rubygems.go @@ -209,7 +209,7 @@ func UploadPackageFile(ctx *context.Context) { defer upload.Close() } - buf, err := packages_module.CreateHashedBufferFromReader(upload) + buf, err := packages_module.CreateHashedBufferFromReader(upload, 32*1024*1024) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return diff --git a/routers/api/packages/swift/swift.go b/routers/api/packages/swift/swift.go index 06f592dd648f9..f78f703778ba0 100644 --- a/routers/api/packages/swift/swift.go +++ b/routers/api/packages/swift/swift.go @@ -300,7 +300,7 @@ func UploadPackageFile(ctx *context.Context) { } defer file.Close() - buf, err := packages_module.CreateHashedBufferFromReader(file) + buf, err := packages_module.CreateHashedBufferFromReader(file, 32*1024*1024) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return diff --git a/routers/api/packages/vagrant/vagrant.go b/routers/api/packages/vagrant/vagrant.go index cefdc45b10ff4..7b76ab79b0580 100644 --- a/routers/api/packages/vagrant/vagrant.go +++ b/routers/api/packages/vagrant/vagrant.go @@ -158,7 +158,7 @@ func UploadPackageFile(ctx *context.Context) { defer upload.Close() } - buf, err := packages_module.CreateHashedBufferFromReader(upload) + buf, err := packages_module.CreateHashedBufferFromReader(upload, 32*1024*1024) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return diff --git a/routers/api/v1/packages/package.go b/routers/api/v1/packages/package.go index ac48eb8a53fec..200dc5aaf1401 100644 --- a/routers/api/v1/packages/package.go +++ b/routers/api/v1/packages/package.go @@ -40,7 +40,7 @@ func ListPackages(ctx *context.APIContext) { // in: query // description: package type filter // type: string - // enum: [cargo, chef, composer, conan, conda, container, debian, generic, helm, maven, npm, nuget, pub, pypi, rubygems, swift, vagrant] + // enum: [cargo, chef, composer, conan, conda, container, generic, helm, maven, npm, nuget, pub, pypi, rubygems, swift, vagrant] // - name: q // in: query // description: name filter diff --git a/routers/web/user/package.go b/routers/web/user/package.go index 37ee0b86319b0..a9acc5281feea 100644 --- a/routers/web/user/package.go +++ b/routers/web/user/package.go @@ -14,10 +14,8 @@ import ( access_model "code.gitea.io/gitea/models/perm/access" repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/modules/base" - "code.gitea.io/gitea/modules/container" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/log" - debian_module "code.gitea.io/gitea/modules/packages/debian" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/modules/web" @@ -165,32 +163,6 @@ func ViewPackageVersion(ctx *context.Context) { ctx.Data["IsPackagesPage"] = true ctx.Data["PackageDescriptor"] = pd - switch pd.Package.Type { - case packages_model.TypeContainer: - ctx.Data["RegistryHost"] = setting.Packages.RegistryHost - case packages_model.TypeDebian: - distributions := make(container.Set[string]) - components := make(container.Set[string]) - architectures := make(container.Set[string]) - - for _, f := range pd.Files { - for _, pp := range f.Properties { - switch pp.Name { - case debian_module.PropertyDistribution: - distributions.Add(pp.Value) - case debian_module.PropertyComponent: - components.Add(pp.Value) - case debian_module.PropertyArchitecture: - architectures.Add(pp.Value) - } - } - } - - ctx.Data["Distributions"] = distributions.Values() - ctx.Data["Components"] = components.Values() - ctx.Data["Architectures"] = architectures.Values() - } - var ( total int64 pvs []*packages_model.PackageVersion @@ -198,6 +170,8 @@ func ViewPackageVersion(ctx *context.Context) { ) switch pd.Package.Type { case packages_model.TypeContainer: + ctx.Data["RegistryHost"] = setting.Packages.RegistryHost + pvs, total, err = container_model.SearchImageTags(ctx, &container_model.ImageTagsSearchOptions{ Paginator: db.NewAbsoluteListOptions(0, 5), PackageID: pd.Package.ID, @@ -209,6 +183,10 @@ func ViewPackageVersion(ctx *context.Context) { PackageID: pd.Package.ID, IsInternal: util.OptionalBoolFalse, }) + if err != nil { + ctx.ServerError("SearchVersions", err) + return + } } if err != nil { ctx.ServerError("", err) diff --git a/services/forms/package_form.go b/services/forms/package_form.go index efe4a0d84a246..699d0fe44f967 100644 --- a/services/forms/package_form.go +++ b/services/forms/package_form.go @@ -15,7 +15,7 @@ import ( type PackageCleanupRuleForm struct { ID int64 Enabled bool - Type string `binding:"Required;In(cargo,chef,composer,conan,conda,container,debian,generic,helm,maven,npm,nuget,pub,pypi,rubygems,swift,vagrant)"` + Type string `binding:"Required;In(cargo,chef,composer,conan,conda,container,generic,helm,maven,npm,nuget,pub,pypi,rubygems,swift,vagrant)"` KeepCount int `binding:"In(0,1,5,10,25,50,100)"` KeepPattern string `binding:"RegexPattern"` RemoveDays int `binding:"In(0,7,14,30,60,90,180)"` diff --git a/services/packages/cleanup/cleanup.go b/services/packages/cleanup/cleanup.go index 43fbc1ad9b113..2d62a028a4c6d 100644 --- a/services/packages/cleanup/cleanup.go +++ b/services/packages/cleanup/cleanup.go @@ -17,7 +17,6 @@ import ( packages_service "code.gitea.io/gitea/services/packages" cargo_service "code.gitea.io/gitea/services/packages/cargo" container_service "code.gitea.io/gitea/services/packages/container" - debian_service "code.gitea.io/gitea/services/packages/debian" ) // Cleanup removes expired package data @@ -46,7 +45,6 @@ func Cleanup(taskCtx context.Context, olderThan time.Duration) error { return fmt.Errorf("CleanupRule [%d]: GetPackagesByType failed: %w", pcr.ID, err) } - anyVersionDeleted := false for _, p := range packages { pvs, _, err := packages_model.SearchVersions(ctx, &packages_model.PackageSearchOptions{ PackageID: p.ID, @@ -93,7 +91,6 @@ func Cleanup(taskCtx context.Context, olderThan time.Duration) error { } versionDeleted = true - anyVersionDeleted = true } if versionDeleted { @@ -108,14 +105,6 @@ func Cleanup(taskCtx context.Context, olderThan time.Duration) error { } } } - - if anyVersionDeleted { - if pcr.Type == packages_model.TypeDebian { - if err := debian_service.BuildAllRepositoryFiles(ctx, pcr.OwnerID); err != nil { - return fmt.Errorf("CleanupRule [%d]: debian.BuildAllRepositoryFiles failed: %w", pcr.ID, err) - } - } - } return nil }) if err != nil { diff --git a/services/packages/debian/repository.go b/services/packages/debian/repository.go deleted file mode 100644 index eac878256ab7b..0000000000000 --- a/services/packages/debian/repository.go +++ /dev/null @@ -1,443 +0,0 @@ -// Copyright 2023 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package debian - -import ( - "bytes" - "compress/gzip" - "context" - "errors" - "fmt" - "io" - "sort" - "strings" - "time" - - "code.gitea.io/gitea/models/db" - packages_model "code.gitea.io/gitea/models/packages" - debian_model "code.gitea.io/gitea/models/packages/debian" - user_model "code.gitea.io/gitea/models/user" - "code.gitea.io/gitea/modules/log" - packages_module "code.gitea.io/gitea/modules/packages" - debian_module "code.gitea.io/gitea/modules/packages/debian" - "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/modules/util" - packages_service "code.gitea.io/gitea/services/packages" - - "github.com/keybase/go-crypto/openpgp" - "github.com/keybase/go-crypto/openpgp/armor" - "github.com/keybase/go-crypto/openpgp/clearsign" - "github.com/keybase/go-crypto/openpgp/packet" - "github.com/ulikunitz/xz" -) - -// GetOrCreateRepositoryVersion gets or creates the internal repository package -// The Debian registry needs multiple index files which are stored in this package. -func GetOrCreateRepositoryVersion(ownerID int64) (*packages_model.PackageVersion, error) { - var repositoryVersion *packages_model.PackageVersion - - return repositoryVersion, db.WithTx(db.DefaultContext, func(ctx context.Context) error { - p := &packages_model.Package{ - OwnerID: ownerID, - Type: packages_model.TypeDebian, - Name: debian_module.RepositoryPackage, - LowerName: debian_module.RepositoryPackage, - IsInternal: true, - } - var err error - if p, err = packages_model.TryInsertPackage(ctx, p); err != nil { - if err != packages_model.ErrDuplicatePackage { - log.Error("Error inserting package: %v", err) - return err - } - } - - pv := &packages_model.PackageVersion{ - PackageID: p.ID, - CreatorID: ownerID, - Version: debian_module.RepositoryVersion, - LowerVersion: debian_module.RepositoryVersion, - IsInternal: true, - MetadataJSON: "null", - } - if pv, err = packages_model.GetOrInsertVersion(ctx, pv); err != nil { - if err != packages_model.ErrDuplicatePackageVersion { - log.Error("Error inserting package version: %v", err) - return err - } - } - - repositoryVersion = pv - - return nil - }) -} - -// GetOrCreateKeyPair gets or creates the PGP keys used to sign repository files -func GetOrCreateKeyPair(ownerID int64) (string, string, error) { - priv, err := user_model.GetSetting(ownerID, debian_module.SettingKeyPrivate) - if err != nil && !errors.Is(err, util.ErrNotExist) { - return "", "", err - } - - pub, err := user_model.GetSetting(ownerID, debian_module.SettingKeyPublic) - if err != nil && !errors.Is(err, util.ErrNotExist) { - return "", "", err - } - - if priv == "" || pub == "" { - priv, pub, err = generateKeypair() - if err != nil { - return "", "", err - } - - if err := user_model.SetUserSetting(ownerID, debian_module.SettingKeyPrivate, priv); err != nil { - return "", "", err - } - - if err := user_model.SetUserSetting(ownerID, debian_module.SettingKeyPublic, pub); err != nil { - return "", "", err - } - } - - return priv, pub, nil -} - -func generateKeypair() (string, string, error) { - e, err := openpgp.NewEntity(setting.AppName, "Debian Registry", "", nil) - if err != nil { - return "", "", err - } - - var priv strings.Builder - var pub strings.Builder - - w, err := armor.Encode(&priv, openpgp.PrivateKeyType, nil) - if err != nil { - return "", "", err - } - if err := e.SerializePrivate(w, nil); err != nil { - return "", "", err - } - w.Close() - - w, err = armor.Encode(&pub, openpgp.PublicKeyType, nil) - if err != nil { - return "", "", err - } - if err := e.Serialize(w); err != nil { - return "", "", err - } - w.Close() - - return priv.String(), pub.String(), nil -} - -// BuildAllRepositoryFiles (re)builds all repository files for every available distributions, components and architectures -func BuildAllRepositoryFiles(ctx context.Context, ownerID int64) error { - pv, err := GetOrCreateRepositoryVersion(ownerID) - if err != nil { - return err - } - - // 1. Delete all existing repository files - pfs, err := packages_model.GetFilesByVersionID(ctx, pv.ID) - if err != nil { - return err - } - - for _, pf := range pfs { - if err := packages_model.DeleteAllProperties(ctx, packages_model.PropertyTypeFile, pf.ID); err != nil { - return err - } - if err := packages_model.DeleteFileByID(ctx, pf.ID); err != nil { - return err - } - } - - // 2. (Re)Build repository files for existing packages - distributions, err := debian_model.GetDistributions(ctx, ownerID) - if err != nil { - return err - } - for _, distribution := range distributions { - components, err := debian_model.GetComponents(ctx, ownerID, distribution) - if err != nil { - return err - } - architectures, err := debian_model.GetArchitectures(ctx, ownerID, distribution) - if err != nil { - return err - } - - for _, component := range components { - for _, architecture := range architectures { - if err := buildRepositoryFiles(ctx, ownerID, pv, distribution, component, architecture); err != nil { - return fmt.Errorf("failed to build repository files [%s/%s/%s]: %w", distribution, component, architecture, err) - } - } - } - } - - return nil -} - -// BuildSpecificRepositoryFiles builds index files for the repository -func BuildSpecificRepositoryFiles(ctx context.Context, ownerID int64, distribution, component, architecture string) error { - pv, err := GetOrCreateRepositoryVersion(ownerID) - if err != nil { - return err - } - - return buildRepositoryFiles(ctx, ownerID, pv, distribution, component, architecture) -} - -func buildRepositoryFiles(ctx context.Context, ownerID int64, repoVersion *packages_model.PackageVersion, distribution, component, architecture string) error { - if err := buildPackagesIndices(ctx, ownerID, repoVersion, distribution, component, architecture); err != nil { - return err - } - - return buildReleaseFiles(ctx, ownerID, repoVersion, distribution) -} - -// https://wiki.debian.org/DebianRepository/Format#A.22Packages.22_Indices -func buildPackagesIndices(ctx context.Context, ownerID int64, repoVersion *packages_model.PackageVersion, distribution, component, architecture string) error { - pfds, err := debian_model.SearchLatestPackages(ctx, &debian_model.PackageSearchOptions{ - OwnerID: ownerID, - Distribution: distribution, - Component: component, - Architecture: architecture, - }) - if err != nil { - return err - } - - // Delete the package indices if there are no packages - if len(pfds) == 0 { - key := fmt.Sprintf("%s|%s|%s", distribution, component, architecture) - for _, filename := range []string{"Packages", "Packages.gz", "Packages.xz"} { - pf, err := packages_model.GetFileForVersionByName(ctx, repoVersion.ID, filename, key) - if err != nil && !errors.Is(err, util.ErrNotExist) { - return err - } - - if err := packages_model.DeleteAllProperties(ctx, packages_model.PropertyTypeFile, pf.ID); err != nil { - return err - } - if err := packages_model.DeleteFileByID(ctx, pf.ID); err != nil { - return err - } - } - - return nil - } - - packagesContent, _ := packages_module.NewHashedBuffer() - - packagesGzipContent, _ := packages_module.NewHashedBuffer() - gzw := gzip.NewWriter(packagesGzipContent) - - packagesXzContent, _ := packages_module.NewHashedBuffer() - xzw, _ := xz.NewWriter(packagesXzContent) - - w := io.MultiWriter(packagesContent, gzw, xzw) - - addSeperator := false - for _, pfd := range pfds { - if addSeperator { - fmt.Fprintln(w) - } - addSeperator = true - - fmt.Fprint(w, pfd.Properties.GetByName(debian_module.PropertyControl)) - - fmt.Fprintf(w, "Filename: pool/%s/%s/%s\n", distribution, component, pfd.File.Name) - fmt.Fprintf(w, "Size: %d\n", pfd.Blob.Size) - fmt.Fprintf(w, "MD5sum: %s\n", pfd.Blob.HashMD5) - fmt.Fprintf(w, "SHA1: %s\n", pfd.Blob.HashSHA1) - fmt.Fprintf(w, "SHA256: %s\n", pfd.Blob.HashSHA256) - fmt.Fprintf(w, "SHA512: %s\n", pfd.Blob.HashSHA512) - } - - gzw.Close() - xzw.Close() - - for _, file := range []struct { - Name string - Data packages_module.HashedSizeReader - }{ - {"Packages", packagesContent}, - {"Packages.gz", packagesGzipContent}, - {"Packages.xz", packagesXzContent}, - } { - _, err = packages_service.AddFileToPackageVersionInternal( - repoVersion, - &packages_service.PackageFileCreationInfo{ - PackageFileInfo: packages_service.PackageFileInfo{ - Filename: file.Name, - CompositeKey: fmt.Sprintf("%s|%s|%s", distribution, component, architecture), - }, - Creator: user_model.NewGhostUser(), - Data: file.Data, - IsLead: false, - OverwriteExisting: true, - Properties: map[string]string{ - debian_module.PropertyRepositoryIncludeInRelease: "", - debian_module.PropertyDistribution: distribution, - debian_module.PropertyComponent: component, - debian_module.PropertyArchitecture: architecture, - }, - }, - ) - if err != nil { - return err - } - } - - return nil -} - -// https://wiki.debian.org/DebianRepository/Format#A.22Release.22_files -func buildReleaseFiles(ctx context.Context, ownerID int64, repoVersion *packages_model.PackageVersion, distribution string) error { - pfs, _, err := packages_model.SearchFiles(ctx, &packages_model.PackageFileSearchOptions{ - VersionID: repoVersion.ID, - Properties: map[string]string{ - debian_module.PropertyRepositoryIncludeInRelease: "", - debian_module.PropertyDistribution: distribution, - }, - }) - if err != nil { - return err - } - - // Delete the release files if there are no packages - if len(pfs) == 0 { - for _, filename := range []string{"Release", "Release.gpg", "InRelease"} { - pf, err := packages_model.GetFileForVersionByName(ctx, repoVersion.ID, filename, distribution) - if err != nil && !errors.Is(err, util.ErrNotExist) { - return err - } - - if err := packages_model.DeleteAllProperties(ctx, packages_model.PropertyTypeFile, pf.ID); err != nil { - return err - } - if err := packages_model.DeleteFileByID(ctx, pf.ID); err != nil { - return err - } - } - - return nil - } - - components, err := debian_model.GetComponents(ctx, ownerID, distribution) - if err != nil { - return err - } - - sort.Strings(components) - - architectures, err := debian_model.GetArchitectures(ctx, ownerID, distribution) - if err != nil { - return err - } - - sort.Strings(architectures) - - priv, _, err := GetOrCreateKeyPair(ownerID) - if err != nil { - return err - } - - block, err := armor.Decode(strings.NewReader(priv)) - if err != nil { - return err - } - - e, err := openpgp.ReadEntity(packet.NewReader(block.Body)) - if err != nil { - return err - } - - inReleaseContent, _ := packages_module.NewHashedBuffer() - sw, err := clearsign.Encode(inReleaseContent, e.PrivateKey, nil) - if err != nil { - return err - } - - var buf bytes.Buffer - - w := io.MultiWriter(sw, &buf) - - fmt.Fprintf(w, "Origin: %s\n", setting.AppName) - fmt.Fprintf(w, "Label: %s\n", setting.AppName) - fmt.Fprintf(w, "Suite: %s\n", distribution) - fmt.Fprintf(w, "Codename: %s\n", distribution) - fmt.Fprintf(w, "Components: %s\n", strings.Join(components, " ")) - fmt.Fprintf(w, "Architectures: %s\n", strings.Join(architectures, " ")) - fmt.Fprintf(w, "Date: %s\n", time.Now().UTC().Format(time.RFC1123)) - fmt.Fprint(w, "Acquire-By-Hash: yes") - - pfds, err := packages_model.GetPackageFileDescriptors(ctx, pfs) - if err != nil { - return err - } - - var md5, sha1, sha256, sha512 strings.Builder - for _, pfd := range pfds { - path := fmt.Sprintf("%s/binary-%s/%s", pfd.Properties.GetByName(debian_module.PropertyComponent), pfd.Properties.GetByName(debian_module.PropertyArchitecture), pfd.File.Name) - fmt.Fprintf(&md5, " %s %d %s\n", pfd.Blob.HashMD5, pfd.Blob.Size, path) - fmt.Fprintf(&sha1, " %s %d %s\n", pfd.Blob.HashSHA1, pfd.Blob.Size, path) - fmt.Fprintf(&sha256, " %s %d %s\n", pfd.Blob.HashSHA256, pfd.Blob.Size, path) - fmt.Fprintf(&sha512, " %s %d %s\n", pfd.Blob.HashSHA512, pfd.Blob.Size, path) - } - - fmt.Fprintln(w, "MD5Sum:") - fmt.Fprint(w, md5.String()) - fmt.Fprintln(w, "SHA1:") - fmt.Fprint(w, sha1.String()) - fmt.Fprintln(w, "SHA256:") - fmt.Fprint(w, sha256.String()) - fmt.Fprintln(w, "SHA512:") - fmt.Fprint(w, sha512.String()) - - sw.Close() - - releaseGpgContent, _ := packages_module.NewHashedBuffer() - if err := openpgp.ArmoredDetachSign(releaseGpgContent, e, bytes.NewReader(buf.Bytes()), nil); err != nil { - return err - } - - releaseContent, _ := packages_module.CreateHashedBufferFromReader(&buf) - - for _, file := range []struct { - Name string - Data packages_module.HashedSizeReader - }{ - {"Release", releaseContent}, - {"Release.gpg", releaseGpgContent}, - {"InRelease", inReleaseContent}, - } { - _, err = packages_service.AddFileToPackageVersionInternal( - repoVersion, - &packages_service.PackageFileCreationInfo{ - PackageFileInfo: packages_service.PackageFileInfo{ - Filename: file.Name, - CompositeKey: distribution, - }, - Creator: user_model.NewGhostUser(), - Data: file.Data, - IsLead: false, - OverwriteExisting: true, - Properties: map[string]string{ - debian_module.PropertyDistribution: distribution, - }, - }, - ) - if err != nil { - return err - } - } - - return nil -} diff --git a/services/packages/packages.go b/services/packages/packages.go index 735e52c854ba0..dd5c63470b8b2 100644 --- a/services/packages/packages.go +++ b/services/packages/packages.go @@ -187,33 +187,19 @@ func createPackageAndVersion(ctx context.Context, pvci *PackageCreationInfo, all } // AddFileToExistingPackage adds a file to an existing package. If the package does not exist, ErrPackageNotExist is returned -func AddFileToExistingPackage(pvi *PackageInfo, pfci *PackageFileCreationInfo) (*packages_model.PackageFile, error) { - return addFileToPackageWrapper(func(ctx context.Context) (*packages_model.PackageFile, *packages_model.PackageBlob, bool, error) { - pv, err := packages_model.GetVersionByNameAndVersion(ctx, pvi.Owner.ID, pvi.PackageType, pvi.Name, pvi.Version) - if err != nil { - return nil, nil, false, err - } - - return addFileToPackageVersion(ctx, pv, pvi, pfci) - }) -} - -// AddFileToPackageVersionInternal adds a file to the package -// This method skips quota checks and should only be used for system-managed packages. -func AddFileToPackageVersionInternal(pv *packages_model.PackageVersion, pfci *PackageFileCreationInfo) (*packages_model.PackageFile, error) { - return addFileToPackageWrapper(func(ctx context.Context) (*packages_model.PackageFile, *packages_model.PackageBlob, bool, error) { - return addFileToPackageVersionUnchecked(ctx, pv, pfci) - }) -} - -func addFileToPackageWrapper(fn func(ctx context.Context) (*packages_model.PackageFile, *packages_model.PackageBlob, bool, error)) (*packages_model.PackageFile, error) { +func AddFileToExistingPackage(pvi *PackageInfo, pfci *PackageFileCreationInfo) (*packages_model.PackageVersion, *packages_model.PackageFile, error) { ctx, committer, err := db.TxContext(db.DefaultContext) if err != nil { - return nil, err + return nil, nil, err } defer committer.Close() - pf, pb, blobCreated, err := fn(ctx) + pv, err := packages_model.GetVersionByNameAndVersion(ctx, pvi.Owner.ID, pvi.PackageType, pvi.Name, pvi.Version) + if err != nil { + return nil, nil, err + } + + pf, pb, blobCreated, err := addFileToPackageVersion(ctx, pv, pvi, pfci) removeBlob := false defer func() { if removeBlob { @@ -225,15 +211,15 @@ func addFileToPackageWrapper(fn func(ctx context.Context) (*packages_model.Packa }() if err != nil { removeBlob = blobCreated - return nil, err + return nil, nil, err } if err := committer.Commit(); err != nil { removeBlob = blobCreated - return nil, err + return nil, nil, err } - return pf, nil + return pv, pf, nil } // NewPackageBlob creates a package blob instance @@ -250,16 +236,12 @@ func NewPackageBlob(hsr packages_module.HashedSizeReader) *packages_model.Packag } func addFileToPackageVersion(ctx context.Context, pv *packages_model.PackageVersion, pvi *PackageInfo, pfci *PackageFileCreationInfo) (*packages_model.PackageFile, *packages_model.PackageBlob, bool, error) { + log.Trace("Adding package file: %v, %s", pv.ID, pfci.Filename) + if err := CheckSizeQuotaExceeded(ctx, pfci.Creator, pvi.Owner, pvi.PackageType, pfci.Data.Size()); err != nil { return nil, nil, false, err } - return addFileToPackageVersionUnchecked(ctx, pv, pfci) -} - -func addFileToPackageVersionUnchecked(ctx context.Context, pv *packages_model.PackageVersion, pfci *PackageFileCreationInfo) (*packages_model.PackageFile, *packages_model.PackageBlob, bool, error) { - log.Trace("Adding package file: %v, %s", pv.ID, pfci.Filename) - pb, exists, err := packages_model.GetOrInsertBlob(ctx, NewPackageBlob(pfci.Data)) if err != nil { log.Error("Error inserting package blob: %v", err) @@ -363,8 +345,6 @@ func CheckSizeQuotaExceeded(ctx context.Context, doer, owner *user_model.User, p typeSpecificSize = setting.Packages.LimitSizeConda case packages_model.TypeContainer: typeSpecificSize = setting.Packages.LimitSizeContainer - case packages_model.TypeDebian: - typeSpecificSize = setting.Packages.LimitSizeDebian case packages_model.TypeGeneric: typeSpecificSize = setting.Packages.LimitSizeGeneric case packages_model.TypeHelm: diff --git a/templates/package/content/debian.tmpl b/templates/package/content/debian.tmpl deleted file mode 100644 index c10c3042b92c4..0000000000000 --- a/templates/package/content/debian.tmpl +++ /dev/null @@ -1,65 +0,0 @@ -{{if eq .PackageDescriptor.Package.Type "debian"}} -

{{.locale.Tr "packages.installation"}}

-
-
-
- - 
sudo curl  -o /etc/apt/trusted.gpg.d/gitea-{{$.PackageDescriptor.Owner.Name}}.asc
-echo "deb  <distribution> <component>" | sudo tee -a /etc/apt/sources.list.d/gitea.list
-sudo apt update
-

{{.locale.Tr "packages.debian.registry.info" | Safe}}

-
-
- -
- 
sudo apt install {{$.PackageDescriptor.Package.Name}}={{$.PackageDescriptor.Version.Version}}
-
-
-
- -
-
-
- -

{{.locale.Tr "packages.debian.repository"}}

-
- - - - - - - - - - - - - - - -
{{.locale.Tr "packages.debian.repository.distributions"}}
{{Join .Distributions ", "}}
{{.locale.Tr "packages.debian.repository.components"}}
{{Join .Components ", "}}
{{.locale.Tr "packages.debian.repository.architectures"}}
{{Join .Architectures ", "}}
-
- - {{if .PackageDescriptor.Metadata.Description}} -

{{.locale.Tr "packages.about"}}

-
- {{.PackageDescriptor.Metadata.Description}} -
- {{end}} - - {{if .PackageDescriptor.Metadata.Dependencies}} -

{{.locale.Tr "packages.dependencies"}}

-
- - - {{range .PackageDescriptor.Metadata.Dependencies}} - - - - {{end}} - -
{{.}}
-
- {{end}} -{{end}} diff --git a/templates/package/metadata/debian.tmpl b/templates/package/metadata/debian.tmpl deleted file mode 100644 index 93b6db3bd240e..0000000000000 --- a/templates/package/metadata/debian.tmpl +++ /dev/null @@ -1,4 +0,0 @@ -{{if eq .PackageDescriptor.Package.Type "debian"}} - {{if .PackageDescriptor.Metadata.Maintainer}}
{{svg "octicon-person" 16 "mr-3"}} {{.PackageDescriptor.Metadata.Maintainer}}
{{end}} - {{if .PackageDescriptor.Metadata.ProjectURL}}
{{svg "octicon-link-external" 16 "mr-3"}} {{.locale.Tr "packages.details.project_site"}}
{{end}} -{{end}} diff --git a/templates/package/view.tmpl b/templates/package/view.tmpl index 30e737d09ceae..3cb130851fee1 100644 --- a/templates/package/view.tmpl +++ b/templates/package/view.tmpl @@ -25,7 +25,6 @@ {{template "package/content/conan" .}} {{template "package/content/conda" .}} {{template "package/content/container" .}} - {{template "package/content/debian" .}} {{template "package/content/generic" .}} {{template "package/content/helm" .}} {{template "package/content/maven" .}} @@ -53,7 +52,6 @@ {{template "package/metadata/conan" .}} {{template "package/metadata/conda" .}} {{template "package/metadata/container" .}} - {{template "package/metadata/debian" .}} {{template "package/metadata/generic" .}} {{template "package/metadata/helm" .}} {{template "package/metadata/maven" .}} diff --git a/templates/swagger/v1_json.tmpl b/templates/swagger/v1_json.tmpl index d656664367cdd..2db950b57a706 100644 --- a/templates/swagger/v1_json.tmpl +++ b/templates/swagger/v1_json.tmpl @@ -2415,7 +2415,6 @@ "conan", "conda", "container", - "debian", "generic", "helm", "maven", diff --git a/tests/integration/api_packages_debian_test.go b/tests/integration/api_packages_debian_test.go deleted file mode 100644 index 3e25acd8cff2c..0000000000000 --- a/tests/integration/api_packages_debian_test.go +++ /dev/null @@ -1,252 +0,0 @@ -// Copyright 2023 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package integration - -import ( - "archive/tar" - "bytes" - "compress/gzip" - "fmt" - "io" - "net/http" - "strings" - "testing" - - "code.gitea.io/gitea/models/db" - "code.gitea.io/gitea/models/packages" - "code.gitea.io/gitea/models/unittest" - user_model "code.gitea.io/gitea/models/user" - "code.gitea.io/gitea/modules/base" - debian_module "code.gitea.io/gitea/modules/packages/debian" - "code.gitea.io/gitea/tests" - - "github.com/blakesmith/ar" - "github.com/stretchr/testify/assert" -) - -func TestPackageDebian(t *testing.T) { - defer tests.PrepareTestEnv(t)() - user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) - - packageName := "gitea" - packageVersion := "1.0.3" - packageDescription := "Package Description" - - createArchive := func(name, version, architecture string) io.Reader { - var cbuf bytes.Buffer - zw := gzip.NewWriter(&cbuf) - tw := tar.NewWriter(zw) - tw.WriteHeader(&tar.Header{ - Name: "control", - Mode: 0o600, - Size: 50, - }) - fmt.Fprintf(tw, "Package: %s\nVersion: %s\nArchitecture: %s\nDescription: %s\n", name, version, architecture, packageDescription) - tw.Close() - zw.Close() - - var buf bytes.Buffer - aw := ar.NewWriter(&buf) - aw.WriteGlobalHeader() - hdr := &ar.Header{ - Name: "control.tar.gz", - Mode: 0o600, - Size: int64(cbuf.Len()), - } - aw.WriteHeader(hdr) - aw.Write(cbuf.Bytes()) - return &buf - } - - distributions := []string{"test", "gitea"} - components := []string{"main", "stable"} - architectures := []string{"all", "amd64"} - - rootURL := fmt.Sprintf("/api/packages/%s/debian", user.Name) - - t.Run("RepositoryKey", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - - req := NewRequest(t, "GET", rootURL+"/repository.key") - resp := MakeRequest(t, req, http.StatusOK) - - assert.Equal(t, "application/pgp-keys", resp.Header().Get("Content-Type")) - assert.Contains(t, resp.Body.String(), "-----BEGIN PGP PUBLIC KEY BLOCK-----") - }) - - for _, distribution := range distributions { - t.Run(fmt.Sprintf("[Distribution:%s]", distribution), func(t *testing.T) { - for _, component := range components { - for _, architecture := range architectures { - t.Run(fmt.Sprintf("[Component:%s,Architecture:%s]", component, architecture), func(t *testing.T) { - t.Run("Upload", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - - uploadURL := fmt.Sprintf("%s/pool/%s/%s/upload", rootURL, distribution, component) - - req := NewRequestWithBody(t, "PUT", uploadURL, bytes.NewReader([]byte{})) - MakeRequest(t, req, http.StatusUnauthorized) - - req = NewRequestWithBody(t, "PUT", uploadURL, bytes.NewReader([]byte{})) - AddBasicAuthHeader(req, user.Name) - MakeRequest(t, req, http.StatusBadRequest) - - req = NewRequestWithBody(t, "PUT", uploadURL, createArchive("", "", "")) - AddBasicAuthHeader(req, user.Name) - MakeRequest(t, req, http.StatusBadRequest) - - req = NewRequestWithBody(t, "PUT", uploadURL, createArchive(packageName, packageVersion, architecture)) - AddBasicAuthHeader(req, user.Name) - MakeRequest(t, req, http.StatusCreated) - - pvs, err := packages.GetVersionsByPackageType(db.DefaultContext, user.ID, packages.TypeDebian) - assert.NoError(t, err) - assert.Len(t, pvs, 1) - - pd, err := packages.GetPackageDescriptor(db.DefaultContext, pvs[0]) - assert.NoError(t, err) - assert.Nil(t, pd.SemVer) - assert.IsType(t, &debian_module.Metadata{}, pd.Metadata) - assert.Equal(t, packageName, pd.Package.Name) - assert.Equal(t, packageVersion, pd.Version.Version) - - pfs, err := packages.GetFilesByVersionID(db.DefaultContext, pvs[0].ID) - assert.NoError(t, err) - assert.NotEmpty(t, pfs) - assert.Condition(t, func() bool { - seen := false - expectedFilename := fmt.Sprintf("%s_%s_%s.deb", packageName, packageVersion, architecture) - expectedCompositeKey := fmt.Sprintf("%s|%s", distribution, component) - for _, pf := range pfs { - if pf.Name == expectedFilename && pf.CompositeKey == expectedCompositeKey { - if seen { - return false - } - seen = true - - assert.True(t, pf.IsLead) - - pfps, err := packages.GetProperties(db.DefaultContext, packages.PropertyTypeFile, pf.ID) - assert.NoError(t, err) - - for _, pfp := range pfps { - switch pfp.Name { - case debian_module.PropertyDistribution: - assert.Equal(t, distribution, pfp.Value) - case debian_module.PropertyComponent: - assert.Equal(t, component, pfp.Value) - case debian_module.PropertyArchitecture: - assert.Equal(t, architecture, pfp.Value) - } - } - } - } - return seen - }) - }) - - t.Run("Download", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - - req := NewRequest(t, "GET", fmt.Sprintf("%s/pool/%s/%s/%s_%s_%s.deb", rootURL, distribution, component, packageName, packageVersion, architecture)) - resp := MakeRequest(t, req, http.StatusOK) - - assert.Equal(t, "application/vnd.debian.binary-package", resp.Header().Get("Content-Type")) - }) - - t.Run("Packages", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - - url := fmt.Sprintf("%s/dists/%s/%s/binary-%s/Packages", rootURL, distribution, component, architecture) - - req := NewRequest(t, "GET", url) - resp := MakeRequest(t, req, http.StatusOK) - - body := resp.Body.String() - - assert.Contains(t, body, "Package: "+packageName) - assert.Contains(t, body, "Version: "+packageVersion) - assert.Contains(t, body, "Architecture: "+architecture) - assert.Contains(t, body, fmt.Sprintf("Filename: pool/%s/%s/%s_%s_%s.deb", distribution, component, packageName, packageVersion, architecture)) - - req = NewRequest(t, "GET", url+".gz") - MakeRequest(t, req, http.StatusOK) - - req = NewRequest(t, "GET", url+".xz") - MakeRequest(t, req, http.StatusOK) - - url = fmt.Sprintf("%s/dists/%s/%s/%s/by-hash/SHA256/%s", rootURL, distribution, component, architecture, base.EncodeSha256(body)) - req = NewRequest(t, "GET", url) - resp = MakeRequest(t, req, http.StatusOK) - - assert.Equal(t, body, resp.Body.String()) - }) - }) - } - } - - t.Run("Release", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - - req := NewRequest(t, "GET", fmt.Sprintf("%s/dists/%s/Release", rootURL, distribution)) - resp := MakeRequest(t, req, http.StatusOK) - - body := resp.Body.String() - - assert.Contains(t, body, "Components: "+strings.Join(components, " ")) - assert.Contains(t, body, "Architectures: "+strings.Join(architectures, " ")) - - for _, component := range components { - for _, architecture := range architectures { - assert.Contains(t, body, fmt.Sprintf("%s/binary-%s/Packages", component, architecture)) - assert.Contains(t, body, fmt.Sprintf("%s/binary-%s/Packages.gz", component, architecture)) - assert.Contains(t, body, fmt.Sprintf("%s/binary-%s/Packages.xz", component, architecture)) - } - } - - req = NewRequest(t, "GET", fmt.Sprintf("%s/dists/%s/by-hash/SHA256/%s", rootURL, distribution, base.EncodeSha256(body))) - resp = MakeRequest(t, req, http.StatusOK) - - assert.Equal(t, body, resp.Body.String()) - - req = NewRequest(t, "GET", fmt.Sprintf("%s/dists/%s/Release.gpg", rootURL, distribution)) - resp = MakeRequest(t, req, http.StatusOK) - - assert.Contains(t, resp.Body.String(), "-----BEGIN PGP SIGNATURE-----") - - req = NewRequest(t, "GET", fmt.Sprintf("%s/dists/%s/InRelease", rootURL, distribution)) - resp = MakeRequest(t, req, http.StatusOK) - - assert.Contains(t, resp.Body.String(), "-----BEGIN PGP SIGNED MESSAGE-----") - }) - }) - } - - t.Run("Delete", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - - distribution := distributions[0] - architecture := architectures[0] - - for _, component := range components { - req := NewRequest(t, "DELETE", fmt.Sprintf("%s/pool/%s/%s/%s/%s/%s", rootURL, distribution, component, packageName, packageVersion, architecture)) - MakeRequest(t, req, http.StatusUnauthorized) - - req = NewRequest(t, "DELETE", fmt.Sprintf("%s/pool/%s/%s/%s/%s/%s", rootURL, distribution, component, packageName, packageVersion, architecture)) - AddBasicAuthHeader(req, user.Name) - MakeRequest(t, req, http.StatusNoContent) - - req = NewRequest(t, "GET", fmt.Sprintf("%s/dists/%s/%s/binary-%s/Packages", rootURL, distribution, component, architecture)) - MakeRequest(t, req, http.StatusNotFound) - } - - req := NewRequest(t, "GET", fmt.Sprintf("%s/dists/%s/Release", rootURL, distribution)) - resp := MakeRequest(t, req, http.StatusOK) - - body := resp.Body.String() - - assert.Contains(t, body, "Components: "+strings.Join(components, " ")) - assert.Contains(t, body, "Architectures: "+architectures[1]) - }) -} diff --git a/web_src/svg/gitea-debian.svg b/web_src/svg/gitea-debian.svg deleted file mode 100644 index 4046f7fc7cb17..0000000000000 --- a/web_src/svg/gitea-debian.svg +++ /dev/null @@ -1,9 +0,0 @@ - - - - - - - - - From ef9e0ce9c93ed2209adaf69094e909e053110990 Mon Sep 17 00:00:00 2001 From: GiteaBot Date: Sat, 29 Apr 2023 00:23:39 +0000 Subject: [PATCH 10/65] [skip ci] Updated translations via Crowdin --- options/locale/locale_zh-CN.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index aab7781dc2eb1..f1921828349c7 100644 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -61,7 +61,7 @@ new_project_column=创建列 manage_org=管理我的组织 admin_panel=管理后台 account_settings=帐户设置 -settings=帐户设置 +settings=设置 your_profile=个人信息 your_starred=已点赞 your_settings=设置 From 0bd05a9f1cbf7839d943e5812f84bdeb7d95dfee Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sat, 29 Apr 2023 04:38:22 +0200 Subject: [PATCH 11/65] Add integration test for API raw content reference formats (#24388) This pull request adds an integration test to validate the behavior of raw content API's reference handling for all supported formats . close #24242 Co-authored-by: Lunny Xiao --- .../integration/api_repo_get_contents_test.go | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests/integration/api_repo_get_contents_test.go b/tests/integration/api_repo_get_contents_test.go index f7d9c716aaf11..8b193b03c93e0 100644 --- a/tests/integration/api_repo_get_contents_test.go +++ b/tests/integration/api_repo_get_contents_test.go @@ -4,6 +4,7 @@ package integration import ( + "io" "net/http" "net/url" "testing" @@ -159,3 +160,30 @@ func testAPIGetContents(t *testing.T, u *url.URL) { req = NewRequestf(t, "GET", "/api/v1/repos/%s/%s/contents/%s?token=%s", user3.Name, repo3.Name, treePath, token2) MakeRequest(t, req, http.StatusOK) } + +func TestAPIGetContentsRefFormats(t *testing.T) { + onGiteaRun(t, func(t *testing.T, u *url.URL) { + file := "README.md" + sha := "65f1bf27bc3bf70f64657658635e66094edbcb4d" + content := "# repo1\n\nDescription for repo1" + + noRef := setting.AppURL + "api/v1/repos/user2/repo1/raw/" + file + refInPath := setting.AppURL + "api/v1/repos/user2/repo1/raw/" + sha + "/" + file + refInQuery := setting.AppURL + "api/v1/repos/user2/repo1/raw/" + file + "?ref=" + sha + + resp := MakeRequest(t, NewRequest(t, http.MethodGet, noRef), http.StatusOK) + raw, err := io.ReadAll(resp.Body) + assert.NoError(t, err) + assert.EqualValues(t, content, string(raw)) + + resp = MakeRequest(t, NewRequest(t, http.MethodGet, refInPath), http.StatusOK) + raw, err = io.ReadAll(resp.Body) + assert.NoError(t, err) + assert.EqualValues(t, content, string(raw)) + + resp = MakeRequest(t, NewRequest(t, http.MethodGet, refInQuery), http.StatusOK) + raw, err = io.ReadAll(resp.Body) + assert.NoError(t, err) + assert.EqualValues(t, content, string(raw)) + }) +} From 9cf721e44601382f8086bdcebfdbae9b9107da69 Mon Sep 17 00:00:00 2001 From: Earl Warren <109468362+earl-warren@users.noreply.github.com> Date: Sat, 29 Apr 2023 06:40:55 +0200 Subject: [PATCH 12/65] getting the tag list does not require being signed in (#24413) Fixes: https://codeberg.org/forgejo/forgejo/issues/681 --- routers/web/web.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routers/web/web.go b/routers/web/web.go index a4ef96ecbe0f1..bb2442fec40ec 100644 --- a/routers/web/web.go +++ b/routers/web/web.go @@ -1093,7 +1093,7 @@ func registerRoutes(m *web.Route) { repo.MustBeNotEmpty, reqRepoCodeReader, context.RepoRefByType(context.RepoRefTag, true)) m.Post("/tags/delete", repo.DeleteTag, reqSignIn, repo.MustBeNotEmpty, context.RepoMustNotBeArchived(), reqRepoCodeWriter, context.RepoRef()) - }, reqSignIn, context.RepoAssignment, context.UnitTypes()) + }, ignSignIn, context.RepoAssignment, context.UnitTypes()) // Releases m.Group("/{username}/{reponame}", func() { From fc62992518158ca6ceb03a57db5b3767c8dc6fc5 Mon Sep 17 00:00:00 2001 From: silverwind Date: Sat, 29 Apr 2023 11:53:11 +0200 Subject: [PATCH 13/65] Skip known flaky `queue` tests on CI environment (#24419) Random CI failures are annoying. It's better to just skip the affected tests so maintainers can use their valuable time for more productive topics. Related: https://github.com/go-gitea/gitea/issues/23608 Related: https://github.com/go-gitea/gitea/issues/23977 Related: https://github.com/go-gitea/gitea/issues/18703 --- modules/queue/queue_channel_test.go | 4 ++++ modules/queue/unique_queue_disk_channel_test.go | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/modules/queue/queue_channel_test.go b/modules/queue/queue_channel_test.go index 9b92398bacd96..f9dae742e29d3 100644 --- a/modules/queue/queue_channel_test.go +++ b/modules/queue/queue_channel_test.go @@ -4,6 +4,7 @@ package queue import ( + "os" "sync" "testing" "time" @@ -101,6 +102,9 @@ func TestChannelQueue_Batch(t *testing.T) { } func TestChannelQueue_Pause(t *testing.T) { + if os.Getenv("CI") != "" { + t.Skip("Skipping because test is flaky on CI") + } lock := sync.Mutex{} var queue Queue var err error diff --git a/modules/queue/unique_queue_disk_channel_test.go b/modules/queue/unique_queue_disk_channel_test.go index f75c69f785150..e2fe4aceee576 100644 --- a/modules/queue/unique_queue_disk_channel_test.go +++ b/modules/queue/unique_queue_disk_channel_test.go @@ -4,6 +4,7 @@ package queue import ( + "os" "strconv" "sync" "testing" @@ -15,6 +16,10 @@ import ( ) func TestPersistableChannelUniqueQueue(t *testing.T) { + if os.Getenv("CI") != "" { + t.Skip("Skipping because test is flaky on CI") + } + tmpDir := t.TempDir() _ = log.NewLogger(1000, "console", "console", `{"level":"warn","stacktracelevel":"NONE","stderr":true}`) From 72e956b79a3b2e055bb5d4d5e20e88eaa2eeec96 Mon Sep 17 00:00:00 2001 From: Hester Gong Date: Sat, 29 Apr 2023 18:44:52 +0800 Subject: [PATCH 14/65] Improve protected branch setting page (#24379) Main changes: 1. Change html structure of protected branch page, use [`grouped fields`](https://fomantic-ui.com/collections/form.html#grouped-fields) instead of `fields` for better margin, and wrap `grouped fields` around related `field`s, remove unnecessary `
` outer div 2. Changed some order of field to make them more categorized, used `ui dividing header` for categorization and fine tune css. Before: Screen Shot 2023-04-27 at 14 56 19 Screen Shot 2023-04-27 at 14 56 30 Screen Shot 2023-04-27 at 14 56 36 After: https://user-images.githubusercontent.com/17645053/235114568-da010aad-7654-4410-ab8c-5d0fce7edadb.mov 3. Changed "Enable Merge Whitelist" to radio checkbox, and added "Enable Merge" radio checkbox, which are exclusive Before: Screen Shot 2023-04-28 at 13 08 29 After: Screen Shot 2023-04-28 at 13 09 28 4. Add a link to set default branch on branch list page (with reference to github) https://user-images.githubusercontent.com/17645053/234787404-61c1c7b6-aabf-429f-a109-5b690e4e0b5a.mov 5. Removed dead codes. --------- Co-authored-by: wxiaoguang Co-authored-by: silverwind Co-authored-by: Giteabot --- options/locale/locale_en-US.ini | 8 +- routers/web/repo/setting_protected_branch.go | 9 - templates/repo/branch/list.tmpl | 7 +- templates/repo/settings/branches.tmpl | 14 +- templates/repo/settings/protected_branch.tmpl | 286 +++++++++--------- web_src/css/base.css | 8 +- web_src/css/helpers.css | 2 + web_src/css/repository.css | 33 +- web_src/js/features/repo-settings.js | 26 +- 9 files changed, 181 insertions(+), 212 deletions(-) diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 0072ac6fc36fa..32d9bebc8b1a3 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -1901,6 +1901,7 @@ settings.sync_mirror = Synchronize Now settings.mirror_sync_in_progress = Mirror synchronization is in progress. Check back in a minute. settings.site = Website settings.update_settings = Update Settings +settings.branches.switch_default_branch = Switch Default Branch settings.branches.update_default_branch = Update Default Branch settings.branches.add_new_rule = Add New Rule settings.advanced_settings = Advanced Settings @@ -2096,6 +2097,8 @@ settings.event_pull_request_review = Pull Request Reviewed settings.event_pull_request_review_desc = Pull request approved, rejected, or review comment. settings.event_pull_request_sync = Pull Request Synchronized settings.event_pull_request_sync_desc = Pull request synchronized. +settings.event_pull_request_approvals = Pull Request Approvals +settings.event_pull_request_merge = Pull Request Merge settings.event_package = Package settings.event_package_desc = Package created or deleted in a repository. settings.branch_filter = Branch filter @@ -2151,13 +2154,15 @@ settings.protected_branch.delete_rule = Delete Rule settings.protected_branch_can_push = Allow push? settings.protected_branch_can_push_yes = You can push settings.protected_branch_can_push_no = You cannot push -settings.branch_protection = Branch Protection for Branch '%s' +settings.branch_protection = Branch Protection Rules for Branch '%s' settings.protect_this_branch = Enable Branch Protection settings.protect_this_branch_desc = Prevents deletion and restricts Git pushing and merging to the branch. settings.protect_disable_push = Disable Push settings.protect_disable_push_desc = No pushing will be allowed to this branch. settings.protect_enable_push = Enable Push settings.protect_enable_push_desc = Anyone with write access will be allowed to push to this branch (but not force push). +settings.protect_enable_merge = Enable Merge +settings.protect_enable_merge_desc = Anyone with write access will be allowed to merge the pull requests into this branch. settings.protect_whitelist_committers = Whitelist Restricted Push settings.protect_whitelist_committers_desc = Only whitelisted users or teams will be allowed to push to this branch (but not force push). settings.protect_whitelist_deploy_keys = Whitelist deploy keys with write access to push. @@ -2183,6 +2188,7 @@ settings.dismiss_stale_approvals_desc = When new commits that change the content settings.require_signed_commits = Require Signed Commits settings.require_signed_commits_desc = Reject pushes to this branch if they are unsigned or unverifiable. settings.protect_branch_name_pattern = Protected Branch Name Pattern +settings.protect_patterns = Patterns settings.protect_protected_file_patterns = "Protected file patterns (separated using semicolon ';'):" settings.protect_protected_file_patterns_desc = "Protected files are not allowed to be changed directly even if user has rights to add, edit, or delete files in this branch. Multiple patterns can be separated using semicolon (';'). See github.com/gobwas/glob documentation for pattern syntax. Examples: .drone.yml, /docs/**/*.txt." settings.protect_unprotected_file_patterns = "Unprotected file patterns (separated using semicolon ';'):" diff --git a/routers/web/repo/setting_protected_branch.go b/routers/web/repo/setting_protected_branch.go index 34e84c4656827..3beb79f20f2b6 100644 --- a/routers/web/repo/setting_protected_branch.go +++ b/routers/web/repo/setting_protected_branch.go @@ -130,15 +130,6 @@ func SettingsProtectedBranch(c *context.Context) { } c.Data["branch_status_check_contexts"] = contexts - c.Data["is_context_required"] = func(context string) bool { - for _, c := range rule.StatusCheckContexts { - if c == context { - return true - } - } - return false - } - if c.Repo.Owner.IsOrganization() { teams, err := organization.OrgFromUser(c.Repo.Owner).TeamsWithAccessToRepo(c.Repo.Repository.ID, perm.AccessModeRead) if err != nil { diff --git a/templates/repo/branch/list.tmpl b/templates/repo/branch/list.tmpl index 596d9ae78bd67..40d6351301dee 100644 --- a/templates/repo/branch/list.tmpl +++ b/templates/repo/branch/list.tmpl @@ -5,8 +5,13 @@ {{template "base/alert" .}} {{template "repo/sub_menu" .}} {{if .DefaultBranchBranch}} -

+

{{.locale.Tr "repo.default_branch"}} + {{if and $.IsWriter (not $.Repository.IsArchived) (not .IsDeleted)}} + + {{svg "octicon-arrow-switch"}} + + {{end}}

diff --git a/templates/repo/settings/branches.tmpl b/templates/repo/settings/branches.tmpl index f74a2a837db66..8023d39fed0e3 100644 --- a/templates/repo/settings/branches.tmpl +++ b/templates/repo/settings/branches.tmpl @@ -12,18 +12,13 @@

{{.locale.Tr "repo.settings.default_branch_desc"}}

-
+ {{.CsrfTokenHtml}} {{if not .Repository.IsEmpty}} -
-
- {{svg "octicon-triangle-down" 14 "dropdown icon"}} +
+ {{svg "octicon-triangle-down" 14 "dropdown icon"}} +
{{.Repository.DefaultBranch}}
{{range .Branches}} @@ -32,7 +27,6 @@
-
{{end}}
diff --git a/templates/repo/settings/protected_branch.tmpl b/templates/repo/settings/protected_branch.tmpl index 31ba0bf888798..2102dac641027 100644 --- a/templates/repo/settings/protected_branch.tmpl +++ b/templates/repo/settings/protected_branch.tmpl @@ -5,39 +5,49 @@ {{.locale.Tr "repo.settings.branch_protection" (.Rule.RuleName|Escape) | Str2html}}
+
{{.locale.Tr "repo.settings.protect_patterns"}}
- +
- -
+
+ + +

{{.locale.Tr "repo.settings.protect_protected_file_patterns_desc" | Safe}}

+
+
+ + +

{{.locale.Tr "repo.settings.protect_unprotected_file_patterns_desc" | Safe}}

+
{{.CsrfTokenHtml}} -
-
-
- - -

{{.locale.Tr "repo.settings.protect_disable_push_desc"}}

-
+
{{.locale.Tr "repo.settings.event_push"}}
+
+
+ + +

{{.locale.Tr "repo.settings.protect_disable_push_desc"}}

-
-
- - -

{{.locale.Tr "repo.settings.protect_enable_push_desc"}}

-
+
+
+
+ + +

{{.locale.Tr "repo.settings.protect_enable_push_desc"}}

+
+
- +

{{.locale.Tr "repo.settings.protect_whitelist_committers_desc"}}

-
-
+
+
@@ -52,8 +62,7 @@
{{if .Owner.IsOrganization}} -
-
+
@@ -69,105 +78,37 @@
{{end}} -
-
+
- -
-
-
- -
- -
-
- - -

{{.locale.Tr "repo.settings.protect_merge_whitelist_committers_desc"}}

-
-
-
-
- -
- -
{{.locale.Tr "repo.settings.protect_whitelist_search_users"}}
-
- {{range .Users}} -
- {{avatar $.Context . 28 "mini"}}{{template "repo/search_name" .}} -
- {{end}} -
+
- {{if .Owner.IsOrganization}} -
-
- -
- -
{{.locale.Tr "repo.settings.protect_whitelist_search_teams"}}
-
- {{range .Teams}} -
- {{svg "octicon-people"}} - {{.Name}} -
- {{end}} -
-
-
- {{end}} -
- -
-
- - -

{{.locale.Tr "repo.settings.protect_check_status_contexts_desc"}}

-
-
- -
-
- - - - - - - {{range $.branch_status_check_contexts}} - - {{end}} - -
- {{.locale.Tr "repo.settings.protect_check_status_contexts_list"}} -
- - - - {{.}} - {{if $.is_context_required}}{{if call $.is_context_required .}}
Required
{{end}}{{end}} -
-
- -
- - -

{{.locale.Tr "repo.settings.protect_required_approvals_desc"}}

+
+
+
+ + +

{{.locale.Tr "repo.settings.require_signed_commits_desc"}}

+
+
{{.locale.Tr "repo.settings.event_pull_request_approvals"}}
+
+ + +

{{.locale.Tr "repo.settings.protect_required_approvals_desc"}}

+
+
- +

{{.locale.Tr "repo.settings.protect_approvals_whitelist_enabled_desc"}}

-
-
+
+
@@ -182,8 +123,7 @@
{{if .Owner.IsOrganization}} -
-
+
@@ -200,53 +140,117 @@
{{end}}
-
-
- - -

{{.locale.Tr "repo.settings.block_rejected_reviews_desc"}}

-
+
+
+
+ + +

{{.locale.Tr "repo.settings.dismiss_stale_approvals_desc"}}

+
+
- - -

{{.locale.Tr "repo.settings.block_on_official_review_requests_desc"}}

+ + +

{{.locale.Tr "repo.settings.protect_check_status_contexts_desc"}}

+
+ + + + + + + + {{range $.branch_status_check_contexts}} + + + + {{else}} + + {{end}} + +
{{.locale.Tr "repo.settings.protect_check_status_contexts_list"}}
+ + + + +
N/A
+
+
+
{{.locale.Tr "repo.settings.event_pull_request_merge"}}
+
-
- - -

{{.locale.Tr "repo.settings.dismiss_stale_approvals_desc"}}

+
+ + +

{{.locale.Tr "repo.settings.protect_enable_merge_desc"}}

-
- - -

{{.locale.Tr "repo.settings.require_signed_commits_desc"}}

+
+ + +

{{.locale.Tr "repo.settings.protect_merge_whitelist_committers_desc"}}

-
-
- - -

{{.locale.Tr "repo.settings.block_outdated_branch_desc"}}

+
+
+ +
+ +
{{.locale.Tr "repo.settings.protect_whitelist_search_users"}}
+
+ {{range .Users}} +
+ {{avatar $.Context . 28 "mini"}}{{template "repo/search_name" .}} +
+ {{end}} +
+
+ {{if .Owner.IsOrganization}} +
+ +
+ +
{{.locale.Tr "repo.settings.protect_whitelist_search_teams"}}
+
+ {{range .Teams}} +
+ {{svg "octicon-people"}} + {{.Name}} +
+ {{end}} +
+
+
+ {{end}}
-
- - -

{{.locale.Tr "repo.settings.protect_protected_file_patterns_desc" | Safe}}

+
+
+
+ + +

{{.locale.Tr "repo.settings.block_rejected_reviews_desc"}}

-
- - -

{{.locale.Tr "repo.settings.protect_unprotected_file_patterns_desc" | Safe}}

+
+
+
+ + +

{{.locale.Tr "repo.settings.block_on_official_review_requests_desc"}}

+
+
+
+
+ + +

{{.locale.Tr "repo.settings.block_outdated_branch_desc"}}

-
diff --git a/web_src/css/base.css b/web_src/css/base.css index c7ea0e47c8f21..95f83ef00a1ac 100644 --- a/web_src/css/base.css +++ b/web_src/css/base.css @@ -2533,12 +2533,12 @@ a.ui.basic.label:hover { right: 0.78571429rem; top: 0; bottom: 0; - height: 30px; - margin-top: auto; - margin-bottom: auto; + display: flex; + align-items: center; } -/* https://github.com/go-gitea/gitea/issues/10210 */ +/* if a .top.attached.header is followed by a .segment, add some margin */ +.ui.segments ~ .ui.top.attached.header, .ui.attached.segment ~ .ui.top.attached.header { margin-top: 1rem; } diff --git a/web_src/css/helpers.css b/web_src/css/helpers.css index 22a284d43c091..0b4b47f560ffa 100644 --- a/web_src/css/helpers.css +++ b/web_src/css/helpers.css @@ -60,6 +60,8 @@ Gitea's private styles use `g-` prefix. text-overflow: ellipsis; } +.gt-max-width-24rem { max-width: 24rem !important; } + /* below class names match Tailwind CSS */ .gt-break-all { word-break: break-all !important; } .gt-content-center { align-content: center !important; } diff --git a/web_src/css/repository.css b/web_src/css/repository.css index 8b25775bb4440..af2dd2be242f1 100644 --- a/web_src/css/repository.css +++ b/web_src/css/repository.css @@ -2012,35 +2012,10 @@ margin-top: -3px; } -.repository.settings.branches .protected-branches .selection.dropdown { - width: 300px; -} - -.repository.settings.branches .protected-branches .item { - border: 1px solid var(--color-secondary); - padding: 10px 15px; -} - -.repository.settings.branches .protected-branches .item:not(:last-child) { - border-bottom: 0; -} - -.repository.settings.branches .branch-protection .help { - margin-left: 26px; - padding-top: 0; -} - -.repository.settings.branches .branch-protection .fields { - margin-left: 20px; - display: block; -} - -.repository.settings.branches .branch-protection .whitelist { - margin-left: 26px; -} - -.repository.settings.branches .branch-protection .whitelist .dropdown img { - display: inline-block; +/* if the element is for a checkbox, then it should have a padding-left to align to the checkbox's text */ +.repository.settings.branches .branch-protection .ui.checkbox .help, +.repository.settings.branches .branch-protection .checkbox-sub-item { + padding-left: 26px; } .repository.settings.webhook .events .column { diff --git a/web_src/js/features/repo-settings.js b/web_src/js/features/repo-settings.js index 3d02a82bb6554..9f094cd2d1624 100644 --- a/web_src/js/features/repo-settings.js +++ b/web_src/js/features/repo-settings.js @@ -1,6 +1,5 @@ import $ from 'jquery'; import {createMonaco} from './codeeditor.js'; -import {initRepoCommonFilterSearchDropdown} from './repo-common.js'; const {appSubUrl, csrfToken} = window.config; @@ -73,20 +72,13 @@ export function initRepoSettingGitHook() { } export function initRepoSettingBranches() { - // Branches - if ($('.repository.settings.branches').length > 0) { - initRepoCommonFilterSearchDropdown('.protected-branches .dropdown'); - $('.enable-protection, .enable-whitelist, .enable-statuscheck').on('change', function () { - if (this.checked) { - $($(this).data('target')).removeClass('disabled'); - } else { - $($(this).data('target')).addClass('disabled'); - } - }); - $('.disable-whitelist').on('change', function () { - if (this.checked) { - $($(this).data('target')).addClass('disabled'); - } - }); - } + if (!$('.repository.settings.branches').length) return; + $('.toggle-target-enabled').on('change', function () { + const $target = $($(this).attr('data-target')); + $target.toggleClass('disabled', !this.checked); + }); + $('.toggle-target-disabled').on('change', function () { + const $target = $($(this).attr('data-target')); + if (this.checked) $target.addClass('disabled'); // only disable, do not auto enable + }); } From 5a5ab8ef5ac5fbdb893707933f06ff6bcd8e834a Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Sat, 29 Apr 2023 19:35:59 +0800 Subject: [PATCH 15/65] Start cleaning the messy ".ui.left / .ui.right", improve label list page, fix stackable menu (#24393) Since 2015/2016, there is a global pollution: ".ui.left" / ".ui.right". Fomantic UI doesn't work this way, it just conflicts with many Fomantic definitions. This PR starts the cleaning work of such techinical debts. And, the "label list" page has been quite messy for long time, for example, why "li" appears in "div" ...... And fix #24296
![image](https://user-images.githubusercontent.com/2114189/235051281-54c5374c-b5fd-4b5f-9aa2-02d4bb2d9112.png) ![image](https://user-images.githubusercontent.com/2114189/235055703-2ba042e0-4db7-4e63-8646-02f390d496b5.png) ![image](https://user-images.githubusercontent.com/2114189/235056310-4f6ffdc2-5758-4927-8fb8-314d9fb72a6b.png) ![image](https://user-images.githubusercontent.com/2114189/235058400-dab1c9ec-3325-4671-8345-aee6b0b68042.png) ![image](https://user-images.githubusercontent.com/2114189/235058424-85509532-b9bc-43ad-b00f-a87184c60f22.png)
--- templates/admin/dashboard.tmpl | 22 ++-- templates/base/footer_content.tmpl | 6 +- templates/base/head.tmpl | 2 +- templates/org/team/members.tmpl | 4 +- templates/org/team/repositories.tmpl | 4 +- templates/repo/cite/cite_modal.tmpl | 2 +- templates/repo/diff/conversation.tmpl | 4 +- templates/repo/header.tmpl | 20 ++- templates/repo/issue/labels/label_list.tmpl | 123 +++++++++--------- templates/repo/issue/list.tmpl | 2 +- templates/user/dashboard/navbar.tmpl | 2 +- web_src/css/base.css | 59 ++------- .../combomarkdowneditor.css} | 0 web_src/css/helpers.css | 8 +- web_src/css/home.css | 21 ++- web_src/css/index.css | 5 +- web_src/css/repository.css | 2 +- web_src/css/repository/issue-label.css | 44 +++++++ .../release-tag.css} | 0 19 files changed, 168 insertions(+), 162 deletions(-) rename web_src/css/{editor-markdown.css => editor/combomarkdowneditor.css} (100%) create mode 100644 web_src/css/repository/issue-label.css rename web_src/css/{repository-release-tag.css => repository/release-tag.css} (100%) diff --git a/templates/admin/dashboard.tmpl b/templates/admin/dashboard.tmpl index 678445137660c..91a84aebf5514 100644 --- a/templates/admin/dashboard.tmpl +++ b/templates/admin/dashboard.tmpl @@ -23,51 +23,51 @@ {{.locale.Tr "admin.dashboard.delete_inactive_accounts"}} - + {{.locale.Tr "admin.dashboard.delete_repo_archives"}} - + {{.locale.Tr "admin.dashboard.delete_missing_repos"}} - + {{.locale.Tr "admin.dashboard.git_gc_repos"}} - + {{if and (not .SSH.Disabled) (not .SSH.StartBuiltinServer)}} {{.locale.Tr "admin.dashboard.resync_all_sshkeys"}}
{{.locale.Tr "admin.dashboard.resync_all_sshkeys.desc"}} - + {{.locale.Tr "admin.dashboard.resync_all_sshprincipals"}}
{{.locale.Tr "admin.dashboard.resync_all_sshprincipals.desc"}} - + {{end}} {{.locale.Tr "admin.dashboard.resync_all_hooks"}} - + {{.locale.Tr "admin.dashboard.reinit_missing_repos"}} - + {{.locale.Tr "admin.dashboard.sync_external_users"}} - + {{.locale.Tr "admin.dashboard.repo_health_check"}} - + {{.locale.Tr "admin.dashboard.delete_generated_repository_avatars"}} - + diff --git a/templates/base/footer_content.tmpl b/templates/base/footer_content.tmpl index 17c75c6874052..f5272744846cd 100644 --- a/templates/base/footer_content.tmpl +++ b/templates/base/footer_content.tmpl @@ -1,5 +1,5 @@ -