diff --git a/.gitignore b/.gitignore index 2cbc7ccfa7dd4..79198ffcfea76 100644 --- a/.gitignore +++ b/.gitignore @@ -41,5 +41,6 @@ coverage.out /dist /custom /data +/indexers /log /public/img/avatar diff --git a/conf/app.ini b/conf/app.ini index 303b006b202ad..254b1bc549e06 100644 --- a/conf/app.ini +++ b/conf/app.ini @@ -158,6 +158,10 @@ SSL_MODE = disable ; For "sqlite3" and "tidb", use absolute path when you start as service PATH = data/gitea.db +[indexer] +ISSUE_INDEXER_PATH = indexers/issues.bleve +UPDATE_BUFFER_LEN = 20 + [admin] [security] diff --git a/models/issue.go b/models/issue.go index ac50d2dfba313..d9261613810ab 100644 --- a/models/issue.go +++ b/models/issue.go @@ -17,6 +17,7 @@ import ( "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/util" ) var ( @@ -451,8 +452,11 @@ func (issue *Issue) ReadBy(userID int64) error { } func updateIssueCols(e Engine, issue *Issue, cols ...string) error { - _, err := e.Id(issue.ID).Cols(cols...).Update(issue) - return err + if _, err := e.Id(issue.ID).Cols(cols...).Update(issue); err != nil { + return err + } + UpdateIssueIndexer(issue) + return nil } // UpdateIssueCols only updates values of specific columns for given issue. @@ -733,6 +737,8 @@ func newIssue(e *xorm.Session, opts NewIssueOptions) (err error) { return err } + UpdateIssueIndexer(opts.Issue) + if len(opts.Attachments) > 0 { attachments, err := getAttachmentsByUUIDs(e, opts.Attachments) if err != nil { @@ -865,10 +871,11 @@ type IssuesOptions struct { MilestoneID int64 RepoIDs []int64 Page int - IsClosed bool - IsPull bool + IsClosed util.OptionalBool + IsPull util.OptionalBool Labels string SortType string + IssueIDs []int64 } // sortIssuesSession sort an issues-related session based on the provided @@ -894,11 +901,23 @@ func sortIssuesSession(sess *xorm.Session, sortType string) { // Issues returns a list of issues by given conditions. func Issues(opts *IssuesOptions) ([]*Issue, error) { - if opts.Page <= 0 { - opts.Page = 1 + var sess *xorm.Session + if opts.Page >= 0 { + var start int + if opts.Page == 0 { + start = 0 + } else { + start = (opts.Page - 1) * setting.UI.IssuePagingNum + } + sess = x.Limit(setting.UI.IssuePagingNum, start) + } else { + sess = x.NewSession() + defer sess.Close() } - sess := x.Limit(setting.UI.IssuePagingNum, (opts.Page-1)*setting.UI.IssuePagingNum) + if len(opts.IssueIDs) > 0 { + sess.In("issue.id", opts.IssueIDs) + } if opts.RepoID > 0 { sess.And("issue.repo_id=?", opts.RepoID) @@ -906,7 +925,13 @@ func Issues(opts *IssuesOptions) ([]*Issue, error) { // In case repository IDs are provided but actually no repository has issue. sess.In("issue.repo_id", opts.RepoIDs) } - sess.And("issue.is_closed=?", opts.IsClosed) + + switch opts.IsClosed { + case util.OptionalBoolTrue: + sess.And("issue.is_closed=true") + case util.OptionalBoolFalse: + sess.And("issue.is_closed=false") + } if opts.AssigneeID > 0 { sess.And("issue.assignee_id=?", opts.AssigneeID) @@ -926,7 +951,12 @@ func Issues(opts *IssuesOptions) ([]*Issue, error) { sess.And("issue.milestone_id=?", opts.MilestoneID) } - sess.And("issue.is_pull=?", opts.IsPull) + switch opts.IsPull { + case util.OptionalBoolTrue: + sess.And("issue.is_pull=true") + case util.OptionalBoolFalse: + sess.And("issue.is_pull=false") + } sortIssuesSession(sess, opts.SortType) @@ -1168,10 +1198,11 @@ type IssueStatsOptions struct { MentionedID int64 PosterID int64 IsPull bool + IssueIDs []int64 } // GetIssueStats returns issue statistic information by given conditions. -func GetIssueStats(opts *IssueStatsOptions) *IssueStats { +func GetIssueStats(opts *IssueStatsOptions) (*IssueStats, error) { stats := &IssueStats{} countSession := func(opts *IssueStatsOptions) *xorm.Session { @@ -1179,6 +1210,10 @@ func GetIssueStats(opts *IssueStatsOptions) *IssueStats { Where("issue.repo_id = ?", opts.RepoID). And("is_pull = ?", opts.IsPull) + if len(opts.IssueIDs) > 0 { + sess.In("issue.id", opts.IssueIDs) + } + if len(opts.Labels) > 0 && opts.Labels != "0" { labelIDs, err := base.StringsToInt64s(strings.Split(opts.Labels, ",")) if err != nil { @@ -1210,13 +1245,20 @@ func GetIssueStats(opts *IssueStatsOptions) *IssueStats { return sess } - stats.OpenCount, _ = countSession(opts). + var err error + stats.OpenCount, err = countSession(opts). And("is_closed = ?", false). Count(&Issue{}) - stats.ClosedCount, _ = countSession(opts). + if err != nil { + return nil, err + } + stats.ClosedCount, err = countSession(opts). And("is_closed = ?", true). Count(&Issue{}) - return stats + if err != nil { + return nil, err + } + return stats, nil } // GetUserIssueStats returns issue statistic information for dashboard by given conditions. @@ -1294,7 +1336,11 @@ func GetRepoIssueStats(repoID, uid int64, filterMode int, isPull bool) (numOpen func updateIssue(e Engine, issue *Issue) error { _, err := e.Id(issue.ID).AllCols().Update(issue) - return err + if err != nil { + return err + } + UpdateIssueIndexer(issue) + return nil } // UpdateIssue updates all fields of given issue. diff --git a/models/issue_comment.go b/models/issue_comment.go index e9a401b864299..a17be97e722a2 100644 --- a/models/issue_comment.go +++ b/models/issue_comment.go @@ -454,28 +454,20 @@ func UpdateComment(c *Comment) error { return err } -// DeleteCommentByID deletes the comment by given ID. -func DeleteCommentByID(id int64) error { - comment, err := GetCommentByID(id) - if err != nil { - if IsErrCommentNotExist(err) { - return nil - } - return err - } - +// DeleteComment deletes the comment +func DeleteComment(comment *Comment) error { sess := x.NewSession() defer sessionRelease(sess) - if err = sess.Begin(); err != nil { + if err := sess.Begin(); err != nil { return err } - if _, err = sess.Id(comment.ID).Delete(new(Comment)); err != nil { + if _, err := sess.Id(comment.ID).Delete(new(Comment)); err != nil { return err } if comment.Type == CommentTypeComment { - if _, err = sess.Exec("UPDATE `issue` SET num_comments = num_comments - 1 WHERE id = ?", comment.IssueID); err != nil { + if _, err := sess.Exec("UPDATE `issue` SET num_comments = num_comments - 1 WHERE id = ?", comment.IssueID); err != nil { return err } } diff --git a/models/issue_indexer.go b/models/issue_indexer.go new file mode 100644 index 0000000000000..bbaf0e64bc271 --- /dev/null +++ b/models/issue_indexer.go @@ -0,0 +1,183 @@ +// Copyright 2017 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package models + +import ( + "fmt" + "os" + "strconv" + "strings" + + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/util" + "github.com/blevesearch/bleve" + "github.com/blevesearch/bleve/analysis/analyzer/simple" + "github.com/blevesearch/bleve/search/query" +) + +// issueIndexerUpdateQueue queue of issues that need to be updated in the issues +// indexer +var issueIndexerUpdateQueue chan *Issue + +// issueIndexer (thread-safe) index for searching issues +var issueIndexer bleve.Index + +// issueIndexerData data stored in the issue indexer +type issueIndexerData struct { + ID int64 + RepoID int64 + + Title string + Content string +} + +// numericQuery an numeric-equality query for the given value and field +func numericQuery(value int64, field string) *query.NumericRangeQuery { + f := float64(value) + tru := true + q := bleve.NewNumericRangeInclusiveQuery(&f, &f, &tru, &tru) + q.SetField(field) + return q +} + +// SearchIssuesByKeyword searches for issues by given conditions. +// Returns the matching issue IDs +func SearchIssuesByKeyword(repoID int64, keyword string) ([]int64, error) { + fields := strings.Fields(strings.ToLower(keyword)) + indexerQuery := bleve.NewConjunctionQuery( + numericQuery(repoID, "RepoID"), + bleve.NewDisjunctionQuery( + bleve.NewPhraseQuery(fields, "Title"), + bleve.NewPhraseQuery(fields, "Content"), + )) + search := bleve.NewSearchRequestOptions(indexerQuery, 2147483647, 0, false) + search.Fields = []string{"ID"} + + result, err := issueIndexer.Search(search) + if err != nil { + return nil, err + } + + issueIDs := make([]int64, len(result.Hits)) + for i, hit := range result.Hits { + issueIDs[i] = int64(hit.Fields["ID"].(float64)) + } + return issueIDs, nil +} + +// InitIssueIndexer initialize issue indexer +func InitIssueIndexer() { + _, err := os.Stat(setting.Indexer.IssuePath) + if err != nil { + if os.IsNotExist(err) { + if err = createIssueIndexer(); err != nil { + log.Fatal(4, "CreateIssuesIndexer: %v", err) + } + if err = populateIssueIndexer(); err != nil { + log.Fatal(4, "PopulateIssuesIndex: %v", err) + } + } else { + log.Fatal(4, "InitIssuesIndexer: %v", err) + } + } else { + issueIndexer, err = bleve.Open(setting.Indexer.IssuePath) + if err != nil { + log.Fatal(4, "InitIssuesIndexer, open index: %v", err) + } + } + issueIndexerUpdateQueue = make(chan *Issue, setting.Indexer.UpdateQueueLength) + go processIssueIndexerUpdateQueue() + // TODO close issueIndexer when Gitea closes +} + +// createIssueIndexer create an issue indexer if one does not already exist +func createIssueIndexer() error { + mapping := bleve.NewIndexMapping() + docMapping := bleve.NewDocumentMapping() + + docMapping.AddFieldMappingsAt("ID", bleve.NewNumericFieldMapping()) + docMapping.AddFieldMappingsAt("RepoID", bleve.NewNumericFieldMapping()) + + textFieldMapping := bleve.NewTextFieldMapping() + textFieldMapping.Analyzer = simple.Name + docMapping.AddFieldMappingsAt("Title", textFieldMapping) + docMapping.AddFieldMappingsAt("Content", textFieldMapping) + + mapping.AddDocumentMapping("issues", docMapping) + + var err error + issueIndexer, err = bleve.New(setting.Indexer.IssuePath, mapping) + return err +} + +// populateIssueIndexer populate the issue indexer with issue data +func populateIssueIndexer() error { + for page := 1; ; page++ { + repos, err := Repositories(&SearchRepoOptions{ + Page: page, + PageSize: 10, + }) + if err != nil { + return fmt.Errorf("Repositories: %v", err) + } + if len(repos) == 0 { + return nil + } + batch := issueIndexer.NewBatch() + for _, repo := range repos { + issues, err := Issues(&IssuesOptions{ + RepoID: repo.ID, + IsClosed: util.OptionalBoolNone, + IsPull: util.OptionalBoolNone, + Page: -1, // do not page + }) + if err != nil { + return fmt.Errorf("Issues: %v", err) + } + for _, issue := range issues { + err = batch.Index(issue.indexUID(), issue.issueData()) + if err != nil { + return fmt.Errorf("batch.Index: %v", err) + } + } + } + if err = issueIndexer.Batch(batch); err != nil { + return fmt.Errorf("index.Batch: %v", err) + } + } +} + +func processIssueIndexerUpdateQueue() { + for { + select { + case issue := <-issueIndexerUpdateQueue: + if err := issueIndexer.Index(issue.indexUID(), issue.issueData()); err != nil { + log.Error(4, "issuesIndexer.Index: %v", err) + } + } + } +} + +// indexUID a unique identifier for an issue used in full-text indices +func (issue *Issue) indexUID() string { + return strconv.FormatInt(issue.ID, 36) +} + +func (issue *Issue) issueData() *issueIndexerData { + return &issueIndexerData{ + ID: issue.ID, + RepoID: issue.RepoID, + Title: issue.Title, + Content: issue.Content, + } +} + +// UpdateIssueIndexer add/update an issue to the issue indexer +func UpdateIssueIndexer(issue *Issue) { + go func() { + issueIndexerUpdateQueue <- issue + }() +} diff --git a/models/models.go b/models/models.go index d9716e79bd229..1ce704a9e4c49 100644 --- a/models/models.go +++ b/models/models.go @@ -138,6 +138,10 @@ func LoadConfigs() { } DbCfg.SSLMode = sec.Key("SSL_MODE").String() DbCfg.Path = sec.Key("PATH").MustString("data/gitea.db") + + sec = setting.Cfg.Section("indexer") + setting.Indexer.IssuePath = sec.Key("ISSUE_INDEXER_PATH").MustString("indexers/issues.bleve") + setting.Indexer.UpdateQueueLength = sec.Key("UPDATE_BUFFER_LEN").MustInt(20) } // parsePostgreSQLHostPort parses given input in various forms defined in diff --git a/modules/indexer/indexer.go b/modules/indexer/indexer.go new file mode 100644 index 0000000000000..2b7b76f7f267a --- /dev/null +++ b/modules/indexer/indexer.go @@ -0,0 +1,14 @@ +// Copyright 2016 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package indexer + +import ( + "code.gitea.io/gitea/models" +) + +// NewContext start indexer service +func NewContext() { + models.InitIssueIndexer() +} diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 910ec9302177a..fd0f5085c9f1b 100644 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -123,6 +123,12 @@ var ( UsePostgreSQL bool UseTiDB bool + // Indexer settings + Indexer struct { + IssuePath string + UpdateQueueLength int + } + // Webhook settings Webhook = struct { QueueLength int diff --git a/modules/util/util.go b/modules/util/util.go new file mode 100644 index 0000000000000..4859965388b5b --- /dev/null +++ b/modules/util/util.go @@ -0,0 +1,25 @@ +// Copyright 2017 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package util + +// OptionalBool a boolean that can be "null" +type OptionalBool byte + +const ( + // OptionalBoolNone a "null" boolean value + OptionalBoolNone = iota + // OptionalBoolTrue a "true" boolean value + OptionalBoolTrue + // OptionalBoolFalse a "false" boolean value + OptionalBoolFalse +) + +// OptionalBoolOf get the corresponding OptionalBool of a bool +func OptionalBoolOf(b bool) OptionalBool { + if b { + return OptionalBoolTrue + } + return OptionalBoolFalse +} diff --git a/public/css/index.css b/public/css/index.css index 7c84cf8517aa9..d755e6127b973 100644 --- a/public/css/index.css +++ b/public/css/index.css @@ -2926,6 +2926,10 @@ footer .ui.language .menu { width: 16px; text-align: center; } +.navbar { + display: flex; + justify-content: space-between; +} .ui.repository.list .item { padding-bottom: 25px; } diff --git a/routers/api/v1/repo/issue.go b/routers/api/v1/repo/issue.go index 908b5aeb961f5..ac0289c412050 100644 --- a/routers/api/v1/repo/issue.go +++ b/routers/api/v1/repo/issue.go @@ -13,14 +13,16 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/util" ) // ListIssues list the issues of a repository func ListIssues(ctx *context.APIContext) { + isClosed := ctx.Query("state") == "closed" issueOpts := models.IssuesOptions{ RepoID: ctx.Repo.Repository.ID, Page: ctx.QueryInt("page"), - IsClosed: ctx.Query("state") == "closed", + IsClosed: util.OptionalBoolOf(isClosed), } issues, err := models.Issues(&issueOpts) @@ -29,7 +31,7 @@ func ListIssues(ctx *context.APIContext) { return } if ctx.Query("state") == "all" { - issueOpts.IsClosed = !issueOpts.IsClosed + issueOpts.IsClosed = util.OptionalBoolOf(!isClosed) tempIssues, err := models.Issues(&issueOpts) if err != nil { ctx.Error(500, "Issues", err) diff --git a/routers/api/v1/repo/issue_comment.go b/routers/api/v1/repo/issue_comment.go index 1afd3e9d78fc9..13e3ec6ab86b9 100644 --- a/routers/api/v1/repo/issue_comment.go +++ b/routers/api/v1/repo/issue_comment.go @@ -125,7 +125,7 @@ func DeleteIssueComment(ctx *context.APIContext) { return } - if err = models.DeleteCommentByID(comment.ID); err != nil { + if err = models.DeleteComment(comment); err != nil { ctx.Error(500, "DeleteCommentByID", err) return } diff --git a/routers/init.go b/routers/init.go index 697f33835cde1..38a456639d8c8 100644 --- a/routers/init.go +++ b/routers/init.go @@ -18,6 +18,7 @@ import ( "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/ssh" macaron "gopkg.in/macaron.v1" + "code.gitea.io/gitea/modules/indexer" ) func checkRunMode() { @@ -36,6 +37,7 @@ func checkRunMode() { func NewServices() { setting.NewServices() mailer.NewContext() + indexer.NewContext() } // GlobalInit is for global configuration reload-able. diff --git a/routers/repo/issue.go b/routers/repo/issue.go index 50995e70fe51c..6ef34746ede5b 100644 --- a/routers/repo/issue.go +++ b/routers/repo/issue.go @@ -5,6 +5,7 @@ package repo import ( + "bytes" "errors" "fmt" "io" @@ -25,6 +26,7 @@ import ( "code.gitea.io/gitea/modules/markdown" "code.gitea.io/gitea/modules/notification" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/util" ) const ( @@ -158,20 +160,39 @@ func Issues(ctx *context.Context) { milestoneID := ctx.QueryInt64("milestone") isShowClosed := ctx.Query("state") == "closed" + keyword := ctx.Query("q") + if bytes.Contains([]byte(keyword), []byte{0x00}) { + keyword = "" + } + + var issueIDs []int64 + var err error + if len(keyword) > 0 { + issueIDs, err = models.SearchIssuesByKeyword(repo.ID, keyword) + if len(issueIDs) == 0 { + forceEmpty = true + } + } + var issueStats *models.IssueStats if forceEmpty { issueStats = &models.IssueStats{} } else { - issueStats = models.GetIssueStats(&models.IssueStatsOptions{ + var err error + issueStats, err = models.GetIssueStats(&models.IssueStatsOptions{ RepoID: repo.ID, Labels: selectLabels, MilestoneID: milestoneID, AssigneeID: assigneeID, MentionedID: mentionedID, IsPull: isPullList, + IssueIDs: issueIDs, }) + if err != nil { + ctx.Error(500, "GetSearchIssueStats") + return + } } - page := ctx.QueryInt("page") if page <= 1 { page = 1 @@ -190,7 +211,6 @@ func Issues(ctx *context.Context) { if forceEmpty { issues = []*models.Issue{} } else { - var err error issues, err = models.Issues(&models.IssuesOptions{ AssigneeID: assigneeID, RepoID: repo.ID, @@ -198,10 +218,11 @@ func Issues(ctx *context.Context) { MentionedID: mentionedID, MilestoneID: milestoneID, Page: pager.Current(), - IsClosed: isShowClosed, - IsPull: isPullList, + IsClosed: util.OptionalBoolOf(isShowClosed), + IsPull: util.OptionalBoolOf(isPullList), Labels: selectLabels, SortType: sortType, + IssueIDs: issueIDs, }) if err != nil { ctx.Handle(500, "Issues", err) @@ -258,6 +279,7 @@ func Issues(ctx *context.Context) { ctx.Data["MilestoneID"] = milestoneID ctx.Data["AssigneeID"] = assigneeID ctx.Data["IsShowClosed"] = isShowClosed + ctx.Data["Keyword"] = keyword if isShowClosed { ctx.Data["State"] = "closed" } else { @@ -934,7 +956,7 @@ func DeleteComment(ctx *context.Context) { return } - if err = models.DeleteCommentByID(comment.ID); err != nil { + if err = models.DeleteComment(comment); err != nil { ctx.Handle(500, "DeleteCommentByID", err) return } diff --git a/routers/user/home.go b/routers/user/home.go index 571849df3041e..db2fe84f91f7d 100644 --- a/routers/user/home.go +++ b/routers/user/home.go @@ -15,6 +15,7 @@ import ( "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/util" ) const ( @@ -277,8 +278,8 @@ func Issues(ctx *context.Context) { PosterID: posterID, RepoIDs: repoIDs, Page: page, - IsClosed: isShowClosed, - IsPull: isPullList, + IsClosed: util.OptionalBoolOf(isShowClosed), + IsPull: util.OptionalBoolOf(isPullList), SortType: sortType, }) if err != nil { diff --git a/templates/repo/issue/list.tmpl b/templates/repo/issue/list.tmpl index d00c9aea21144..bb7327ebf15e9 100644 --- a/templates/repo/issue/list.tmpl +++ b/templates/repo/issue/list.tmpl @@ -4,6 +4,7 @@
@@ -105,7 +106,7 @@ {{.Title}} {{range .Labels}} - {{.Name}} + {{.Name}} {{end}} {{if .NumComments}} @@ -115,7 +116,7 @@

{{$.i18n.Tr "repo.issues.opened_by" $timeStr .Poster.HomeLink .Poster.Name | Safe}} {{if .Milestone}} - + {{.Milestone.Name}} {{end}} @@ -132,17 +133,17 @@ {{if gt .TotalPages 1}}

diff --git a/templates/repo/issue/navbar.tmpl b/templates/repo/issue/navbar.tmpl index a3e9d2660d1aa..1e864458e7d9a 100644 --- a/templates/repo/issue/navbar.tmpl +++ b/templates/repo/issue/navbar.tmpl @@ -1,4 +1,4 @@ -