From 45b5aa0b39aa281c4741d722dda32c0f008aa5be Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Wed, 1 Apr 2020 13:17:56 +0200 Subject: [PATCH 01/15] Cache PullRequest Divergence --- models/migrations/migrations.go | 3 +++ models/migrations/v999.go | 17 +++++++++++++++++ models/pull.go | 22 ++++++++++++++++++++++ routers/repo/pull.go | 8 +------- services/pull/check.go | 8 +++++++- services/pull/pull.go | 16 ++++++++++++++++ 6 files changed, 66 insertions(+), 8 deletions(-) create mode 100644 models/migrations/v999.go diff --git a/models/migrations/migrations.go b/models/migrations/migrations.go index 847cd75d521f6..242052266d0e4 100644 --- a/models/migrations/migrations.go +++ b/models/migrations/migrations.go @@ -204,6 +204,9 @@ var migrations = []Migration{ NewMigration("Refix merge base for merged pull requests", refixMergeBase), // v135 -> 136 NewMigration("Add OrgID column to Labels table", addOrgIDLabelColumn), + + // v999 not finalized + NewMigration("Add CommitsAhead and CommitsBehind Column to PullRequest Table", addCommitDivergenceToPulls), } // Migrate database to current version diff --git a/models/migrations/v999.go b/models/migrations/v999.go new file mode 100644 index 0000000000000..d1b3cb91f8059 --- /dev/null +++ b/models/migrations/v999.go @@ -0,0 +1,17 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package migrations + +import ( + "xorm.io/xorm" +) + +func addCommitDivergenceToPulls(x *xorm.Engine) error { + type PullRequest struct { + CommitsAhead int + CommitsBehind int + } + return x.Sync2(new(PullRequest)) +} diff --git a/models/pull.go b/models/pull.go index 48d2340544377..acdd4df2392ce 100644 --- a/models/pull.go +++ b/models/pull.go @@ -43,6 +43,8 @@ type PullRequest struct { Type PullRequestType Status PullRequestStatus ConflictedFiles []string `xorm:"TEXT JSON"` + CommitsAhead int + CommitsBehind int IssueID int64 `xorm:"INDEX"` Issue *Issue `xorm:"-"` @@ -352,6 +354,26 @@ func (pr *PullRequest) GetCommitMessages() string { return stringBuilder.String() } +// GetCommitDivergence get Divergence of a pull request +func (pr *PullRequest) GetCommitDivergence() *git.DivergeObject { + return &git.DivergeObject{Ahead: pr.CommitsAhead, Behind: pr.CommitsBehind} +} + +// UpdateCommitDivergence update Divergence of a pull request +func (pr *PullRequest) UpdateCommitDivergence(div *git.DivergeObject) error { + return pr.updateCommitDivergence(x, div) +} + +func (pr *PullRequest) updateCommitDivergence(e Engine, div *git.DivergeObject) error { + if div == nil || pr.ID == 0 { + return fmt.Errorf("updateCommitDivergence: diverge nil or PR-ID 0") + } + pr.CommitsAhead = div.Ahead + pr.CommitsBehind = div.Behind + _, err := e.ID(pr.ID).Cols("commits_ahead", "commits_behind").Update(pr) + return err +} + // ReviewCount represents a count of Reviews type ReviewCount struct { IssueID int64 diff --git a/routers/repo/pull.go b/routers/repo/pull.go index c29cfb81b22d0..7c616e8e9bd52 100644 --- a/routers/repo/pull.go +++ b/routers/repo/pull.go @@ -426,13 +426,7 @@ func PrepareViewPullInfo(ctx *context.Context, issue *models.Issue) *git.Compare return nil } ctx.Data["UpdateAllowed"] = allowUpdate - - divergence, err := pull_service.GetDiverging(pull) - if err != nil { - ctx.ServerError("GetDiverging", err) - return nil - } - ctx.Data["Divergence"] = divergence + ctx.Data["Divergence"] = pull.GetCommitDivergence() } sha, err := baseGitRepo.GetRefCommitID(pull.GetGitRefName()) diff --git a/services/pull/check.go b/services/pull/check.go index ea91f3a862475..76b24ec671456 100644 --- a/services/pull/check.go +++ b/services/pull/check.go @@ -32,7 +32,13 @@ func AddToTaskQueue(pr *models.PullRequest) { go func() { err := prQueue.PushFunc(strconv.FormatInt(pr.ID, 10), func() error { pr.Status = models.PullRequestStatusChecking - err := pr.UpdateColsIfNotMerged("status") + divergence, err := GetDiverging(pr) + if err != nil { + log.Error("AddToTaskQueue.GetDiverging[%d]: %v", pr.ID, err) + } + pr.CommitsAhead = divergence.Ahead + pr.CommitsBehind = divergence.Behind + err = pr.UpdateColsIfNotMerged("status", "commits_ahead", "commits_behind") if err != nil { log.Error("AddToTaskQueue.UpdateCols[%d].(add to queue): %v", pr.ID, err) } else { diff --git a/services/pull/pull.go b/services/pull/pull.go index ce5c4ff22ffc2..e8bd2339ea1aa 100644 --- a/services/pull/pull.go +++ b/services/pull/pull.go @@ -30,6 +30,13 @@ func NewPullRequest(repo *models.Repository, pull *models.Issue, labelIDs []int6 return err } + divergence, err := GetDiverging(pr) + if err != nil { + return err + } + pr.CommitsAhead = divergence.Ahead + pr.CommitsBehind = divergence.Behind + if err := models.NewPullRequest(repo, pull, labelIDs, uuids, pr); err != nil { return err } @@ -211,6 +218,15 @@ func AddTestPullRequestTask(doer *models.User, repoID int64, branch string, isSy if err := models.MarkReviewsAsNotStale(pr.IssueID, newCommitID); err != nil { log.Error("MarkReviewsAsNotStale: %v", err) } + divergence, err := GetDiverging(pr) + if err != nil { + log.Error("GetDiverging: %v", err) + } else { + err = pr.UpdateCommitDivergence(divergence) + if err != nil { + log.Error("UpdateCommitDivergence: %v", err) + } + } } pr.Issue.PullRequest = pr From 3d2ca637e6d50946166509cf88068aa0c0759a10 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Fri, 3 Apr 2020 23:47:48 +0200 Subject: [PATCH 02/15] only re-calc divergence if AddTestPullRequestTask() is exec --- services/pull/check.go | 8 +------- services/pull/pull.go | 9 +++++++++ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/services/pull/check.go b/services/pull/check.go index 76b24ec671456..ea91f3a862475 100644 --- a/services/pull/check.go +++ b/services/pull/check.go @@ -32,13 +32,7 @@ func AddToTaskQueue(pr *models.PullRequest) { go func() { err := prQueue.PushFunc(strconv.FormatInt(pr.ID, 10), func() error { pr.Status = models.PullRequestStatusChecking - divergence, err := GetDiverging(pr) - if err != nil { - log.Error("AddToTaskQueue.GetDiverging[%d]: %v", pr.ID, err) - } - pr.CommitsAhead = divergence.Ahead - pr.CommitsBehind = divergence.Behind - err = pr.UpdateColsIfNotMerged("status", "commits_ahead", "commits_behind") + err := pr.UpdateColsIfNotMerged("status") if err != nil { log.Error("AddToTaskQueue.UpdateCols[%d].(add to queue): %v", pr.ID, err) } else { diff --git a/services/pull/pull.go b/services/pull/pull.go index e8bd2339ea1aa..5db7fcf3ac3dd 100644 --- a/services/pull/pull.go +++ b/services/pull/pull.go @@ -244,6 +244,15 @@ func AddTestPullRequestTask(doer *models.User, repoID int64, branch string, isSy return } for _, pr := range prs { + divergence, err := GetDiverging(pr) + if err != nil { + log.Error("GetDiverging: %v", err) + } else { + err = pr.UpdateCommitDivergence(divergence) + if err != nil { + log.Error("UpdateCommitDivergence: %v", err) + } + } AddToTaskQueue(pr) } }) From 1f7effec03768034c5009367f9b67a1bb4f5a41d Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sat, 4 Apr 2020 00:23:29 +0200 Subject: [PATCH 03/15] migrate already open pulls --- models/migrations/v999.go | 47 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/models/migrations/v999.go b/models/migrations/v999.go index d1b3cb91f8059..dff07568e6cdf 100644 --- a/models/migrations/v999.go +++ b/models/migrations/v999.go @@ -5,6 +5,12 @@ package migrations import ( + "fmt" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/setting" + pull_service "code.gitea.io/gitea/services/pull" + "xorm.io/xorm" ) @@ -13,5 +19,44 @@ func addCommitDivergenceToPulls(x *xorm.Engine) error { CommitsAhead int CommitsBehind int } - return x.Sync2(new(PullRequest)) + if err := x.Sync2(new(PullRequest)); err != nil { + return fmt.Errorf("Sync2: %v", err) + } + + var last int + batchSize := setting.Database.IterateBufferSize + sess := x.NewSession() + defer sess.Close() + for { + if err := sess.Begin(); err != nil { + return err + } + var results = make([]*models.PullRequest, 0, batchSize) + err := sess.Where("has_merged = ?", false).OrderBy("id").Limit(batchSize, last).Find(&results) + if err != nil { + return err + } + if len(results) == 0 { + break + } + last += len(results) + + for _, pr := range results { + divergence, err := pull_service.GetDiverging(pr) + if err != nil { + return fmt.Errorf("GetDiverging: %v", err) + } else { + pr.CommitsAhead = divergence.Ahead + pr.CommitsBehind = divergence.Behind + if _, err = sess.ID(pr.ID).Cols("commits_ahead", "commits_behind").Update(pr); err != nil { + return fmt.Errorf("Update Cols: %v", err) + } + } + } + + if err := sess.Commit(); err != nil { + return err + } + } + return nil } From 846c9243fc64c38543193451f439abfc0d25b8e7 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Wed, 1 Apr 2020 13:19:36 +0200 Subject: [PATCH 04/15] finalize --- models/migrations/migrations.go | 3 +-- models/migrations/{v999.go => v136.go} | 0 2 files changed, 1 insertion(+), 2 deletions(-) rename models/migrations/{v999.go => v136.go} (100%) diff --git a/models/migrations/migrations.go b/models/migrations/migrations.go index 242052266d0e4..8731bc1d80050 100644 --- a/models/migrations/migrations.go +++ b/models/migrations/migrations.go @@ -204,8 +204,7 @@ var migrations = []Migration{ NewMigration("Refix merge base for merged pull requests", refixMergeBase), // v135 -> 136 NewMigration("Add OrgID column to Labels table", addOrgIDLabelColumn), - - // v999 not finalized + // v136 -> 137 NewMigration("Add CommitsAhead and CommitsBehind Column to PullRequest Table", addCommitDivergenceToPulls), } diff --git a/models/migrations/v999.go b/models/migrations/v136.go similarity index 100% rename from models/migrations/v999.go rename to models/migrations/v136.go From 870cda8fcd1c283a3243eab1200e069482f2ff0e Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sat, 4 Apr 2020 00:47:38 +0200 Subject: [PATCH 05/15] take care of closed¬-merged+deleted-branch pull requests --- models/migrations/v136.go | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/models/migrations/v136.go b/models/migrations/v136.go index dff07568e6cdf..b11f576334bab 100644 --- a/models/migrations/v136.go +++ b/models/migrations/v136.go @@ -8,6 +8,7 @@ import ( "fmt" "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" pull_service "code.gitea.io/gitea/services/pull" @@ -44,13 +45,21 @@ func addCommitDivergenceToPulls(x *xorm.Engine) error { for _, pr := range results { divergence, err := pull_service.GetDiverging(pr) if err != nil { - return fmt.Errorf("GetDiverging: %v", err) - } else { - pr.CommitsAhead = divergence.Ahead - pr.CommitsBehind = divergence.Behind - if _, err = sess.ID(pr.ID).Cols("commits_ahead", "commits_behind").Update(pr); err != nil { - return fmt.Errorf("Update Cols: %v", err) + if err = pr.LoadIssue(); err != nil { + return fmt.Errorf("pr.LoadIssue()[%d]: %v", pr.ID, err) } + if !pr.Issue.IsClosed { + return fmt.Errorf("GetDiverging: %v", err) + } + log.Error("Could not recalculate Divergence for pull: %d", pr.ID) + pr.CommitsAhead = 0 + pr.CommitsBehind = 0 + } + + pr.CommitsAhead = divergence.Ahead + pr.CommitsBehind = divergence.Behind + if _, err = sess.ID(pr.ID).Cols("commits_ahead", "commits_behind").Update(pr); err != nil { + return fmt.Errorf("Update Cols: %v", err) } } From cd342867370efa5279df8aadf2517bebf85e8e4f Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sat, 4 Apr 2020 00:54:39 +0200 Subject: [PATCH 06/15] fix nil pointer exeption Signed-off-by: 6543 <6543@obermui.de> --- models/migrations/v136.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/models/migrations/v136.go b/models/migrations/v136.go index b11f576334bab..5631688e8edac 100644 --- a/models/migrations/v136.go +++ b/models/migrations/v136.go @@ -55,9 +55,10 @@ func addCommitDivergenceToPulls(x *xorm.Engine) error { pr.CommitsAhead = 0 pr.CommitsBehind = 0 } - - pr.CommitsAhead = divergence.Ahead - pr.CommitsBehind = divergence.Behind + if divergence != nil { + pr.CommitsAhead = divergence.Ahead + pr.CommitsBehind = divergence.Behind + } if _, err = sess.ID(pr.ID).Cols("commits_ahead", "commits_behind").Update(pr); err != nil { return fmt.Errorf("Update Cols: %v", err) } From 6e27c62757a906a81fd1262a6386fc13363fd8e1 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sat, 4 Apr 2020 01:12:48 +0200 Subject: [PATCH 07/15] try this --- models/migrations/v136.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/models/migrations/v136.go b/models/migrations/v136.go index 5631688e8edac..d3def805d6e0d 100644 --- a/models/migrations/v136.go +++ b/models/migrations/v136.go @@ -16,11 +16,8 @@ import ( ) func addCommitDivergenceToPulls(x *xorm.Engine) error { - type PullRequest struct { - CommitsAhead int - CommitsBehind int - } - if err := x.Sync2(new(PullRequest)); err != nil { + + if err := x.Sync2(new(models.PullRequest)); err != nil { return fmt.Errorf("Sync2: %v", err) } From 235f1e4b0d1257f7df0d4be0b47ea00fe77e643b Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sat, 4 Apr 2020 01:24:14 +0200 Subject: [PATCH 08/15] no error its a warn --- models/migrations/v136.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/migrations/v136.go b/models/migrations/v136.go index d3def805d6e0d..115861f344cce 100644 --- a/models/migrations/v136.go +++ b/models/migrations/v136.go @@ -48,7 +48,7 @@ func addCommitDivergenceToPulls(x *xorm.Engine) error { if !pr.Issue.IsClosed { return fmt.Errorf("GetDiverging: %v", err) } - log.Error("Could not recalculate Divergence for pull: %d", pr.ID) + log.Warn("Could not recalculate Divergence for pull: %d", pr.ID) pr.CommitsAhead = 0 pr.CommitsBehind = 0 } From 39e12baf9eb6456b868e229e8cb375b2527db747 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Wed, 8 Apr 2020 04:49:39 +0200 Subject: [PATCH 09/15] init gitea-repositories-meta --- integrations/migration-test/migration_test.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/integrations/migration-test/migration_test.go b/integrations/migration-test/migration_test.go index a4d01efef3c18..4ee045db38348 100644 --- a/integrations/migration-test/migration_test.go +++ b/integrations/migration-test/migration_test.go @@ -12,6 +12,7 @@ import ( "io/ioutil" "os" "path" + "path/filepath" "regexp" "sort" "strings" @@ -25,6 +26,7 @@ import ( "code.gitea.io/gitea/modules/setting" "github.com/stretchr/testify/assert" + "github.com/unknwon/com" "xorm.io/xorm" ) @@ -54,6 +56,11 @@ func initMigrationTest(t *testing.T) func() { } setting.NewContext() + + assert.True(t, len(setting.RepoRootPath) != 0) + assert.NoError(t, os.RemoveAll(setting.RepoRootPath)) + assert.NoError(t, com.CopyDir(path.Join(filepath.Dir(setting.AppPath), "integrations/gitea-repositories-meta"), setting.RepoRootPath)) + setting.CheckLFSVersion() setting.InitDBConfig() setting.NewLogServices(true) From 0f12098b173fb6990aa2f9a76468adbb8ab40619 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Thu, 9 Apr 2020 13:29:35 +0200 Subject: [PATCH 10/15] dont use gitDivergence type --- models/pull.go | 19 +++++++------------ routers/repo/pull.go | 3 +-- services/pull/pull.go | 4 ++-- templates/repo/issue/view_content/pull.tmpl | 2 +- 4 files changed, 11 insertions(+), 17 deletions(-) diff --git a/models/pull.go b/models/pull.go index acdd4df2392ce..2d51deeb4721a 100644 --- a/models/pull.go +++ b/models/pull.go @@ -354,22 +354,17 @@ func (pr *PullRequest) GetCommitMessages() string { return stringBuilder.String() } -// GetCommitDivergence get Divergence of a pull request -func (pr *PullRequest) GetCommitDivergence() *git.DivergeObject { - return &git.DivergeObject{Ahead: pr.CommitsAhead, Behind: pr.CommitsBehind} -} - // UpdateCommitDivergence update Divergence of a pull request -func (pr *PullRequest) UpdateCommitDivergence(div *git.DivergeObject) error { - return pr.updateCommitDivergence(x, div) +func (pr *PullRequest) UpdateCommitDivergence(ahead, behind int) error { + return pr.updateCommitDivergence(x, ahead, behind) } -func (pr *PullRequest) updateCommitDivergence(e Engine, div *git.DivergeObject) error { - if div == nil || pr.ID == 0 { - return fmt.Errorf("updateCommitDivergence: diverge nil or PR-ID 0") +func (pr *PullRequest) updateCommitDivergence(e Engine, ahead, behind int) error { + if pr.ID == 0 { + return fmt.Errorf("pull ID is 0") } - pr.CommitsAhead = div.Ahead - pr.CommitsBehind = div.Behind + pr.CommitsAhead = ahead + pr.CommitsBehind = behind _, err := e.ID(pr.ID).Cols("commits_ahead", "commits_behind").Update(pr) return err } diff --git a/routers/repo/pull.go b/routers/repo/pull.go index 32f5c697a7d70..86359a6f28414 100644 --- a/routers/repo/pull.go +++ b/routers/repo/pull.go @@ -419,14 +419,13 @@ func PrepareViewPullInfo(ctx *context.Context, issue *models.Issue) *git.Compare } } - if headBranchExist { + if headBranchExist && pull.CommitsBehind > 0 { allowUpdate, err := pull_service.IsUserAllowedToUpdate(pull, ctx.User) if err != nil { ctx.ServerError("IsUserAllowedToUpdate", err) return nil } ctx.Data["UpdateAllowed"] = allowUpdate - ctx.Data["Divergence"] = pull.GetCommitDivergence() } sha, err := baseGitRepo.GetRefCommitID(pull.GetGitRefName()) diff --git a/services/pull/pull.go b/services/pull/pull.go index 5db7fcf3ac3dd..18bc113170172 100644 --- a/services/pull/pull.go +++ b/services/pull/pull.go @@ -222,7 +222,7 @@ func AddTestPullRequestTask(doer *models.User, repoID int64, branch string, isSy if err != nil { log.Error("GetDiverging: %v", err) } else { - err = pr.UpdateCommitDivergence(divergence) + err = pr.UpdateCommitDivergence(divergence.Ahead, divergence.Behind) if err != nil { log.Error("UpdateCommitDivergence: %v", err) } @@ -248,7 +248,7 @@ func AddTestPullRequestTask(doer *models.User, repoID int64, branch string, isSy if err != nil { log.Error("GetDiverging: %v", err) } else { - err = pr.UpdateCommitDivergence(divergence) + err = pr.UpdateCommitDivergence(divergence.Ahead, divergence.Behind) if err != nil { log.Error("UpdateCommitDivergence: %v", err) } diff --git a/templates/repo/issue/view_content/pull.tmpl b/templates/repo/issue/view_content/pull.tmpl index 09c7cd1d9f465..a44c0f15cb597 100644 --- a/templates/repo/issue/view_content/pull.tmpl +++ b/templates/repo/issue/view_content/pull.tmpl @@ -306,7 +306,7 @@ {{end}} {{end}} - {{if and .Divergence (gt .Divergence.Behind 0)}} + {{if gt .Issue.PullRequest.CommitsBehind 0}}
From fe503efd1948a5b45587159e2631b1320df60865 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Thu, 9 Apr 2020 14:17:50 +0200 Subject: [PATCH 11/15] CI.restart() From b65da306afc395a307f65f9f8e269fa14f64d87f Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Thu, 9 Apr 2020 18:53:54 +0200 Subject: [PATCH 12/15] CI.restart() From e8d67208c0f6fdefb6917cb91b5dd58003a5e1eb Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Mon, 13 Apr 2020 04:33:22 +0200 Subject: [PATCH 13/15] CI.restart() From 0894c42b26f9082ee8d11ac73ca34df257b61d7e Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Mon, 13 Apr 2020 05:16:05 +0200 Subject: [PATCH 14/15] CI.restart() From 13440b2474798985338f8d02a88e63ce8bd61e28 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Tue, 14 Apr 2020 14:34:41 +0200 Subject: [PATCH 15/15] check IsUserAllowedToUpdate independend from CommitsBehind --- routers/repo/pull.go | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/routers/repo/pull.go b/routers/repo/pull.go index 361ef59e0831b..00195fd02a1e3 100644 --- a/routers/repo/pull.go +++ b/routers/repo/pull.go @@ -418,12 +418,10 @@ func PrepareViewPullInfo(ctx *context.Context, issue *models.Issue) *git.Compare } if headBranchExist { - if pull.CommitsBehind > 0 { - ctx.Data["UpdateAllowed"], err = pull_service.IsUserAllowedToUpdate(pull, ctx.User) - if err != nil { - ctx.ServerError("IsUserAllowedToUpdate", err) - return nil - } + ctx.Data["UpdateAllowed"], err = pull_service.IsUserAllowedToUpdate(pull, ctx.User) + if err != nil { + ctx.ServerError("IsUserAllowedToUpdate", err) + return nil } ctx.Data["GetCommitMessages"] = pull_service.GetCommitMessages(pull) }