From df49449de5b80ab8695506cee8d6e92804eb7145 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sat, 4 Feb 2023 10:52:06 +0800 Subject: [PATCH 01/56] sync branches into databases --- models/git/branches.go | 33 ++++++++++++++ modules/repository/branch.go | 83 ++++++++++++++++++++++++++++++++++++ 2 files changed, 116 insertions(+) create mode 100644 modules/repository/branch.go diff --git a/models/git/branches.go b/models/git/branches.go index b94ea329599a1..3b2c3b706ff93 100644 --- a/models/git/branches.go +++ b/models/git/branches.go @@ -15,6 +15,16 @@ import ( "code.gitea.io/gitea/modules/timeutil" ) +// Branch represents a branch of a repository +// For those repository who have many branches, stored into database is a good choice +// for pagination and search +type Branch struct { + ID int64 + RepoID int64 `xorm:"index UNIQUE(s)"` + Name string `xorm:"UNIQUE(s) NOT NULL"` + CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"` +} + // DeletedBranch struct type DeletedBranch struct { ID int64 `xorm:"pk autoincr"` @@ -27,10 +37,33 @@ type DeletedBranch struct { } func init() { + db.RegisterModel(new(Branch)) db.RegisterModel(new(DeletedBranch)) db.RegisterModel(new(RenamedBranch)) } +func LoadAllBranches(ctx context.Context, repoID int64) ([]*Branch, error) { + var branches []*Branch + err := db.GetEngine(ctx).Find(&branches) + return branches, err +} + +func AddBranches(ctx context.Context, repoID int64, branches []string) error { + var dbBranches []*Branch + for _, branch := range branches { + dbBranches = append(dbBranches, &Branch{ + RepoID: repoID, + Name: branch, + }) + } + return db.Insert(ctx, dbBranches) +} + +func DeleteBranches(ctx context.Context, repoID int64, branches []int64) error { + _, err := db.GetEngine(ctx).Where("repo_id=?", repoID).In("id", branches).Delete(new(Branch)) + return err +} + // AddDeletedBranch adds a deleted branch to the database func AddDeletedBranch(ctx context.Context, repoID int64, branchName, commit string, deletedByID int64) error { deletedBranch := &DeletedBranch{ diff --git a/modules/repository/branch.go b/modules/repository/branch.go new file mode 100644 index 0000000000000..e3dbd52e2d3f9 --- /dev/null +++ b/modules/repository/branch.go @@ -0,0 +1,83 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package repository + +import ( + "context" + + "code.gitea.io/gitea/models/db" + git_model "code.gitea.io/gitea/models/git" + repo_model "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/log" +) + +// SyncBranches synchronizes branch table with repository branches +func SyncBranches(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository) error { + log.Debug("SyncBranches: in Repo[%d:%s/%s]", repo.ID, repo.OwnerName, repo.Name) + + const limit = 100 + var allBranches []string + for page := 1; ; page++ { + branches, _, err := gitRepo.GetBranchNames(page*limit, limit) + if err != nil { + return err + } + if len(branches) == 0 { + break + } + allBranches = append(allBranches, branches...) + } + + dbBranches, err := git_model.LoadAllBranches(ctx, repo.ID) + if err != nil { + return err + } + + var toAdd []string + var toRemove []int64 + for _, branch := range allBranches { + var found bool + for _, dbBranch := range dbBranches { + if branch == dbBranch.Name { + found = true + break + } + } + if !found { + toAdd = append(toAdd, branch) + } + } + + for _, dbBranch := range dbBranches { + var found bool + for _, branch := range allBranches { + if branch == dbBranch.Name { + found = true + break + } + } + if !found { + toRemove = append(toRemove, dbBranch.ID) + } + } + + return db.WithTx(ctx, func(ctx context.Context) error { + if len(toAdd) > 0 { + err = git_model.AddBranches(ctx, repo.ID, toAdd) + if err != nil { + return err + } + } + + if len(toRemove) > 0 { + err = git_model.DeleteBranches(ctx, repo.ID, toRemove) + if err != nil { + return err + } + } + + return nil + }) +} From a495e754cd70b5c928905db9934c07bfbe146539 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sun, 9 Apr 2023 13:19:32 +0800 Subject: [PATCH 02/56] add migration --- cmd/admin.go | 58 +++++++++++++++++ models/git/branches.go | 112 ++++++++++++++++++++++++++------ models/migrations/migrations.go | 6 ++ models/migrations/v1_20/v241.go | 25 +++++++ modules/repository/branch.go | 4 +- services/repository/branch.go | 9 --- services/repository/push.go | 8 ++- 7 files changed, 188 insertions(+), 34 deletions(-) create mode 100644 models/migrations/v1_20/v241.go diff --git a/cmd/admin.go b/cmd/admin.go index 8f8c68f9810c9..e64640aa5c50e 100644 --- a/cmd/admin.go +++ b/cmd/admin.go @@ -42,6 +42,7 @@ var ( Subcommands: []cli.Command{ subcmdUser, subcmdRepoSyncReleases, + subcmdRepoSyncBranches, subcmdRegenerate, subcmdAuth, subcmdSendMail, @@ -195,6 +196,12 @@ var ( Action: runRepoSyncReleases, } + subcmdRepoSyncBranches = cli.Command{ + Name: "repo-sync-branches", + Usage: "Synchronize repository branches", + Action: runRepoSyncBranches, + } + subcmdRegenerate = cli.Command{ Name: "regenerate", Usage: "Regenerate specific files", @@ -787,6 +794,57 @@ func runRepoSyncReleases(_ *cli.Context) error { return nil } +func runRepoSyncBranches(_ *cli.Context) error { + ctx, cancel := installSignals() + defer cancel() + + if err := initDB(ctx); err != nil { + return err + } + + doer, err := user_model.GetAdminUser() + if err != nil { + return err + } + + log.Trace("Synchronizing repository branches (this may take a while)") + for page := 1; ; page++ { + repos, count, err := repo_model.SearchRepositoryByName(ctx, &repo_model.SearchRepoOptions{ + ListOptions: db.ListOptions{ + PageSize: repo_model.RepositoryListDefaultPageSize, + Page: page, + }, + Private: true, + }) + if err != nil { + return fmt.Errorf("SearchRepositoryByName: %w", err) + } + if len(repos) == 0 { + break + } + log.Trace("Processing next %d repos of %d", len(repos), count) + for _, repo := range repos { + log.Trace("Synchronizing repo %s with path %s", repo.FullName(), repo.RepoPath()) + gitRepo, err := git.OpenRepository(ctx, repo.RepoPath()) + if err != nil { + log.Warn("OpenRepository: %v", err) + continue + } + + if err = repo_module.SyncBranches(ctx, repo, doer.ID, gitRepo); err != nil { + log.Warn("repo_module.SyncBranches: %v", err) + gitRepo.Close() + continue + } + + log.Trace("repo %s branches synchronized") + gitRepo.Close() + } + } + + return nil +} + func getReleaseCount(id int64) (int64, error) { return repo_model.GetReleaseCountByRepoID( db.DefaultContext, diff --git a/models/git/branches.go b/models/git/branches.go index 3b2c3b706ff93..5f366123ebcfa 100644 --- a/models/git/branches.go +++ b/models/git/branches.go @@ -20,9 +20,13 @@ import ( // for pagination and search type Branch struct { ID int64 - RepoID int64 `xorm:"index UNIQUE(s)"` - Name string `xorm:"UNIQUE(s) NOT NULL"` - CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"` + RepoID int64 `xorm:"index UNIQUE(s)"` + Name string `xorm:"UNIQUE(s) NOT NULL"` + Commit string + PusherID int64 + CommitTime timeutil.TimeStamp // The commit + CreatedUnix timeutil.TimeStamp `xorm:"created"` + UpdatedUnix timeutil.TimeStamp `xorm:"updated"` } // DeletedBranch struct @@ -59,22 +63,67 @@ func AddBranches(ctx context.Context, repoID int64, branches []string) error { return db.Insert(ctx, dbBranches) } -func DeleteBranches(ctx context.Context, repoID int64, branches []int64) error { - _, err := db.GetEngine(ctx).Where("repo_id=?", repoID).In("id", branches).Delete(new(Branch)) - return err +func DeleteBranches(ctx context.Context, repoID, doerID int64, branchIDs []int64) error { + return db.WithTx(ctx, func(ctx context.Context) error { + branches := make([]*Branch, 0, len(branchIDs)) + if err := db.GetEngine(ctx).In("id", branchIDs).Find(&branches); err != nil { + return err + } + for _, branch := range branches { + if err := AddDeletedBranch(ctx, repoID, branch.Name, branch.Commit, doerID); err != nil { + return err + } + } + _, err := db.GetEngine(ctx).In("id", branchIDs).Delete(new(Branch)) + return err + }) +} + +// UpdateBranch updates the branch information in the database. If the branch exist, it will update latest commit of this branch information +// If it doest not exist, insert a new record into database +func UpdateBranch(ctx context.Context, repoID int64, branchName string, commitID string, pusherID int64, commitTime time.Time) error { + if err := removeDeletedBranchByName(ctx, repoID, branchName); err != nil { + return err + } + cnt, err := db.GetEngine(ctx).Where("repo_id=? AND name=?", repoID, branchName). + Cols("commit, pusher_id, commit_time, updated_unix"). + Update(&Branch{ + Commit: commitID, + PusherID: pusherID, + CommitTime: timeutil.TimeStamp(commitTime.Unix()), + }) + if err != nil { + return err + } + if cnt > 0 { + return nil + } + + return db.Insert(ctx, &Branch{ + RepoID: repoID, + Name: branchName, + Commit: commitID, + PusherID: pusherID, + CommitTime: timeutil.TimeStamp(commitTime.Unix()), + }) } // AddDeletedBranch adds a deleted branch to the database func AddDeletedBranch(ctx context.Context, repoID int64, branchName, commit string, deletedByID int64) error { - deletedBranch := &DeletedBranch{ - RepoID: repoID, - Name: branchName, - Commit: commit, - DeletedByID: deletedByID, - } + return db.WithTx(ctx, func(ctx context.Context) error { + if _, err := db.GetEngine(ctx).Where("repo_id = ? AND name = ?", repoID, branchName).Delete(new(Branch)); err != nil { + return err + } - _, err := db.GetEngine(ctx).Insert(deletedBranch) - return err + deletedBranch := &DeletedBranch{ + RepoID: repoID, + Name: branchName, + Commit: commit, + DeletedByID: deletedByID, + } + _, err := db.GetEngine(ctx).Insert(deletedBranch) + return err + }) } // GetDeletedBranches returns all the deleted branches @@ -122,8 +171,8 @@ func (deletedBranch *DeletedBranch) LoadUser(ctx context.Context) { deletedBranch.DeletedBy = user } -// RemoveDeletedBranchByName removes all deleted branches -func RemoveDeletedBranchByName(ctx context.Context, repoID int64, branch string) error { +// removeDeletedBranchByName removes all deleted branches +func removeDeletedBranchByName(ctx context.Context, repoID int64, branch string) error { _, err := db.GetEngine(ctx).Where("repo_id=? AND name=?", repoID, branch).Delete(new(DeletedBranch)) return err } @@ -170,7 +219,26 @@ func RenameBranch(ctx context.Context, repo *repo_model.Repository, from, to str defer committer.Close() sess := db.GetEngine(ctx) - // 1. update default branch if needed + + // 1. update deleted branch + if _, err := sess.Where("repo_id = ? AND name=?", repo.ID, from).Update(&DeletedBranch{ + RepoID: repo.ID, + Name: to, + }); err != nil { + return err + } + + // 2. update branch in database + if n, err := sess.Where("repo_id=? AND name=?", repo.ID, from).Update(&Branch{ + Name: to, + }); err != nil { + return err + } else if n <= 0 { + // branch does not exist in the database, so we think branch is not existed + return nil + } + + // 3. update default branch if needed isDefault := repo.DefaultBranch == from if isDefault { repo.DefaultBranch = to @@ -180,19 +248,21 @@ func RenameBranch(ctx context.Context, repo *repo_model.Repository, from, to str } } - // 2. Update protected branch if needed + // 4. Update protected branch if needed protectedBranch, err := GetProtectedBranchRuleByName(ctx, repo.ID, from) if err != nil { return err } if protectedBranch != nil { + // there is a protect rule for this branch protectedBranch.RuleName = to _, err = sess.ID(protectedBranch.ID).Cols("branch_name").Update(protectedBranch) if err != nil { return err } } else { + // some glob protect rules may match this branch protected, err := IsBranchProtected(ctx, repo.ID, from) if err != nil { return err @@ -202,7 +272,7 @@ func RenameBranch(ctx context.Context, repo *repo_model.Repository, from, to str } } - // 3. Update all not merged pull request base branch name + // 5. Update all not merged pull request base branch name _, err = sess.Table("pull_request").Where("base_repo_id=? AND base_branch=? AND has_merged=?", repo.ID, from, false). Update(map[string]interface{}{"base_branch": to}) @@ -210,12 +280,12 @@ func RenameBranch(ctx context.Context, repo *repo_model.Repository, from, to str return err } - // 4. do git action + // 6. do git action if err = gitAction(isDefault); err != nil { return err } - // 5. insert renamed branch record + // 7. insert renamed branch record renamedBranch := &RenamedBranch{ RepoID: repo.ID, From: from, diff --git a/models/migrations/migrations.go b/models/migrations/migrations.go index 15600f057cfb1..661a8d966a2a6 100644 --- a/models/migrations/migrations.go +++ b/models/migrations/migrations.go @@ -19,6 +19,7 @@ import ( "code.gitea.io/gitea/models/migrations/v1_17" "code.gitea.io/gitea/models/migrations/v1_18" "code.gitea.io/gitea/models/migrations/v1_19" + "code.gitea.io/gitea/models/migrations/v1_20" "code.gitea.io/gitea/models/migrations/v1_6" "code.gitea.io/gitea/models/migrations/v1_7" "code.gitea.io/gitea/models/migrations/v1_8" @@ -455,6 +456,11 @@ var migrations = []Migration{ NewMigration("Add scope for access_token", v1_19.AddScopeForAccessTokens), // v240 -> v241 NewMigration("Add actions tables", v1_19.AddActionsTables), + + // Gitea 1.19.0 ends at v241 + + // v241 -> v242 + NewMigration("Add branch table", v1_20.AddBranchTable), } // GetCurrentDBVersion returns the current db version diff --git a/models/migrations/v1_20/v241.go b/models/migrations/v1_20/v241.go new file mode 100644 index 0000000000000..128f8e57edfae --- /dev/null +++ b/models/migrations/v1_20/v241.go @@ -0,0 +1,25 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package v1_20 //nolint + +import ( + "code.gitea.io/gitea/modules/timeutil" + + "xorm.io/xorm" +) + +func AddBranchTable(x *xorm.Engine) error { + type Branch struct { + ID int64 + RepoID int64 `xorm:"index UNIQUE(s)"` + Name string `xorm:"UNIQUE(s) NOT NULL"` + Commit string + PusherID int64 + CommitTime timeutil.TimeStamp // The commit + CreatedUnix timeutil.TimeStamp `xorm:"created"` + UpdatedUnix timeutil.TimeStamp `xorm:"updated"` + } + + return x.Sync(new(Branch)) +} diff --git a/modules/repository/branch.go b/modules/repository/branch.go index e3dbd52e2d3f9..957e41e2504f6 100644 --- a/modules/repository/branch.go +++ b/modules/repository/branch.go @@ -14,7 +14,7 @@ import ( ) // SyncBranches synchronizes branch table with repository branches -func SyncBranches(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository) error { +func SyncBranches(ctx context.Context, repo *repo_model.Repository, doerID int64, gitRepo *git.Repository) error { log.Debug("SyncBranches: in Repo[%d:%s/%s]", repo.ID, repo.OwnerName, repo.Name) const limit = 100 @@ -72,7 +72,7 @@ func SyncBranches(ctx context.Context, repo *repo_model.Repository, gitRepo *git } if len(toRemove) > 0 { - err = git_model.DeleteBranches(ctx, repo.ID, toRemove) + err = git_model.DeleteBranches(ctx, repo.ID, doerID, toRemove) if err != nil { return err } diff --git a/services/repository/branch.go b/services/repository/branch.go index 291fb4a92b374..10aea17f7dc1b 100644 --- a/services/repository/branch.go +++ b/services/repository/branch.go @@ -18,7 +18,6 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/notification" repo_module "code.gitea.io/gitea/modules/repository" - pull_service "code.gitea.io/gitea/services/pull" ) // CreateNewBranch creates a new repository branch @@ -177,10 +176,6 @@ func DeleteBranch(doer *user_model.User, repo *repo_model.Repository, gitRepo *g return err } - if err := pull_service.CloseBranchPulls(doer, repo.ID, branchName); err != nil { - return err - } - // Don't return error below this if err := PushUpdate( &repo_module.PushUpdateOptions{ @@ -195,9 +190,5 @@ func DeleteBranch(doer *user_model.User, repo *repo_model.Repository, gitRepo *g log.Error("Update: %v", err) } - if err := git_model.AddDeletedBranch(db.DefaultContext, repo.ID, branchName, commit.ID.String(), doer.ID); err != nil { - log.Warn("AddDeletedBranch: %v", err) - } - return nil } diff --git a/services/repository/push.go b/services/repository/push.go index ef6460cef4231..82493072b8a29 100644 --- a/services/repository/push.go +++ b/services/repository/push.go @@ -261,8 +261,8 @@ func pushUpdates(optsList []*repo_module.PushUpdateOptions) error { notification.NotifyPushCommits(db.DefaultContext, pusher, repo, opts, commits) - if err = git_model.RemoveDeletedBranchByName(ctx, repo.ID, branch); err != nil { - log.Error("models.RemoveDeletedBranch %s/%s failed: %v", repo.ID, branch, err) + if err = git_model.UpdateBranch(ctx, repo.ID, branch, newCommit.ID.String(), opts.PusherID, newCommit.Committer.When); err != nil { + log.Error("git_model.UpdateBranch %s/%s failed: %v", repo.ID, branch, err) } // Cache for big repository @@ -275,6 +275,10 @@ func pushUpdates(optsList []*repo_module.PushUpdateOptions) error { // close all related pulls log.Error("close related pull request failed: %v", err) } + + if err := git_model.AddDeletedBranch(db.DefaultContext, repo.ID, branch, opts.OldCommitID, pusher.ID); err != nil { + log.Warn("AddDeletedBranch: %v", err) + } } // Even if user delete a branch on a repository which he didn't watch, he will be watch that. From 0a8cbc9a33bf44a13e3582d22684fac950bf92bd Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sun, 9 Apr 2023 15:10:01 +0800 Subject: [PATCH 03/56] some improvements --- cmd/admin.go | 2 +- models/git/branches.go | 36 +++++++++--- modules/repository/branch.go | 98 -------------------------------- services/repository/branch.go | 102 ++++++++++++++++++++++++++++++++++ 4 files changed, 132 insertions(+), 106 deletions(-) delete mode 100644 modules/repository/branch.go diff --git a/cmd/admin.go b/cmd/admin.go index f3a5b35a260b0..3f2b49511b30b 100644 --- a/cmd/admin.go +++ b/cmd/admin.go @@ -445,7 +445,7 @@ func runRepoSyncBranches(_ *cli.Context) error { continue } - if err = repo_module.SyncBranches(ctx, repo, doer.ID, gitRepo); err != nil { + if err = repo_service.SyncRepoBranches(ctx, repo, doer.ID, gitRepo); err != nil { log.Warn("repo_module.SyncBranches: %v", err) gitRepo.Close() continue diff --git a/models/git/branches.go b/models/git/branches.go index ac08a04091b73..cff7d8f11e97e 100644 --- a/models/git/branches.go +++ b/models/git/branches.go @@ -52,15 +52,37 @@ func LoadAllBranches(ctx context.Context, repoID int64) ([]*Branch, error) { return branches, err } -func AddBranches(ctx context.Context, repoID int64, branches []string) error { - var dbBranches []*Branch +type FindBranchOptions struct { + db.ListOptions + RepoID int64 +} + +func FindBranches(ctx context.Context, opts FindBranchOptions) ([]*Branch, error) { + sess := db.GetEngine(ctx).Where("repo_id=?", opts.RepoID) + if opts.PageSize > 0 { + sess = db.SetSessionPagination(sess, &opts.ListOptions) + } + var branches []*Branch + return branches, sess.Find(&branches) +} + +func AddBranch(ctx context.Context, branch *Branch) error { + return db.WithTx(ctx, func(ctx context.Context) error { + if _, err := db.GetEngine(ctx).Insert(branch); err != nil { + return err + } + + return removeDeletedBranchByName(ctx, branch.RepoID, branch.Name) + }) +} + +func AddBranches(ctx context.Context, branches []*Branch) error { for _, branch := range branches { - dbBranches = append(dbBranches, &Branch{ - RepoID: repoID, - Name: branch, - }) + if err := AddBranch(ctx, branch); err != nil { + return err + } } - return db.Insert(ctx, dbBranches) + return nil } func DeleteBranches(ctx context.Context, repoID, doerID int64, branchIDs []int64) error { diff --git a/modules/repository/branch.go b/modules/repository/branch.go deleted file mode 100644 index 0a23763ff26f6..0000000000000 --- a/modules/repository/branch.go +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright 2023 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package repository - -import ( - "context" - - "code.gitea.io/gitea/models/db" - git_model "code.gitea.io/gitea/models/git" - repo_model "code.gitea.io/gitea/models/repo" - "code.gitea.io/gitea/modules/git" - "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/timeutil" -) - -// SyncBranches synchronizes branch table with repository branches -func SyncBranches(ctx context.Context, repo *repo_model.Repository, doerID int64, gitRepo *git.Repository) error { - log.Debug("SyncBranches: in Repo[%d:%s/%s]", repo.ID, repo.OwnerName, repo.Name) - - const limit = 100 - var allBranches []string - for page := 1; ; page++ { - branches, _, err := gitRepo.GetBranchNames(page*limit, limit) - if err != nil { - return err - } - if len(branches) == 0 { - break - } - allBranches = append(allBranches, branches...) - } - - dbBranches, err := git_model.LoadAllBranches(ctx, repo.ID) - if err != nil { - return err - } - - var toAdd []*git_model.Branch - var toRemove []int64 - for _, branch := range allBranches { - var found bool - for _, dbBranch := range dbBranches { - if branch == dbBranch.Name { - found = true - break - } - } - if !found { - commit, err := gitRepo.GetBranchCommit(branch) - if err != nil { - return err - } - toAdd = append(toAdd, &git_model.Branch{ - RepoID: repo.ID, - Name: branch, - Commit: commit.ID.String(), - PusherID: doerID, - CommitTime: timeutil.TimeStamp(commit.Author.When.Unix()), - }) - } - } - - for _, dbBranch := range dbBranches { - var found bool - for _, branch := range allBranches { - if branch == dbBranch.Name { - found = true - break - } - } - if !found { - toRemove = append(toRemove, dbBranch.ID) - } - } - - if len(toAdd) <= 0 && len(toRemove) <= 0 { - return nil - } - - return db.WithTx(ctx, func(ctx context.Context) error { - if len(toAdd) > 0 { - err = db.Insert(ctx, toAdd) - if err != nil { - return err - } - } - - if len(toRemove) > 0 { - err = git_model.DeleteBranches(ctx, repo.ID, doerID, toRemove) - if err != nil { - return err - } - } - - return nil - }) -} diff --git a/services/repository/branch.go b/services/repository/branch.go index 60cbc7b94ae1d..26da481e45ab8 100644 --- a/services/repository/branch.go +++ b/services/repository/branch.go @@ -10,6 +10,7 @@ import ( "strings" "code.gitea.io/gitea/models" + "code.gitea.io/gitea/models/db" git_model "code.gitea.io/gitea/models/git" repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" @@ -17,6 +18,7 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/notification" repo_module "code.gitea.io/gitea/modules/repository" + "code.gitea.io/gitea/modules/timeutil" ) // CreateNewBranch creates a new repository branch @@ -191,3 +193,103 @@ func DeleteBranch(ctx context.Context, doer *user_model.User, repo *repo_model.R return nil } + +// SyncRepoBranches synchronizes branch table with repository branches +func SyncRepoBranches(ctx context.Context, repo *repo_model.Repository, doerID int64, gitRepo *git.Repository) error { + log.Debug("SyncRepoBranches: in Repo[%d:%s/%s]", repo.ID, repo.OwnerName, repo.Name) + + const limit = 100 + var allBranches []string + for page := 1; ; page++ { + branches, _, err := gitRepo.GetBranchNames(page*limit, limit) + if err != nil { + return err + } + if len(branches) == 0 { + break + } + allBranches = append(allBranches, branches...) + } + + dbBranches, err := git_model.LoadAllBranches(ctx, repo.ID) + if err != nil { + return err + } + + var toAdd []*git_model.Branch + var toUpdate []*git_model.Branch + var toRemove []int64 + for _, branch := range allBranches { + var dbb *git_model.Branch + for _, dbBranch := range dbBranches { + if branch == dbBranch.Name { + dbb = dbBranch + break + } + } + commit, err := gitRepo.GetBranchCommit(branch) + if err != nil { + return err + } + if dbb == nil { + toAdd = append(toAdd, &git_model.Branch{ + RepoID: repo.ID, + Name: branch, + Commit: commit.ID.String(), + PusherID: doerID, + CommitTime: timeutil.TimeStamp(commit.Author.When.Unix()), + }) + } else if commit.ID.String() != dbb.Commit { + toUpdate = append(toUpdate, &git_model.Branch{ + ID: dbb.ID, + RepoID: repo.ID, + Name: branch, + Commit: commit.ID.String(), + PusherID: doerID, + CommitTime: timeutil.TimeStamp(commit.Author.When.Unix()), + }) + } + } + + for _, dbBranch := range dbBranches { + var found bool + for _, branch := range allBranches { + if branch == dbBranch.Name { + found = true + break + } + } + if !found { + toRemove = append(toRemove, dbBranch.ID) + } + } + + if len(toAdd) <= 0 && len(toRemove) <= 0 && len(toUpdate) <= 0 { + return nil + } + + return db.WithTx(ctx, func(ctx context.Context) error { + if len(toAdd) > 0 { + if err := git_model.AddBranches(ctx, toAdd); err != nil { + return err + } + } + + if len(toUpdate) > 0 { + for _, b := range toUpdate { + if _, err := db.GetEngine(ctx).ID(b.ID).Cols("commit, pusher_id, commit_time").Update(b); err != nil { + return err + } + } + } + + if len(toRemove) > 0 { + err = git_model.DeleteBranches(ctx, repo.ID, doerID, toRemove) + if err != nil { + return err + } + } + + return nil + }) +} From 9effdb89acf47e0ab026be7ff7ea0808267dc880 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Wed, 14 Jun 2023 13:25:33 +0800 Subject: [PATCH 04/56] Add new columns --- models/git/branches.go | 19 ++++++++++--------- services/repository/branch.go | 26 ++++++++++++++------------ 2 files changed, 24 insertions(+), 21 deletions(-) diff --git a/models/git/branches.go b/models/git/branches.go index cff7d8f11e97e..28069f3db42e4 100644 --- a/models/git/branches.go +++ b/models/git/branches.go @@ -17,16 +17,17 @@ import ( // Branch represents a branch of a repository // For those repository who have many branches, stored into database is a good choice -// for pagination and search +// for pagination, keyword search and filtering type Branch struct { - ID int64 - RepoID int64 `xorm:"index UNIQUE(s)"` - Name string `xorm:"UNIQUE(s) NOT NULL"` - Commit string - PusherID int64 - CommitTime timeutil.TimeStamp // The commit - CreatedUnix timeutil.TimeStamp `xorm:"created"` - UpdatedUnix timeutil.TimeStamp `xorm:"updated"` + ID int64 + RepoID int64 `xorm:"index UNIQUE(s)"` + Name string `xorm:"UNIQUE(s) NOT NULL"` + CommitSHA string + CommitMessage string `xorm:"TEXT"` + PusherID int64 + CommitTime timeutil.TimeStamp // The commit + CreatedUnix timeutil.TimeStamp `xorm:"created"` + UpdatedUnix timeutil.TimeStamp `xorm:"updated"` } // DeletedBranch struct diff --git a/services/repository/branch.go b/services/repository/branch.go index 71203e63ca9c7..7e3e94444f8c6 100644 --- a/services/repository/branch.go +++ b/services/repository/branch.go @@ -238,20 +238,22 @@ func SyncRepoBranches(ctx context.Context, repo *repo_model.Repository, doerID i } if dbb == nil { toAdd = append(toAdd, &git_model.Branch{ - RepoID: repo.ID, - Name: branch, - Commit: commit.ID.String(), - PusherID: doerID, - CommitTime: timeutil.TimeStamp(commit.Author.When.Unix()), + RepoID: repo.ID, + Name: branch, + CommitSHA: commit.ID.String(), + CommitMessage: commit.CommitMessage, + PusherID: doerID, + CommitTime: timeutil.TimeStamp(commit.Author.When.Unix()), }) - } else if commit.ID.String() != dbb.Commit { + } else if commit.ID.String() != dbb.CommitSHA { toUpdate = append(toUpdate, &git_model.Branch{ - ID: dbb.ID, - RepoID: repo.ID, - Name: branch, - Commit: commit.ID.String(), - PusherID: doerID, - CommitTime: timeutil.TimeStamp(commit.Author.When.Unix()), + ID: dbb.ID, + RepoID: repo.ID, + Name: branch, + CommitSHA: commit.ID.String(), + CommitMessage: commit.CommitMessage, + PusherID: doerID, + CommitTime: timeutil.TimeStamp(commit.Author.When.Unix()), }) } } From 458052bcb842bdbfa7dd1d8033e25014505b631c Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Thu, 15 Jun 2023 16:36:32 +0800 Subject: [PATCH 05/56] rewrite branches page --- models/git/branches.go | 56 ++++++--- modules/repository/init.go | 19 +++ routers/web/repo/branch.go | 203 ++---------------------------- services/repository/branch.go | 149 +++++++++++++++++++++- services/repository/push.go | 2 +- templates/repo/branch/list.tmpl | 216 ++++++++++++++++---------------- 6 files changed, 320 insertions(+), 325 deletions(-) diff --git a/models/git/branches.go b/models/git/branches.go index 28069f3db42e4..c23691f7b8cd1 100644 --- a/models/git/branches.go +++ b/models/git/branches.go @@ -11,8 +11,11 @@ import ( "code.gitea.io/gitea/models/db" repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/timeutil" + + "xorm.io/builder" ) // Branch represents a branch of a repository @@ -53,18 +56,41 @@ func LoadAllBranches(ctx context.Context, repoID int64) ([]*Branch, error) { return branches, err } +func GetDefaultBranch(ctx context.Context, repo *repo_model.Repository) (*Branch, error) { + var branch Branch + has, err := db.GetEngine(ctx).Where("repo_id=?", repo.ID).And("name=?", repo.DefaultBranch).Get(&branch) + if err != nil { + return nil, err + } else if !has { + return nil, git.ErrBranchNotExist{Name: repo.DefaultBranch} + } + return &branch, nil +} + type FindBranchOptions struct { db.ListOptions - RepoID int64 + RepoID int64 + IncludeDefaultBranch bool + IncludeDeletedBranch bool } -func FindBranches(ctx context.Context, opts FindBranchOptions) ([]*Branch, error) { +func FindBranches(ctx context.Context, opts FindBranchOptions) ([]*Branch, int64, error) { sess := db.GetEngine(ctx).Where("repo_id=?", opts.RepoID) if opts.PageSize > 0 { sess = db.SetSessionPagination(sess, &opts.ListOptions) } + if !opts.IncludeDefaultBranch { + sess = sess.And(builder.Neq{"name": builder.Select("default_branch").From("repository").Where(builder.Eq{"id": opts.RepoID})}) + } + if opts.IncludeDeletedBranch { + // FIXME: xxxxxx + } var branches []*Branch - return branches, sess.Find(&branches) + total, err := sess.FindAndCount(&branches) + if err != nil { + return nil, 0, err + } + return branches, total, err } func AddBranch(ctx context.Context, branch *Branch) error { @@ -93,7 +119,7 @@ func DeleteBranches(ctx context.Context, repoID, doerID int64, branchIDs []int64 return err } for _, branch := range branches { - if err := AddDeletedBranch(ctx, repoID, branch.Name, branch.Commit, doerID); err != nil { + if err := AddDeletedBranch(ctx, repoID, branch.Name, branch.CommitSHA, doerID); err != nil { return err } } @@ -104,16 +130,17 @@ func DeleteBranches(ctx context.Context, repoID, doerID int64, branchIDs []int64 // UpdateBranch updates the branch information in the database. If the branch exist, it will update latest commit of this branch information // If it doest not exist, insert a new record into database -func UpdateBranch(ctx context.Context, repoID int64, branchName string, commitID string, pusherID int64, commitTime time.Time) error { +func UpdateBranch(ctx context.Context, repoID int64, branchName, commitID, commitMessage string, pusherID int64, commitTime time.Time) error { if err := removeDeletedBranchByName(ctx, repoID, branchName); err != nil { return err } cnt, err := db.GetEngine(ctx).Where("repo_id=? AND name=?", repoID, branchName). - Cols("commit, pusher_id, commit_time, updated_unix"). + Cols("commit_sha, commit_message, pusher_id, commit_time, updated_unix"). Update(&Branch{ - Commit: commitID, - PusherID: pusherID, - CommitTime: timeutil.TimeStamp(commitTime.Unix()), + CommitSHA: commitID, + CommitMessage: commitMessage, + PusherID: pusherID, + CommitTime: timeutil.TimeStamp(commitTime.Unix()), }) if err != nil { return err @@ -123,11 +150,12 @@ func UpdateBranch(ctx context.Context, repoID int64, branchName string, commitID } return db.Insert(ctx, &Branch{ - RepoID: repoID, - Name: branchName, - Commit: commitID, - PusherID: pusherID, - CommitTime: timeutil.TimeStamp(commitTime.Unix()), + RepoID: repoID, + Name: branchName, + CommitSHA: commitID, + CommitMessage: commitMessage, + PusherID: pusherID, + CommitTime: timeutil.TimeStamp(commitTime.Unix()), }) } diff --git a/modules/repository/init.go b/modules/repository/init.go index f079f72b77116..e028bce8a9fbc 100644 --- a/modules/repository/init.go +++ b/modules/repository/init.go @@ -13,6 +13,8 @@ import ( "strings" "time" + "code.gitea.io/gitea/models/db" + git_model "code.gitea.io/gitea/models/git" issues_model "code.gitea.io/gitea/models/issues" repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" @@ -22,6 +24,7 @@ import ( "code.gitea.io/gitea/modules/options" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/templates/vars" + "code.gitea.io/gitea/modules/timeutil" "code.gitea.io/gitea/modules/util" asymkey_service "code.gitea.io/gitea/services/asymkey" ) @@ -351,6 +354,22 @@ func initRepository(ctx context.Context, repoPath string, u *user_model.User, re if err = gitRepo.SetDefaultBranch(repo.DefaultBranch); err != nil { return fmt.Errorf("setDefaultBranch: %w", err) } + + commit, err := gitRepo.GetBranchCommit(repo.DefaultBranch) + if err != nil { + return fmt.Errorf("getBranchCommit: %w", err) + } + + if err := db.Insert(ctx, &git_model.Branch{ + Name: repo.DefaultBranch, + RepoID: repo.ID, + CommitSHA: commit.ID.String(), + CommitMessage: commit.CommitMessage, + CommitTime: timeutil.TimeStamp(commit.Committer.When.Unix()), + PusherID: u.ID, + }); err != nil { + return fmt.Errorf("insert default branch for repo %d: %w", repo.ID, err) + } } if err = UpdateRepository(ctx, repo, false); err != nil { diff --git a/routers/web/repo/branch.go b/routers/web/repo/branch.go index ea2c01856d45f..ec48c2ba75f7f 100644 --- a/routers/web/repo/branch.go +++ b/routers/web/repo/branch.go @@ -13,7 +13,6 @@ import ( "code.gitea.io/gitea/models" git_model "code.gitea.io/gitea/models/git" - issues_model "code.gitea.io/gitea/models/issues" repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/models/unit" "code.gitea.io/gitea/modules/base" @@ -28,27 +27,12 @@ import ( "code.gitea.io/gitea/services/forms" release_service "code.gitea.io/gitea/services/release" repo_service "code.gitea.io/gitea/services/repository" - files_service "code.gitea.io/gitea/services/repository/files" ) const ( tplBranch base.TplName = "repo/branch/list" ) -// Branch contains the branch information -type Branch struct { - Name string - Commit *git.Commit - IsProtected bool - IsDeleted bool - IsIncluded bool - DeletedBranch *git_model.DeletedBranch - CommitsAhead int - CommitsBehind int - LatestPullRequest *issues_model.PullRequest - MergeMovedOn bool -} - // Branches render repository branch page func Branches(ctx *context.Context) { ctx.Data["Title"] = "Branches" @@ -68,15 +52,16 @@ func Branches(ctx *context.Context) { } pageSize := setting.Git.BranchesRangeSize - skip := (page - 1) * pageSize - log.Debug("Branches: skip: %d limit: %d", skip, pageSize) - defaultBranchBranch, branches, branchesCount := loadBranches(ctx, skip, pageSize) - if ctx.Written() { + log.Debug("Branches: page: %d pageSize: %d", page, pageSize) + defaultBranch, branches, branchesCount, err := repo_service.LoadBranches(ctx, ctx.Repo.Repository, ctx.Repo.GitRepo, true, page, pageSize) + if err != nil { + ctx.ServerError("LoadBranches", err) return } + ctx.Data["Branches"] = branches - ctx.Data["DefaultBranchBranch"] = defaultBranchBranch - pager := context.NewPagination(branchesCount, pageSize, page, 5) + ctx.Data["DefaultBranch"] = defaultBranch + pager := context.NewPagination(int(branchesCount), pageSize, page, 5) pager.SetDefaultParams(ctx) ctx.Data["Page"] = pager @@ -166,180 +151,6 @@ func redirect(ctx *context.Context) { }) } -// loadBranches loads branches from the repository limited by page & pageSize. -// NOTE: May write to context on error. -func loadBranches(ctx *context.Context, skip, limit int) (*Branch, []*Branch, int) { - defaultBranch, err := ctx.Repo.GitRepo.GetBranch(ctx.Repo.Repository.DefaultBranch) - if err != nil { - if !git.IsErrBranchNotExist(err) { - log.Error("loadBranches: get default branch: %v", err) - ctx.ServerError("GetDefaultBranch", err) - return nil, nil, 0 - } - log.Warn("loadBranches: missing default branch %s for %-v", ctx.Repo.Repository.DefaultBranch, ctx.Repo.Repository) - } - - rawBranches, totalNumOfBranches, err := ctx.Repo.GitRepo.GetBranches(skip, limit) - if err != nil { - log.Error("GetBranches: %v", err) - ctx.ServerError("GetBranches", err) - return nil, nil, 0 - } - - rules, err := git_model.FindRepoProtectedBranchRules(ctx, ctx.Repo.Repository.ID) - if err != nil { - ctx.ServerError("FindRepoProtectedBranchRules", err) - return nil, nil, 0 - } - - repoIDToRepo := map[int64]*repo_model.Repository{} - repoIDToRepo[ctx.Repo.Repository.ID] = ctx.Repo.Repository - - repoIDToGitRepo := map[int64]*git.Repository{} - repoIDToGitRepo[ctx.Repo.Repository.ID] = ctx.Repo.GitRepo - - var branches []*Branch - for i := range rawBranches { - if defaultBranch != nil && rawBranches[i].Name == defaultBranch.Name { - // Skip default branch - continue - } - - branch := loadOneBranch(ctx, rawBranches[i], defaultBranch, &rules, repoIDToRepo, repoIDToGitRepo) - if branch == nil { - return nil, nil, 0 - } - - branches = append(branches, branch) - } - - var defaultBranchBranch *Branch - if defaultBranch != nil { - // Always add the default branch - log.Debug("loadOneBranch: load default: '%s'", defaultBranch.Name) - defaultBranchBranch = loadOneBranch(ctx, defaultBranch, defaultBranch, &rules, repoIDToRepo, repoIDToGitRepo) - branches = append(branches, defaultBranchBranch) - } - - if ctx.Repo.CanWrite(unit.TypeCode) { - deletedBranches, err := getDeletedBranches(ctx) - if err != nil { - ctx.ServerError("getDeletedBranches", err) - return nil, nil, 0 - } - branches = append(branches, deletedBranches...) - } - - return defaultBranchBranch, branches, totalNumOfBranches -} - -func loadOneBranch(ctx *context.Context, rawBranch, defaultBranch *git.Branch, protectedBranches *git_model.ProtectedBranchRules, - repoIDToRepo map[int64]*repo_model.Repository, - repoIDToGitRepo map[int64]*git.Repository, -) *Branch { - log.Trace("loadOneBranch: '%s'", rawBranch.Name) - - commit, err := rawBranch.GetCommit() - if err != nil { - ctx.ServerError("GetCommit", err) - return nil - } - - branchName := rawBranch.Name - p := protectedBranches.GetFirstMatched(branchName) - isProtected := p != nil - - divergence := &git.DivergeObject{ - Ahead: -1, - Behind: -1, - } - if defaultBranch != nil { - divergence, err = files_service.CountDivergingCommits(ctx, ctx.Repo.Repository, git.BranchPrefix+branchName) - if err != nil { - log.Error("CountDivergingCommits", err) - } - } - - pr, err := issues_model.GetLatestPullRequestByHeadInfo(ctx.Repo.Repository.ID, branchName) - if err != nil { - ctx.ServerError("GetLatestPullRequestByHeadInfo", err) - return nil - } - headCommit := commit.ID.String() - - mergeMovedOn := false - if pr != nil { - pr.HeadRepo = ctx.Repo.Repository - if err := pr.LoadIssue(ctx); err != nil { - ctx.ServerError("LoadIssue", err) - return nil - } - if repo, ok := repoIDToRepo[pr.BaseRepoID]; ok { - pr.BaseRepo = repo - } else if err := pr.LoadBaseRepo(ctx); err != nil { - ctx.ServerError("LoadBaseRepo", err) - return nil - } else { - repoIDToRepo[pr.BaseRepoID] = pr.BaseRepo - } - pr.Issue.Repo = pr.BaseRepo - - if pr.HasMerged { - baseGitRepo, ok := repoIDToGitRepo[pr.BaseRepoID] - if !ok { - baseGitRepo, err = git.OpenRepository(ctx, pr.BaseRepo.RepoPath()) - if err != nil { - ctx.ServerError("OpenRepository", err) - return nil - } - defer baseGitRepo.Close() - repoIDToGitRepo[pr.BaseRepoID] = baseGitRepo - } - pullCommit, err := baseGitRepo.GetRefCommitID(pr.GetGitRefName()) - if err != nil && !git.IsErrNotExist(err) { - ctx.ServerError("GetBranchCommitID", err) - return nil - } - if err == nil && headCommit != pullCommit { - // the head has moved on from the merge - we shouldn't delete - mergeMovedOn = true - } - } - } - - isIncluded := divergence.Ahead == 0 && ctx.Repo.Repository.DefaultBranch != branchName - return &Branch{ - Name: branchName, - Commit: commit, - IsProtected: isProtected, - IsIncluded: isIncluded, - CommitsAhead: divergence.Ahead, - CommitsBehind: divergence.Behind, - LatestPullRequest: pr, - MergeMovedOn: mergeMovedOn, - } -} - -func getDeletedBranches(ctx *context.Context) ([]*Branch, error) { - branches := []*Branch{} - - deletedBranches, err := git_model.GetDeletedBranches(ctx, ctx.Repo.Repository.ID) - if err != nil { - return branches, err - } - - for i := range deletedBranches { - deletedBranches[i].LoadUser(ctx) - branches = append(branches, &Branch{ - Name: deletedBranches[i].Name, - IsDeleted: true, - DeletedBranch: deletedBranches[i], - }) - } - - return branches, nil -} - // CreateBranch creates new branch in repository func CreateBranch(ctx *context.Context) { form := web.GetForm(ctx).(*forms.NewBranchForm) diff --git a/services/repository/branch.go b/services/repository/branch.go index 7e3e94444f8c6..b3c9ae17d7781 100644 --- a/services/repository/branch.go +++ b/services/repository/branch.go @@ -12,6 +12,7 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/models/db" git_model "code.gitea.io/gitea/models/git" + issues_model "code.gitea.io/gitea/models/issues" repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/git" @@ -19,6 +20,7 @@ import ( "code.gitea.io/gitea/modules/notification" repo_module "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/timeutil" + files_service "code.gitea.io/gitea/services/repository/files" ) // CreateNewBranch creates a new repository branch @@ -48,10 +50,147 @@ func CreateNewBranch(ctx context.Context, doer *user_model.User, repo *repo_mode return nil } -// GetBranches returns branches from the repository, skipping skip initial branches and -// returning at most limit branches, or all branches if limit is 0. -func GetBranches(ctx context.Context, repo *repo_model.Repository, skip, limit int) ([]*git.Branch, int, error) { - return git.GetBranchesByPath(ctx, repo.RepoPath(), skip, limit) +// Branch contains the branch information +type Branch struct { + RawBranch *git_model.Branch + IsProtected bool + IsDeleted bool + IsIncluded bool + DeletedBranch *git_model.DeletedBranch + CommitsAhead int + CommitsBehind int + LatestPullRequest *issues_model.PullRequest + MergeMovedOn bool +} + +// LoadBranches loads branches from the repository limited by page & pageSize. +// NOTE: May write to context on error. +func LoadBranches(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository, includeDeletedBranch bool, page, pageSize int) (*Branch, []*Branch, int64, error) { + defaultRawBranch, err := git_model.GetDefaultBranch(ctx, repo) + if err != nil { + return nil, nil, 0, err + } + + rawBranches, totalNumOfBranches, err := git_model.FindBranches(ctx, git_model.FindBranchOptions{ + RepoID: repo.ID, + IncludeDefaultBranch: false, + IncludeDeletedBranch: includeDeletedBranch, + ListOptions: db.ListOptions{ + Page: page, + PageSize: pageSize, + }, + }) + if err != nil { + return nil, nil, 0, err + } + + rules, err := git_model.FindRepoProtectedBranchRules(ctx, repo.ID) + if err != nil { + return nil, nil, 0, err + } + + repoIDToRepo := map[int64]*repo_model.Repository{} + repoIDToRepo[repo.ID] = repo + + repoIDToGitRepo := map[int64]*git.Repository{} + repoIDToGitRepo[repo.ID] = gitRepo + + var branches []*Branch + for i := range rawBranches { + branch, err := loadOneBranch(ctx, repo, rawBranches[i], &rules, repoIDToRepo, repoIDToGitRepo) + if err != nil { + return nil, nil, 0, fmt.Errorf("loadOneBranch: %v", err) + } + + branches = append(branches, branch) + } + + // Always add the default branch + log.Debug("loadOneBranch: load default: '%s'", defaultRawBranch.Name) + defaultBranch, err := loadOneBranch(ctx, repo, defaultRawBranch, &rules, repoIDToRepo, repoIDToGitRepo) + if err != nil { + return nil, nil, 0, fmt.Errorf("loadOneBranch: %v", err) + } + + return defaultBranch, branches, totalNumOfBranches, nil +} + +func loadOneBranch(ctx context.Context, repo *repo_model.Repository, rawBranch *git_model.Branch, protectedBranches *git_model.ProtectedBranchRules, + repoIDToRepo map[int64]*repo_model.Repository, + repoIDToGitRepo map[int64]*git.Repository, +) (*Branch, error) { + log.Trace("loadOneBranch: '%s'", rawBranch.Name) + + branchName := rawBranch.Name + p := protectedBranches.GetFirstMatched(branchName) + isProtected := p != nil + + divergence := &git.DivergeObject{ + Ahead: -1, + Behind: -1, + } + + // it's not default branch + if repo.DefaultBranch != rawBranch.Name { + var err error + divergence, err = files_service.CountDivergingCommits(ctx, repo, git.BranchPrefix+branchName) + if err != nil { + log.Error("CountDivergingCommits", err) + } + } + + pr, err := issues_model.GetLatestPullRequestByHeadInfo(repo.ID, branchName) + if err != nil { + return nil, fmt.Errorf("GetLatestPullRequestByHeadInfo: %v", err) + } + headCommit := rawBranch.CommitSHA + + mergeMovedOn := false + if pr != nil { + pr.HeadRepo = repo + if err := pr.LoadIssue(ctx); err != nil { + return nil, fmt.Errorf("LoadIssue: %v", err) + } + if repo, ok := repoIDToRepo[pr.BaseRepoID]; ok { + pr.BaseRepo = repo + } else if err := pr.LoadBaseRepo(ctx); err != nil { + return nil, fmt.Errorf("LoadBaseRepo: %v", err) + } else { + repoIDToRepo[pr.BaseRepoID] = pr.BaseRepo + } + pr.Issue.Repo = pr.BaseRepo + + if pr.HasMerged { + baseGitRepo, ok := repoIDToGitRepo[pr.BaseRepoID] + if !ok { + baseGitRepo, err = git.OpenRepository(ctx, pr.BaseRepo.RepoPath()) + if err != nil { + return nil, fmt.Errorf("OpenRepository: %v", err) + } + defer baseGitRepo.Close() + repoIDToGitRepo[pr.BaseRepoID] = baseGitRepo + } + pullCommit, err := baseGitRepo.GetRefCommitID(pr.GetGitRefName()) + if err != nil && !git.IsErrNotExist(err) { + return nil, fmt.Errorf("GetBranchCommitID: %v", err) + } + if err == nil && headCommit != pullCommit { + // the head has moved on from the merge - we shouldn't delete + mergeMovedOn = true + } + } + } + + isIncluded := divergence.Ahead == 0 && repo.DefaultBranch != branchName + return &Branch{ + RawBranch: rawBranch, + IsProtected: isProtected, + IsIncluded: isIncluded, + CommitsAhead: divergence.Ahead, + CommitsBehind: divergence.Behind, + LatestPullRequest: pr, + MergeMovedOn: mergeMovedOn, + }, nil } func GetBranchCommitID(ctx context.Context, repo *repo_model.Repository, branch string) (string, error) { @@ -103,7 +242,7 @@ func CreateNewBranchFromCommit(ctx context.Context, doer *user_model.User, repo if git.IsErrPushOutOfDate(err) || git.IsErrPushRejected(err) { return err } - return fmt.Errorf("Push: %w", err) + return fmt.Errorf("push: %w", err) } return nil diff --git a/services/repository/push.go b/services/repository/push.go index 2093714cf488b..f159fb71b92b2 100644 --- a/services/repository/push.go +++ b/services/repository/push.go @@ -259,7 +259,7 @@ func pushUpdates(optsList []*repo_module.PushUpdateOptions) error { notification.NotifyPushCommits(ctx, pusher, repo, opts, commits) - if err = git_model.UpdateBranch(ctx, repo.ID, branch, newCommit.ID.String(), opts.PusherID, newCommit.Committer.When); err != nil { + if err = git_model.UpdateBranch(ctx, repo.ID, branch, newCommit.ID.String(), newCommit.CommitMessage, opts.PusherID, newCommit.Committer.When); err != nil { log.Error("git_model.UpdateBranch %s/%s failed: %v", repo.ID, branch, err) } diff --git a/templates/repo/branch/list.tmpl b/templates/repo/branch/list.tmpl index d68feb43b7893..6542548dadcd4 100644 --- a/templates/repo/branch/list.tmpl +++ b/templates/repo/branch/list.tmpl @@ -4,7 +4,7 @@
{{template "base/alert" .}} {{template "repo/sub_menu" .}} - {{if .DefaultBranchBranch}} + {{if .DefaultBranch}}

{{.locale.Tr "repo.default_branch"}} {{if and $.IsWriter (not $.Repository.IsArchived) (not .IsDeleted)}} @@ -19,32 +19,32 @@ - {{if .DefaultBranchBranch.IsProtected}} + {{if .DefaultBranch.IsProtected}} {{svg "octicon-shield-lock"}} {{end}} - {{.DefaultBranch}} -

{{svg "octicon-git-commit" 16 "gt-mr-2"}}{{ShortSha .DefaultBranchBranch.Commit.ID.String}} · {{RenderCommitMessage $.Context .DefaultBranchBranch.Commit.CommitMessage .RepoLink .Repository.ComposeMetas}} · {{.locale.Tr "org.repo_updated"}} {{TimeSince .DefaultBranchBranch.Commit.Committer.When .locale}}

+ {{.DefaultBranch.RawBranch.Name}} +

{{svg "octicon-git-commit" 16 "gt-mr-2"}}{{ShortSha .DefaultBranch.RawBranch.CommitSHA}} · {{RenderCommitMessage $.Context .DefaultBranch.RawBranch.CommitMessage .RepoLink .Repository.ComposeMetas}} · {{.locale.Tr "org.repo_updated"}} {{TimeSince .DefaultBranch.RawBranch.CommitTime.AsTime .locale}}

{{if and $.IsWriter (not $.Repository.IsArchived) (not .IsDeleted)}} {{end}} {{if .EnableFeed}} - {{svg "octicon-rss"}} + {{svg "octicon-rss"}} {{end}} {{if not $.DisableDownloadSourceArchives}} -
+
{{svg "octicon-download"}}
- {{svg "octicon-file-zip"}} ZIP - {{svg "octicon-file-zip"}} TAR.GZ + {{svg "octicon-file-zip"}} ZIP + {{svg "octicon-file-zip"}} TAR.GZ
{{end}} @@ -53,7 +53,7 @@ data-is-default-branch="true" data-modal="#rename-branch-modal" data-old-branch-name="{{$.DefaultBranch}}" - data-tooltip-content="{{$.locale.Tr "repo.branch.rename" ($.DefaultBranch)}}" + data-tooltip-content="{{$.locale.Tr "repo.branch.rename" ($.DefaultBranch.RawBranch.Name)}}" > {{svg "octicon-pencil"}} @@ -65,7 +65,7 @@
{{end}} - {{if gt (len .Branches) 1}} + {{if .Branches}}

{{.locale.Tr "repo.branches"}}

@@ -73,112 +73,110 @@ {{range .Branches}} - {{if ne .Name $.DefaultBranch}} - - + + + - + - - - - {{end}} + {{end}} + + {{end}}
- {{if .IsDeleted}} - {{.Name}} -

{{$.locale.Tr "repo.branch.deleted_by" .DeletedBranch.DeletedBy.Name}} {{TimeSinceUnix .DeletedBranch.DeletedUnix $.locale}}

+
+ {{if .IsDeleted}} + {{.RawBranch.Name}} +

{{$.locale.Tr "repo.branch.deleted_by" .DeletedBranch.DeletedBy.Name}} {{TimeSinceUnix .DeletedBranch.DeletedUnix $.locale}}

+ {{else}} + {{if .IsProtected}} + {{svg "octicon-shield-lock"}} + {{end}} + {{.RawBranch.Name}} +

{{svg "octicon-git-commit" 16 "gt-mr-2"}}{{ShortSha .RawBranch.CommitSHA}} · {{RenderCommitMessage $.Context .RawBranch.CommitMessage $.RepoLink $.Repository.ComposeMetas}} · {{$.locale.Tr "org.repo_updated"}} {{TimeSince .RawBranch.CommitTime.AsTime $.locale}}

+ {{end}} + 		+ {{if and (not .IsDeleted) $.DefaultBranch}} +
+
+
{{.CommitsBehind}}
+ {{/* old code bears 0/0.0 = NaN output, so it might output invalid "width: NaNpx", it just works and doesn't caues any problem. */}} +
+
+
+
{{.CommitsAhead}}
+
+
+
+ {{end}} + 		+ {{if not .LatestPullRequest}} + {{if .IsIncluded}} + + {{svg "octicon-git-pull-request"}} {{$.locale.Tr "repo.branch.included"}} + + {{else if and (not .IsDeleted) $.AllowsPulls (gt .CommitsAhead 0)}} + + + + {{end}} + {{else if and .LatestPullRequest.HasMerged .MergeMovedOn}} + {{if and (not .IsDeleted) $.AllowsPulls (gt .CommitsAhead 0)}} + + + + {{end}} {{else}} - {{if .IsProtected}} - {{svg "octicon-shield-lock"}} + {{if not .LatestPullRequest.IsSameRepo}}{{.LatestPullRequest.BaseRepo.FullName}}{{end}}#{{.LatestPullRequest.Issue.Index}} + {{if .LatestPullRequest.HasMerged}} + {{svg "octicon-git-merge" 16 "gt-mr-2"}}{{$.locale.Tr "repo.pulls.merged"}} + {{else if .LatestPullRequest.Issue.IsClosed}} + {{svg "octicon-git-pull-request" 16 "gt-mr-2"}}{{$.locale.Tr "repo.issues.closed_title"}} + {{else}} + {{svg "octicon-git-pull-request" 16 "gt-mr-2"}}{{$.locale.Tr "repo.issues.open_title"}} {{end}} - {{.Name}} -

{{svg "octicon-git-commit" 16 "gt-mr-2"}}{{ShortSha .Commit.ID.String}} · {{RenderCommitMessage $.Context .Commit.CommitMessage $.RepoLink $.Repository.ComposeMetas}} · {{$.locale.Tr "org.repo_updated"}} {{TimeSince .Commit.Committer.When $.locale}}

{{end}} - 		- {{if and (not .IsDeleted) $.DefaultBranchBranch}} -
-
-
{{.CommitsBehind}}
- {{/* old code bears 0/0.0 = NaN output, so it might output invalid "width: NaNpx", it just works and doesn't caues any problem. */}} -
-
-
-
{{.CommitsAhead}}
-
+
+ {{if and $.IsWriter (not $.Repository.IsArchived) (not .IsDeleted)}} + + {{end}} + {{if $.EnableFeed}} + {{svg "octicon-rss"}} + {{end}} + {{if and (not .IsDeleted) (not $.DisableDownloadSourceArchives)}} +
+ {{svg "octicon-download"}} +
+ {{svg "octicon-file-zip"}} ZIP + {{svg "octicon-file-zip"}} TAR.GZ
- {{end}} - 		- {{if not .LatestPullRequest}} - {{if .IsIncluded}} - - {{svg "octicon-git-pull-request"}} {{$.locale.Tr "repo.branch.included"}} + {{end}} + {{if and $.IsWriter (not $.Repository.IsArchived) (not .IsDeleted) (not $.IsMirror)}} + + {{end}} + {{if and $.IsWriter (not $.IsMirror) (not $.Repository.IsArchived) (not .IsProtected)}} + {{if .IsDeleted}} + - - {{end}} - {{else if and .LatestPullRequest.HasMerged .MergeMovedOn}} - {{if and (not .IsDeleted) $.AllowsPulls (gt .CommitsAhead 0)}} - - - - {{end}} - {{else}} - {{if not .LatestPullRequest.IsSameRepo}}{{.LatestPullRequest.BaseRepo.FullName}}{{end}}#{{.LatestPullRequest.Issue.Index}} - {{if .LatestPullRequest.HasMerged}} - {{svg "octicon-git-merge" 16 "gt-mr-2"}}{{$.locale.Tr "repo.pulls.merged"}} - {{else if .LatestPullRequest.Issue.IsClosed}} - {{svg "octicon-git-pull-request" 16 "gt-mr-2"}}{{$.locale.Tr "repo.issues.closed_title"}} - {{else}} - {{svg "octicon-git-pull-request" 16 "gt-mr-2"}}{{$.locale.Tr "repo.issues.open_title"}} - {{end}} - {{end}} - - {{if and $.IsWriter (not $.Repository.IsArchived) (not .IsDeleted)}} - - {{end}} - {{if $.EnableFeed}} - {{svg "octicon-rss"}} - {{end}} - {{if and (not .IsDeleted) (not $.DisableDownloadSourceArchives)}} -
- {{svg "octicon-download"}} -
- {{svg "octicon-file-zip"}} ZIP - {{svg "octicon-file-zip"}} TAR.GZ -
-
- {{end}} - {{if and $.IsWriter (not $.Repository.IsArchived) (not .IsDeleted) (not $.IsMirror)}} - {{end}} - {{if and $.IsWriter (not $.IsMirror) (not $.Repository.IsArchived) (not .IsProtected)}} - {{if .IsDeleted}} - - {{else}} - - {{end}} - {{end}} -
From 2c10f0c1836f864e44751a3cdfe2e5a3ed95a69f Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Thu, 15 Jun 2023 17:55:57 +0800 Subject: [PATCH 06/56] Remove deletedbranch table and migrate data to branch --- models/git/branches.go | 134 ++++++++++---------------------- models/migrations/v1_21/v261.go | 61 +++++++++++++-- models/repo.go | 2 +- routers/web/repo/branch.go | 4 +- services/repository/branch.go | 3 - services/repository/push.go | 2 +- 6 files changed, 100 insertions(+), 106 deletions(-) diff --git a/models/git/branches.go b/models/git/branches.go index c23691f7b8cd1..44cb2e8405506 100644 --- a/models/git/branches.go +++ b/models/git/branches.go @@ -5,12 +5,10 @@ package git import ( "context" - "fmt" "time" "code.gitea.io/gitea/models/db" repo_model "code.gitea.io/gitea/models/repo" - user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/timeutil" @@ -28,25 +26,16 @@ type Branch struct { CommitSHA string CommitMessage string `xorm:"TEXT"` PusherID int64 + IsDeleted bool + DeletedByID int64 + DeletedUnix timeutil.TimeStamp CommitTime timeutil.TimeStamp // The commit CreatedUnix timeutil.TimeStamp `xorm:"created"` UpdatedUnix timeutil.TimeStamp `xorm:"updated"` } -// DeletedBranch struct -type DeletedBranch struct { - ID int64 `xorm:"pk autoincr"` - RepoID int64 `xorm:"UNIQUE(s) INDEX NOT NULL"` - Name string `xorm:"UNIQUE(s) NOT NULL"` - Commit string `xorm:"UNIQUE(s) NOT NULL"` - DeletedByID int64 `xorm:"INDEX"` - DeletedBy *user_model.User `xorm:"-"` - DeletedUnix timeutil.TimeStamp `xorm:"INDEX created"` -} - func init() { db.RegisterModel(new(Branch)) - db.RegisterModel(new(DeletedBranch)) db.RegisterModel(new(RenamedBranch)) } @@ -82,8 +71,8 @@ func FindBranches(ctx context.Context, opts FindBranchOptions) ([]*Branch, int64 if !opts.IncludeDefaultBranch { sess = sess.And(builder.Neq{"name": builder.Select("default_branch").From("repository").Where(builder.Eq{"id": opts.RepoID})}) } - if opts.IncludeDeletedBranch { - // FIXME: xxxxxx + if !opts.IncludeDeletedBranch { + sess.And(builder.Eq{"is_deleted": false}) } var branches []*Branch total, err := sess.FindAndCount(&branches) @@ -112,6 +101,23 @@ func AddBranches(ctx context.Context, branches []*Branch) error { return nil } +func GetDeletedBranchByID(ctx context.Context, repoID, branchID int64) (*Branch, error) { + var branch Branch + has, err := db.GetEngine(ctx).ID(branchID).Get(&branch) + if err != nil { + return nil, err + } else if !has { + return nil, git.ErrNotExist{} + } + if branch.RepoID != repoID { + return nil, git.ErrNotExist{} + } + if !branch.IsDeleted { + return nil, git.ErrNotExist{} + } + return &branch, nil +} + func DeleteBranches(ctx context.Context, repoID, doerID int64, branchIDs []int64) error { return db.WithTx(ctx, func(ctx context.Context) error { branches := make([]*Branch, 0, len(branchIDs)) @@ -119,12 +125,11 @@ func DeleteBranches(ctx context.Context, repoID, doerID int64, branchIDs []int64 return err } for _, branch := range branches { - if err := AddDeletedBranch(ctx, repoID, branch.Name, branch.CommitSHA, doerID); err != nil { + if err := AddDeletedBranch(ctx, repoID, branch.Name, doerID); err != nil { return err } } - _, err := db.GetEngine(ctx).In("id", branchIDs).Delete(new(Branch)) - return err + return nil }) } @@ -160,71 +165,20 @@ func UpdateBranch(ctx context.Context, repoID int64, branchName, commitID, commi } // AddDeletedBranch adds a deleted branch to the database -func AddDeletedBranch(ctx context.Context, repoID int64, branchName, commit string, deletedByID int64) error { - return db.WithTx(ctx, func(ctx context.Context) error { - if _, err := db.GetEngine(ctx).Where("repo_id = ? AND name = ?", repoID, branchName).Delete(new(Branch)); err != nil { - return err - } - - deletedBranch := &DeletedBranch{ - RepoID: repoID, - Name: branchName, - Commit: commit, +func AddDeletedBranch(ctx context.Context, repoID int64, branchName string, deletedByID int64) error { + _, err := db.GetEngine(ctx).Where("repo_id=? AND name=?", repoID, branchName). + Cols("is_deleted, deleted_by_id, deleted_unix"). + Update(&Branch{ + IsDeleted: true, DeletedByID: deletedByID, - } - _, err := db.GetEngine(ctx).Insert(deletedBranch) - return err - }) -} - -// GetDeletedBranches returns all the deleted branches -func GetDeletedBranches(ctx context.Context, repoID int64) ([]*DeletedBranch, error) { - deletedBranches := make([]*DeletedBranch, 0) - return deletedBranches, db.GetEngine(ctx).Where("repo_id = ?", repoID).Desc("deleted_unix").Find(&deletedBranches) -} - -// GetDeletedBranchByID get a deleted branch by its ID -func GetDeletedBranchByID(ctx context.Context, repoID, id int64) (*DeletedBranch, error) { - deletedBranch := &DeletedBranch{} - has, err := db.GetEngine(ctx).Where("repo_id = ?", repoID).And("id = ?", id).Get(deletedBranch) - if err != nil { - return nil, err - } - if !has { - return nil, nil - } - return deletedBranch, nil -} - -// RemoveDeletedBranchByID removes a deleted branch from the database -func RemoveDeletedBranchByID(ctx context.Context, repoID, id int64) (err error) { - deletedBranch := &DeletedBranch{ - RepoID: repoID, - ID: id, - } - - if affected, err := db.GetEngine(ctx).Delete(deletedBranch); err != nil { - return err - } else if affected != 1 { - return fmt.Errorf("remove deleted branch ID(%v) failed", id) - } - - return nil -} - -// LoadUser loads the user that deleted the branch -// When there's no user found it returns a user_model.NewGhostUser -func (deletedBranch *DeletedBranch) LoadUser(ctx context.Context) { - user, err := user_model.GetUserByID(ctx, deletedBranch.DeletedByID) - if err != nil { - user = user_model.NewGhostUser() - } - deletedBranch.DeletedBy = user + DeletedUnix: timeutil.TimeStampNow(), + }) + return err } // removeDeletedBranchByName removes all deleted branches func removeDeletedBranchByName(ctx context.Context, repoID int64, branch string) error { - _, err := db.GetEngine(ctx).Where("repo_id=? AND name=?", repoID, branch).Delete(new(DeletedBranch)) + _, err := db.GetEngine(ctx).Where("repo_id=? AND name=? AND is_deleted = ?", repoID, branch, true).Delete(new(Branch)) return err } @@ -234,7 +188,7 @@ func RemoveOldDeletedBranches(ctx context.Context, olderThan time.Duration) { log.Trace("Doing: DeletedBranchesCleanup") deleteBefore := time.Now().Add(-olderThan) - _, err := db.GetEngine(ctx).Where("deleted_unix < ?", deleteBefore.Unix()).Delete(new(DeletedBranch)) + _, err := db.GetEngine(ctx).Where("deleted_unix < ?", deleteBefore.Unix()).Delete(new(Branch)) if err != nil { log.Error("DeletedBranchesCleanup: %v", err) } @@ -271,15 +225,7 @@ func RenameBranch(ctx context.Context, repo *repo_model.Repository, from, to str sess := db.GetEngine(ctx) - // 1. update deleted branch - if _, err := sess.Where("repo_id = ? AND name=?", repo.ID, from).Update(&DeletedBranch{ - RepoID: repo.ID, - Name: to, - }); err != nil { - return err - } - - // 2. update branch in database + // 1. update branch in database if n, err := sess.Where("repo_id=? AND name=?", repo.ID, from).Update(&Branch{ Name: to, }); err != nil { @@ -289,7 +235,7 @@ func RenameBranch(ctx context.Context, repo *repo_model.Repository, from, to str return nil } - // 3. update default branch if needed + // 2. update default branch if needed isDefault := repo.DefaultBranch == from if isDefault { repo.DefaultBranch = to @@ -299,7 +245,7 @@ func RenameBranch(ctx context.Context, repo *repo_model.Repository, from, to str } } - // 4. Update protected branch if needed + // 3. Update protected branch if needed protectedBranch, err := GetProtectedBranchRuleByName(ctx, repo.ID, from) if err != nil { return err @@ -323,7 +269,7 @@ func RenameBranch(ctx context.Context, repo *repo_model.Repository, from, to str } } - // 5. Update all not merged pull request base branch name + // 4. Update all not merged pull request base branch name _, err = sess.Table("pull_request").Where("base_repo_id=? AND base_branch=? AND has_merged=?", repo.ID, from, false). Update(map[string]interface{}{"base_branch": to}) @@ -331,12 +277,12 @@ func RenameBranch(ctx context.Context, repo *repo_model.Repository, from, to str return err } - // 6. do git action + // 5. do git action if err = gitAction(isDefault); err != nil { return err } - // 7. insert renamed branch record + // 6. insert renamed branch record renamedBranch := &RenamedBranch{ RepoID: repo.ID, From: from, diff --git a/models/migrations/v1_21/v261.go b/models/migrations/v1_21/v261.go index 3e8e258d26430..e063f30af4935 100644 --- a/models/migrations/v1_21/v261.go +++ b/models/migrations/v1_21/v261.go @@ -4,6 +4,9 @@ package v1_21 //nolint import ( + "context" + + "code.gitea.io/gitea/models/db" "code.gitea.io/gitea/modules/timeutil" "xorm.io/xorm" @@ -11,15 +14,63 @@ import ( func AddBranchTable(x *xorm.Engine) error { type Branch struct { + ID int64 + RepoID int64 `xorm:"index UNIQUE(s)"` + Name string `xorm:"UNIQUE(s) NOT NULL"` + CommitSHA string + CommitMessage string `xorm:"TEXT"` + PusherID int64 + IsDeleted bool + DeletedByID int64 + DeletedUnix timeutil.TimeStamp + CommitTime timeutil.TimeStamp // The commit + CreatedUnix timeutil.TimeStamp `xorm:"created"` + UpdatedUnix timeutil.TimeStamp `xorm:"updated"` + } + + if err := x.Sync(new(Branch)); err != nil { + return err + } + + type DeletedBranch struct { ID int64 RepoID int64 `xorm:"index UNIQUE(s)"` Name string `xorm:"UNIQUE(s) NOT NULL"` Commit string - PusherID int64 - CommitTime timeutil.TimeStamp // The commit - CreatedUnix timeutil.TimeStamp `xorm:"created"` - UpdatedUnix timeutil.TimeStamp `xorm:"updated"` + IsDeleted bool + DeletedByID int64 + DeletedUnix timeutil.TimeStamp + } + + branches := make([]Branch, 0, 100) + if err := db.Iterate(context.Background(), nil, func(ctx context.Context, deletedBranch *DeletedBranch) error { + branches = append(branches, Branch{ + RepoID: deletedBranch.RepoID, + Name: deletedBranch.Name, + CommitSHA: deletedBranch.Commit, + // CommitMessage: string, FIXME: how to get this? + // CommitTime: timeutil.TimeStamp, FIXME: how to get this? + IsDeleted: true, + DeletedByID: deletedBranch.DeletedByID, + DeletedUnix: deletedBranch.DeletedUnix, + }) + if len(branches) >= 100 { + _, err := x.Insert(&branches) + if err != nil { + return err + } + branches = branches[:0] + } + return nil + }); err != nil { + return err + } + + if len(branches) > 0 { + if _, err := x.Insert(&branches); err != nil { + return err + } } - return x.Sync(new(Branch)) + return x.DropTables("deleted_branches") } diff --git a/models/repo.go b/models/repo.go index 2e0e8af16c4a5..933f7e56a3a04 100644 --- a/models/repo.go +++ b/models/repo.go @@ -147,7 +147,7 @@ func DeleteRepository(doer *user_model.User, uid, repoID int64) error { &repo_model.Collaboration{RepoID: repoID}, &issues_model.Comment{RefRepoID: repoID}, &git_model.CommitStatus{RepoID: repoID}, - &git_model.DeletedBranch{RepoID: repoID}, + &git_model.Branch{RepoID: repoID}, &git_model.LFSLock{RepoID: repoID}, &repo_model.LanguageStat{RepoID: repoID}, &issues_model.Milestone{RepoID: repoID}, diff --git a/routers/web/repo/branch.go b/routers/web/repo/branch.go index ec48c2ba75f7f..edbc76ba72b68 100644 --- a/routers/web/repo/branch.go +++ b/routers/web/repo/branch.go @@ -115,7 +115,7 @@ func RestoreBranchPost(ctx *context.Context) { if err := git.Push(ctx, ctx.Repo.Repository.RepoPath(), git.PushOptions{ Remote: ctx.Repo.Repository.RepoPath(), - Branch: fmt.Sprintf("%s:%s%s", deletedBranch.Commit, git.BranchPrefix, deletedBranch.Name), + Branch: fmt.Sprintf("%s:%s%s", deletedBranch.CommitSHA, git.BranchPrefix, deletedBranch.Name), Env: repo_module.PushingEnvironment(ctx.Doer, ctx.Repo.Repository), }); err != nil { if strings.Contains(err.Error(), "already exists") { @@ -133,7 +133,7 @@ func RestoreBranchPost(ctx *context.Context) { &repo_module.PushUpdateOptions{ RefFullName: git.RefNameFromBranch(deletedBranch.Name), OldCommitID: git.EmptySHA, - NewCommitID: deletedBranch.Commit, + NewCommitID: deletedBranch.CommitSHA, PusherID: ctx.Doer.ID, PusherName: ctx.Doer.Name, RepoUserName: ctx.Repo.Owner.Name, diff --git a/services/repository/branch.go b/services/repository/branch.go index b3c9ae17d7781..ef084eb577aef 100644 --- a/services/repository/branch.go +++ b/services/repository/branch.go @@ -54,9 +54,7 @@ func CreateNewBranch(ctx context.Context, doer *user_model.User, repo *repo_mode type Branch struct { RawBranch *git_model.Branch IsProtected bool - IsDeleted bool IsIncluded bool - DeletedBranch *git_model.DeletedBranch CommitsAhead int CommitsBehind int LatestPullRequest *issues_model.PullRequest @@ -64,7 +62,6 @@ type Branch struct { } // LoadBranches loads branches from the repository limited by page & pageSize. -// NOTE: May write to context on error. func LoadBranches(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository, includeDeletedBranch bool, page, pageSize int) (*Branch, []*Branch, int64, error) { defaultRawBranch, err := git_model.GetDefaultBranch(ctx, repo) if err != nil { diff --git a/services/repository/push.go b/services/repository/push.go index f159fb71b92b2..1564eb5563559 100644 --- a/services/repository/push.go +++ b/services/repository/push.go @@ -274,7 +274,7 @@ func pushUpdates(optsList []*repo_module.PushUpdateOptions) error { log.Error("close related pull request failed: %v", err) } - if err := git_model.AddDeletedBranch(db.DefaultContext, repo.ID, branch, opts.OldCommitID, pusher.ID); err != nil { + if err := git_model.AddDeletedBranch(db.DefaultContext, repo.ID, branch, pusher.ID); err != nil { log.Warn("AddDeletedBranch: %v", err) } } From 60d12586cefdb5b5426c86330b47d707904bcf47 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Thu, 15 Jun 2023 18:14:58 +0800 Subject: [PATCH 07/56] Fix test --- models/git/branch_list.go | 65 +++++++++++++++++++++++++++++++++ models/git/branches.go | 36 +----------------- services/repository/branch.go | 10 +++-- templates/repo/branch/list.tmpl | 14 +++---- 4 files changed, 81 insertions(+), 44 deletions(-) create mode 100644 models/git/branch_list.go diff --git a/models/git/branch_list.go b/models/git/branch_list.go new file mode 100644 index 0000000000000..182ef3df33ff8 --- /dev/null +++ b/models/git/branch_list.go @@ -0,0 +1,65 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package git + +import ( + "context" + + "code.gitea.io/gitea/models/db" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/container" + "xorm.io/builder" +) + +type BranchList []*Branch + +func (branches BranchList) LoadDeletedBy(ctx context.Context) error { + ids := container.Set[int64]{} + for _, branch := range branches { + ids.Add(branch.DeletedByID) + } + usersMap := make(map[int64]*user_model.User, len(ids)) + if err := db.GetEngine(ctx).In("id", ids.Values()).Find(&usersMap); err != nil { + return err + } + for _, branch := range branches { + branch.DeletedBy = usersMap[branch.DeletedByID] + if branch.DeletedBy == nil { + branch.DeletedBy = user_model.NewGhostUser() + } + } + return nil +} + +func LoadAllBranches(ctx context.Context, repoID int64) ([]*Branch, error) { + var branches []*Branch + err := db.GetEngine(ctx).Where("repo_id=?", repoID).Find(&branches) + return branches, err +} + +type FindBranchOptions struct { + db.ListOptions + RepoID int64 + IncludeDefaultBranch bool + IncludeDeletedBranch bool +} + +func FindBranches(ctx context.Context, opts FindBranchOptions) (BranchList, int64, error) { + sess := db.GetEngine(ctx).Where("repo_id=?", opts.RepoID) + if opts.PageSize > 0 { + sess = db.SetSessionPagination(sess, &opts.ListOptions) + } + if !opts.IncludeDefaultBranch { + sess = sess.And(builder.Neq{"name": builder.Select("default_branch").From("repository").Where(builder.Eq{"id": opts.RepoID})}) + } + if !opts.IncludeDeletedBranch { + sess.And(builder.Eq{"is_deleted": false}) + } + var branches []*Branch + total, err := sess.FindAndCount(&branches) + if err != nil { + return nil, 0, err + } + return branches, total, err +} diff --git a/models/git/branches.go b/models/git/branches.go index 44cb2e8405506..0d57e847c9b0e 100644 --- a/models/git/branches.go +++ b/models/git/branches.go @@ -9,11 +9,10 @@ import ( "code.gitea.io/gitea/models/db" repo_model "code.gitea.io/gitea/models/repo" + user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/timeutil" - - "xorm.io/builder" ) // Branch represents a branch of a repository @@ -28,6 +27,7 @@ type Branch struct { PusherID int64 IsDeleted bool DeletedByID int64 + DeletedBy *user_model.User `xorm:"-"` DeletedUnix timeutil.TimeStamp CommitTime timeutil.TimeStamp // The commit CreatedUnix timeutil.TimeStamp `xorm:"created"` @@ -39,12 +39,6 @@ func init() { db.RegisterModel(new(RenamedBranch)) } -func LoadAllBranches(ctx context.Context, repoID int64) ([]*Branch, error) { - var branches []*Branch - err := db.GetEngine(ctx).Where("repo_id=?", repoID).Find(&branches) - return branches, err -} - func GetDefaultBranch(ctx context.Context, repo *repo_model.Repository) (*Branch, error) { var branch Branch has, err := db.GetEngine(ctx).Where("repo_id=?", repo.ID).And("name=?", repo.DefaultBranch).Get(&branch) @@ -56,32 +50,6 @@ func GetDefaultBranch(ctx context.Context, repo *repo_model.Repository) (*Branch return &branch, nil } -type FindBranchOptions struct { - db.ListOptions - RepoID int64 - IncludeDefaultBranch bool - IncludeDeletedBranch bool -} - -func FindBranches(ctx context.Context, opts FindBranchOptions) ([]*Branch, int64, error) { - sess := db.GetEngine(ctx).Where("repo_id=?", opts.RepoID) - if opts.PageSize > 0 { - sess = db.SetSessionPagination(sess, &opts.ListOptions) - } - if !opts.IncludeDefaultBranch { - sess = sess.And(builder.Neq{"name": builder.Select("default_branch").From("repository").Where(builder.Eq{"id": opts.RepoID})}) - } - if !opts.IncludeDeletedBranch { - sess.And(builder.Eq{"is_deleted": false}) - } - var branches []*Branch - total, err := sess.FindAndCount(&branches) - if err != nil { - return nil, 0, err - } - return branches, total, err -} - func AddBranch(ctx context.Context, branch *Branch) error { return db.WithTx(ctx, func(ctx context.Context) error { if _, err := db.GetEngine(ctx).Insert(branch); err != nil { diff --git a/services/repository/branch.go b/services/repository/branch.go index ef084eb577aef..9dad8d1dc5dcc 100644 --- a/services/repository/branch.go +++ b/services/repository/branch.go @@ -81,6 +81,10 @@ func LoadBranches(ctx context.Context, repo *repo_model.Repository, gitRepo *git return nil, nil, 0, err } + if err := rawBranches.LoadDeletedBy(ctx); err != nil { + return nil, nil, 0, err + } + rules, err := git_model.FindRepoProtectedBranchRules(ctx, repo.ID) if err != nil { return nil, nil, 0, err @@ -92,7 +96,7 @@ func LoadBranches(ctx context.Context, repo *repo_model.Repository, gitRepo *git repoIDToGitRepo := map[int64]*git.Repository{} repoIDToGitRepo[repo.ID] = gitRepo - var branches []*Branch + branches := make([]*Branch, 0, len(rawBranches)) for i := range rawBranches { branch, err := loadOneBranch(ctx, repo, rawBranches[i], &rules, repoIDToRepo, repoIDToGitRepo) if err != nil { @@ -128,11 +132,11 @@ func loadOneBranch(ctx context.Context, repo *repo_model.Repository, rawBranch * } // it's not default branch - if repo.DefaultBranch != rawBranch.Name { + if repo.DefaultBranch != rawBranch.Name && !rawBranch.IsDeleted { var err error divergence, err = files_service.CountDivergingCommits(ctx, repo, git.BranchPrefix+branchName) if err != nil { - log.Error("CountDivergingCommits", err) + log.Error("CountDivergingCommits: %v", err) } } diff --git a/templates/repo/branch/list.tmpl b/templates/repo/branch/list.tmpl index 6542548dadcd4..9fe69e0624078 100644 --- a/templates/repo/branch/list.tmpl +++ b/templates/repo/branch/list.tmpl @@ -75,9 +75,9 @@ {{range .Branches}} - {{if .IsDeleted}} + {{if .RawBranch.IsDeleted}} {{.RawBranch.Name}} -

{{$.locale.Tr "repo.branch.deleted_by" .DeletedBranch.DeletedBy.Name}} {{TimeSinceUnix .DeletedBranch.DeletedUnix $.locale}}

+

{{$.locale.Tr "repo.branch.deleted_by" .RawBranch.DeletedBy.Name}} {{TimeSinceUnix .RawBranch.DeletedUnix $.locale}}

{{else}} {{if .IsProtected}} {{svg "octicon-shield-lock"}} @@ -87,7 +87,7 @@ {{end}} - {{if and (not .IsDeleted) $.DefaultBranch}} + {{if and (not .RawBranch.IsDeleted) $.DefaultBranch}}
{{.CommitsBehind}}
@@ -130,7 +130,7 @@ {{end}} - {{if and $.IsWriter (not $.Repository.IsArchived) (not .IsDeleted)}} + {{if and $.IsWriter (not $.Repository.IsArchived) (not .RawBranch.IsDeleted)}} {{end}} {{else if and .LatestPullRequest.HasMerged .MergeMovedOn}} - {{if and (not .IsDeleted) $.AllowsPulls (gt .CommitsAhead 0)}} + {{if and (not .RawBranch.IsDeleted) $.AllowsPulls (gt .CommitsAhead 0)}} @@ -164,7 +164,7 @@ {{end}} {{if and $.IsWriter (not $.IsMirror) (not $.Repository.IsArchived) (not .IsProtected)}} {{if .RawBranch.IsDeleted}} - + + {{.locale.Tr "admin.dashboard.sync_repo_sync"}} + + From b282988376e2a03ccd4f8242388a97cef2fefc57 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Thu, 22 Jun 2023 17:01:08 +0800 Subject: [PATCH 30/56] Revert unnecessary cron job and fix lint --- routers/web/repo/compare.go | 2 +- services/cron/tasks_extended.go | 14 -------------- 2 files changed, 1 insertion(+), 15 deletions(-) diff --git a/routers/web/repo/compare.go b/routers/web/repo/compare.go index 62414fadf429e..d89117ffae6e0 100644 --- a/routers/web/repo/compare.go +++ b/routers/web/repo/compare.go @@ -741,7 +741,7 @@ func CompareDiff(ctx *context.Context) { } headBranches, err := git_model.FindBranchNames(ctx, git_model.FindBranchOptions{ - RepoID: ctx.Repo.Repository.ID, + RepoID: ci.HeadRepo.ID, ListOptions: db.ListOptions{ PageSize: 1, }, diff --git a/services/cron/tasks_extended.go b/services/cron/tasks_extended.go index 2309438858f2c..acf2d3373c8ea 100644 --- a/services/cron/tasks_extended.go +++ b/services/cron/tasks_extended.go @@ -213,20 +213,6 @@ func registerGCLFS() { }) } -func registerSyncBranches() { - RegisterTaskFatal("sync_repo_sync", &OlderThanConfig{ - BaseConfig: BaseConfig{ - Enabled: false, - RunAtStart: false, - Schedule: "@every 168h", - }, - OlderThan: 365 * 24 * time.Hour, - }, func(ctx context.Context, _ *user_model.User, config Config) error { - olderThanConfig := config.(*OlderThanConfig) - return system.DeleteOldSystemNotices(olderThanConfig.OlderThan) - }) -} - func initExtendedTasks() { registerDeleteInactiveUsers() registerDeleteRepositoryArchives() From 968e1ec0554ca463189e87e28f2570323dddeec6 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Fri, 23 Jun 2023 00:14:25 +0800 Subject: [PATCH 31/56] Add error track --- services/repository/branch.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/repository/branch.go b/services/repository/branch.go index 916ef57bfd479..97461db94c418 100644 --- a/services/repository/branch.go +++ b/services/repository/branch.go @@ -393,7 +393,7 @@ func initBranchSyncQueue() error { admin, err := user_model.GetAdminUser() if err != nil { - return err + return fmt.Errorf("user_model.GetAdminUser: %v", err) } cnt, err := git_model.CountBranches(context.Background(), git_model.FindBranchOptions{ @@ -401,7 +401,7 @@ func initBranchSyncQueue() error { IsDeletedBranch: util.OptionalBoolFalse, }) if err != nil { - return err + return fmt.Errorf("CountBranches: %v", err) } if cnt == 0 { From efa33010631108dd711853c4acada169d4fd08e1 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sat, 24 Jun 2023 14:24:06 +0800 Subject: [PATCH 32/56] Prepare data when database is ready --- cmd/web.go | 2 +- routers/init.go | 7 ++++++- tests/test_utils.go | 4 +++- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/cmd/web.go b/cmd/web.go index 7a257a62a277d..a7827ce0f72ac 100644 --- a/cmd/web.go +++ b/cmd/web.go @@ -171,7 +171,7 @@ func serveInstalled(ctx *cli.Context) error { } } - routers.InitWebInstalled(graceful.GetManager().HammerContext()) + routers.InitWebInstalled(graceful.GetManager().HammerContext(), nil) // We check that AppDataPath exists here (it should have been created during installation) // We can't check it in `InitWebInstalled`, because some integration tests diff --git a/routers/init.go b/routers/init.go index ddbabcc397447..a3a1dcec458d4 100644 --- a/routers/init.go +++ b/routers/init.go @@ -107,7 +107,7 @@ func InitWebInstallPage(ctx context.Context) { } // InitWebInstalled is for global installed configuration. -func InitWebInstalled(ctx context.Context) { +func InitWebInstalled(ctx context.Context, extraDBPrepare func() error) { mustInitCtx(ctx, git.InitFull) log.Info("Git version: %s (home: %s)", git.VersionInfo(), git.HomeDir()) @@ -134,6 +134,11 @@ func InitWebInstalled(ctx context.Context) { mustInitCtx(ctx, common.InitDBEngine) log.Info("ORM engine initialization successful!") + if extraDBPrepare != nil { + if err := extraDBPrepare(); err != nil { + log.Fatal("prepare database data failed: %v", err) + } + } mustInit(system.Init) mustInit(oauth2.Init) diff --git a/tests/test_utils.go b/tests/test_utils.go index bf7d1b3fbe63b..51afbfe4f152f 100644 --- a/tests/test_utils.go +++ b/tests/test_utils.go @@ -173,7 +173,9 @@ func InitTest(requireGitea bool) { defer db.Close() } - routers.InitWebInstalled(graceful.GetManager().HammerContext()) + routers.InitWebInstalled(graceful.GetManager().HammerContext(), func() error { + return unittest.LoadFixtures() + }) } func PrepareTestEnv(t testing.TB, skip ...int) func() { From adbfecd59f1fb57c8f1fc5743dae7c350c86e45e Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sat, 24 Jun 2023 22:12:55 +0800 Subject: [PATCH 33/56] fix test --- tests/test_utils.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/test_utils.go b/tests/test_utils.go index 51afbfe4f152f..8d27e4c822980 100644 --- a/tests/test_utils.go +++ b/tests/test_utils.go @@ -174,6 +174,13 @@ func InitTest(requireGitea bool) { } routers.InitWebInstalled(graceful.GetManager().HammerContext(), func() error { + if err := unittest.InitFixtures( + unittest.FixturesOptions{ + Dir: filepath.Join(filepath.Dir(setting.AppPath), "models/fixtures/"), + }, + ); err != nil { + return err + } return unittest.LoadFixtures() }) } From b1c8b39db9c8fec77d33d15d4a2a7f83337cbb5c Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sun, 25 Jun 2023 07:54:01 +0800 Subject: [PATCH 34/56] Fix lint --- services/repository/branch.go | 1 + 1 file changed, 1 insertion(+) diff --git a/services/repository/branch.go b/services/repository/branch.go index 97461db94c418..7c75cce9f5389 100644 --- a/services/repository/branch.go +++ b/services/repository/branch.go @@ -23,6 +23,7 @@ import ( repo_module "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/util" files_service "code.gitea.io/gitea/services/repository/files" + "xorm.io/builder" ) From 6e90189d204bdfb236fb33205be3141904f93a82 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sun, 25 Jun 2023 08:05:27 +0800 Subject: [PATCH 35/56] Fix check --- templates/swagger/v1_json.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/swagger/v1_json.tmpl b/templates/swagger/v1_json.tmpl index e78c077fc3b32..000df5bd7ead7 100644 --- a/templates/swagger/v1_json.tmpl +++ b/templates/swagger/v1_json.tmpl @@ -22923,4 +22923,4 @@ "TOTPHeader": [] } ] -} +} \ No newline at end of file From 2761e08c50add7b2b3a698af53489e86f78f9f62 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sun, 25 Jun 2023 08:19:11 +0800 Subject: [PATCH 36/56] Add lin ending --- templates/swagger/v1_json.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/swagger/v1_json.tmpl b/templates/swagger/v1_json.tmpl index 000df5bd7ead7..e78c077fc3b32 100644 --- a/templates/swagger/v1_json.tmpl +++ b/templates/swagger/v1_json.tmpl @@ -22923,4 +22923,4 @@ "TOTPHeader": [] } ] -} \ No newline at end of file +} From 67a83b962e09118eccba4e15fb11f57b9c9cb077 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sun, 25 Jun 2023 09:58:35 +0800 Subject: [PATCH 37/56] Fix bug --- routers/web/repo/compare.go | 1 + 1 file changed, 1 insertion(+) diff --git a/routers/web/repo/compare.go b/routers/web/repo/compare.go index d89117ffae6e0..059ea6bb79a29 100644 --- a/routers/web/repo/compare.go +++ b/routers/web/repo/compare.go @@ -689,6 +689,7 @@ func getBranchesAndTagsForRepo(ctx gocontext.Context, repo *repo_model.Repositor ListOptions: db.ListOptions{ PageSize: -1, }, + IsDeletedBranch: util.OptionalBoolFalse, }) if err != nil { return nil, nil, err From 8279c91c39573e9dac0a2a22c64a42158cafd222 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sun, 25 Jun 2023 23:58:37 +0800 Subject: [PATCH 38/56] Follow wxiaoguang's suggestion --- cmd/admin.go | 36 ----------------------------------- models/git/branch_list.go | 7 +++---- models/git/branches.go | 31 +++++++++++++++++------------- modules/repository/branch.go | 13 +++---------- routers/api/v1/repo/branch.go | 6 +++++- 5 files changed, 29 insertions(+), 64 deletions(-) diff --git a/cmd/admin.go b/cmd/admin.go index 55b1be815355a..f9fb1b6c68f07 100644 --- a/cmd/admin.go +++ b/cmd/admin.go @@ -16,7 +16,6 @@ import ( auth_model "code.gitea.io/gitea/models/auth" "code.gitea.io/gitea/models/db" repo_model "code.gitea.io/gitea/models/repo" - user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/graceful" "code.gitea.io/gitea/modules/log" @@ -38,7 +37,6 @@ var ( Subcommands: []cli.Command{ subcmdUser, subcmdRepoSyncReleases, - subcmdRepoSyncBranches, subcmdRegenerate, subcmdAuth, subcmdSendMail, @@ -51,19 +49,6 @@ var ( Action: runRepoSyncReleases, } - subcmdRepoSyncBranches = cli.Command{ - Name: "repo-sync-branches", - Usage: "Synchronize repository branches", - Flags: []cli.Flag{ - cli.IntFlag{ - Name: "repo-id", - Usage: "Repository id to be synchronized", - Value: 0, - }, - }, - Action: runRepoSyncBranches, - } - subcmdRegenerate = cli.Command{ Name: "regenerate", Usage: "Regenerate specific files", @@ -415,27 +400,6 @@ func runRepoSyncReleases(_ *cli.Context) error { return nil } -func runRepoSyncBranches(cli *cli.Context) error { - ctx, cancel := installSignals() - defer cancel() - - if err := initDB(ctx); err != nil { - return err - } - - doer, err := user_model.GetAdminUser() - if err != nil { - return err - } - - if !cli.IsSet("repo-id") { - return repo_module.SyncAllBranches(ctx, doer.ID) - } - - repoID := cli.Int64("repo-id") - return repo_module.SyncRepoBranches(ctx, repoID, doer.ID) -} - func getReleaseCount(id int64) (int64, error) { return repo_model.GetReleaseCountByRepoID( db.DefaultContext, diff --git a/models/git/branch_list.go b/models/git/branch_list.go index 8b663c20d827d..de712042bd33b 100644 --- a/models/git/branch_list.go +++ b/models/git/branch_list.go @@ -35,7 +35,7 @@ func (branches BranchList) LoadDeletedBy(ctx context.Context) error { } // LoadAllBranches loads all branches of a repository from database includes deleted branches -func LoadAllBranches(ctx context.Context, repoID int64) ([]*Branch, error) { +func LoadAllBranches(ctx context.Context, repoID int64) (BranchList, error) { var branches []*Branch err := db.GetEngine(ctx).Where("repo_id=?", repoID). Find(&branches) @@ -51,9 +51,8 @@ type FindBranchOptions struct { func (opts *FindBranchOptions) Cond() builder.Cond { cond := builder.NewCond() - if opts.RepoID > 0 { - cond = cond.And(builder.Eq{"repo_id": opts.RepoID}) - } + cond = cond.And(builder.Eq{"repo_id": opts.RepoID}) + if !opts.IncludeDefaultBranch { cond = cond.And(builder.Neq{"name": builder.Select("default_branch").From("repository").Where(builder.Eq{"id": opts.RepoID})}) } diff --git a/models/git/branches.go b/models/git/branches.go index 0089656a62de4..4006088dee851 100644 --- a/models/git/branches.go +++ b/models/git/branches.go @@ -11,29 +11,28 @@ import ( "code.gitea.io/gitea/models/db" repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" - "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/timeutil" "code.gitea.io/gitea/modules/util" ) -// ErrBranchDoesNotExist represents an error that branch with such name does not exist. -type ErrBranchDoesNotExist struct { +// ErrBranchNotExist represents an error that branch with such name does not exist. +type ErrBranchNotExist struct { RepoID int64 BranchName string } -// IsErrBranchDoesNotExist checks if an error is an ErrBranchDoesNotExist. -func IsErrBranchDoesNotExist(err error) bool { - _, ok := err.(ErrBranchDoesNotExist) +// IsErrBranchNotExist checks if an error is an ErrBranchDoesNotExist. +func IsErrBranchNotExist(err error) bool { + _, ok := err.(ErrBranchNotExist) return ok } -func (err ErrBranchDoesNotExist) Error() string { +func (err ErrBranchNotExist) Error() string { return fmt.Sprintf("branch does not exist [repo_id: %d name: %s]", err.RepoID, err.BranchName) } -func (err ErrBranchDoesNotExist) Unwrap() error { +func (err ErrBranchNotExist) Unwrap() error { return util.ErrNotExist } @@ -100,7 +99,7 @@ func (err ErrBranchesEqual) Unwrap() error { // for pagination, keyword search and filtering type Branch struct { ID int64 - RepoID int64 `xorm:"index UNIQUE(s)"` + RepoID int64 `xorm:"UNIQUE(s)"` Name string `xorm:"UNIQUE(s) NOT NULL"` CommitSHA string CommitMessage string `xorm:"TEXT"` @@ -136,7 +135,7 @@ func GetBranch(ctx context.Context, repoID int64, branchName string) (*Branch, e if err != nil { return nil, err } else if !has { - return nil, ErrBranchDoesNotExist{ + return nil, ErrBranchNotExist{ RepoID: repoID, BranchName: branchName, } @@ -159,13 +158,19 @@ func GetDeletedBranchByID(ctx context.Context, repoID, branchID int64) (*Branch, if err != nil { return nil, err } else if !has { - return nil, git.ErrNotExist{} + return nil, ErrBranchNotExist{ + RepoID: repoID, + } } if branch.RepoID != repoID { - return nil, git.ErrNotExist{} + return nil, ErrBranchNotExist{ + RepoID: repoID, + } } if !branch.IsDeleted { - return nil, git.ErrNotExist{} + return nil, ErrBranchNotExist{ + RepoID: repoID, + } } return &branch, nil } diff --git a/modules/repository/branch.go b/modules/repository/branch.go index 497096926f5f3..31fe388b08bed 100644 --- a/modules/repository/branch.go +++ b/modules/repository/branch.go @@ -43,16 +43,9 @@ func SyncRepoBranches(ctx context.Context, repoID, doerID int64) error { } func SyncRepoBranchesWithRepo(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository, doerID int64) error { - var allBranches []string - for page := 0; ; page++ { - branches, _, err := gitRepo.GetBranchNames(page*100, 100) - if err != nil { - return err - } - allBranches = append(allBranches, branches...) - if len(branches) < 100 { - break - } + allBranches, _, err := gitRepo.GetBranchNames(0, 0) + if err != nil { + return err } log.Trace("SyncRepoBranches[%s]: branches[%d]: %v", repo.FullName(), len(allBranches), allBranches) diff --git a/routers/api/v1/repo/branch.go b/routers/api/v1/repo/branch.go index 3a026504f0b69..7b18eb758e879 100644 --- a/routers/api/v1/repo/branch.go +++ b/routers/api/v1/repo/branch.go @@ -16,6 +16,7 @@ import ( "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/git" api "code.gitea.io/gitea/modules/structs" + "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/modules/web" "code.gitea.io/gitea/routers/api/v1/utils" "code.gitea.io/gitea/services/convert" @@ -288,7 +289,10 @@ func ListBranches(ctx *context.APIContext) { } branches, total, err := git_model.FindBranches(ctx, git_model.FindBranchOptions{ - ListOptions: listOptions, + ListOptions: listOptions, + RepoID: ctx.Repo.Repository.ID, + IncludeDefaultBranch: true, + IsDeletedBranch: util.OptionalBoolFalse, }) if err != nil { ctx.Error(http.StatusInternalServerError, "GetBranches", err) From 178c9657b838d2ae84445c4eecd0903a25fb1423 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Mon, 26 Jun 2023 08:52:07 +0800 Subject: [PATCH 39/56] Follow wxiaoguang's suggestion --- models/git/branch_list.go | 24 ++++++++++------------- models/git/branches.go | 2 +- models/user/user.go | 4 ++-- modules/context/repo.go | 12 +++++------- routers/api/v1/repo/branch.go | 9 ++++----- routers/api/v1/repo/file.go | 2 +- routers/api/v1/repo/patch.go | 2 +- routers/web/admin/admin.go | 1 - routers/web/repo/branch.go | 4 +--- routers/web/repo/issue.go | 2 +- services/migrations/dump.go | 4 ++-- services/pull/pull.go | 2 +- services/pull/temp_repo.go | 2 +- services/pull/update.go | 2 +- services/repository/branch.go | 23 +++++++++++----------- services/repository/repository.go | 3 ++- templates/repo/branch/list.tmpl | 32 +++++++++++++++---------------- 17 files changed, 60 insertions(+), 70 deletions(-) diff --git a/models/git/branch_list.go b/models/git/branch_list.go index de712042bd33b..2dc4b6c8f46f8 100644 --- a/models/git/branch_list.go +++ b/models/git/branch_list.go @@ -44,17 +44,19 @@ func LoadAllBranches(ctx context.Context, repoID int64) (BranchList, error) { type FindBranchOptions struct { db.ListOptions - RepoID int64 - IncludeDefaultBranch bool - IsDeletedBranch util.OptionalBool + RepoID int64 + ExcludeBranchNames []string + IsDeletedBranch util.OptionalBool } func (opts *FindBranchOptions) Cond() builder.Cond { cond := builder.NewCond() - cond = cond.And(builder.Eq{"repo_id": opts.RepoID}) + if opts.RepoID > 0 { + cond = cond.And(builder.Eq{"repo_id": opts.RepoID}) + } - if !opts.IncludeDefaultBranch { - cond = cond.And(builder.Neq{"name": builder.Select("default_branch").From("repository").Where(builder.Eq{"id": opts.RepoID})}) + if len(opts.ExcludeBranchNames) > 0 { + cond = cond.And(builder.NotIn("name", opts.ExcludeBranchNames)) } if !opts.IsDeletedBranch.IsNone() { cond = cond.And(builder.Eq{"is_deleted": opts.IsDeletedBranch.IsTrue()}) @@ -68,7 +70,7 @@ func CountBranches(ctx context.Context, opts FindBranchOptions) (int64, error) { func FindBranches(ctx context.Context, opts FindBranchOptions) (BranchList, int64, error) { sess := db.GetEngine(ctx).Where(opts.Cond()) - if opts.PageSize > 0 { + if opts.PageSize > 0 && !opts.IsListAll() { sess = db.SetSessionPagination(sess, &opts.ListOptions) } @@ -81,16 +83,10 @@ func FindBranches(ctx context.Context, opts FindBranchOptions) (BranchList, int6 } func FindBranchNames(ctx context.Context, opts FindBranchOptions) ([]string, error) { - sess := db.GetEngine(ctx).Select("name").Where("repo_id=?", opts.RepoID) + sess := db.GetEngine(ctx).Select("name").Where(opts.Cond()) if opts.PageSize > 0 { sess = db.SetSessionPagination(sess, &opts.ListOptions) } - if !opts.IncludeDefaultBranch { - sess = sess.And(builder.Neq{"name": builder.Select("default_branch").From("repository").Where(builder.Eq{"id": opts.RepoID})}) - } - if !opts.IsDeletedBranch.IsNone() { - sess.And(builder.Eq{"is_deleted": opts.IsDeletedBranch.IsTrue()}) - } var branches []string if err := sess.Table("branch").Find(&branches); err != nil { return nil, err diff --git a/models/git/branches.go b/models/git/branches.go index 4006088dee851..3032fdf601954 100644 --- a/models/git/branches.go +++ b/models/git/branches.go @@ -291,7 +291,7 @@ func RenameBranch(ctx context.Context, repo *repo_model.Repository, from, to str }); err != nil { return err } else if n <= 0 { - return ErrBranchDoesNotExist{ + return ErrBranchNotExist{ RepoID: repo.ID, BranchName: from, } diff --git a/models/user/user.go b/models/user/user.go index 2077d55f513e0..6f9c2f5b35a8f 100644 --- a/models/user/user.go +++ b/models/user/user.go @@ -1171,9 +1171,9 @@ func GetUserByOpenID(uri string) (*User, error) { } // GetAdminUser returns the first administrator -func GetAdminUser() (*User, error) { +func GetAdminUser(ctx context.Context) (*User, error) { var admin User - has, err := db.GetEngine(db.DefaultContext). + has, err := db.GetEngine(ctx). Where("is_admin=?", true). Asc("id"). // Reliably get the admin with the lowest ID. Get(&admin) diff --git a/modules/context/repo.go b/modules/context/repo.go index b073ace4556e8..7cbfd65cbd702 100644 --- a/modules/context/repo.go +++ b/modules/context/repo.go @@ -668,9 +668,8 @@ func RepoAssignment(ctx *Context) (cancel context.CancelFunc) { ctx.Data["Tags"] = tags brs, err := git_model.FindBranchNames(ctx, git_model.FindBranchOptions{ - RepoID: ctx.Repo.Repository.ID, - IncludeDefaultBranch: true, - IsDeletedBranch: util.OptionalBoolFalse, + RepoID: ctx.Repo.Repository.ID, + IsDeletedBranch: util.OptionalBoolFalse, ListOptions: db.ListOptions{ PageSize: -1, }, @@ -684,7 +683,7 @@ func RepoAssignment(ctx *Context) (cancel context.CancelFunc) { if ctx.Doer != nil { doerID = ctx.Doer.ID } else { - admin, err := user_model.GetAdminUser() + admin, err := user_model.GetAdminUser(ctx) if err != nil { ctx.ServerError("GetAdminUser", err) return @@ -696,9 +695,8 @@ func RepoAssignment(ctx *Context) (cancel context.CancelFunc) { return } brs, err = git_model.FindBranchNames(ctx, git_model.FindBranchOptions{ - RepoID: ctx.Repo.Repository.ID, - IncludeDefaultBranch: true, - IsDeletedBranch: util.OptionalBoolFalse, + RepoID: ctx.Repo.Repository.ID, + IsDeletedBranch: util.OptionalBoolFalse, ListOptions: db.ListOptions{ PageSize: -1, }, diff --git a/routers/api/v1/repo/branch.go b/routers/api/v1/repo/branch.go index 7b18eb758e879..dbdfd4c9e0c4c 100644 --- a/routers/api/v1/repo/branch.go +++ b/routers/api/v1/repo/branch.go @@ -204,7 +204,7 @@ func CreateBranch(ctx *context.APIContext) { err = repo_service.CreateNewBranchFromCommit(ctx, ctx.Doer, ctx.Repo.Repository, oldCommit.ID.String(), opt.BranchName) if err != nil { - if git_model.IsErrBranchDoesNotExist(err) { + if git_model.IsErrBranchNotExist(err) { ctx.Error(http.StatusNotFound, "", "The old branch does not exist") } if models.IsErrTagAlreadyExists(err) { @@ -289,10 +289,9 @@ func ListBranches(ctx *context.APIContext) { } branches, total, err := git_model.FindBranches(ctx, git_model.FindBranchOptions{ - ListOptions: listOptions, - RepoID: ctx.Repo.Repository.ID, - IncludeDefaultBranch: true, - IsDeletedBranch: util.OptionalBoolFalse, + ListOptions: listOptions, + RepoID: ctx.Repo.Repository.ID, + IsDeletedBranch: util.OptionalBoolFalse, }) if err != nil { ctx.Error(http.StatusInternalServerError, "GetBranches", err) diff --git a/routers/api/v1/repo/file.go b/routers/api/v1/repo/file.go index 6d9573c83b89c..48f890ee552b1 100644 --- a/routers/api/v1/repo/file.go +++ b/routers/api/v1/repo/file.go @@ -692,7 +692,7 @@ func handleCreateOrUpdateFileError(ctx *context.APIContext, err error) { ctx.Error(http.StatusUnprocessableEntity, "Invalid", err) return } - if git_model.IsErrBranchDoesNotExist(err) || git.IsErrBranchNotExist(err) { + if git_model.IsErrBranchNotExist(err) || git.IsErrBranchNotExist(err) { ctx.Error(http.StatusNotFound, "BranchDoesNotExist", err) return } diff --git a/routers/api/v1/repo/patch.go b/routers/api/v1/repo/patch.go index eedcb4d38c3cb..d2f055355de0d 100644 --- a/routers/api/v1/repo/patch.go +++ b/routers/api/v1/repo/patch.go @@ -97,7 +97,7 @@ func ApplyDiffPatch(ctx *context.APIContext) { ctx.Error(http.StatusUnprocessableEntity, "Invalid", err) return } - if git_model.IsErrBranchDoesNotExist(err) || git.IsErrBranchNotExist(err) { + if git_model.IsErrBranchNotExist(err) || git.IsErrBranchNotExist(err) { ctx.Error(http.StatusNotFound, "BranchDoesNotExist", err) return } diff --git a/routers/web/admin/admin.go b/routers/web/admin/admin.go index 8a6430b637974..2e710019e2e22 100644 --- a/routers/web/admin/admin.go +++ b/routers/web/admin/admin.go @@ -142,7 +142,6 @@ func DashboardPost(ctx *context.Context) { log.Error("AddAllRepoBranchesToSyncQueue: %v: %v", ctx.Doer.ID, err) } }() - case "": default: task := cron.GetTask(form.Op) if task != nil { diff --git a/routers/web/repo/branch.go b/routers/web/repo/branch.go index 2fb8b66a75a5b..2670da85c5deb 100644 --- a/routers/web/repo/branch.go +++ b/routers/web/repo/branch.go @@ -37,7 +37,6 @@ const ( func Branches(ctx *context.Context) { ctx.Data["Title"] = "Branches" ctx.Data["IsRepoToolbarBranches"] = true - ctx.Data["DefaultBranch"] = ctx.Repo.Repository.DefaultBranch ctx.Data["AllowsPulls"] = ctx.Repo.Repository.AllowsPulls() ctx.Data["IsWriter"] = ctx.Repo.CanWrite(unit.TypeCode) ctx.Data["IsMirror"] = ctx.Repo.Repository.IsMirror @@ -52,7 +51,6 @@ func Branches(ctx *context.Context) { } pageSize := setting.Git.BranchesRangeSize - log.Debug("Branches: page: %d pageSize: %d", page, pageSize) defaultBranch, branches, branchesCount, err := repo_service.LoadBranches(ctx, ctx.Repo.Repository, ctx.Repo.GitRepo, util.OptionalBoolNone, page, pageSize) if err != nil { ctx.ServerError("LoadBranches", err) @@ -60,7 +58,7 @@ func Branches(ctx *context.Context) { } ctx.Data["Branches"] = branches - ctx.Data["DefaultBranch"] = defaultBranch + ctx.Data["DefaultBranchBranch"] = defaultBranch pager := context.NewPagination(int(branchesCount), pageSize, page, 5) pager.SetDefaultParams(ctx) ctx.Data["Page"] = pager diff --git a/routers/web/repo/issue.go b/routers/web/repo/issue.go index 706b8a9c4caf8..4f14cc381f9d3 100644 --- a/routers/web/repo/issue.go +++ b/routers/web/repo/issue.go @@ -788,7 +788,7 @@ func RetrieveRepoMetas(ctx *context.Context, repo *repo_model.Repository, isPull brs, err := git_model.FindBranchNames(ctx, git_model.FindBranchOptions{ RepoID: ctx.Repo.Repository.ID, ListOptions: db.ListOptions{ - PageSize: 1, + ListAll: true, }, IsDeletedBranch: util.OptionalBoolFalse, }) diff --git a/services/migrations/dump.go b/services/migrations/dump.go index cc8518d4a25c6..729112bcd2398 100644 --- a/services/migrations/dump.go +++ b/services/migrations/dump.go @@ -642,7 +642,7 @@ func (g *RepositoryDumper) Finish() error { // DumpRepository dump repository according MigrateOptions to a local directory func DumpRepository(ctx context.Context, baseDir, ownerName string, opts base.MigrateOptions) error { - doer, err := user_model.GetAdminUser() + doer, err := user_model.GetAdminUser(ctx) if err != nil { return err } @@ -705,7 +705,7 @@ func updateOptionsUnits(opts *base.MigrateOptions, units []string) error { // RestoreRepository restore a repository from the disk directory func RestoreRepository(ctx context.Context, baseDir, ownerName, repoName string, units []string, validation bool) error { - doer, err := user_model.GetAdminUser() + doer, err := user_model.GetAdminUser(ctx) if err != nil { return err } diff --git a/services/pull/pull.go b/services/pull/pull.go index 118aaabe28324..0f562b9ee35da 100644 --- a/services/pull/pull.go +++ b/services/pull/pull.go @@ -338,7 +338,7 @@ func AddTestPullRequestTask(doer *user_model.User, repoID int64, branch string, for _, pr := range prs { divergence, err := GetDiverging(ctx, pr) if err != nil { - if git_model.IsErrBranchDoesNotExist(err) && !git.IsBranchExist(ctx, pr.HeadRepo.RepoPath(), pr.HeadBranch) { + if git_model.IsErrBranchNotExist(err) && !git.IsBranchExist(ctx, pr.HeadRepo.RepoPath(), pr.HeadBranch) { log.Warn("Cannot test PR %s/%d: head_branch %s no longer exists", pr.BaseRepo.Name, pr.IssueID, pr.HeadBranch) } else { log.Error("GetDiverging: %v", err) diff --git a/services/pull/temp_repo.go b/services/pull/temp_repo.go index 2bce41464e890..db32940e3835a 100644 --- a/services/pull/temp_repo.go +++ b/services/pull/temp_repo.go @@ -181,7 +181,7 @@ func createTemporaryRepoForPR(ctx context.Context, pr *issues_model.PullRequest) Run(prCtx.RunOpts()); err != nil { cancel() if !git.IsBranchExist(ctx, pr.HeadRepo.RepoPath(), pr.HeadBranch) { - return nil, nil, git_model.ErrBranchDoesNotExist{ + return nil, nil, git_model.ErrBranchNotExist{ BranchName: pr.HeadBranch, } } diff --git a/services/pull/update.go b/services/pull/update.go index b4e2fa9ca4992..bc8c4a25e5f61 100644 --- a/services/pull/update.go +++ b/services/pull/update.go @@ -167,7 +167,7 @@ func GetDiverging(ctx context.Context, pr *issues_model.PullRequest) (*git.Diver log.Trace("GetDiverging[%-v]: compare commits", pr) prCtx, cancel, err := createTemporaryRepoForPR(ctx, pr) if err != nil { - if !git_model.IsErrBranchDoesNotExist(err) { + if !git_model.IsErrBranchNotExist(err) { log.Error("CreateTemporaryRepoForPR %-v: %v", pr, err) } return nil, err diff --git a/services/repository/branch.go b/services/repository/branch.go index 7c75cce9f5389..0c72b8eacc530 100644 --- a/services/repository/branch.go +++ b/services/repository/branch.go @@ -35,7 +35,7 @@ func CreateNewBranch(ctx context.Context, doer *user_model.User, repo *repo_mode } if !git.IsBranchExist(ctx, repo.RepoPath(), oldBranchName) { - return git_model.ErrBranchDoesNotExist{ + return git_model.ErrBranchNotExist{ BranchName: oldBranchName, } } @@ -73,9 +73,9 @@ func LoadBranches(ctx context.Context, repo *repo_model.Repository, gitRepo *git } rawBranches, totalNumOfBranches, err := git_model.FindBranches(ctx, git_model.FindBranchOptions{ - RepoID: repo.ID, - IncludeDefaultBranch: false, - IsDeletedBranch: isDeletedBranch, + RepoID: repo.ID, + ExcludeBranchNames: []string{repo.DefaultBranch}, + IsDeletedBranch: isDeletedBranch, ListOptions: db.ListOptions{ Page: page, PageSize: pageSize, @@ -370,7 +370,7 @@ var branchSyncQueue *queue.WorkerPoolQueue[*BranchSyncOptions] func handlerBranchSync(items ...*BranchSyncOptions) []*BranchSyncOptions { var failedOptions []*BranchSyncOptions for _, opts := range items { - if err := repo_module.SyncRepoBranches(context.Background(), opts.RepoID, opts.DoerID); err != nil { + if err := repo_module.SyncRepoBranches(graceful.GetManager().ShutdownContext(), opts.RepoID, opts.DoerID); err != nil { log.Error("syncRepoBranches [%d:%d] failed: %v", opts.RepoID, opts.DoerID, err) failedOptions = append(failedOptions, opts) } @@ -385,21 +385,20 @@ func addRepoToBranchSyncQueue(repoID, doerID int64) error { }) } -func initBranchSyncQueue() error { - branchSyncQueue = queue.CreateSimpleQueue(graceful.GetManager().ShutdownContext(), "branch_sync", handlerBranchSync) +func initBranchSyncQueue(ctx context.Context) error { + branchSyncQueue = queue.CreateSimpleQueue(ctx, "branch_sync", handlerBranchSync) if branchSyncQueue == nil { return errors.New("unable to create branch_sync queue") } go graceful.GetManager().RunWithCancel(branchSyncQueue) - admin, err := user_model.GetAdminUser() + admin, err := user_model.GetAdminUser(ctx) if err != nil { return fmt.Errorf("user_model.GetAdminUser: %v", err) } - cnt, err := git_model.CountBranches(context.Background(), git_model.FindBranchOptions{ - IncludeDefaultBranch: true, - IsDeletedBranch: util.OptionalBoolFalse, + cnt, err := git_model.CountBranches(ctx, git_model.FindBranchOptions{ + IsDeletedBranch: util.OptionalBoolFalse, }) if err != nil { return fmt.Errorf("CountBranches: %v", err) @@ -407,7 +406,7 @@ func initBranchSyncQueue() error { if cnt == 0 { go func() { - if err := AddAllRepoBranchesToSyncQueue(graceful.GetManager().ShutdownContext(), admin.ID); err != nil { + if err := AddAllRepoBranchesToSyncQueue(ctx, admin.ID); err != nil { log.Error("AddAllRepoBranchesToSyncQueue: %v", err) } }() diff --git a/services/repository/repository.go b/services/repository/repository.go index aba4df18d94c1..cd3658dcd8e06 100644 --- a/services/repository/repository.go +++ b/services/repository/repository.go @@ -17,6 +17,7 @@ import ( system_model "code.gitea.io/gitea/models/system" "code.gitea.io/gitea/models/unit" user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/graceful" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/notification" repo_module "code.gitea.io/gitea/modules/repository" @@ -103,7 +104,7 @@ func Init() error { if err := initPushQueue(); err != nil { return err } - return initBranchSyncQueue() + return initBranchSyncQueue(graceful.GetManager().ShutdownContext()) } // UpdateRepository updates a repository diff --git a/templates/repo/branch/list.tmpl b/templates/repo/branch/list.tmpl index 6f0707b95c197..4b21a31ba2662 100644 --- a/templates/repo/branch/list.tmpl +++ b/templates/repo/branch/list.tmpl @@ -4,7 +4,7 @@
{{template "base/alert" .}} {{template "repo/sub_menu" .}} - {{if .DefaultBranch}} + {{if .DefaultBranchBranch}}

{{.locale.Tr "repo.default_branch"}} {{if and $.IsWriter (not $.Repository.IsArchived) (not .IsDeleted)}} @@ -19,32 +19,32 @@ - {{if .DefaultBranch.IsProtected}} + {{if .DefaultBranchBranch.IsProtected}} {{svg "octicon-shield-lock"}} {{end}} - {{.DefaultBranch.RawBranch.Name}} -

{{svg "octicon-git-commit" 16 "gt-mr-2"}}{{ShortSha .DefaultBranch.RawBranch.CommitSHA}} · {{RenderCommitMessage $.Context .DefaultBranch.RawBranch.CommitMessage .RepoLink .Repository.ComposeMetas}} · {{.locale.Tr "org.repo_updated"}} {{TimeSince .DefaultBranch.RawBranch.CommitTime.AsTime .locale}}

+ {{.DefaultBranchBranch.RawBranch.Name}} +

{{svg "octicon-git-commit" 16 "gt-mr-2"}}{{ShortSha .DefaultBranchBranch.RawBranch.CommitSHA}} · {{RenderCommitMessage $.Context .DefaultBranch.RawBranch.CommitMessage .RepoLink .Repository.ComposeMetas}} · {{.locale.Tr "org.repo_updated"}} {{TimeSince .DefaultBranch.RawBranch.CommitTime.AsTime .locale}}

{{if and $.IsWriter (not $.Repository.IsArchived) (not .IsDeleted)}} {{end}} {{if .EnableFeed}} - {{svg "octicon-rss"}} + {{svg "octicon-rss"}} {{end}} {{if not $.DisableDownloadSourceArchives}} -
+
{{svg "octicon-download"}}
- {{svg "octicon-file-zip"}} ZIP - {{svg "octicon-file-zip"}} TAR.GZ + {{svg "octicon-file-zip"}} ZIP + {{svg "octicon-file-zip"}} TAR.GZ
{{end}} @@ -52,8 +52,8 @@ @@ -87,7 +87,7 @@ {{end}} - {{if and (not .RawBranch.IsDeleted) $.DefaultBranch}} + {{if and (not .RawBranch.IsDeleted) $.DefaultBranchBranch}}
{{.CommitsBehind}}
@@ -108,13 +108,13 @@ {{svg "octicon-git-pull-request"}} {{$.locale.Tr "repo.branch.included"}} {{else if and (not .RawBranch.IsDeleted) $.AllowsPulls (gt .CommitsAhead 0)}} - + {{end}} {{else if and .LatestPullRequest.HasMerged .MergeMovedOn}} {{if and (not .RawBranch.IsDeleted) $.AllowsPulls (gt .CommitsAhead 0)}} - + {{end}} From 9bc6efb557cf1c06f7cd12f4c906d20a8cc4ce2b Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Mon, 26 Jun 2023 09:28:55 +0800 Subject: [PATCH 40/56] Fix test --- modules/repository/repo.go | 12 ++++++------ templates/repo/branch/list.tmpl | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/modules/repository/repo.go b/modules/repository/repo.go index 9f4a45d73596f..71ce8fdeb8aa3 100644 --- a/modules/repository/repo.go +++ b/modules/repository/repo.go @@ -173,7 +173,7 @@ func MigrateRepositoryGitData(ctx context.Context, u *user_model.User, } } - subCtx, committer, err := db.TxContext(ctx) + ctx, committer, err := db.TxContext(ctx) if err != nil { return nil, err } @@ -210,28 +210,28 @@ func MigrateRepositoryGitData(ctx context.Context, u *user_model.User, } } - if err = repo_model.InsertMirror(subCtx, &mirrorModel); err != nil { + if err = repo_model.InsertMirror(ctx, &mirrorModel); err != nil { return repo, fmt.Errorf("InsertOne: %w", err) } repo.IsMirror = true - if err = UpdateRepository(subCtx, repo, false); err != nil { + if err = UpdateRepository(ctx, repo, false); err != nil { return nil, err } // this is necessary for sync local tags from remote configName := fmt.Sprintf("remote.%s.fetch", mirrorModel.GetRemoteName()) - if stdout, _, err := git.NewCommand(subCtx, "config"). + if stdout, _, err := git.NewCommand(ctx, "config"). AddOptionValues("--add", configName, `+refs/tags/*:refs/tags/*`). RunStdString(&git.RunOpts{Dir: repoPath}); err != nil { log.Error("MigrateRepositoryGitData(git config --add +refs/tags/*:refs/tags/*) in %v: Stdout: %s\nError: %v", repo, stdout, err) return repo, fmt.Errorf("error in MigrateRepositoryGitData(git config --add +refs/tags/*:refs/tags/*): %w", err) } } else { - if err = UpdateRepoSize(subCtx, repo); err != nil { + if err = UpdateRepoSize(ctx, repo); err != nil { log.Error("Failed to update size for repository: %v", err) } - if repo, err = CleanUpMigrateInfo(subCtx, repo); err != nil { + if repo, err = CleanUpMigrateInfo(ctx, repo); err != nil { return nil, err } } diff --git a/templates/repo/branch/list.tmpl b/templates/repo/branch/list.tmpl index 4b21a31ba2662..1cffd25d9a536 100644 --- a/templates/repo/branch/list.tmpl +++ b/templates/repo/branch/list.tmpl @@ -23,7 +23,7 @@ {{svg "octicon-shield-lock"}} {{end}} {{.DefaultBranchBranch.RawBranch.Name}} -

{{svg "octicon-git-commit" 16 "gt-mr-2"}}{{ShortSha .DefaultBranchBranch.RawBranch.CommitSHA}} · {{RenderCommitMessage $.Context .DefaultBranch.RawBranch.CommitMessage .RepoLink .Repository.ComposeMetas}} · {{.locale.Tr "org.repo_updated"}} {{TimeSince .DefaultBranch.RawBranch.CommitTime.AsTime .locale}}

+

{{svg "octicon-git-commit" 16 "gt-mr-2"}}{{ShortSha .DefaultBranchBranch.RawBranch.CommitSHA}} · {{RenderCommitMessage $.Context .DefaultBranchBranch.RawBranch.CommitMessage .RepoLink .Repository.ComposeMetas}} · {{.locale.Tr "org.repo_updated"}} {{TimeSince .DefaultBranchBranch.RawBranch.CommitTime.AsTime .locale}}

{{if and $.IsWriter (not $.Repository.IsArchived) (not .IsDeleted)}} From 19d3b4c51c69cdc4709d37fa5699929afff1407b Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Mon, 26 Jun 2023 11:43:02 +0800 Subject: [PATCH 41/56] Use sync instead of insert data --- modules/repository/init.go | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/modules/repository/init.go b/modules/repository/init.go index 5277e377fd5df..29cb974c27484 100644 --- a/modules/repository/init.go +++ b/modules/repository/init.go @@ -13,8 +13,6 @@ import ( "strings" "time" - "code.gitea.io/gitea/models/db" - git_model "code.gitea.io/gitea/models/git" issues_model "code.gitea.io/gitea/models/issues" repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" @@ -24,7 +22,6 @@ import ( "code.gitea.io/gitea/modules/options" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/templates/vars" - "code.gitea.io/gitea/modules/timeutil" "code.gitea.io/gitea/modules/util" asymkey_service "code.gitea.io/gitea/services/asymkey" ) @@ -356,20 +353,8 @@ func initRepository(ctx context.Context, repoPath string, u *user_model.User, re } if !repo.IsEmpty { - commit, err := gitRepo.GetBranchCommit(repo.DefaultBranch) - if err != nil { - return fmt.Errorf("getBranchCommit: %w", err) - } - - if err := db.Insert(ctx, &git_model.Branch{ - Name: repo.DefaultBranch, - RepoID: repo.ID, - CommitSHA: commit.ID.String(), - CommitMessage: commit.CommitMessage, - CommitTime: timeutil.TimeStamp(commit.Committer.When.Unix()), - PusherID: u.ID, - }); err != nil { - return fmt.Errorf("insert default branch for repo %d: %w", repo.ID, err) + if err := SyncRepoBranches(ctx, repo.ID, u.ID); err != nil { + return fmt.Errorf("SyncRepoBranches: %w", err) } } } From ed46267fa887f62b44d181fe05be420bbf0f325b Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Wed, 28 Jun 2023 00:36:13 +0800 Subject: [PATCH 42/56] Follow wolfogre's suggestion --- models/git/branches.go | 2 +- models/git/protected_branch_list.go | 4 +++- models/migrations/v1_21/v263.go | 10 ++++------ 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/models/git/branches.go b/models/git/branches.go index 3032fdf601954..9a7213d3f3d18 100644 --- a/models/git/branches.go +++ b/models/git/branches.go @@ -74,7 +74,7 @@ func (err ErrBranchNameConflict) Unwrap() error { return util.ErrAlreadyExist } -// ErrBranchesEqual represents an error that branch name conflicts with other branch. +// ErrBranchesEqual represents an error that base branch is equal to the head branch. type ErrBranchesEqual struct { BaseBranchName string HeadBranchName string diff --git a/models/git/protected_branch_list.go b/models/git/protected_branch_list.go index 3f009e27177d8..eeb307e2454e7 100644 --- a/models/git/protected_branch_list.go +++ b/models/git/protected_branch_list.go @@ -8,6 +8,7 @@ import ( "sort" "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/modules/util" "github.com/gobwas/glob" ) @@ -54,7 +55,8 @@ func FindAllMatchedBranches(ctx context.Context, repoID int64, ruleName string) PageSize: 100, Page: page, }, - RepoID: repoID, + RepoID: repoID, + IsDeletedBranch: util.OptionalBoolFalse, }) if err != nil { return nil, err diff --git a/models/migrations/v1_21/v263.go b/models/migrations/v1_21/v263.go index 07695b0df4e37..0a86db7842831 100644 --- a/models/migrations/v1_21/v263.go +++ b/models/migrations/v1_21/v263.go @@ -63,12 +63,10 @@ func AddBranchTable(x *xorm.Engine) error { branches := make([]Branch, 0, 100) if err := db.Iterate(context.Background(), nil, func(ctx context.Context, deletedBranch *DeletedBranch) error { branches = append(branches, Branch{ - RepoID: deletedBranch.RepoID, - Name: deletedBranch.Name, - CommitSHA: deletedBranch.Commit, - PusherID: adminUserID, - // CommitMessage: string, FIXME: how to get this? - // CommitTime: timeutil.TimeStamp, FIXME: how to get this? + RepoID: deletedBranch.RepoID, + Name: deletedBranch.Name, + CommitSHA: deletedBranch.Commit, + PusherID: adminUserID, IsDeleted: true, DeletedByID: deletedBranch.DeletedByID, DeletedUnix: deletedBranch.DeletedUnix, From c68beb6c34ac210a0ec2524219ecff427c30896d Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Wed, 28 Jun 2023 08:58:13 +0800 Subject: [PATCH 43/56] directly read git repository branches --- modules/context/repo.go | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/modules/context/repo.go b/modules/context/repo.go index 7cbfd65cbd702..8754e1c1b4551 100644 --- a/modules/context/repo.go +++ b/modules/context/repo.go @@ -931,16 +931,9 @@ func RepoRefByType(refType RepoRefType, ignoreNotExistErr ...bool) func(*Context if len(ctx.Params("*")) == 0 { refName = ctx.Repo.Repository.DefaultBranch if !ctx.Repo.GitRepo.IsBranchExist(refName) { - // get the first branch - brs, err := git_model.FindBranchNames(ctx, git_model.FindBranchOptions{ - RepoID: ctx.Repo.Repository.ID, - ListOptions: db.ListOptions{ - PageSize: 1, - }, - IsDeletedBranch: util.OptionalBoolFalse, - }) + brs, _, err := ctx.Repo.GitRepo.GetBranches(0, 1) if err == nil && len(brs) != 0 { - refName = brs[0] + refName = brs[0].Name } else if len(brs) == 0 { log.Error("No branches in non-empty repository %s", ctx.Repo.GitRepo.Path) ctx.Repo.Repository.MarkAsBrokenEmpty() From 3fa69142bf4e5156893b4e3443998151ea82f932 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Wed, 28 Jun 2023 12:14:23 +0800 Subject: [PATCH 44/56] Display pusher on branches list page --- cmd/web.go | 2 +- models/git/{branches.go => branch.go} | 14 ++++++++++- models/git/branch_list.go | 23 +++++++++++++++++++ .../git/{branches_test.go => branch_test.go} | 0 routers/init.go | 7 +----- services/repository/branch.go | 12 +++++----- templates/repo/branch/list.tmpl | 10 ++++---- tests/test_utils.go | 11 +-------- 8 files changed, 50 insertions(+), 29 deletions(-) rename models/git/{branches.go => branch.go} (96%) rename models/git/{branches_test.go => branch_test.go} (100%) diff --git a/cmd/web.go b/cmd/web.go index a7827ce0f72ac..7a257a62a277d 100644 --- a/cmd/web.go +++ b/cmd/web.go @@ -171,7 +171,7 @@ func serveInstalled(ctx *cli.Context) error { } } - routers.InitWebInstalled(graceful.GetManager().HammerContext(), nil) + routers.InitWebInstalled(graceful.GetManager().HammerContext()) // We check that AppDataPath exists here (it should have been created during installation) // We can't check it in `InitWebInstalled`, because some integration tests diff --git a/models/git/branches.go b/models/git/branch.go similarity index 96% rename from models/git/branches.go rename to models/git/branch.go index 9a7213d3f3d18..cb28fafa37115 100644 --- a/models/git/branches.go +++ b/models/git/branch.go @@ -104,6 +104,7 @@ type Branch struct { CommitSHA string CommitMessage string `xorm:"TEXT"` PusherID int64 + Pusher *user_model.User `xorm:"-"` IsDeleted bool DeletedByID int64 DeletedBy *user_model.User `xorm:"-"` @@ -113,7 +114,7 @@ type Branch struct { UpdatedUnix timeutil.TimeStamp `xorm:"updated"` } -func (b *Branch) LoadUser(ctx context.Context) (err error) { +func (b *Branch) LoadDeletedBy(ctx context.Context) (err error) { if b.DeletedBy == nil { b.DeletedBy, err = user_model.GetUserByID(ctx, b.DeletedByID) if user_model.IsErrUserNotExist(err) { @@ -124,6 +125,17 @@ func (b *Branch) LoadUser(ctx context.Context) (err error) { return err } +func (b *Branch) LoadPusher(ctx context.Context) (err error) { + if b.Pusher == nil && b.PusherID > 0 { + b.Pusher, err = user_model.GetUserByID(ctx, b.PusherID) + if user_model.IsErrUserNotExist(err) { + b.Pusher = user_model.NewGhostUser() + err = nil + } + } + return err +} + func init() { db.RegisterModel(new(Branch)) db.RegisterModel(new(RenamedBranch)) diff --git a/models/git/branch_list.go b/models/git/branch_list.go index 2dc4b6c8f46f8..a77ebc016ac46 100644 --- a/models/git/branch_list.go +++ b/models/git/branch_list.go @@ -34,6 +34,29 @@ func (branches BranchList) LoadDeletedBy(ctx context.Context) error { return nil } +func (branches BranchList) LoadPusher(ctx context.Context) error { + ids := container.Set[int64]{} + for _, branch := range branches { + if branch.PusherID > 0 { + ids.Add(branch.PusherID) + } + } + usersMap := make(map[int64]*user_model.User, len(ids)) + if err := db.GetEngine(ctx).In("id", ids.Values()).Find(&usersMap); err != nil { + return err + } + for _, branch := range branches { + if branch.PusherID <= 0 { + continue + } + branch.Pusher = usersMap[branch.PusherID] + if branch.Pusher == nil { + branch.Pusher = user_model.NewGhostUser() + } + } + return nil +} + // LoadAllBranches loads all branches of a repository from database includes deleted branches func LoadAllBranches(ctx context.Context, repoID int64) (BranchList, error) { var branches []*Branch diff --git a/models/git/branches_test.go b/models/git/branch_test.go similarity index 100% rename from models/git/branches_test.go rename to models/git/branch_test.go diff --git a/routers/init.go b/routers/init.go index a3a1dcec458d4..ddbabcc397447 100644 --- a/routers/init.go +++ b/routers/init.go @@ -107,7 +107,7 @@ func InitWebInstallPage(ctx context.Context) { } // InitWebInstalled is for global installed configuration. -func InitWebInstalled(ctx context.Context, extraDBPrepare func() error) { +func InitWebInstalled(ctx context.Context) { mustInitCtx(ctx, git.InitFull) log.Info("Git version: %s (home: %s)", git.VersionInfo(), git.HomeDir()) @@ -134,11 +134,6 @@ func InitWebInstalled(ctx context.Context, extraDBPrepare func() error) { mustInitCtx(ctx, common.InitDBEngine) log.Info("ORM engine initialization successful!") - if extraDBPrepare != nil { - if err := extraDBPrepare(); err != nil { - log.Fatal("prepare database data failed: %v", err) - } - } mustInit(system.Init) mustInit(oauth2.Init) diff --git a/services/repository/branch.go b/services/repository/branch.go index 0c72b8eacc530..b020bdfe195bd 100644 --- a/services/repository/branch.go +++ b/services/repository/branch.go @@ -88,6 +88,9 @@ func LoadBranches(ctx context.Context, repo *repo_model.Repository, gitRepo *git if err := rawBranches.LoadDeletedBy(ctx); err != nil { return nil, nil, 0, err } + if err := rawBranches.LoadPusher(ctx); err != nil { + return nil, nil, 0, err + } rules, err := git_model.FindRepoProtectedBranchRules(ctx, repo.ID) if err != nil { @@ -392,11 +395,8 @@ func initBranchSyncQueue(ctx context.Context) error { } go graceful.GetManager().RunWithCancel(branchSyncQueue) - admin, err := user_model.GetAdminUser(ctx) - if err != nil { - return fmt.Errorf("user_model.GetAdminUser: %v", err) - } - + // for test env, this will return empty because data hasn't been loaded yet + // but that env don't need the sync cnt, err := git_model.CountBranches(ctx, git_model.FindBranchOptions{ IsDeletedBranch: util.OptionalBoolFalse, }) @@ -406,7 +406,7 @@ func initBranchSyncQueue(ctx context.Context) error { if cnt == 0 { go func() { - if err := AddAllRepoBranchesToSyncQueue(ctx, admin.ID); err != nil { + if err := AddAllRepoBranchesToSyncQueue(ctx, 0); err != nil { log.Error("AddAllRepoBranchesToSyncQueue: %v", err) } }() diff --git a/templates/repo/branch/list.tmpl b/templates/repo/branch/list.tmpl index 1cffd25d9a536..c7fd09e615166 100644 --- a/templates/repo/branch/list.tmpl +++ b/templates/repo/branch/list.tmpl @@ -23,7 +23,7 @@ {{svg "octicon-shield-lock"}} {{end}} {{.DefaultBranchBranch.RawBranch.Name}} -

{{svg "octicon-git-commit" 16 "gt-mr-2"}}{{ShortSha .DefaultBranchBranch.RawBranch.CommitSHA}} · {{RenderCommitMessage $.Context .DefaultBranchBranch.RawBranch.CommitMessage .RepoLink .Repository.ComposeMetas}} · {{.locale.Tr "org.repo_updated"}} {{TimeSince .DefaultBranchBranch.RawBranch.CommitTime.AsTime .locale}}

+

{{svg "octicon-git-commit" 16 "gt-mr-2"}}{{ShortSha .DefaultBranchBranch.RawBranch.CommitSHA}} · {{RenderCommitMessage $.Context .DefaultBranchBranch.RawBranch.CommitMessage .RepoLink .Repository.ComposeMetas}} · {{.locale.Tr "org.repo_updated"}} {{TimeSince .DefaultBranchBranch.RawBranch.CommitTime.AsTime .locale}}{{if .DefaultBranchBranch.RawBranch.Pusher}}  {{template "shared/user/avatarlink" dict "Context" $.Context "user" .DefaultBranchBranch.RawBranch.Pusher}}{{template "shared/user/namelink" .DefaultBranchBranch.RawBranch.Pusher}}{{end}}

{{if and $.IsWriter (not $.Repository.IsArchived) (not .IsDeleted)}} @@ -74,7 +74,7 @@ {{range .Branches}} - + {{if .RawBranch.IsDeleted}} {{.RawBranch.Name}}

{{$.locale.Tr "repo.branch.deleted_by" .RawBranch.DeletedBy.Name}} {{TimeSinceUnix .RawBranch.DeletedUnix $.locale}}

@@ -83,10 +83,10 @@ {{svg "octicon-shield-lock"}} {{end}} {{.RawBranch.Name}} -

{{svg "octicon-git-commit" 16 "gt-mr-2"}}{{ShortSha .RawBranch.CommitSHA}} · {{RenderCommitMessage $.Context .RawBranch.CommitMessage $.RepoLink $.Repository.ComposeMetas}} · {{$.locale.Tr "org.repo_updated"}} {{TimeSince .RawBranch.CommitTime.AsTime $.locale}}

+

{{svg "octicon-git-commit" 16 "gt-mr-2"}}{{ShortSha .RawBranch.CommitSHA}} · {{RenderCommitMessage $.Context .RawBranch.CommitMessage $.RepoLink $.Repository.ComposeMetas}} · {{$.locale.Tr "org.repo_updated"}} {{TimeSince .RawBranch.CommitTime.AsTime $.locale}}{{if .RawBranch.Pusher}}  {{template "shared/user/avatarlink" dict "Context" $.Context "user" .RawBranch.Pusher}}  {{template "shared/user/namelink" .RawBranch.Pusher}}{{end}}

{{end}} - + {{if and (not .RawBranch.IsDeleted) $.DefaultBranchBranch}}
@@ -101,7 +101,7 @@
{{end}} - + {{if not .LatestPullRequest}} {{if .IsIncluded}} diff --git a/tests/test_utils.go b/tests/test_utils.go index 8d27e4c822980..bf7d1b3fbe63b 100644 --- a/tests/test_utils.go +++ b/tests/test_utils.go @@ -173,16 +173,7 @@ func InitTest(requireGitea bool) { defer db.Close() } - routers.InitWebInstalled(graceful.GetManager().HammerContext(), func() error { - if err := unittest.InitFixtures( - unittest.FixturesOptions{ - Dir: filepath.Join(filepath.Dir(setting.AppPath), "models/fixtures/"), - }, - ); err != nil { - return err - } - return unittest.LoadFixtures() - }) + routers.InitWebInstalled(graceful.GetManager().HammerContext()) } func PrepareTestEnv(t testing.TB, skip ...int) func() { From 625de6edffee55e87b79c915467eb59d1cf0f225 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Wed, 28 Jun 2023 12:20:36 +0800 Subject: [PATCH 45/56] Some improvements --- routers/web/admin/admin.go | 3 ++- services/repository/branch.go | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/routers/web/admin/admin.go b/routers/web/admin/admin.go index 2e710019e2e22..74524edb5f5c2 100644 --- a/routers/web/admin/admin.go +++ b/routers/web/admin/admin.go @@ -14,6 +14,7 @@ import ( activities_model "code.gitea.io/gitea/models/activities" "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/graceful" "code.gitea.io/gitea/modules/json" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" @@ -138,7 +139,7 @@ func DashboardPost(ctx *context.Context) { switch form.Op { case "sync_repo_sync": go func() { - if err := repo_service.AddAllRepoBranchesToSyncQueue(ctx, ctx.Doer.ID); err != nil { + if err := repo_service.AddAllRepoBranchesToSyncQueue(graceful.GetManager().HammerContext(), ctx.Doer.ID); err != nil { log.Error("AddAllRepoBranchesToSyncQueue: %v: %v", ctx.Doer.ID, err) } }() diff --git a/services/repository/branch.go b/services/repository/branch.go index b020bdfe195bd..47975aebbf64f 100644 --- a/services/repository/branch.go +++ b/services/repository/branch.go @@ -406,6 +406,7 @@ func initBranchSyncQueue(ctx context.Context) error { if cnt == 0 { go func() { + // doerID is 0 means no pusher will be displayed in the branch list ui if err := AddAllRepoBranchesToSyncQueue(ctx, 0); err != nil { log.Error("AddAllRepoBranchesToSyncQueue: %v", err) } @@ -415,7 +416,7 @@ func initBranchSyncQueue(ctx context.Context) error { } func AddAllRepoBranchesToSyncQueue(ctx context.Context, doerID int64) error { - if err := db.Iterate(graceful.GetManager().ShutdownContext(), builder.Eq{"is_empty": false}, func(ctx context.Context, repo *repo_model.Repository) error { + if err := db.Iterate(ctx, builder.Eq{"is_empty": false}, func(ctx context.Context, repo *repo_model.Repository) error { return addRepoToBranchSyncQueue(repo.ID, doerID) }); err != nil { return fmt.Errorf("run sync all branches failed: %v", err) From 425929bb8381f258e46d6fb8501c662ea9db0340 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Wed, 28 Jun 2023 12:23:20 +0800 Subject: [PATCH 46/56] Update modules/repository/branch.go Co-authored-by: Jason Song --- modules/repository/branch.go | 59 +++++++++++++++++------------------- 1 file changed, 28 insertions(+), 31 deletions(-) diff --git a/modules/repository/branch.go b/modules/repository/branch.go index 31fe388b08bed..076ffa2adcbde 100644 --- a/modules/repository/branch.go +++ b/modules/repository/branch.go @@ -43,28 +43,34 @@ func SyncRepoBranches(ctx context.Context, repoID, doerID int64) error { } func SyncRepoBranchesWithRepo(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository, doerID int64) error { - allBranches, _, err := gitRepo.GetBranchNames(0, 0) - if err != nil { - return err + allBranches := make(map[string]struct{}) + { + branches, _, err := gitRepo.GetBranchNames(0, 0) + if err != nil { + return err + } + log.Trace("SyncRepoBranches[%s]: branches[%d]: %v", repo.FullName(), len(branches), branches) + for _, branch := range branches { + allBranches[branch] = struct{}{} + } } - log.Trace("SyncRepoBranches[%s]: branches[%d]: %v", repo.FullName(), len(allBranches), allBranches) - dbBranches, err := git_model.LoadAllBranches(ctx, repo.ID) - if err != nil { - return err + dbBranches := make(map[string]*git_model.Branch) + { + branches, err := git_model.LoadAllBranches(ctx, repo.ID) + if err != nil { + return err + } + for _, branch := range branches { + dbBranches[branch.Name] = branch + } } var toAdd []*git_model.Branch var toUpdate []*git_model.Branch var toRemove []int64 - for _, branch := range allBranches { - var dbb *git_model.Branch - for _, dbBranch := range dbBranches { - if branch == dbBranch.Name { - dbb = dbBranch - break - } - } + for branch := range allBranches { + dbb := dbBranches[branch] commit, err := gitRepo.GetBranchCommit(branch) if err != nil { return err @@ -92,13 +98,7 @@ func SyncRepoBranchesWithRepo(ctx context.Context, repo *repo_model.Repository, } for _, dbBranch := range dbBranches { - var found bool - for _, branch := range allBranches { - if branch == dbBranch.Name { - found = true - break - } - } + _, found := dbBranches[dbBranch.Name] if !found && !dbBranch.IsDeleted { toRemove = append(toRemove, dbBranch.ID) } @@ -117,19 +117,16 @@ func SyncRepoBranchesWithRepo(ctx context.Context, repo *repo_model.Repository, } } - if len(toUpdate) > 0 { - for _, b := range toUpdate { - if _, err := db.GetEngine(subCtx).ID(b.ID). - Cols("commit_sha, commit_message, pusher_id, commit_time, is_deleted"). - Update(b); err != nil { - return err - } + for _, b := range toUpdate { + if _, err := db.GetEngine(subCtx).ID(b.ID). + Cols("commit_sha, commit_message, pusher_id, commit_time, is_deleted"). + Update(b); err != nil { + return err } } if len(toRemove) > 0 { - err = git_model.DeleteBranches(subCtx, repo.ID, doerID, toRemove) - if err != nil { + if err := git_model.DeleteBranches(subCtx, repo.ID, doerID, toRemove); err != nil { return err } } From 9c102e8254f076b0f237eda0653be4d745fb8335 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Wed, 28 Jun 2023 12:29:27 +0800 Subject: [PATCH 47/56] Don't use admin user as branch pusher --- modules/context/repo.go | 7 ------- 1 file changed, 7 deletions(-) diff --git a/modules/context/repo.go b/modules/context/repo.go index 8754e1c1b4551..381f7c822d441 100644 --- a/modules/context/repo.go +++ b/modules/context/repo.go @@ -682,13 +682,6 @@ func RepoAssignment(ctx *Context) (cancel context.CancelFunc) { var doerID int64 if ctx.Doer != nil { doerID = ctx.Doer.ID - } else { - admin, err := user_model.GetAdminUser(ctx) - if err != nil { - ctx.ServerError("GetAdminUser", err) - return - } - doerID = admin.ID } if err := repo_module.SyncRepoBranches(ctx, ctx.Repo.Repository.ID, doerID); err != nil { ctx.ServerError("SyncRepoBranches", err) From fd6db0fbb924cc7e479e93370b53f68b9c6ad350 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Wed, 28 Jun 2023 13:06:06 +0800 Subject: [PATCH 48/56] Fix bug --- models/git/branch_list.go | 2 +- models/git/branch_test.go | 4 ++-- modules/context/repo.go | 39 ++++++++++++++++-------------------- routers/web/repo/compare.go | 4 ++-- services/repository/adopt.go | 2 +- 5 files changed, 23 insertions(+), 28 deletions(-) diff --git a/models/git/branch_list.go b/models/git/branch_list.go index a77ebc016ac46..6a0a9b700ed51 100644 --- a/models/git/branch_list.go +++ b/models/git/branch_list.go @@ -107,7 +107,7 @@ func FindBranches(ctx context.Context, opts FindBranchOptions) (BranchList, int6 func FindBranchNames(ctx context.Context, opts FindBranchOptions) ([]string, error) { sess := db.GetEngine(ctx).Select("name").Where(opts.Cond()) - if opts.PageSize > 0 { + if opts.PageSize > 0 && !opts.IsListAll() { sess = db.SetSessionPagination(sess, &opts.ListOptions) } var branches []string diff --git a/models/git/branch_test.go b/models/git/branch_test.go index 3a14f7450c6c7..e700a56b13d1e 100644 --- a/models/git/branch_test.go +++ b/models/git/branch_test.go @@ -55,13 +55,13 @@ func TestDeletedBranchLoadUser(t *testing.T) { branch := getDeletedBranch(t, firstBranch) assert.Nil(t, branch.DeletedBy) - branch.LoadUser(db.DefaultContext) + branch.LoadDeletedBy(db.DefaultContext) assert.NotNil(t, branch.DeletedBy) assert.Equal(t, "user1", branch.DeletedBy.Name) branch = getDeletedBranch(t, secondBranch) assert.Nil(t, branch.DeletedBy) - branch.LoadUser(db.DefaultContext) + branch.LoadDeletedBy(db.DefaultContext) assert.NotNil(t, branch.DeletedBy) assert.Equal(t, "Ghost", branch.DeletedBy.Name) } diff --git a/modules/context/repo.go b/modules/context/repo.go index 381f7c822d441..5ba98ddd52884 100644 --- a/modules/context/repo.go +++ b/modules/context/repo.go @@ -667,40 +667,35 @@ func RepoAssignment(ctx *Context) (cancel context.CancelFunc) { } ctx.Data["Tags"] = tags - brs, err := git_model.FindBranchNames(ctx, git_model.FindBranchOptions{ + branchOpts := git_model.FindBranchOptions{ RepoID: ctx.Repo.Repository.ID, IsDeletedBranch: util.OptionalBoolFalse, ListOptions: db.ListOptions{ - PageSize: -1, + ListAll: true, }, - }) + } + branchesTotal, err := git_model.CountBranches(ctx, branchOpts) if err != nil { - ctx.ServerError("GetBranches", err) + ctx.ServerError("CountBranches", err) return } - if len(brs) == 0 { // fallback to do a sync immediately - var doerID int64 - if ctx.Doer != nil { - doerID = ctx.Doer.ID - } - if err := repo_module.SyncRepoBranches(ctx, ctx.Repo.Repository.ID, doerID); err != nil { + + // non empty repo should have at least 1 branch, so this repository's branches haven't been synced yet + if branchesTotal == 0 { // fallback to do a sync immediately + if err := repo_module.SyncRepoBranches(ctx, ctx.Repo.Repository.ID, 0); err != nil { ctx.ServerError("SyncRepoBranches", err) return } - brs, err = git_model.FindBranchNames(ctx, git_model.FindBranchOptions{ - RepoID: ctx.Repo.Repository.ID, - IsDeletedBranch: util.OptionalBoolFalse, - ListOptions: db.ListOptions{ - PageSize: -1, - }, - }) - if err != nil { - ctx.ServerError("GetBranches", err) - return - } + } + + // FIXME: use paganation and async loading + brs, err := git_model.FindBranchNames(ctx, branchOpts) + if err != nil { + ctx.ServerError("GetBranches", err) + return } ctx.Data["Branches"] = brs - ctx.Data["BranchesCount"] = len(brs) + ctx.Data["BranchesCount"] = branchesTotal // If not branch selected, try default one. // If default branch doesn't exist, fall back to some other branch. diff --git a/routers/web/repo/compare.go b/routers/web/repo/compare.go index 059ea6bb79a29..7089c219ad25c 100644 --- a/routers/web/repo/compare.go +++ b/routers/web/repo/compare.go @@ -687,7 +687,7 @@ func getBranchesAndTagsForRepo(ctx gocontext.Context, repo *repo_model.Repositor branches, err = git_model.FindBranchNames(ctx, git_model.FindBranchOptions{ RepoID: repo.ID, ListOptions: db.ListOptions{ - PageSize: -1, + ListAll: true, }, IsDeletedBranch: util.OptionalBoolFalse, }) @@ -744,7 +744,7 @@ func CompareDiff(ctx *context.Context) { headBranches, err := git_model.FindBranchNames(ctx, git_model.FindBranchOptions{ RepoID: ci.HeadRepo.ID, ListOptions: db.ListOptions{ - PageSize: 1, + ListAll: true, }, IsDeletedBranch: util.OptionalBoolFalse, }) diff --git a/services/repository/adopt.go b/services/repository/adopt.go index 1f01af602445f..f95fb5988f66b 100644 --- a/services/repository/adopt.go +++ b/services/repository/adopt.go @@ -151,7 +151,7 @@ func adoptRepository(ctx context.Context, repoPath string, u *user_model.User, r branches, _ := git_model.FindBranchNames(ctx, git_model.FindBranchOptions{ RepoID: repo.ID, ListOptions: db.ListOptions{ - PageSize: -1, + ListAll: true, }, IsDeletedBranch: util.OptionalBoolFalse, }) From e96621cdf2b9884bf6d47df8404e2ea81effbf1b Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Wed, 28 Jun 2023 15:13:40 +0800 Subject: [PATCH 49/56] Follow wxiaoguang's suggestion --- models/git/branch.go | 12 +++--- models/git/branch_list.go | 31 +++++++-------- models/git/branch_test.go | 13 +++++-- models/migrations/v1_21/v263.go | 10 ++--- modules/context/repo.go | 4 +- modules/repository/branch.go | 21 ++++++---- options/locale/locale_en-US.ini | 3 +- routers/web/admin/admin.go | 5 ++- routers/web/repo/branch.go | 4 +- services/repository/branch.go | 35 +++++++++-------- templates/admin/dashboard.tmpl | 4 +- templates/repo/branch/list.tmpl | 68 ++++++++++++++++----------------- 12 files changed, 110 insertions(+), 100 deletions(-) diff --git a/models/git/branch.go b/models/git/branch.go index cb28fafa37115..70b7218f09fa3 100644 --- a/models/git/branch.go +++ b/models/git/branch.go @@ -101,14 +101,14 @@ type Branch struct { ID int64 RepoID int64 `xorm:"UNIQUE(s)"` Name string `xorm:"UNIQUE(s) NOT NULL"` - CommitSHA string + CommitID string CommitMessage string `xorm:"TEXT"` PusherID int64 Pusher *user_model.User `xorm:"-"` - IsDeleted bool + IsDeleted bool `xorm:"index"` DeletedByID int64 - DeletedBy *user_model.User `xorm:"-"` - DeletedUnix timeutil.TimeStamp + DeletedBy *user_model.User `xorm:"-"` + DeletedUnix timeutil.TimeStamp `xorm:"index"` CommitTime timeutil.TimeStamp // The commit CreatedUnix timeutil.TimeStamp `xorm:"created"` UpdatedUnix timeutil.TimeStamp `xorm:"updated"` @@ -208,7 +208,7 @@ func UpdateBranch(ctx context.Context, repoID int64, branchName, commitID, commi cnt, err := db.GetEngine(ctx).Where("repo_id=? AND name=?", repoID, branchName). Cols("commit_sha, commit_message, pusher_id, commit_time, is_deleted, updated_unix"). Update(&Branch{ - CommitSHA: commitID, + CommitID: commitID, CommitMessage: commitMessage, PusherID: pusherID, CommitTime: timeutil.TimeStamp(commitTime.Unix()), @@ -224,7 +224,7 @@ func UpdateBranch(ctx context.Context, repoID int64, branchName, commitID, commi return db.Insert(ctx, &Branch{ RepoID: repoID, Name: branchName, - CommitSHA: commitID, + CommitID: commitID, CommitMessage: commitMessage, PusherID: pusherID, CommitTime: timeutil.TimeStamp(commitTime.Unix()), diff --git a/models/git/branch_list.go b/models/git/branch_list.go index 6a0a9b700ed51..f2fcbcee06eb6 100644 --- a/models/git/branch_list.go +++ b/models/git/branch_list.go @@ -57,19 +57,17 @@ func (branches BranchList) LoadPusher(ctx context.Context) error { return nil } -// LoadAllBranches loads all branches of a repository from database includes deleted branches -func LoadAllBranches(ctx context.Context, repoID int64) (BranchList, error) { - var branches []*Branch - err := db.GetEngine(ctx).Where("repo_id=?", repoID). - Find(&branches) - return branches, err -} +const ( + BranchOrderByNameAsc = "name ASC" + BranchOrderByCommitTimeDesc = "commit_time DESC" +) type FindBranchOptions struct { db.ListOptions RepoID int64 ExcludeBranchNames []string IsDeletedBranch util.OptionalBool + OrderBy string } func (opts *FindBranchOptions) Cond() builder.Cond { @@ -96,6 +94,10 @@ func FindBranches(ctx context.Context, opts FindBranchOptions) (BranchList, int6 if opts.PageSize > 0 && !opts.IsListAll() { sess = db.SetSessionPagination(sess, &opts.ListOptions) } + if opts.OrderBy == "" { + opts.OrderBy = BranchOrderByCommitTimeDesc + } + sess = sess.OrderBy(opts.OrderBy) var branches []*Branch total, err := sess.FindAndCount(&branches) @@ -110,20 +112,13 @@ func FindBranchNames(ctx context.Context, opts FindBranchOptions) ([]string, err if opts.PageSize > 0 && !opts.IsListAll() { sess = db.SetSessionPagination(sess, &opts.ListOptions) } + if opts.OrderBy == "" { + opts.OrderBy = BranchOrderByCommitTimeDesc + } + sess = sess.OrderBy(opts.OrderBy) var branches []string if err := sess.Table("branch").Find(&branches); err != nil { return nil, err } return branches, nil } - -func GetDeletedBranches(ctx context.Context, repoID int64) (BranchList, error) { - branches, _, err := FindBranches(ctx, FindBranchOptions{ - ListOptions: db.ListOptions{ - PageSize: -1, - }, - RepoID: repoID, - IsDeletedBranch: util.OptionalBoolTrue, - }) - return branches, err -} diff --git a/models/git/branch_test.go b/models/git/branch_test.go index e700a56b13d1e..6fe7ee2783d64 100644 --- a/models/git/branch_test.go +++ b/models/git/branch_test.go @@ -11,6 +11,7 @@ import ( issues_model "code.gitea.io/gitea/models/issues" repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/models/unittest" + "code.gitea.io/gitea/modules/util" "github.com/stretchr/testify/assert" ) @@ -27,7 +28,7 @@ func TestAddDeletedBranch(t *testing.T) { secondBranch := unittest.AssertExistsAndLoadBean(t, &git_model.Branch{RepoID: repo.ID, Name: "branch2"}) assert.True(t, secondBranch.IsDeleted) - err := git_model.UpdateBranch(db.DefaultContext, repo.ID, secondBranch.Name, secondBranch.CommitSHA, secondBranch.CommitMessage, secondBranch.PusherID, secondBranch.CommitTime.AsLocalTime()) + err := git_model.UpdateBranch(db.DefaultContext, repo.ID, secondBranch.Name, secondBranch.CommitID, secondBranch.CommitMessage, secondBranch.PusherID, secondBranch.CommitTime.AsLocalTime()) assert.NoError(t, err) } @@ -35,7 +36,13 @@ func TestGetDeletedBranches(t *testing.T) { assert.NoError(t, unittest.PrepareTestDatabase()) repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 1}) - branches, err := git_model.GetDeletedBranches(db.DefaultContext, repo.ID) + branches, _, err := git_model.FindBranches(db.DefaultContext, git_model.FindBranchOptions{ + ListOptions: db.ListOptions{ + ListAll: true, + }, + RepoID: repo.ID, + IsDeletedBranch: util.OptionalBoolTrue, + }) assert.NoError(t, err) assert.Len(t, branches, 2) } @@ -85,7 +92,7 @@ func getDeletedBranch(t *testing.T, branch *git_model.Branch) *git_model.Branch assert.NoError(t, err) assert.Equal(t, branch.ID, deletedBranch.ID) assert.Equal(t, branch.Name, deletedBranch.Name) - assert.Equal(t, branch.CommitSHA, deletedBranch.CommitSHA) + assert.Equal(t, branch.CommitID, deletedBranch.CommitID) assert.Equal(t, branch.DeletedByID, deletedBranch.DeletedByID) return deletedBranch diff --git a/models/migrations/v1_21/v263.go b/models/migrations/v1_21/v263.go index 0a86db7842831..60b7a7acf7d0c 100644 --- a/models/migrations/v1_21/v263.go +++ b/models/migrations/v1_21/v263.go @@ -16,14 +16,14 @@ import ( func AddBranchTable(x *xorm.Engine) error { type Branch struct { ID int64 - RepoID int64 `xorm:"index UNIQUE(s)"` + RepoID int64 `xorm:"UNIQUE(s)"` Name string `xorm:"UNIQUE(s) NOT NULL"` - CommitSHA string + CommitID string CommitMessage string `xorm:"TEXT"` PusherID int64 - IsDeleted bool + IsDeleted bool `xorm:"index"` DeletedByID int64 - DeletedUnix timeutil.TimeStamp + DeletedUnix timeutil.TimeStamp `xorm:"index"` CommitTime timeutil.TimeStamp // The commit CreatedUnix timeutil.TimeStamp `xorm:"created"` UpdatedUnix timeutil.TimeStamp `xorm:"updated"` @@ -65,7 +65,7 @@ func AddBranchTable(x *xorm.Engine) error { branches = append(branches, Branch{ RepoID: deletedBranch.RepoID, Name: deletedBranch.Name, - CommitSHA: deletedBranch.Commit, + CommitID: deletedBranch.Commit, PusherID: adminUserID, IsDeleted: true, DeletedByID: deletedBranch.DeletedByID, diff --git a/modules/context/repo.go b/modules/context/repo.go index 5ba98ddd52884..e40b45f453987 100644 --- a/modules/context/repo.go +++ b/modules/context/repo.go @@ -689,12 +689,14 @@ func RepoAssignment(ctx *Context) (cancel context.CancelFunc) { } // FIXME: use paganation and async loading + branchOpts.ExcludeBranchNames = []string{ctx.Repo.Repository.DefaultBranch} brs, err := git_model.FindBranchNames(ctx, branchOpts) if err != nil { ctx.ServerError("GetBranches", err) return } - ctx.Data["Branches"] = brs + // always put default branch on the top + ctx.Data["Branches"] = append(branchOpts.ExcludeBranchNames, brs...) ctx.Data["BranchesCount"] = branchesTotal // If not branch selected, try default one. diff --git a/modules/repository/branch.go b/modules/repository/branch.go index 076ffa2adcbde..88c445003a8ca 100644 --- a/modules/repository/branch.go +++ b/modules/repository/branch.go @@ -9,6 +9,7 @@ import ( "code.gitea.io/gitea/models/db" git_model "code.gitea.io/gitea/models/git" repo_model "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/modules/container" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/timeutil" @@ -43,7 +44,7 @@ func SyncRepoBranches(ctx context.Context, repoID, doerID int64) error { } func SyncRepoBranchesWithRepo(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository, doerID int64) error { - allBranches := make(map[string]struct{}) + allBranches := container.Set[string]{} { branches, _, err := gitRepo.GetBranchNames(0, 0) if err != nil { @@ -51,13 +52,18 @@ func SyncRepoBranchesWithRepo(ctx context.Context, repo *repo_model.Repository, } log.Trace("SyncRepoBranches[%s]: branches[%d]: %v", repo.FullName(), len(branches), branches) for _, branch := range branches { - allBranches[branch] = struct{}{} + allBranches.Add(branch) } } dbBranches := make(map[string]*git_model.Branch) { - branches, err := git_model.LoadAllBranches(ctx, repo.ID) + branches, _, err := git_model.FindBranches(ctx, git_model.FindBranchOptions{ + ListOptions: db.ListOptions{ + ListAll: true, + }, + RepoID: repo.ID, + }) if err != nil { return err } @@ -79,17 +85,17 @@ func SyncRepoBranchesWithRepo(ctx context.Context, repo *repo_model.Repository, toAdd = append(toAdd, &git_model.Branch{ RepoID: repo.ID, Name: branch, - CommitSHA: commit.ID.String(), + CommitID: commit.ID.String(), CommitMessage: commit.CommitMessage, PusherID: doerID, CommitTime: timeutil.TimeStamp(commit.Author.When.Unix()), }) - } else if commit.ID.String() != dbb.CommitSHA { + } else if commit.ID.String() != dbb.CommitID { toUpdate = append(toUpdate, &git_model.Branch{ ID: dbb.ID, RepoID: repo.ID, Name: branch, - CommitSHA: commit.ID.String(), + CommitID: commit.ID.String(), CommitMessage: commit.CommitMessage, PusherID: doerID, CommitTime: timeutil.TimeStamp(commit.Author.When.Unix()), @@ -98,8 +104,7 @@ func SyncRepoBranchesWithRepo(ctx context.Context, repo *repo_model.Repository, } for _, dbBranch := range dbBranches { - _, found := dbBranches[dbBranch.Name] - if !found && !dbBranch.IsDeleted { + if !allBranches.Contains(dbBranch.Name) && !dbBranch.IsDeleted { toRemove = append(toRemove, dbBranch.ID) } } diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index a18616db68beb..315a8efb6cc7b 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -2659,7 +2659,7 @@ dashboard.delete_repo_archives.started = Delete all repository archives task sta dashboard.delete_missing_repos = Delete all repositories missing their Git files dashboard.delete_missing_repos.started = Delete all repositories missing their Git files task started. dashboard.delete_generated_repository_avatars = Delete generated repository avatars -dashboard.sync_repo_sync = Sync missed branches from git data to databases +dashboard.sync_repo_branches = Sync missed branches from git data to databases dashboard.update_mirrors = Update Mirrors dashboard.repo_health_check = Health check all repositories dashboard.check_repo_stats = Check all repository statistics @@ -2713,6 +2713,7 @@ dashboard.gc_lfs = Garbage collect LFS meta objects dashboard.stop_zombie_tasks = Stop zombie tasks dashboard.stop_endless_tasks = Stop endless tasks dashboard.cancel_abandoned_jobs = Cancel abandoned jobs +dashboard.sync_branch.started = Branches Sync started users.user_manage_panel = User Account Management users.new_account = Create User Account diff --git a/routers/web/admin/admin.go b/routers/web/admin/admin.go index 74524edb5f5c2..2643caae0ae1b 100644 --- a/routers/web/admin/admin.go +++ b/routers/web/admin/admin.go @@ -137,12 +137,13 @@ func DashboardPost(ctx *context.Context) { // Run operation. if form.Op != "" { switch form.Op { - case "sync_repo_sync": + case "sync_repo_branches": go func() { - if err := repo_service.AddAllRepoBranchesToSyncQueue(graceful.GetManager().HammerContext(), ctx.Doer.ID); err != nil { + if err := repo_service.AddAllRepoBranchesToSyncQueue(graceful.GetManager().ShutdownContext(), ctx.Doer.ID); err != nil { log.Error("AddAllRepoBranchesToSyncQueue: %v: %v", ctx.Doer.ID, err) } }() + ctx.Flash.Success(ctx.Tr("admin.dashboard.sync_branch.started", ctx.Tr("admin.dashboard."+form.Op))) default: task := cron.GetTask(form.Op) if task != nil { diff --git a/routers/web/repo/branch.go b/routers/web/repo/branch.go index 2670da85c5deb..f0282a71b876b 100644 --- a/routers/web/repo/branch.go +++ b/routers/web/repo/branch.go @@ -113,7 +113,7 @@ func RestoreBranchPost(ctx *context.Context) { if err := git.Push(ctx, ctx.Repo.Repository.RepoPath(), git.PushOptions{ Remote: ctx.Repo.Repository.RepoPath(), - Branch: fmt.Sprintf("%s:%s%s", deletedBranch.CommitSHA, git.BranchPrefix, deletedBranch.Name), + Branch: fmt.Sprintf("%s:%s%s", deletedBranch.CommitID, git.BranchPrefix, deletedBranch.Name), Env: repo_module.PushingEnvironment(ctx.Doer, ctx.Repo.Repository), }); err != nil { if strings.Contains(err.Error(), "already exists") { @@ -131,7 +131,7 @@ func RestoreBranchPost(ctx *context.Context) { &repo_module.PushUpdateOptions{ RefFullName: git.RefNameFromBranch(deletedBranch.Name), OldCommitID: git.EmptySHA, - NewCommitID: deletedBranch.CommitSHA, + NewCommitID: deletedBranch.CommitID, PusherID: ctx.Doer.ID, PusherName: ctx.Doer.Name, RepoUserName: ctx.Repo.Owner.Name, diff --git a/services/repository/branch.go b/services/repository/branch.go index 47975aebbf64f..1b401e2c71de3 100644 --- a/services/repository/branch.go +++ b/services/repository/branch.go @@ -56,7 +56,7 @@ func CreateNewBranch(ctx context.Context, doer *user_model.User, repo *repo_mode // Branch contains the branch information type Branch struct { - RawBranch *git_model.Branch + DBBranch *git_model.Branch IsProtected bool IsIncluded bool CommitsAhead int @@ -67,12 +67,12 @@ type Branch struct { // LoadBranches loads branches from the repository limited by page & pageSize. func LoadBranches(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository, isDeletedBranch util.OptionalBool, page, pageSize int) (*Branch, []*Branch, int64, error) { - defaultRawBranch, err := git_model.GetBranch(ctx, repo.ID, repo.DefaultBranch) + defaultDBBranch, err := git_model.GetBranch(ctx, repo.ID, repo.DefaultBranch) if err != nil { return nil, nil, 0, err } - rawBranches, totalNumOfBranches, err := git_model.FindBranches(ctx, git_model.FindBranchOptions{ + dbBranches, totalNumOfBranches, err := git_model.FindBranches(ctx, git_model.FindBranchOptions{ RepoID: repo.ID, ExcludeBranchNames: []string{repo.DefaultBranch}, IsDeletedBranch: isDeletedBranch, @@ -85,10 +85,10 @@ func LoadBranches(ctx context.Context, repo *repo_model.Repository, gitRepo *git return nil, nil, 0, err } - if err := rawBranches.LoadDeletedBy(ctx); err != nil { + if err := dbBranches.LoadDeletedBy(ctx); err != nil { return nil, nil, 0, err } - if err := rawBranches.LoadPusher(ctx); err != nil { + if err := dbBranches.LoadPusher(ctx); err != nil { return nil, nil, 0, err } @@ -103,9 +103,9 @@ func LoadBranches(ctx context.Context, repo *repo_model.Repository, gitRepo *git repoIDToGitRepo := map[int64]*git.Repository{} repoIDToGitRepo[repo.ID] = gitRepo - branches := make([]*Branch, 0, len(rawBranches)) - for i := range rawBranches { - branch, err := loadOneBranch(ctx, repo, rawBranches[i], &rules, repoIDToRepo, repoIDToGitRepo) + branches := make([]*Branch, 0, len(dbBranches)) + for i := range dbBranches { + branch, err := loadOneBranch(ctx, repo, dbBranches[i], &rules, repoIDToRepo, repoIDToGitRepo) if err != nil { return nil, nil, 0, fmt.Errorf("loadOneBranch: %v", err) } @@ -114,8 +114,8 @@ func LoadBranches(ctx context.Context, repo *repo_model.Repository, gitRepo *git } // Always add the default branch - log.Debug("loadOneBranch: load default: '%s'", defaultRawBranch.Name) - defaultBranch, err := loadOneBranch(ctx, repo, defaultRawBranch, &rules, repoIDToRepo, repoIDToGitRepo) + log.Debug("loadOneBranch: load default: '%s'", defaultDBBranch.Name) + defaultBranch, err := loadOneBranch(ctx, repo, defaultDBBranch, &rules, repoIDToRepo, repoIDToGitRepo) if err != nil { return nil, nil, 0, fmt.Errorf("loadOneBranch: %v", err) } @@ -123,13 +123,13 @@ func LoadBranches(ctx context.Context, repo *repo_model.Repository, gitRepo *git return defaultBranch, branches, totalNumOfBranches, nil } -func loadOneBranch(ctx context.Context, repo *repo_model.Repository, rawBranch *git_model.Branch, protectedBranches *git_model.ProtectedBranchRules, +func loadOneBranch(ctx context.Context, repo *repo_model.Repository, dbBranch *git_model.Branch, protectedBranches *git_model.ProtectedBranchRules, repoIDToRepo map[int64]*repo_model.Repository, repoIDToGitRepo map[int64]*git.Repository, ) (*Branch, error) { - log.Trace("loadOneBranch: '%s'", rawBranch.Name) + log.Trace("loadOneBranch: '%s'", dbBranch.Name) - branchName := rawBranch.Name + branchName := dbBranch.Name p := protectedBranches.GetFirstMatched(branchName) isProtected := p != nil @@ -139,7 +139,7 @@ func loadOneBranch(ctx context.Context, repo *repo_model.Repository, rawBranch * } // it's not default branch - if repo.DefaultBranch != rawBranch.Name && !rawBranch.IsDeleted { + if repo.DefaultBranch != dbBranch.Name && !dbBranch.IsDeleted { var err error divergence, err = files_service.CountDivergingCommits(ctx, repo, git.BranchPrefix+branchName) if err != nil { @@ -151,7 +151,7 @@ func loadOneBranch(ctx context.Context, repo *repo_model.Repository, rawBranch * if err != nil { return nil, fmt.Errorf("GetLatestPullRequestByHeadInfo: %v", err) } - headCommit := rawBranch.CommitSHA + headCommit := dbBranch.CommitID mergeMovedOn := false if pr != nil { @@ -191,7 +191,7 @@ func loadOneBranch(ctx context.Context, repo *repo_model.Repository, rawBranch * isIncluded := divergence.Ahead == 0 && repo.DefaultBranch != branchName return &Branch{ - RawBranch: rawBranch, + DBBranch: dbBranch, IsProtected: isProtected, IsIncluded: isIncluded, CommitsAhead: divergence.Ahead, @@ -366,10 +366,9 @@ type BranchSyncOptions struct { DoerID int64 } -// branchSyncQueue represents a queue to handle branch s +// branchSyncQueue represents a queue to handle branch sync jobs. var branchSyncQueue *queue.WorkerPoolQueue[*BranchSyncOptions] -// handle passed PR IDs and test the PRs func handlerBranchSync(items ...*BranchSyncOptions) []*BranchSyncOptions { var failedOptions []*BranchSyncOptions for _, opts := range items { diff --git a/templates/admin/dashboard.tmpl b/templates/admin/dashboard.tmpl index f81b266e99891..1dd920c2dd0e7 100644 --- a/templates/admin/dashboard.tmpl +++ b/templates/admin/dashboard.tmpl @@ -62,8 +62,8 @@ - {{.locale.Tr "admin.dashboard.sync_repo_sync"}} - + {{.locale.Tr "admin.dashboard.sync_repo_branches"}} + diff --git a/templates/repo/branch/list.tmpl b/templates/repo/branch/list.tmpl index c7fd09e615166..cf81f461327d7 100644 --- a/templates/repo/branch/list.tmpl +++ b/templates/repo/branch/list.tmpl @@ -22,29 +22,29 @@ {{if .DefaultBranchBranch.IsProtected}} {{svg "octicon-shield-lock"}} {{end}} - {{.DefaultBranchBranch.RawBranch.Name}} -

{{svg "octicon-git-commit" 16 "gt-mr-2"}}{{ShortSha .DefaultBranchBranch.RawBranch.CommitSHA}} · {{RenderCommitMessage $.Context .DefaultBranchBranch.RawBranch.CommitMessage .RepoLink .Repository.ComposeMetas}} · {{.locale.Tr "org.repo_updated"}} {{TimeSince .DefaultBranchBranch.RawBranch.CommitTime.AsTime .locale}}{{if .DefaultBranchBranch.RawBranch.Pusher}}  {{template "shared/user/avatarlink" dict "Context" $.Context "user" .DefaultBranchBranch.RawBranch.Pusher}}{{template "shared/user/namelink" .DefaultBranchBranch.RawBranch.Pusher}}{{end}}

+ {{.DefaultBranchBranch.DBBranch.Name}} +

{{svg "octicon-git-commit" 16 "gt-mr-2"}}{{ShortSha .DefaultBranchBranch.DBBranch.CommitID}} · {{RenderCommitMessage $.Context .DefaultBranchBranch.DBBranch.CommitMessage .RepoLink .Repository.ComposeMetas}} · {{.locale.Tr "org.repo_updated"}} {{TimeSince .DefaultBranchBranch.DBBranch.CommitTime.AsTime .locale}}{{if .DefaultBranchBranch.DBBranch.Pusher}}  {{template "shared/user/avatarlink" dict "Context" $.Context "user" .DefaultBranchBranch.DBBranch.Pusher}}{{template "shared/user/namelink" .DefaultBranchBranch.DBBranch.Pusher}}{{end}}

{{if and $.IsWriter (not $.Repository.IsArchived) (not .IsDeleted)}} {{end}} {{if .EnableFeed}} - {{svg "octicon-rss"}} + {{svg "octicon-rss"}} {{end}} {{if not $.DisableDownloadSourceArchives}} -
+
{{svg "octicon-download"}}
- {{svg "octicon-file-zip"}} ZIP - {{svg "octicon-file-zip"}} TAR.GZ + {{svg "octicon-file-zip"}} ZIP + {{svg "octicon-file-zip"}} TAR.GZ
{{end}} @@ -53,7 +53,7 @@ data-is-default-branch="true" data-modal="#rename-branch-modal" data-old-branch-name="{{$.DefaultBranchBranch}}" - data-tooltip-content="{{$.locale.Tr "repo.branch.rename" ($.DefaultBranchBranch.RawBranch.Name)}}" + data-tooltip-content="{{$.locale.Tr "repo.branch.rename" ($.DefaultBranchBranch.DBBranch.Name)}}" > {{svg "octicon-pencil"}} @@ -75,19 +75,19 @@ {{range .Branches}} - {{if .RawBranch.IsDeleted}} - {{.RawBranch.Name}} -

{{$.locale.Tr "repo.branch.deleted_by" .RawBranch.DeletedBy.Name}} {{TimeSinceUnix .RawBranch.DeletedUnix $.locale}}

+ {{if .DBBranch.IsDeleted}} + {{.DBBranch.Name}} +

{{$.locale.Tr "repo.branch.deleted_by" .DBBranch.DeletedBy.Name}} {{TimeSinceUnix .DBBranch.DeletedUnix $.locale}}

{{else}} {{if .IsProtected}} {{svg "octicon-shield-lock"}} {{end}} - {{.RawBranch.Name}} -

{{svg "octicon-git-commit" 16 "gt-mr-2"}}{{ShortSha .RawBranch.CommitSHA}} · {{RenderCommitMessage $.Context .RawBranch.CommitMessage $.RepoLink $.Repository.ComposeMetas}} · {{$.locale.Tr "org.repo_updated"}} {{TimeSince .RawBranch.CommitTime.AsTime $.locale}}{{if .RawBranch.Pusher}}  {{template "shared/user/avatarlink" dict "Context" $.Context "user" .RawBranch.Pusher}}  {{template "shared/user/namelink" .RawBranch.Pusher}}{{end}}

+ {{.DBBranch.Name}} +

{{svg "octicon-git-commit" 16 "gt-mr-2"}}{{ShortSha .DBBranch.CommitID}} · {{RenderCommitMessage $.Context .DBBranch.CommitMessage $.RepoLink $.Repository.ComposeMetas}} · {{$.locale.Tr "org.repo_updated"}} {{TimeSince .DBBranch.CommitTime.AsTime $.locale}}{{if .DBBranch.Pusher}}  {{template "shared/user/avatarlink" dict "Context" $.Context "user" .DBBranch.Pusher}}  {{template "shared/user/namelink" .DBBranch.Pusher}}{{end}}

{{end}} - {{if and (not .RawBranch.IsDeleted) $.DefaultBranchBranch}} + {{if and (not .DBBranch.IsDeleted) $.DefaultBranchBranch}}
{{.CommitsBehind}}
@@ -107,13 +107,13 @@ {{svg "octicon-git-pull-request"}} {{$.locale.Tr "repo.branch.included"}} - {{else if and (not .RawBranch.IsDeleted) $.AllowsPulls (gt .CommitsAhead 0)}} - + {{else if and (not .DBBranch.IsDeleted) $.AllowsPulls (gt .CommitsAhead 0)}} + {{end}} {{else if and .LatestPullRequest.HasMerged .MergeMovedOn}} - {{if and (not .RawBranch.IsDeleted) $.AllowsPulls (gt .CommitsAhead 0)}} + {{if and (not .DBBranch.IsDeleted) $.AllowsPulls (gt .CommitsAhead 0)}} @@ -130,47 +130,47 @@ {{end}} - {{if and $.IsWriter (not $.Repository.IsArchived) (not .RawBranch.IsDeleted)}} + {{if and $.IsWriter (not $.Repository.IsArchived) (not .DBBranch.IsDeleted)}} {{end}} {{if $.EnableFeed}} - {{svg "octicon-rss"}} + {{svg "octicon-rss"}} {{end}} - {{if and (not .RawBranch.IsDeleted) (not $.DisableDownloadSourceArchives)}} -
+ {{if and (not .DBBranch.IsDeleted) (not $.DisableDownloadSourceArchives)}} +
{{svg "octicon-download"}}
- {{svg "octicon-file-zip"}} ZIP - {{svg "octicon-file-zip"}} TAR.GZ + {{svg "octicon-file-zip"}} ZIP + {{svg "octicon-file-zip"}} TAR.GZ
{{end}} - {{if and $.IsWriter (not $.Repository.IsArchived) (not .RawBranch.IsDeleted) (not $.IsMirror)}} + {{if and $.IsWriter (not $.Repository.IsArchived) (not .DBBranch.IsDeleted) (not $.IsMirror)}} {{end}} {{if and $.IsWriter (not $.IsMirror) (not $.Repository.IsArchived) (not .IsProtected)}} - {{if .RawBranch.IsDeleted}} - {{else}} - {{end}} From 61fa367abb1f1ab4ca44b0afea316a0d6ff8b2dc Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Wed, 28 Jun 2023 15:50:14 +0800 Subject: [PATCH 50/56] Remove unused function --- models/fixtures/branch.yml | 8 ++++---- modules/repository/branch.go | 9 --------- 2 files changed, 4 insertions(+), 13 deletions(-) diff --git a/models/fixtures/branch.yml b/models/fixtures/branch.yml index 6aeca6f0354ce..93003049c67fb 100644 --- a/models/fixtures/branch.yml +++ b/models/fixtures/branch.yml @@ -2,7 +2,7 @@ id: 1 repo_id: 1 name: 'foo' - commit_sha: '65f1bf27bc3bf70f64657658635e66094edbcb4d' + commit_id: '65f1bf27bc3bf70f64657658635e66094edbcb4d' commit_message: 'first commit' commit_time: 978307100 pusher_id: 1 @@ -14,7 +14,7 @@ id: 2 repo_id: 1 name: 'bar' - commit_sha: '62fb502a7172d4453f0322a2cc85bddffa57f07a' + commit_id: '62fb502a7172d4453f0322a2cc85bddffa57f07a' commit_message: 'second commit' commit_time: 978307100 pusher_id: 1 @@ -26,7 +26,7 @@ id: 3 repo_id: 1 name: 'branch2' - commit_sha: '985f0301dba5e7b34be866819cd15ad3d8f508ee' + commit_id: '985f0301dba5e7b34be866819cd15ad3d8f508ee' commit_message: 'make pull5 outdated' commit_time: 1579166279 pusher_id: 1 @@ -38,7 +38,7 @@ id: 4 repo_id: 1 name: 'master' - commit_sha: '65f1bf27bc3bf70f64657658635e66094edbcb4d' + commit_id: '65f1bf27bc3bf70f64657658635e66094edbcb4d' commit_message: 'Initial commit' commit_time: 1489927679 pusher_id: 1 diff --git a/modules/repository/branch.go b/modules/repository/branch.go index 88c445003a8ca..74de19fa97eee 100644 --- a/modules/repository/branch.go +++ b/modules/repository/branch.go @@ -13,17 +13,8 @@ import ( "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/timeutil" - - "xorm.io/builder" ) -func SyncAllBranches(ctx context.Context, doerID int64) error { - log.Trace("Synchronizing repository branches (this may take a while)") - return db.Iterate(ctx, builder.Eq{"is_empty": false}, func(ctx context.Context, repo *repo_model.Repository) error { - return SyncRepoBranches(ctx, repo.ID, doerID) - }) -} - // SyncRepoBranches synchronizes branch table with repository branches func SyncRepoBranches(ctx context.Context, repoID, doerID int64) error { repo, err := repo_model.GetRepositoryByID(ctx, repoID) From 3cdf0d93d4b040571cf44a9ef55bd6b2cdec2b45 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Wed, 28 Jun 2023 15:56:52 +0800 Subject: [PATCH 51/56] Add some comment for LoadDeletedBy --- models/git/branch_list.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/models/git/branch_list.go b/models/git/branch_list.go index f2fcbcee06eb6..91aa7869f2258 100644 --- a/models/git/branch_list.go +++ b/models/git/branch_list.go @@ -19,6 +19,9 @@ type BranchList []*Branch func (branches BranchList) LoadDeletedBy(ctx context.Context) error { ids := container.Set[int64]{} for _, branch := range branches { + if !branch.IsDeleted { + continue + } ids.Add(branch.DeletedByID) } usersMap := make(map[int64]*user_model.User, len(ids)) @@ -26,6 +29,9 @@ func (branches BranchList) LoadDeletedBy(ctx context.Context) error { return err } for _, branch := range branches { + if !branch.IsDeleted { + continue + } branch.DeletedBy = usersMap[branch.DeletedByID] if branch.DeletedBy == nil { branch.DeletedBy = user_model.NewGhostUser() @@ -37,7 +43,7 @@ func (branches BranchList) LoadDeletedBy(ctx context.Context) error { func (branches BranchList) LoadPusher(ctx context.Context) error { ids := container.Set[int64]{} for _, branch := range branches { - if branch.PusherID > 0 { + if branch.PusherID > 0 { // pusher_id maybe zero because some branches are sync by backend with no pusher ids.Add(branch.PusherID) } } From 2f8fbfe473e12a7c820c990ae2daa4ec8284be13 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Wed, 28 Jun 2023 16:56:39 +0800 Subject: [PATCH 52/56] Add sync for ListBranch and DeleteBranch API --- models/git/branch_list.go | 8 ++--- models/git/branch_test.go | 2 +- modules/context/repo.go | 3 +- modules/repository/branch.go | 25 +++++++------ modules/repository/init.go | 2 +- modules/repository/repo.go | 2 +- routers/api/v1/repo/branch.go | 67 +++++++++++++++++++++++++++++------ routers/web/admin/admin.go | 2 +- services/repository/branch.go | 38 ++++++++------------ services/repository/fork.go | 3 +- 10 files changed, 95 insertions(+), 57 deletions(-) diff --git a/models/git/branch_list.go b/models/git/branch_list.go index 91aa7869f2258..4f13e328d4ad4 100644 --- a/models/git/branch_list.go +++ b/models/git/branch_list.go @@ -95,7 +95,7 @@ func CountBranches(ctx context.Context, opts FindBranchOptions) (int64, error) { return db.GetEngine(ctx).Where(opts.Cond()).Count(&Branch{}) } -func FindBranches(ctx context.Context, opts FindBranchOptions) (BranchList, int64, error) { +func FindBranches(ctx context.Context, opts FindBranchOptions) (BranchList, error) { sess := db.GetEngine(ctx).Where(opts.Cond()) if opts.PageSize > 0 && !opts.IsListAll() { sess = db.SetSessionPagination(sess, &opts.ListOptions) @@ -106,11 +106,7 @@ func FindBranches(ctx context.Context, opts FindBranchOptions) (BranchList, int6 sess = sess.OrderBy(opts.OrderBy) var branches []*Branch - total, err := sess.FindAndCount(&branches) - if err != nil { - return nil, 0, err - } - return branches, total, err + return branches, sess.Find(&branches) } func FindBranchNames(ctx context.Context, opts FindBranchOptions) ([]string, error) { diff --git a/models/git/branch_test.go b/models/git/branch_test.go index 6fe7ee2783d64..b6674edbe32e0 100644 --- a/models/git/branch_test.go +++ b/models/git/branch_test.go @@ -36,7 +36,7 @@ func TestGetDeletedBranches(t *testing.T) { assert.NoError(t, unittest.PrepareTestDatabase()) repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 1}) - branches, _, err := git_model.FindBranches(db.DefaultContext, git_model.FindBranchOptions{ + branches, err := git_model.FindBranches(db.DefaultContext, git_model.FindBranchOptions{ ListOptions: db.ListOptions{ ListAll: true, }, diff --git a/modules/context/repo.go b/modules/context/repo.go index e40b45f453987..e99908525128f 100644 --- a/modules/context/repo.go +++ b/modules/context/repo.go @@ -682,7 +682,8 @@ func RepoAssignment(ctx *Context) (cancel context.CancelFunc) { // non empty repo should have at least 1 branch, so this repository's branches haven't been synced yet if branchesTotal == 0 { // fallback to do a sync immediately - if err := repo_module.SyncRepoBranches(ctx, ctx.Repo.Repository.ID, 0); err != nil { + branchesTotal, err = repo_module.SyncRepoBranches(ctx, ctx.Repo.Repository.ID, 0) + if err != nil { ctx.ServerError("SyncRepoBranches", err) return } diff --git a/modules/repository/branch.go b/modules/repository/branch.go index 74de19fa97eee..372c0e57be76d 100644 --- a/modules/repository/branch.go +++ b/modules/repository/branch.go @@ -16,10 +16,10 @@ import ( ) // SyncRepoBranches synchronizes branch table with repository branches -func SyncRepoBranches(ctx context.Context, repoID, doerID int64) error { +func SyncRepoBranches(ctx context.Context, repoID, doerID int64) (int64, error) { repo, err := repo_model.GetRepositoryByID(ctx, repoID) if err != nil { - return err + return 0, err } log.Debug("SyncRepoBranches: in Repo[%d:%s]", repo.ID, repo.FullName()) @@ -27,19 +27,19 @@ func SyncRepoBranches(ctx context.Context, repoID, doerID int64) error { gitRepo, err := git.OpenRepository(ctx, repo.RepoPath()) if err != nil { log.Error("OpenRepository[%s]: %w", repo.RepoPath(), err) - return nil + return 0, err } defer gitRepo.Close() return SyncRepoBranchesWithRepo(ctx, repo, gitRepo, doerID) } -func SyncRepoBranchesWithRepo(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository, doerID int64) error { +func SyncRepoBranchesWithRepo(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository, doerID int64) (int64, error) { allBranches := container.Set[string]{} { branches, _, err := gitRepo.GetBranchNames(0, 0) if err != nil { - return err + return 0, err } log.Trace("SyncRepoBranches[%s]: branches[%d]: %v", repo.FullName(), len(branches), branches) for _, branch := range branches { @@ -49,14 +49,14 @@ func SyncRepoBranchesWithRepo(ctx context.Context, repo *repo_model.Repository, dbBranches := make(map[string]*git_model.Branch) { - branches, _, err := git_model.FindBranches(ctx, git_model.FindBranchOptions{ + branches, err := git_model.FindBranches(ctx, git_model.FindBranchOptions{ ListOptions: db.ListOptions{ ListAll: true, }, RepoID: repo.ID, }) if err != nil { - return err + return 0, err } for _, branch := range branches { dbBranches[branch.Name] = branch @@ -70,7 +70,7 @@ func SyncRepoBranchesWithRepo(ctx context.Context, repo *repo_model.Repository, dbb := dbBranches[branch] commit, err := gitRepo.GetBranchCommit(branch) if err != nil { - return err + return 0, err } if dbb == nil { toAdd = append(toAdd, &git_model.Branch{ @@ -103,10 +103,10 @@ func SyncRepoBranchesWithRepo(ctx context.Context, repo *repo_model.Repository, log.Trace("SyncRepoBranches[%s]: toAdd: %v, toUpdate: %v, toRemove: %v", repo.FullName(), toAdd, toUpdate, toRemove) if len(toAdd) == 0 && len(toRemove) == 0 && len(toUpdate) == 0 { - return nil + return int64(len(allBranches)), nil } - return db.WithTx(ctx, func(subCtx context.Context) error { + if err := db.WithTx(ctx, func(subCtx context.Context) error { if len(toAdd) > 0 { if err := git_model.AddBranches(subCtx, toAdd); err != nil { return err @@ -128,5 +128,8 @@ func SyncRepoBranchesWithRepo(ctx context.Context, repo *repo_model.Repository, } return nil - }) + }); err != nil { + return 0, err + } + return int64(len(allBranches)), nil } diff --git a/modules/repository/init.go b/modules/repository/init.go index 29cb974c27484..84648f45ebf4f 100644 --- a/modules/repository/init.go +++ b/modules/repository/init.go @@ -353,7 +353,7 @@ func initRepository(ctx context.Context, repoPath string, u *user_model.User, re } if !repo.IsEmpty { - if err := SyncRepoBranches(ctx, repo.ID, u.ID); err != nil { + if _, err := SyncRepoBranches(ctx, repo.ID, u.ID); err != nil { return fmt.Errorf("SyncRepoBranches: %w", err) } } diff --git a/modules/repository/repo.go b/modules/repository/repo.go index 71ce8fdeb8aa3..6a11315cc4043 100644 --- a/modules/repository/repo.go +++ b/modules/repository/repo.go @@ -151,7 +151,7 @@ func MigrateRepositoryGitData(ctx context.Context, u *user_model.User, } } - if err := SyncRepoBranchesWithRepo(ctx, repo, gitRepo, u.ID); err != nil { + if _, err := SyncRepoBranchesWithRepo(ctx, repo, gitRepo, u.ID); err != nil { return repo, fmt.Errorf("SyncRepoBranchesWithRepo: %v", err) } diff --git a/routers/api/v1/repo/branch.go b/routers/api/v1/repo/branch.go index dbdfd4c9e0c4c..4900ecf4d08be 100644 --- a/routers/api/v1/repo/branch.go +++ b/routers/api/v1/repo/branch.go @@ -15,6 +15,7 @@ import ( user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/git" + repo_module "code.gitea.io/gitea/modules/repository" api "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/modules/web" @@ -119,6 +120,37 @@ func DeleteBranch(ctx *context.APIContext) { branchName := ctx.Params("*") + if ctx.Repo.Repository.IsEmpty { + ctx.Error(http.StatusForbidden, "", "Git Repository is empty.") + return + } + + // check whether branches of this repository has been synced + totalNumOfBranches, err := git_model.CountBranches(ctx, git_model.FindBranchOptions{ + RepoID: ctx.Repo.Repository.ID, + IsDeletedBranch: util.OptionalBoolFalse, + }) + if err != nil { + ctx.Error(http.StatusInternalServerError, "CountBranches", err) + return + } + if totalNumOfBranches == 0 { // sync branches immediately because non-empty repository should have at least 1 branch + _, err = repo_module.SyncRepoBranches(ctx, ctx.Repo.Repository.ID, 0) + if err != nil { + ctx.ServerError("SyncRepoBranches", err) + return + } + } + + if ctx.Repo.Repository.IsArchived { + ctx.Error(http.StatusForbidden, "IsArchived", fmt.Errorf("can not delete branch of an archived repository")) + return + } + if ctx.Repo.Repository.IsMirror { + ctx.Error(http.StatusForbidden, "IsMirrored", fmt.Errorf("can not delete branch of an mirror repository")) + return + } + if err := repo_service.DeleteBranch(ctx, ctx.Doer, ctx.Repo.Repository, ctx.Repo.GitRepo, branchName); err != nil { switch { case git.IsErrBranchNotExist(err): @@ -276,23 +308,38 @@ func ListBranches(ctx *context.APIContext) { // "200": // "$ref": "#/responses/BranchList" - var totalNumOfBranches int + var totalNumOfBranches int64 var apiBranches []*api.Branch listOptions := utils.GetListOptions(ctx) if !ctx.Repo.Repository.IsEmpty && ctx.Repo.GitRepo != nil { + branchOpts := git_model.FindBranchOptions{ + ListOptions: listOptions, + RepoID: ctx.Repo.Repository.ID, + IsDeletedBranch: util.OptionalBoolFalse, + } + var err error + totalNumOfBranches, err = git_model.CountBranches(ctx, branchOpts) + if err != nil { + ctx.Error(http.StatusInternalServerError, "CountBranches", err) + return + } + if totalNumOfBranches == 0 { // sync branches immediately because non-empty repository should have at least 1 branch + totalNumOfBranches, err = repo_module.SyncRepoBranches(ctx, ctx.Repo.Repository.ID, 0) + if err != nil { + ctx.ServerError("SyncRepoBranches", err) + return + } + } + rules, err := git_model.FindRepoProtectedBranchRules(ctx, ctx.Repo.Repository.ID) if err != nil { ctx.Error(http.StatusInternalServerError, "FindMatchedProtectedBranchRules", err) return } - branches, total, err := git_model.FindBranches(ctx, git_model.FindBranchOptions{ - ListOptions: listOptions, - RepoID: ctx.Repo.Repository.ID, - IsDeletedBranch: util.OptionalBoolFalse, - }) + branches, err := git_model.FindBranches(ctx, branchOpts) if err != nil { ctx.Error(http.StatusInternalServerError, "GetBranches", err) return @@ -304,7 +351,7 @@ func ListBranches(ctx *context.APIContext) { if err != nil { // Skip if this branch doesn't exist anymore. if git.IsErrNotExist(err) { - total-- + totalNumOfBranches-- continue } ctx.Error(http.StatusInternalServerError, "GetCommit", err) @@ -319,12 +366,10 @@ func ListBranches(ctx *context.APIContext) { } apiBranches = append(apiBranches, apiBranch) } - - totalNumOfBranches = int(total) } - ctx.SetLinkHeader(totalNumOfBranches, listOptions.PageSize) - ctx.SetTotalCountHeader(int64(totalNumOfBranches)) + ctx.SetLinkHeader(int(totalNumOfBranches), listOptions.PageSize) + ctx.SetTotalCountHeader(totalNumOfBranches) ctx.JSON(http.StatusOK, apiBranches) } diff --git a/routers/web/admin/admin.go b/routers/web/admin/admin.go index 2643caae0ae1b..225a8c6705368 100644 --- a/routers/web/admin/admin.go +++ b/routers/web/admin/admin.go @@ -143,7 +143,7 @@ func DashboardPost(ctx *context.Context) { log.Error("AddAllRepoBranchesToSyncQueue: %v: %v", ctx.Doer.ID, err) } }() - ctx.Flash.Success(ctx.Tr("admin.dashboard.sync_branch.started", ctx.Tr("admin.dashboard."+form.Op))) + ctx.Flash.Success(ctx.Tr("admin.dashboard.sync_branch.started")) default: task := cron.GetTask(form.Op) if task != nil { diff --git a/services/repository/branch.go b/services/repository/branch.go index 1b401e2c71de3..bf8936009b72c 100644 --- a/services/repository/branch.go +++ b/services/repository/branch.go @@ -72,15 +72,23 @@ func LoadBranches(ctx context.Context, repo *repo_model.Repository, gitRepo *git return nil, nil, 0, err } - dbBranches, totalNumOfBranches, err := git_model.FindBranches(ctx, git_model.FindBranchOptions{ - RepoID: repo.ID, - ExcludeBranchNames: []string{repo.DefaultBranch}, - IsDeletedBranch: isDeletedBranch, + branchOpts := git_model.FindBranchOptions{ + RepoID: repo.ID, + IsDeletedBranch: isDeletedBranch, ListOptions: db.ListOptions{ Page: page, PageSize: pageSize, }, - }) + } + + totalNumOfBranches, err := git_model.CountBranches(ctx, branchOpts) + if err != nil { + return nil, nil, 0, err + } + + branchOpts.ExcludeBranchNames = []string{repo.DefaultBranch} + + dbBranches, err := git_model.FindBranches(ctx, branchOpts) if err != nil { return nil, nil, 0, err } @@ -372,7 +380,8 @@ var branchSyncQueue *queue.WorkerPoolQueue[*BranchSyncOptions] func handlerBranchSync(items ...*BranchSyncOptions) []*BranchSyncOptions { var failedOptions []*BranchSyncOptions for _, opts := range items { - if err := repo_module.SyncRepoBranches(graceful.GetManager().ShutdownContext(), opts.RepoID, opts.DoerID); err != nil { + _, err := repo_module.SyncRepoBranches(graceful.GetManager().ShutdownContext(), opts.RepoID, opts.DoerID) + if err != nil { log.Error("syncRepoBranches [%d:%d] failed: %v", opts.RepoID, opts.DoerID, err) failedOptions = append(failedOptions, opts) } @@ -394,23 +403,6 @@ func initBranchSyncQueue(ctx context.Context) error { } go graceful.GetManager().RunWithCancel(branchSyncQueue) - // for test env, this will return empty because data hasn't been loaded yet - // but that env don't need the sync - cnt, err := git_model.CountBranches(ctx, git_model.FindBranchOptions{ - IsDeletedBranch: util.OptionalBoolFalse, - }) - if err != nil { - return fmt.Errorf("CountBranches: %v", err) - } - - if cnt == 0 { - go func() { - // doerID is 0 means no pusher will be displayed in the branch list ui - if err := AddAllRepoBranchesToSyncQueue(ctx, 0); err != nil { - log.Error("AddAllRepoBranchesToSyncQueue: %v", err) - } - }() - } return nil } diff --git a/services/repository/fork.go b/services/repository/fork.go index 1b9c30b114257..59aa173373314 100644 --- a/services/repository/fork.go +++ b/services/repository/fork.go @@ -164,7 +164,8 @@ func ForkRepository(ctx context.Context, doer, owner *user_model.User, opts Fork } defer gitRepo.Close() - return repo_module.SyncRepoBranchesWithRepo(txCtx, repo, gitRepo, doer.ID) + _, err = repo_module.SyncRepoBranchesWithRepo(txCtx, repo, gitRepo, doer.ID) + return err }) needsRollbackInPanic = false if err != nil { From 41359d8b2bc2d8e43e3ad01fe266731f7d3ea6c3 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Wed, 28 Jun 2023 17:02:16 +0800 Subject: [PATCH 53/56] Don't put the failure sync branches task back to queue --- services/repository/branch.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/services/repository/branch.go b/services/repository/branch.go index bf8936009b72c..71ef07fe99c4f 100644 --- a/services/repository/branch.go +++ b/services/repository/branch.go @@ -378,15 +378,13 @@ type BranchSyncOptions struct { var branchSyncQueue *queue.WorkerPoolQueue[*BranchSyncOptions] func handlerBranchSync(items ...*BranchSyncOptions) []*BranchSyncOptions { - var failedOptions []*BranchSyncOptions for _, opts := range items { _, err := repo_module.SyncRepoBranches(graceful.GetManager().ShutdownContext(), opts.RepoID, opts.DoerID) if err != nil { log.Error("syncRepoBranches [%d:%d] failed: %v", opts.RepoID, opts.DoerID, err) - failedOptions = append(failedOptions, opts) } } - return failedOptions + return nil } func addRepoToBranchSyncQueue(repoID, doerID int64) error { From 1ab19bd61dcb7c26ccb4e0a1e8b0752c05fe573c Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Wed, 28 Jun 2023 17:33:36 +0800 Subject: [PATCH 54/56] Fix some bugs --- models/git/branch.go | 10 +++++++++- models/git/branch_list.go | 22 ++++++++++++++-------- modules/repository/branch.go | 2 +- 3 files changed, 24 insertions(+), 10 deletions(-) diff --git a/models/git/branch.go b/models/git/branch.go index 70b7218f09fa3..adf8b0a78d223 100644 --- a/models/git/branch.go +++ b/models/git/branch.go @@ -206,7 +206,7 @@ func DeleteBranches(ctx context.Context, repoID, doerID int64, branchIDs []int64 // If it doest not exist, insert a new record into database func UpdateBranch(ctx context.Context, repoID int64, branchName, commitID, commitMessage string, pusherID int64, commitTime time.Time) error { cnt, err := db.GetEngine(ctx).Where("repo_id=? AND name=?", repoID, branchName). - Cols("commit_sha, commit_message, pusher_id, commit_time, is_deleted, updated_unix"). + Cols("commit_id, commit_message, pusher_id, commit_time, is_deleted, updated_unix"). Update(&Branch{ CommitID: commitID, CommitMessage: commitMessage, @@ -233,6 +233,14 @@ func UpdateBranch(ctx context.Context, repoID int64, branchName, commitID, commi // AddDeletedBranch adds a deleted branch to the database func AddDeletedBranch(ctx context.Context, repoID int64, branchName string, deletedByID int64) error { + branch, err := GetBranch(ctx, repoID, branchName) + if err != nil { + return err + } + if branch.IsDeleted { + return nil + } + cnt, err := db.GetEngine(ctx).Where("repo_id=? AND name=? AND is_deleted=?", repoID, branchName, false). Cols("is_deleted, deleted_by_id, deleted_unix"). Update(&Branch{ diff --git a/models/git/branch_list.go b/models/git/branch_list.go index 4f13e328d4ad4..da78248c0bc01 100644 --- a/models/git/branch_list.go +++ b/models/git/branch_list.go @@ -12,6 +12,7 @@ import ( "code.gitea.io/gitea/modules/util" "xorm.io/builder" + "xorm.io/xorm" ) type BranchList []*Branch @@ -95,15 +96,23 @@ func CountBranches(ctx context.Context, opts FindBranchOptions) (int64, error) { return db.GetEngine(ctx).Where(opts.Cond()).Count(&Branch{}) } +func orderByBranches(sess *xorm.Session, opts FindBranchOptions) *xorm.Session { + if !opts.IsDeletedBranch.IsFalse() { // if deleted branch included, put them at the end + sess = sess.OrderBy("is_deleted ASC") + } + + if opts.OrderBy == "" { + opts.OrderBy = BranchOrderByCommitTimeDesc + } + return sess.OrderBy(opts.OrderBy) +} + func FindBranches(ctx context.Context, opts FindBranchOptions) (BranchList, error) { sess := db.GetEngine(ctx).Where(opts.Cond()) if opts.PageSize > 0 && !opts.IsListAll() { sess = db.SetSessionPagination(sess, &opts.ListOptions) } - if opts.OrderBy == "" { - opts.OrderBy = BranchOrderByCommitTimeDesc - } - sess = sess.OrderBy(opts.OrderBy) + sess = orderByBranches(sess, opts) var branches []*Branch return branches, sess.Find(&branches) @@ -114,10 +123,7 @@ func FindBranchNames(ctx context.Context, opts FindBranchOptions) ([]string, err if opts.PageSize > 0 && !opts.IsListAll() { sess = db.SetSessionPagination(sess, &opts.ListOptions) } - if opts.OrderBy == "" { - opts.OrderBy = BranchOrderByCommitTimeDesc - } - sess = sess.OrderBy(opts.OrderBy) + sess = orderByBranches(sess, opts) var branches []string if err := sess.Table("branch").Find(&branches); err != nil { return nil, err diff --git a/modules/repository/branch.go b/modules/repository/branch.go index 372c0e57be76d..7fd29e3f7d027 100644 --- a/modules/repository/branch.go +++ b/modules/repository/branch.go @@ -115,7 +115,7 @@ func SyncRepoBranchesWithRepo(ctx context.Context, repo *repo_model.Repository, for _, b := range toUpdate { if _, err := db.GetEngine(subCtx).ID(b.ID). - Cols("commit_sha, commit_message, pusher_id, commit_time, is_deleted"). + Cols("commit_id, commit_message, pusher_id, commit_time, is_deleted"). Update(b); err != nil { return err } From c41ad1699cc5b752e2a9707a781b55d24939ec50 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Wed, 28 Jun 2023 17:54:10 +0800 Subject: [PATCH 55/56] Delete a deleted branch now return no error --- models/git/branch_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/git/branch_test.go b/models/git/branch_test.go index b6674edbe32e0..bb63660d078a9 100644 --- a/models/git/branch_test.go +++ b/models/git/branch_test.go @@ -22,7 +22,7 @@ func TestAddDeletedBranch(t *testing.T) { firstBranch := unittest.AssertExistsAndLoadBean(t, &git_model.Branch{ID: 1}) assert.True(t, firstBranch.IsDeleted) - assert.Error(t, git_model.AddDeletedBranch(db.DefaultContext, repo.ID, firstBranch.Name, firstBranch.DeletedByID)) + assert.NoError(t, git_model.AddDeletedBranch(db.DefaultContext, repo.ID, firstBranch.Name, firstBranch.DeletedByID)) assert.NoError(t, git_model.AddDeletedBranch(db.DefaultContext, repo.ID, "branch2", int64(1))) secondBranch := unittest.AssertExistsAndLoadBean(t, &git_model.Branch{RepoID: repo.ID, Name: "branch2"}) From cd195ab947e404ee05ce705f699c24d67b715635 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Wed, 28 Jun 2023 18:05:14 +0800 Subject: [PATCH 56/56] Use unique queue --- services/repository/branch.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/services/repository/branch.go b/services/repository/branch.go index 71ef07fe99c4f..11a8b20531578 100644 --- a/services/repository/branch.go +++ b/services/repository/branch.go @@ -371,7 +371,6 @@ func DeleteBranch(ctx context.Context, doer *user_model.User, repo *repo_model.R type BranchSyncOptions struct { RepoID int64 - DoerID int64 } // branchSyncQueue represents a queue to handle branch sync jobs. @@ -379,9 +378,9 @@ var branchSyncQueue *queue.WorkerPoolQueue[*BranchSyncOptions] func handlerBranchSync(items ...*BranchSyncOptions) []*BranchSyncOptions { for _, opts := range items { - _, err := repo_module.SyncRepoBranches(graceful.GetManager().ShutdownContext(), opts.RepoID, opts.DoerID) + _, err := repo_module.SyncRepoBranches(graceful.GetManager().ShutdownContext(), opts.RepoID, 0) if err != nil { - log.Error("syncRepoBranches [%d:%d] failed: %v", opts.RepoID, opts.DoerID, err) + log.Error("syncRepoBranches [%d] failed: %v", opts.RepoID, err) } } return nil @@ -390,12 +389,11 @@ func handlerBranchSync(items ...*BranchSyncOptions) []*BranchSyncOptions { func addRepoToBranchSyncQueue(repoID, doerID int64) error { return branchSyncQueue.Push(&BranchSyncOptions{ RepoID: repoID, - DoerID: doerID, }) } func initBranchSyncQueue(ctx context.Context) error { - branchSyncQueue = queue.CreateSimpleQueue(ctx, "branch_sync", handlerBranchSync) + branchSyncQueue = queue.CreateUniqueQueue(ctx, "branch_sync", handlerBranchSync) if branchSyncQueue == nil { return errors.New("unable to create branch_sync queue") }