From 726748a7e78a2563acb026c1fa2512487106ba1e Mon Sep 17 00:00:00 2001 From: Bo-Yi Wu Date: Tue, 20 Feb 2018 19:49:44 +0800 Subject: [PATCH 1/2] refactor: reduce sql query in retrieveFeeds --- models/action.go | 11 ++++- models/action_list.go | 98 +++++++++++++++++++++++++++++++++++++++++++ routers/user/home.go | 39 ++++------------- 3 files changed, 116 insertions(+), 32 deletions(-) create mode 100644 models/action_list.go diff --git a/models/action.go b/models/action.go index 5333f6277206c..b551d79bb8cf4 100644 --- a/models/action.go +++ b/models/action.go @@ -742,5 +742,14 @@ func GetFeeds(opts GetFeedsOptions) ([]*Action, error) { } actions := make([]*Action, 0, 20) - return actions, x.Limit(20).Desc("id").Where(cond).Find(&actions) + + if err := x.Limit(20).Desc("id").Where(cond).Find(&actions); err != nil { + return nil, fmt.Errorf("Find: %v", err) + } + + if err := ActionList(actions).LoadAttributes(); err != nil { + return nil, fmt.Errorf("LoadAttributes: %v", err) + } + + return actions, nil } diff --git a/models/action_list.go b/models/action_list.go new file mode 100644 index 0000000000000..6f726f4b34c08 --- /dev/null +++ b/models/action_list.go @@ -0,0 +1,98 @@ +// Copyright 2018 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package models + +import "fmt" + +// ActionList defines a list of actions +type ActionList []*Action + +func (actions ActionList) getUserIDs() []int64 { + userIDs := make(map[int64]struct{}, len(actions)) + for _, action := range actions { + if _, ok := userIDs[action.ActUserID]; !ok { + userIDs[action.ActUserID] = struct{}{} + } + } + return keysInt64(userIDs) +} + +func (actions ActionList) loadUsers(e Engine) ([]*User, error) { + if len(actions) == 0 { + return nil, nil + } + + userIDs := actions.getUserIDs() + userMaps := make(map[int64]*User, len(userIDs)) + err := e. + In("id", userIDs). + Find(&userMaps) + if err != nil { + return nil, fmt.Errorf("find user: %v", err) + } + + for _, action := range actions { + action.ActUser = userMaps[action.ActUserID] + } + return valuesUser(userMaps), nil +} + +// LoadUsers loads actions' all users +func (actions ActionList) LoadUsers() ([]*User, error) { + return actions.loadUsers(x) +} + +func (actions ActionList) getRepoIDs() []int64 { + repoIDs := make(map[int64]struct{}, len(actions)) + for _, action := range actions { + if _, ok := repoIDs[action.RepoID]; !ok { + repoIDs[action.RepoID] = struct{}{} + } + } + return keysInt64(repoIDs) +} + +func (actions ActionList) loadRepositories(e Engine) ([]*Repository, error) { + if len(actions) == 0 { + return nil, nil + } + + repoIDs := actions.getRepoIDs() + repoMaps := make(map[int64]*Repository, len(repoIDs)) + err := e. + In("id", repoIDs). + Find(&repoMaps) + if err != nil { + return nil, fmt.Errorf("find repository: %v", err) + } + + for _, action := range actions { + action.Repo = repoMaps[action.RepoID] + } + return valuesRepository(repoMaps), nil +} + +// LoadRepositories loads actions' all repositories +func (actions ActionList) LoadRepositories() ([]*Repository, error) { + return actions.loadRepositories(x) +} + +// loadAttributes loads all attributes +func (actions ActionList) loadAttributes(e Engine) (err error) { + if _, err = actions.loadUsers(e); err != nil { + return + } + + if _, err = actions.loadRepositories(e); err != nil { + return + } + + return nil +} + +// LoadAttributes loads attributes of the actions +func (actions ActionList) LoadAttributes() error { + return actions.loadAttributes(x) +} diff --git a/routers/user/home.go b/routers/user/home.go index 5687cb4f1a3e0..2a193bbdef519 100644 --- a/routers/user/home.go +++ b/routers/user/home.go @@ -66,39 +66,14 @@ func retrieveFeeds(ctx *context.Context, options models.GetFeedsOptions) { if ctx.User != nil { userCache[ctx.User.ID] = ctx.User } - repoCache := map[int64]*models.Repository{} for _, act := range actions { - // Cache results to reduce queries. - u, ok := userCache[act.ActUserID] - if !ok { - u, err = models.GetUserByID(act.ActUserID) - if err != nil { - if models.IsErrUserNotExist(err) { - continue - } - ctx.ServerError("GetUserByID", err) - return - } - userCache[act.ActUserID] = u - } - act.ActUser = u - - repo, ok := repoCache[act.RepoID] - if !ok { - repo, err = models.GetRepositoryByID(act.RepoID) - if err != nil { - if models.IsErrRepoNotExist(err) { - continue - } - ctx.ServerError("GetRepositoryByID", err) - return - } + if act.ActUser != nil { + userCache[act.ActUserID] = act.ActUser } - act.Repo = repo - repoOwner, ok := userCache[repo.OwnerID] + repoOwner, ok := userCache[act.Repo.OwnerID] if !ok { - repoOwner, err = models.GetUserByID(repo.OwnerID) + repoOwner, err = models.GetUserByID(act.Repo.OwnerID) if err != nil { if models.IsErrUserNotExist(err) { continue @@ -106,8 +81,9 @@ func retrieveFeeds(ctx *context.Context, options models.GetFeedsOptions) { ctx.ServerError("GetUserByID", err) return } + userCache[repoOwner.ID] = repoOwner } - repo.Owner = repoOwner + act.Repo.Owner = repoOwner } ctx.Data["Feeds"] = actions } @@ -154,7 +130,8 @@ func Dashboard(ctx *context.Context) { ctx.Data["MirrorCount"] = len(mirrors) ctx.Data["Mirrors"] = mirrors - retrieveFeeds(ctx, models.GetFeedsOptions{RequestedUser: ctxUser, + retrieveFeeds(ctx, models.GetFeedsOptions{ + RequestedUser: ctxUser, IncludePrivate: true, OnlyPerformedBy: false, IncludeDeleted: false, From a4ad6a92123330c23c843895f6b7230b97584fd8 Mon Sep 17 00:00:00 2001 From: Bo-Yi Wu Date: Fri, 23 Feb 2018 08:27:09 +0800 Subject: [PATCH 2/2] =?UTF-8?q?fix:=20if=20Mirrors=20repo=20no=20content?= =?UTF-8?q?=20is=20fetched,=20updated=20time=20should=20not=20b=E2=80=A6?= =?UTF-8?q?=20(#3551)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: if Mirrors repo no content is fetched, updated time should not be changed * fix: sync update time from mirror repo. * fix: one single session. * update comment. Signed-off-by: Bo-Yi Wu --- models/repo_mirror.go | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/models/repo_mirror.go b/models/repo_mirror.go index 197889e19ac69..97fe2406c65ff 100644 --- a/models/repo_mirror.go +++ b/models/repo_mirror.go @@ -244,6 +244,8 @@ func MirrorUpdate() { // SyncMirrors checks and syncs mirrors. // TODO: sync more mirrors at same time. func SyncMirrors() { + sess := x.NewSession() + defer sess.Close() // Start listening on new sync requests. for repoID := range MirrorQueue.Queue() { log.Trace("SyncMirrors [repo_id: %v]", repoID) @@ -260,10 +262,22 @@ func SyncMirrors() { } m.ScheduleNextUpdate() - if err = UpdateMirror(m); err != nil { + if err = updateMirror(sess, m); err != nil { log.Error(4, "UpdateMirror [%s]: %v", repoID, err) continue } + + // Get latest commit date and update to current repository updated time + commitDate, err := git.GetLatestCommitTime(m.Repo.RepoPath()) + if err != nil { + log.Error(2, "GetLatestCommitDate [%s]: %v", m.RepoID, err) + continue + } + + if _, err = sess.Exec("UPDATE repository SET updated_unix = ? WHERE id = ?", commitDate.Unix(), m.RepoID); err != nil { + log.Error(2, "Update repository 'updated_unix' [%s]: %v", m.RepoID, err) + continue + } } }