API add/generalize pagination (#9452)

* paginate results

* fixed deadlock

* prevented breaking change

* updated swagger

* go fmt

* fixed find topic

* go mod tidy

* go mod vendor with go1.13.5

* fixed repo find topics

* fixed unit test

* added Limit method to Engine struct; use engine variable when provided; fixed gitignore

* use ItemsPerPage for default pagesize; fix GetWatchers, getOrgUsersByOrgID and GetStargazers; fix GetAllCommits headers; reverted some changed behaviors

* set Page value on Home route

* improved memory allocations

* fixed response headers

* removed logfiles

* fixed import order

* import order

* improved swagger

* added function to get models.ListOptions from context

* removed pagesize diff on unit test

* fixed imports

* removed unnecessary struct field

* fixed go fmt

* scoped PR

* code improvements

* code improvements

* go mod tidy

* fixed import order

* fixed commit statuses session

* fixed files headers

* fixed headers; added pagination for notifications

* go mod tidy

* go fmt

* removed Private from user search options; added setting.UI.IssuePagingNum as default valeu on repo's issues list

* Apply suggestions from code review

Co-Authored-By: 6543 <[email protected]>
Co-Authored-By: zeripath <[email protected]>

* fixed build error

* CI.restart()

* fixed merge conflicts resolve

* fixed conflicts resolve

* improved FindTrackedTimesOptions.ToOptions() method

* added backwards compatibility on ListReleases request; fixed issue tracked time ToSession

* fixed build error; fixed swagger template

* fixed swagger template

* fixed ListReleases backwards compatibility

* added page to user search route

Co-authored-by: techknowlogick <[email protected]>
Co-authored-by: 6543 <[email protected]>
Co-authored-by: zeripath <[email protected]>

Author: 4 people

cmd/admin.go

@@ -351,9 +351,11 @@ func runRepoSyncReleases(c *cli.Context) error {
 	log.Trace("Synchronizing repository releases (this may take a while)")
 	for page := 1; ; page++ {
 		repos, count, err := models.SearchRepositoryByName(&models.SearchRepoOptions{
-			Page:     page,
-			PageSize: models.RepositoryListDefaultPageSize,
-			Private:  true,
+			ListOptions: models.ListOptions{
+				PageSize: models.RepositoryListDefaultPageSize,
+				Page:     page,
+			},
+			Private: true,
 		})
 		if err != nil {
 			return fmt.Errorf("SearchRepositoryByName: %v", err)

models/access.go

@@ -215,7 +215,7 @@ func (repo *Repository) refreshAccesses(e Engine, accessMap map[int64]*userAcces
 
 // refreshCollaboratorAccesses retrieves repository collaborations with their access modes.
 func (repo *Repository) refreshCollaboratorAccesses(e Engine, accessMap map[int64]*userAccess) error {
-	collaborators, err := repo.getCollaborators(e)
+	collaborators, err := repo.getCollaborators(e, ListOptions{})
 	if err != nil {
 		return fmt.Errorf("getCollaborations: %v", err)
 	}

models/commit_status.go

@@ -104,7 +104,7 @@ func CalcCommitStatus(statuses []*CommitStatus) *CommitStatus {
 
 // CommitStatusOptions holds the options for query commit statuses
 type CommitStatusOptions struct {
-	Page     int
+	ListOptions
 	State    string
 	SortType string
 }
@@ -114,18 +114,22 @@ func GetCommitStatuses(repo *Repository, sha string, opts *CommitStatusOptions)
 	if opts.Page <= 0 {
 		opts.Page = 1
 	}
+	if opts.PageSize <= 0 {
+		opts.Page = ItemsPerPage
+	}
 
 	countSession := listCommitStatusesStatement(repo, sha, opts)
+	countSession = opts.setSessionPagination(countSession)
 	maxResults, err := countSession.Count(new(CommitStatus))
 	if err != nil {
 		log.Error("Count PRs: %v", err)
 		return nil, maxResults, err
 	}
 
-	statuses := make([]*CommitStatus, 0, ItemsPerPage)
+	statuses := make([]*CommitStatus, 0, opts.PageSize)
 	findSession := listCommitStatusesStatement(repo, sha, opts)
+	findSession = opts.setSessionPagination(findSession)
 	sortCommitStatusesSession(findSession, opts.SortType)
-	findSession.Limit(ItemsPerPage, (opts.Page-1)*ItemsPerPage)
 	return statuses, maxResults, findSession.Find(&statuses)
 }
 

models/gpg_key.go

@@ -64,9 +64,14 @@ func (key *GPGKey) AfterLoad(session *xorm.Session) {
 }
 
 // ListGPGKeys returns a list of public keys belongs to given user.
-func ListGPGKeys(uid int64) ([]*GPGKey, error) {
-	keys := make([]*GPGKey, 0, 5)
-	return keys, x.Where("owner_id=? AND primary_key_id=''", uid).Find(&keys)
+func ListGPGKeys(uid int64, listOptions ListOptions) ([]*GPGKey, error) {
+	sess := x.Where("owner_id=? AND primary_key_id=''", uid)
+	if listOptions.Page != 0 {
+		sess = listOptions.setSessionPagination(sess)
+	}
+
+	keys := make([]*GPGKey, 0, 2)
+	return keys, sess.Find(&keys)
 }
 
 // GetGPGKeyByID returns public key by given ID.
@@ -628,7 +633,7 @@ func ParseCommitWithSignature(c *git.Commit) *CommitVerification {
 
 	// Now try to associate the signature with the committer, if present
 	if committer.ID != 0 {
-		keys, err := ListGPGKeys(committer.ID)
+		keys, err := ListGPGKeys(committer.ID, ListOptions{})
 		if err != nil { //Skipping failed to get gpg keys of user
 			log.Error("ListGPGKeys: %v", err)
 			return &CommitVerification{

models/issue.go

@@ -1103,13 +1103,12 @@ func GetIssuesByIDs(issueIDs []int64) ([]*Issue, error) {
 
 // IssuesOptions represents options of an issue.
 type IssuesOptions struct {
+	ListOptions
 	RepoIDs     []int64 // include all repos if empty
 	AssigneeID  int64
 	PosterID    int64
 	MentionedID int64
 	MilestoneID int64
-	Page        int
-	PageSize    int
 	IsClosed    util.OptionalBool
 	IsPull      util.OptionalBool
 	LabelIDs    []int64

models/issue_comment.go

@@ -777,6 +777,7 @@ func GetCommentByID(id int64) (*Comment, error) {
 
 // FindCommentsOptions describes the conditions to Find comments
 type FindCommentsOptions struct {
+	ListOptions
 	RepoID   int64
 	IssueID  int64
 	ReviewID int64
@@ -814,6 +815,11 @@ func findComments(e Engine, opts FindCommentsOptions) ([]*Comment, error) {
 	if opts.RepoID > 0 {
 		sess.Join("INNER", "issue", "issue.id = comment.issue_id")
 	}
+
+	if opts.Page != 0 {
+		sess = opts.setSessionPagination(sess)
+	}
+
 	return comments, sess.
 		Asc("comment.created_unix").
 		Asc("comment.id").

models/issue_label.go

@@ -1,4 +1,5 @@
 // Copyright 2016 The Gogs Authors. All rights reserved.
+// Copyright 2020 The Gitea Authors.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 
@@ -298,7 +299,7 @@ func GetLabelsInRepoByIDs(repoID int64, labelIDs []int64) ([]*Label, error) {
 		Find(&labels)
 }
 
-func getLabelsByRepoID(e Engine, repoID int64, sortType string) ([]*Label, error) {
+func getLabelsByRepoID(e Engine, repoID int64, sortType string, listOptions ListOptions) ([]*Label, error) {
 	labels := make([]*Label, 0, 10)
 	sess := e.Where("repo_id = ?", repoID)
 
@@ -313,12 +314,16 @@ func getLabelsByRepoID(e Engine, repoID int64, sortType string) ([]*Label, error
 		sess.Asc("name")
 	}
 
+	if listOptions.Page != 0 {
+		sess = listOptions.setSessionPagination(sess)
+	}
+
 	return labels, sess.Find(&labels)
 }
 
 // GetLabelsByRepoID returns all labels that belong to given repository by ID.
-func GetLabelsByRepoID(repoID int64, sortType string) ([]*Label, error) {
-	return getLabelsByRepoID(x, repoID, sortType)
+func GetLabelsByRepoID(repoID int64, sortType string, listOptions ListOptions) ([]*Label, error) {
+	return getLabelsByRepoID(x, repoID, sortType, listOptions)
 }
 
 func getLabelsByIssueID(e Engine, issueID int64) ([]*Label, error) {

models/issue_label_test.go

@@ -131,7 +131,7 @@ func TestGetLabelsInRepoByIDs(t *testing.T) {
 func TestGetLabelsByRepoID(t *testing.T) {
 	assert.NoError(t, PrepareTestDatabase())
 	testSuccess := func(repoID int64, sortType string, expectedIssueIDs []int64) {
-		labels, err := GetLabelsByRepoID(repoID, sortType)
+		labels, err := GetLabelsByRepoID(repoID, sortType, ListOptions{})
 		assert.NoError(t, err)
 		assert.Len(t, labels, len(expectedIssueIDs))
 		for i, label := range labels {

models/issue_milestone.go

@@ -221,7 +221,7 @@ func (milestones MilestoneList) getMilestoneIDs() []int64 {
 }
 
 // GetMilestonesByRepoID returns all opened milestones of a repository.
-func GetMilestonesByRepoID(repoID int64, state api.StateType) (MilestoneList, error) {
+func GetMilestonesByRepoID(repoID int64, state api.StateType, listOptions ListOptions) (MilestoneList, error) {
 	sess := x.Where("repo_id = ?", repoID)
 
 	switch state {
@@ -238,7 +238,11 @@ func GetMilestonesByRepoID(repoID int64, state api.StateType) (MilestoneList, er
 		sess = sess.And("is_closed = ?", false)
 	}
 
-	miles := make([]*Milestone, 0, 10)
+	if listOptions.Page != 0 {
+		sess = listOptions.setSessionPagination(sess)
+	}
+
+	miles := make([]*Milestone, 0, listOptions.PageSize)
 	return miles, sess.Asc("deadline_unix").Asc("id").Find(&miles)
 }
 

models/issue_milestone_test.go

@@ -71,7 +71,7 @@ func TestGetMilestonesByRepoID(t *testing.T) {
 	assert.NoError(t, PrepareTestDatabase())
 	test := func(repoID int64, state api.StateType) {
 		repo := AssertExistsAndLoadBean(t, &Repository{ID: repoID}).(*Repository)
-		milestones, err := GetMilestonesByRepoID(repo.ID, state)
+		milestones, err := GetMilestonesByRepoID(repo.ID, state, ListOptions{})
 		assert.NoError(t, err)
 
 		var n int
@@ -105,7 +105,7 @@ func TestGetMilestonesByRepoID(t *testing.T) {
 	test(3, api.StateClosed)
 	test(3, api.StateAll)
 
-	milestones, err := GetMilestonesByRepoID(NonexistentID, api.StateOpen)
+	milestones, err := GetMilestonesByRepoID(NonexistentID, api.StateOpen, ListOptions{})
 	assert.NoError(t, err)
 	assert.Len(t, milestones, 0)
 }