From 2892d1c2faa263b9930c619bc2e1946bf8f6ece8 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sat, 23 Apr 2022 22:34:35 +0100 Subject: [PATCH 1/9] Cache statistics and provide estimation methods Currently whenever the prometheus metrics endpoint or `/admin` endpoint are viewed the statistics are recalculated immediately - using COUNT rather than a less expensive method. This PR provides a mechanism to cache these statistics, avoids generating all of the metrics on the admin page and provides an estimation method for the plain table counts. Fix #17506 Signed-off-by: Andrew Thornton --- .../doc/advanced/config-cheat-sheet.en-us.md | 4 + models/db/context.go | 22 ++++ models/statistic.go | 28 +++-- modules/metrics/collector.go | 118 +++++++++--------- modules/metrics/statistics.go | 38 ++++++ modules/setting/cache.go | 16 +-- modules/setting/setting.go | 8 ++ routers/web/admin/admin.go | 6 +- templates/admin/dashboard.tmpl | 1 + 9 files changed, 165 insertions(+), 76 deletions(-) create mode 100644 modules/metrics/statistics.go diff --git a/docs/content/doc/advanced/config-cheat-sheet.en-us.md b/docs/content/doc/advanced/config-cheat-sheet.en-us.md index 9d70269bf2fd4..ce81a0891f106 100644 --- a/docs/content/doc/advanced/config-cheat-sheet.en-us.md +++ b/docs/content/doc/advanced/config-cheat-sheet.en-us.md @@ -197,6 +197,8 @@ The following configuration set `Content-Type: application/vnd.android.package-a - `REPO_PAGING_NUM`: **50**: Number of repos that are shown in one page. - `NOTICE_PAGING_NUM`: **25**: Number of notices that are shown in one page. - `ORG_PAGING_NUM`: **50**: Number of organizations that are shown in one page. +- `ESTIMATE_COUNTS`: **false**: Estimate counts for summary statistics instead counting directly. +- `STATISTICS_TTL`: **5m**: Cache summary statistics for this period of time. ### UI - Metadata (`ui.meta`) @@ -974,6 +976,8 @@ Default templates for project boards: - `ENABLED_ISSUE_BY_LABEL`: **false**: Enable issue by label metrics with format `gitea_issues_by_label{label="bug"} 2`. - `ENABLED_ISSUE_BY_REPOSITORY`: **false**: Enable issue by repository metrics with format `gitea_issues_by_repository{repository="org/repo"} 5`. - `TOKEN`: **\**: You need to specify the token, if you want to include in the authorization the metrics . The same token need to be used in prometheus parameters `bearer_token` or `bearer_token_file`. +- `ESTIMATE_COUNTS`: **false**: Estimate counts for statistics instead counting directly. +- `STATISTICS_TTL`: **5m**: Cache summary statistics for this period of time. ## API (`api`) diff --git a/models/db/context.go b/models/db/context.go index 1cd23d453ce4c..d918a6af76a4c 100644 --- a/models/db/context.go +++ b/models/db/context.go @@ -11,6 +11,7 @@ import ( "code.gitea.io/gitea/modules/setting" "xorm.io/builder" + "xorm.io/xorm/schemas" ) // DefaultContext is the default context to run xorm queries in @@ -168,3 +169,24 @@ func CountByBean(ctx context.Context, bean interface{}) (int64, error) { func TableName(bean interface{}) string { return x.TableName(bean) } + +// EstimateTotal returns an estimate of total number of rows in table +func EstimateTotal(bean interface{}) (int64, error) { + tablename := x.TableName(bean) + switch x.Dialect().URI().DBType { + case schemas.MYSQL: + var rows int64 + _, err := x.SQL("SELECT table_rows FROM information_schema.tables WHERE tables.table_name = ? AND tables.table_schema = ?;", tablename, x.Dialect().URI().DBName).Get(&rows) + return rows, err + case schemas.POSTGRES: + var rows int64 + _, err := x.SQL("SELECT reltuples AS estimate FROM pg_class WHERE relname = ?;", tablename).Get(&rows) + return rows, err + case schemas.MSSQL: + var rows int64 + _, err := x.SQL("sp_spaceused ?;", tablename).Get(&rows) + return rows, err + default: + return x.Count(tablename) + } +} diff --git a/models/statistic.go b/models/statistic.go index 87c1bd6d754eb..6aeef790be0c7 100644 --- a/models/statistic.go +++ b/models/statistic.go @@ -5,6 +5,8 @@ package models import ( + "time" + asymkey_model "code.gitea.io/gitea/models/asymkey" "code.gitea.io/gitea/models/auth" "code.gitea.io/gitea/models/db" @@ -31,6 +33,7 @@ type Statistic struct { IssueByLabel []IssueByLabelCount IssueByRepository []IssueByRepositoryCount } + Time time.Time } // IssueByLabelCount contains the number of issue group by label @@ -47,23 +50,31 @@ type IssueByRepositoryCount struct { } // GetStatistic returns the database statistics -func GetStatistic() (stats Statistic) { +func GetStatistic(estimate, metrics bool) (stats Statistic) { e := db.GetEngine(db.DefaultContext) stats.Counter.User = user_model.CountUsers() stats.Counter.Org = organization.CountOrganizations() - stats.Counter.PublicKey, _ = e.Count(new(asymkey_model.PublicKey)) stats.Counter.Repo = repo_model.CountRepositories(true) - stats.Counter.Watch, _ = e.Count(new(repo_model.Watch)) - stats.Counter.Star, _ = e.Count(new(repo_model.Star)) - stats.Counter.Action, _ = e.Count(new(Action)) - stats.Counter.Access, _ = e.Count(new(Access)) + if estimate { + stats.Counter.PublicKey, _ = db.EstimateTotal(new(asymkey_model.PublicKey)) + stats.Counter.Watch, _ = db.EstimateTotal(new(repo_model.Watch)) + stats.Counter.Star, _ = db.EstimateTotal(new(repo_model.Star)) + stats.Counter.Action, _ = db.EstimateTotal(new(Action)) + stats.Counter.Access, _ = db.EstimateTotal(new(Access)) + } else { + stats.Counter.PublicKey, _ = e.Count(new(asymkey_model.PublicKey)) + stats.Counter.Watch, _ = e.Count(new(repo_model.Watch)) + stats.Counter.Star, _ = e.Count(new(repo_model.Star)) + stats.Counter.Action, _ = e.Count(new(Action)) + stats.Counter.Access, _ = e.Count(new(Access)) + } type IssueCount struct { Count int64 IsClosed bool } - if setting.Metrics.EnabledIssueByLabel { + if metrics && setting.Metrics.EnabledIssueByLabel { stats.Counter.IssueByLabel = []IssueByLabelCount{} _ = e.Select("COUNT(*) AS count, l.name AS label"). @@ -73,7 +84,7 @@ func GetStatistic() (stats Statistic) { Find(&stats.Counter.IssueByLabel) } - if setting.Metrics.EnabledIssueByRepository { + if metrics && setting.Metrics.EnabledIssueByRepository { stats.Counter.IssueByRepository = []IssueByRepositoryCount{} _ = e.Select("COUNT(*) AS count, r.owner_name, r.name AS repository"). @@ -110,5 +121,6 @@ func GetStatistic() (stats Statistic) { stats.Counter.Attachment, _ = e.Count(new(repo_model.Attachment)) stats.Counter.Project, _ = e.Count(new(project_model.Project)) stats.Counter.ProjectBoard, _ = e.Count(new(project_model.Board)) + stats.Time = time.Now() return } diff --git a/modules/metrics/collector.go b/modules/metrics/collector.go index 069633a565a20..4529550cef4c9 100755 --- a/modules/metrics/collector.go +++ b/modules/metrics/collector.go @@ -5,7 +5,7 @@ package metrics import ( - "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/setting" "github.com/prometheus/client_golang/prometheus" ) @@ -43,6 +43,8 @@ type Collector struct { Users *prometheus.Desc Watches *prometheus.Desc Webhooks *prometheus.Desc + + StatisticsTime *prometheus.Desc } // NewCollector returns a new Collector with all prometheus.Desc initialized @@ -225,152 +227,152 @@ func (c Collector) Describe(ch chan<- *prometheus.Desc) { // Collect returns the metrics with values func (c Collector) Collect(ch chan<- prometheus.Metric) { - stats := models.GetStatistic() + stats := GetStatistic(setting.Metrics.EstimateCounts, setting.Metrics.StatisticTTL) - ch <- prometheus.MustNewConstMetric( + ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric( c.Accesses, prometheus.GaugeValue, float64(stats.Counter.Access), - ) - ch <- prometheus.MustNewConstMetric( + )) + ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric( c.Actions, prometheus.GaugeValue, float64(stats.Counter.Action), - ) - ch <- prometheus.MustNewConstMetric( + )) + ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric( c.Attachments, prometheus.GaugeValue, float64(stats.Counter.Attachment), - ) - ch <- prometheus.MustNewConstMetric( + )) + ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric( c.Comments, prometheus.GaugeValue, float64(stats.Counter.Comment), - ) - ch <- prometheus.MustNewConstMetric( + )) + ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric( c.Follows, prometheus.GaugeValue, float64(stats.Counter.Follow), - ) - ch <- prometheus.MustNewConstMetric( + )) + ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric( c.HookTasks, prometheus.GaugeValue, float64(stats.Counter.HookTask), - ) - ch <- prometheus.MustNewConstMetric( + )) + ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric( c.Issues, prometheus.GaugeValue, float64(stats.Counter.Issue), - ) + )) for _, il := range stats.Counter.IssueByLabel { - ch <- prometheus.MustNewConstMetric( + ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric( c.IssuesByLabel, prometheus.GaugeValue, float64(il.Count), il.Label, - ) + )) } for _, ir := range stats.Counter.IssueByRepository { - ch <- prometheus.MustNewConstMetric( + ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric( c.IssuesByRepository, prometheus.GaugeValue, float64(ir.Count), ir.OwnerName+"/"+ir.Repository, - ) + )) } - ch <- prometheus.MustNewConstMetric( + ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric( c.IssuesClosed, prometheus.GaugeValue, float64(stats.Counter.IssueClosed), - ) - ch <- prometheus.MustNewConstMetric( + )) + ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric( c.IssuesOpen, prometheus.GaugeValue, float64(stats.Counter.IssueOpen), - ) - ch <- prometheus.MustNewConstMetric( + )) + ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric( c.Labels, prometheus.GaugeValue, float64(stats.Counter.Label), - ) - ch <- prometheus.MustNewConstMetric( + )) + ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric( c.LoginSources, prometheus.GaugeValue, float64(stats.Counter.AuthSource), - ) - ch <- prometheus.MustNewConstMetric( + )) + ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric( c.Milestones, prometheus.GaugeValue, float64(stats.Counter.Milestone), - ) - ch <- prometheus.MustNewConstMetric( + )) + ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric( c.Mirrors, prometheus.GaugeValue, float64(stats.Counter.Mirror), - ) - ch <- prometheus.MustNewConstMetric( + )) + ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric( c.Oauths, prometheus.GaugeValue, float64(stats.Counter.Oauth), - ) - ch <- prometheus.MustNewConstMetric( + )) + ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric( c.Organizations, prometheus.GaugeValue, float64(stats.Counter.Org), - ) - ch <- prometheus.MustNewConstMetric( + )) + ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric( c.Projects, prometheus.GaugeValue, float64(stats.Counter.Project), - ) - ch <- prometheus.MustNewConstMetric( + )) + ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric( c.ProjectBoards, prometheus.GaugeValue, float64(stats.Counter.ProjectBoard), - ) - ch <- prometheus.MustNewConstMetric( + )) + ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric( c.PublicKeys, prometheus.GaugeValue, float64(stats.Counter.PublicKey), - ) - ch <- prometheus.MustNewConstMetric( + )) + ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric( c.Releases, prometheus.GaugeValue, float64(stats.Counter.Release), - ) - ch <- prometheus.MustNewConstMetric( + )) + ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric( c.Repositories, prometheus.GaugeValue, float64(stats.Counter.Repo), - ) - ch <- prometheus.MustNewConstMetric( + )) + ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric( c.Stars, prometheus.GaugeValue, float64(stats.Counter.Star), - ) - ch <- prometheus.MustNewConstMetric( + )) + ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric( c.Teams, prometheus.GaugeValue, float64(stats.Counter.Team), - ) - ch <- prometheus.MustNewConstMetric( + )) + ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric( c.UpdateTasks, prometheus.GaugeValue, float64(stats.Counter.UpdateTask), - ) - ch <- prometheus.MustNewConstMetric( + )) + ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric( c.Users, prometheus.GaugeValue, float64(stats.Counter.User), - ) - ch <- prometheus.MustNewConstMetric( + )) + ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric( c.Watches, prometheus.GaugeValue, float64(stats.Counter.Watch), - ) - ch <- prometheus.MustNewConstMetric( + )) + ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric( c.Webhooks, prometheus.GaugeValue, float64(stats.Counter.Webhook), - ) + )) } diff --git a/modules/metrics/statistics.go b/modules/metrics/statistics.go new file mode 100644 index 0000000000000..90dd40cee412d --- /dev/null +++ b/modules/metrics/statistics.go @@ -0,0 +1,38 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package metrics + +import ( + "strconv" + "sync" + "time" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/cache" + "code.gitea.io/gitea/modules/setting" +) + +var statisticsLock sync.Mutex + +func GetStatistic(estimate bool, statisticsTTL time.Duration, metrics bool) models.Statistic { + if statisticsTTL > 0 { + c := cache.GetCache() + if c != nil { + statisticsLock.Lock() + defer statisticsLock.Unlock() + cacheKey := "models/statistic.Statistic." + strconv.FormatBool(estimate) + strconv.FormatBool(metrics) + + if stats, ok := c.Get(cacheKey).(*models.Statistic); ok { + return *stats + } + + stats := models.GetStatistic(estimate, metrics) + c.Put(cacheKey, &stats, setting.DurationToCacheTTL(statisticsTTL)) + return stats + } + } + + return models.GetStatistic(estimate, metrics) +} diff --git a/modules/setting/cache.go b/modules/setting/cache.go index 9a44965124446..3e96200f58f78 100644 --- a/modules/setting/cache.go +++ b/modules/setting/cache.go @@ -94,16 +94,18 @@ func newCacheService() { // TTLSeconds returns the TTLSeconds or unix timestamp for memcache func (c Cache) TTLSeconds() int64 { - if c.Adapter == "memcache" && c.TTL > MemcacheMaxTTL { - return time.Now().Add(c.TTL).Unix() - } - return int64(c.TTL.Seconds()) + return DurationToCacheTTL(c.TTL) } // LastCommitCacheTTLSeconds returns the TTLSeconds or unix timestamp for memcache func LastCommitCacheTTLSeconds() int64 { - if CacheService.Adapter == "memcache" && CacheService.LastCommit.TTL > MemcacheMaxTTL { - return time.Now().Add(CacheService.LastCommit.TTL).Unix() + return DurationToCacheTTL(CacheService.LastCommit.TTL) +} + +// DurationToCacheTTL converts a time.Duration to a TTL +func DurationToCacheTTL(duration time.Duration) int64 { + if CacheService.Adapter == "memcache" && duration > MemcacheMaxTTL { + return time.Now().Add(duration).Unix() } - return int64(CacheService.LastCommit.TTL.Seconds()) + return int64(duration.Seconds()) } diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 5e317b39ea289..5d26f4589c812 100644 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -249,6 +249,8 @@ var ( RepoPagingNum int NoticePagingNum int OrgPagingNum int + EstimateCounts bool + StatisticTTL time.Duration `ini:"STATISTICS_TTL"` } `ini:"ui.admin"` User struct { RepoPagingNum int @@ -302,11 +304,14 @@ var ( RepoPagingNum int NoticePagingNum int OrgPagingNum int + EstimateCounts bool + StatisticTTL time.Duration `ini:"STATISTICS_TTL"` }{ UserPagingNum: 50, RepoPagingNum: 50, NoticePagingNum: 25, OrgPagingNum: 50, + StatisticTTL: 5 * time.Minute, }, User: struct { RepoPagingNum int @@ -411,11 +416,14 @@ var ( Token string EnabledIssueByLabel bool EnabledIssueByRepository bool + EstimateCounts bool + StatisticTTL time.Duration `ini:"STATISTICS_TTL"` }{ Enabled: false, Token: "", EnabledIssueByLabel: false, EnabledIssueByRepository: false, + StatisticTTL: 5 * time.Minute, } // I18n settings diff --git a/routers/web/admin/admin.go b/routers/web/admin/admin.go index d4093f2049ac4..46a63550aceb8 100644 --- a/routers/web/admin/admin.go +++ b/routers/web/admin/admin.go @@ -15,12 +15,12 @@ import ( "strings" "time" - "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/json" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/metrics" "code.gitea.io/gitea/modules/process" "code.gitea.io/gitea/modules/queue" "code.gitea.io/gitea/modules/setting" @@ -126,7 +126,7 @@ func Dashboard(ctx *context.Context) { ctx.Data["Title"] = ctx.Tr("admin.dashboard") ctx.Data["PageIsAdmin"] = true ctx.Data["PageIsAdminDashboard"] = true - ctx.Data["Stats"] = models.GetStatistic() + ctx.Data["Stats"] = metrics.GetStatistic(setting.UI.Admin.EstimateCounts, setting.UI.Admin.StatisticTTL) ctx.Data["NeedUpdate"] = updatechecker.GetNeedUpdate() ctx.Data["RemoteVersion"] = updatechecker.GetRemoteVersion() // FIXME: update periodically @@ -142,7 +142,7 @@ func DashboardPost(ctx *context.Context) { ctx.Data["Title"] = ctx.Tr("admin.dashboard") ctx.Data["PageIsAdmin"] = true ctx.Data["PageIsAdminDashboard"] = true - ctx.Data["Stats"] = models.GetStatistic() + ctx.Data["Stats"] = metrics.GetStatistic(setting.UI.Admin.EstimateCounts, setting.UI.Admin.StatisticTTL) updateSystemStatus() ctx.Data["SysStatus"] = sysStatus diff --git a/templates/admin/dashboard.tmpl b/templates/admin/dashboard.tmpl index 5a4b82f1b963e..74f77a36a4d17 100644 --- a/templates/admin/dashboard.tmpl +++ b/templates/admin/dashboard.tmpl @@ -14,6 +14,7 @@

{{.i18n.Tr "admin.dashboard.statistic_info" .Stats.Counter.User .Stats.Counter.Org .Stats.Counter.PublicKey .Stats.Counter.Repo .Stats.Counter.Watch .Stats.Counter.Star .Stats.Counter.Action .Stats.Counter.Access .Stats.Counter.Issue .Stats.Counter.Comment .Stats.Counter.Oauth .Stats.Counter.Follow .Stats.Counter.Mirror .Stats.Counter.Release .Stats.Counter.AuthSource .Stats.Counter.Webhook .Stats.Counter.Milestone .Stats.Counter.Label .Stats.Counter.HookTask .Stats.Counter.Team .Stats.Counter.UpdateTask .Stats.Counter.Attachment | Str2html}} + {{TimeSince .Stats.Time $.i18n.Lang}}

From 7b3974f39deff77c7ced7fd30d04c8f8257d0ddc Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sun, 24 Apr 2022 14:58:33 +0100 Subject: [PATCH 2/9] cache results locally instead of using the cache service and ensure that statistics are only calculated one at a time. Signed-off-by: Andrew Thornton --- modules/metrics/collector.go | 2 +- modules/metrics/statistics.go | 75 ++++++++++++++++++++++++++------- options/locale/locale_en-US.ini | 4 +- routers/web/admin/admin.go | 18 +++++++- templates/admin/dashboard.tmpl | 12 ++++-- 5 files changed, 87 insertions(+), 24 deletions(-) diff --git a/modules/metrics/collector.go b/modules/metrics/collector.go index 4529550cef4c9..53c2665a47ec0 100755 --- a/modules/metrics/collector.go +++ b/modules/metrics/collector.go @@ -227,7 +227,7 @@ func (c Collector) Describe(ch chan<- *prometheus.Desc) { // Collect returns the metrics with values func (c Collector) Collect(ch chan<- prometheus.Metric) { - stats := GetStatistic(setting.Metrics.EstimateCounts, setting.Metrics.StatisticTTL) + stats := <-GetStatistic(setting.Metrics.EstimateCounts, setting.Metrics.StatisticTTL, true) ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric( c.Accesses, diff --git a/modules/metrics/statistics.go b/modules/metrics/statistics.go index 90dd40cee412d..1c16002c34d44 100644 --- a/modules/metrics/statistics.go +++ b/modules/metrics/statistics.go @@ -10,29 +10,72 @@ import ( "time" "code.gitea.io/gitea/models" - "code.gitea.io/gitea/modules/cache" - "code.gitea.io/gitea/modules/setting" ) -var statisticsLock sync.Mutex +var ( + statisticsLock sync.Mutex + statisticsMap = map[string]*models.Statistic{} + statisticsWorkingChan = map[string]chan struct{}{} +) + +func GetStatistic(estimate bool, statisticsTTL time.Duration, metrics bool) <-chan *models.Statistic { + cacheKey := "models/statistic.Statistic." + strconv.FormatBool(estimate) + strconv.FormatBool(metrics) -func GetStatistic(estimate bool, statisticsTTL time.Duration, metrics bool) models.Statistic { + statisticsLock.Lock() // CAREFUL: no defer! + ourChan := make(chan *models.Statistic, 1) + + // Check for a cached statistic if statisticsTTL > 0 { - c := cache.GetCache() - if c != nil { - statisticsLock.Lock() - defer statisticsLock.Unlock() - cacheKey := "models/statistic.Statistic." + strconv.FormatBool(estimate) + strconv.FormatBool(metrics) + if stats, ok := statisticsMap[cacheKey]; ok && stats.Time.Add(statisticsTTL).After(time.Now()) { + // Found a valid cached statistic for these params, so unlock and send this down the channel + statisticsLock.Unlock() // Unlock from line 24 + + ourChan <- stats + close(ourChan) + return ourChan + } + } + + // We need to calculate a statistic - however, we should only do this one at a time (NOTE: we are still within the lock) + // + // So check if we have a worker already and get a marker channel + workingChan, ok := statisticsWorkingChan[cacheKey] - if stats, ok := c.Get(cacheKey).(*models.Statistic); ok { - return *stats - } + if !ok { + // we need to make our own worker... (NOTE: we are still within the lock) + // create a marker channel which will be closed when our worker is finished + // and assign it to the working map. + workingChan = make(chan struct{}) + statisticsWorkingChan[cacheKey] = workingChan + + // Create the working go-routine + go func() { stats := models.GetStatistic(estimate, metrics) - c.Put(cacheKey, &stats, setting.DurationToCacheTTL(statisticsTTL)) - return stats - } + + // cache the result, remove this worker and inform anyone waiting we are done + statisticsLock.Lock() // Lock within goroutine + statisticsMap[cacheKey] = &stats + delete(statisticsWorkingChan, cacheKey) + close(workingChan) + statisticsLock.Unlock() // Unlock within goroutine + }() } - return models.GetStatistic(estimate, metrics) + statisticsLock.Unlock() // Unlock from line 24 + + // Create our goroutine for the channel waiting for the statistics to be generated + go func() { + <-workingChan // Wait for the worker to finish + + // Now lock and get the last stats completed + statisticsLock.Lock() + stats := statisticsMap[cacheKey] + statisticsLock.Unlock() + + ourChan <- stats + close(ourChan) + }() + + return ourChan } diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 21bf0c49ea198..50c4adac6b2fa 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -2397,6 +2397,8 @@ dashboard.statistic = Summary dashboard.operations = Maintenance Operations dashboard.system_status = System Status dashboard.statistic_info = The Gitea database holds %d users, %d organizations, %d public keys, %d repositories, %d watches, %d stars, %d actions, %d accesses, %d issues, %d comments, %d social accounts, %d follows, %d mirrors, %d releases, %d authentication sources, %d webhooks, %d milestones, %d labels, %d hook tasks, %d teams, %d update tasks, %d attachments. +dashboard.statistic_info_last_updated = Last updated %s +dashboard.statistic_info_in_progress = Statistics are being calculated dashboard.operation_name = Operation Name dashboard.operation_switch = Switch dashboard.operation_run = Run @@ -3086,7 +3088,7 @@ settings.link = Link this package to a repository settings.link.description = If you link a package with a repository, the package is listed in the repository's package list. settings.link.select = Select Repository settings.link.button = Update Repository Link -settings.link.success = Repository link was successfully updated. +settings.link.success = Repository link was successfully updated. settings.link.error = Failed to update repository link. settings.delete = Delete package settings.delete.description = Deleting a package is permanent and cannot be undone. diff --git a/routers/web/admin/admin.go b/routers/web/admin/admin.go index 46a63550aceb8..1baa54e83ffa8 100644 --- a/routers/web/admin/admin.go +++ b/routers/web/admin/admin.go @@ -15,6 +15,7 @@ import ( "strings" "time" + "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/git" @@ -121,12 +122,25 @@ func updateSystemStatus() { sysStatus.NumGC = m.NumGC } +func getStatistics() *models.Statistic { + if setting.UI.Admin.StatisticTTL > 0 { + select { + case stats := <-metrics.GetStatistic(setting.UI.Admin.EstimateCounts, setting.UI.Admin.StatisticTTL, false): + return stats + case <-time.After(1 * time.Second): + return nil + } + } + + return <-metrics.GetStatistic(setting.UI.Admin.EstimateCounts, setting.UI.Admin.StatisticTTL, false) +} + // Dashboard show admin panel dashboard func Dashboard(ctx *context.Context) { ctx.Data["Title"] = ctx.Tr("admin.dashboard") ctx.Data["PageIsAdmin"] = true ctx.Data["PageIsAdminDashboard"] = true - ctx.Data["Stats"] = metrics.GetStatistic(setting.UI.Admin.EstimateCounts, setting.UI.Admin.StatisticTTL) + ctx.Data["Stats"] = getStatistics() ctx.Data["NeedUpdate"] = updatechecker.GetNeedUpdate() ctx.Data["RemoteVersion"] = updatechecker.GetRemoteVersion() // FIXME: update periodically @@ -142,7 +156,7 @@ func DashboardPost(ctx *context.Context) { ctx.Data["Title"] = ctx.Tr("admin.dashboard") ctx.Data["PageIsAdmin"] = true ctx.Data["PageIsAdminDashboard"] = true - ctx.Data["Stats"] = metrics.GetStatistic(setting.UI.Admin.EstimateCounts, setting.UI.Admin.StatisticTTL) + ctx.Data["Stats"] = getStatistics() updateSystemStatus() ctx.Data["SysStatus"] = sysStatus diff --git a/templates/admin/dashboard.tmpl b/templates/admin/dashboard.tmpl index 74f77a36a4d17..0ec9ea110950d 100644 --- a/templates/admin/dashboard.tmpl +++ b/templates/admin/dashboard.tmpl @@ -12,10 +12,14 @@ {{.i18n.Tr "admin.dashboard.statistic"}}

-

- {{.i18n.Tr "admin.dashboard.statistic_info" .Stats.Counter.User .Stats.Counter.Org .Stats.Counter.PublicKey .Stats.Counter.Repo .Stats.Counter.Watch .Stats.Counter.Star .Stats.Counter.Action .Stats.Counter.Access .Stats.Counter.Issue .Stats.Counter.Comment .Stats.Counter.Oauth .Stats.Counter.Follow .Stats.Counter.Mirror .Stats.Counter.Release .Stats.Counter.AuthSource .Stats.Counter.Webhook .Stats.Counter.Milestone .Stats.Counter.Label .Stats.Counter.HookTask .Stats.Counter.Team .Stats.Counter.UpdateTask .Stats.Counter.Attachment | Str2html}} - {{TimeSince .Stats.Time $.i18n.Lang}} -

+ {{if .Stats}} +

+ {{.i18n.Tr "admin.dashboard.statistic_info" .Stats.Counter.User .Stats.Counter.Org .Stats.Counter.PublicKey .Stats.Counter.Repo .Stats.Counter.Watch .Stats.Counter.Star .Stats.Counter.Action .Stats.Counter.Access .Stats.Counter.Issue .Stats.Counter.Comment .Stats.Counter.Oauth .Stats.Counter.Follow .Stats.Counter.Mirror .Stats.Counter.Release .Stats.Counter.AuthSource .Stats.Counter.Webhook .Stats.Counter.Milestone .Stats.Counter.Label .Stats.Counter.HookTask .Stats.Counter.Team .Stats.Counter.UpdateTask .Stats.Counter.Attachment | Str2html}} + {{.i18n.Tr "admin.dashboard.statistic_info_last_updated" (TimeSince .Stats.Time $.i18n.Lang) | Str2html}} +

+ {{else}} +

{{.i18n.Tr "admin.dashboard.statistic_info_in_progress"}}

+ {{end}}

{{.i18n.Tr "admin.dashboard.operations"}} From d487e08a5291320e3ea40114a7e25bab8e6eaef2 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sun, 24 Apr 2022 15:26:03 +0100 Subject: [PATCH 3/9] more estimates Signed-off-by: Andrew Thornton --- models/statistic.go | 39 +++++++++++++++++++++++++++------------ 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/models/statistic.go b/models/statistic.go index 6aeef790be0c7..9b29b808b697b 100644 --- a/models/statistic.go +++ b/models/statistic.go @@ -107,20 +107,35 @@ func GetStatistic(estimate, metrics bool) (stats Statistic) { stats.Counter.Issue = stats.Counter.IssueClosed + stats.Counter.IssueOpen - stats.Counter.Comment, _ = e.Count(new(Comment)) + if estimate { + stats.Counter.Comment, _ = db.EstimateTotal(new(Comment)) + stats.Counter.Follow, _ = db.EstimateTotal(new(user_model.Follow)) + stats.Counter.Mirror, _ = db.EstimateTotal(new(repo_model.Mirror)) + stats.Counter.Release, _ = db.EstimateTotal(new(Release)) + stats.Counter.Webhook, _ = db.EstimateTotal(new(webhook.Webhook)) + stats.Counter.Milestone, _ = db.EstimateTotal(new(issues_model.Milestone)) + stats.Counter.Label, _ = db.EstimateTotal(new(Label)) + stats.Counter.HookTask, _ = db.EstimateTotal(new(webhook.HookTask)) + stats.Counter.Team, _ = db.EstimateTotal(new(organization.Team)) + stats.Counter.Attachment, _ = db.EstimateTotal(new(repo_model.Attachment)) + stats.Counter.Project, _ = db.EstimateTotal(new(project_model.Project)) + stats.Counter.ProjectBoard, _ = db.EstimateTotal(new(project_model.Board)) + } else { + stats.Counter.Comment, _ = e.Count(new(Comment)) + stats.Counter.Follow, _ = e.Count(new(user_model.Follow)) + stats.Counter.Mirror, _ = e.Count(new(repo_model.Mirror)) + stats.Counter.Release, _ = e.Count(new(Release)) + stats.Counter.Webhook, _ = e.Count(new(webhook.Webhook)) + stats.Counter.Milestone, _ = e.Count(new(issues_model.Milestone)) + stats.Counter.Label, _ = e.Count(new(Label)) + stats.Counter.HookTask, _ = e.Count(new(webhook.HookTask)) + stats.Counter.Team, _ = e.Count(new(organization.Team)) + stats.Counter.Attachment, _ = e.Count(new(repo_model.Attachment)) + stats.Counter.Project, _ = e.Count(new(project_model.Project)) + stats.Counter.ProjectBoard, _ = e.Count(new(project_model.Board)) + } stats.Counter.Oauth = 0 - stats.Counter.Follow, _ = e.Count(new(user_model.Follow)) - stats.Counter.Mirror, _ = e.Count(new(repo_model.Mirror)) - stats.Counter.Release, _ = e.Count(new(Release)) stats.Counter.AuthSource = auth.CountSources() - stats.Counter.Webhook, _ = e.Count(new(webhook.Webhook)) - stats.Counter.Milestone, _ = e.Count(new(issues_model.Milestone)) - stats.Counter.Label, _ = e.Count(new(Label)) - stats.Counter.HookTask, _ = e.Count(new(webhook.HookTask)) - stats.Counter.Team, _ = e.Count(new(organization.Team)) - stats.Counter.Attachment, _ = e.Count(new(repo_model.Attachment)) - stats.Counter.Project, _ = e.Count(new(project_model.Project)) - stats.Counter.ProjectBoard, _ = e.Count(new(project_model.Board)) stats.Time = time.Now() return } From a9026aa79e63fb1c88721b40e0531db7b09c03db Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Mon, 25 Apr 2022 20:01:00 +0100 Subject: [PATCH 4/9] placate lint Signed-off-by: Andrew Thornton --- models/db/context.go | 17 +++++---- models/statistic.go | 71 ++++++++++++++--------------------- modules/metrics/collector.go | 5 +++ modules/metrics/statistics.go | 18 ++++++++- 4 files changed, 60 insertions(+), 51 deletions(-) diff --git a/models/db/context.go b/models/db/context.go index d918a6af76a4c..938a8b1347bff 100644 --- a/models/db/context.go +++ b/models/db/context.go @@ -170,23 +170,26 @@ func TableName(bean interface{}) string { return x.TableName(bean) } -// EstimateTotal returns an estimate of total number of rows in table -func EstimateTotal(bean interface{}) (int64, error) { - tablename := x.TableName(bean) +// EstimateCount returns an estimate of total number of rows in table +func EstimateCount(ctx context.Context, bean interface{}) (int64, error) { + e := GetEngine(ctx) + e.Context(ctx) + + tablename := TableName(bean) switch x.Dialect().URI().DBType { case schemas.MYSQL: var rows int64 - _, err := x.SQL("SELECT table_rows FROM information_schema.tables WHERE tables.table_name = ? AND tables.table_schema = ?;", tablename, x.Dialect().URI().DBName).Get(&rows) + _, err := e.Context(ctx).SQL("SELECT table_rows FROM information_schema.tables WHERE tables.table_name = ? AND tables.table_schema = ?;", tablename, x.Dialect().URI().DBName).Get(&rows) return rows, err case schemas.POSTGRES: var rows int64 - _, err := x.SQL("SELECT reltuples AS estimate FROM pg_class WHERE relname = ?;", tablename).Get(&rows) + _, err := e.Context(ctx).SQL("SELECT reltuples AS estimate FROM pg_class WHERE relname = ?;", tablename).Get(&rows) return rows, err case schemas.MSSQL: var rows int64 - _, err := x.SQL("sp_spaceused ?;", tablename).Get(&rows) + _, err := e.Context(ctx).SQL("sp_spaceused ?;", tablename).Get(&rows) return rows, err default: - return x.Count(tablename) + return e.Context(ctx).Count(tablename) } } diff --git a/models/statistic.go b/models/statistic.go index 9b29b808b697b..d2fa2cd027c9c 100644 --- a/models/statistic.go +++ b/models/statistic.go @@ -5,6 +5,7 @@ package models import ( + "context" "time" asymkey_model "code.gitea.io/gitea/models/asymkey" @@ -50,24 +51,25 @@ type IssueByRepositoryCount struct { } // GetStatistic returns the database statistics -func GetStatistic(estimate, metrics bool) (stats Statistic) { - e := db.GetEngine(db.DefaultContext) +func GetStatistic(ctx context.Context, estimate, metrics bool) (stats Statistic) { + e := db.GetEngine(ctx) + countFn := func(bean interface{}) (int64, error) { + return e.Context(ctx).Count(bean) + } + if estimate { + countFn = func(bean interface{}) (int64, error) { + return db.EstimateCount(ctx, bean) + } + } + stats.Counter.User = user_model.CountUsers() stats.Counter.Org = organization.CountOrganizations() stats.Counter.Repo = repo_model.CountRepositories(true) - if estimate { - stats.Counter.PublicKey, _ = db.EstimateTotal(new(asymkey_model.PublicKey)) - stats.Counter.Watch, _ = db.EstimateTotal(new(repo_model.Watch)) - stats.Counter.Star, _ = db.EstimateTotal(new(repo_model.Star)) - stats.Counter.Action, _ = db.EstimateTotal(new(Action)) - stats.Counter.Access, _ = db.EstimateTotal(new(Access)) - } else { - stats.Counter.PublicKey, _ = e.Count(new(asymkey_model.PublicKey)) - stats.Counter.Watch, _ = e.Count(new(repo_model.Watch)) - stats.Counter.Star, _ = e.Count(new(repo_model.Star)) - stats.Counter.Action, _ = e.Count(new(Action)) - stats.Counter.Access, _ = e.Count(new(Access)) - } + stats.Counter.PublicKey, _ = countFn(new(asymkey_model.PublicKey)) + stats.Counter.Watch, _ = countFn(new(repo_model.Watch)) + stats.Counter.Star, _ = countFn(new(repo_model.Star)) + stats.Counter.Action, _ = countFn(new(Action)) + stats.Counter.Access, _ = countFn(new(Access)) type IssueCount struct { Count int64 @@ -107,33 +109,18 @@ func GetStatistic(estimate, metrics bool) (stats Statistic) { stats.Counter.Issue = stats.Counter.IssueClosed + stats.Counter.IssueOpen - if estimate { - stats.Counter.Comment, _ = db.EstimateTotal(new(Comment)) - stats.Counter.Follow, _ = db.EstimateTotal(new(user_model.Follow)) - stats.Counter.Mirror, _ = db.EstimateTotal(new(repo_model.Mirror)) - stats.Counter.Release, _ = db.EstimateTotal(new(Release)) - stats.Counter.Webhook, _ = db.EstimateTotal(new(webhook.Webhook)) - stats.Counter.Milestone, _ = db.EstimateTotal(new(issues_model.Milestone)) - stats.Counter.Label, _ = db.EstimateTotal(new(Label)) - stats.Counter.HookTask, _ = db.EstimateTotal(new(webhook.HookTask)) - stats.Counter.Team, _ = db.EstimateTotal(new(organization.Team)) - stats.Counter.Attachment, _ = db.EstimateTotal(new(repo_model.Attachment)) - stats.Counter.Project, _ = db.EstimateTotal(new(project_model.Project)) - stats.Counter.ProjectBoard, _ = db.EstimateTotal(new(project_model.Board)) - } else { - stats.Counter.Comment, _ = e.Count(new(Comment)) - stats.Counter.Follow, _ = e.Count(new(user_model.Follow)) - stats.Counter.Mirror, _ = e.Count(new(repo_model.Mirror)) - stats.Counter.Release, _ = e.Count(new(Release)) - stats.Counter.Webhook, _ = e.Count(new(webhook.Webhook)) - stats.Counter.Milestone, _ = e.Count(new(issues_model.Milestone)) - stats.Counter.Label, _ = e.Count(new(Label)) - stats.Counter.HookTask, _ = e.Count(new(webhook.HookTask)) - stats.Counter.Team, _ = e.Count(new(organization.Team)) - stats.Counter.Attachment, _ = e.Count(new(repo_model.Attachment)) - stats.Counter.Project, _ = e.Count(new(project_model.Project)) - stats.Counter.ProjectBoard, _ = e.Count(new(project_model.Board)) - } + stats.Counter.Comment, _ = countFn(new(Comment)) + stats.Counter.Follow, _ = countFn(new(user_model.Follow)) + stats.Counter.Mirror, _ = countFn(new(repo_model.Mirror)) + stats.Counter.Release, _ = countFn(new(Release)) + stats.Counter.Webhook, _ = countFn(new(webhook.Webhook)) + stats.Counter.Milestone, _ = countFn(new(issues_model.Milestone)) + stats.Counter.Label, _ = countFn(new(Label)) + stats.Counter.HookTask, _ = countFn(new(webhook.HookTask)) + stats.Counter.Team, _ = countFn(new(organization.Team)) + stats.Counter.Attachment, _ = countFn(new(repo_model.Attachment)) + stats.Counter.Project, _ = countFn(new(project_model.Project)) + stats.Counter.ProjectBoard, _ = countFn(new(project_model.Board)) stats.Counter.Oauth = 0 stats.Counter.AuthSource = auth.CountSources() stats.Time = time.Now() diff --git a/modules/metrics/collector.go b/modules/metrics/collector.go index 53c2665a47ec0..32cc86ff1f551 100755 --- a/modules/metrics/collector.go +++ b/modules/metrics/collector.go @@ -5,6 +5,7 @@ package metrics import ( + "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/setting" "github.com/prometheus/client_golang/prometheus" @@ -228,6 +229,10 @@ func (c Collector) Describe(ch chan<- *prometheus.Desc) { // Collect returns the metrics with values func (c Collector) Collect(ch chan<- prometheus.Metric) { stats := <-GetStatistic(setting.Metrics.EstimateCounts, setting.Metrics.StatisticTTL, true) + if stats == nil { + // This will happen if the statistics generation was cancelled midway through + stats = &models.Statistic{} + } ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric( c.Accesses, diff --git a/modules/metrics/statistics.go b/modules/metrics/statistics.go index 1c16002c34d44..6d3695648bf2f 100644 --- a/modules/metrics/statistics.go +++ b/modules/metrics/statistics.go @@ -5,11 +5,14 @@ package metrics import ( + "fmt" "strconv" "sync" "time" "code.gitea.io/gitea/models" + "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/modules/process" ) var ( @@ -51,11 +54,22 @@ func GetStatistic(estimate bool, statisticsTTL time.Duration, metrics bool) <-ch // Create the working go-routine go func() { - stats := models.GetStatistic(estimate, metrics) + ctx, _, finished := process.GetManager().AddContext(db.DefaultContext, fmt.Sprintf("Statistics: Estimated: %t Metrics: %t", estimate, metrics)) + defer finished() + stats := models.GetStatistic(ctx, estimate, metrics) + statsPtr := &stats + select { + case <-ctx.Done(): + // The above stats likely have been cancelled part way through generation and should be ignored + statsPtr = nil + default: + } // cache the result, remove this worker and inform anyone waiting we are done statisticsLock.Lock() // Lock within goroutine - statisticsMap[cacheKey] = &stats + if statsPtr != nil { + statisticsMap[cacheKey] = statsPtr + } delete(statisticsWorkingChan, cacheKey) close(workingChan) statisticsLock.Unlock() // Unlock within goroutine From f3e881d1a39262516fa04d038fe0945eeaf8b9cc Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Fri, 6 May 2022 20:03:15 +0100 Subject: [PATCH 5/9] Only provide estimates Signed-off-by: Andrew Thornton --- .../doc/advanced/config-cheat-sheet.en-us.md | 2 - models/statistic.go | 46 +++++++-------- modules/metrics/collector.go | 2 +- modules/metrics/statistics.go | 56 +++++++++++++------ modules/setting/setting.go | 3 - routers/web/admin/admin.go | 4 +- 6 files changed, 62 insertions(+), 51 deletions(-) diff --git a/docs/content/doc/advanced/config-cheat-sheet.en-us.md b/docs/content/doc/advanced/config-cheat-sheet.en-us.md index ac2d978c77113..04ee245670433 100644 --- a/docs/content/doc/advanced/config-cheat-sheet.en-us.md +++ b/docs/content/doc/advanced/config-cheat-sheet.en-us.md @@ -197,7 +197,6 @@ The following configuration set `Content-Type: application/vnd.android.package-a - `REPO_PAGING_NUM`: **50**: Number of repos that are shown in one page. - `NOTICE_PAGING_NUM`: **25**: Number of notices that are shown in one page. - `ORG_PAGING_NUM`: **50**: Number of organizations that are shown in one page. -- `ESTIMATE_COUNTS`: **false**: Estimate counts for summary statistics instead counting directly. - `STATISTICS_TTL`: **5m**: Cache summary statistics for this period of time. ### UI - Metadata (`ui.meta`) @@ -977,7 +976,6 @@ Default templates for project boards: - `ENABLED_ISSUE_BY_LABEL`: **false**: Enable issue by label metrics with format `gitea_issues_by_label{label="bug"} 2`. - `ENABLED_ISSUE_BY_REPOSITORY`: **false**: Enable issue by repository metrics with format `gitea_issues_by_repository{repository="org/repo"} 5`. - `TOKEN`: **\**: You need to specify the token, if you want to include in the authorization the metrics . The same token need to be used in prometheus parameters `bearer_token` or `bearer_token_file`. -- `ESTIMATE_COUNTS`: **false**: Estimate counts for statistics instead counting directly. - `STATISTICS_TTL`: **5m**: Cache summary statistics for this period of time. ## API (`api`) diff --git a/models/statistic.go b/models/statistic.go index 2f48ad386bd56..b627ca9498dd2 100644 --- a/models/statistic.go +++ b/models/statistic.go @@ -51,25 +51,17 @@ type IssueByRepositoryCount struct { } // GetStatistic returns the database statistics -func GetStatistic(ctx context.Context, estimate, metrics bool) (stats Statistic) { +func GetStatistic(ctx context.Context, metrics bool) (stats Statistic) { e := db.GetEngine(ctx) - countFn := func(bean interface{}) (int64, error) { - return e.Context(ctx).Count(bean) - } - if estimate { - countFn = func(bean interface{}) (int64, error) { - return db.EstimateCount(ctx, bean) - } - } stats.Counter.User = user_model.CountUsers(nil) stats.Counter.Org = organization.CountOrganizations() - stats.Counter.Repo = repo_model.CountRepositories(true) - stats.Counter.PublicKey, _ = countFn(new(asymkey_model.PublicKey)) - stats.Counter.Watch, _ = countFn(new(repo_model.Watch)) - stats.Counter.Star, _ = countFn(new(repo_model.Star)) - stats.Counter.Action, _ = countFn(new(Action)) - stats.Counter.Access, _ = countFn(new(Access)) + stats.Counter.Repo, _ = db.EstimateCount(ctx, new(repo_model.Repository)) + stats.Counter.PublicKey, _ = db.EstimateCount(ctx, new(asymkey_model.PublicKey)) + stats.Counter.Watch, _ = db.EstimateCount(ctx, new(repo_model.Watch)) + stats.Counter.Star, _ = db.EstimateCount(ctx, new(repo_model.Star)) + stats.Counter.Action, _ = db.EstimateCount(ctx, new(Action)) + stats.Counter.Access, _ = db.EstimateCount(ctx, new(Access)) type IssueCount struct { Count int64 @@ -109,18 +101,18 @@ func GetStatistic(ctx context.Context, estimate, metrics bool) (stats Statistic) stats.Counter.Issue = stats.Counter.IssueClosed + stats.Counter.IssueOpen - stats.Counter.Comment, _ = countFn(new(Comment)) - stats.Counter.Follow, _ = countFn(new(user_model.Follow)) - stats.Counter.Mirror, _ = countFn(new(repo_model.Mirror)) - stats.Counter.Release, _ = countFn(new(Release)) - stats.Counter.Webhook, _ = countFn(new(webhook.Webhook)) - stats.Counter.Milestone, _ = countFn(new(issues_model.Milestone)) - stats.Counter.Label, _ = countFn(new(Label)) - stats.Counter.HookTask, _ = countFn(new(webhook.HookTask)) - stats.Counter.Team, _ = countFn(new(organization.Team)) - stats.Counter.Attachment, _ = countFn(new(repo_model.Attachment)) - stats.Counter.Project, _ = countFn(new(project_model.Project)) - stats.Counter.ProjectBoard, _ = countFn(new(project_model.Board)) + stats.Counter.Comment, _ = db.EstimateCount(ctx, new(Comment)) + stats.Counter.Follow, _ = db.EstimateCount(ctx, new(user_model.Follow)) + stats.Counter.Mirror, _ = db.EstimateCount(ctx, new(repo_model.Mirror)) + stats.Counter.Release, _ = db.EstimateCount(ctx, new(Release)) + stats.Counter.Webhook, _ = db.EstimateCount(ctx, new(webhook.Webhook)) + stats.Counter.Milestone, _ = db.EstimateCount(ctx, new(issues_model.Milestone)) + stats.Counter.Label, _ = db.EstimateCount(ctx, new(Label)) + stats.Counter.HookTask, _ = db.EstimateCount(ctx, new(webhook.HookTask)) + stats.Counter.Team, _ = db.EstimateCount(ctx, new(organization.Team)) + stats.Counter.Attachment, _ = db.EstimateCount(ctx, new(repo_model.Attachment)) + stats.Counter.Project, _ = db.EstimateCount(ctx, new(project_model.Project)) + stats.Counter.ProjectBoard, _ = db.EstimateCount(ctx, new(project_model.Board)) stats.Counter.Oauth = 0 stats.Counter.AuthSource = auth.CountSources() stats.Time = time.Now() diff --git a/modules/metrics/collector.go b/modules/metrics/collector.go index 32cc86ff1f551..b5b249f59a63d 100755 --- a/modules/metrics/collector.go +++ b/modules/metrics/collector.go @@ -228,7 +228,7 @@ func (c Collector) Describe(ch chan<- *prometheus.Desc) { // Collect returns the metrics with values func (c Collector) Collect(ch chan<- prometheus.Metric) { - stats := <-GetStatistic(setting.Metrics.EstimateCounts, setting.Metrics.StatisticTTL, true) + stats := <-GetStatistic(setting.Metrics.StatisticTTL, true) if stats == nil { // This will happen if the statistics generation was cancelled midway through stats = &models.Statistic{} diff --git a/modules/metrics/statistics.go b/modules/metrics/statistics.go index 6d3695648bf2f..028c64114c652 100644 --- a/modules/metrics/statistics.go +++ b/modules/metrics/statistics.go @@ -6,7 +6,6 @@ package metrics import ( "fmt" - "strconv" "sync" "time" @@ -16,20 +15,26 @@ import ( ) var ( - statisticsLock sync.Mutex - statisticsMap = map[string]*models.Statistic{} - statisticsWorkingChan = map[string]chan struct{}{} -) + statisticsLock sync.Mutex + + shortStatistic *models.Statistic + fullStatistic *models.Statistic -func GetStatistic(estimate bool, statisticsTTL time.Duration, metrics bool) <-chan *models.Statistic { - cacheKey := "models/statistic.Statistic." + strconv.FormatBool(estimate) + strconv.FormatBool(metrics) + shortStatisticWorkingChan chan struct{} + fullStatisticWorkingChan chan struct{} +) +func GetStatistic(statisticsTTL time.Duration, full bool) <-chan *models.Statistic { statisticsLock.Lock() // CAREFUL: no defer! ourChan := make(chan *models.Statistic, 1) // Check for a cached statistic if statisticsTTL > 0 { - if stats, ok := statisticsMap[cacheKey]; ok && stats.Time.Add(statisticsTTL).After(time.Now()) { + stats := fullStatistic + if !full && shortStatistic != nil { + stats = shortStatistic + } + if stats != nil && stats.Time.Add(statisticsTTL).After(time.Now()) { // Found a valid cached statistic for these params, so unlock and send this down the channel statisticsLock.Unlock() // Unlock from line 24 @@ -42,21 +47,30 @@ func GetStatistic(estimate bool, statisticsTTL time.Duration, metrics bool) <-ch // We need to calculate a statistic - however, we should only do this one at a time (NOTE: we are still within the lock) // // So check if we have a worker already and get a marker channel - workingChan, ok := statisticsWorkingChan[cacheKey] + var workingChan chan struct{} + if full { + workingChan = fullStatisticWorkingChan + } else { + workingChan = shortStatisticWorkingChan + } - if !ok { + if workingChan == nil { // we need to make our own worker... (NOTE: we are still within the lock) // create a marker channel which will be closed when our worker is finished // and assign it to the working map. workingChan = make(chan struct{}) - statisticsWorkingChan[cacheKey] = workingChan + if full { + fullStatisticWorkingChan = workingChan + } else { + shortStatisticWorkingChan = workingChan + } // Create the working go-routine go func() { - ctx, _, finished := process.GetManager().AddContext(db.DefaultContext, fmt.Sprintf("Statistics: Estimated: %t Metrics: %t", estimate, metrics)) + ctx, _, finished := process.GetManager().AddContext(db.DefaultContext, fmt.Sprintf("Statistics: Full: %t", full)) defer finished() - stats := models.GetStatistic(ctx, estimate, metrics) + stats := models.GetStatistic(ctx, full) statsPtr := &stats select { case <-ctx.Done(): @@ -68,9 +82,16 @@ func GetStatistic(estimate bool, statisticsTTL time.Duration, metrics bool) <-ch // cache the result, remove this worker and inform anyone waiting we are done statisticsLock.Lock() // Lock within goroutine if statsPtr != nil { - statisticsMap[cacheKey] = statsPtr + shortStatistic = statsPtr + if full { + fullStatistic = statsPtr + } + } + if full { + fullStatisticWorkingChan = nil + } else { + shortStatisticWorkingChan = nil } - delete(statisticsWorkingChan, cacheKey) close(workingChan) statisticsLock.Unlock() // Unlock within goroutine }() @@ -84,7 +105,10 @@ func GetStatistic(estimate bool, statisticsTTL time.Duration, metrics bool) <-ch // Now lock and get the last stats completed statisticsLock.Lock() - stats := statisticsMap[cacheKey] + stats := fullStatistic + if !full { + stats = shortStatistic + } statisticsLock.Unlock() ourChan <- stats diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 5d26f4589c812..9755e413f5bf4 100644 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -249,7 +249,6 @@ var ( RepoPagingNum int NoticePagingNum int OrgPagingNum int - EstimateCounts bool StatisticTTL time.Duration `ini:"STATISTICS_TTL"` } `ini:"ui.admin"` User struct { @@ -304,7 +303,6 @@ var ( RepoPagingNum int NoticePagingNum int OrgPagingNum int - EstimateCounts bool StatisticTTL time.Duration `ini:"STATISTICS_TTL"` }{ UserPagingNum: 50, @@ -416,7 +414,6 @@ var ( Token string EnabledIssueByLabel bool EnabledIssueByRepository bool - EstimateCounts bool StatisticTTL time.Duration `ini:"STATISTICS_TTL"` }{ Enabled: false, diff --git a/routers/web/admin/admin.go b/routers/web/admin/admin.go index 1baa54e83ffa8..67891e626aaa2 100644 --- a/routers/web/admin/admin.go +++ b/routers/web/admin/admin.go @@ -125,14 +125,14 @@ func updateSystemStatus() { func getStatistics() *models.Statistic { if setting.UI.Admin.StatisticTTL > 0 { select { - case stats := <-metrics.GetStatistic(setting.UI.Admin.EstimateCounts, setting.UI.Admin.StatisticTTL, false): + case stats := <-metrics.GetStatistic(setting.UI.Admin.StatisticTTL, false): return stats case <-time.After(1 * time.Second): return nil } } - return <-metrics.GetStatistic(setting.UI.Admin.EstimateCounts, setting.UI.Admin.StatisticTTL, false) + return <-metrics.GetStatistic(setting.UI.Admin.StatisticTTL, false) } // Dashboard show admin panel dashboard From aa64b2c31855136fe52c9b09cd8fb8e00ebdeb82 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sat, 21 May 2022 13:51:05 +0100 Subject: [PATCH 6/9] fix mismerge Signed-off-by: Andrew Thornton --- models/statistic.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/statistic.go b/models/statistic.go index 8d609f4a0a559..09ce840f4f72c 100644 --- a/models/statistic.go +++ b/models/statistic.go @@ -56,7 +56,7 @@ func GetStatistic(ctx context.Context, metrics bool) (stats Statistic) { e := db.GetEngine(ctx) stats.Counter.User = user_model.CountUsers(nil) - stats.Counter.Org = organization.CountOrganizations(organization.FindOrgOptions{IncludePrivate: true}) + stats.Counter.Org, _ = organization.CountOrgs(organization.FindOrgOptions{IncludePrivate: true}) stats.Counter.Repo, _ = db.EstimateCount(ctx, new(repo_model.Repository)) stats.Counter.PublicKey, _ = db.EstimateCount(ctx, new(asymkey_model.PublicKey)) stats.Counter.Watch, _ = db.EstimateCount(ctx, new(repo_model.Watch)) From da9579dc38fab0df824c16ddc4331c5a85a921de Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sun, 22 May 2022 12:14:05 +0100 Subject: [PATCH 7/9] reduce the number of estimated counts and clearly mark them Signed-off-by: Andrew Thornton --- models/statistic.go | 26 +++++++++++++------------- options/locale/locale_en-US.ini | 2 +- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/models/statistic.go b/models/statistic.go index 09ce840f4f72c..a5cb72896a108 100644 --- a/models/statistic.go +++ b/models/statistic.go @@ -57,10 +57,10 @@ func GetStatistic(ctx context.Context, metrics bool) (stats Statistic) { stats.Counter.User = user_model.CountUsers(nil) stats.Counter.Org, _ = organization.CountOrgs(organization.FindOrgOptions{IncludePrivate: true}) - stats.Counter.Repo, _ = db.EstimateCount(ctx, new(repo_model.Repository)) - stats.Counter.PublicKey, _ = db.EstimateCount(ctx, new(asymkey_model.PublicKey)) - stats.Counter.Watch, _ = db.EstimateCount(ctx, new(repo_model.Watch)) - stats.Counter.Star, _ = db.EstimateCount(ctx, new(repo_model.Star)) + stats.Counter.PublicKey, _ = e.Count(new(asymkey_model.PublicKey)) + stats.Counter.Repo, _ = repo_model.CountRepositories(db.DefaultContext, repo_model.CountRepositoryOptions{}) + stats.Counter.Watch, _ = e.Count(new(repo_model.Watch)) + stats.Counter.Star, _ = e.Count(new(repo_model.Star)) stats.Counter.Action, _ = db.EstimateCount(ctx, new(Action)) stats.Counter.Access, _ = db.EstimateCount(ctx, new(access_model.Access)) @@ -103,17 +103,17 @@ func GetStatistic(ctx context.Context, metrics bool) (stats Statistic) { stats.Counter.Issue = stats.Counter.IssueClosed + stats.Counter.IssueOpen stats.Counter.Comment, _ = db.EstimateCount(ctx, new(Comment)) - stats.Counter.Follow, _ = db.EstimateCount(ctx, new(user_model.Follow)) - stats.Counter.Mirror, _ = db.EstimateCount(ctx, new(repo_model.Mirror)) - stats.Counter.Release, _ = db.EstimateCount(ctx, new(Release)) - stats.Counter.Webhook, _ = db.EstimateCount(ctx, new(webhook.Webhook)) - stats.Counter.Milestone, _ = db.EstimateCount(ctx, new(issues_model.Milestone)) - stats.Counter.Label, _ = db.EstimateCount(ctx, new(Label)) + stats.Counter.Follow, _ = e.Count(new(user_model.Follow)) + stats.Counter.Mirror, _ = e.Count(new(repo_model.Mirror)) + stats.Counter.Release, _ = e.Count(new(Release)) + stats.Counter.Webhook, _ = e.Count(new(webhook.Webhook)) + stats.Counter.Milestone, _ = e.Count(new(issues_model.Milestone)) + stats.Counter.Label, _ = e.Count(new(Label)) stats.Counter.HookTask, _ = db.EstimateCount(ctx, new(webhook.HookTask)) - stats.Counter.Team, _ = db.EstimateCount(ctx, new(organization.Team)) + stats.Counter.Team, _ = e.Count(new(organization.Team)) stats.Counter.Attachment, _ = db.EstimateCount(ctx, new(repo_model.Attachment)) - stats.Counter.Project, _ = db.EstimateCount(ctx, new(project_model.Project)) - stats.Counter.ProjectBoard, _ = db.EstimateCount(ctx, new(project_model.Board)) + stats.Counter.Project, _ = e.Count(new(project_model.Project)) + stats.Counter.ProjectBoard, _ = e.Count(new(project_model.Board)) stats.Counter.Oauth = 0 stats.Counter.AuthSource = auth.CountSources() stats.Time = time.Now() diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 019577860d4ed..1b9ed4e3c2691 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -2423,7 +2423,7 @@ dashboard.new_version_hint = Gitea %s is now available, you are running %s. Chec dashboard.statistic = Summary dashboard.operations = Maintenance Operations dashboard.system_status = System Status -dashboard.statistic_info = The Gitea database holds %d users, %d organizations, %d public keys, %d repositories, %d watches, %d stars, ~%d actions, %d accesses, %d issues, %d comments, %d social accounts, %d follows, %d mirrors, %d releases, %d authentication sources, %d webhooks, %d milestones, %d labels, %d hook tasks, %d teams, %d update tasks, %d attachments. +dashboard.statistic_info = The Gitea database holds %d users, %d organizations, %d public keys, %d repositories, %d watches, %d stars, ~%d actions, ~%d accesses, %d issues, ~%d comments, %d social accounts, %d follows, %d mirrors, %d releases, %d authentication sources, %d webhooks, %d milestones, %d labels, ~%d hook tasks, %d teams, %d update tasks, ~%d attachments. dashboard.statistic_info_last_updated = Last updated %s dashboard.statistic_info_in_progress = Statistics are being calculated dashboard.operation_name = Operation Name From 27f8b14a0c79fcf13dc37b0098022d3718c55456 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sun, 22 May 2022 12:15:24 +0100 Subject: [PATCH 8/9] as per review Signed-off-by: Andrew Thornton --- modules/metrics/statistics.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/metrics/statistics.go b/modules/metrics/statistics.go index 028c64114c652..94a381cba4350 100644 --- a/modules/metrics/statistics.go +++ b/modules/metrics/statistics.go @@ -36,7 +36,7 @@ func GetStatistic(statisticsTTL time.Duration, full bool) <-chan *models.Statist } if stats != nil && stats.Time.Add(statisticsTTL).After(time.Now()) { // Found a valid cached statistic for these params, so unlock and send this down the channel - statisticsLock.Unlock() // Unlock from line 24 + statisticsLock.Unlock() // Unlock from above ourChan <- stats close(ourChan) @@ -97,7 +97,7 @@ func GetStatistic(statisticsTTL time.Duration, full bool) <-chan *models.Statist }() } - statisticsLock.Unlock() // Unlock from line 24 + statisticsLock.Unlock() // Unlock from above // Create our goroutine for the channel waiting for the statistics to be generated go func() { From 626d23ff769730a823455a5254b81b7122a0ea13 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sat, 13 Aug 2022 18:08:03 +0100 Subject: [PATCH 9/9] add examples to app.example.ini Signed-off-by: Andrew Thornton --- custom/conf/app.example.ini | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index dd4d5a4cbb3fe..b994fb940adea 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -1160,6 +1160,9 @@ ROUTER = console ;; ;; Number of organizations that are displayed on one page ;ORG_PAGING_NUM = 50 +;; +;; Cache summary statistics for this period of time. +;STATISTICS_TTL = 5m ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -2217,6 +2220,9 @@ ROUTER = console ;ENABLED_ISSUE_BY_LABEL = false ;; Enable issue by repository metrics; default is false ;ENABLED_ISSUE_BY_REPOSITORY = false +;; +;; Cache statistics for this period of time. +;STATISTICS_TTL = 5m ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;