From 3ecbdbe70a84c20a21a18c4dd961b20fc6773ad2 Mon Sep 17 00:00:00 2001
From: Guillermo Prandi <guillep2k@users.noreply.github.com>
Date: Sun, 11 Aug 2019 19:19:37 -0300
Subject: [PATCH] Make token deduplication optional for the repo indexer

---
 custom/conf/app.ini.sample                    |  1 +
 .../doc/advanced/config-cheat-sheet.en-us.md  |  1 +
 modules/indexer/repo.go                       |  6 ++++-
 modules/setting/indexer.go                    | 22 ++++++++++---------
 4 files changed, 19 insertions(+), 11 deletions(-)

diff --git a/custom/conf/app.ini.sample b/custom/conf/app.ini.sample
index 99e50ffba9b72..bcd01113c637f 100644
--- a/custom/conf/app.ini.sample
+++ b/custom/conf/app.ini.sample
@@ -296,6 +296,7 @@ REPO_INDEXER_ENABLED = false
 REPO_INDEXER_PATH = indexers/repos.bleve
 UPDATE_BUFFER_LEN = 20
 MAX_FILE_SIZE = 1048576
+REPO_INDEXER_DEDUPLICATE = true
 
 [admin]
 ; Disallow regular (non-admin) users from creating organizations.
diff --git a/docs/content/doc/advanced/config-cheat-sheet.en-us.md b/docs/content/doc/advanced/config-cheat-sheet.en-us.md
index 89f57d9b0c0d4..a60eb6983bea3 100644
--- a/docs/content/doc/advanced/config-cheat-sheet.en-us.md
+++ b/docs/content/doc/advanced/config-cheat-sheet.en-us.md
@@ -180,6 +180,7 @@ Values containing `#` or `;` must be quoted using `` ` `` or `"""`.
 - `REPO_INDEXER_PATH`: **indexers/repos.bleve**: Index file used for code search.
 - `UPDATE_BUFFER_LEN`: **20**: Buffer length of index request.
 - `MAX_FILE_SIZE`: **1048576**: Maximum size in bytes of files to be indexed.
+- `REPO_INDEXER_DEDUPLICATE`: **true** Deduplicate terms from the index, reducing the index size while limiting the search capabilities.
 
 ## Security (`security`)
 
diff --git a/modules/indexer/repo.go b/modules/indexer/repo.go
index 787f9a9467ab0..a6a5736d7fb84 100644
--- a/modules/indexer/repo.go
+++ b/modules/indexer/repo.go
@@ -103,6 +103,10 @@ func createRepoIndexer(path string, latestVersion int) error {
 	textFieldMapping.IncludeInAll = false
 	docMapping.AddFieldMappingsAt("Content", textFieldMapping)
 
+	tokenFilters := []string{unicodeNormalizeName, lowercase.Name}
+	if setting.Indexer.RepoIndexerDeduplicate {
+		tokenFilters = append(tokenFilters, unique.Name)
+	}
 	mapping := bleve.NewIndexMapping()
 	if err = addUnicodeNormalizeTokenFilter(mapping); err != nil {
 		return err
@@ -110,7 +114,7 @@ func createRepoIndexer(path string, latestVersion int) error {
 		"type":          custom.Name,
 		"char_filters":  []string{},
 		"tokenizer":     unicode.Name,
-		"token_filters": []string{unicodeNormalizeName, lowercase.Name, unique.Name},
+		"token_filters": tokenFilters,
 	}); err != nil {
 		return err
 	}
diff --git a/modules/setting/indexer.go b/modules/setting/indexer.go
index 36fd4a020b2ce..16bbc5c7520ee 100644
--- a/modules/setting/indexer.go
+++ b/modules/setting/indexer.go
@@ -19,16 +19,17 @@ const (
 var (
 	// Indexer settings
 	Indexer = struct {
-		IssueType             string
-		IssuePath             string
-		RepoIndexerEnabled    bool
-		RepoPath              string
-		UpdateQueueLength     int
-		MaxIndexerFileSize    int64
-		IssueQueueType        string
-		IssueQueueDir         string
-		IssueQueueConnStr     string
-		IssueQueueBatchNumber int
+		IssueType              string
+		IssuePath              string
+		RepoIndexerEnabled     bool
+		RepoPath               string
+		UpdateQueueLength      int
+		MaxIndexerFileSize     int64
+		RepoIndexerDeduplicate bool
+		IssueQueueType         string
+		IssueQueueDir          string
+		IssueQueueConnStr      string
+		IssueQueueBatchNumber  int
 	}{
 		IssueType:             "bleve",
 		IssuePath:             "indexers/issues.bleve",
@@ -51,6 +52,7 @@ func newIndexerService() {
 	if !filepath.IsAbs(Indexer.RepoPath) {
 		Indexer.RepoPath = path.Join(AppWorkPath, Indexer.RepoPath)
 	}
+	Indexer.RepoIndexerDeduplicate = sec.Key("REPO_INDEXER_DEDUPLICATE").MustBool(true)
 	Indexer.UpdateQueueLength = sec.Key("UPDATE_BUFFER_LEN").MustInt(20)
 	Indexer.MaxIndexerFileSize = sec.Key("MAX_FILE_SIZE").MustInt64(1024 * 1024)
 	Indexer.IssueQueueType = sec.Key("ISSUE_INDEXER_QUEUE_TYPE").MustString(LevelQueueType)