From e161de0fd507643dbb7ef6eeae1221a93f558d82 Mon Sep 17 00:00:00 2001 From: fuxiaohei Date: Sat, 13 May 2023 08:29:39 +0800 Subject: [PATCH 1/6] fix minio storage iterator path --- modules/storage/minio.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/storage/minio.go b/modules/storage/minio.go index 250f17827ff5b..81963f5e660b2 100644 --- a/modules/storage/minio.go +++ b/modules/storage/minio.go @@ -244,7 +244,7 @@ func (m *MinioStorage) IterateObjects(prefix string, fn func(path string, obj Ob } if err := func(object *minio.Object, fn func(path string, obj Object) error) error { defer object.Close() - return fn(strings.TrimPrefix(mObjInfo.Key, basePath), &minioObject{object}) + return fn(strings.TrimPrefix(mObjInfo.Key, m.basePath), &minioObject{object}) }(object, fn); err != nil { return convertMinioErr(err) } From 3c1ca3d34ea26586db79193ae3f05278ead700ed Mon Sep 17 00:00:00 2001 From: fuxiaohei Date: Sat, 13 May 2023 18:46:17 +0800 Subject: [PATCH 2/6] add minio storage iterator tests --- .github/workflows/pull-db-tests.yml | 7 ++++ modules/storage/minio.go | 3 +- modules/storage/minio_test.go | 56 +++++++++++++++++++++++++++++ 3 files changed, 65 insertions(+), 1 deletion(-) create mode 100644 modules/storage/minio_test.go diff --git a/.github/workflows/pull-db-tests.yml b/.github/workflows/pull-db-tests.yml index 4011b4201be27..3446b711559d6 100644 --- a/.github/workflows/pull-db-tests.yml +++ b/.github/workflows/pull-db-tests.yml @@ -102,6 +102,13 @@ jobs: --health-retries 10 ports: - 6379:6379 + minio: + image: bitnami/minio:2021.3.17 + env: + MINIO_ACCESS_KEY: 123456 + MINIO_SECRET_KEY: 12345678 + ports: + - "9000:9000" steps: - uses: actions/checkout@v3 - uses: actions/setup-go@v4 diff --git a/modules/storage/minio.go b/modules/storage/minio.go index 81963f5e660b2..a772e2d7e419a 100644 --- a/modules/storage/minio.go +++ b/modules/storage/minio.go @@ -231,7 +231,8 @@ func (m *MinioStorage) IterateObjects(prefix string, fn func(path string, obj Ob basePath := m.basePath if prefix != "" { - basePath = m.buildMinioPath(prefix) + // ending slash is required for avoiding matching like "foo/" and "foobar/" with prefix "foo" + basePath = m.buildMinioPath(prefix) + "/" } for mObjInfo := range m.client.ListObjects(lobjectCtx, m.bucket, minio.ListObjectsOptions{ diff --git a/modules/storage/minio_test.go b/modules/storage/minio_test.go new file mode 100644 index 0000000000000..daf4d64bbf4ba --- /dev/null +++ b/modules/storage/minio_test.go @@ -0,0 +1,56 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package storage + +import ( + "bytes" + "context" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestMinioStorageIterator(t *testing.T) { + config := MinioStorageConfig{ + Endpoint: "127.0.0.1:9000", + AccessKeyID: "123456", + SecretAccessKey: "12345678", + Bucket: "gitea", + Location: "us-east-1", + } + l, err := NewMinioStorage(context.Background(), config) + assert.NoError(t, err) + + testFiles := [][]string{ + {"m/1.txt", "m1"}, + {"/mn/1.txt", "mn1"}, + {"b/1.txt", "b1"}, + {"b/2.txt", "b2"}, + {"b/3.txt", "b3"}, + {"b/x 4.txt", "bx4"}, + } + for _, f := range testFiles { + _, err = l.Save(f[0], bytes.NewBufferString(f[1]), -1) + assert.NoError(t, err) + } + + expectedList := map[string][]string{ + "mn": {"mn/1.txt"}, + "m": {"m/1.txt"}, + "b": {"b/1.txt", "b/2.txt", "b/3.txt", "b/x 4.txt"}, + "m/b/../../m": {"m/1.txt"}, + } + for dir, expected := range expectedList { + count := 0 + err = l.IterateObjects(dir, func(path string, f Object) error { + defer f.Close() + assert.Contains(t, expected, path) + count++ + return nil + }) + assert.NoError(t, err) + assert.Len(t, expected, count) + } + +} From 23f7faf29c0f240cfae62fb9ef7176d7cdd9b752 Mon Sep 17 00:00:00 2001 From: fuxiaohei Date: Sat, 13 May 2023 19:03:45 +0800 Subject: [PATCH 3/6] add tests cases for storage iterator --- modules/storage/local_test.go | 5 +++-- modules/storage/minio.go | 1 + modules/storage/minio_test.go | 13 +++++++------ 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/modules/storage/local_test.go b/modules/storage/local_test.go index 0873f8e14ef03..26541a2e88775 100644 --- a/modules/storage/local_test.go +++ b/modules/storage/local_test.go @@ -63,6 +63,7 @@ func TestLocalStorageIterator(t *testing.T) { testFiles := [][]string{ {"a/1.txt", "a1"}, {"/a/1.txt", "aa1"}, // same as above, but with leading slash that will be trim + {"ab/1.txt", "ab1"}, {"b/1.txt", "b1"}, {"b/2.txt", "b2"}, {"b/3.txt", "b3"}, @@ -76,8 +77,8 @@ func TestLocalStorageIterator(t *testing.T) { expectedList := map[string][]string{ "a": {"a/1.txt"}, "b": {"b/1.txt", "b/2.txt", "b/3.txt", "b/x 4.txt"}, - "": {"a/1.txt", "b/1.txt", "b/2.txt", "b/3.txt", "b/x 4.txt"}, - "/": {"a/1.txt", "b/1.txt", "b/2.txt", "b/3.txt", "b/x 4.txt"}, + "": {"a/1.txt", "b/1.txt", "b/2.txt", "b/3.txt", "b/x 4.txt", "ab/1.txt"}, + "/": {"a/1.txt", "b/1.txt", "b/2.txt", "b/3.txt", "b/x 4.txt", "ab/1.txt"}, "a/b/../../a": {"a/1.txt"}, } for dir, expected := range expectedList { diff --git a/modules/storage/minio.go b/modules/storage/minio.go index a772e2d7e419a..12a206d2fccf9 100644 --- a/modules/storage/minio.go +++ b/modules/storage/minio.go @@ -129,6 +129,7 @@ func NewMinioStorage(ctx context.Context, cfg interface{}) (ObjectStorage, error } func (m *MinioStorage) buildMinioPath(p string) string { + p = strings.TrimPrefix(p, "/") // url path prefix should not start with / return util.PathJoinRelX(m.basePath, p) } diff --git a/modules/storage/minio_test.go b/modules/storage/minio_test.go index daf4d64bbf4ba..7b8ebe94b6cce 100644 --- a/modules/storage/minio_test.go +++ b/modules/storage/minio_test.go @@ -23,8 +23,9 @@ func TestMinioStorageIterator(t *testing.T) { assert.NoError(t, err) testFiles := [][]string{ - {"m/1.txt", "m1"}, - {"/mn/1.txt", "mn1"}, + {"a/1.txt", "a1"}, + {"/a/1.txt", "aa1"}, // same as above, but with leading slash that will be trim + {"ab/1.txt", "ab1"}, {"b/1.txt", "b1"}, {"b/2.txt", "b2"}, {"b/3.txt", "b3"}, @@ -36,10 +37,11 @@ func TestMinioStorageIterator(t *testing.T) { } expectedList := map[string][]string{ - "mn": {"mn/1.txt"}, - "m": {"m/1.txt"}, + "a": {"a/1.txt"}, "b": {"b/1.txt", "b/2.txt", "b/3.txt", "b/x 4.txt"}, - "m/b/../../m": {"m/1.txt"}, + "": {"a/1.txt", "b/1.txt", "b/2.txt", "b/3.txt", "b/x 4.txt", "ab/1.txt"}, + "/": {"a/1.txt", "b/1.txt", "b/2.txt", "b/3.txt", "b/x 4.txt", "ab/1.txt"}, + "a/b/../../a": {"a/1.txt"}, } for dir, expected := range expectedList { count := 0 @@ -52,5 +54,4 @@ func TestMinioStorageIterator(t *testing.T) { assert.NoError(t, err) assert.Len(t, expected, count) } - } From ddba317e3e48de096634abbd26ef2ef3f899264a Mon Sep 17 00:00:00 2001 From: fuxiaohei Date: Sat, 13 May 2023 19:05:47 +0800 Subject: [PATCH 4/6] storege iterator prefix should be dirName --- modules/storage/local.go | 4 ++-- modules/storage/minio.go | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/modules/storage/local.go b/modules/storage/local.go index d22974a65add0..73ef306979abe 100644 --- a/modules/storage/local.go +++ b/modules/storage/local.go @@ -133,8 +133,8 @@ func (l *LocalStorage) URL(path, name string) (*url.URL, error) { } // IterateObjects iterates across the objects in the local storage -func (l *LocalStorage) IterateObjects(prefix string, fn func(path string, obj Object) error) error { - dir := l.buildLocalPath(prefix) +func (l *LocalStorage) IterateObjects(dirName string, fn func(path string, obj Object) error) error { + dir := l.buildLocalPath(dirName) return filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error { if err != nil { return err diff --git a/modules/storage/minio.go b/modules/storage/minio.go index 12a206d2fccf9..144ebcf1722d3 100644 --- a/modules/storage/minio.go +++ b/modules/storage/minio.go @@ -225,15 +225,15 @@ func (m *MinioStorage) URL(path, name string) (*url.URL, error) { } // IterateObjects iterates across the objects in the miniostorage -func (m *MinioStorage) IterateObjects(prefix string, fn func(path string, obj Object) error) error { +func (m *MinioStorage) IterateObjects(dirName string, fn func(path string, obj Object) error) error { opts := minio.GetObjectOptions{} lobjectCtx, cancel := context.WithCancel(m.ctx) defer cancel() basePath := m.basePath - if prefix != "" { + if dirName != "" { // ending slash is required for avoiding matching like "foo/" and "foobar/" with prefix "foo" - basePath = m.buildMinioPath(prefix) + "/" + basePath = m.buildMinioPath(dirName) + "/" } for mObjInfo := range m.client.ListObjects(lobjectCtx, m.bucket, minio.ListObjectsOptions{ From 4f3c992feafdb7fd3c2375670992d95ba0841e2a Mon Sep 17 00:00:00 2001 From: fuxiaohei Date: Sat, 13 May 2023 20:48:27 +0800 Subject: [PATCH 5/6] use test helper in storage iterator tests --- modules/storage/local_test.go | 38 +------------------------ modules/storage/minio_test.go | 43 ++--------------------------- modules/storage/storage_test.go | 49 +++++++++++++++++++++++++++++++++ 3 files changed, 52 insertions(+), 78 deletions(-) create mode 100644 modules/storage/storage_test.go diff --git a/modules/storage/local_test.go b/modules/storage/local_test.go index 26541a2e88775..1c4b856ab6afd 100644 --- a/modules/storage/local_test.go +++ b/modules/storage/local_test.go @@ -4,8 +4,6 @@ package storage import ( - "bytes" - "context" "os" "path/filepath" "testing" @@ -57,39 +55,5 @@ func TestBuildLocalPath(t *testing.T) { func TestLocalStorageIterator(t *testing.T) { dir := filepath.Join(os.TempDir(), "TestLocalStorageIteratorTestDir") - l, err := NewLocalStorage(context.Background(), LocalStorageConfig{Path: dir}) - assert.NoError(t, err) - - testFiles := [][]string{ - {"a/1.txt", "a1"}, - {"/a/1.txt", "aa1"}, // same as above, but with leading slash that will be trim - {"ab/1.txt", "ab1"}, - {"b/1.txt", "b1"}, - {"b/2.txt", "b2"}, - {"b/3.txt", "b3"}, - {"b/x 4.txt", "bx4"}, - } - for _, f := range testFiles { - _, err = l.Save(f[0], bytes.NewBufferString(f[1]), -1) - assert.NoError(t, err) - } - - expectedList := map[string][]string{ - "a": {"a/1.txt"}, - "b": {"b/1.txt", "b/2.txt", "b/3.txt", "b/x 4.txt"}, - "": {"a/1.txt", "b/1.txt", "b/2.txt", "b/3.txt", "b/x 4.txt", "ab/1.txt"}, - "/": {"a/1.txt", "b/1.txt", "b/2.txt", "b/3.txt", "b/x 4.txt", "ab/1.txt"}, - "a/b/../../a": {"a/1.txt"}, - } - for dir, expected := range expectedList { - count := 0 - err = l.IterateObjects(dir, func(path string, f Object) error { - defer f.Close() - assert.Contains(t, expected, path) - count++ - return nil - }) - assert.NoError(t, err) - assert.Len(t, expected, count) - } + testStorageIterator(t, string(LocalStorageType), LocalStorageConfig{Path: dir}) } diff --git a/modules/storage/minio_test.go b/modules/storage/minio_test.go index 7b8ebe94b6cce..bee1b86318607 100644 --- a/modules/storage/minio_test.go +++ b/modules/storage/minio_test.go @@ -4,54 +4,15 @@ package storage import ( - "bytes" - "context" "testing" - - "github.com/stretchr/testify/assert" ) func TestMinioStorageIterator(t *testing.T) { - config := MinioStorageConfig{ + testStorageIterator(t, string(MinioStorageType), MinioStorageConfig{ Endpoint: "127.0.0.1:9000", AccessKeyID: "123456", SecretAccessKey: "12345678", Bucket: "gitea", Location: "us-east-1", - } - l, err := NewMinioStorage(context.Background(), config) - assert.NoError(t, err) - - testFiles := [][]string{ - {"a/1.txt", "a1"}, - {"/a/1.txt", "aa1"}, // same as above, but with leading slash that will be trim - {"ab/1.txt", "ab1"}, - {"b/1.txt", "b1"}, - {"b/2.txt", "b2"}, - {"b/3.txt", "b3"}, - {"b/x 4.txt", "bx4"}, - } - for _, f := range testFiles { - _, err = l.Save(f[0], bytes.NewBufferString(f[1]), -1) - assert.NoError(t, err) - } - - expectedList := map[string][]string{ - "a": {"a/1.txt"}, - "b": {"b/1.txt", "b/2.txt", "b/3.txt", "b/x 4.txt"}, - "": {"a/1.txt", "b/1.txt", "b/2.txt", "b/3.txt", "b/x 4.txt", "ab/1.txt"}, - "/": {"a/1.txt", "b/1.txt", "b/2.txt", "b/3.txt", "b/x 4.txt", "ab/1.txt"}, - "a/b/../../a": {"a/1.txt"}, - } - for dir, expected := range expectedList { - count := 0 - err = l.IterateObjects(dir, func(path string, f Object) error { - defer f.Close() - assert.Contains(t, expected, path) - count++ - return nil - }) - assert.NoError(t, err) - assert.Len(t, expected, count) - } + }) } diff --git a/modules/storage/storage_test.go b/modules/storage/storage_test.go new file mode 100644 index 0000000000000..b517a9e71a16e --- /dev/null +++ b/modules/storage/storage_test.go @@ -0,0 +1,49 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package storage + +import ( + "bytes" + "testing" + + "github.com/stretchr/testify/assert" +) + +func testStorageIterator(t *testing.T, typStr string, cfg interface{}) { + l, err := NewStorage(typStr, cfg) + assert.NoError(t, err) + + testFiles := [][]string{ + {"a/1.txt", "a1"}, + {"/a/1.txt", "aa1"}, // same as above, but with leading slash that will be trim + {"ab/1.txt", "ab1"}, + {"b/1.txt", "b1"}, + {"b/2.txt", "b2"}, + {"b/3.txt", "b3"}, + {"b/x 4.txt", "bx4"}, + } + for _, f := range testFiles { + _, err = l.Save(f[0], bytes.NewBufferString(f[1]), -1) + assert.NoError(t, err) + } + + expectedList := map[string][]string{ + "a": {"a/1.txt"}, + "b": {"b/1.txt", "b/2.txt", "b/3.txt", "b/x 4.txt"}, + "": {"a/1.txt", "b/1.txt", "b/2.txt", "b/3.txt", "b/x 4.txt", "ab/1.txt"}, + "/": {"a/1.txt", "b/1.txt", "b/2.txt", "b/3.txt", "b/x 4.txt", "ab/1.txt"}, + "a/b/../../a": {"a/1.txt"}, + } + for dir, expected := range expectedList { + count := 0 + err = l.IterateObjects(dir, func(path string, f Object) error { + defer f.Close() + assert.Contains(t, expected, path) + count++ + return nil + }) + assert.NoError(t, err) + assert.Len(t, expected, count) + } +} From 72b27a26f8f9db17aba4ed432c99dc534050667b Mon Sep 17 00:00:00 2001 From: fuxiaohei Date: Sat, 13 May 2023 22:31:09 +0800 Subject: [PATCH 6/6] correct trim for minio storage path dirName --- modules/storage/minio.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/modules/storage/minio.go b/modules/storage/minio.go index 144ebcf1722d3..c78f351e9ce03 100644 --- a/modules/storage/minio.go +++ b/modules/storage/minio.go @@ -129,8 +129,11 @@ func NewMinioStorage(ctx context.Context, cfg interface{}) (ObjectStorage, error } func (m *MinioStorage) buildMinioPath(p string) string { - p = strings.TrimPrefix(p, "/") // url path prefix should not start with / - return util.PathJoinRelX(m.basePath, p) + p = util.PathJoinRelX(m.basePath, p) + if p == "." { + p = "" // minio doesn't use dot as relative path + } + return p } // Open opens a file