Skip to content

Commit 7ba137a

Browse files
ibuildthecloudibuildthecloud
authored
Merge pull request #110 from ibuildthecloud/main
Add github metadata to internal data structures
2 parents abdf8d3 + 057ad68 commit 7ba137a

File tree

10 files changed

+247
-95
lines changed

10 files changed

+247
-95
lines changed

‎main.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,9 @@ package main
33
import (
44
"github.com/acorn-io/cmd"
55
"github.com/gptscript-ai/gptscript/pkg/cli"
6+
7+
// Load all VCS
8+
_ "github.com/gptscript-ai/gptscript/pkg/loader/vcs"
69
)
710

811
func main() {

‎pkg/builtin/builtin.go

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -216,6 +216,10 @@ func SysFind(ctx context.Context, env []string, input string) (string, error) {
216216
if err != nil {
217217
return "", nil
218218
}
219+
if len(result) == 0 {
220+
return "No files found", nil
221+
}
222+
219223
sort.Strings(result)
220224
return strings.Join(result, "\n"), nil
221225
}
@@ -455,7 +459,11 @@ func SysStat(ctx context.Context, env []string, input string) (string, error) {
455459
return "", err
456460
}
457461

458-
return fmt.Sprintf("File %s mode: %s, size: %d bytes, modtime: %s", params.Filepath, stat.Mode().String(), stat.Size(), stat.ModTime().String()), nil
462+
title := "File"
463+
if stat.IsDir() {
464+
title = "Directory"
465+
}
466+
return fmt.Sprintf("%s %s mode: %s, size: %d bytes, modtime: %s", title, params.Filepath, stat.Mode().String(), stat.Size(), stat.ModTime().String()), nil
459467
}
460468

461469
func SysDownload(ctx context.Context, env []string, input string) (_ string, err error) {

‎pkg/loader/github/github.go

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
package github
2+
3+
import (
4+
"encoding/json"
5+
"fmt"
6+
"io"
7+
"net/http"
8+
"path/filepath"
9+
"strings"
10+
11+
"github.com/gptscript-ai/gptscript/pkg/loader"
12+
"github.com/gptscript-ai/gptscript/pkg/system"
13+
)
14+
15+
const (
16+
GithubPrefix = "github.com/"
17+
githubRepoURL = "https://github.com/%s/%s.git"
18+
githubDownloadURL = "https://raw.githubusercontent.com/%s/%s/%s/%s"
19+
githubCommitURL = "https://api.github.com/repos/%s/%s/commits/%s"
20+
)
21+
22+
func init() {
23+
loader.AddVSC(Load)
24+
}
25+
26+
func getCommit(account, repo, ref string) (string, error) {
27+
url := fmt.Sprintf(githubCommitURL, account, repo, ref)
28+
resp, err := http.Get(url)
29+
if err != nil {
30+
return "", err
31+
} else if resp.StatusCode != http.StatusOK {
32+
c, _ := io.ReadAll(resp.Body)
33+
resp.Body.Close()
34+
return "", fmt.Errorf("failed to GitHub commit of %s/%s at %s: %s %s",
35+
account, repo, ref, resp.Status, c)
36+
}
37+
defer resp.Body.Close()
38+
39+
var commit struct {
40+
SHA string `json:"sha,omitempty"`
41+
}
42+
if err := json.NewDecoder(resp.Body).Decode(&commit); err != nil {
43+
return "", fmt.Errorf("failed to decode GitHub commit of %s/%s at %s: %w", account, repo, url, err)
44+
}
45+
46+
if commit.SHA == "" {
47+
return "", fmt.Errorf("failed to find commit in response of %s, got empty string", url)
48+
}
49+
50+
return commit.SHA, nil
51+
}
52+
53+
func Load(urlName string) (string, *loader.Repo, bool, error) {
54+
if !strings.HasPrefix(urlName, GithubPrefix) {
55+
return "", nil, false, nil
56+
}
57+
58+
url, ref, _ := strings.Cut(urlName, "@")
59+
if ref == "" {
60+
ref = "HEAD"
61+
}
62+
63+
parts := strings.Split(url, "/")
64+
// Must be at least 4 parts github.com/ACCOUNT/REPO/FILE
65+
if len(parts) < 4 {
66+
return "", nil, false, nil
67+
}
68+
69+
account, repo := parts[1], parts[2]
70+
path := strings.Join(parts[3:], "/")
71+
72+
if path == "" || path == "/" {
73+
path = "tool.gpt"
74+
} else if !strings.HasSuffix(path, system.Suffix) {
75+
path += "/tool.gpt"
76+
}
77+
78+
ref, err := getCommit(account, repo, ref)
79+
if err != nil {
80+
return "", nil, false, err
81+
}
82+
83+
downloadURL := fmt.Sprintf(githubDownloadURL, account, repo, ref, path)
84+
return downloadURL, &loader.Repo{
85+
VCS: "github",
86+
Root: fmt.Sprintf(githubRepoURL, account, repo),
87+
Path: filepath.Dir(path),
88+
Name: filepath.Base(path),
89+
Revision: ref,
90+
}, true, nil
91+
}

‎pkg/loader/loader.go

Lines changed: 38 additions & 86 deletions
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,6 @@ import (
1111
"fmt"
1212
"io"
1313
"io/fs"
14-
"net/http"
15-
url2 "net/url"
1614
"os"
1715
"path/filepath"
1816
"regexp"
@@ -22,20 +20,37 @@ import (
2220
"github.com/gptscript-ai/gptscript/pkg/builtin"
2321
"github.com/gptscript-ai/gptscript/pkg/engine"
2422
"github.com/gptscript-ai/gptscript/pkg/parser"
23+
"github.com/gptscript-ai/gptscript/pkg/system"
2524
"github.com/gptscript-ai/gptscript/pkg/types"
2625
)
2726

28-
const (
29-
GithubPrefix = "github.com/"
30-
githubRawURL = "https://raw.githubusercontent.com/"
31-
)
32-
3327
type source struct {
28+
// Content The content of the source
3429
Content io.ReadCloser
35-
Remote bool
36-
Path string
37-
Name string
38-
File string
30+
// Remote indicates that this file was loaded from a remote source (not local disk)
31+
Remote bool
32+
// Path is the path of this source used to find any relative references to this source
33+
Path string
34+
// Name is the filename of this source, it does not include the path in it
35+
Name string
36+
// Location is a string representation representing the source. It's not assume to
37+
// be a valid URI or URL, used primarily for display.
38+
Location string
39+
// Repo The VCS repo where this tool was found, used to clone and provide the local tool code content
40+
Repo *Repo
41+
}
42+
43+
type Repo struct {
44+
// VCS The VCS type, such as "github"
45+
VCS string
46+
// The URL where the VCS repo can be found
47+
Root string
48+
// The path in the repo of this source. This should refer to a directory and not the actual file
49+
Path string
50+
// The filename of the source in the repo, relative to Path
51+
Name string
52+
// The revision of this source
53+
Revision string
3954
}
4055

4156
func (s *source) String() string {
@@ -67,74 +82,11 @@ func loadLocal(base *source, name string) (*source, bool, error) {
6782
log.Debugf("opened %s", path)
6883

6984
return &source{
70-
Content: content,
71-
Remote: false,
72-
Path: filepath.Dir(path),
73-
Name: filepath.Base(path),
74-
File: path,
75-
}, true, nil
76-
}
77-
78-
func githubURL(urlName string) (string, bool) {
79-
if !strings.HasPrefix(urlName, GithubPrefix) {
80-
return "", false
81-
}
82-
83-
url, version, _ := strings.Cut(urlName, "@")
84-
if version == "" {
85-
version = "HEAD"
86-
}
87-
88-
parts := strings.Split(url, "/")
89-
// Must be at least 4 parts github.com/ACCOUNT/REPO/FILE
90-
if len(parts) < 4 {
91-
return "", false
92-
}
93-
94-
url = githubRawURL + parts[1] + "/" + parts[2] + "/" + version + "/" + strings.Join(parts[3:], "/")
95-
return url, true
96-
}
97-
98-
func loadURL(ctx context.Context, base *source, name string) (*source, bool, error) {
99-
url := name
100-
if base.Path != "" {
101-
url = base.Path + "/" + name
102-
}
103-
if githubURL, ok := githubURL(url); ok {
104-
url = githubURL
105-
}
106-
if !strings.HasPrefix(url, "http://") && !strings.HasPrefix(url, "https://") {
107-
return nil, false, nil
108-
}
109-
110-
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
111-
if err != nil {
112-
return nil, false, err
113-
}
114-
115-
resp, err := http.DefaultClient.Do(req)
116-
if err != nil {
117-
return nil, false, err
118-
} else if resp.StatusCode != http.StatusOK {
119-
return nil, false, fmt.Errorf("error loading %s: %s", url, resp.Status)
120-
}
121-
122-
log.Debugf("opened %s", url)
123-
124-
parsed, err := url2.Parse(url)
125-
if err != nil {
126-
return nil, false, err
127-
}
128-
129-
pathURL := *parsed
130-
pathURL.Path = filepath.Dir(parsed.Path)
131-
132-
return &source{
133-
Content: resp.Body,
134-
Remote: true,
135-
Path: pathURL.String(),
136-
Name: filepath.Base(parsed.Path),
137-
File: url,
85+
Content: content,
86+
Remote: false,
87+
Path: filepath.Dir(path),
88+
Name: filepath.Base(path),
89+
Location: path,
13890
}, true, nil
13991
}
14092

@@ -201,7 +153,7 @@ func readTool(ctx context.Context, prg *types.Program, base *source, targetToolN
201153

202154
for i, tool := range tools {
203155
tool.WorkingDir = base.Path
204-
tool.Source.File = base.File
156+
tool.Source.Location = base.Location
205157

206158
// Probably a better way to come up with an ID
207159
tool.ID = tool.Source.String()
@@ -211,16 +163,16 @@ func readTool(ctx context.Context, prg *types.Program, base *source, targetToolN
211163
}
212164

213165
if i != 0 && tool.Parameters.Name == "" {
214-
return types.Tool{}, parser.NewErrLine(tool.Source.File, tool.Source.LineNo, fmt.Errorf("only the first tool in a file can have no name"))
166+
return types.Tool{}, parser.NewErrLine(tool.Source.Location, tool.Source.LineNo, fmt.Errorf("only the first tool in a file can have no name"))
215167
}
216168

217169
if targetToolName != "" && tool.Parameters.Name == targetToolName {
218170
mainTool = tool
219171
}
220172

221173
if existing, ok := localTools[tool.Parameters.Name]; ok {
222-
return types.Tool{}, parser.NewErrLine(tool.Source.File, tool.Source.LineNo,
223-
fmt.Errorf("duplicate tool name [%s] in %s found at lines %d and %d", tool.Parameters.Name, tool.Source.File,
174+
return types.Tool{}, parser.NewErrLine(tool.Source.Location, tool.Source.LineNo,
175+
fmt.Errorf("duplicate tool name [%s] in %s found at lines %d and %d", tool.Parameters.Name, tool.Source.Location,
224176
tool.Source.LineNo, existing.Source.LineNo))
225177
}
226178

@@ -238,7 +190,7 @@ var (
238190
func ToolNormalizer(tool string) string {
239191
parts := strings.Split(tool, "/")
240192
tool = parts[len(parts)-1]
241-
if strings.HasSuffix(tool, ".gpt") {
193+
if strings.HasSuffix(tool, system.Suffix) {
242194
tool = strings.TrimSuffix(tool, filepath.Ext(tool))
243195
}
244196

@@ -349,8 +301,8 @@ func ProgramFromSource(ctx context.Context, content, subToolName string) (types.
349301
ToolSet: types.ToolSet{},
350302
}
351303
tool, err := readTool(ctx, &prg, &source{
352-
Content: io.NopCloser(strings.NewReader(content)),
353-
File: "inline",
304+
Content: io.NopCloser(strings.NewReader(content)),
305+
Location: "inline",
354306
}, subToolName)
355307
if err != nil {
356308
return types.Program{}, err

‎pkg/loader/url.go

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
package loader
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"net/http"
7+
url2 "net/url"
8+
"path/filepath"
9+
"strings"
10+
)
11+
12+
type VCSLookup func(string) (string, *Repo, bool, error)
13+
14+
var vcsLookups []VCSLookup
15+
16+
func AddVSC(lookup VCSLookup) {
17+
vcsLookups = append(vcsLookups, lookup)
18+
}
19+
20+
func loadURL(ctx context.Context, base *source, name string) (*source, bool, error) {
21+
var (
22+
repo *Repo
23+
url = name
24+
)
25+
26+
if base.Path != "" {
27+
url = base.Path + "/" + name
28+
}
29+
30+
if base.Repo != nil {
31+
newRepo := *base.Repo
32+
newPath := filepath.Join(newRepo.Path, name)
33+
newRepo.Path = filepath.Dir(newPath)
34+
newRepo.Name = filepath.Base(newPath)
35+
repo = &newRepo
36+
}
37+
38+
if repo == nil {
39+
for _, vcs := range vcsLookups {
40+
newURL, newRepo, ok, err := vcs(url)
41+
if err != nil {
42+
return nil, false, err
43+
} else if ok {
44+
repo = newRepo
45+
url = newURL
46+
break
47+
}
48+
}
49+
}
50+
51+
if !strings.HasPrefix(url, "http://") && !strings.HasPrefix(url, "https://") {
52+
return nil, false, nil
53+
}
54+
55+
parsed, err := url2.Parse(url)
56+
if err != nil {
57+
return nil, false, err
58+
}
59+
60+
pathURL := *parsed
61+
pathURL.Path = filepath.Dir(parsed.Path)
62+
path := pathURL.String()
63+
name = filepath.Base(parsed.Path)
64+
url = path + "/" + name
65+
66+
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
67+
if err != nil {
68+
return nil, false, err
69+
}
70+
71+
resp, err := http.DefaultClient.Do(req)
72+
if err != nil {
73+
return nil, false, err
74+
} else if resp.StatusCode != http.StatusOK {
75+
return nil, false, fmt.Errorf("error loading %s: %s", url, resp.Status)
76+
}
77+
78+
log.Debugf("opened %s", url)
79+
80+
return &source{
81+
Content: resp.Body,
82+
Remote: true,
83+
Path: path,
84+
Name: name,
85+
Location: url,
86+
Repo: repo,
87+
}, true, nil
88+
}

0 commit comments

Comments
 (0)