diff --git a/main.go b/main.go index 3ac87972..597bf11a 100644 --- a/main.go +++ b/main.go @@ -3,6 +3,9 @@ package main import ( "github.com/acorn-io/cmd" "github.com/gptscript-ai/gptscript/pkg/cli" + + // Load all VCS + _ "github.com/gptscript-ai/gptscript/pkg/loader/vcs" ) func main() { diff --git a/pkg/builtin/builtin.go b/pkg/builtin/builtin.go index 62c526b5..027c3ed6 100644 --- a/pkg/builtin/builtin.go +++ b/pkg/builtin/builtin.go @@ -216,6 +216,10 @@ func SysFind(ctx context.Context, env []string, input string) (string, error) { if err != nil { return "", nil } + if len(result) == 0 { + return "No files found", nil + } + sort.Strings(result) return strings.Join(result, "\n"), nil } @@ -455,7 +459,11 @@ func SysStat(ctx context.Context, env []string, input string) (string, error) { return "", err } - return fmt.Sprintf("File %s mode: %s, size: %d bytes, modtime: %s", params.Filepath, stat.Mode().String(), stat.Size(), stat.ModTime().String()), nil + title := "File" + if stat.IsDir() { + title = "Directory" + } + return fmt.Sprintf("%s %s mode: %s, size: %d bytes, modtime: %s", title, params.Filepath, stat.Mode().String(), stat.Size(), stat.ModTime().String()), nil } func SysDownload(ctx context.Context, env []string, input string) (_ string, err error) { diff --git a/pkg/loader/github/github.go b/pkg/loader/github/github.go new file mode 100644 index 00000000..9c52f819 --- /dev/null +++ b/pkg/loader/github/github.go @@ -0,0 +1,91 @@ +package github + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "path/filepath" + "strings" + + "github.com/gptscript-ai/gptscript/pkg/loader" + "github.com/gptscript-ai/gptscript/pkg/system" +) + +const ( + GithubPrefix = "github.com/" + githubRepoURL = "https://github.com/%s/%s.git" + githubDownloadURL = "https://raw.githubusercontent.com/%s/%s/%s/%s" + githubCommitURL = "https://api.github.com/repos/%s/%s/commits/%s" +) + +func init() { + loader.AddVSC(Load) +} + +func getCommit(account, repo, ref string) (string, error) { + url := fmt.Sprintf(githubCommitURL, account, repo, ref) + resp, err := http.Get(url) + if err != nil { + return "", err + } else if resp.StatusCode != http.StatusOK { + c, _ := io.ReadAll(resp.Body) + resp.Body.Close() + return "", fmt.Errorf("failed to GitHub commit of %s/%s at %s: %s %s", + account, repo, ref, resp.Status, c) + } + defer resp.Body.Close() + + var commit struct { + SHA string `json:"sha,omitempty"` + } + if err := json.NewDecoder(resp.Body).Decode(&commit); err != nil { + return "", fmt.Errorf("failed to decode GitHub commit of %s/%s at %s: %w", account, repo, url, err) + } + + if commit.SHA == "" { + return "", fmt.Errorf("failed to find commit in response of %s, got empty string", url) + } + + return commit.SHA, nil +} + +func Load(urlName string) (string, *loader.Repo, bool, error) { + if !strings.HasPrefix(urlName, GithubPrefix) { + return "", nil, false, nil + } + + url, ref, _ := strings.Cut(urlName, "@") + if ref == "" { + ref = "HEAD" + } + + parts := strings.Split(url, "/") + // Must be at least 4 parts github.com/ACCOUNT/REPO/FILE + if len(parts) < 4 { + return "", nil, false, nil + } + + account, repo := parts[1], parts[2] + path := strings.Join(parts[3:], "/") + + if path == "" || path == "/" { + path = "tool.gpt" + } else if !strings.HasSuffix(path, system.Suffix) { + path += "/tool.gpt" + } + + ref, err := getCommit(account, repo, ref) + if err != nil { + return "", nil, false, err + } + + downloadURL := fmt.Sprintf(githubDownloadURL, account, repo, ref, path) + return downloadURL, &loader.Repo{ + VCS: "github", + Root: fmt.Sprintf(githubRepoURL, account, repo), + Path: filepath.Dir(path), + Name: filepath.Base(path), + Revision: ref, + }, true, nil +} diff --git a/pkg/loader/loader.go b/pkg/loader/loader.go index 22856290..141b901b 100644 --- a/pkg/loader/loader.go +++ b/pkg/loader/loader.go @@ -11,8 +11,6 @@ import ( "fmt" "io" "io/fs" - "net/http" - url2 "net/url" "os" "path/filepath" "regexp" @@ -22,20 +20,37 @@ import ( "github.com/gptscript-ai/gptscript/pkg/builtin" "github.com/gptscript-ai/gptscript/pkg/engine" "github.com/gptscript-ai/gptscript/pkg/parser" + "github.com/gptscript-ai/gptscript/pkg/system" "github.com/gptscript-ai/gptscript/pkg/types" ) -const ( - GithubPrefix = "github.com/" - githubRawURL = "https://raw.githubusercontent.com/" -) - type source struct { + // Content The content of the source Content io.ReadCloser - Remote bool - Path string - Name string - File string + // Remote indicates that this file was loaded from a remote source (not local disk) + Remote bool + // Path is the path of this source used to find any relative references to this source + Path string + // Name is the filename of this source, it does not include the path in it + Name string + // Location is a string representation representing the source. It's not assume to + // be a valid URI or URL, used primarily for display. + Location string + // Repo The VCS repo where this tool was found, used to clone and provide the local tool code content + Repo *Repo +} + +type Repo struct { + // VCS The VCS type, such as "github" + VCS string + // The URL where the VCS repo can be found + Root string + // The path in the repo of this source. This should refer to a directory and not the actual file + Path string + // The filename of the source in the repo, relative to Path + Name string + // The revision of this source + Revision string } func (s *source) String() string { @@ -67,74 +82,11 @@ func loadLocal(base *source, name string) (*source, bool, error) { log.Debugf("opened %s", path) return &source{ - Content: content, - Remote: false, - Path: filepath.Dir(path), - Name: filepath.Base(path), - File: path, - }, true, nil -} - -func githubURL(urlName string) (string, bool) { - if !strings.HasPrefix(urlName, GithubPrefix) { - return "", false - } - - url, version, _ := strings.Cut(urlName, "@") - if version == "" { - version = "HEAD" - } - - parts := strings.Split(url, "/") - // Must be at least 4 parts github.com/ACCOUNT/REPO/FILE - if len(parts) < 4 { - return "", false - } - - url = githubRawURL + parts[1] + "/" + parts[2] + "/" + version + "/" + strings.Join(parts[3:], "/") - return url, true -} - -func loadURL(ctx context.Context, base *source, name string) (*source, bool, error) { - url := name - if base.Path != "" { - url = base.Path + "/" + name - } - if githubURL, ok := githubURL(url); ok { - url = githubURL - } - if !strings.HasPrefix(url, "http://") && !strings.HasPrefix(url, "https://") { - return nil, false, nil - } - - req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) - if err != nil { - return nil, false, err - } - - resp, err := http.DefaultClient.Do(req) - if err != nil { - return nil, false, err - } else if resp.StatusCode != http.StatusOK { - return nil, false, fmt.Errorf("error loading %s: %s", url, resp.Status) - } - - log.Debugf("opened %s", url) - - parsed, err := url2.Parse(url) - if err != nil { - return nil, false, err - } - - pathURL := *parsed - pathURL.Path = filepath.Dir(parsed.Path) - - return &source{ - Content: resp.Body, - Remote: true, - Path: pathURL.String(), - Name: filepath.Base(parsed.Path), - File: url, + Content: content, + Remote: false, + Path: filepath.Dir(path), + Name: filepath.Base(path), + Location: path, }, true, nil } @@ -201,7 +153,7 @@ func readTool(ctx context.Context, prg *types.Program, base *source, targetToolN for i, tool := range tools { tool.WorkingDir = base.Path - tool.Source.File = base.File + tool.Source.Location = base.Location // Probably a better way to come up with an ID tool.ID = tool.Source.String() @@ -211,7 +163,7 @@ func readTool(ctx context.Context, prg *types.Program, base *source, targetToolN } if i != 0 && tool.Parameters.Name == "" { - return types.Tool{}, parser.NewErrLine(tool.Source.File, tool.Source.LineNo, fmt.Errorf("only the first tool in a file can have no name")) + return types.Tool{}, parser.NewErrLine(tool.Source.Location, tool.Source.LineNo, fmt.Errorf("only the first tool in a file can have no name")) } if targetToolName != "" && tool.Parameters.Name == targetToolName { @@ -219,8 +171,8 @@ func readTool(ctx context.Context, prg *types.Program, base *source, targetToolN } if existing, ok := localTools[tool.Parameters.Name]; ok { - return types.Tool{}, parser.NewErrLine(tool.Source.File, tool.Source.LineNo, - fmt.Errorf("duplicate tool name [%s] in %s found at lines %d and %d", tool.Parameters.Name, tool.Source.File, + return types.Tool{}, parser.NewErrLine(tool.Source.Location, tool.Source.LineNo, + fmt.Errorf("duplicate tool name [%s] in %s found at lines %d and %d", tool.Parameters.Name, tool.Source.Location, tool.Source.LineNo, existing.Source.LineNo)) } @@ -238,7 +190,7 @@ var ( func ToolNormalizer(tool string) string { parts := strings.Split(tool, "/") tool = parts[len(parts)-1] - if strings.HasSuffix(tool, ".gpt") { + if strings.HasSuffix(tool, system.Suffix) { tool = strings.TrimSuffix(tool, filepath.Ext(tool)) } @@ -349,8 +301,8 @@ func ProgramFromSource(ctx context.Context, content, subToolName string) (types. ToolSet: types.ToolSet{}, } tool, err := readTool(ctx, &prg, &source{ - Content: io.NopCloser(strings.NewReader(content)), - File: "inline", + Content: io.NopCloser(strings.NewReader(content)), + Location: "inline", }, subToolName) if err != nil { return types.Program{}, err diff --git a/pkg/loader/url.go b/pkg/loader/url.go new file mode 100644 index 00000000..896d30ff --- /dev/null +++ b/pkg/loader/url.go @@ -0,0 +1,88 @@ +package loader + +import ( + "context" + "fmt" + "net/http" + url2 "net/url" + "path/filepath" + "strings" +) + +type VCSLookup func(string) (string, *Repo, bool, error) + +var vcsLookups []VCSLookup + +func AddVSC(lookup VCSLookup) { + vcsLookups = append(vcsLookups, lookup) +} + +func loadURL(ctx context.Context, base *source, name string) (*source, bool, error) { + var ( + repo *Repo + url = name + ) + + if base.Path != "" { + url = base.Path + "/" + name + } + + if base.Repo != nil { + newRepo := *base.Repo + newPath := filepath.Join(newRepo.Path, name) + newRepo.Path = filepath.Dir(newPath) + newRepo.Name = filepath.Base(newPath) + repo = &newRepo + } + + if repo == nil { + for _, vcs := range vcsLookups { + newURL, newRepo, ok, err := vcs(url) + if err != nil { + return nil, false, err + } else if ok { + repo = newRepo + url = newURL + break + } + } + } + + if !strings.HasPrefix(url, "http://") && !strings.HasPrefix(url, "https://") { + return nil, false, nil + } + + parsed, err := url2.Parse(url) + if err != nil { + return nil, false, err + } + + pathURL := *parsed + pathURL.Path = filepath.Dir(parsed.Path) + path := pathURL.String() + name = filepath.Base(parsed.Path) + url = path + "/" + name + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, false, err + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, false, err + } else if resp.StatusCode != http.StatusOK { + return nil, false, fmt.Errorf("error loading %s: %s", url, resp.Status) + } + + log.Debugf("opened %s", url) + + return &source{ + Content: resp.Body, + Remote: true, + Path: path, + Name: name, + Location: url, + Repo: repo, + }, true, nil +} diff --git a/pkg/loader/vcs/init.go b/pkg/loader/vcs/init.go new file mode 100644 index 00000000..4456b52d --- /dev/null +++ b/pkg/loader/vcs/init.go @@ -0,0 +1,6 @@ +package vcs + +import ( + // Load all VCS + _ "github.com/gptscript-ai/gptscript/pkg/loader/github" +) diff --git a/pkg/monitor/display.go b/pkg/monitor/display.go index cecebb72..c315633c 100644 --- a/pkg/monitor/display.go +++ b/pkg/monitor/display.go @@ -343,7 +343,7 @@ func (c callName) String() string { tool := c.prg.ToolSet[currentCall.ToolID] name := tool.Parameters.Name if name == "" { - name = tool.Source.File + name = tool.Source.Location } if currentCall.ID != "1" { name += "(" + c.prettyID + ")" diff --git a/pkg/server/server.go b/pkg/server/server.go index e61b7dc7..d24a8d06 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -20,6 +20,7 @@ import ( "github.com/gptscript-ai/gptscript/pkg/engine" "github.com/gptscript-ai/gptscript/pkg/loader" "github.com/gptscript-ai/gptscript/pkg/runner" + "github.com/gptscript-ai/gptscript/pkg/system" "github.com/gptscript-ai/gptscript/pkg/types" "github.com/gptscript-ai/gptscript/static" "github.com/olahol/melody" @@ -99,7 +100,7 @@ func (s *Server) list(rw http.ResponseWriter, req *http.Request) { if req.URL.Path == "/sys" { _ = enc.Encode(builtin.SysProgram()) return - } else if strings.HasSuffix(path, ".gpt") { + } else if strings.HasSuffix(path, system.Suffix) { prg, err := loader.Program(req.Context(), path, req.URL.Query().Get("tool")) if err != nil { http.Error(rw, err.Error(), http.StatusInternalServerError) @@ -121,7 +122,7 @@ func (s *Server) list(rw http.ResponseWriter, req *http.Request) { return nil } - if !d.IsDir() && strings.HasSuffix(d.Name(), ".gpt") { + if !d.IsDir() && strings.HasSuffix(d.Name(), system.Suffix) { result = append(result, path) } @@ -137,8 +138,8 @@ func (s *Server) list(rw http.ResponseWriter, req *http.Request) { func (s *Server) run(rw http.ResponseWriter, req *http.Request) { path := filepath.Join(".", req.URL.Path) - if !strings.HasSuffix(path, ".gpt") { - path += ".gpt" + if !strings.HasSuffix(path, system.Suffix) { + path += system.Suffix } prg, err := loader.Program(req.Context(), path, req.URL.Query().Get("tool")) diff --git a/pkg/system/prompt.go b/pkg/system/prompt.go index 4de97b85..4044092e 100644 --- a/pkg/system/prompt.go +++ b/pkg/system/prompt.go @@ -8,6 +8,9 @@ import ( "github.com/gptscript-ai/gptscript/pkg/types" ) +// Suffix is default suffix of gptscript files +const Suffix = ".gpt" + // InternalSystemPrompt is added to all threads. Changing this is very dangerous as it has a // terrible global effect and changes the behavior of all scripts. var InternalSystemPrompt = ` diff --git a/pkg/types/tool.go b/pkg/types/tool.go index 5cf581a6..3c4e6623 100644 --- a/pkg/types/tool.go +++ b/pkg/types/tool.go @@ -97,12 +97,12 @@ func (t Tool) String() string { } type ToolSource struct { - File string `json:"file,omitempty"` - LineNo int `json:"lineNo,omitempty"` + Location string `json:"location,omitempty"` + LineNo int `json:"lineNo,omitempty"` } func (t ToolSource) String() string { - return fmt.Sprintf("%s:%d", t.File, t.LineNo) + return fmt.Sprintf("%s:%d", t.Location, t.LineNo) } func (t Tool) IsCommand() bool {