Add github metadata to internal data structures

main.go

@@ -3,6 +3,9 @@ package main
 import (
 	"github.com/acorn-io/cmd"
 	"github.com/gptscript-ai/gptscript/pkg/cli"
+
+	// Load all VCS
+	_ "github.com/gptscript-ai/gptscript/pkg/loader/vcs"
 )
 
 func main() {

pkg/builtin/builtin.go

@@ -216,6 +216,10 @@ func SysFind(ctx context.Context, env []string, input string) (string, error) {
 	if err != nil {
 		return "", nil
 	}
+	if len(result) == 0 {
+		return "No files found", nil
+	}
+
 	sort.Strings(result)
 	return strings.Join(result, "\n"), nil
 }
@@ -455,7 +459,11 @@ func SysStat(ctx context.Context, env []string, input string) (string, error) {
 		return "", err
 	}
 
-	return fmt.Sprintf("File %s mode: %s, size: %d bytes, modtime: %s", params.Filepath, stat.Mode().String(), stat.Size(), stat.ModTime().String()), nil
+	title := "File"
+	if stat.IsDir() {
+		title = "Directory"
+	}
+	return fmt.Sprintf("%s %s mode: %s, size: %d bytes, modtime: %s", title, params.Filepath, stat.Mode().String(), stat.Size(), stat.ModTime().String()), nil
 }
 
 func SysDownload(ctx context.Context, env []string, input string) (_ string, err error) {

pkg/loader/github/github.go

@@ -0,0 +1,91 @@
+package github
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"path/filepath"
+	"strings"
+
+	"github.com/gptscript-ai/gptscript/pkg/loader"
+	"github.com/gptscript-ai/gptscript/pkg/system"
+)
+
+const (
+	GithubPrefix      = "github.com/"
+	githubRepoURL     = "https://github.com/%s/%s.git"
+	githubDownloadURL = "https://raw.githubusercontent.com/%s/%s/%s/%s"
+	githubCommitURL   = "https://api.github.com/repos/%s/%s/commits/%s"
+)
+
+func init() {
+	loader.AddVSC(Load)
+}
+
+func getCommit(account, repo, ref string) (string, error) {
+	url := fmt.Sprintf(githubCommitURL, account, repo, ref)
+	resp, err := http.Get(url)
+	if err != nil {
+		return "", err
+	} else if resp.StatusCode != http.StatusOK {
+		c, _ := io.ReadAll(resp.Body)
+		resp.Body.Close()
+		return "", fmt.Errorf("failed to GitHub commit of %s/%s at %s: %s %s",
+			account, repo, ref, resp.Status, c)
+	}
+	defer resp.Body.Close()
+
+	var commit struct {
+		SHA string `json:"sha,omitempty"`
+	}
+	if err := json.NewDecoder(resp.Body).Decode(&commit); err != nil {
+		return "", fmt.Errorf("failed to decode GitHub commit of %s/%s at %s: %w", account, repo, url, err)
+	}
+
+	if commit.SHA == "" {
+		return "", fmt.Errorf("failed to find commit in response of %s, got empty string", url)
+	}
+
+	return commit.SHA, nil
+}
+
+func Load(urlName string) (string, *loader.Repo, bool, error) {
+	if !strings.HasPrefix(urlName, GithubPrefix) {
+		return "", nil, false, nil
+	}
+
+	url, ref, _ := strings.Cut(urlName, "@")
+	if ref == "" {
+		ref = "HEAD"
+	}
+
+	parts := strings.Split(url, "/")
+	// Must be at least 4 parts github.com/ACCOUNT/REPO/FILE
+	if len(parts) < 4 {
+		return "", nil, false, nil
+	}
+
+	account, repo := parts[1], parts[2]
+	path := strings.Join(parts[3:], "/")
+
+	if path == "" || path == "/" {
+		path = "tool.gpt"
+	} else if !strings.HasSuffix(path, system.Suffix) {
+		path += "/tool.gpt"
+	}
+
+	ref, err := getCommit(account, repo, ref)
+	if err != nil {
+		return "", nil, false, err
+	}
+
+	downloadURL := fmt.Sprintf(githubDownloadURL, account, repo, ref, path)
+	return downloadURL, &loader.Repo{
+		VCS:      "github",
+		Root:     fmt.Sprintf(githubRepoURL, account, repo),
+		Path:     filepath.Dir(path),
+		Name:     filepath.Base(path),
+		Revision: ref,
+	}, true, nil
+}

pkg/loader/loader.go

@@ -11,8 +11,6 @@ import (
 	"fmt"
 	"io"
 	"io/fs"
-	"net/http"
-	url2 "net/url"
 	"os"
 	"path/filepath"
 	"regexp"
@@ -22,20 +20,37 @@ import (
 	"github.com/gptscript-ai/gptscript/pkg/builtin"
 	"github.com/gptscript-ai/gptscript/pkg/engine"
 	"github.com/gptscript-ai/gptscript/pkg/parser"
+	"github.com/gptscript-ai/gptscript/pkg/system"
 	"github.com/gptscript-ai/gptscript/pkg/types"
 )
 
-const (
-	GithubPrefix = "github.com/"
-	githubRawURL = "https://raw.githubusercontent.com/"
-)
-
 type source struct {
+	// Content The content of the source
 	Content io.ReadCloser
-	Remote  bool
-	Path    string
-	Name    string
-	File    string
+	// Remote indicates that this file was loaded from a remote source (not local disk)
+	Remote bool
+	// Path is the path of this source used to find any relative references to this source
+	Path string
+	// Name is the filename of this source, it does not include the path in it
+	Name string
+	// Location is a string representation representing the source. It's not assume to
+	// be a valid URI or URL, used primarily for display.
+	Location string
+	// Repo The VCS repo where this tool was found, used to clone and provide the local tool code content
+	Repo *Repo
+}
+
+type Repo struct {
+	// VCS The VCS type, such as "github"
+	VCS string
+	// The URL where the VCS repo can be found
+	Root string
+	// The path in the repo of this source. This should refer to a directory and not the actual file
+	Path string
+	// The filename of the source in the repo, relative to Path
+	Name string
+	// The revision of this source
+	Revision string
 }
 
 func (s *source) String() string {
@@ -67,74 +82,11 @@ func loadLocal(base *source, name string) (*source, bool, error) {
 	log.Debugf("opened %s", path)
 
 	return &source{
-		Content: content,
-		Remote:  false,
-		Path:    filepath.Dir(path),
-		Name:    filepath.Base(path),
-		File:    path,
-	}, true, nil
-}
-
-func githubURL(urlName string) (string, bool) {
-	if !strings.HasPrefix(urlName, GithubPrefix) {
-		return "", false
-	}
-
-	url, version, _ := strings.Cut(urlName, "@")
-	if version == "" {
-		version = "HEAD"
-	}
-
-	parts := strings.Split(url, "/")
-	// Must be at least 4 parts github.com/ACCOUNT/REPO/FILE
-	if len(parts) < 4 {
-		return "", false
-	}
-
-	url = githubRawURL + parts[1] + "/" + parts[2] + "/" + version + "/" + strings.Join(parts[3:], "/")
-	return url, true
-}
-
-func loadURL(ctx context.Context, base *source, name string) (*source, bool, error) {
-	url := name
-	if base.Path != "" {
-		url = base.Path + "/" + name
-	}
-	if githubURL, ok := githubURL(url); ok {
-		url = githubURL
-	}
-	if !strings.HasPrefix(url, "http://") && !strings.HasPrefix(url, "https://") {
-		return nil, false, nil
-	}
-
-	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
-	if err != nil {
-		return nil, false, err
-	}
-
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return nil, false, err
-	} else if resp.StatusCode != http.StatusOK {
-		return nil, false, fmt.Errorf("error loading %s: %s", url, resp.Status)
-	}
-
-	log.Debugf("opened %s", url)
-
-	parsed, err := url2.Parse(url)
-	if err != nil {
-		return nil, false, err
-	}
-
-	pathURL := *parsed
-	pathURL.Path = filepath.Dir(parsed.Path)
-
-	return &source{
-		Content: resp.Body,
-		Remote:  true,
-		Path:    pathURL.String(),
-		Name:    filepath.Base(parsed.Path),
-		File:    url,
+		Content:  content,
+		Remote:   false,
+		Path:     filepath.Dir(path),
+		Name:     filepath.Base(path),
+		Location: path,
 	}, true, nil
 }
 
@@ -201,7 +153,7 @@ func readTool(ctx context.Context, prg *types.Program, base *source, targetToolN
 
 	for i, tool := range tools {
 		tool.WorkingDir = base.Path
-		tool.Source.File = base.File
+		tool.Source.Location = base.Location
 
 		// Probably a better way to come up with an ID
 		tool.ID = tool.Source.String()
@@ -211,16 +163,16 @@ func readTool(ctx context.Context, prg *types.Program, base *source, targetToolN
 		}
 
 		if i != 0 && tool.Parameters.Name == "" {
-			return types.Tool{}, parser.NewErrLine(tool.Source.File, tool.Source.LineNo, fmt.Errorf("only the first tool in a file can have no name"))
+			return types.Tool{}, parser.NewErrLine(tool.Source.Location, tool.Source.LineNo, fmt.Errorf("only the first tool in a file can have no name"))
 		}
 
 		if targetToolName != "" && tool.Parameters.Name == targetToolName {
 			mainTool = tool
 		}
 
 		if existing, ok := localTools[tool.Parameters.Name]; ok {
-			return types.Tool{}, parser.NewErrLine(tool.Source.File, tool.Source.LineNo,
-				fmt.Errorf("duplicate tool name [%s] in %s found at lines %d and %d", tool.Parameters.Name, tool.Source.File,
+			return types.Tool{}, parser.NewErrLine(tool.Source.Location, tool.Source.LineNo,
+				fmt.Errorf("duplicate tool name [%s] in %s found at lines %d and %d", tool.Parameters.Name, tool.Source.Location,
 					tool.Source.LineNo, existing.Source.LineNo))
 		}
 
@@ -238,7 +190,7 @@ var (
 func ToolNormalizer(tool string) string {
 	parts := strings.Split(tool, "/")
 	tool = parts[len(parts)-1]
-	if strings.HasSuffix(tool, ".gpt") {
+	if strings.HasSuffix(tool, system.Suffix) {
 		tool = strings.TrimSuffix(tool, filepath.Ext(tool))
 	}
 
@@ -349,8 +301,8 @@ func ProgramFromSource(ctx context.Context, content, subToolName string) (types.
 		ToolSet: types.ToolSet{},
 	}
 	tool, err := readTool(ctx, &prg, &source{
-		Content: io.NopCloser(strings.NewReader(content)),
-		File:    "inline",
+		Content:  io.NopCloser(strings.NewReader(content)),
+		Location: "inline",
 	}, subToolName)
 	if err != nil {
 		return types.Program{}, err

pkg/loader/url.go

@@ -0,0 +1,88 @@
+package loader
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	url2 "net/url"
+	"path/filepath"
+	"strings"
+)
+
+type VCSLookup func(string) (string, *Repo, bool, error)
+
+var vcsLookups []VCSLookup
+
+func AddVSC(lookup VCSLookup) {
+	vcsLookups = append(vcsLookups, lookup)
+}
+
+func loadURL(ctx context.Context, base *source, name string) (*source, bool, error) {
+	var (
+		repo *Repo
+		url  = name
+	)
+
+	if base.Path != "" {
+		url = base.Path + "/" + name
+	}
+
+	if base.Repo != nil {
+		newRepo := *base.Repo
+		newPath := filepath.Join(newRepo.Path, name)
+		newRepo.Path = filepath.Dir(newPath)
+		newRepo.Name = filepath.Base(newPath)
+		repo = &newRepo
+	}
+
+	if repo == nil {
+		for _, vcs := range vcsLookups {
+			newURL, newRepo, ok, err := vcs(url)
+			if err != nil {
+				return nil, false, err
+			} else if ok {
+				repo = newRepo
+				url = newURL
+				break
+			}
+		}
+	}
+
+	if !strings.HasPrefix(url, "http://") && !strings.HasPrefix(url, "https://") {
+		return nil, false, nil
+	}
+
+	parsed, err := url2.Parse(url)
+	if err != nil {
+		return nil, false, err
+	}
+
+	pathURL := *parsed
+	pathURL.Path = filepath.Dir(parsed.Path)
+	path := pathURL.String()
+	name = filepath.Base(parsed.Path)
+	url = path + "/" + name
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	if err != nil {
+		return nil, false, err
+	}
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return nil, false, err
+	} else if resp.StatusCode != http.StatusOK {
+		return nil, false, fmt.Errorf("error loading %s: %s", url, resp.Status)
+	}
+
+	log.Debugf("opened %s", url)
+
+	return &source{
+		Content:  resp.Body,
+		Remote:   true,
+		Path:     path,
+		Name:     name,
+		Location: url,
+		Repo:     repo,
+	}, true, nil
+}