From e20b3c38e4f446b31b386ef5df7c9db093af3f58 Mon Sep 17 00:00:00 2001 From: Darren Shepherd Date: Sat, 9 Mar 2024 00:04:52 -0700 Subject: [PATCH] feat: Add golang tool support --- pkg/engine/cmd.go | 2 +- pkg/repos/download/extract.go | 2 + pkg/repos/runtimes/default.go | 4 + pkg/repos/runtimes/golang/digests.txt | 10 ++ pkg/repos/runtimes/golang/golang.go | 119 +++++++++++++++++++++ pkg/repos/runtimes/golang/golang_test.go | 35 ++++++ pkg/repos/runtimes/golang/log.go | 5 + pkg/repos/runtimes/golang/testdata/go.mod | 3 + pkg/repos/runtimes/golang/testdata/main.go | 7 ++ 9 files changed, 186 insertions(+), 1 deletion(-) create mode 100644 pkg/repos/runtimes/golang/digests.txt create mode 100644 pkg/repos/runtimes/golang/golang.go create mode 100644 pkg/repos/runtimes/golang/golang_test.go create mode 100644 pkg/repos/runtimes/golang/log.go create mode 100644 pkg/repos/runtimes/golang/testdata/go.mod create mode 100644 pkg/repos/runtimes/golang/testdata/main.go diff --git a/pkg/engine/cmd.go b/pkg/engine/cmd.go index 03fc0fb4..59b1cf03 100644 --- a/pkg/engine/cmd.go +++ b/pkg/engine/cmd.go @@ -57,7 +57,6 @@ func (e *Engine) runCommand(ctx context.Context, tool types.Tool, input string) output := &bytes.Buffer{} all := &bytes.Buffer{} - cmd.Stdin = strings.NewReader(input) cmd.Stderr = io.MultiWriter(all, os.Stderr) cmd.Stdout = io.MultiWriter(all, output) @@ -142,6 +141,7 @@ func appendInputAsEnv(env []string, input string) []string { } } + env = appendEnv(env, "GPTSCRIPT_INPUT", input) return env } diff --git a/pkg/repos/download/extract.go b/pkg/repos/download/extract.go index 3ad6b053..448d04c8 100644 --- a/pkg/repos/download/extract.go +++ b/pkg/repos/download/extract.go @@ -30,6 +30,8 @@ func Extract(ctx context.Context, downloadURL, digest, targetDir string) error { } defer resp.Body.Close() + // NOTE: Because I'm validating the hash at the same time as extracting this isn't actually secure. + // Security is still assumed the source is trusted. Which is bad and should be changed. digester := sha256.New() input := io.TeeReader(resp.Body, digester) diff --git a/pkg/repos/runtimes/default.go b/pkg/repos/runtimes/default.go index e97a5122..d37cca8f 100644 --- a/pkg/repos/runtimes/default.go +++ b/pkg/repos/runtimes/default.go @@ -3,6 +3,7 @@ package runtimes import ( "github.com/gptscript-ai/gptscript/pkg/engine" "github.com/gptscript-ai/gptscript/pkg/repos" + "github.com/gptscript-ai/gptscript/pkg/repos/runtimes/golang" "github.com/gptscript-ai/gptscript/pkg/repos/runtimes/node" "github.com/gptscript-ai/gptscript/pkg/repos/runtimes/python" ) @@ -22,6 +23,9 @@ var Runtimes = []repos.Runtime{ Version: "21", Default: true, }, + &golang.Runtime{ + Version: "1.22.1", + }, } func Default(cacheDir string) engine.RuntimeManager { diff --git a/pkg/repos/runtimes/golang/digests.txt b/pkg/repos/runtimes/golang/digests.txt new file mode 100644 index 00000000..8a1b82c6 --- /dev/null +++ b/pkg/repos/runtimes/golang/digests.txt @@ -0,0 +1,10 @@ +3bc971772f4712fec0364f4bc3de06af22a00a12daab10b6f717fdcd13156cc0 go1.22.1.darwin-amd64.tar.gz +943e4f9f038239f9911c44366f52ab9202f6ee13610322a668fe42406fb3deef go1.22.1.darwin-amd64.pkg +f6a9cec6b8a002fcc9c0ee24ec04d67f430a52abc3cfd613836986bcc00d8383 go1.22.1.darwin-arm64.tar.gz +5f10b95e2678618f85ba9d87fbed506b3b87efc9d5a8cafda939055cb97949ba go1.22.1.darwin-arm64.pkg +8484df36d3d40139eaf0fe5e647b006435d826cc12f9ae72973bf7ec265e0ae4 go1.22.1.linux-386.tar.gz +aab8e15785c997ae20f9c88422ee35d962c4562212bb0f879d052a35c8307c7f go1.22.1.linux-amd64.tar.gz +e56685a245b6a0c592fc4a55f0b7803af5b3f827aaa29feab1f40e491acf35b8 go1.22.1.linux-arm64.tar.gz +8cb7a90e48c20daed39a6ac8b8a40760030ba5e93c12274c42191d868687c281 go1.22.1.linux-armv6l.tar.gz +0c5ebb7eb39b7884ec99f92b425d4c03a96a72443562aafbf6e7d15c42a3108a go1.22.1.windows-386.zip +cf9c66a208a106402a527f5b956269ca506cfe535fc388e828d249ea88ed28ba go1.22.1.windows-amd64.zip diff --git a/pkg/repos/runtimes/golang/golang.go b/pkg/repos/runtimes/golang/golang.go new file mode 100644 index 00000000..e809d49e --- /dev/null +++ b/pkg/repos/runtimes/golang/golang.go @@ -0,0 +1,119 @@ +package golang + +import ( + "bufio" + "bytes" + "context" + _ "embed" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "runtime" + "strings" + + "github.com/gptscript-ai/gptscript/pkg/debugcmd" + "github.com/gptscript-ai/gptscript/pkg/hash" + "github.com/gptscript-ai/gptscript/pkg/repos/download" + runtimeEnv "github.com/gptscript-ai/gptscript/pkg/repos/runtimes/env" +) + +//go:embed digests.txt +var releasesData []byte + +const downloadURL = "https://go.dev/dl/" + +type Runtime struct { + // version something like "1.22.1" + Version string +} + +func (r *Runtime) ID() string { + return "go" + r.Version +} + +func (r *Runtime) Supports(cmd []string) bool { + return len(cmd) > 0 && cmd[0] == "${GPTSCRIPT_TOOL_DIR}/bin/gptscript-go-tool" +} + +func (r *Runtime) Setup(ctx context.Context, dataRoot, toolSource string, env []string) ([]string, error) { + binPath, err := r.getRuntime(ctx, dataRoot) + if err != nil { + return nil, err + } + + newEnv := runtimeEnv.AppendPath(env, binPath) + if err := r.runBuild(ctx, toolSource, binPath, append(env, newEnv...)); err != nil { + return nil, err + } + + return newEnv, nil +} + +func (r *Runtime) getReleaseAndDigest() (string, string, error) { + scanner := bufio.NewScanner(bytes.NewReader(releasesData)) + key := r.ID() + "." + runtime.GOOS + "-" + runtime.GOARCH + for scanner.Scan() { + line := strings.Split(scanner.Text(), " ") + file, digest := strings.TrimSpace(line[1]), strings.TrimSpace(line[0]) + if strings.HasPrefix(file, key) { + return downloadURL + file, digest, nil + } + } + + return "", "", fmt.Errorf("failed to find %s release for os=%s arch=%s", r.ID(), runtime.GOOS, runtime.GOARCH) +} + +func stripGo(env []string) (result []string) { + for _, env := range env { + if strings.HasPrefix(env, "GO") { + continue + } + result = append(result, env) + } + return +} + +func (r *Runtime) runBuild(ctx context.Context, toolSource, binDir string, env []string) error { + cmd := debugcmd.New(ctx, filepath.Join(binDir, "go"), "build", "-o", "bin/gptscript-go-tool") + cmd.Env = stripGo(env) + cmd.Dir = toolSource + return cmd.Run() +} + +func (r *Runtime) binDir(rel string) string { + return filepath.Join(rel, "go", "bin") +} + +func (r *Runtime) getRuntime(ctx context.Context, cwd string) (string, error) { + url, sha, err := r.getReleaseAndDigest() + if err != nil { + return "", err + } + + target := filepath.Join(cwd, "golang", hash.ID(url, sha)) + if _, err := os.Stat(target); err == nil { + return r.binDir(target), nil + } else if !errors.Is(err, fs.ErrNotExist) { + return "", err + } + + log.Infof("Downloading Go %s", r.Version) + tmp := target + ".download" + defer os.RemoveAll(tmp) + + if err := os.MkdirAll(tmp, 0755); err != nil { + return "", err + } + + if err := download.Extract(ctx, url, sha, tmp); err != nil { + return "", err + } + + if err := os.Rename(tmp, target); err != nil { + return "", err + } + + return r.binDir(target), nil +} diff --git a/pkg/repos/runtimes/golang/golang_test.go b/pkg/repos/runtimes/golang/golang_test.go new file mode 100644 index 00000000..e559e833 --- /dev/null +++ b/pkg/repos/runtimes/golang/golang_test.go @@ -0,0 +1,35 @@ +package golang + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/adrg/xdg" + "github.com/samber/lo" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +var ( + testCacheHome = lo.Must(xdg.CacheFile("gptscript-test-cache/runtime")) +) + +func TestRuntime(t *testing.T) { + t.Cleanup(func() { + os.RemoveAll("testdata/bin") + }) + r := Runtime{ + Version: "1.22.1", + } + + s, err := r.Setup(context.Background(), testCacheHome, "testdata", os.Environ()) + require.NoError(t, err) + p, v, _ := strings.Cut(s[0], "=") + v, _, _ = strings.Cut(v, string(filepath.ListSeparator)) + assert.Equal(t, "PATH", p) + _, err = os.Stat(filepath.Join(v, "gofmt")) + assert.NoError(t, err) +} diff --git a/pkg/repos/runtimes/golang/log.go b/pkg/repos/runtimes/golang/log.go new file mode 100644 index 00000000..69cca8c2 --- /dev/null +++ b/pkg/repos/runtimes/golang/log.go @@ -0,0 +1,5 @@ +package golang + +import "github.com/gptscript-ai/gptscript/pkg/mvl" + +var log = mvl.Package() diff --git a/pkg/repos/runtimes/golang/testdata/go.mod b/pkg/repos/runtimes/golang/testdata/go.mod new file mode 100644 index 00000000..93eac7fe --- /dev/null +++ b/pkg/repos/runtimes/golang/testdata/go.mod @@ -0,0 +1,3 @@ +module example.com + +go 1.22.1 diff --git a/pkg/repos/runtimes/golang/testdata/main.go b/pkg/repos/runtimes/golang/testdata/main.go new file mode 100644 index 00000000..00f2021b --- /dev/null +++ b/pkg/repos/runtimes/golang/testdata/main.go @@ -0,0 +1,7 @@ +package main + +import "fmt" + +func main() { + fmt.Println("Hello AI World!") +}