From 8f3b5249a9b8583b26606a0f42152513de311310 Mon Sep 17 00:00:00 2001 From: Darren Shepherd Date: Sun, 2 Jun 2024 22:46:37 -0700 Subject: [PATCH] feat: add basic bash support for windows --- pkg/engine/cmd.go | 12 ++- pkg/repos/download/extract.go | 14 +++ pkg/repos/get.go | 47 +++++++-- pkg/repos/runtimes/busybox/SHASUMS256.txt | 1 + pkg/repos/runtimes/busybox/busybox.go | 107 ++++++++++++++++++++ pkg/repos/runtimes/busybox/busybox_test.go | 41 ++++++++ pkg/repos/runtimes/busybox/log.go | 5 + pkg/repos/runtimes/default.go | 2 + pkg/tests/runner_test.go | 3 - pkg/tests/testdata/TestContextArg/other.gpt | 4 +- pkg/tests/testdata/TestContextArg/test.gpt | 2 +- pkg/tests/tester/runner.go | 12 ++- 12 files changed, 232 insertions(+), 18 deletions(-) create mode 100644 pkg/repos/runtimes/busybox/SHASUMS256.txt create mode 100644 pkg/repos/runtimes/busybox/busybox.go create mode 100644 pkg/repos/runtimes/busybox/busybox_test.go create mode 100644 pkg/repos/runtimes/busybox/log.go diff --git a/pkg/engine/cmd.go b/pkg/engine/cmd.go index 9e4b94fc..60965340 100644 --- a/pkg/engine/cmd.go +++ b/pkg/engine/cmd.go @@ -8,6 +8,7 @@ import ( "io" "os" "os/exec" + "path" "path/filepath" "runtime" "sort" @@ -172,10 +173,9 @@ var ignoreENV = map[string]struct{}{ } func appendEnv(envs []string, k, v string) []string { - for _, k := range []string{k, env.ToEnvLike(k)} { - if _, ignore := ignoreENV[k]; !ignore { - envs = append(envs, k+"="+v) - } + k = env.ToEnvLike(k) + if _, ignore := ignoreENV[k]; !ignore { + envs = append(envs, k+"="+v) } return envs } @@ -238,6 +238,10 @@ func (e *Engine) newCommand(ctx context.Context, extraEnv []string, tool types.T }) } + if runtime.GOOS == "windows" && (args[0] == "/bin/bash" || args[0] == "/bin/sh") { + args[0] = path.Base(args[0]) + } + if runtime.GOOS == "windows" && (args[0] == "/usr/bin/env" || args[0] == "/bin/env") { args = args[1:] } diff --git a/pkg/repos/download/extract.go b/pkg/repos/download/extract.go index 95a82d74..4cf09f0c 100644 --- a/pkg/repos/download/extract.go +++ b/pkg/repos/download/extract.go @@ -9,7 +9,9 @@ import ( "net/http" "net/url" "os" + "path" "path/filepath" + "strings" "time" "github.com/mholt/archiver/v4" @@ -60,6 +62,18 @@ func Extract(ctx context.Context, downloadURL, digest, targetDir string) error { return err } + bin := path.Base(parsedURL.Path) + if strings.HasSuffix(bin, ".exe") { + dst, err := os.Create(filepath.Join(targetDir, bin)) + if err != nil { + return err + } + defer dst.Close() + + _, err = io.Copy(dst, tmpFile) + return err + } + format, input, err := archiver.Identify(filepath.Base(parsedURL.Path), tmpFile) if err != nil { return err diff --git a/pkg/repos/get.go b/pkg/repos/get.go index 2f96d8b3..6c66f9ff 100644 --- a/pkg/repos/get.go +++ b/pkg/repos/get.go @@ -15,6 +15,7 @@ import ( "github.com/BurntSushi/locker" "github.com/gptscript-ai/gptscript/pkg/config" "github.com/gptscript-ai/gptscript/pkg/credentials" + "github.com/gptscript-ai/gptscript/pkg/hash" "github.com/gptscript-ai/gptscript/pkg/loader/github" "github.com/gptscript-ai/gptscript/pkg/repos/git" "github.com/gptscript-ai/gptscript/pkg/repos/runtimes/golang" @@ -51,6 +52,7 @@ type Manager struct { credHelperDirs credentials.CredentialHelperDirs runtimes []Runtime credHelperConfig *credHelperConfig + supportLocal bool } type credHelperConfig struct { @@ -58,6 +60,14 @@ type credHelperConfig struct { initialized bool cliCfg *config.CLIConfig env []string + storageDir string + gitDir string + runtimeDir string + runtimes []Runtime +} + +func (m *Manager) SetSupportLocal() { + m.supportLocal = true } func New(cacheDir string, runtimes ...Runtime) *Manager { @@ -200,8 +210,14 @@ func (m *Manager) setup(ctx context.Context, runtime Runtime, tool types.Tool, e _ = os.RemoveAll(doneFile) _ = os.RemoveAll(target) - if err := git.Checkout(ctx, m.gitDir, tool.Source.Repo.Root, tool.Source.Repo.Revision, target); err != nil { - return "", nil, err + if tool.Source.Repo.VCS == "git" { + if err := git.Checkout(ctx, m.gitDir, tool.Source.Repo.Root, tool.Source.Repo.Revision, target); err != nil { + return "", nil, err + } + } else { + if err := os.MkdirAll(target, 0755); err != nil { + return "", nil, err + } } newEnv, err := runtime.Setup(ctx, m.runtimeDir, targetFinal, env) @@ -227,12 +243,25 @@ func (m *Manager) setup(ctx context.Context, runtime Runtime, tool types.Tool, e } func (m *Manager) GetContext(ctx context.Context, tool types.Tool, cmd, env []string) (string, []string, error) { - if tool.Source.Repo == nil { - return tool.WorkingDir, env, nil - } + var isLocal bool + if !m.supportLocal { + if tool.Source.Repo == nil { + return tool.WorkingDir, env, nil + } - if tool.Source.Repo.VCS != "git" { - return "", nil, fmt.Errorf("only git is supported, found VCS %s for %s", tool.Source.Repo.VCS, tool.ID) + if tool.Source.Repo.VCS != "git" { + return "", nil, fmt.Errorf("only git is supported, found VCS %s for %s", tool.Source.Repo.VCS, tool.ID) + } + } else if tool.Source.Repo == nil { + isLocal = true + id := hash.Digest(tool)[:12] + tool.Source.Repo = &types.Repo{ + VCS: "", + Root: id, + Path: "/", + Name: id, + Revision: id, + } } for _, runtime := range m.runtimes { @@ -242,5 +271,9 @@ func (m *Manager) GetContext(ctx context.Context, tool types.Tool, cmd, env []st } } + if isLocal { + return tool.WorkingDir, env, nil + } + return m.setup(ctx, &noopRuntime{}, tool, env) } diff --git a/pkg/repos/runtimes/busybox/SHASUMS256.txt b/pkg/repos/runtimes/busybox/SHASUMS256.txt new file mode 100644 index 00000000..7da1aff6 --- /dev/null +++ b/pkg/repos/runtimes/busybox/SHASUMS256.txt @@ -0,0 +1 @@ +6d2dfd1c1412c3550a89071a1b36a6f6073844320e687343d1dfc72719ecb8d9 FRP-5301-gda71f7c57/busybox-w64-FRP-5301-gda71f7c57.exe \ No newline at end of file diff --git a/pkg/repos/runtimes/busybox/busybox.go b/pkg/repos/runtimes/busybox/busybox.go new file mode 100644 index 00000000..b0c00a0c --- /dev/null +++ b/pkg/repos/runtimes/busybox/busybox.go @@ -0,0 +1,107 @@ +package busybox + +import ( + "bufio" + "bytes" + "context" + _ "embed" + "errors" + "fmt" + "io/fs" + "os" + "os/exec" + "path" + "path/filepath" + "runtime" + "strings" + + runtimeEnv "github.com/gptscript-ai/gptscript/pkg/env" + "github.com/gptscript-ai/gptscript/pkg/hash" + "github.com/gptscript-ai/gptscript/pkg/repos/download" +) + +//go:embed SHASUMS256.txt +var releasesData []byte + +const downloadURL = "https://github.com/gptscript-ai/busybox-w32/releases/download/%s" + +type Runtime struct { +} + +func (r *Runtime) ID() string { + return "busybox" +} + +func (r *Runtime) Supports(cmd []string) bool { + if runtime.GOOS != "windows" { + return false + } + for _, bin := range []string{"bash", "sh", "/bin/sh", "/bin/bash"} { + if runtimeEnv.Matches(cmd, bin) { + return true + } + } + return false +} + +func (r *Runtime) Setup(ctx context.Context, dataRoot, _ string, env []string) ([]string, error) { + binPath, err := r.getRuntime(ctx, dataRoot) + if err != nil { + return nil, err + } + + newEnv := runtimeEnv.AppendPath(env, binPath) + return newEnv, nil +} + +func (r *Runtime) getReleaseAndDigest() (string, string, error) { + scanner := bufio.NewScanner(bytes.NewReader(releasesData)) + for scanner.Scan() { + line := scanner.Text() + fields := strings.Fields(line) + return fmt.Sprintf(downloadURL, fields[1]), fields[0], nil + } + + return "", "", fmt.Errorf("failed to find %s release", r.ID()) +} + +func (r *Runtime) getRuntime(ctx context.Context, cwd string) (string, error) { + url, sha, err := r.getReleaseAndDigest() + if err != nil { + return "", err + } + + target := filepath.Join(cwd, "busybox", hash.ID(url, sha)) + if _, err := os.Stat(target); err == nil { + return target, nil + } else if !errors.Is(err, fs.ErrNotExist) { + return "", err + } + + log.Infof("Downloading Busybox") + tmp := target + ".download" + defer os.RemoveAll(tmp) + + if err := os.MkdirAll(tmp, 0755); err != nil { + return "", err + } + + if err := download.Extract(ctx, url, sha, tmp); err != nil { + return "", err + } + + bbExe := filepath.Join(tmp, path.Base(url)) + + cmd := exec.Command(bbExe, "--install", ".") + cmd.Dir = filepath.Dir(bbExe) + + if err := cmd.Run(); err != nil { + return "", err + } + + if err := os.Rename(tmp, target); err != nil { + return "", err + } + + return target, nil +} diff --git a/pkg/repos/runtimes/busybox/busybox_test.go b/pkg/repos/runtimes/busybox/busybox_test.go new file mode 100644 index 00000000..f3add18a --- /dev/null +++ b/pkg/repos/runtimes/busybox/busybox_test.go @@ -0,0 +1,41 @@ +package busybox + +import ( + "context" + "errors" + "io/fs" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/adrg/xdg" + "github.com/samber/lo" + "github.com/stretchr/testify/require" +) + +var ( + testCacheHome = lo.Must(xdg.CacheFile("gptscript-test-cache/runtime")) +) + +func firstPath(s []string) string { + _, p, _ := strings.Cut(s[0], "=") + return strings.Split(p, string(os.PathListSeparator))[0] +} + +func TestRuntime(t *testing.T) { + if runtime.GOOS != "windows" { + t.Skip() + } + + r := Runtime{} + + s, err := r.Setup(context.Background(), testCacheHome, "testdata", os.Environ()) + require.NoError(t, err) + _, err = os.Stat(filepath.Join(firstPath(s), "busybox.exe")) + if errors.Is(err, fs.ErrNotExist) { + _, err = os.Stat(filepath.Join(firstPath(s), "busybox")) + } + require.NoError(t, err) +} diff --git a/pkg/repos/runtimes/busybox/log.go b/pkg/repos/runtimes/busybox/log.go new file mode 100644 index 00000000..b7e486f1 --- /dev/null +++ b/pkg/repos/runtimes/busybox/log.go @@ -0,0 +1,5 @@ +package busybox + +import "github.com/gptscript-ai/gptscript/pkg/mvl" + +var log = mvl.Package() diff --git a/pkg/repos/runtimes/default.go b/pkg/repos/runtimes/default.go index d37cca8f..3782e26e 100644 --- a/pkg/repos/runtimes/default.go +++ b/pkg/repos/runtimes/default.go @@ -3,12 +3,14 @@ package runtimes import ( "github.com/gptscript-ai/gptscript/pkg/engine" "github.com/gptscript-ai/gptscript/pkg/repos" + "github.com/gptscript-ai/gptscript/pkg/repos/runtimes/busybox" "github.com/gptscript-ai/gptscript/pkg/repos/runtimes/golang" "github.com/gptscript-ai/gptscript/pkg/repos/runtimes/node" "github.com/gptscript-ai/gptscript/pkg/repos/runtimes/python" ) var Runtimes = []repos.Runtime{ + &busybox.Runtime{}, &python.Runtime{ Version: "3.12", Default: true, diff --git a/pkg/tests/runner_test.go b/pkg/tests/runner_test.go index 70d5346c..f76bcd3e 100644 --- a/pkg/tests/runner_test.go +++ b/pkg/tests/runner_test.go @@ -748,9 +748,6 @@ func TestGlobalErr(t *testing.T) { } func TestContextArg(t *testing.T) { - if runtime.GOOS == "windows" { - t.Skip() - } runner := tester.NewRunner(t) x, err := runner.Run("", `{ "file": "foo.db" diff --git a/pkg/tests/testdata/TestContextArg/other.gpt b/pkg/tests/testdata/TestContextArg/other.gpt index b1acd66a..f97b4ba6 100644 --- a/pkg/tests/testdata/TestContextArg/other.gpt +++ b/pkg/tests/testdata/TestContextArg/other.gpt @@ -2,5 +2,5 @@ name: fromcontext args: first: an arg args: second: an arg -#!/bin/bash -echo this is from other context ${first} and then ${second} \ No newline at end of file +#!/usr/bin/env bash +echo this is from other context ${FIRST} and then ${SECOND} \ No newline at end of file diff --git a/pkg/tests/testdata/TestContextArg/test.gpt b/pkg/tests/testdata/TestContextArg/test.gpt index 9569aaf9..50d2ccf2 100644 --- a/pkg/tests/testdata/TestContextArg/test.gpt +++ b/pkg/tests/testdata/TestContextArg/test.gpt @@ -9,4 +9,4 @@ name: fromcontext args: first: an arg #!/bin/bash -echo this is from context -- ${first} \ No newline at end of file +echo this is from context -- ${FIRST} \ No newline at end of file diff --git a/pkg/tests/tester/runner.go b/pkg/tests/tester/runner.go index 775f0248..ef75c0f5 100644 --- a/pkg/tests/tester/runner.go +++ b/pkg/tests/tester/runner.go @@ -8,8 +8,11 @@ import ( "path/filepath" "testing" + "github.com/adrg/xdg" "github.com/gptscript-ai/gptscript/pkg/credentials" "github.com/gptscript-ai/gptscript/pkg/loader" + "github.com/gptscript-ai/gptscript/pkg/repos" + "github.com/gptscript-ai/gptscript/pkg/repos/runtimes" "github.com/gptscript-ai/gptscript/pkg/runner" "github.com/gptscript-ai/gptscript/pkg/types" "github.com/hexops/autogold/v2" @@ -171,8 +174,15 @@ func NewRunner(t *testing.T) *Runner { t: t, } + cacheDir, err := xdg.CacheFile("gptscript-test-cache/runtime") + require.NoError(t, err) + + rm := runtimes.Default(cacheDir) + rm.(*repos.Manager).SetSupportLocal() + run, err := runner.New(c, credentials.NoopStore{}, runner.Options{ - Sequential: true, + Sequential: true, + RuntimeManager: rm, }) require.NoError(t, err)