From 62290963ddd73a4e86e3213a042393abf42ee499 Mon Sep 17 00:00:00 2001 From: Darren Shepherd Date: Wed, 13 Mar 2024 16:44:28 -0700 Subject: [PATCH] Windows path fixes --- pkg/engine/cmd.go | 19 +++++--- pkg/env/env.go | 73 +++++++++++++++++++++++++++++ pkg/loader/url.go | 18 +++---- pkg/repos/download/extract.go | 33 +++++++++---- pkg/repos/get.go | 4 ++ pkg/repos/runtimes/env/env.go | 40 ---------------- pkg/repos/runtimes/golang/golang.go | 2 +- pkg/repos/runtimes/node/node.go | 9 +++- pkg/repos/runtimes/python/python.go | 2 +- 9 files changed, 131 insertions(+), 69 deletions(-) create mode 100644 pkg/env/env.go delete mode 100644 pkg/repos/runtimes/env/env.go diff --git a/pkg/engine/cmd.go b/pkg/engine/cmd.go index 59b1cf03..5dc099a7 100644 --- a/pkg/engine/cmd.go +++ b/pkg/engine/cmd.go @@ -8,11 +8,13 @@ import ( "io" "os" "os/exec" + "runtime" "sort" "strings" "sync/atomic" "github.com/google/shlex" + "github.com/gptscript-ai/gptscript/pkg/env" "github.com/gptscript-ai/gptscript/pkg/types" "github.com/gptscript-ai/gptscript/pkg/version" ) @@ -103,6 +105,7 @@ func envAsMapAndDeDup(env []string) (sortedEnv []string, _ map[string]string) { var ignoreENV = map[string]struct{}{ "PATH": {}, + "Path": {}, "GPTSCRIPT_TOOL_DIR": {}, } @@ -146,8 +149,8 @@ func appendInputAsEnv(env []string, input string) []string { } func (e *Engine) newCommand(ctx context.Context, extraEnv []string, tool types.Tool, input string) (*exec.Cmd, func(), error) { - env := append(e.Env[:], extraEnv...) - env = appendInputAsEnv(env, input) + envvars := append(e.Env[:], extraEnv...) + envvars = appendInputAsEnv(envvars, input) interpreter, rest, _ := strings.Cut(tool.Instructions, "\n") interpreter = strings.TrimSpace(interpreter)[2:] @@ -157,18 +160,22 @@ func (e *Engine) newCommand(ctx context.Context, extraEnv []string, tool types.T return nil, nil, err } - env, err = e.getRuntimeEnv(ctx, tool, args, env) + envvars, err = e.getRuntimeEnv(ctx, tool, args, envvars) if err != nil { return nil, nil, err } - env, envMap := envAsMapAndDeDup(env) + envvars, envMap := envAsMapAndDeDup(envvars) for i, arg := range args { args[i] = os.Expand(arg, func(s string) string { return envMap[s] }) } + if runtime.GOOS == "windows" && (args[0] == "/usr/bin/env" || args[0] == "/bin/env") { + args = args[1:] + } + var ( cmdArgs = args[1:] stop = func() {} @@ -192,7 +199,7 @@ func (e *Engine) newCommand(ctx context.Context, extraEnv []string, tool types.T cmdArgs = append(cmdArgs, f.Name()) } - cmd := exec.CommandContext(ctx, args[0], cmdArgs...) - cmd.Env = env + cmd := exec.CommandContext(ctx, env.Lookup(envvars, args[0]), cmdArgs...) + cmd.Env = envvars return cmd, stop, nil } diff --git a/pkg/env/env.go b/pkg/env/env.go new file mode 100644 index 00000000..f1b71c5b --- /dev/null +++ b/pkg/env/env.go @@ -0,0 +1,73 @@ +package env + +import ( + "fmt" + "os" + "path/filepath" + "strings" +) + +func execEquals(bin, check string) bool { + return bin == check || + bin == check+".exe" +} + +func Matches(cmd []string, bin string) bool { + switch len(cmd) { + case 0: + return false + case 1: + return execEquals(cmd[0], bin) + } + if cmd[0] == bin { + return true + } + if cmd[0] == "/usr/bin/env" || cmd[0] == "/bin/env" { + return execEquals(cmd[1], bin) + } + return false +} + +func AppendPath(env []string, binPath string) []string { + var newEnv []string + for _, path := range env { + v, ok := strings.CutPrefix(path, "PATH=") + if ok { + newEnv = append(newEnv, fmt.Sprintf("PATH=%s%s%s", + binPath, string(os.PathListSeparator), v)) + } + } + return newEnv +} + +// Lookup will try to find bin in the PATH in env. It will refer to PATHEXT for Windows support. +// If bin can not be resolved to anything the original bin string is returned. +func Lookup(env []string, bin string) string { + for _, env := range env { + for _, prefix := range []string{"PATH=", "Path="} { + suffix, ok := strings.CutPrefix(env, prefix) + if !ok { + continue + } + for _, path := range strings.Split(suffix, string(os.PathListSeparator)) { + testPath := filepath.Join(path, bin) + + if stat, err := os.Stat(testPath); err == nil && !stat.IsDir() { + return testPath + } + + for _, ext := range strings.Split(os.Getenv("PATHEXT"), string(os.PathListSeparator)) { + if ext == "" { + continue + } + + if stat, err := os.Stat(testPath + ext); err == nil && !stat.IsDir() { + return testPath + ext + } + } + } + } + } + + return bin +} diff --git a/pkg/loader/url.go b/pkg/loader/url.go index 7d56b2ec..1762a235 100644 --- a/pkg/loader/url.go +++ b/pkg/loader/url.go @@ -5,7 +5,7 @@ import ( "fmt" "net/http" url2 "net/url" - "path/filepath" + "path" "strings" "github.com/gptscript-ai/gptscript/pkg/types" @@ -32,9 +32,9 @@ func loadURL(ctx context.Context, base *source, name string) (*source, bool, err if base.Repo != nil { newRepo := *base.Repo - newPath := filepath.Join(newRepo.Path, name) - newRepo.Path = filepath.Dir(newPath) - newRepo.Name = filepath.Base(newPath) + newPath := path.Join(newRepo.Path, name) + newRepo.Path = path.Dir(newPath) + newRepo.Name = path.Base(newPath) repo = &newRepo } @@ -61,10 +61,10 @@ func loadURL(ctx context.Context, base *source, name string) (*source, bool, err } pathURL := *parsed - pathURL.Path = filepath.Dir(parsed.Path) - path := pathURL.String() - name = filepath.Base(parsed.Path) - url = path + "/" + name + pathURL.Path = path.Dir(parsed.Path) + pathString := pathURL.String() + name = path.Base(parsed.Path) + url = pathString + "/" + name req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) if err != nil { @@ -83,7 +83,7 @@ func loadURL(ctx context.Context, base *source, name string) (*source, bool, err return &source{ Content: resp.Body, Remote: true, - Path: path, + Path: pathString, Name: name, Location: url, Repo: repo, diff --git a/pkg/repos/download/extract.go b/pkg/repos/download/extract.go index 448d04c8..95a82d74 100644 --- a/pkg/repos/download/extract.go +++ b/pkg/repos/download/extract.go @@ -24,23 +24,43 @@ func Extract(ctx context.Context, downloadURL, digest, targetDir string) error { return fmt.Errorf("mkdir %s: %w", targetDir, err) } + tmpFile, err := os.CreateTemp("", "gptscript-download") + if err != nil { + return err + } + defer os.Remove(tmpFile.Name()) + defer tmpFile.Close() + resp, err := http.Get(downloadURL) if err != nil { return err } defer resp.Body.Close() - // NOTE: Because I'm validating the hash at the same time as extracting this isn't actually secure. - // Security is still assumed the source is trusted. Which is bad and should be changed. digester := sha256.New() input := io.TeeReader(resp.Body, digester) + if _, err = io.Copy(tmpFile, input); err != nil { + return err + } + + resultDigest := digester.Sum(nil) + resultDigestString := hex.EncodeToString(resultDigest[:]) + + if resultDigestString != digest { + return fmt.Errorf("downloaded %s and expected digest %s but got %s", downloadURL, digest, resultDigestString) + } + parsedURL, err := url.Parse(downloadURL) if err != nil { return err } - format, input, err := archiver.Identify(filepath.Base(parsedURL.Path), input) + if _, err := tmpFile.Seek(0, 0); err != nil { + return err + } + + format, input, err := archiver.Identify(filepath.Base(parsedURL.Path), tmpFile) if err != nil { return err } @@ -90,12 +110,5 @@ func Extract(ctx context.Context, downloadURL, digest, targetDir string) error { return err } - resultDigest := digester.Sum(nil) - resultDigestString := hex.EncodeToString(resultDigest[:]) - - if resultDigestString != digest { - return fmt.Errorf("downloaded %s and expected digest %s but got %s", downloadURL, digest, resultDigestString) - } - return nil } diff --git a/pkg/repos/get.go b/pkg/repos/get.go index db96dc8a..9c5f100c 100644 --- a/pkg/repos/get.go +++ b/pkg/repos/get.go @@ -93,6 +93,10 @@ func (m *Manager) setup(ctx context.Context, runtime Runtime, tool types.Tool, e return "", nil, err } + if err := out.Close(); err != nil { + return "", nil, err + } + return targetFinal, append(env, newEnv...), os.Rename(doneFile+".tmp", doneFile) } diff --git a/pkg/repos/runtimes/env/env.go b/pkg/repos/runtimes/env/env.go deleted file mode 100644 index 5f52ba9d..00000000 --- a/pkg/repos/runtimes/env/env.go +++ /dev/null @@ -1,40 +0,0 @@ -package env - -import ( - "fmt" - "os" - "strings" -) - -func execEquals(bin, check string) bool { - return bin == check || - bin == check+".exe" -} - -func Matches(cmd []string, bin string) bool { - switch len(cmd) { - case 0: - return false - case 1: - return execEquals(cmd[0], bin) - } - if cmd[0] == bin { - return true - } - if cmd[0] == "/usr/bin/env" || cmd[0] == "/bin/env" { - return execEquals(cmd[1], bin) - } - return false -} - -func AppendPath(env []string, binPath string) []string { - var newEnv []string - for _, path := range env { - v, ok := strings.CutPrefix(path, "PATH=") - if ok { - newEnv = append(newEnv, fmt.Sprintf("PATH=%s%s%s", - binPath, string(os.PathListSeparator), v)) - } - } - return newEnv -} diff --git a/pkg/repos/runtimes/golang/golang.go b/pkg/repos/runtimes/golang/golang.go index 0bd890e3..7f8a8a29 100644 --- a/pkg/repos/runtimes/golang/golang.go +++ b/pkg/repos/runtimes/golang/golang.go @@ -14,9 +14,9 @@ import ( "strings" "github.com/gptscript-ai/gptscript/pkg/debugcmd" + runtimeEnv "github.com/gptscript-ai/gptscript/pkg/env" "github.com/gptscript-ai/gptscript/pkg/hash" "github.com/gptscript-ai/gptscript/pkg/repos/download" - runtimeEnv "github.com/gptscript-ai/gptscript/pkg/repos/runtimes/env" ) //go:embed digests.txt diff --git a/pkg/repos/runtimes/node/node.go b/pkg/repos/runtimes/node/node.go index 8b5a1c00..9280d3b0 100644 --- a/pkg/repos/runtimes/node/node.go +++ b/pkg/repos/runtimes/node/node.go @@ -14,9 +14,9 @@ import ( "strings" "github.com/gptscript-ai/gptscript/pkg/debugcmd" + runtimeEnv "github.com/gptscript-ai/gptscript/pkg/env" "github.com/gptscript-ai/gptscript/pkg/hash" "github.com/gptscript-ai/gptscript/pkg/repos/download" - runtimeEnv "github.com/gptscript-ai/gptscript/pkg/repos/runtimes/env" ) //go:embed SHASUMS256.txt.asc @@ -84,7 +84,7 @@ func arch() string { func (r *Runtime) getReleaseAndDigest() (string, string, error) { scanner := bufio.NewScanner(bytes.NewReader(releasesData)) - key := osName() + "-" + arch() + key := "-" + osName() + "-" + arch() for scanner.Scan() { line := scanner.Text() if strings.Contains(line, "node-v"+r.Version) && strings.Contains(line, key) { @@ -116,6 +116,11 @@ func (r *Runtime) binDir(rel string) (string, error) { for _, entry := range entries { if entry.IsDir() { + if _, err := os.Stat(filepath.Join(rel, entry.Name(), "node.exe")); err == nil { + return filepath.Join(rel, entry.Name()), nil + } else if !errors.Is(err, fs.ErrNotExist) { + return "", err + } return filepath.Join(rel, entry.Name(), "bin"), nil } } diff --git a/pkg/repos/runtimes/python/python.go b/pkg/repos/runtimes/python/python.go index 1a66a1a9..cb85e639 100644 --- a/pkg/repos/runtimes/python/python.go +++ b/pkg/repos/runtimes/python/python.go @@ -13,9 +13,9 @@ import ( "runtime" "github.com/gptscript-ai/gptscript/pkg/debugcmd" + runtimeEnv "github.com/gptscript-ai/gptscript/pkg/env" "github.com/gptscript-ai/gptscript/pkg/hash" "github.com/gptscript-ai/gptscript/pkg/repos/download" - runtimeEnv "github.com/gptscript-ai/gptscript/pkg/repos/runtimes/env" ) //go:embed python.json