Skip to content

Commit 9a110e7

Browse files
Add tools distribution
1 parent ecd4f4d commit 9a110e7

File tree

25 files changed

+1298
-64
lines changed

25 files changed

+1298
-64
lines changed

go.mod

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,10 @@ require (
1010
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
1111
github.com/hexops/autogold/v2 v2.1.0
1212
github.com/jaytaylor/html2text v0.0.0-20230321000545-74c2419ad056
13+
github.com/mholt/archiver/v4 v4.0.0-alpha.8
1314
github.com/olahol/melody v1.1.4
1415
github.com/rs/cors v1.10.1
16+
github.com/samber/lo v1.38.1
1517
github.com/sashabaranov/go-openai v1.20.1
1618
github.com/sirupsen/logrus v1.9.3
1719
github.com/spf13/cobra v1.8.0
@@ -22,33 +24,49 @@ require (
2224

2325
require (
2426
github.com/acorn-io/baaah v0.0.0-20240119160309-2a58ee757bbd // indirect
27+
github.com/andybalholm/brotli v1.0.4 // indirect
28+
github.com/bodgit/plumbing v1.2.0 // indirect
29+
github.com/bodgit/sevenzip v1.3.0 // indirect
30+
github.com/bodgit/windows v1.0.0 // indirect
2531
github.com/bombsimon/logrusr/v4 v4.0.0 // indirect
32+
github.com/connesc/cipherio v0.2.1 // indirect
2633
github.com/davecgh/go-spew v1.1.1 // indirect
34+
github.com/dsnet/compress v0.0.1 // indirect
2735
github.com/fatih/color v1.16.0 // indirect
2836
github.com/go-logr/logr v1.4.1 // indirect
37+
github.com/golang/snappy v0.0.4 // indirect
2938
github.com/google/go-cmp v0.6.0 // indirect
3039
github.com/google/go-containerregistry v0.16.1 // indirect
3140
github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 // indirect
3241
github.com/gorilla/websocket v1.5.0 // indirect
42+
github.com/hashicorp/errwrap v1.0.0 // indirect
43+
github.com/hashicorp/go-multierror v1.1.1 // indirect
3344
github.com/hexops/gotextdiff v1.0.3 // indirect
3445
github.com/hexops/valast v1.4.3 // indirect
3546
github.com/inconshreveable/mousetrap v1.1.0 // indirect
47+
github.com/klauspost/compress v1.16.5 // indirect
48+
github.com/klauspost/pgzip v1.2.5 // indirect
3649
github.com/mattn/go-colorable v0.1.13 // indirect
3750
github.com/mattn/go-isatty v0.0.20 // indirect
3851
github.com/mattn/go-runewidth v0.0.9 // indirect
3952
github.com/nightlyone/lockfile v1.0.0 // indirect
53+
github.com/nwaples/rardecode/v2 v2.0.0-beta.2 // indirect
4054
github.com/olekukonko/tablewriter v0.0.5 // indirect
4155
github.com/onsi/ginkgo/v2 v2.13.0 // indirect
4256
github.com/onsi/gomega v1.29.0 // indirect
57+
github.com/pierrec/lz4/v4 v4.1.15 // indirect
4358
github.com/pmezard/go-difflib v1.0.0 // indirect
44-
github.com/samber/lo v1.38.1 // indirect
4559
github.com/samber/slog-logrus v1.0.0 // indirect
4660
github.com/spf13/pflag v1.0.5 // indirect
4761
github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf // indirect
62+
github.com/therootcompany/xz v1.0.1 // indirect
63+
github.com/ulikunitz/xz v0.5.10 // indirect
64+
go4.org v0.0.0-20200411211856-f5505b9728dd // indirect
4865
golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc // indirect
4966
golang.org/x/mod v0.15.0 // indirect
5067
golang.org/x/net v0.20.0 // indirect
5168
golang.org/x/sys v0.16.0 // indirect
69+
golang.org/x/text v0.14.0 // indirect
5270
golang.org/x/tools v0.17.0 // indirect
5371
gopkg.in/yaml.v3 v3.0.1 // indirect
5472
k8s.io/klog/v2 v2.110.1 // indirect

go.sum

Lines changed: 240 additions & 0 deletions
Large diffs are not rendered by default.

pkg/debugcmd/debug.go

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
package debugcmd
2+
3+
import (
4+
"context"
5+
"os"
6+
"os/exec"
7+
)
8+
9+
func New(ctx context.Context, arg string, args ...string) *exec.Cmd {
10+
cmd := exec.CommandContext(ctx, arg, args...)
11+
SetupDebug(cmd)
12+
return cmd
13+
}
14+
15+
func SetupDebug(cmd *exec.Cmd) {
16+
cmd.Stdout = os.Stdout
17+
cmd.Stderr = os.Stderr
18+
}

pkg/engine/cmd.go

Lines changed: 72 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ func (e *Engine) runCommand(ctx context.Context, tool types.Tool, input string)
4545
extraEnv = append(extraEnv, "GPTSCRIPT_TOOL_DIR="+tool.WorkingDir)
4646
}
4747

48-
cmd, stop, err := e.newCommand(ctx, extraEnv, tool.Instructions, input)
48+
cmd, stop, err := e.newCommand(ctx, extraEnv, tool, input)
4949
if err != nil {
5050
return "", err
5151
}
@@ -74,62 +74,95 @@ func (e *Engine) runCommand(ctx context.Context, tool types.Tool, input string)
7474
return output.String(), nil
7575
}
7676

77-
func (e *Engine) newCommand(ctx context.Context, extraEnv []string, instructions, input string) (*exec.Cmd, func(), error) {
78-
env := append(e.Env[:], extraEnv...)
79-
data := map[string]any{}
80-
81-
dec := json.NewDecoder(bytes.NewReader([]byte(input)))
82-
dec.UseNumber()
77+
func (e *Engine) getRuntimeEnv(ctx context.Context, tool types.Tool, cmd, env []string) ([]string, error) {
78+
var (
79+
workdir = tool.WorkingDir
80+
err error
81+
)
82+
if e.RuntimeManager != nil {
83+
workdir, env, err = e.RuntimeManager.GetContext(ctx, tool, cmd, env)
84+
if err != nil {
85+
return nil, err
86+
}
87+
}
88+
return append(env, "GPTSCRIPT_TOOL_DIR="+workdir), nil
89+
}
8390

91+
func envAsMap(env []string) map[string]string {
8492
envMap := map[string]string{}
8593
for _, env := range env {
8694
key, value, _ := strings.Cut(env, "=")
8795
envMap[key] = value
8896
}
97+
return envMap
98+
}
99+
100+
var ignoreENV = map[string]struct{}{
101+
"PATH": {},
102+
}
103+
104+
func appendEnv(env []string, k, v string) []string {
105+
for _, k := range []string{k, strings.ToUpper(strings.ReplaceAll(k, "-", "_"))} {
106+
if _, ignore := ignoreENV[k]; !ignore {
107+
env = append(env, k+"="+v)
108+
}
109+
}
110+
return env
111+
}
89112

90-
if err := json.Unmarshal([]byte(input), &data); err == nil {
91-
for k, v := range data {
92-
envName := strings.ToUpper(strings.ReplaceAll(k, "-", "_"))
93-
switch val := v.(type) {
94-
case string:
95-
envMap[envName] = val
96-
env = append(env, envName+"="+val)
97-
envMap[k] = val
98-
env = append(env, k+"="+val)
99-
case json.Number:
100-
envMap[envName] = string(val)
101-
env = append(env, envName+"="+string(val))
102-
envMap[k] = string(val)
103-
env = append(env, k+"="+string(val))
104-
case bool:
105-
envMap[envName] = fmt.Sprint(val)
106-
env = append(env, envName+"="+fmt.Sprint(val))
107-
envMap[k] = fmt.Sprint(val)
108-
env = append(env, k+"="+fmt.Sprint(val))
109-
default:
110-
data, err := json.Marshal(val)
111-
if err == nil {
112-
envMap[envName] = string(data)
113-
env = append(env, envName+"="+string(data))
114-
envMap[k] = string(data)
115-
env = append(env, k+"="+string(data))
116-
}
113+
func appendInputAsEnv(env []string, input string) []string {
114+
data := map[string]any{}
115+
dec := json.NewDecoder(bytes.NewReader([]byte(input)))
116+
dec.UseNumber()
117+
118+
if err := json.Unmarshal([]byte(input), &data); err != nil {
119+
// ignore invalid JSON
120+
return env
121+
}
122+
123+
for k, v := range data {
124+
switch val := v.(type) {
125+
case string:
126+
env = appendEnv(env, k, val)
127+
case json.Number:
128+
env = appendEnv(env, k, string(val))
129+
case bool:
130+
env = appendEnv(env, k, fmt.Sprint(val))
131+
default:
132+
data, err := json.Marshal(val)
133+
if err == nil {
134+
env = appendEnv(env, k, string(data))
117135
}
118136
}
119137
}
120138

121-
interpreter, rest, _ := strings.Cut(instructions, "\n")
122-
interpreter = strings.TrimSpace(interpreter)[2:]
139+
return env
140+
}
123141

124-
interpreter = os.Expand(interpreter, func(s string) string {
125-
return envMap[s]
126-
})
142+
func (e *Engine) newCommand(ctx context.Context, extraEnv []string, tool types.Tool, input string) (*exec.Cmd, func(), error) {
143+
env := append(e.Env[:], extraEnv...)
144+
env = appendInputAsEnv(env, input)
145+
envMap := envAsMap(env)
146+
147+
interpreter, rest, _ := strings.Cut(tool.Instructions, "\n")
148+
interpreter = strings.TrimSpace(interpreter)[2:]
127149

128150
args, err := shlex.Split(interpreter)
129151
if err != nil {
130152
return nil, nil, err
131153
}
132154

155+
env, err = e.getRuntimeEnv(ctx, tool, args, env)
156+
if err != nil {
157+
return nil, nil, err
158+
}
159+
160+
for i, arg := range args {
161+
args[i] = os.Expand(arg, func(s string) string {
162+
return envMap[s]
163+
})
164+
}
165+
133166
var (
134167
cmdArgs = args[1:]
135168
stop = func() {}

pkg/engine/daemon.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,7 @@ func (e *Engine) startDaemon(_ context.Context, tool types.Tool) (string, error)
7474

7575
instructions := strings.TrimPrefix(tool.Instructions, types.DaemonPrefix)
7676
instructions, path := getPath(instructions)
77+
tool.Instructions = types.CommandPrefix + instructions
7778

7879
port, ok := daemonPorts[tool.ID]
7980
url := fmt.Sprintf("http://127.0.0.1:%d%s", port, path)
@@ -92,7 +93,7 @@ func (e *Engine) startDaemon(_ context.Context, tool types.Tool) (string, error)
9293
cmd, stop, err := e.newCommand(ctx, []string{
9394
fmt.Sprintf("PORT=%d", port),
9495
},
95-
types.CommandPrefix+instructions,
96+
tool,
9697
"{}",
9798
)
9899
if err != nil {

pkg/engine/engine.go

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -26,10 +26,15 @@ type Model interface {
2626
Call(ctx context.Context, messageRequest types.CompletionRequest, status chan<- types.CompletionStatus) (*types.CompletionMessage, error)
2727
}
2828

29+
type RuntimeManager interface {
30+
GetContext(ctx context.Context, tool types.Tool, cmd, env []string) (string, []string, error)
31+
}
32+
2933
type Engine struct {
30-
Model Model
31-
Env []string
32-
Progress chan<- types.CompletionStatus
34+
Model Model
35+
RuntimeManager RuntimeManager
36+
Env []string
37+
Progress chan<- types.CompletionStatus
3338
}
3439

3540
type State struct {

pkg/hash/sha256.go

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,13 +6,25 @@ import (
66
"encoding/json"
77
)
88

9+
func ID(parts ...string) string {
10+
d := sha256.New()
11+
for i, part := range parts {
12+
if i > 0 {
13+
d.Write([]byte{0x00})
14+
}
15+
d.Write([]byte(part))
16+
}
17+
hash := d.Sum(nil)
18+
return hex.EncodeToString(hash[:])
19+
}
20+
921
func Digest(obj any) string {
1022
data, err := json.Marshal(obj)
1123
if err != nil {
1224
panic(err)
1325
}
1426

15-
hash := sha256.Sum224(data)
27+
hash := sha256.Sum256(data)
1628
return hex.EncodeToString(hash[:])
1729
}
1830

@@ -32,6 +44,6 @@ func Encode(obj any) string {
3244
panic(err)
3345
}
3446

35-
hash := sha256.Sum224(data)
47+
hash := sha256.Sum256(data)
3648
return hex.EncodeToString(hash[:])
3749
}

pkg/loader/github/github.go

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import (
1010

1111
"github.com/gptscript-ai/gptscript/pkg/loader"
1212
"github.com/gptscript-ai/gptscript/pkg/system"
13+
"github.com/gptscript-ai/gptscript/pkg/types"
1314
)
1415

1516
const (
@@ -50,7 +51,7 @@ func getCommit(account, repo, ref string) (string, error) {
5051
return commit.SHA, nil
5152
}
5253

53-
func Load(urlName string) (string, *loader.Repo, bool, error) {
54+
func Load(urlName string) (string, *types.Repo, bool, error) {
5455
if !strings.HasPrefix(urlName, GithubPrefix) {
5556
return "", nil, false, nil
5657
}
@@ -81,7 +82,7 @@ func Load(urlName string) (string, *loader.Repo, bool, error) {
8182
}
8283

8384
downloadURL := fmt.Sprintf(githubDownloadURL, account, repo, ref, path)
84-
return downloadURL, &loader.Repo{
85+
return downloadURL, &types.Repo{
8586
VCS: "github",
8687
Root: fmt.Sprintf(githubRepoURL, account, repo),
8788
Path: filepath.Dir(path),

pkg/loader/loader.go

Lines changed: 2 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -34,20 +34,7 @@ type source struct {
3434
// be a valid URI or URL, used primarily for display.
3535
Location string
3636
// Repo The VCS repo where this tool was found, used to clone and provide the local tool code content
37-
Repo *Repo
38-
}
39-
40-
type Repo struct {
41-
// VCS The VCS type, such as "github"
42-
VCS string
43-
// The URL where the VCS repo can be found
44-
Root string
45-
// The path in the repo of this source. This should refer to a directory and not the actual file
46-
Path string
47-
// The filename of the source in the repo, relative to Path
48-
Name string
49-
// The revision of this source
50-
Revision string
37+
Repo *types.Repo
5138
}
5239

5340
func (s *source) String() string {
@@ -151,6 +138,7 @@ func readTool(ctx context.Context, prg *types.Program, base *source, targetToolN
151138
for i, tool := range tools {
152139
tool.WorkingDir = base.Path
153140
tool.Source.Location = base.Location
141+
tool.Source.Repo = base.Repo
154142

155143
// Probably a better way to come up with an ID
156144
tool.ID = tool.Source.String()

pkg/loader/url.go

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,11 @@ import (
77
url2 "net/url"
88
"path/filepath"
99
"strings"
10+
11+
"github.com/gptscript-ai/gptscript/pkg/types"
1012
)
1113

12-
type VCSLookup func(string) (string, *Repo, bool, error)
14+
type VCSLookup func(string) (string, *types.Repo, bool, error)
1315

1416
var vcsLookups []VCSLookup
1517

@@ -19,7 +21,7 @@ func AddVSC(lookup VCSLookup) {
1921

2022
func loadURL(ctx context.Context, base *source, name string) (*source, bool, error) {
2123
var (
22-
repo *Repo
24+
repo *types.Repo
2325
url = name
2426
)
2527

0 commit comments

Comments
 (0)