Skip to content

Commit 736fd5c

Browse files
ibuildthecloudibuildthecloud
authored
Merge pull request #121 from ibuildthecloud/distribution
Add tools distribution for python
2 parents ea1691a + 9fb4106 commit 736fd5c

File tree

31 files changed

+1365
-83
lines changed

31 files changed

+1365
-83
lines changed

‎go.mod

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,10 @@ require (
1010
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
1111
github.com/hexops/autogold/v2 v2.1.0
1212
github.com/jaytaylor/html2text v0.0.0-20230321000545-74c2419ad056
13+
github.com/mholt/archiver/v4 v4.0.0-alpha.8
1314
github.com/olahol/melody v1.1.4
1415
github.com/rs/cors v1.10.1
16+
github.com/samber/lo v1.38.1
1517
github.com/sashabaranov/go-openai v1.20.1
1618
github.com/sirupsen/logrus v1.9.3
1719
github.com/spf13/cobra v1.8.0
@@ -22,33 +24,49 @@ require (
2224

2325
require (
2426
github.com/acorn-io/baaah v0.0.0-20240119160309-2a58ee757bbd // indirect
27+
github.com/andybalholm/brotli v1.0.4 // indirect
28+
github.com/bodgit/plumbing v1.2.0 // indirect
29+
github.com/bodgit/sevenzip v1.3.0 // indirect
30+
github.com/bodgit/windows v1.0.0 // indirect
2531
github.com/bombsimon/logrusr/v4 v4.0.0 // indirect
32+
github.com/connesc/cipherio v0.2.1 // indirect
2633
github.com/davecgh/go-spew v1.1.1 // indirect
34+
github.com/dsnet/compress v0.0.1 // indirect
2735
github.com/fatih/color v1.16.0 // indirect
2836
github.com/go-logr/logr v1.4.1 // indirect
37+
github.com/golang/snappy v0.0.4 // indirect
2938
github.com/google/go-cmp v0.6.0 // indirect
3039
github.com/google/go-containerregistry v0.16.1 // indirect
3140
github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 // indirect
3241
github.com/gorilla/websocket v1.5.0 // indirect
42+
github.com/hashicorp/errwrap v1.0.0 // indirect
43+
github.com/hashicorp/go-multierror v1.1.1 // indirect
3344
github.com/hexops/gotextdiff v1.0.3 // indirect
3445
github.com/hexops/valast v1.4.3 // indirect
3546
github.com/inconshreveable/mousetrap v1.1.0 // indirect
47+
github.com/klauspost/compress v1.16.5 // indirect
48+
github.com/klauspost/pgzip v1.2.5 // indirect
3649
github.com/mattn/go-colorable v0.1.13 // indirect
3750
github.com/mattn/go-isatty v0.0.20 // indirect
3851
github.com/mattn/go-runewidth v0.0.9 // indirect
3952
github.com/nightlyone/lockfile v1.0.0 // indirect
53+
github.com/nwaples/rardecode/v2 v2.0.0-beta.2 // indirect
4054
github.com/olekukonko/tablewriter v0.0.5 // indirect
4155
github.com/onsi/ginkgo/v2 v2.13.0 // indirect
4256
github.com/onsi/gomega v1.29.0 // indirect
57+
github.com/pierrec/lz4/v4 v4.1.15 // indirect
4358
github.com/pmezard/go-difflib v1.0.0 // indirect
44-
github.com/samber/lo v1.38.1 // indirect
4559
github.com/samber/slog-logrus v1.0.0 // indirect
4660
github.com/spf13/pflag v1.0.5 // indirect
4761
github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf // indirect
62+
github.com/therootcompany/xz v1.0.1 // indirect
63+
github.com/ulikunitz/xz v0.5.10 // indirect
64+
go4.org v0.0.0-20200411211856-f5505b9728dd // indirect
4865
golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc // indirect
4966
golang.org/x/mod v0.15.0 // indirect
5067
golang.org/x/net v0.20.0 // indirect
5168
golang.org/x/sys v0.16.0 // indirect
69+
golang.org/x/text v0.14.0 // indirect
5270
golang.org/x/tools v0.17.0 // indirect
5371
gopkg.in/yaml.v3 v3.0.1 // indirect
5472
k8s.io/klog/v2 v2.110.1 // indirect

‎go.sum

Lines changed: 240 additions & 0 deletions
Large diffs are not rendered by default.

‎pkg/cache/cache.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ type Options struct {
2222
CacheDir string `usage:"Directory to store cache (default: $XDG_CACHE_HOME/gptscript)"`
2323
}
2424

25-
func complete(opts ...Options) (result Options) {
25+
func Complete(opts ...Options) (result Options) {
2626
for _, opt := range opts {
2727
result.CacheDir = types.FirstSet(opt.CacheDir, result.CacheDir)
2828
result.Cache = types.FirstSet(opt.Cache, result.Cache)
@@ -48,7 +48,7 @@ func WithNoCache(ctx context.Context) context.Context {
4848
}
4949

5050
func New(opts ...Options) (*Client, error) {
51-
opt := complete(opts...)
51+
opt := Complete(opts...)
5252
if err := os.MkdirAll(opt.CacheDir, 0755); err != nil {
5353
return nil, err
5454
}

‎pkg/cli/gptscript.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ import (
1818
"github.com/gptscript-ai/gptscript/pkg/monitor"
1919
"github.com/gptscript-ai/gptscript/pkg/mvl"
2020
"github.com/gptscript-ai/gptscript/pkg/openai"
21+
"github.com/gptscript-ai/gptscript/pkg/repos/runtimes"
2122
"github.com/gptscript-ai/gptscript/pkg/runner"
2223
"github.com/gptscript-ai/gptscript/pkg/server"
2324
"github.com/gptscript-ai/gptscript/pkg/types"
@@ -225,6 +226,7 @@ func (r *GPTScript) Run(cmd *cobra.Command, args []string) error {
225226
MonitorFactory: monitor.NewConsole(monitor.Options(r.DisplayOptions), monitor.Options{
226227
DisplayProgress: !*r.Quiet,
227228
}),
229+
RuntimeManager: runtimes.Default(cache.Complete(cache.Options(r.CacheOptions)).CacheDir),
228230
})
229231
if err != nil {
230232
return err

‎pkg/debugcmd/debug.go

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
package debugcmd
2+
3+
import (
4+
"context"
5+
"os"
6+
"os/exec"
7+
)
8+
9+
func New(ctx context.Context, arg string, args ...string) *exec.Cmd {
10+
cmd := exec.CommandContext(ctx, arg, args...)
11+
SetupDebug(cmd)
12+
return cmd
13+
}
14+
15+
func SetupDebug(cmd *exec.Cmd) {
16+
if log.IsDebug() {
17+
cmd.Stdout = os.Stdout
18+
}
19+
cmd.Stderr = os.Stderr
20+
}

‎pkg/debugcmd/log.go

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
package debugcmd
2+
3+
import "github.com/gptscript-ai/gptscript/pkg/mvl"
4+
5+
var log = mvl.Package()

‎pkg/engine/cmd.go

Lines changed: 85 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@ import (
88
"io"
99
"os"
1010
"os/exec"
11+
"path/filepath"
12+
"sort"
1113
"strings"
1214
"sync/atomic"
1315

@@ -40,12 +42,7 @@ func (e *Engine) runCommand(ctx context.Context, tool types.Tool, input string)
4042
return tool.BuiltinFunc(ctx, e.Env, input)
4143
}
4244

43-
var extraEnv []string
44-
if tool.WorkingDir != "" {
45-
extraEnv = append(extraEnv, "GPTSCRIPT_TOOL_DIR="+tool.WorkingDir)
46-
}
47-
48-
cmd, stop, err := e.newCommand(ctx, extraEnv, tool.Instructions, input)
45+
cmd, stop, err := e.newCommand(ctx, nil, tool, input)
4946
if err != nil {
5047
return "", err
5148
}
@@ -74,62 +71,106 @@ func (e *Engine) runCommand(ctx context.Context, tool types.Tool, input string)
7471
return output.String(), nil
7572
}
7673

77-
func (e *Engine) newCommand(ctx context.Context, extraEnv []string, instructions, input string) (*exec.Cmd, func(), error) {
78-
env := append(e.Env[:], extraEnv...)
79-
data := map[string]any{}
80-
81-
dec := json.NewDecoder(bytes.NewReader([]byte(input)))
82-
dec.UseNumber()
74+
func (e *Engine) getRuntimeEnv(ctx context.Context, tool types.Tool, cmd, env []string) ([]string, error) {
75+
var (
76+
workdir = tool.WorkingDir
77+
err error
78+
)
79+
if e.RuntimeManager != nil {
80+
workdir, env, err = e.RuntimeManager.GetContext(ctx, tool, cmd, env)
81+
if err != nil {
82+
return nil, err
83+
}
84+
workdir = filepath.Join(workdir, tool.Source.Repo.Path)
85+
}
86+
return append(env, "GPTSCRIPT_TOOL_DIR="+workdir), nil
87+
}
8388

89+
func envAsMapAndDeDup(env []string) (sortedEnv []string, _ map[string]string) {
8490
envMap := map[string]string{}
91+
var keys []string
8592
for _, env := range env {
8693
key, value, _ := strings.Cut(env, "=")
94+
if _, existing := envMap[key]; !existing {
95+
keys = append(keys, key)
96+
}
8797
envMap[key] = value
8898
}
99+
sort.Strings(keys)
100+
for _, key := range keys {
101+
sortedEnv = append(sortedEnv, key+"="+envMap[key])
102+
}
103+
104+
return sortedEnv, envMap
105+
}
106+
107+
var ignoreENV = map[string]struct{}{
108+
"PATH": {},
109+
"GPTSCRIPT_TOOL_DIR": {},
110+
}
111+
112+
func appendEnv(env []string, k, v string) []string {
113+
for _, k := range []string{k, strings.ToUpper(strings.ReplaceAll(k, "-", "_"))} {
114+
if _, ignore := ignoreENV[k]; !ignore {
115+
env = append(env, k+"="+v)
116+
}
117+
}
118+
return env
119+
}
120+
121+
func appendInputAsEnv(env []string, input string) []string {
122+
data := map[string]any{}
123+
dec := json.NewDecoder(bytes.NewReader([]byte(input)))
124+
dec.UseNumber()
125+
126+
if err := json.Unmarshal([]byte(input), &data); err != nil {
127+
// ignore invalid JSON
128+
return env
129+
}
89130

90-
if err := json.Unmarshal([]byte(input), &data); err == nil {
91-
for k, v := range data {
92-
envName := strings.ToUpper(strings.ReplaceAll(k, "-", "_"))
93-
switch val := v.(type) {
94-
case string:
95-
envMap[envName] = val
96-
env = append(env, envName+"="+val)
97-
envMap[k] = val
98-
env = append(env, k+"="+val)
99-
case json.Number:
100-
envMap[envName] = string(val)
101-
env = append(env, envName+"="+string(val))
102-
envMap[k] = string(val)
103-
env = append(env, k+"="+string(val))
104-
case bool:
105-
envMap[envName] = fmt.Sprint(val)
106-
env = append(env, envName+"="+fmt.Sprint(val))
107-
envMap[k] = fmt.Sprint(val)
108-
env = append(env, k+"="+fmt.Sprint(val))
109-
default:
110-
data, err := json.Marshal(val)
111-
if err == nil {
112-
envMap[envName] = string(data)
113-
env = append(env, envName+"="+string(data))
114-
envMap[k] = string(data)
115-
env = append(env, k+"="+string(data))
116-
}
131+
for k, v := range data {
132+
switch val := v.(type) {
133+
case string:
134+
env = appendEnv(env, k, val)
135+
case json.Number:
136+
env = appendEnv(env, k, string(val))
137+
case bool:
138+
env = appendEnv(env, k, fmt.Sprint(val))
139+
default:
140+
data, err := json.Marshal(val)
141+
if err == nil {
142+
env = appendEnv(env, k, string(data))
117143
}
118144
}
119145
}
120146

121-
interpreter, rest, _ := strings.Cut(instructions, "\n")
122-
interpreter = strings.TrimSpace(interpreter)[2:]
147+
return env
148+
}
123149

124-
interpreter = os.Expand(interpreter, func(s string) string {
125-
return envMap[s]
126-
})
150+
func (e *Engine) newCommand(ctx context.Context, extraEnv []string, tool types.Tool, input string) (*exec.Cmd, func(), error) {
151+
env := append(e.Env[:], extraEnv...)
152+
env = appendInputAsEnv(env, input)
153+
154+
interpreter, rest, _ := strings.Cut(tool.Instructions, "\n")
155+
interpreter = strings.TrimSpace(interpreter)[2:]
127156

128157
args, err := shlex.Split(interpreter)
129158
if err != nil {
130159
return nil, nil, err
131160
}
132161

162+
env, err = e.getRuntimeEnv(ctx, tool, args, env)
163+
if err != nil {
164+
return nil, nil, err
165+
}
166+
167+
env, envMap := envAsMapAndDeDup(env)
168+
for i, arg := range args {
169+
args[i] = os.Expand(arg, func(s string) string {
170+
return envMap[s]
171+
})
172+
}
173+
133174
var (
134175
cmdArgs = args[1:]
135176
stop = func() {}

‎pkg/engine/daemon.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,7 @@ func (e *Engine) startDaemon(_ context.Context, tool types.Tool) (string, error)
7474

7575
instructions := strings.TrimPrefix(tool.Instructions, types.DaemonPrefix)
7676
instructions, path := getPath(instructions)
77+
tool.Instructions = types.CommandPrefix + instructions
7778

7879
port, ok := daemonPorts[tool.ID]
7980
url := fmt.Sprintf("http://127.0.0.1:%d%s", port, path)
@@ -92,7 +93,7 @@ func (e *Engine) startDaemon(_ context.Context, tool types.Tool) (string, error)
9293
cmd, stop, err := e.newCommand(ctx, []string{
9394
fmt.Sprintf("PORT=%d", port),
9495
},
95-
types.CommandPrefix+instructions,
96+
tool,
9697
"{}",
9798
)
9899
if err != nil {

‎pkg/engine/engine.go

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -26,10 +26,15 @@ type Model interface {
2626
Call(ctx context.Context, messageRequest types.CompletionRequest, status chan<- types.CompletionStatus) (*types.CompletionMessage, error)
2727
}
2828

29+
type RuntimeManager interface {
30+
GetContext(ctx context.Context, tool types.Tool, cmd, env []string) (string, []string, error)
31+
}
32+
2933
type Engine struct {
30-
Model Model
31-
Env []string
32-
Progress chan<- types.CompletionStatus
34+
Model Model
35+
RuntimeManager RuntimeManager
36+
Env []string
37+
Progress chan<- types.CompletionStatus
3338
}
3439

3540
type State struct {

‎pkg/hash/sha256.go

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,13 +6,25 @@ import (
66
"encoding/json"
77
)
88

9+
func ID(parts ...string) string {
10+
d := sha256.New()
11+
for i, part := range parts {
12+
if i > 0 {
13+
d.Write([]byte{0x00})
14+
}
15+
d.Write([]byte(part))
16+
}
17+
hash := d.Sum(nil)
18+
return hex.EncodeToString(hash[:])
19+
}
20+
921
func Digest(obj any) string {
1022
data, err := json.Marshal(obj)
1123
if err != nil {
1224
panic(err)
1325
}
1426

15-
hash := sha256.Sum224(data)
27+
hash := sha256.Sum256(data)
1628
return hex.EncodeToString(hash[:])
1729
}
1830

@@ -32,6 +44,6 @@ func Encode(obj any) string {
3244
panic(err)
3345
}
3446

35-
hash := sha256.Sum224(data)
47+
hash := sha256.Sum256(data)
3648
return hex.EncodeToString(hash[:])
3749
}

0 commit comments

Comments
 (0)