Skip to content

feat: add gptscript eval sub command #206

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Apr 1, 2024
Merged

feat: add gptscript eval sub command #206

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
71 changes: 71 additions & 0 deletions pkg/cli/eval.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
package cli

import (
"fmt"
"os"
"strconv"
"strings"

"github.com/gptscript-ai/gptscript/pkg/gptscript"
"github.com/gptscript-ai/gptscript/pkg/input"
"github.com/gptscript-ai/gptscript/pkg/loader"
"github.com/gptscript-ai/gptscript/pkg/types"
"github.com/spf13/cobra"
)

type Eval struct {
Tools []string `usage:"Tools available to call"`
MaxTokens int `usage:"Maximum number of tokens to output"`
Model string `usage:"The model to use"`
JSON bool `usage:"Output JSON"`
Temperature string `usage:"Set the temperature, \"creativity\""`
InternalPrompt *bool `Usage:"Set to false to disable the internal prompt"`

gptscript *GPTScript
}

func (e *Eval) Run(cmd *cobra.Command, args []string) error {
tool := types.Tool{
Parameters: types.Parameters{
Description: "inline script",
Tools: e.Tools,
MaxTokens: e.MaxTokens,
ModelName: e.Model,
JSONResponse: e.JSON,
InternalPrompt: e.InternalPrompt,
},
Instructions: strings.Join(args, " "),
}

if e.Temperature != "" {
temp, err := strconv.ParseFloat(e.Temperature, 32)
if err != nil {
return fmt.Errorf("failed to parse %v: %v", e.Temperature, err)
}
temp32 := float32(temp)
tool.Temperature = &temp32
}

prg, err := loader.ProgramFromSource(cmd.Context(), tool.String(), "")
if err != nil {
return err
}

opts := e.gptscript.NewGPTScriptOpts()
runner, err := gptscript.New(&opts)
if err != nil {
return err
}

toolInput, err := input.FromFile(e.gptscript.Input)
if err != nil {
return err
}

toolOutput, err := runner.Run(e.gptscript.NewRunContext(cmd), prg, os.Environ(), toolInput)
if err != nil {
return err
}

return e.gptscript.PrintOutput("", toolOutput)
}
182 changes: 110 additions & 72 deletions pkg/cli/gptscript.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
package cli

import (
"context"
"fmt"
"io"
"os"
"sort"
"strings"

"github.com/acorn-io/cmd"
Expand Down Expand Up @@ -41,17 +43,38 @@ type GPTScript struct {
Quiet *bool `usage:"No output logging (set --quiet=false to force on even when there is no TTY)" short:"q"`
Output string `usage:"Save output to a file, or - for stdout" short:"o"`
Input string `usage:"Read input from a file (\"-\" for stdin)" short:"f"`
SubTool string `usage:"Use tool of this name, not the first tool in file"`
Assemble bool `usage:"Assemble tool to a single artifact, saved to --output" hidden:"true"`
ListModels bool `usage:"List the models available and exit"`
ListTools bool `usage:"List built-in tools and exit"`
Server bool `usage:"Start server"`
ListenAddress string `usage:"Server listen address" default:"127.0.0.1:9090"`
SubTool string `usage:"Use tool of this name, not the first tool in file" local:"true"`
Assemble bool `usage:"Assemble tool to a single artifact, saved to --output" hidden:"true" local:"true"`
ListModels bool `usage:"List the models available and exit" local:"true"`
ListTools bool `usage:"List built-in tools and exit" local:"true"`
Server bool `usage:"Start server" local:"true"`
ListenAddress string `usage:"Server listen address" default:"127.0.0.1:9090" local:"true"`
Chdir string `usage:"Change current working directory" short:"C"`
}

func New() *cobra.Command {
return cmd.Command(&GPTScript{})
root := &GPTScript{}
return cmd.Command(root, &Eval{
gptscript: root,
})
}

func (r *GPTScript) NewRunContext(cmd *cobra.Command) context.Context {
ctx := cmd.Context()
if r.Confirm {
ctx = confirm.WithConfirm(ctx, confirm.TextPrompt{})
}
return ctx
}

func (r *GPTScript) NewGPTScriptOpts() gptscript.Options {
return gptscript.Options{
Cache: cache.Options(r.CacheOptions),
OpenAI: openai.Options(r.OpenAIOptions),
Monitor: monitor.Options(r.DisplayOptions),
Quiet: r.Quiet,
Env: os.Environ(),
}
}

func (r *GPTScript) Customize(cmd *cobra.Command) {
Expand All @@ -74,16 +97,27 @@ func (r *GPTScript) Customize(cmd *cobra.Command) {
}
}

func (r *GPTScript) listTools() error {
func (r *GPTScript) listTools(ctx context.Context, gptScript *gptscript.GPTScript, prg types.Program) error {
tools := gptScript.ListTools(ctx, prg)
sort.Slice(tools, func(i, j int) bool {
return tools[i].Name < tools[j].Name
})
var lines []string
for _, tool := range builtin.ListTools() {
for _, tool := range tools {
if tool.Name == "" {
tool.Name = prg.Name
}

// Don't print instructions
tool.Instructions = ""

lines = append(lines, tool.String())
}
fmt.Println(strings.Join(lines, "\n---\n"))
return nil
}

func (r *GPTScript) Pre(*cobra.Command, []string) error {
func (r *GPTScript) PersistentPre(*cobra.Command, []string) error {
// chdir as soon as possible
if r.Chdir != "" {
if err := os.Chdir(r.Chdir); err != nil {
Expand Down Expand Up @@ -111,10 +145,7 @@ func (r *GPTScript) Pre(*cobra.Command, []string) error {
mvl.SetError()
}
}
return nil
}

func (r *GPTScript) Run(cmd *cobra.Command, args []string) error {
if r.Color != nil {
color.NoColor = !*r.Color
}
Expand All @@ -123,14 +154,64 @@ func (r *GPTScript) Run(cmd *cobra.Command, args []string) error {
log.Infof("WARNING: Changing the default model can have unknown behavior for existing tools. Use the model field per tool instead.")
}

gptOpt := gptscript.Options{
Cache: cache.Options(r.CacheOptions),
OpenAI: openai.Options(r.OpenAIOptions),
Monitor: monitor.Options(r.DisplayOptions),
Quiet: r.Quiet,
Env: os.Environ(),
return nil
}

func (r *GPTScript) listModels(ctx context.Context, gptScript *gptscript.GPTScript, args []string) error {
models, err := gptScript.ListModels(ctx, args...)
if err != nil {
return err
}
fmt.Println(strings.Join(models, "\n"))
return nil
}

func (r *GPTScript) readProgram(ctx context.Context, args []string) (prg types.Program, err error) {
if len(args) == 0 {
return
}

if args[0] == "-" {
data, err := io.ReadAll(os.Stdin)
if err != nil {
return prg, err
}
prg, err = loader.ProgramFromSource(ctx, string(data), r.SubTool)
if err != nil {
return prg, err
}
}

return loader.Program(ctx, args[0], r.SubTool)
}

func (r *GPTScript) PrintOutput(toolInput, toolOutput string) (err error) {
if r.Output != "" {
err = os.WriteFile(r.Output, []byte(toolOutput), 0644)
if err != nil {
return err
}
} else {
if !*r.Quiet {
if toolInput != "" {
_, _ = fmt.Fprint(os.Stderr, "\nINPUT:\n\n")
_, _ = fmt.Fprintln(os.Stderr, toolInput)
}
_, _ = fmt.Fprint(os.Stderr, "\nOUTPUT:\n\n")
}
fmt.Print(toolOutput)
if !strings.HasSuffix(toolOutput, "\n") {
fmt.Println()
}
}

return
}

func (r *GPTScript) Run(cmd *cobra.Command, args []string) error {
gptOpt := r.NewGPTScriptOpts()
ctx := cmd.Context()

if r.Server {
s, err := server.New(&server.Options{
ListenAddress: r.ListenAddress,
Expand All @@ -140,7 +221,7 @@ func (r *GPTScript) Run(cmd *cobra.Command, args []string) error {
return err
}
defer s.Close()
return s.Start(cmd.Context())
return s.Start(ctx)
}

gptScript, err := gptscript.New(&gptOpt)
Expand All @@ -150,42 +231,22 @@ func (r *GPTScript) Run(cmd *cobra.Command, args []string) error {
defer gptScript.Close()

if r.ListModels {
models, err := gptScript.ListModels(cmd.Context(), args...)
if err != nil {
return err
}
fmt.Println(strings.Join(models, "\n"))
return nil
return r.listModels(ctx, gptScript, args)
}

prg, err := r.readProgram(ctx, args)
if err != nil {
return err
}

if r.ListTools {
return r.listTools()
return r.listTools(ctx, gptScript, prg)
}

if len(args) == 0 {
return cmd.Help()
}

var (
prg types.Program
)

if args[0] == "-" {
data, err := io.ReadAll(os.Stdin)
if err != nil {
return err
}
prg, err = loader.ProgramFromSource(cmd.Context(), string(data), r.SubTool)
if err != nil {
return err
}
} else {
prg, err = loader.Program(cmd.Context(), args[0], r.SubTool)
if err != nil {
return err
}
}

if r.Assemble {
var out io.Writer = os.Stdout
if r.Output != "" && r.Output != "-" {
Expand All @@ -205,33 +266,10 @@ func (r *GPTScript) Run(cmd *cobra.Command, args []string) error {
return err
}

ctx := cmd.Context()
if r.Confirm {
ctx = confirm.WithConfirm(ctx, confirm.TextPrompt{})
}
s, err := gptScript.Run(ctx, prg, os.Environ(), toolInput)
s, err := gptScript.Run(r.NewRunContext(cmd), prg, os.Environ(), toolInput)
if err != nil {
return err
}

if r.Output != "" {
err = os.WriteFile(r.Output, []byte(s), 0644)
if err != nil {
return err
}
} else {
if !*r.Quiet {
if toolInput != "" {
_, _ = fmt.Fprint(os.Stderr, "\nINPUT:\n\n")
_, _ = fmt.Fprintln(os.Stderr, toolInput)
}
_, _ = fmt.Fprint(os.Stderr, "\nOUTPUT:\n\n")
}
fmt.Print(s)
if !strings.HasSuffix(s, "\n") {
fmt.Println()
}
}

return nil
return r.PrintOutput(toolInput, s)
}
Loading