From b07aec14df75321b01aca5460420419d9bc7b7dc Mon Sep 17 00:00:00 2001 From: Darren Shepherd Date: Tue, 30 Apr 2024 12:02:41 -0700 Subject: [PATCH] feat: add parse and fmt CLI commands --- pkg/cli/fmt.go | 51 ++++++++++++++++ pkg/cli/gptscript.go | 12 +++- pkg/cli/parse.go | 47 +++++++++++++++ pkg/loader/loader.go | 2 +- pkg/parser/parser.go | 124 +++++++++++++++++++++++++++++++++----- pkg/parser/parser_test.go | 87 ++++++++++++++++---------- pkg/types/tool.go | 4 +- 7 files changed, 272 insertions(+), 55 deletions(-) create mode 100644 pkg/cli/fmt.go create mode 100644 pkg/cli/parse.go diff --git a/pkg/cli/fmt.go b/pkg/cli/fmt.go new file mode 100644 index 00000000..72696756 --- /dev/null +++ b/pkg/cli/fmt.go @@ -0,0 +1,51 @@ +package cli + +import ( + "encoding/json" + "fmt" + "os" + "strings" + + "github.com/gptscript-ai/gptscript/pkg/input" + "github.com/gptscript-ai/gptscript/pkg/parser" + "github.com/spf13/cobra" +) + +type Fmt struct { + Write bool `usage:"Write output to file instead of stdout" short:"w"` +} + +func (e *Fmt) Customize(cmd *cobra.Command) { + cmd.Args = cobra.ExactArgs(1) +} + +func (e *Fmt) Run(_ *cobra.Command, args []string) error { + input, err := input.FromFile(args[0]) + if err != nil { + return err + } + + var ( + doc parser.Document + loc = locationName(args[0]) + ) + if strings.HasPrefix(input, "{") { + if err := json.Unmarshal([]byte(input), &doc); err != nil { + return err + } + } else { + doc, err = parser.Parse(strings.NewReader(input), parser.Options{ + Location: locationName(args[0]), + }) + if err != nil { + return err + } + } + + if e.Write && loc != "" { + return os.WriteFile(loc, []byte(doc.String()), 0644) + } + + fmt.Print(doc.String()) + return nil +} diff --git a/pkg/cli/gptscript.go b/pkg/cli/gptscript.go index caebc68c..1f5ff899 100644 --- a/pkg/cli/gptscript.go +++ b/pkg/cli/gptscript.go @@ -68,9 +68,15 @@ type GPTScript struct { func New() *cobra.Command { root := &GPTScript{} - command := cmd.Command(root, &Eval{ - gptscript: root, - }, &Credential{root: root}) + command := cmd.Command( + root, + &Eval{ + gptscript: root, + }, + &Credential{root: root}, + &Parse{}, + &Fmt{}, + ) // Hide all the global flags for the credential subcommand. for _, child := range command.Commands() { diff --git a/pkg/cli/parse.go b/pkg/cli/parse.go new file mode 100644 index 00000000..016e107b --- /dev/null +++ b/pkg/cli/parse.go @@ -0,0 +1,47 @@ +package cli + +import ( + "encoding/json" + "os" + "strings" + + "github.com/gptscript-ai/gptscript/pkg/input" + "github.com/gptscript-ai/gptscript/pkg/parser" + "github.com/spf13/cobra" +) + +type Parse struct { + PrettyPrint bool `usage:"Indent the json output" short:"p"` +} + +func (e *Parse) Customize(cmd *cobra.Command) { + cmd.Args = cobra.ExactArgs(1) +} + +func locationName(l string) string { + if l == "-" { + return "" + } + return l +} + +func (e *Parse) Run(_ *cobra.Command, args []string) error { + input, err := input.FromFile(args[0]) + if err != nil { + return err + } + + docs, err := parser.Parse(strings.NewReader(input), parser.Options{ + Location: locationName(args[0]), + }) + if err != nil { + return err + } + + enc := json.NewEncoder(os.Stdout) + if e.PrettyPrint { + enc.SetIndent("", " ") + } + + return enc.Encode(docs) +} diff --git a/pkg/loader/loader.go b/pkg/loader/loader.go index 51c74921..250276d0 100644 --- a/pkg/loader/loader.go +++ b/pkg/loader/loader.go @@ -147,7 +147,7 @@ func readTool(ctx context.Context, prg *types.Program, base *source, targetToolN // If we didn't get any tools from trying to parse it as OpenAPI, try to parse it as a GPTScript if len(tools) == 0 { - tools, err = parser.Parse(bytes.NewReader(data), parser.Options{ + tools, err = parser.ParseTools(bytes.NewReader(data), parser.Options{ AssignGlobals: true, }) if err != nil { diff --git a/pkg/parser/parser.go b/pkg/parser/parser.go index 33f42c80..84226fba 100644 --- a/pkg/parser/parser.go +++ b/pkg/parser/parser.go @@ -174,42 +174,124 @@ type context struct { instructions []string inBody bool skipNode bool + skipLines []string seenParam bool } -func (c *context) finish(tools *[]types.Tool) { +func (c *context) finish(tools *[]Node) { c.tool.Instructions = strings.TrimSpace(strings.Join(c.instructions, "")) if c.tool.Instructions != "" || c.tool.Parameters.Name != "" || len(c.tool.Export) > 0 || len(c.tool.Tools) > 0 || c.tool.GlobalModelName != "" || len(c.tool.GlobalTools) > 0 || c.tool.Chat { - *tools = append(*tools, c.tool) + *tools = append(*tools, Node{ + ToolNode: &ToolNode{ + Tool: c.tool, + }, + }) + } + if c.skipNode && len(c.skipLines) > 0 { + *tools = append(*tools, Node{ + TextNode: &TextNode{ + Text: strings.Join(c.skipLines, ""), + }, + }) } *c = context{} } type Options struct { AssignGlobals bool + Location string } func complete(opts ...Options) (result Options) { for _, opt := range opts { result.AssignGlobals = types.FirstSet(opt.AssignGlobals, result.AssignGlobals) + result.Location = types.FirstSet(opt.Location, result.Location) } return } -func Parse(input io.Reader, opts ...Options) ([]types.Tool, error) { - tools, err := parse(input) +type Document struct { + Nodes []Node `json:"nodes,omitempty"` +} + +func writeSep(buf *strings.Builder, lastText bool) { + if buf.Len() > 0 { + if !lastText { + buf.WriteString("\n") + } + buf.WriteString("---\n") + } +} + +func (d Document) String() string { + buf := strings.Builder{} + lastText := false + for _, node := range d.Nodes { + if node.TextNode != nil { + writeSep(&buf, lastText) + buf.WriteString(node.TextNode.Text) + lastText = true + } + if node.ToolNode != nil { + writeSep(&buf, lastText) + buf.WriteString(node.ToolNode.Tool.String()) + lastText = false + } + } + return buf.String() +} + +type Node struct { + TextNode *TextNode `json:"textNode,omitempty"` + ToolNode *ToolNode `json:"toolNode,omitempty"` +} + +type TextNode struct { + Text string `json:"text,omitempty"` +} + +type ToolNode struct { + Tool types.Tool `json:"tool,omitempty"` +} + +func ParseTools(input io.Reader, opts ...Options) (result []types.Tool, _ error) { + doc, err := Parse(input, opts...) if err != nil { return nil, err } + for _, node := range doc.Nodes { + if node.ToolNode != nil { + result = append(result, node.ToolNode.Tool) + } + } + + return +} + +func Parse(input io.Reader, opts ...Options) (Document, error) { + nodes, err := parse(input) + if err != nil { + return Document{}, err + } opt := complete(opts...) + if opt.Location != "" { + for _, node := range nodes { + if node.ToolNode != nil && node.ToolNode.Tool.Source.Location == "" { + node.ToolNode.Tool.Source.Location = opt.Location + } + } + } + if !opt.AssignGlobals { - return tools, nil + return Document{ + Nodes: nodes, + }, nil } var ( @@ -218,10 +300,14 @@ func Parse(input io.Reader, opts ...Options) ([]types.Tool, error) { globalTools []string ) - for _, tool := range tools { + for _, node := range nodes { + if node.ToolNode == nil { + continue + } + tool := node.ToolNode.Tool if tool.GlobalModelName != "" { if globalModel != "" { - return nil, fmt.Errorf("global model name defined multiple times") + return Document{}, fmt.Errorf("global model name defined multiple times") } globalModel = tool.GlobalModelName } @@ -234,26 +320,30 @@ func Parse(input io.Reader, opts ...Options) ([]types.Tool, error) { } } - for i, tool := range tools { - if globalModel != "" && tool.ModelName == "" { - tool.ModelName = globalModel + for _, node := range nodes { + if node.ToolNode == nil { + continue + } + if globalModel != "" && node.ToolNode.Tool.ModelName == "" { + node.ToolNode.Tool.ModelName = globalModel } for _, globalTool := range globalTools { - if !slices.Contains(tool.Tools, globalTool) { - tool.Tools = append(tool.Tools, globalTool) + if !slices.Contains(node.ToolNode.Tool.Tools, globalTool) { + node.ToolNode.Tool.Tools = append(node.ToolNode.Tool.Tools, globalTool) } } - tools[i] = tool } - return tools, nil + return Document{ + Nodes: nodes, + }, nil } -func parse(input io.Reader) ([]types.Tool, error) { +func parse(input io.Reader) ([]Node, error) { scan := bufio.NewScanner(input) var ( - tools []types.Tool + tools []Node context context lineNo int ) @@ -277,6 +367,7 @@ func parse(input io.Reader) ([]types.Tool, error) { } if context.skipNode { + context.skipLines = append(context.skipLines, line) continue } @@ -292,6 +383,7 @@ func parse(input io.Reader) ([]types.Tool, error) { } if !context.seenParam && skipRegex.MatchString(line) { + context.skipLines = append(context.skipLines, line) context.skipNode = true continue } diff --git a/pkg/parser/parser_test.go b/pkg/parser/parser_test.go index 0127b566..2f150aee 100644 --- a/pkg/parser/parser_test.go +++ b/pkg/parser/parser_test.go @@ -21,23 +21,25 @@ tools: bar AssignGlobals: true, }) require.NoError(t, err) - autogold.Expect([]types.Tool{ - { - Parameters: types.Parameters{ - ModelName: "the model", - Tools: []string{ - "foo", - "bar", + autogold.Expect(Document{Nodes: []Node{ + {ToolNode: &ToolNode{ + Tool: types.Tool{ + Parameters: types.Parameters{ + ModelName: "the model", + Tools: []string{ + "foo", + "bar", + }, + GlobalTools: []string{ + "foo", + "bar", + }, + GlobalModelName: "the model", }, - GlobalTools: []string{ - "foo", - "bar", - }, - GlobalModelName: "the model", + Source: types.ToolSource{LineNo: 1}, }, - Source: types.ToolSource{LineNo: 1}, - }, - { + }}, + {ToolNode: &ToolNode{Tool: types.Tool{ Parameters: types.Parameters{ Name: "bar", ModelName: "the model", @@ -47,8 +49,8 @@ tools: bar }, }, Source: types.ToolSource{LineNo: 5}, - }, - }).Equal(t, out) + }}}, + }}).Equal(t, out) } func TestParseSkip(t *testing.T) { @@ -85,28 +87,47 @@ name: seven ` out, err := Parse(strings.NewReader(input)) require.NoError(t, err) - autogold.Expect([]types.Tool{ - { - Instructions: "first", - Source: types.ToolSource{LineNo: 1}, - }, - { + autogold.Expect(Document{Nodes: []Node{ + {ToolNode: &ToolNode{ + Tool: types.Tool{ + Instructions: "first", + Source: types.ToolSource{ + LineNo: 1, + }, + }, + }}, + {ToolNode: &ToolNode{Tool: types.Tool{ Parameters: types.Parameters{Name: "second"}, Source: types.ToolSource{LineNo: 4}, - }, - { + }}}, + {TextNode: &TextNode{Text: "!third\n\nname: third\n"}}, + {ToolNode: &ToolNode{Tool: types.Tool{ Parameters: types.Parameters{Name: "fourth"}, Instructions: "!forth dont skip", Source: types.ToolSource{LineNo: 11}, - }, - { + }}}, + {ToolNode: &ToolNode{Tool: types.Tool{ Parameters: types.Parameters{Name: "fifth"}, Instructions: "#!ignore", Source: types.ToolSource{LineNo: 14}, - }, - { - Parameters: types.Parameters{Name: "seven"}, - Source: types.ToolSource{LineNo: 30}, - }, - }).Equal(t, out) + }}}, + {TextNode: &TextNode{Text: `!skip +name: six + +---- +name: bad + --- +name: bad +-- +name: bad +--- +name: bad +`}}, + {ToolNode: &ToolNode{Tool: types.Tool{ + Parameters: types.Parameters{ + Name: "seven", + }, + Source: types.ToolSource{LineNo: 30}, + }}}, + }}).Equal(t, out) } diff --git a/pkg/types/tool.go b/pkg/types/tool.go index 3b43b8d6..b6eb8528 100644 --- a/pkg/types/tool.go +++ b/pkg/types/tool.go @@ -251,7 +251,7 @@ func (t Tool) String() string { _, _ = fmt.Fprintln(buf, "Cache: false") } if t.Parameters.Temperature != nil { - _, _ = fmt.Fprintf(buf, "Temperature: %f", *t.Parameters.Temperature) + _, _ = fmt.Fprintf(buf, "Temperature: %f\n", *t.Parameters.Temperature) } if t.Parameters.Arguments != nil { var keys []string @@ -275,7 +275,7 @@ func (t Tool) String() string { _, _ = fmt.Fprintf(buf, "Credentials: %s\n", strings.Join(t.Parameters.Credentials, ", ")) } if t.Chat { - _, _ = fmt.Fprintf(buf, "Chat: true") + _, _ = fmt.Fprintf(buf, "Chat: true\n") } return buf.String()