feat: add parse and fmt CLI commands

pkg/cli/fmt.go

@@ -0,0 +1,51 @@
+package cli
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/gptscript-ai/gptscript/pkg/input"
+	"github.com/gptscript-ai/gptscript/pkg/parser"
+	"github.com/spf13/cobra"
+)
+
+type Fmt struct {
+	Write bool `usage:"Write output to file instead of stdout" short:"w"`
+}
+
+func (e *Fmt) Customize(cmd *cobra.Command) {
+	cmd.Args = cobra.ExactArgs(1)
+}
+
+func (e *Fmt) Run(_ *cobra.Command, args []string) error {
+	input, err := input.FromFile(args[0])
+	if err != nil {
+		return err
+	}
+
+	var (
+		doc parser.Document
+		loc = locationName(args[0])
+	)
+	if strings.HasPrefix(input, "{") {
+		if err := json.Unmarshal([]byte(input), &doc); err != nil {
+			return err
+		}
+	} else {
+		doc, err = parser.Parse(strings.NewReader(input), parser.Options{
+			Location: locationName(args[0]),
+		})
+		if err != nil {
+			return err
+		}
+	}
+
+	if e.Write && loc != "" {
+		return os.WriteFile(loc, []byte(doc.String()), 0644)
+	}
+
+	fmt.Print(doc.String())
+	return nil
+}

pkg/cli/gptscript.go

@@ -68,9 +68,15 @@ type GPTScript struct {
 
 func New() *cobra.Command {
 	root := &GPTScript{}
-	command := cmd.Command(root, &Eval{
-		gptscript: root,
-	}, &Credential{root: root})
+	command := cmd.Command(
+		root,
+		&Eval{
+			gptscript: root,
+		},
+		&Credential{root: root},
+		&Parse{},
+		&Fmt{},
+	)
 
 	// Hide all the global flags for the credential subcommand.
 	for _, child := range command.Commands() {

pkg/cli/parse.go

@@ -0,0 +1,47 @@
+package cli
+
+import (
+	"encoding/json"
+	"os"
+	"strings"
+
+	"github.com/gptscript-ai/gptscript/pkg/input"
+	"github.com/gptscript-ai/gptscript/pkg/parser"
+	"github.com/spf13/cobra"
+)
+
+type Parse struct {
+	PrettyPrint bool `usage:"Indent the json output" short:"p"`
+}
+
+func (e *Parse) Customize(cmd *cobra.Command) {
+	cmd.Args = cobra.ExactArgs(1)
+}
+
+func locationName(l string) string {
+	if l == "-" {
+		return ""
+	}
+	return l
+}
+
+func (e *Parse) Run(_ *cobra.Command, args []string) error {
+	input, err := input.FromFile(args[0])
+	if err != nil {
+		return err
+	}
+
+	docs, err := parser.Parse(strings.NewReader(input), parser.Options{
+		Location: locationName(args[0]),
+	})
+	if err != nil {
+		return err
+	}
+
+	enc := json.NewEncoder(os.Stdout)
+	if e.PrettyPrint {
+		enc.SetIndent("", "  ")
+	}
+
+	return enc.Encode(docs)
+}

pkg/loader/loader.go

@@ -147,7 +147,7 @@ func readTool(ctx context.Context, prg *types.Program, base *source, targetToolN
 
 	// If we didn't get any tools from trying to parse it as OpenAPI, try to parse it as a GPTScript
 	if len(tools) == 0 {
-		tools, err = parser.Parse(bytes.NewReader(data), parser.Options{
+		tools, err = parser.ParseTools(bytes.NewReader(data), parser.Options{
 			AssignGlobals: true,
 		})
 		if err != nil {

pkg/parser/parser.go

@@ -174,42 +174,124 @@ type context struct {
 	instructions []string
 	inBody       bool
 	skipNode     bool
+	skipLines    []string
 	seenParam    bool
 }
 
-func (c *context) finish(tools *[]types.Tool) {
+func (c *context) finish(tools *[]Node) {
 	c.tool.Instructions = strings.TrimSpace(strings.Join(c.instructions, ""))
 	if c.tool.Instructions != "" || c.tool.Parameters.Name != "" ||
 		len(c.tool.Export) > 0 || len(c.tool.Tools) > 0 ||
 		c.tool.GlobalModelName != "" ||
 		len(c.tool.GlobalTools) > 0 ||
 		c.tool.Chat {
-		*tools = append(*tools, c.tool)
+		*tools = append(*tools, Node{
+			ToolNode: &ToolNode{
+				Tool: c.tool,
+			},
+		})
+	}
+	if c.skipNode && len(c.skipLines) > 0 {
+		*tools = append(*tools, Node{
+			TextNode: &TextNode{
+				Text: strings.Join(c.skipLines, ""),
+			},
+		})
 	}
 	*c = context{}
 }
 
 type Options struct {
 	AssignGlobals bool
+	Location      string
 }
 
 func complete(opts ...Options) (result Options) {
 	for _, opt := range opts {
 		result.AssignGlobals = types.FirstSet(opt.AssignGlobals, result.AssignGlobals)
+		result.Location = types.FirstSet(opt.Location, result.Location)
 	}
 	return
 }
 
-func Parse(input io.Reader, opts ...Options) ([]types.Tool, error) {
-	tools, err := parse(input)
+type Document struct {
+	Nodes []Node `json:"nodes,omitempty"`
+}
+
+func writeSep(buf *strings.Builder, lastText bool) {
+	if buf.Len() > 0 {
+		if !lastText {
+			buf.WriteString("\n")
+		}
+		buf.WriteString("---\n")
+	}
+}
+
+func (d Document) String() string {
+	buf := strings.Builder{}
+	lastText := false
+	for _, node := range d.Nodes {
+		if node.TextNode != nil {
+			writeSep(&buf, lastText)
+			buf.WriteString(node.TextNode.Text)
+			lastText = true
+		}
+		if node.ToolNode != nil {
+			writeSep(&buf, lastText)
+			buf.WriteString(node.ToolNode.Tool.String())
+			lastText = false
+		}
+	}
+	return buf.String()
+}
+
+type Node struct {
+	TextNode *TextNode `json:"textNode,omitempty"`
+	ToolNode *ToolNode `json:"toolNode,omitempty"`
+}
+
+type TextNode struct {
+	Text string `json:"text,omitempty"`
+}
+
+type ToolNode struct {
+	Tool types.Tool `json:"tool,omitempty"`
+}
+
+func ParseTools(input io.Reader, opts ...Options) (result []types.Tool, _ error) {
+	doc, err := Parse(input, opts...)
 	if err != nil {
 		return nil, err
 	}
+	for _, node := range doc.Nodes {
+		if node.ToolNode != nil {
+			result = append(result, node.ToolNode.Tool)
+		}
+	}
+
+	return
+}
+
+func Parse(input io.Reader, opts ...Options) (Document, error) {
+	nodes, err := parse(input)
+	if err != nil {
+		return Document{}, err
+	}
 
 	opt := complete(opts...)
 
+	if opt.Location != "" {
+		for _, node := range nodes {
+			if node.ToolNode != nil && node.ToolNode.Tool.Source.Location == "" {
+				node.ToolNode.Tool.Source.Location = opt.Location
+			}
+		}
+	}
+
 	if !opt.AssignGlobals {
-		return tools, nil
+		return Document{
+			Nodes: nodes,
+		}, nil
 	}
 
 	var (
@@ -218,10 +300,14 @@ func Parse(input io.Reader, opts ...Options) ([]types.Tool, error) {
 		globalTools     []string
 	)
 
-	for _, tool := range tools {
+	for _, node := range nodes {
+		if node.ToolNode == nil {
+			continue
+		}
+		tool := node.ToolNode.Tool
 		if tool.GlobalModelName != "" {
 			if globalModel != "" {
-				return nil, fmt.Errorf("global model name defined multiple times")
+				return Document{}, fmt.Errorf("global model name defined multiple times")
 			}
 			globalModel = tool.GlobalModelName
 		}
@@ -234,26 +320,30 @@ func Parse(input io.Reader, opts ...Options) ([]types.Tool, error) {
 		}
 	}
 
-	for i, tool := range tools {
-		if globalModel != "" && tool.ModelName == "" {
-			tool.ModelName = globalModel
+	for _, node := range nodes {
+		if node.ToolNode == nil {
+			continue
+		}
+		if globalModel != "" && node.ToolNode.Tool.ModelName == "" {
+			node.ToolNode.Tool.ModelName = globalModel
 		}
 		for _, globalTool := range globalTools {
-			if !slices.Contains(tool.Tools, globalTool) {
-				tool.Tools = append(tool.Tools, globalTool)
+			if !slices.Contains(node.ToolNode.Tool.Tools, globalTool) {
+				node.ToolNode.Tool.Tools = append(node.ToolNode.Tool.Tools, globalTool)
 			}
 		}
-		tools[i] = tool
 	}
 
-	return tools, nil
+	return Document{
+		Nodes: nodes,
+	}, nil
 }
 
-func parse(input io.Reader) ([]types.Tool, error) {
+func parse(input io.Reader) ([]Node, error) {
 	scan := bufio.NewScanner(input)
 
 	var (
-		tools   []types.Tool
+		tools   []Node
 		context context
 		lineNo  int
 	)
@@ -277,6 +367,7 @@ func parse(input io.Reader) ([]types.Tool, error) {
 		}
 
 		if context.skipNode {
+			context.skipLines = append(context.skipLines, line)
 			continue
 		}
 
@@ -292,6 +383,7 @@ func parse(input io.Reader) ([]types.Tool, error) {
 			}
 
 			if !context.seenParam && skipRegex.MatchString(line) {
+				context.skipLines = append(context.skipLines, line)
 				context.skipNode = true
 				continue
 			}