From 04dd074694d7928ecbb124635e2ea5ccad77f0a6 Mon Sep 17 00:00:00 2001 From: Darren Shepherd Date: Tue, 27 Aug 2024 09:52:59 -0700 Subject: [PATCH 1/3] chore: add with * syntax to context tools Basic example chat: true context: foo with * Say hi --- name: foo #!/bin/bash echo This is the input: ${GPTSCRIPT_INPUT} --- pkg/runner/runner.go | 14 +++- pkg/tests/runner2_test.go | 34 ++++++++++ .../TestContextWithAsterick/call1-resp.golden | 9 +++ .../TestContextWithAsterick/call1.golden | 25 +++++++ .../TestContextWithAsterick/call2-resp.golden | 9 +++ .../TestContextWithAsterick/call2.golden | 43 ++++++++++++ .../TestContextWithAsterick/step1.golden | 48 ++++++++++++++ .../TestContextWithAsterick/step2.golden | 66 +++++++++++++++++++ pkg/tests/tester/runner.go | 18 ++++- 9 files changed, 262 insertions(+), 4 deletions(-) create mode 100644 pkg/tests/runner2_test.go create mode 100644 pkg/tests/testdata/TestContextWithAsterick/call1-resp.golden create mode 100644 pkg/tests/testdata/TestContextWithAsterick/call1.golden create mode 100644 pkg/tests/testdata/TestContextWithAsterick/call2-resp.golden create mode 100644 pkg/tests/testdata/TestContextWithAsterick/call2.golden create mode 100644 pkg/tests/testdata/TestContextWithAsterick/step1.golden create mode 100644 pkg/tests/testdata/TestContextWithAsterick/step2.golden diff --git a/pkg/runner/runner.go b/pkg/runner/runner.go index 93e40670..30737a4c 100644 --- a/pkg/runner/runner.go +++ b/pkg/runner/runner.go @@ -260,6 +260,10 @@ func getToolRefInput(prg *types.Program, ref types.ToolReference, input string) targetArgs := prg.ToolSet[ref.ToolID].Arguments targetKeys := map[string]string{} + if ref.Arg == "*" { + return input, nil + } + if targetArgs == nil { return "", nil } @@ -647,13 +651,17 @@ func (r *Runner) resume(callCtx engine.Context, monitor Monitor, env []string, s Env: env, } - var contentInput string + var contextInput string if state.Continuation != nil && state.Continuation.State != nil { - contentInput = state.Continuation.State.Input + contextInput = state.Continuation.State.Input + } + + if state.ResumeInput != nil { + contextInput = *state.ResumeInput } - callCtx.InputContext, state, err = r.getContext(callCtx, state, monitor, env, contentInput) + callCtx.InputContext, state, err = r.getContext(callCtx, state, monitor, env, contextInput) if err != nil || state.InputContextContinuation != nil { return state, err } diff --git a/pkg/tests/runner2_test.go b/pkg/tests/runner2_test.go new file mode 100644 index 00000000..12ac4fa0 --- /dev/null +++ b/pkg/tests/runner2_test.go @@ -0,0 +1,34 @@ +package tests + +import ( + "context" + "testing" + + "github.com/gptscript-ai/gptscript/pkg/loader" + "github.com/gptscript-ai/gptscript/pkg/tests/tester" + "github.com/stretchr/testify/require" +) + +func TestContextWithAsterick(t *testing.T) { + r := tester.NewRunner(t) + prg, err := loader.ProgramFromSource(context.Background(), ` +chat: true +context: foo with * + +Say hi + +--- +name: foo + +#!/bin/bash + +echo This is the input: ${GPTSCRIPT_INPUT} +`, "") + require.NoError(t, err) + + resp, err := r.Chat(context.Background(), nil, prg, nil, "input 1") + r.AssertStep(t, resp, err) + + resp, err = r.Chat(context.Background(), resp.State, prg, nil, "input 2") + r.AssertStep(t, resp, err) +} diff --git a/pkg/tests/testdata/TestContextWithAsterick/call1-resp.golden b/pkg/tests/testdata/TestContextWithAsterick/call1-resp.golden new file mode 100644 index 00000000..2861a036 --- /dev/null +++ b/pkg/tests/testdata/TestContextWithAsterick/call1-resp.golden @@ -0,0 +1,9 @@ +`{ + "role": "assistant", + "content": [ + { + "text": "TEST RESULT CALL: 1" + } + ], + "usage": {} +}` diff --git a/pkg/tests/testdata/TestContextWithAsterick/call1.golden b/pkg/tests/testdata/TestContextWithAsterick/call1.golden new file mode 100644 index 00000000..6d9538ce --- /dev/null +++ b/pkg/tests/testdata/TestContextWithAsterick/call1.golden @@ -0,0 +1,25 @@ +`{ + "model": "gpt-4o", + "internalSystemPrompt": false, + "messages": [ + { + "role": "system", + "content": [ + { + "text": "This is the input: input 1\n\nSay hi" + } + ], + "usage": {} + }, + { + "role": "user", + "content": [ + { + "text": "input 1" + } + ], + "usage": {} + } + ], + "chat": true +}` diff --git a/pkg/tests/testdata/TestContextWithAsterick/call2-resp.golden b/pkg/tests/testdata/TestContextWithAsterick/call2-resp.golden new file mode 100644 index 00000000..997ca1b9 --- /dev/null +++ b/pkg/tests/testdata/TestContextWithAsterick/call2-resp.golden @@ -0,0 +1,9 @@ +`{ + "role": "assistant", + "content": [ + { + "text": "TEST RESULT CALL: 2" + } + ], + "usage": {} +}` diff --git a/pkg/tests/testdata/TestContextWithAsterick/call2.golden b/pkg/tests/testdata/TestContextWithAsterick/call2.golden new file mode 100644 index 00000000..f159014c --- /dev/null +++ b/pkg/tests/testdata/TestContextWithAsterick/call2.golden @@ -0,0 +1,43 @@ +`{ + "model": "gpt-4o", + "internalSystemPrompt": false, + "messages": [ + { + "role": "system", + "content": [ + { + "text": "This is the input: input 2\n\nSay hi" + } + ], + "usage": {} + }, + { + "role": "user", + "content": [ + { + "text": "input 1" + } + ], + "usage": {} + }, + { + "role": "assistant", + "content": [ + { + "text": "TEST RESULT CALL: 1" + } + ], + "usage": {} + }, + { + "role": "user", + "content": [ + { + "text": "input 2" + } + ], + "usage": {} + } + ], + "chat": true +}` diff --git a/pkg/tests/testdata/TestContextWithAsterick/step1.golden b/pkg/tests/testdata/TestContextWithAsterick/step1.golden new file mode 100644 index 00000000..cc42e6a6 --- /dev/null +++ b/pkg/tests/testdata/TestContextWithAsterick/step1.golden @@ -0,0 +1,48 @@ +`{ + "done": false, + "content": "TEST RESULT CALL: 1", + "toolID": "inline:", + "state": { + "continuation": { + "state": { + "input": "input 1", + "completion": { + "model": "gpt-4o", + "internalSystemPrompt": false, + "messages": [ + { + "role": "system", + "content": [ + { + "text": "This is the input: input 1\n\nSay hi" + } + ], + "usage": {} + }, + { + "role": "user", + "content": [ + { + "text": "input 1" + } + ], + "usage": {} + }, + { + "role": "assistant", + "content": [ + { + "text": "TEST RESULT CALL: 1" + } + ], + "usage": {} + } + ], + "chat": true + } + }, + "result": "TEST RESULT CALL: 1" + }, + "continuationToolID": "inline:" + } +}` diff --git a/pkg/tests/testdata/TestContextWithAsterick/step2.golden b/pkg/tests/testdata/TestContextWithAsterick/step2.golden new file mode 100644 index 00000000..02bc92fe --- /dev/null +++ b/pkg/tests/testdata/TestContextWithAsterick/step2.golden @@ -0,0 +1,66 @@ +`{ + "done": false, + "content": "TEST RESULT CALL: 2", + "toolID": "inline:", + "state": { + "continuation": { + "state": { + "input": "input 1", + "completion": { + "model": "gpt-4o", + "internalSystemPrompt": false, + "messages": [ + { + "role": "system", + "content": [ + { + "text": "This is the input: input 2\n\nSay hi" + } + ], + "usage": {} + }, + { + "role": "user", + "content": [ + { + "text": "input 1" + } + ], + "usage": {} + }, + { + "role": "assistant", + "content": [ + { + "text": "TEST RESULT CALL: 1" + } + ], + "usage": {} + }, + { + "role": "user", + "content": [ + { + "text": "input 2" + } + ], + "usage": {} + }, + { + "role": "assistant", + "content": [ + { + "text": "TEST RESULT CALL: 2" + } + ], + "usage": {} + } + ], + "chat": true + } + }, + "result": "TEST RESULT CALL: 2" + }, + "continuationToolID": "inline:" + } +}` diff --git a/pkg/tests/tester/runner.go b/pkg/tests/tester/runner.go index 66337ff5..b460ce18 100644 --- a/pkg/tests/tester/runner.go +++ b/pkg/tests/tester/runner.go @@ -135,7 +135,8 @@ func (c *Client) Call(_ context.Context, messageRequest types.CompletionRequest, type Runner struct { *runner.Runner - Client *Client + Client *Client + StepAsserted int } func (r *Runner) RunDefault() string { @@ -166,6 +167,21 @@ func (r *Runner) AssertResponded(t *testing.T) { require.Len(t, r.Client.result, 0) } +func toJSONString(t *testing.T, v interface{}) string { + t.Helper() + x, err := json.MarshalIndent(v, "", " ") + require.NoError(t, err) + return string(x) +} + +func (r *Runner) AssertStep(t *testing.T, resp runner.ChatResponse, err error) { + t.Helper() + r.StepAsserted++ + require.NoError(t, err) + r.AssertResponded(t) + autogold.ExpectFile(t, toJSONString(t, resp), autogold.Name(t.Name()+fmt.Sprintf("/step%d", r.StepAsserted))) +} + func (r *Runner) RespondWith(result ...Result) { r.Client.result = append(r.Client.result, result...) } From 1c75954594d59abe8ae371bc7c96ce953eec4101 Mon Sep 17 00:00:00 2001 From: Darren Shepherd Date: Tue, 27 Aug 2024 10:30:30 -0700 Subject: [PATCH 2/3] chore: drop input continuations feature --- pkg/runner/runner.go | 97 ++------- pkg/tests/runner_test.go | 78 +------- .../TestContextSubChat/call10-resp.golden | 9 - .../testdata/TestContextSubChat/call10.golden | 43 ---- .../TestContextSubChat/call3-resp.golden | 16 -- .../testdata/TestContextSubChat/call3.golden | 61 ------ .../TestContextSubChat/call4-resp.golden | 9 - .../testdata/TestContextSubChat/call4.golden | 64 ------ .../TestContextSubChat/call5-resp.golden | 9 - .../testdata/TestContextSubChat/call5.golden | 25 --- .../TestContextSubChat/call6-resp.golden | 16 -- .../testdata/TestContextSubChat/call6.golden | 31 --- .../TestContextSubChat/call7-resp.golden | 9 - .../testdata/TestContextSubChat/call7.golden | 43 ---- .../TestContextSubChat/call8-resp.golden | 16 -- .../testdata/TestContextSubChat/call8.golden | 61 ------ .../TestContextSubChat/call9-resp.golden | 9 - .../testdata/TestContextSubChat/call9.golden | 64 ------ .../testdata/TestContextSubChat/step1.golden | 146 -------------- .../testdata/TestContextSubChat/step2.golden | 48 ----- .../testdata/TestContextSubChat/step3.golden | 188 ------------------ .../testdata/TestContextSubChat/step4.golden | 66 ------ 22 files changed, 20 insertions(+), 1088 deletions(-) delete mode 100644 pkg/tests/testdata/TestContextSubChat/call10-resp.golden delete mode 100644 pkg/tests/testdata/TestContextSubChat/call10.golden delete mode 100644 pkg/tests/testdata/TestContextSubChat/call3-resp.golden delete mode 100644 pkg/tests/testdata/TestContextSubChat/call3.golden delete mode 100644 pkg/tests/testdata/TestContextSubChat/call4-resp.golden delete mode 100644 pkg/tests/testdata/TestContextSubChat/call4.golden delete mode 100644 pkg/tests/testdata/TestContextSubChat/call5-resp.golden delete mode 100644 pkg/tests/testdata/TestContextSubChat/call5.golden delete mode 100644 pkg/tests/testdata/TestContextSubChat/call6-resp.golden delete mode 100644 pkg/tests/testdata/TestContextSubChat/call6.golden delete mode 100644 pkg/tests/testdata/TestContextSubChat/call7-resp.golden delete mode 100644 pkg/tests/testdata/TestContextSubChat/call7.golden delete mode 100644 pkg/tests/testdata/TestContextSubChat/call8-resp.golden delete mode 100644 pkg/tests/testdata/TestContextSubChat/call8.golden delete mode 100644 pkg/tests/testdata/TestContextSubChat/call9-resp.golden delete mode 100644 pkg/tests/testdata/TestContextSubChat/call9.golden delete mode 100644 pkg/tests/testdata/TestContextSubChat/step1.golden delete mode 100644 pkg/tests/testdata/TestContextSubChat/step2.golden delete mode 100644 pkg/tests/testdata/TestContextSubChat/step3.golden delete mode 100644 pkg/tests/testdata/TestContextSubChat/step4.golden diff --git a/pkg/runner/runner.go b/pkg/runner/runner.go index 30737a4c..3035a1d1 100644 --- a/pkg/runner/runner.go +++ b/pkg/runner/runner.go @@ -172,11 +172,7 @@ func (r *Runner) Chat(ctx context.Context, prevState ChatState, prg types.Progra return resp, err } - if state == nil || state.StartContinuation { - if state != nil { - state = state.WithResumeInput(&input) - input = state.InputContextContinuationInput - } + if state == nil { state, err = r.start(callCtx, state, monitor, env, input) if err != nil { return resp, err @@ -186,11 +182,9 @@ func (r *Runner) Chat(ctx context.Context, prevState ChatState, prg types.Progra state.ResumeInput = &input } - if !state.StartContinuation { - state, err = r.resume(callCtx, monitor, env, state) - if err != nil { - return resp, err - } + state, err = r.resume(callCtx, monitor, env, state) + if err != nil { + return resp, err } if state.Result != nil { @@ -335,24 +329,10 @@ func getToolRefInput(prg *types.Program, ref types.ToolReference, input string) return string(output), err } -func (r *Runner) getContext(callCtx engine.Context, state *State, monitor Monitor, env []string, input string) (result []engine.InputContext, _ *State, _ error) { +func (r *Runner) getContext(callCtx engine.Context, state *State, monitor Monitor, env []string, input string) (result []engine.InputContext, _ error) { toolRefs, err := callCtx.Tool.GetContextTools(*callCtx.Program) if err != nil { - return nil, nil, err - } - - var newState *State - if state != nil { - cp := *state - newState = &cp - if newState.InputContextContinuation != nil { - newState.InputContexts = nil - newState.InputContextContinuation = nil - newState.InputContextContinuationInput = "" - newState.ResumeInput = state.InputContextContinuationResumeInput - - input = state.InputContextContinuationInput - } + return nil, err } for i, toolRef := range toolRefs { @@ -363,29 +343,16 @@ func (r *Runner) getContext(callCtx engine.Context, state *State, monitor Monito contextInput, err := getToolRefInput(callCtx.Program, toolRef, input) if err != nil { - return nil, nil, err + return nil, err } var content *State - if state != nil && state.InputContextContinuation != nil { - content, err = r.subCallResume(callCtx.Ctx, callCtx, monitor, env, toolRef.ToolID, "", state.InputContextContinuation.WithResumeInput(state.ResumeInput), engine.ContextToolCategory) - } else { - content, err = r.subCall(callCtx.Ctx, callCtx, monitor, env, toolRef.ToolID, contextInput, "", engine.ContextToolCategory) - } + content, err = r.subCall(callCtx.Ctx, callCtx, monitor, env, toolRef.ToolID, contextInput, "", engine.ContextToolCategory) if err != nil { - return nil, nil, err + return nil, err } if content.Continuation != nil { - if newState == nil { - newState = &State{} - } - newState.InputContexts = result - newState.InputContextContinuation = content - newState.InputContextContinuationInput = input - if state != nil { - newState.InputContextContinuationResumeInput = state.ResumeInput - } - return nil, newState, nil + return nil, fmt.Errorf("invalid state: context tool [%s] can not result in a continuation", toolRef.ToolID) } result = append(result, engine.InputContext{ ToolID: toolRef.ToolID, @@ -393,7 +360,7 @@ func (r *Runner) getContext(callCtx engine.Context, state *State, monitor Monito }) } - return result, newState, nil + return result, nil } func (r *Runner) call(callCtx engine.Context, monitor Monitor, env []string, input string) (*State, error) { @@ -401,9 +368,6 @@ func (r *Runner) call(callCtx engine.Context, monitor Monitor, env []string, inp if err != nil { return nil, err } - if result.StartContinuation { - return result, nil - } return r.resume(callCtx, monitor, env, result) } @@ -435,15 +399,10 @@ func (r *Runner) start(callCtx engine.Context, state *State, monitor Monitor, en } } - var newState *State - callCtx.InputContext, newState, err = r.getContext(callCtx, state, monitor, env, input) + callCtx.InputContext, err = r.getContext(callCtx, state, monitor, env, input) if err != nil { return nil, err } - if newState != nil && newState.InputContextContinuation != nil { - newState.StartContinuation = true - return newState, nil - } e := engine.Engine{ Model: r.c, @@ -493,11 +452,7 @@ type State struct { SubCalls []SubCallResult `json:"subCalls,omitempty"` SubCallID string `json:"subCallID,omitempty"` - InputContexts []engine.InputContext `json:"inputContexts,omitempty"` - InputContextContinuation *State `json:"inputContextContinuation,omitempty"` - InputContextContinuationInput string `json:"inputContextContinuationInput,omitempty"` - InputContextContinuationResumeInput *string `json:"inputContextContinuationResumeInput,omitempty"` - StartContinuation bool `json:"startContinuation,omitempty"` + InputContexts []engine.InputContext `json:"inputContexts,omitempty"` } func (s State) WithResumeInput(input *string) *State { @@ -510,10 +465,6 @@ func (s State) ContinuationContentToolID() (string, error) { return s.ContinuationToolID, nil } - if s.InputContextContinuation != nil { - return s.InputContextContinuation.ContinuationContentToolID() - } - for _, subCall := range s.SubCalls { if s.SubCallID == subCall.CallID { return subCall.State.ContinuationContentToolID() @@ -527,10 +478,6 @@ func (s State) ContinuationContent() (string, error) { return *s.Continuation.Result, nil } - if s.InputContextContinuation != nil { - return s.InputContextContinuation.ContinuationContent() - } - for _, subCall := range s.SubCalls { if s.SubCallID == subCall.CallID { return subCall.State.ContinuationContent() @@ -549,10 +496,6 @@ func (r *Runner) resume(callCtx engine.Context, monitor Monitor, env []string, s retState, retErr = r.handleOutput(callCtx, monitor, env, retState, retErr) }() - if state.StartContinuation { - return nil, fmt.Errorf("invalid state, resume should not have StartContinuation set to true") - } - if state.Continuation == nil { return nil, errors.New("invalid state, resume should have Continuation data") } @@ -651,18 +594,18 @@ func (r *Runner) resume(callCtx engine.Context, monitor Monitor, env []string, s Env: env, } - var contextInput string + var contentInput string if state.Continuation != nil && state.Continuation.State != nil { - contextInput = state.Continuation.State.Input + contentInput = state.Continuation.State.Input } if state.ResumeInput != nil { - contextInput = *state.ResumeInput + contentInput = *state.ResumeInput } - callCtx.InputContext, state, err = r.getContext(callCtx, state, monitor, env, contextInput) - if err != nil || state.InputContextContinuation != nil { + callCtx.InputContext, err = r.getContext(callCtx, state, monitor, env, contentInput) + if err != nil { return state, err } @@ -772,10 +715,6 @@ func (r *Runner) subCalls(callCtx engine.Context, monitor Monitor, env []string, callCtx.LastReturn = state.Continuation } - if state.InputContextContinuation != nil { - return state, nil, nil - } - if state.SubCallID != "" { if state.ResumeInput == nil { return nil, nil, fmt.Errorf("invalid state, input must be set for sub call continuation on callID [%s]", state.SubCallID) diff --git a/pkg/tests/runner_test.go b/pkg/tests/runner_test.go index 141e6aff..f21f936e 100644 --- a/pkg/tests/runner_test.go +++ b/pkg/tests/runner_test.go @@ -212,82 +212,8 @@ func TestContextSubChat(t *testing.T) { prg, err := r.Load("") require.NoError(t, err) - resp, err := r.Chat(context.Background(), nil, prg, os.Environ(), "User 1") - require.NoError(t, err) - r.AssertResponded(t) - assert.False(t, resp.Done) - autogold.Expect("Assistant Response 1 - from chatbot1").Equal(t, resp.Content) - autogold.ExpectFile(t, toJSONString(t, resp), autogold.Name(t.Name()+"/step1")) - - r.RespondWith(tester.Result{ - Content: []types.ContentPart{ - { - ToolCall: &types.CompletionToolCall{ - ID: "call_2", - Function: types.CompletionFunctionCall{ - Name: types.ToolNormalizer("sys.chat.finish"), - Arguments: "Response from context chatbot", - }, - }, - }, - }, - }, tester.Result{ - Text: "Assistant Response 2 - from context tool", - }, tester.Result{ - Text: "Assistant Response 3 - from main chat tool", - }) - resp, err = r.Chat(context.Background(), resp.State, prg, os.Environ(), "User 2") - require.NoError(t, err) - r.AssertResponded(t) - assert.False(t, resp.Done) - autogold.Expect("Assistant Response 3 - from main chat tool").Equal(t, resp.Content) - autogold.ExpectFile(t, toJSONString(t, resp), autogold.Name(t.Name()+"/step2")) - - r.RespondWith(tester.Result{ - Content: []types.ContentPart{ - { - ToolCall: &types.CompletionToolCall{ - ID: "call_3", - Function: types.CompletionFunctionCall{ - Name: "chatbot", - Arguments: "Input to chatbot1 on resume", - }, - }, - }, - }, - }, tester.Result{ - Text: "Assistant Response 4 - from chatbot1", - }) - resp, err = r.Chat(context.Background(), resp.State, prg, os.Environ(), "User 3") - require.NoError(t, err) - r.AssertResponded(t) - assert.False(t, resp.Done) - autogold.Expect("Assistant Response 3 - from main chat tool").Equal(t, resp.Content) - autogold.ExpectFile(t, toJSONString(t, resp), autogold.Name(t.Name()+"/step3")) - - r.RespondWith(tester.Result{ - Content: []types.ContentPart{ - { - ToolCall: &types.CompletionToolCall{ - ID: "call_4", - Function: types.CompletionFunctionCall{ - Name: types.ToolNormalizer("sys.chat.finish"), - Arguments: "Response from context chatbot after resume", - }, - }, - }, - }, - }, tester.Result{ - Text: "Assistant Response 5 - from context tool resume", - }, tester.Result{ - Text: "Assistant Response 6 - from main chat tool resume", - }) - resp, err = r.Chat(context.Background(), resp.State, prg, os.Environ(), "User 4") - require.NoError(t, err) - r.AssertResponded(t) - assert.False(t, resp.Done) - autogold.Expect("Assistant Response 6 - from main chat tool resume").Equal(t, resp.Content) - autogold.ExpectFile(t, toJSONString(t, resp), autogold.Name(t.Name()+"/step4")) + _, err = r.Chat(context.Background(), nil, prg, os.Environ(), "User 1") + autogold.Expect("invalid state: context tool [testdata/TestContextSubChat/test.gpt:subtool] can not result in a continuation").Equal(t, err.Error()) } func TestSubChat(t *testing.T) { diff --git a/pkg/tests/testdata/TestContextSubChat/call10-resp.golden b/pkg/tests/testdata/TestContextSubChat/call10-resp.golden deleted file mode 100644 index 144ca8d9..00000000 --- a/pkg/tests/testdata/TestContextSubChat/call10-resp.golden +++ /dev/null @@ -1,9 +0,0 @@ -`{ - "role": "assistant", - "content": [ - { - "text": "Assistant Response 6 - from main chat tool resume" - } - ], - "usage": {} -}` diff --git a/pkg/tests/testdata/TestContextSubChat/call10.golden b/pkg/tests/testdata/TestContextSubChat/call10.golden deleted file mode 100644 index c8c98651..00000000 --- a/pkg/tests/testdata/TestContextSubChat/call10.golden +++ /dev/null @@ -1,43 +0,0 @@ -`{ - "model": "gpt-4o", - "internalSystemPrompt": false, - "messages": [ - { - "role": "system", - "content": [ - { - "text": "Assistant Response 5 - from context tool resume\nHello" - } - ], - "usage": {} - }, - { - "role": "user", - "content": [ - { - "text": "User 1" - } - ], - "usage": {} - }, - { - "role": "assistant", - "content": [ - { - "text": "Assistant Response 3 - from main chat tool" - } - ], - "usage": {} - }, - { - "role": "user", - "content": [ - { - "text": "User 3" - } - ], - "usage": {} - } - ], - "chat": true -}` diff --git a/pkg/tests/testdata/TestContextSubChat/call3-resp.golden b/pkg/tests/testdata/TestContextSubChat/call3-resp.golden deleted file mode 100644 index b116d066..00000000 --- a/pkg/tests/testdata/TestContextSubChat/call3-resp.golden +++ /dev/null @@ -1,16 +0,0 @@ -`{ - "role": "assistant", - "content": [ - { - "toolCall": { - "index": 0, - "id": "call_2", - "function": { - "name": "chatFinish", - "arguments": "Response from context chatbot" - } - } - } - ], - "usage": {} -}` diff --git a/pkg/tests/testdata/TestContextSubChat/call3.golden b/pkg/tests/testdata/TestContextSubChat/call3.golden deleted file mode 100644 index 55ad402f..00000000 --- a/pkg/tests/testdata/TestContextSubChat/call3.golden +++ /dev/null @@ -1,61 +0,0 @@ -`{ - "model": "gpt-4o", - "internalSystemPrompt": false, - "tools": [ - { - "function": { - "toolID": "sys.chat.finish", - "name": "chatFinish", - "description": "Concludes the conversation. This can not be used to ask a question.", - "parameters": { - "properties": { - "return": { - "description": "The instructed value to return or a summary of the dialog if no value is instructed", - "type": "string" - } - }, - "type": "object" - } - } - } - ], - "messages": [ - { - "role": "system", - "content": [ - { - "text": "This is a chatbot" - } - ], - "usage": {} - }, - { - "role": "user", - "content": [ - { - "text": "Input to chatbot1" - } - ], - "usage": {} - }, - { - "role": "assistant", - "content": [ - { - "text": "Assistant Response 1 - from chatbot1" - } - ], - "usage": {} - }, - { - "role": "user", - "content": [ - { - "text": "User 1" - } - ], - "usage": {} - } - ], - "chat": true -}` diff --git a/pkg/tests/testdata/TestContextSubChat/call4-resp.golden b/pkg/tests/testdata/TestContextSubChat/call4-resp.golden deleted file mode 100644 index a86ae187..00000000 --- a/pkg/tests/testdata/TestContextSubChat/call4-resp.golden +++ /dev/null @@ -1,9 +0,0 @@ -`{ - "role": "assistant", - "content": [ - { - "text": "Assistant Response 2 - from context tool" - } - ], - "usage": {} -}` diff --git a/pkg/tests/testdata/TestContextSubChat/call4.golden b/pkg/tests/testdata/TestContextSubChat/call4.golden deleted file mode 100644 index e1fb91ea..00000000 --- a/pkg/tests/testdata/TestContextSubChat/call4.golden +++ /dev/null @@ -1,64 +0,0 @@ -`{ - "model": "gpt-4o", - "tools": [ - { - "function": { - "toolID": "testdata/TestContextSubChat/test.gpt:chatbot", - "name": "chatbot", - "parameters": { - "properties": { - "defaultPromptParameter": { - "description": "Prompt to send to the assistant. This may be an instruction or question.", - "type": "string" - } - }, - "type": "object" - } - } - } - ], - "messages": [ - { - "role": "system", - "content": [ - { - "text": "Call chatbot" - } - ], - "usage": {} - }, - { - "role": "assistant", - "content": [ - { - "toolCall": { - "index": 0, - "id": "call_1", - "function": { - "name": "chatbot", - "arguments": "Input to chatbot1" - } - } - } - ], - "usage": {} - }, - { - "role": "tool", - "content": [ - { - "text": "Response from context chatbot" - } - ], - "toolCall": { - "index": 0, - "id": "call_1", - "function": { - "name": "chatbot", - "arguments": "Input to chatbot1" - } - }, - "usage": {} - } - ] -}` diff --git a/pkg/tests/testdata/TestContextSubChat/call5-resp.golden b/pkg/tests/testdata/TestContextSubChat/call5-resp.golden deleted file mode 100644 index e49a8481..00000000 --- a/pkg/tests/testdata/TestContextSubChat/call5-resp.golden +++ /dev/null @@ -1,9 +0,0 @@ -`{ - "role": "assistant", - "content": [ - { - "text": "Assistant Response 3 - from main chat tool" - } - ], - "usage": {} -}` diff --git a/pkg/tests/testdata/TestContextSubChat/call5.golden b/pkg/tests/testdata/TestContextSubChat/call5.golden deleted file mode 100644 index 2b8cf41e..00000000 --- a/pkg/tests/testdata/TestContextSubChat/call5.golden +++ /dev/null @@ -1,25 +0,0 @@ -`{ - "model": "gpt-4o", - "internalSystemPrompt": false, - "messages": [ - { - "role": "system", - "content": [ - { - "text": "Assistant Response 2 - from context tool\nHello" - } - ], - "usage": {} - }, - { - "role": "user", - "content": [ - { - "text": "User 1" - } - ], - "usage": {} - } - ], - "chat": true -}` diff --git a/pkg/tests/testdata/TestContextSubChat/call6-resp.golden b/pkg/tests/testdata/TestContextSubChat/call6-resp.golden deleted file mode 100644 index 6807fce9..00000000 --- a/pkg/tests/testdata/TestContextSubChat/call6-resp.golden +++ /dev/null @@ -1,16 +0,0 @@ -`{ - "role": "assistant", - "content": [ - { - "toolCall": { - "index": 0, - "id": "call_3", - "function": { - "name": "chatbot", - "arguments": "Input to chatbot1 on resume" - } - } - } - ], - "usage": {} -}` diff --git a/pkg/tests/testdata/TestContextSubChat/call6.golden b/pkg/tests/testdata/TestContextSubChat/call6.golden deleted file mode 100644 index 225401db..00000000 --- a/pkg/tests/testdata/TestContextSubChat/call6.golden +++ /dev/null @@ -1,31 +0,0 @@ -`{ - "model": "gpt-4o", - "tools": [ - { - "function": { - "toolID": "testdata/TestContextSubChat/test.gpt:chatbot", - "name": "chatbot", - "parameters": { - "properties": { - "defaultPromptParameter": { - "description": "Prompt to send to the assistant. This may be an instruction or question.", - "type": "string" - } - }, - "type": "object" - } - } - } - ], - "messages": [ - { - "role": "system", - "content": [ - { - "text": "Call chatbot" - } - ], - "usage": {} - } - ] -}` diff --git a/pkg/tests/testdata/TestContextSubChat/call7-resp.golden b/pkg/tests/testdata/TestContextSubChat/call7-resp.golden deleted file mode 100644 index 3e0c5f3c..00000000 --- a/pkg/tests/testdata/TestContextSubChat/call7-resp.golden +++ /dev/null @@ -1,9 +0,0 @@ -`{ - "role": "assistant", - "content": [ - { - "text": "Assistant Response 4 - from chatbot1" - } - ], - "usage": {} -}` diff --git a/pkg/tests/testdata/TestContextSubChat/call7.golden b/pkg/tests/testdata/TestContextSubChat/call7.golden deleted file mode 100644 index b0ef4e39..00000000 --- a/pkg/tests/testdata/TestContextSubChat/call7.golden +++ /dev/null @@ -1,43 +0,0 @@ -`{ - "model": "gpt-4o", - "internalSystemPrompt": false, - "tools": [ - { - "function": { - "toolID": "sys.chat.finish", - "name": "chatFinish", - "description": "Concludes the conversation. This can not be used to ask a question.", - "parameters": { - "properties": { - "return": { - "description": "The instructed value to return or a summary of the dialog if no value is instructed", - "type": "string" - } - }, - "type": "object" - } - } - } - ], - "messages": [ - { - "role": "system", - "content": [ - { - "text": "This is a chatbot" - } - ], - "usage": {} - }, - { - "role": "user", - "content": [ - { - "text": "Input to chatbot1 on resume" - } - ], - "usage": {} - } - ], - "chat": true -}` diff --git a/pkg/tests/testdata/TestContextSubChat/call8-resp.golden b/pkg/tests/testdata/TestContextSubChat/call8-resp.golden deleted file mode 100644 index 2e608b31..00000000 --- a/pkg/tests/testdata/TestContextSubChat/call8-resp.golden +++ /dev/null @@ -1,16 +0,0 @@ -`{ - "role": "assistant", - "content": [ - { - "toolCall": { - "index": 0, - "id": "call_4", - "function": { - "name": "chatFinish", - "arguments": "Response from context chatbot after resume" - } - } - } - ], - "usage": {} -}` diff --git a/pkg/tests/testdata/TestContextSubChat/call8.golden b/pkg/tests/testdata/TestContextSubChat/call8.golden deleted file mode 100644 index 3d0db61b..00000000 --- a/pkg/tests/testdata/TestContextSubChat/call8.golden +++ /dev/null @@ -1,61 +0,0 @@ -`{ - "model": "gpt-4o", - "internalSystemPrompt": false, - "tools": [ - { - "function": { - "toolID": "sys.chat.finish", - "name": "chatFinish", - "description": "Concludes the conversation. This can not be used to ask a question.", - "parameters": { - "properties": { - "return": { - "description": "The instructed value to return or a summary of the dialog if no value is instructed", - "type": "string" - } - }, - "type": "object" - } - } - } - ], - "messages": [ - { - "role": "system", - "content": [ - { - "text": "This is a chatbot" - } - ], - "usage": {} - }, - { - "role": "user", - "content": [ - { - "text": "Input to chatbot1 on resume" - } - ], - "usage": {} - }, - { - "role": "assistant", - "content": [ - { - "text": "Assistant Response 4 - from chatbot1" - } - ], - "usage": {} - }, - { - "role": "user", - "content": [ - { - "text": "User 4" - } - ], - "usage": {} - } - ], - "chat": true -}` diff --git a/pkg/tests/testdata/TestContextSubChat/call9-resp.golden b/pkg/tests/testdata/TestContextSubChat/call9-resp.golden deleted file mode 100644 index 4424246d..00000000 --- a/pkg/tests/testdata/TestContextSubChat/call9-resp.golden +++ /dev/null @@ -1,9 +0,0 @@ -`{ - "role": "assistant", - "content": [ - { - "text": "Assistant Response 5 - from context tool resume" - } - ], - "usage": {} -}` diff --git a/pkg/tests/testdata/TestContextSubChat/call9.golden b/pkg/tests/testdata/TestContextSubChat/call9.golden deleted file mode 100644 index 33768f26..00000000 --- a/pkg/tests/testdata/TestContextSubChat/call9.golden +++ /dev/null @@ -1,64 +0,0 @@ -`{ - "model": "gpt-4o", - "tools": [ - { - "function": { - "toolID": "testdata/TestContextSubChat/test.gpt:chatbot", - "name": "chatbot", - "parameters": { - "properties": { - "defaultPromptParameter": { - "description": "Prompt to send to the assistant. This may be an instruction or question.", - "type": "string" - } - }, - "type": "object" - } - } - } - ], - "messages": [ - { - "role": "system", - "content": [ - { - "text": "Call chatbot" - } - ], - "usage": {} - }, - { - "role": "assistant", - "content": [ - { - "toolCall": { - "index": 0, - "id": "call_3", - "function": { - "name": "chatbot", - "arguments": "Input to chatbot1 on resume" - } - } - } - ], - "usage": {} - }, - { - "role": "tool", - "content": [ - { - "text": "Response from context chatbot after resume" - } - ], - "toolCall": { - "index": 0, - "id": "call_3", - "function": { - "name": "chatbot", - "arguments": "Input to chatbot1 on resume" - } - }, - "usage": {} - } - ] -}` diff --git a/pkg/tests/testdata/TestContextSubChat/step1.golden b/pkg/tests/testdata/TestContextSubChat/step1.golden deleted file mode 100644 index 2ffb138e..00000000 --- a/pkg/tests/testdata/TestContextSubChat/step1.golden +++ /dev/null @@ -1,146 +0,0 @@ -`{ - "done": false, - "content": "Assistant Response 1 - from chatbot1", - "toolID": "testdata/TestContextSubChat/test.gpt:chatbot", - "state": { - "inputContextContinuation": { - "continuation": { - "state": { - "completion": { - "model": "gpt-4o", - "tools": [ - { - "function": { - "toolID": "testdata/TestContextSubChat/test.gpt:chatbot", - "name": "chatbot", - "parameters": { - "properties": { - "defaultPromptParameter": { - "description": "Prompt to send to the assistant. This may be an instruction or question.", - "type": "string" - } - }, - "type": "object" - } - } - } - ], - "messages": [ - { - "role": "system", - "content": [ - { - "text": "Call chatbot" - } - ], - "usage": {} - }, - { - "role": "assistant", - "content": [ - { - "toolCall": { - "index": 0, - "id": "call_1", - "function": { - "name": "chatbot", - "arguments": "Input to chatbot1" - } - } - } - ], - "usage": {} - } - ] - }, - "pending": { - "call_1": { - "index": 0, - "id": "call_1", - "function": { - "name": "chatbot", - "arguments": "Input to chatbot1" - } - } - } - }, - "calls": { - "call_1": { - "toolID": "testdata/TestContextSubChat/test.gpt:chatbot", - "input": "Input to chatbot1" - } - } - }, - "subCalls": [ - { - "toolId": "testdata/TestContextSubChat/test.gpt:chatbot", - "callId": "call_1", - "state": { - "continuation": { - "state": { - "input": "Input to chatbot1", - "completion": { - "model": "gpt-4o", - "internalSystemPrompt": false, - "tools": [ - { - "function": { - "toolID": "sys.chat.finish", - "name": "chatFinish", - "description": "Concludes the conversation. This can not be used to ask a question.", - "parameters": { - "properties": { - "return": { - "description": "The instructed value to return or a summary of the dialog if no value is instructed", - "type": "string" - } - }, - "type": "object" - } - } - } - ], - "messages": [ - { - "role": "system", - "content": [ - { - "text": "This is a chatbot" - } - ], - "usage": {} - }, - { - "role": "user", - "content": [ - { - "text": "Input to chatbot1" - } - ], - "usage": {} - }, - { - "role": "assistant", - "content": [ - { - "text": "Assistant Response 1 - from chatbot1" - } - ], - "usage": {} - } - ], - "chat": true - } - }, - "result": "Assistant Response 1 - from chatbot1" - }, - "continuationToolID": "testdata/TestContextSubChat/test.gpt:chatbot" - } - } - ], - "subCallID": "call_1" - }, - "inputContextContinuationInput": "User 1", - "startContinuation": true - } -}` diff --git a/pkg/tests/testdata/TestContextSubChat/step2.golden b/pkg/tests/testdata/TestContextSubChat/step2.golden deleted file mode 100644 index dfcb2b96..00000000 --- a/pkg/tests/testdata/TestContextSubChat/step2.golden +++ /dev/null @@ -1,48 +0,0 @@ -`{ - "done": false, - "content": "Assistant Response 3 - from main chat tool", - "toolID": "testdata/TestContextSubChat/test.gpt:", - "state": { - "continuation": { - "state": { - "input": "User 1", - "completion": { - "model": "gpt-4o", - "internalSystemPrompt": false, - "messages": [ - { - "role": "system", - "content": [ - { - "text": "Assistant Response 2 - from context tool\nHello" - } - ], - "usage": {} - }, - { - "role": "user", - "content": [ - { - "text": "User 1" - } - ], - "usage": {} - }, - { - "role": "assistant", - "content": [ - { - "text": "Assistant Response 3 - from main chat tool" - } - ], - "usage": {} - } - ], - "chat": true - } - }, - "result": "Assistant Response 3 - from main chat tool" - }, - "continuationToolID": "testdata/TestContextSubChat/test.gpt:" - } -}` diff --git a/pkg/tests/testdata/TestContextSubChat/step3.golden b/pkg/tests/testdata/TestContextSubChat/step3.golden deleted file mode 100644 index 0ccb188b..00000000 --- a/pkg/tests/testdata/TestContextSubChat/step3.golden +++ /dev/null @@ -1,188 +0,0 @@ -`{ - "done": false, - "content": "Assistant Response 3 - from main chat tool", - "toolID": "testdata/TestContextSubChat/test.gpt:", - "state": { - "continuation": { - "state": { - "input": "User 1", - "completion": { - "model": "gpt-4o", - "internalSystemPrompt": false, - "messages": [ - { - "role": "system", - "content": [ - { - "text": "Assistant Response 2 - from context tool\nHello" - } - ], - "usage": {} - }, - { - "role": "user", - "content": [ - { - "text": "User 1" - } - ], - "usage": {} - }, - { - "role": "assistant", - "content": [ - { - "text": "Assistant Response 3 - from main chat tool" - } - ], - "usage": {} - } - ], - "chat": true - } - }, - "result": "Assistant Response 3 - from main chat tool" - }, - "continuationToolID": "testdata/TestContextSubChat/test.gpt:", - "resumeInput": "User 3", - "inputContextContinuation": { - "continuation": { - "state": { - "completion": { - "model": "gpt-4o", - "tools": [ - { - "function": { - "toolID": "testdata/TestContextSubChat/test.gpt:chatbot", - "name": "chatbot", - "parameters": { - "properties": { - "defaultPromptParameter": { - "description": "Prompt to send to the assistant. This may be an instruction or question.", - "type": "string" - } - }, - "type": "object" - } - } - } - ], - "messages": [ - { - "role": "system", - "content": [ - { - "text": "Call chatbot" - } - ], - "usage": {} - }, - { - "role": "assistant", - "content": [ - { - "toolCall": { - "index": 0, - "id": "call_3", - "function": { - "name": "chatbot", - "arguments": "Input to chatbot1 on resume" - } - } - } - ], - "usage": {} - } - ] - }, - "pending": { - "call_3": { - "index": 0, - "id": "call_3", - "function": { - "name": "chatbot", - "arguments": "Input to chatbot1 on resume" - } - } - } - }, - "calls": { - "call_3": { - "toolID": "testdata/TestContextSubChat/test.gpt:chatbot", - "input": "Input to chatbot1 on resume" - } - } - }, - "subCalls": [ - { - "toolId": "testdata/TestContextSubChat/test.gpt:chatbot", - "callId": "call_3", - "state": { - "continuation": { - "state": { - "input": "Input to chatbot1 on resume", - "completion": { - "model": "gpt-4o", - "internalSystemPrompt": false, - "tools": [ - { - "function": { - "toolID": "sys.chat.finish", - "name": "chatFinish", - "description": "Concludes the conversation. This can not be used to ask a question.", - "parameters": { - "properties": { - "return": { - "description": "The instructed value to return or a summary of the dialog if no value is instructed", - "type": "string" - } - }, - "type": "object" - } - } - } - ], - "messages": [ - { - "role": "system", - "content": [ - { - "text": "This is a chatbot" - } - ], - "usage": {} - }, - { - "role": "user", - "content": [ - { - "text": "Input to chatbot1 on resume" - } - ], - "usage": {} - }, - { - "role": "assistant", - "content": [ - { - "text": "Assistant Response 4 - from chatbot1" - } - ], - "usage": {} - } - ], - "chat": true - } - }, - "result": "Assistant Response 4 - from chatbot1" - }, - "continuationToolID": "testdata/TestContextSubChat/test.gpt:chatbot" - } - } - ], - "subCallID": "call_3" - }, - "inputContextContinuationInput": "User 1", - "inputContextContinuationResumeInput": "User 3" - } -}` diff --git a/pkg/tests/testdata/TestContextSubChat/step4.golden b/pkg/tests/testdata/TestContextSubChat/step4.golden deleted file mode 100644 index 5e95d626..00000000 --- a/pkg/tests/testdata/TestContextSubChat/step4.golden +++ /dev/null @@ -1,66 +0,0 @@ -`{ - "done": false, - "content": "Assistant Response 6 - from main chat tool resume", - "toolID": "testdata/TestContextSubChat/test.gpt:", - "state": { - "continuation": { - "state": { - "input": "User 1", - "completion": { - "model": "gpt-4o", - "internalSystemPrompt": false, - "messages": [ - { - "role": "system", - "content": [ - { - "text": "Assistant Response 5 - from context tool resume\nHello" - } - ], - "usage": {} - }, - { - "role": "user", - "content": [ - { - "text": "User 1" - } - ], - "usage": {} - }, - { - "role": "assistant", - "content": [ - { - "text": "Assistant Response 3 - from main chat tool" - } - ], - "usage": {} - }, - { - "role": "user", - "content": [ - { - "text": "User 3" - } - ], - "usage": {} - }, - { - "role": "assistant", - "content": [ - { - "text": "Assistant Response 6 - from main chat tool resume" - } - ], - "usage": {} - } - ], - "chat": true - } - }, - "result": "Assistant Response 6 - from main chat tool resume" - }, - "continuationToolID": "testdata/TestContextSubChat/test.gpt:" - } -}` From 93e77066e8dc357070728825eaecc5136852b67c Mon Sep 17 00:00:00 2001 From: Darren Shepherd Date: Tue, 27 Aug 2024 10:44:57 -0700 Subject: [PATCH 3/3] bug: handle case where node_modules is deleted in local tool dev --- pkg/repos/runtimes/node/node.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/pkg/repos/runtimes/node/node.go b/pkg/repos/runtimes/node/node.go index aa57b059..4d73c13b 100644 --- a/pkg/repos/runtimes/node/node.go +++ b/pkg/repos/runtimes/node/node.go @@ -26,6 +26,7 @@ var releasesData []byte const ( downloadURL = "https://nodejs.org/dist/%s/" packageJSON = "package.json" + nodeModules = "node_modules" ) type Runtime struct { @@ -64,8 +65,15 @@ func (r *Runtime) supports(testCmd string, cmd []string) bool { func (r *Runtime) GetHash(tool types.Tool) (string, error) { if !tool.Source.IsGit() && tool.WorkingDir != "" { + var prefix string + // This hashes if the node_modules directory was deleted + if s, err := os.Stat(filepath.Join(tool.WorkingDir, nodeModules)); err == nil { + prefix = hash.Digest(tool.WorkingDir + s.ModTime().String())[:7] + } else if s, err := os.Stat(tool.WorkingDir); err == nil { + prefix = hash.Digest(tool.WorkingDir + s.ModTime().String())[:7] + } if s, err := os.Stat(filepath.Join(tool.WorkingDir, packageJSON)); err == nil { - return hash.Digest(tool.WorkingDir + s.ModTime().String())[:7], nil + return prefix + hash.Digest(tool.WorkingDir + s.ModTime().String())[:7], nil } } return "", nil