From 2226e1c24f263d66642e6a78487b13a1f95a1a82 Mon Sep 17 00:00:00 2001 From: Darren Shepherd Date: Sun, 28 Apr 2024 23:31:40 -0700 Subject: [PATCH] change: allow chat continuation in context tools --- pkg/builtin/builtin.go | 4 +- pkg/runner/runner.go | 161 +++++++++++---- pkg/tests/runner_test.go | 99 ++++++++++ .../testdata/TestContextSubChat/call1.golden | 28 +++ .../testdata/TestContextSubChat/call10.golden | 44 +++++ .../testdata/TestContextSubChat/call2.golden | 45 +++++ .../testdata/TestContextSubChat/call3.golden | 61 ++++++ .../testdata/TestContextSubChat/call4.golden | 59 ++++++ .../testdata/TestContextSubChat/call5.golden | 28 +++ .../testdata/TestContextSubChat/call6.golden | 28 +++ .../testdata/TestContextSubChat/call7.golden | 45 +++++ .../testdata/TestContextSubChat/call8.golden | 61 ++++++ .../testdata/TestContextSubChat/call9.golden | 59 ++++++ .../testdata/TestContextSubChat/step1.golden | 143 ++++++++++++++ .../testdata/TestContextSubChat/step2.golden | 50 +++++ .../testdata/TestContextSubChat/step3.golden | 187 ++++++++++++++++++ .../testdata/TestContextSubChat/step4.golden | 66 +++++++ .../testdata/TestContextSubChat/test.gpt | 17 ++ 18 files changed, 1141 insertions(+), 44 deletions(-) create mode 100644 pkg/tests/testdata/TestContextSubChat/call1.golden create mode 100644 pkg/tests/testdata/TestContextSubChat/call10.golden create mode 100644 pkg/tests/testdata/TestContextSubChat/call2.golden create mode 100644 pkg/tests/testdata/TestContextSubChat/call3.golden create mode 100644 pkg/tests/testdata/TestContextSubChat/call4.golden create mode 100644 pkg/tests/testdata/TestContextSubChat/call5.golden create mode 100644 pkg/tests/testdata/TestContextSubChat/call6.golden create mode 100644 pkg/tests/testdata/TestContextSubChat/call7.golden create mode 100644 pkg/tests/testdata/TestContextSubChat/call8.golden create mode 100644 pkg/tests/testdata/TestContextSubChat/call9.golden create mode 100644 pkg/tests/testdata/TestContextSubChat/step1.golden create mode 100644 pkg/tests/testdata/TestContextSubChat/step2.golden create mode 100644 pkg/tests/testdata/TestContextSubChat/step3.golden create mode 100644 pkg/tests/testdata/TestContextSubChat/step4.golden create mode 100644 pkg/tests/testdata/TestContextSubChat/test.gpt diff --git a/pkg/builtin/builtin.go b/pkg/builtin/builtin.go index 0018a45c..f5799baf 100644 --- a/pkg/builtin/builtin.go +++ b/pkg/builtin/builtin.go @@ -633,7 +633,9 @@ func SysChatFinish(ctx context.Context, env []string, input string) (string, err Message string `json:"message,omitempty"` } if err := json.Unmarshal([]byte(input), ¶ms); err != nil { - return "", err + return "", &ErrChatFinish{ + Message: input, + } } return "", &ErrChatFinish{ Message: params.Message, diff --git a/pkg/runner/runner.go b/pkg/runner/runner.go index 4a4f0251..b9c0a103 100644 --- a/pkg/runner/runner.go +++ b/pkg/runner/runner.go @@ -134,21 +134,25 @@ func (r *Runner) Chat(ctx context.Context, prevState ChatState, prg types.Progra }() callCtx := engine.NewContext(ctx, &prg) - if state == nil { - startResult, err := r.start(callCtx, monitor, env, input) + if state == nil || state.StartContinuation { + if state != nil { + state = state.WithResumeInput(&input) + input = state.InputContextContinuationInput + } + state, err = r.start(callCtx, state, monitor, env, input) if err != nil { return resp, err } - state = &State{ - Continuation: startResult, - } } else { + state = state.WithResumeInput(&input) state.ResumeInput = &input } - state, err = r.resume(callCtx, monitor, env, state) - if err != nil { - return resp, err + if !state.StartContinuation { + state, err = r.resume(callCtx, monitor, env, state) + if err != nil { + return resp, err + } } if state.Result != nil { @@ -286,44 +290,79 @@ func getContextInput(prg *types.Program, ref types.ToolReference, input string) return string(output), err } -func (r *Runner) getContext(callCtx engine.Context, monitor Monitor, env []string, input string) (result []engine.InputContext, _ error) { +func (r *Runner) getContext(callCtx engine.Context, state *State, monitor Monitor, env []string, input string) (result []engine.InputContext, _ *State, _ error) { toolRefs, err := callCtx.Program.GetContextToolRefs(callCtx.Tool.ID) if err != nil { - return nil, err + return nil, nil, err } - for _, toolRef := range toolRefs { + var newState *State + if state != nil { + cp := *state + newState = &cp + if newState.InputContextContinuation != nil { + newState.InputContexts = nil + newState.InputContextContinuation = nil + newState.InputContextContinuationInput = "" + newState.ResumeInput = state.InputContextContinuationResumeInput + + input = state.InputContextContinuationInput + } + } + + for i, toolRef := range toolRefs { + if state != nil && i < len(state.InputContexts) { + result = append(result, state.InputContexts[i]) + continue + } + contextInput, err := getContextInput(callCtx.Program, toolRef, input) if err != nil { - return nil, err + return nil, nil, err } - content, err := r.subCall(callCtx.Ctx, callCtx, monitor, env, toolRef.ToolID, contextInput, "", engine.ContextToolCategory) + var content *State + if state != nil && state.InputContextContinuation != nil { + content, err = r.subCallResume(callCtx.Ctx, callCtx, monitor, env, toolRef.ToolID, "", state.InputContextContinuation.WithResumeInput(state.ResumeInput), engine.ContextToolCategory) + } else { + content, err = r.subCall(callCtx.Ctx, callCtx, monitor, env, toolRef.ToolID, contextInput, "", engine.ContextToolCategory) + } if err != nil { - return nil, err + return nil, nil, err } - if content.Result == nil { - return nil, fmt.Errorf("context tool can not result in a chat continuation") + if content.Continuation != nil { + if newState == nil { + newState = &State{} + } + newState.InputContexts = result + newState.InputContextContinuation = content + newState.InputContextContinuationInput = input + if state != nil { + newState.InputContextContinuationResumeInput = state.ResumeInput + } + return nil, newState, nil } result = append(result, engine.InputContext{ ToolID: toolRef.ToolID, Content: *content.Result, }) } - return result, nil + + return result, newState, nil } func (r *Runner) call(callCtx engine.Context, monitor Monitor, env []string, input string) (*State, error) { - result, err := r.start(callCtx, monitor, env, input) + result, err := r.start(callCtx, nil, monitor, env, input) if err != nil { return nil, err } - return r.resume(callCtx, monitor, env, &State{ - Continuation: result, - }) + if result.StartContinuation { + return result, nil + } + return r.resume(callCtx, monitor, env, result) } -func (r *Runner) start(callCtx engine.Context, monitor Monitor, env []string, input string) (*engine.Return, error) { +func (r *Runner) start(callCtx engine.Context, state *State, monitor Monitor, env []string, input string) (*State, error) { progress, progressClose := streamProgress(&callCtx, monitor) defer progressClose() @@ -335,11 +374,18 @@ func (r *Runner) start(callCtx engine.Context, monitor Monitor, env []string, in } } - var err error - callCtx.InputContext, err = r.getContext(callCtx, monitor, env, input) + var ( + err error + newState *State + ) + callCtx.InputContext, newState, err = r.getContext(callCtx, state, monitor, env, input) if err != nil { return nil, err } + if newState != nil && newState.InputContextContinuation != nil { + newState.StartContinuation = true + return newState, nil + } e := engine.Engine{ Model: r.c, @@ -358,7 +404,14 @@ func (r *Runner) start(callCtx engine.Context, monitor Monitor, env []string, in callCtx.Ctx = context2.AddPauseFuncToCtx(callCtx.Ctx, monitor.Pause) - return e.Start(callCtx, input) + ret, err := e.Start(callCtx, input) + if err != nil { + return nil, err + } + + return &State{ + Continuation: ret, + }, nil } type State struct { @@ -369,18 +422,28 @@ type State struct { ResumeInput *string `json:"resumeInput,omitempty"` SubCalls []SubCallResult `json:"subCalls,omitempty"` SubCallID string `json:"subCallID,omitempty"` + + InputContexts []engine.InputContext `json:"inputContexts,omitempty"` + InputContextContinuation *State `json:"inputContextContinuation,omitempty"` + InputContextContinuationInput string `json:"inputContextContinuationInput,omitempty"` + InputContextContinuationResumeInput *string `json:"inputContextContinuationResumeInput,omitempty"` + StartContinuation bool `json:"startContinuation,omitempty"` } -func (s State) WithInput(input *string) *State { +func (s State) WithResumeInput(input *string) *State { s.ResumeInput = input return &s } func (s State) ContinuationContentToolID() (string, error) { - if s.Continuation.Result != nil { + if s.Continuation != nil && s.Continuation.Result != nil { return s.ContinuationToolID, nil } + if s.InputContextContinuation != nil { + return s.InputContextContinuation.ContinuationContentToolID() + } + for _, subCall := range s.SubCalls { if s.SubCallID == subCall.CallID { return subCall.State.ContinuationContentToolID() @@ -390,10 +453,14 @@ func (s State) ContinuationContentToolID() (string, error) { } func (s State) ContinuationContent() (string, error) { - if s.Continuation.Result != nil { + if s.Continuation != nil && s.Continuation.Result != nil { return *s.Continuation.Result, nil } + if s.InputContextContinuation != nil { + return s.InputContextContinuation.ContinuationContent() + } + for _, subCall := range s.SubCalls { if s.SubCallID == subCall.CallID { return subCall.State.ContinuationContent() @@ -408,6 +475,10 @@ type Needed struct { } func (r *Runner) resume(callCtx engine.Context, monitor Monitor, env []string, state *State) (*State, error) { + if state.StartContinuation { + return nil, fmt.Errorf("invalid state, resume should not have StartContinuation set to true") + } + progress, progressClose := streamProgress(&callCtx, monitor) defer progressClose() @@ -451,7 +522,7 @@ func (r *Runner) resume(callCtx engine.Context, monitor Monitor, env []string, s err error ) - state, callResults, err = r.subCalls(callCtx, monitor, env, state) + state, callResults, err = r.subCalls(callCtx, monitor, env, state, engine.NoCategory) if errMessage := (*builtin.ErrChatFinish)(nil); errors.As(err, &errMessage) && callCtx.Tool.Chat { return &State{ Result: &errMessage.Message, @@ -477,12 +548,6 @@ func (r *Runner) resume(callCtx engine.Context, monitor Monitor, env []string, s } } - if state.ResumeInput != nil { - engineResults = append(engineResults, engine.CallResult{ - User: *state.ResumeInput, - }) - } - monitor.Event(Event{ Time: time.Now(), CallContext: callCtx.GetCallContext(), @@ -506,9 +571,15 @@ func (r *Runner) resume(callCtx engine.Context, monitor Monitor, env []string, s contentInput = state.Continuation.State.Input } - callCtx.InputContext, err = r.getContext(callCtx, monitor, env, contentInput) - if err != nil { - return nil, err + callCtx.InputContext, state, err = r.getContext(callCtx, state, monitor, env, contentInput) + if err != nil || state.InputContextContinuation != nil { + return state, err + } + + if state.ResumeInput != nil { + engineResults = append(engineResults, engine.CallResult{ + User: *state.ResumeInput, + }) } nextContinuation, err := e.Continue(callCtx, state.Continuation.State, engineResults...) @@ -571,8 +642,8 @@ func (r *Runner) subCall(ctx context.Context, parentContext engine.Context, moni return r.call(callCtx, monitor, env, input) } -func (r *Runner) subCallResume(ctx context.Context, parentContext engine.Context, monitor Monitor, env []string, toolID, callID string, state *State) (*State, error) { - callCtx, err := parentContext.SubCall(ctx, toolID, callID, engine.NoCategory) +func (r *Runner) subCallResume(ctx context.Context, parentContext engine.Context, monitor Monitor, env []string, toolID, callID string, state *State, toolCategory engine.ToolCategory) (*State, error) { + callCtx, err := parentContext.SubCall(ctx, toolID, callID, toolCategory) if err != nil { return nil, err } @@ -593,11 +664,15 @@ func (r *Runner) newDispatcher(ctx context.Context) dispatcher { return newParallelDispatcher(ctx) } -func (r *Runner) subCalls(callCtx engine.Context, monitor Monitor, env []string, state *State) (_ *State, callResults []SubCallResult, _ error) { +func (r *Runner) subCalls(callCtx engine.Context, monitor Monitor, env []string, state *State, toolCategory engine.ToolCategory) (_ *State, callResults []SubCallResult, _ error) { var ( resultLock sync.Mutex ) + if state.InputContextContinuation != nil { + return state, nil, nil + } + if state.SubCallID != "" { if state.ResumeInput == nil { return nil, nil, fmt.Errorf("invalid state, input must be set for sub call continuation on callID [%s]", state.SubCallID) @@ -608,7 +683,7 @@ func (r *Runner) subCalls(callCtx engine.Context, monitor Monitor, env []string, found = true subState := *subCall.State subState.ResumeInput = state.ResumeInput - result, err := r.subCallResume(callCtx.Ctx, callCtx, monitor, env, subCall.ToolID, subCall.CallID, subCall.State.WithInput(state.ResumeInput)) + result, err := r.subCallResume(callCtx.Ctx, callCtx, monitor, env, subCall.ToolID, subCall.CallID, subCall.State.WithResumeInput(state.ResumeInput), toolCategory) if err != nil { return nil, nil, err } @@ -618,7 +693,7 @@ func (r *Runner) subCalls(callCtx engine.Context, monitor Monitor, env []string, State: result, }) // Clear the input, we have already processed it - state = state.WithInput(nil) + state = state.WithResumeInput(nil) } else { callResults = append(callResults, subCall) } diff --git a/pkg/tests/runner_test.go b/pkg/tests/runner_test.go index 6f94d43c..33c3d2d3 100644 --- a/pkg/tests/runner_test.go +++ b/pkg/tests/runner_test.go @@ -101,6 +101,105 @@ func TestDualSubChat(t *testing.T) { autogold.ExpectFile(t, toJSONString(t, resp), autogold.Name(t.Name()+"/step4")) } +func TestContextSubChat(t *testing.T) { + r := tester.NewRunner(t) + r.RespondWith(tester.Result{ + Content: []types.ContentPart{ + { + ToolCall: &types.CompletionToolCall{ + ID: "call_1", + Function: types.CompletionFunctionCall{ + Name: "chatbot", + Arguments: "Input to chatbot1", + }, + }, + }, + }, + }, tester.Result{ + Text: "Assistant Response 1 - from chatbot1", + }) + + prg, err := r.Load("") + require.NoError(t, err) + + resp, err := r.Chat(context.Background(), nil, prg, os.Environ(), "User 1") + require.NoError(t, err) + r.AssertResponded(t) + assert.False(t, resp.Done) + autogold.Expect("Assistant Response 1 - from chatbot1").Equal(t, resp.Content) + autogold.ExpectFile(t, toJSONString(t, resp), autogold.Name(t.Name()+"/step1")) + + r.RespondWith(tester.Result{ + Content: []types.ContentPart{ + { + ToolCall: &types.CompletionToolCall{ + ID: "call_2", + Function: types.CompletionFunctionCall{ + Name: types.ToolNormalizer("sys.chat.finish"), + Arguments: "Response from context chatbot", + }, + }, + }, + }, + }, tester.Result{ + Text: "Assistant Response 2 - from context tool", + }, tester.Result{ + Text: "Assistant Response 3 - from main chat tool", + }) + resp, err = r.Chat(context.Background(), resp.State, prg, os.Environ(), "User 2") + require.NoError(t, err) + r.AssertResponded(t) + assert.False(t, resp.Done) + autogold.Expect("Assistant Response 3 - from main chat tool").Equal(t, resp.Content) + autogold.ExpectFile(t, toJSONString(t, resp), autogold.Name(t.Name()+"/step2")) + + r.RespondWith(tester.Result{ + Content: []types.ContentPart{ + { + ToolCall: &types.CompletionToolCall{ + ID: "call_3", + Function: types.CompletionFunctionCall{ + Name: "chatbot", + Arguments: "Input to chatbot1 on resume", + }, + }, + }, + }, + }, tester.Result{ + Text: "Assistant Response 4 - from chatbot1", + }) + resp, err = r.Chat(context.Background(), resp.State, prg, os.Environ(), "User 3") + require.NoError(t, err) + r.AssertResponded(t) + assert.False(t, resp.Done) + autogold.Expect("Assistant Response 3 - from main chat tool").Equal(t, resp.Content) + autogold.ExpectFile(t, toJSONString(t, resp), autogold.Name(t.Name()+"/step3")) + + r.RespondWith(tester.Result{ + Content: []types.ContentPart{ + { + ToolCall: &types.CompletionToolCall{ + ID: "call_4", + Function: types.CompletionFunctionCall{ + Name: types.ToolNormalizer("sys.chat.finish"), + Arguments: "Response from context chatbot after resume", + }, + }, + }, + }, + }, tester.Result{ + Text: "Assistant Response 5 - from context tool resume", + }, tester.Result{ + Text: "Assistant Response 6 - from main chat tool resume", + }) + resp, err = r.Chat(context.Background(), resp.State, prg, os.Environ(), "User 4") + require.NoError(t, err) + r.AssertResponded(t) + assert.False(t, resp.Done) + autogold.Expect("Assistant Response 6 - from main chat tool resume").Equal(t, resp.Content) + autogold.ExpectFile(t, toJSONString(t, resp), autogold.Name(t.Name()+"/step4")) +} + func TestSubChat(t *testing.T) { r := tester.NewRunner(t) r.RespondWith(tester.Result{ diff --git a/pkg/tests/testdata/TestContextSubChat/call1.golden b/pkg/tests/testdata/TestContextSubChat/call1.golden new file mode 100644 index 00000000..c3d558e2 --- /dev/null +++ b/pkg/tests/testdata/TestContextSubChat/call1.golden @@ -0,0 +1,28 @@ +`{ + "Model": "gpt-4-turbo-preview", + "InternalSystemPrompt": null, + "Tools": [ + { + "function": { + "toolID": "testdata/TestContextSubChat/test.gpt:13", + "name": "chatbot", + "parameters": null + } + } + ], + "Messages": [ + { + "role": "system", + "content": [ + { + "text": "Call chatbot" + } + ] + } + ], + "MaxTokens": 0, + "Temperature": null, + "JSONResponse": false, + "Grammar": "", + "Cache": null +}` diff --git a/pkg/tests/testdata/TestContextSubChat/call10.golden b/pkg/tests/testdata/TestContextSubChat/call10.golden new file mode 100644 index 00000000..00e2061f --- /dev/null +++ b/pkg/tests/testdata/TestContextSubChat/call10.golden @@ -0,0 +1,44 @@ +`{ + "Model": "gpt-4-turbo-preview", + "InternalSystemPrompt": false, + "Tools": null, + "Messages": [ + { + "role": "system", + "content": [ + { + "text": "Assistant Response 5 - from context tool resume\nHello" + } + ] + }, + { + "role": "user", + "content": [ + { + "text": "User 1" + } + ] + }, + { + "role": "assistant", + "content": [ + { + "text": "Assistant Response 3 - from main chat tool" + } + ] + }, + { + "role": "user", + "content": [ + { + "text": "User 3" + } + ] + } + ], + "MaxTokens": 0, + "Temperature": null, + "JSONResponse": false, + "Grammar": "", + "Cache": null +}` diff --git a/pkg/tests/testdata/TestContextSubChat/call2.golden b/pkg/tests/testdata/TestContextSubChat/call2.golden new file mode 100644 index 00000000..fc0453bb --- /dev/null +++ b/pkg/tests/testdata/TestContextSubChat/call2.golden @@ -0,0 +1,45 @@ +`{ + "Model": "gpt-4-turbo-preview", + "InternalSystemPrompt": false, + "Tools": [ + { + "function": { + "toolID": "sys.chat.finish", + "name": "chatFinish", + "description": "Concludes the conversation. This can not be used to ask a question.", + "parameters": { + "properties": { + "summary": { + "description": "A summary of the dialog", + "type": "string" + } + }, + "type": "object" + } + } + } + ], + "Messages": [ + { + "role": "system", + "content": [ + { + "text": "This is a chatbot" + } + ] + }, + { + "role": "user", + "content": [ + { + "text": "Input to chatbot1" + } + ] + } + ], + "MaxTokens": 0, + "Temperature": null, + "JSONResponse": false, + "Grammar": "", + "Cache": null +}` diff --git a/pkg/tests/testdata/TestContextSubChat/call3.golden b/pkg/tests/testdata/TestContextSubChat/call3.golden new file mode 100644 index 00000000..13aa7bb8 --- /dev/null +++ b/pkg/tests/testdata/TestContextSubChat/call3.golden @@ -0,0 +1,61 @@ +`{ + "Model": "gpt-4-turbo-preview", + "InternalSystemPrompt": false, + "Tools": [ + { + "function": { + "toolID": "sys.chat.finish", + "name": "chatFinish", + "description": "Concludes the conversation. This can not be used to ask a question.", + "parameters": { + "properties": { + "summary": { + "description": "A summary of the dialog", + "type": "string" + } + }, + "type": "object" + } + } + } + ], + "Messages": [ + { + "role": "system", + "content": [ + { + "text": "This is a chatbot" + } + ] + }, + { + "role": "user", + "content": [ + { + "text": "Input to chatbot1" + } + ] + }, + { + "role": "assistant", + "content": [ + { + "text": "Assistant Response 1 - from chatbot1" + } + ] + }, + { + "role": "user", + "content": [ + { + "text": "User 1" + } + ] + } + ], + "MaxTokens": 0, + "Temperature": null, + "JSONResponse": false, + "Grammar": "", + "Cache": null +}` diff --git a/pkg/tests/testdata/TestContextSubChat/call4.golden b/pkg/tests/testdata/TestContextSubChat/call4.golden new file mode 100644 index 00000000..552e0afe --- /dev/null +++ b/pkg/tests/testdata/TestContextSubChat/call4.golden @@ -0,0 +1,59 @@ +`{ + "Model": "gpt-4-turbo-preview", + "InternalSystemPrompt": null, + "Tools": [ + { + "function": { + "toolID": "testdata/TestContextSubChat/test.gpt:13", + "name": "chatbot", + "parameters": null + } + } + ], + "Messages": [ + { + "role": "system", + "content": [ + { + "text": "Call chatbot" + } + ] + }, + { + "role": "assistant", + "content": [ + { + "toolCall": { + "index": 0, + "id": "call_1", + "function": { + "name": "chatbot", + "arguments": "Input to chatbot1" + } + } + } + ] + }, + { + "role": "tool", + "content": [ + { + "text": "Response from context chatbot" + } + ], + "toolCall": { + "index": 0, + "id": "call_1", + "function": { + "name": "chatbot", + "arguments": "Input to chatbot1" + } + } + } + ], + "MaxTokens": 0, + "Temperature": null, + "JSONResponse": false, + "Grammar": "", + "Cache": null +}` diff --git a/pkg/tests/testdata/TestContextSubChat/call5.golden b/pkg/tests/testdata/TestContextSubChat/call5.golden new file mode 100644 index 00000000..24dc949b --- /dev/null +++ b/pkg/tests/testdata/TestContextSubChat/call5.golden @@ -0,0 +1,28 @@ +`{ + "Model": "gpt-4-turbo-preview", + "InternalSystemPrompt": false, + "Tools": null, + "Messages": [ + { + "role": "system", + "content": [ + { + "text": "Assistant Response 2 - from context tool\nHello" + } + ] + }, + { + "role": "user", + "content": [ + { + "text": "User 1" + } + ] + } + ], + "MaxTokens": 0, + "Temperature": null, + "JSONResponse": false, + "Grammar": "", + "Cache": null +}` diff --git a/pkg/tests/testdata/TestContextSubChat/call6.golden b/pkg/tests/testdata/TestContextSubChat/call6.golden new file mode 100644 index 00000000..c3d558e2 --- /dev/null +++ b/pkg/tests/testdata/TestContextSubChat/call6.golden @@ -0,0 +1,28 @@ +`{ + "Model": "gpt-4-turbo-preview", + "InternalSystemPrompt": null, + "Tools": [ + { + "function": { + "toolID": "testdata/TestContextSubChat/test.gpt:13", + "name": "chatbot", + "parameters": null + } + } + ], + "Messages": [ + { + "role": "system", + "content": [ + { + "text": "Call chatbot" + } + ] + } + ], + "MaxTokens": 0, + "Temperature": null, + "JSONResponse": false, + "Grammar": "", + "Cache": null +}` diff --git a/pkg/tests/testdata/TestContextSubChat/call7.golden b/pkg/tests/testdata/TestContextSubChat/call7.golden new file mode 100644 index 00000000..91f1405e --- /dev/null +++ b/pkg/tests/testdata/TestContextSubChat/call7.golden @@ -0,0 +1,45 @@ +`{ + "Model": "gpt-4-turbo-preview", + "InternalSystemPrompt": false, + "Tools": [ + { + "function": { + "toolID": "sys.chat.finish", + "name": "chatFinish", + "description": "Concludes the conversation. This can not be used to ask a question.", + "parameters": { + "properties": { + "summary": { + "description": "A summary of the dialog", + "type": "string" + } + }, + "type": "object" + } + } + } + ], + "Messages": [ + { + "role": "system", + "content": [ + { + "text": "This is a chatbot" + } + ] + }, + { + "role": "user", + "content": [ + { + "text": "Input to chatbot1 on resume" + } + ] + } + ], + "MaxTokens": 0, + "Temperature": null, + "JSONResponse": false, + "Grammar": "", + "Cache": null +}` diff --git a/pkg/tests/testdata/TestContextSubChat/call8.golden b/pkg/tests/testdata/TestContextSubChat/call8.golden new file mode 100644 index 00000000..ba44de50 --- /dev/null +++ b/pkg/tests/testdata/TestContextSubChat/call8.golden @@ -0,0 +1,61 @@ +`{ + "Model": "gpt-4-turbo-preview", + "InternalSystemPrompt": false, + "Tools": [ + { + "function": { + "toolID": "sys.chat.finish", + "name": "chatFinish", + "description": "Concludes the conversation. This can not be used to ask a question.", + "parameters": { + "properties": { + "summary": { + "description": "A summary of the dialog", + "type": "string" + } + }, + "type": "object" + } + } + } + ], + "Messages": [ + { + "role": "system", + "content": [ + { + "text": "This is a chatbot" + } + ] + }, + { + "role": "user", + "content": [ + { + "text": "Input to chatbot1 on resume" + } + ] + }, + { + "role": "assistant", + "content": [ + { + "text": "Assistant Response 4 - from chatbot1" + } + ] + }, + { + "role": "user", + "content": [ + { + "text": "User 4" + } + ] + } + ], + "MaxTokens": 0, + "Temperature": null, + "JSONResponse": false, + "Grammar": "", + "Cache": null +}` diff --git a/pkg/tests/testdata/TestContextSubChat/call9.golden b/pkg/tests/testdata/TestContextSubChat/call9.golden new file mode 100644 index 00000000..d7e65a50 --- /dev/null +++ b/pkg/tests/testdata/TestContextSubChat/call9.golden @@ -0,0 +1,59 @@ +`{ + "Model": "gpt-4-turbo-preview", + "InternalSystemPrompt": null, + "Tools": [ + { + "function": { + "toolID": "testdata/TestContextSubChat/test.gpt:13", + "name": "chatbot", + "parameters": null + } + } + ], + "Messages": [ + { + "role": "system", + "content": [ + { + "text": "Call chatbot" + } + ] + }, + { + "role": "assistant", + "content": [ + { + "toolCall": { + "index": 0, + "id": "call_3", + "function": { + "name": "chatbot", + "arguments": "Input to chatbot1 on resume" + } + } + } + ] + }, + { + "role": "tool", + "content": [ + { + "text": "Response from context chatbot after resume" + } + ], + "toolCall": { + "index": 0, + "id": "call_3", + "function": { + "name": "chatbot", + "arguments": "Input to chatbot1 on resume" + } + } + } + ], + "MaxTokens": 0, + "Temperature": null, + "JSONResponse": false, + "Grammar": "", + "Cache": null +}` diff --git a/pkg/tests/testdata/TestContextSubChat/step1.golden b/pkg/tests/testdata/TestContextSubChat/step1.golden new file mode 100644 index 00000000..826ad25d --- /dev/null +++ b/pkg/tests/testdata/TestContextSubChat/step1.golden @@ -0,0 +1,143 @@ +`{ + "done": false, + "content": "Assistant Response 1 - from chatbot1", + "toolID": "testdata/TestContextSubChat/test.gpt:13", + "state": { + "inputContextContinuation": { + "continuation": { + "state": { + "completion": { + "Model": "gpt-4-turbo-preview", + "InternalSystemPrompt": null, + "Tools": [ + { + "function": { + "toolID": "testdata/TestContextSubChat/test.gpt:13", + "name": "chatbot", + "parameters": null + } + } + ], + "Messages": [ + { + "role": "system", + "content": [ + { + "text": "Call chatbot" + } + ] + }, + { + "role": "assistant", + "content": [ + { + "toolCall": { + "index": 0, + "id": "call_1", + "function": { + "name": "chatbot", + "arguments": "Input to chatbot1" + } + } + } + ] + } + ], + "MaxTokens": 0, + "Temperature": null, + "JSONResponse": false, + "Grammar": "", + "Cache": null + }, + "pending": { + "call_1": { + "index": 0, + "id": "call_1", + "function": { + "name": "chatbot", + "arguments": "Input to chatbot1" + } + } + } + }, + "calls": { + "call_1": { + "toolID": "testdata/TestContextSubChat/test.gpt:13", + "input": "Input to chatbot1" + } + } + }, + "subCalls": [ + { + "toolId": "testdata/TestContextSubChat/test.gpt:13", + "callId": "call_1", + "state": { + "continuation": { + "state": { + "input": "Input to chatbot1", + "completion": { + "Model": "gpt-4-turbo-preview", + "InternalSystemPrompt": false, + "Tools": [ + { + "function": { + "toolID": "sys.chat.finish", + "name": "chatFinish", + "description": "Concludes the conversation. This can not be used to ask a question.", + "parameters": { + "properties": { + "summary": { + "description": "A summary of the dialog", + "type": "string" + } + }, + "type": "object" + } + } + } + ], + "Messages": [ + { + "role": "system", + "content": [ + { + "text": "This is a chatbot" + } + ] + }, + { + "role": "user", + "content": [ + { + "text": "Input to chatbot1" + } + ] + }, + { + "role": "assistant", + "content": [ + { + "text": "Assistant Response 1 - from chatbot1" + } + ] + } + ], + "MaxTokens": 0, + "Temperature": null, + "JSONResponse": false, + "Grammar": "", + "Cache": null + } + }, + "result": "Assistant Response 1 - from chatbot1" + }, + "continuationToolID": "testdata/TestContextSubChat/test.gpt:13" + } + } + ], + "subCallID": "call_1" + }, + "inputContextContinuationInput": "User 1", + "startContinuation": true + } +}` diff --git a/pkg/tests/testdata/TestContextSubChat/step2.golden b/pkg/tests/testdata/TestContextSubChat/step2.golden new file mode 100644 index 00000000..39af5881 --- /dev/null +++ b/pkg/tests/testdata/TestContextSubChat/step2.golden @@ -0,0 +1,50 @@ +`{ + "done": false, + "content": "Assistant Response 3 - from main chat tool", + "toolID": "testdata/TestContextSubChat/test.gpt:1", + "state": { + "continuation": { + "state": { + "input": "User 1", + "completion": { + "Model": "gpt-4-turbo-preview", + "InternalSystemPrompt": false, + "Tools": null, + "Messages": [ + { + "role": "system", + "content": [ + { + "text": "Assistant Response 2 - from context tool\nHello" + } + ] + }, + { + "role": "user", + "content": [ + { + "text": "User 1" + } + ] + }, + { + "role": "assistant", + "content": [ + { + "text": "Assistant Response 3 - from main chat tool" + } + ] + } + ], + "MaxTokens": 0, + "Temperature": null, + "JSONResponse": false, + "Grammar": "", + "Cache": null + } + }, + "result": "Assistant Response 3 - from main chat tool" + }, + "continuationToolID": "testdata/TestContextSubChat/test.gpt:1" + } +}` diff --git a/pkg/tests/testdata/TestContextSubChat/step3.golden b/pkg/tests/testdata/TestContextSubChat/step3.golden new file mode 100644 index 00000000..f4c859d7 --- /dev/null +++ b/pkg/tests/testdata/TestContextSubChat/step3.golden @@ -0,0 +1,187 @@ +`{ + "done": false, + "content": "Assistant Response 3 - from main chat tool", + "toolID": "testdata/TestContextSubChat/test.gpt:1", + "state": { + "continuation": { + "state": { + "input": "User 1", + "completion": { + "Model": "gpt-4-turbo-preview", + "InternalSystemPrompt": false, + "Tools": null, + "Messages": [ + { + "role": "system", + "content": [ + { + "text": "Assistant Response 2 - from context tool\nHello" + } + ] + }, + { + "role": "user", + "content": [ + { + "text": "User 1" + } + ] + }, + { + "role": "assistant", + "content": [ + { + "text": "Assistant Response 3 - from main chat tool" + } + ] + } + ], + "MaxTokens": 0, + "Temperature": null, + "JSONResponse": false, + "Grammar": "", + "Cache": null + } + }, + "result": "Assistant Response 3 - from main chat tool" + }, + "continuationToolID": "testdata/TestContextSubChat/test.gpt:1", + "resumeInput": "User 3", + "inputContextContinuation": { + "continuation": { + "state": { + "completion": { + "Model": "gpt-4-turbo-preview", + "InternalSystemPrompt": null, + "Tools": [ + { + "function": { + "toolID": "testdata/TestContextSubChat/test.gpt:13", + "name": "chatbot", + "parameters": null + } + } + ], + "Messages": [ + { + "role": "system", + "content": [ + { + "text": "Call chatbot" + } + ] + }, + { + "role": "assistant", + "content": [ + { + "toolCall": { + "index": 0, + "id": "call_3", + "function": { + "name": "chatbot", + "arguments": "Input to chatbot1 on resume" + } + } + } + ] + } + ], + "MaxTokens": 0, + "Temperature": null, + "JSONResponse": false, + "Grammar": "", + "Cache": null + }, + "pending": { + "call_3": { + "index": 0, + "id": "call_3", + "function": { + "name": "chatbot", + "arguments": "Input to chatbot1 on resume" + } + } + } + }, + "calls": { + "call_3": { + "toolID": "testdata/TestContextSubChat/test.gpt:13", + "input": "Input to chatbot1 on resume" + } + } + }, + "subCalls": [ + { + "toolId": "testdata/TestContextSubChat/test.gpt:13", + "callId": "call_3", + "state": { + "continuation": { + "state": { + "input": "Input to chatbot1 on resume", + "completion": { + "Model": "gpt-4-turbo-preview", + "InternalSystemPrompt": false, + "Tools": [ + { + "function": { + "toolID": "sys.chat.finish", + "name": "chatFinish", + "description": "Concludes the conversation. This can not be used to ask a question.", + "parameters": { + "properties": { + "summary": { + "description": "A summary of the dialog", + "type": "string" + } + }, + "type": "object" + } + } + } + ], + "Messages": [ + { + "role": "system", + "content": [ + { + "text": "This is a chatbot" + } + ] + }, + { + "role": "user", + "content": [ + { + "text": "Input to chatbot1 on resume" + } + ] + }, + { + "role": "assistant", + "content": [ + { + "text": "Assistant Response 4 - from chatbot1" + } + ] + } + ], + "MaxTokens": 0, + "Temperature": null, + "JSONResponse": false, + "Grammar": "", + "Cache": null + } + }, + "result": "Assistant Response 4 - from chatbot1" + }, + "continuationToolID": "testdata/TestContextSubChat/test.gpt:13" + } + } + ], + "subCallID": "call_3" + }, + "inputContextContinuationInput": "User 1", + "inputContextContinuationResumeInput": "User 3" + } +}` diff --git a/pkg/tests/testdata/TestContextSubChat/step4.golden b/pkg/tests/testdata/TestContextSubChat/step4.golden new file mode 100644 index 00000000..0f7b5a4e --- /dev/null +++ b/pkg/tests/testdata/TestContextSubChat/step4.golden @@ -0,0 +1,66 @@ +`{ + "done": false, + "content": "Assistant Response 6 - from main chat tool resume", + "toolID": "testdata/TestContextSubChat/test.gpt:1", + "state": { + "continuation": { + "state": { + "input": "User 1", + "completion": { + "Model": "gpt-4-turbo-preview", + "InternalSystemPrompt": false, + "Tools": null, + "Messages": [ + { + "role": "system", + "content": [ + { + "text": "Assistant Response 5 - from context tool resume\nHello" + } + ] + }, + { + "role": "user", + "content": [ + { + "text": "User 1" + } + ] + }, + { + "role": "assistant", + "content": [ + { + "text": "Assistant Response 3 - from main chat tool" + } + ] + }, + { + "role": "user", + "content": [ + { + "text": "User 3" + } + ] + }, + { + "role": "assistant", + "content": [ + { + "text": "Assistant Response 6 - from main chat tool resume" + } + ] + } + ], + "MaxTokens": 0, + "Temperature": null, + "JSONResponse": false, + "Grammar": "", + "Cache": null + } + }, + "result": "Assistant Response 6 - from main chat tool resume" + }, + "continuationToolID": "testdata/TestContextSubChat/test.gpt:1" + } +}` diff --git a/pkg/tests/testdata/TestContextSubChat/test.gpt b/pkg/tests/testdata/TestContextSubChat/test.gpt new file mode 100644 index 00000000..57f621f2 --- /dev/null +++ b/pkg/tests/testdata/TestContextSubChat/test.gpt @@ -0,0 +1,17 @@ +chat: true +context: subtool + +Hello + +--- +name: subtool +tools: chatbot + +Call chatbot + +--- +name: chatbot +chat: true +tools: sys.chat.finish + +This is a chatbot \ No newline at end of file