From 331343ab0e74ba97a8b2dba169969c19518bfd6d Mon Sep 17 00:00:00 2001 From: Jason McCartney Date: Mon, 17 Apr 2023 07:51:31 -0700 Subject: [PATCH 1/5] Make reverse prompt option act as a stop token in non-interactive scenarios --- examples/common.cpp | 3 ++- examples/main/main.cpp | 26 ++++++++++++++++++-------- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/examples/common.cpp b/examples/common.cpp index 0772dbfe142ff..8b44926634c93 100644 --- a/examples/common.cpp +++ b/examples/common.cpp @@ -210,7 +210,8 @@ void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) { fprintf(stderr, " --interactive-first run in interactive mode and wait for input right away\n"); fprintf(stderr, " -ins, --instruct run in instruction mode (use with Alpaca models)\n"); fprintf(stderr, " -r PROMPT, --reverse-prompt PROMPT\n"); - fprintf(stderr, " run in interactive mode and poll user input upon seeing PROMPT (can be\n"); + fprintf(stderr, " specify a PROMPT that will cause generation to stop\n"); + fprintf(stderr, " if running interactive, poll user input upon seeing PROMPT (can be\n"); fprintf(stderr, " specified more than once for multiple prompts).\n"); fprintf(stderr, " --color colorise output to distinguish prompt and user input from generations\n"); fprintf(stderr, " -s SEED, --seed SEED RNG seed (default: -1, use random seed for <= 0)\n"); diff --git a/examples/main/main.cpp b/examples/main/main.cpp index 3e4b0034ee977..cfa40941e3379 100644 --- a/examples/main/main.cpp +++ b/examples/main/main.cpp @@ -168,8 +168,8 @@ int main(int argc, char ** argv) { params.antiprompt.push_back("### Instruction:\n\n"); } - // enable interactive mode if reverse prompt or interactive start is specified - if (params.antiprompt.size() != 0 || params.interactive_start) { + // enable interactive mode if interactive start is specified + if (params.interactive_start) { params.interactive = true; } @@ -247,7 +247,7 @@ int main(int argc, char ** argv) { std::vector embd; - while (n_remain != 0 || params.interactive) { + while ((n_remain != 0 && !is_antiprompt) || params.interactive) { // predict if (embd.size() > 0) { // infinite text generation via context swapping @@ -347,9 +347,10 @@ int main(int argc, char ** argv) { set_console_color(con_st, CONSOLE_COLOR_DEFAULT); } - // in interactive mode, and not currently processing queued inputs; + // in not currently processing queued inputs; // check if we should prompt the user for more - if (params.interactive && (int) embd_inp.size() <= n_consumed) { + // or quit + if ((int) embd_inp.size() <= n_consumed) { // check for reverse prompt if (params.antiprompt.size()) { @@ -360,11 +361,20 @@ int main(int argc, char ** argv) { is_antiprompt = false; // Check if each of the reverse prompts appears at the end of the output. + // If we're not running interactively, the reverse prompt might be tokenized with some following characters + // so we'll compensate for that by widening the search window a bit. for (std::string & antiprompt : params.antiprompt) { - if (last_output.find(antiprompt.c_str(), last_output.length() - antiprompt.length(), antiprompt.length()) != std::string::npos) { - is_interacting = true; + size_t extra_padding = params.interactive ? 0 : 2; + size_t search_start_pos = last_output.length() > static_cast(antiprompt.length() + extra_padding) + ? last_output.length() - static_cast(antiprompt.length() + extra_padding) + : 0; + + if (last_output.find(antiprompt.c_str(), search_start_pos) != std::string::npos) { + if (params.interactive) { + is_interacting = true; + set_console_color(con_st, CONSOLE_COLOR_USER_INPUT); + } is_antiprompt = true; - set_console_color(con_st, CONSOLE_COLOR_USER_INPUT); fflush(stdout); break; } From f7229f23d73ba65a93b73f16e80ca982607d6d2d Mon Sep 17 00:00:00 2001 From: Jason McCartney Date: Thu, 11 May 2023 11:14:08 -0700 Subject: [PATCH 2/5] Making requested review changes --- examples/common.cpp | 5 ++--- examples/main/main.cpp | 4 +--- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/examples/common.cpp b/examples/common.cpp index 946c1caa2daeb..7d488beadcea7 100644 --- a/examples/common.cpp +++ b/examples/common.cpp @@ -223,9 +223,8 @@ void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) { fprintf(stderr, " --interactive-first run in interactive mode and wait for input right away\n"); fprintf(stderr, " -ins, --instruct run in instruction mode (use with Alpaca models)\n"); fprintf(stderr, " -r PROMPT, --reverse-prompt PROMPT\n"); - fprintf(stderr, " specify a PROMPT that will cause generation to stop\n"); - fprintf(stderr, " if running interactive, poll user input upon seeing PROMPT (can be\n"); - fprintf(stderr, " specified more than once for multiple prompts).\n"); + fprintf(stderr, " halt generation at PROMPT, return control in interactive mode\n"); + fprintf(stderr, " (can be specified more than once for multiple prompts).\n"); fprintf(stderr, " --color colorise output to distinguish prompt and user input from generations\n"); fprintf(stderr, " -s SEED, --seed SEED RNG seed (default: -1, use random seed for <= 0)\n"); fprintf(stderr, " -t N, --threads N number of threads to use during computation (default: %d)\n", params.n_threads); diff --git a/examples/main/main.cpp b/examples/main/main.cpp index 831a37524b858..d44579aff3295 100644 --- a/examples/main/main.cpp +++ b/examples/main/main.cpp @@ -358,9 +358,7 @@ int main(int argc, char ** argv) { set_console_color(con_st, CONSOLE_COLOR_DEFAULT); } - // in not currently processing queued inputs; - // check if we should prompt the user for more - // or quit + // if not currently processing queued inputs; if ((int) embd_inp.size() <= n_consumed) { // check for reverse prompt From 2bb2ff1748513591ad45b175a75ed1d8089d84c8 Mon Sep 17 00:00:00 2001 From: Jason McCartney Date: Thu, 11 May 2023 21:00:11 -0700 Subject: [PATCH 3/5] Update gpt_params_parse and fix a merge error --- examples/common.cpp | 2 +- examples/main/main.cpp | 26 ++++++++------------------ 2 files changed, 9 insertions(+), 19 deletions(-) diff --git a/examples/common.cpp b/examples/common.cpp index 00535f5580800..73278db0ed67f 100644 --- a/examples/common.cpp +++ b/examples/common.cpp @@ -346,7 +346,7 @@ bool gpt_params_parse(int argc, char ** argv, gpt_params & params) { } if (params.prompt_cache_all && (params.interactive || params.interactive_first || - params.instruct || params.antiprompt.size())) { + params.instruct)) { fprintf(stderr, "error: --prompt-cache-all not supported in interactive mode yet\n"); gpt_print_usage(argc, argv, default_params); exit(1); diff --git a/examples/main/main.cpp b/examples/main/main.cpp index f23a4bc72737c..231a99951bcb5 100644 --- a/examples/main/main.cpp +++ b/examples/main/main.cpp @@ -209,8 +209,8 @@ int main(int argc, char ** argv) { params.antiprompt.push_back("### Instruction:\n\n"); } - // enable interactive mode if interactive start is specified - if (params.interactive_start) { + // enable interactive mode if reverse prompt or interactive start is specified + if (params.interactive_first) { params.interactive = true; } @@ -306,7 +306,7 @@ int main(int argc, char ** argv) { std::vector embd; - while ((n_remain != 0 && !is_antiprompt) || params.interactive) { + while (n_remain != 0 || params.interactive) { // predict if (embd.size() > 0) { // infinite text generation via context swapping @@ -504,8 +504,9 @@ int main(int argc, char ** argv) { console_set_color(con_st, CONSOLE_COLOR_DEFAULT); } - // if not currently processing queued inputs; - if ((int) embd_inp.size() <= n_consumed) { + // in interactive mode, and not currently processing queued inputs; + // check if we should prompt the user for more + if (params.interactive && (int) embd_inp.size() <= n_consumed) { // check for reverse prompt if (params.antiprompt.size()) { @@ -516,21 +517,10 @@ int main(int argc, char ** argv) { is_antiprompt = false; // Check if each of the reverse prompts appears at the end of the output. - // If we're not running interactively, the reverse prompt might be tokenized with some following characters - // so we'll compensate for that by widening the search window a bit. for (std::string & antiprompt : params.antiprompt) { - size_t extra_padding = params.interactive ? 0 : 2; - size_t search_start_pos = last_output.length() > static_cast(antiprompt.length() + extra_padding) - ? last_output.length() - static_cast(antiprompt.length() + extra_padding) - : 0; - - if (last_output.find(antiprompt.c_str(), search_start_pos) != std::string::npos) { - if (params.interactive) { - is_interacting = true; - set_console_color(con_st, CONSOLE_COLOR_USER_INPUT); - } + if (last_output.find(antiprompt.c_str(), last_output.length() - antiprompt.length(), antiprompt.length()) != std::string::npos) { + is_interacting = true; is_antiprompt = true; - fflush(stdout); break; } } From 121c986d02fc0f8b04f584c368626062095b897f Mon Sep 17 00:00:00 2001 From: Jason McCartney Date: Thu, 11 May 2023 21:11:55 -0700 Subject: [PATCH 4/5] Revert "Update gpt_params_parse and fix a merge error" This reverts commit 2bb2ff1748513591ad45b175a75ed1d8089d84c8. --- examples/common.cpp | 2 +- examples/main/main.cpp | 26 ++++++++++++++++++-------- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/examples/common.cpp b/examples/common.cpp index 73278db0ed67f..00535f5580800 100644 --- a/examples/common.cpp +++ b/examples/common.cpp @@ -346,7 +346,7 @@ bool gpt_params_parse(int argc, char ** argv, gpt_params & params) { } if (params.prompt_cache_all && (params.interactive || params.interactive_first || - params.instruct)) { + params.instruct || params.antiprompt.size())) { fprintf(stderr, "error: --prompt-cache-all not supported in interactive mode yet\n"); gpt_print_usage(argc, argv, default_params); exit(1); diff --git a/examples/main/main.cpp b/examples/main/main.cpp index 231a99951bcb5..f23a4bc72737c 100644 --- a/examples/main/main.cpp +++ b/examples/main/main.cpp @@ -209,8 +209,8 @@ int main(int argc, char ** argv) { params.antiprompt.push_back("### Instruction:\n\n"); } - // enable interactive mode if reverse prompt or interactive start is specified - if (params.interactive_first) { + // enable interactive mode if interactive start is specified + if (params.interactive_start) { params.interactive = true; } @@ -306,7 +306,7 @@ int main(int argc, char ** argv) { std::vector embd; - while (n_remain != 0 || params.interactive) { + while ((n_remain != 0 && !is_antiprompt) || params.interactive) { // predict if (embd.size() > 0) { // infinite text generation via context swapping @@ -504,9 +504,8 @@ int main(int argc, char ** argv) { console_set_color(con_st, CONSOLE_COLOR_DEFAULT); } - // in interactive mode, and not currently processing queued inputs; - // check if we should prompt the user for more - if (params.interactive && (int) embd_inp.size() <= n_consumed) { + // if not currently processing queued inputs; + if ((int) embd_inp.size() <= n_consumed) { // check for reverse prompt if (params.antiprompt.size()) { @@ -517,10 +516,21 @@ int main(int argc, char ** argv) { is_antiprompt = false; // Check if each of the reverse prompts appears at the end of the output. + // If we're not running interactively, the reverse prompt might be tokenized with some following characters + // so we'll compensate for that by widening the search window a bit. for (std::string & antiprompt : params.antiprompt) { - if (last_output.find(antiprompt.c_str(), last_output.length() - antiprompt.length(), antiprompt.length()) != std::string::npos) { - is_interacting = true; + size_t extra_padding = params.interactive ? 0 : 2; + size_t search_start_pos = last_output.length() > static_cast(antiprompt.length() + extra_padding) + ? last_output.length() - static_cast(antiprompt.length() + extra_padding) + : 0; + + if (last_output.find(antiprompt.c_str(), search_start_pos) != std::string::npos) { + if (params.interactive) { + is_interacting = true; + set_console_color(con_st, CONSOLE_COLOR_USER_INPUT); + } is_antiprompt = true; + fflush(stdout); break; } } From e052d53e5192c809adad30741bf080f90ef8ac51 Mon Sep 17 00:00:00 2001 From: Jason McCartney Date: Thu, 11 May 2023 21:17:04 -0700 Subject: [PATCH 5/5] Update gpt_params_parse and fix a merge error take 2 --- examples/common.cpp | 2 +- examples/main/main.cpp | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/common.cpp b/examples/common.cpp index 00535f5580800..73278db0ed67f 100644 --- a/examples/common.cpp +++ b/examples/common.cpp @@ -346,7 +346,7 @@ bool gpt_params_parse(int argc, char ** argv, gpt_params & params) { } if (params.prompt_cache_all && (params.interactive || params.interactive_first || - params.instruct || params.antiprompt.size())) { + params.instruct)) { fprintf(stderr, "error: --prompt-cache-all not supported in interactive mode yet\n"); gpt_print_usage(argc, argv, default_params); exit(1); diff --git a/examples/main/main.cpp b/examples/main/main.cpp index f23a4bc72737c..ae073f08cf79d 100644 --- a/examples/main/main.cpp +++ b/examples/main/main.cpp @@ -210,7 +210,7 @@ int main(int argc, char ** argv) { } // enable interactive mode if interactive start is specified - if (params.interactive_start) { + if (params.interactive_first) { params.interactive = true; } @@ -527,7 +527,7 @@ int main(int argc, char ** argv) { if (last_output.find(antiprompt.c_str(), search_start_pos) != std::string::npos) { if (params.interactive) { is_interacting = true; - set_console_color(con_st, CONSOLE_COLOR_USER_INPUT); + console_set_color(con_st, CONSOLE_COLOR_USER_INPUT); } is_antiprompt = true; fflush(stdout);