From 80181c0712e85bd5edd045fcfc80a4a5a8f71f8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Pazdiora?= Date: Sun, 9 Apr 2023 06:11:29 +0200 Subject: [PATCH 01/12] Add support for configs, add configurable prefixes / suffixes, deprecate instruct mode, add stop prompt --- configs/alpaca.txt | 5 + configs/chat-with-bob.txt | 10 ++ configs/vicuna-simple.txt | 4 + configs/vicuna-stop.txt | 4 + configs/vicuna.txt | 5 + examples/common.cpp | 191 +++++++++++++++++++++++++++++--------- examples/common.h | 25 +++-- examples/main/main.cpp | 120 +++++++++++++++++------- prompts/alpaca.txt | 1 - prompts/chat-with-bob.txt | 7 -- 10 files changed, 279 insertions(+), 93 deletions(-) create mode 100644 configs/alpaca.txt create mode 100644 configs/chat-with-bob.txt create mode 100644 configs/vicuna-simple.txt create mode 100644 configs/vicuna-stop.txt create mode 100644 configs/vicuna.txt delete mode 100644 prompts/alpaca.txt delete mode 100644 prompts/chat-with-bob.txt diff --git a/configs/alpaca.txt b/configs/alpaca.txt new file mode 100644 index 0000000000000..06158cdb97b55 --- /dev/null +++ b/configs/alpaca.txt @@ -0,0 +1,5 @@ +--clean-interface --interactive-first --keep -1 +--in-prefix-bos --in-prefix "\n\n### Instruction:\n\n" -r "### Instruction:\n\n" +--in-suffix "\n\n### Response:\n\n" + +-p "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n" diff --git a/configs/chat-with-bob.txt b/configs/chat-with-bob.txt new file mode 100644 index 0000000000000..9019753d084d8 --- /dev/null +++ b/configs/chat-with-bob.txt @@ -0,0 +1,10 @@ +--interactive-first --keep -1 +--in-prefix-bos +--in-prefix "\nUser: " -r "User: " +--in-suffix "\nBob: " + +-p "Transcript of a dialog, where the User interacts with an Assistant named Bob. Bob is helpful, kind, honest, good at writing, and never fails to answer the User's requests immediately and with precision. +User: Hello, Bob. +Bob: Hello. How may I help you today? +User: Please tell me the largest city in Europe. +Bob: Sure. The largest city in Europe is Moscow, the capital of Russia." diff --git a/configs/vicuna-simple.txt b/configs/vicuna-simple.txt new file mode 100644 index 0000000000000..053391479e0d1 --- /dev/null +++ b/configs/vicuna-simple.txt @@ -0,0 +1,4 @@ +--interactive-first --keep -1 +--in-prefix-bos +--in-prefix "\n### Human: " --reverse-prompt "### Human: " +--in-suffix "\n### Assistant: " \ No newline at end of file diff --git a/configs/vicuna-stop.txt b/configs/vicuna-stop.txt new file mode 100644 index 0000000000000..e88a9fd053b79 --- /dev/null +++ b/configs/vicuna-stop.txt @@ -0,0 +1,4 @@ +--interactive-first --keep -1 +--in-prefix-bos +--in-prefix "\n### Human: " --reverse-prompt "### Human: " +--in-suffix "\n### Assistant: " --stop-prompt "### Assistant: " \ No newline at end of file diff --git a/configs/vicuna.txt b/configs/vicuna.txt new file mode 100644 index 0000000000000..bf3b966c7fe64 --- /dev/null +++ b/configs/vicuna.txt @@ -0,0 +1,5 @@ +--interactive-first --keep -1 +--in-prefix-bos --in-prefix "\n### Human: " -r "### Human: " +--in-suffix "\n### Assistant: " + +-p "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions." diff --git a/examples/common.cpp b/examples/common.cpp index f909eed24b325..f149206cdc64b 100644 --- a/examples/common.cpp +++ b/examples/common.cpp @@ -3,9 +3,11 @@ #include #include #include +#include #include #include #include +#include #if defined(_MSC_VER) || defined(__MINGW32__) #include // using malloc.h with MSC/MINGW @@ -29,6 +31,43 @@ extern "C" __declspec(dllimport) int __stdcall WideCharToMultiByte(unsigned int #define CP_UTF8 65001 #endif +void split_args(std::string args_string, std::vector & output_args) +{ + std::string current_arg = ""; + bool in_quotes = false; + char quote_type; + + for (char c : args_string) { + if (c == '"' || c == '\'') { + if (!in_quotes) { + in_quotes = true; + quote_type = c; + } else if (quote_type == c) { + in_quotes = false; + } else { + current_arg += c; + } + } else if (in_quotes) { + current_arg += c; + } else if (std::isspace(c)) { + if (current_arg != "") { + output_args.push_back(current_arg); + current_arg = ""; + } + } else { + current_arg += c; + } + } + + if (current_arg != "") { + output_args.push_back(current_arg); + } +} + +std::string unescape(std::string str) { + return std::regex_replace(str, std::regex("\\\\n"), "\n"); +} + bool gpt_params_parse(int argc, char ** argv, gpt_params & params) { // determine sensible default number of threads. // std::thread::hardware_concurrency may not be equal to the number of cores, or may return 0. @@ -46,35 +85,66 @@ bool gpt_params_parse(int argc, char ** argv, gpt_params & params) { std::string arg; gpt_params default_params; + // get additional arguments from config files + std::vector args; for (int i = 1; i < argc; i++) { arg = argv[i]; + if (arg == "--config") { + if (++i >= argc) { + invalid_param = true; + break; + } + std::ifstream file(argv[i]); + if (!file) { + fprintf(stderr, "error: failed to open file '%s'\n", argv[i]); + invalid_param = true; + break; + } + std::string args_string; + std::copy(std::istreambuf_iterator(file), std::istreambuf_iterator(), back_inserter(args_string)); + if (args_string.back() == '\n') { + args_string.pop_back(); + } + split_args(args_string, args); + for (int j = 0; j < args.size(); j++) { + args[j] = unescape(args[j]); + } + } else { + args.emplace_back(argv[i]); + } + } + + // parse args + int args_c = static_cast(args.size()); + for (int i = 0; i < args_c && !invalid_param; i++) { + arg = args[i]; if (arg == "-s" || arg == "--seed") { - if (++i >= argc) { + if (++i >= args_c) { invalid_param = true; break; } - params.seed = std::stoi(argv[i]); + params.seed = std::stoi(args[i]); } else if (arg == "-t" || arg == "--threads") { - if (++i >= argc) { + if (++i >= args_c) { invalid_param = true; break; } - params.n_threads = std::stoi(argv[i]); + params.n_threads = std::stoi(args[i]); } else if (arg == "-p" || arg == "--prompt") { - if (++i >= argc) { + if (++i >= args_c) { invalid_param = true; break; } - params.prompt = argv[i]; + params.prompt = args[i]; } else if (arg == "-f" || arg == "--file") { - if (++i >= argc) { + if (++i >= args_c) { invalid_param = true; break; } - std::ifstream file(argv[i]); + std::ifstream file(args[i]); if (!file) { - fprintf(stderr, "error: failed to open file '%s'\n", argv[i]); + fprintf(stderr, "error: failed to open file '%s'\n", args[i].c_str()); invalid_param = true; break; } @@ -83,78 +153,96 @@ bool gpt_params_parse(int argc, char ** argv, gpt_params & params) { params.prompt.pop_back(); } } else if (arg == "-n" || arg == "--n_predict") { - if (++i >= argc) { + if (++i >= args_c) { invalid_param = true; break; } - params.n_predict = std::stoi(argv[i]); + params.n_predict = std::stoi(args[i]); } else if (arg == "--top_k") { - if (++i >= argc) { + if (++i >= args_c) { invalid_param = true; break; } - params.top_k = std::stoi(argv[i]); + params.top_k = std::stoi(args[i]); } else if (arg == "-c" || arg == "--ctx_size") { - if (++i >= argc) { + if (++i >= args_c) { invalid_param = true; break; } - params.n_ctx = std::stoi(argv[i]); + params.n_ctx = std::stoi(args[i]); } else if (arg == "--memory_f32") { params.memory_f16 = false; } else if (arg == "--top_p") { - if (++i >= argc) { + if (++i >= args_c) { invalid_param = true; break; } - params.top_p = std::stof(argv[i]); + params.top_p = std::stof(args[i]); } else if (arg == "--temp") { - if (++i >= argc) { + if (++i >= args_c) { invalid_param = true; break; } - params.temp = std::stof(argv[i]); + params.temp = std::stof(args[i]); } else if (arg == "--repeat_last_n") { - if (++i >= argc) { + if (++i >= args_c) { invalid_param = true; break; } - params.repeat_last_n = std::stoi(argv[i]); + params.repeat_last_n = std::stoi(args[i]); } else if (arg == "--repeat_penalty") { - if (++i >= argc) { + if (++i >= args_c) { invalid_param = true; break; } - params.repeat_penalty = std::stof(argv[i]); + params.repeat_penalty = std::stof(args[i]); } else if (arg == "-b" || arg == "--batch_size") { - if (++i >= argc) { + if (++i >= args_c) { invalid_param = true; break; } - params.n_batch = std::stoi(argv[i]); + params.n_batch = std::stoi(args[i]); params.n_batch = std::min(512, params.n_batch); } else if (arg == "--keep") { - if (++i >= argc) { + if (++i >= args_c) { invalid_param = true; break; } - params.n_keep = std::stoi(argv[i]); + params.n_keep = std::stoi(args[i]); } else if (arg == "-m" || arg == "--model") { - if (++i >= argc) { + if (++i >= args_c) { invalid_param = true; break; } - params.model = argv[i]; + params.model = args[i]; } else if (arg == "-i" || arg == "--interactive") { params.interactive = true; } else if (arg == "--embedding") { params.embedding = true; + } else if (arg == "--clean-interface") { + params.clean_interface = true; } else if (arg == "--interactive-start") { params.interactive = true; } else if (arg == "--interactive-first") { params.interactive_start = true; } else if (arg == "-ins" || arg == "--instruct") { - params.instruct = true; + fprintf(stderr, "\n\nWarning: instruct mode is deprecated! Use: \n" + "--clean-interface " + "--interactive-first " + "--keep -1 " + "--in-prefix-bos " + "--in-prefix \"\\n\\n### Instruction:\\n\\n\" " + "--in-suffix \"\\n\\n### Response:\\n\\n\" " + "-r \"### Instruction:\\n\\n\" " + "\n\n"); + // params.instruct = true; + params.clean_interface = true; + params.interactive_start = true; + params.n_keep = -1; + params.input_prefix_bos = true; + params.input_prefix = "\n\n### Instruction:\n\n"; + params.input_suffix = "\n\n### Response:\n\n"; + params.antiprompt.push_back("### Instruction:\n\n"); } else if (arg == "--color") { params.use_color = true; } else if (arg == "--mlock") { @@ -166,55 +254,71 @@ bool gpt_params_parse(int argc, char ** argv, gpt_params & params) { } else if (arg == "--verbose-prompt") { params.verbose_prompt = true; } else if (arg == "-r" || arg == "--reverse-prompt") { - if (++i >= argc) { + if (++i >= args_c) { invalid_param = true; break; } - params.antiprompt.push_back(argv[i]); + params.antiprompt.push_back(args[i]); + } else if (arg == "--stop-prompt") { + if (++i >= args_c) { + invalid_param = true; + break; + } + params.stopprompt.push_back(args[i]); } else if (arg == "--perplexity") { params.perplexity = true; } else if (arg == "--ignore-eos") { params.ignore_eos = true; } else if (arg == "--n_parts") { - if (++i >= argc) { + if (++i >= args_c) { invalid_param = true; break; } - params.n_parts = std::stoi(argv[i]); + params.n_parts = std::stoi(args[i]); } else if (arg == "-h" || arg == "--help") { - gpt_print_usage(argc, argv, default_params); + gpt_print_usage(argv[0], default_params); exit(0); } else if (arg == "--random-prompt") { params.random_prompt = true; + } else if (arg == "--in-prefix-bos") { + params.input_prefix_bos = true; } else if (arg == "--in-prefix") { - if (++i >= argc) { + if (++i >= args_c) { + invalid_param = true; + break; + } + params.input_prefix = args[i]; + } else if (arg == "--in-suffix-bos") { + params.input_suffix_bos = true; + } else if (arg == "--in-suffix") { + if (++i >= args_c) { invalid_param = true; break; } - params.input_prefix = argv[i]; + params.input_suffix = args[i]; } else { fprintf(stderr, "error: unknown argument: %s\n", arg.c_str()); - gpt_print_usage(argc, argv, default_params); + gpt_print_usage(argv[0], default_params); exit(1); } } if (invalid_param) { fprintf(stderr, "error: invalid parameter for argument: %s\n", arg.c_str()); - gpt_print_usage(argc, argv, default_params); + gpt_print_usage(argv[0], default_params); exit(1); } return true; } -void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) { - fprintf(stderr, "usage: %s [options]\n", argv[0]); +void gpt_print_usage(char * argv_0, const gpt_params & params) { + fprintf(stderr, "usage: %s [options]\n", argv_0); fprintf(stderr, "\n"); fprintf(stderr, "options:\n"); fprintf(stderr, " -h, --help show this help message and exit\n"); fprintf(stderr, " -i, --interactive run in interactive mode\n"); fprintf(stderr, " --interactive-first run in interactive mode and wait for input right away\n"); - fprintf(stderr, " -ins, --instruct run in instruction mode (use with Alpaca models)\n"); + fprintf(stderr, " --clean-interface hides input prefix & suffix and displays '>' instead\n"); fprintf(stderr, " -r PROMPT, --reverse-prompt PROMPT\n"); fprintf(stderr, " run in interactive mode and poll user input upon seeing PROMPT (can be\n"); fprintf(stderr, " specified more than once for multiple prompts).\n"); @@ -224,7 +328,10 @@ void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) { fprintf(stderr, " -p PROMPT, --prompt PROMPT\n"); fprintf(stderr, " prompt to start generation with (default: empty)\n"); fprintf(stderr, " --random-prompt start with a randomized prompt.\n"); + fprintf(stderr, " --in-prefix-bos append bos token to input prefix.\n"); fprintf(stderr, " --in-prefix STRING string to prefix user inputs with (default: empty)\n"); + fprintf(stderr, " --in-suffix-bos append bos token to input suffix.\n"); + fprintf(stderr, " --in-suffix STRING string to suffix user inputs with (default: empty)\n"); fprintf(stderr, " -f FNAME, --file FNAME\n"); fprintf(stderr, " prompt file to start generation.\n"); fprintf(stderr, " -n N, --n_predict N number of tokens to predict (default: %d, -1 = infinity)\n", params.n_predict); diff --git a/examples/common.h b/examples/common.h index 1ea6f74451811..3a7f103e5060c 100644 --- a/examples/common.h +++ b/examples/common.h @@ -14,14 +14,14 @@ // struct gpt_params { - int32_t seed = -1; // RNG seed - int32_t n_threads = std::min(4, (int32_t) std::thread::hardware_concurrency()); - int32_t n_predict = 128; // new tokens to predict - int32_t repeat_last_n = 64; // last n tokens to penalize - int32_t n_parts = -1; // amount of model parts (-1 = determine from model dimensions) - int32_t n_ctx = 512; // context size - int32_t n_batch = 8; // batch size for prompt processing - int32_t n_keep = 0; // number of tokens to keep from initial prompt + int32_t seed = -1; // RNG seed + int32_t n_threads = std::min(4, (int32_t) std::thread::hardware_concurrency()); // max 4 threads (default) + int32_t n_predict = 128; // new tokens to predict + int32_t repeat_last_n = 64; // last n tokens to penalize + int32_t n_parts = -1; // amount of model parts (-1 = determine from model dimensions) + int32_t n_ctx = 512; // context size + int32_t n_batch = 8; // batch size for prompt processing + int32_t n_keep = 0; // number of tokens to keep from initial prompt (-1 for all) // sampling parameters int32_t top_k = 40; @@ -32,9 +32,12 @@ struct gpt_params { std::string model = "models/lamma-7B/ggml-model.bin"; // model path std::string prompt = ""; std::string input_prefix = ""; // string to prefix user inputs with - + bool input_prefix_bos = false; // append bos token to input prefix + std::string input_suffix = ""; // string to suffix user inputs with + bool input_suffix_bos = false; // append bos token to input suffix std::vector antiprompt; // string upon seeing which more user input is prompted + std::vector stopprompt; // string upon seeing which more user input is prompted (without adding prefixes and suffixes) bool memory_f16 = true; // use f16 instead of f32 for memory kv bool random_prompt = false; // do not randomize prompt if none provided @@ -51,11 +54,13 @@ struct gpt_params { bool use_mlock = false; // use mlock to keep model in memory bool mem_test = false; // compute maximum memory usage bool verbose_prompt = false; // print prompt tokens before generation + + bool clean_interface = false; // hides input prefix & suffix and displays '>' }; bool gpt_params_parse(int argc, char ** argv, gpt_params & params); -void gpt_print_usage(int argc, char ** argv, const gpt_params & params); +void gpt_print_usage(char * argv_0, const gpt_params & params); std::string gpt_random_prompt(std::mt19937 & rng); diff --git a/examples/main/main.cpp b/examples/main/main.cpp index d333d0dbacd20..744c968392d80 100644 --- a/examples/main/main.cpp +++ b/examples/main/main.cpp @@ -148,22 +148,18 @@ int main(int argc, char ** argv) { } // number of tokens to keep when resetting context - if (params.n_keep < 0 || params.n_keep > (int)embd_inp.size() || params.instruct) { + if (params.n_keep < 0 || params.n_keep > (int)embd_inp.size()) { params.n_keep = (int)embd_inp.size(); } // prefix & suffix for instruct mode - const auto inp_pfx = ::llama_tokenize(ctx, "\n\n### Instruction:\n\n", true); - const auto inp_sfx = ::llama_tokenize(ctx, "\n\n### Response:\n\n", false); - - // in instruct mode, we inject a prefix and a suffix to each input by the user - if (params.instruct) { - params.interactive_start = true; - params.antiprompt.push_back("### Instruction:\n\n"); - } + const auto inp_pfx = ::llama_tokenize(ctx, params.input_prefix, params.input_prefix_bos); + std::string input_suffix = params.input_suffix; + if (input_suffix.back() == ' ') { input_suffix.pop_back(); } // (remove trailing space workaround) + const auto inp_sfx = ::llama_tokenize(ctx, input_suffix, params.input_suffix_bos); // enable interactive mode if reverse prompt or interactive start is specified - if (params.antiprompt.size() != 0 || params.interactive_start) { + if (params.antiprompt.size() != 0 || params.stopprompt.size() != 0 || params.interactive_start) { params.interactive = true; } @@ -205,9 +201,17 @@ int main(int argc, char ** argv) { fprintf(stderr, "Reverse prompt: '%s'\n", antiprompt.c_str()); } } + if (params.stopprompt.size()) { + for (auto stopprompt : params.stopprompt) { + fprintf(stderr, "Stop prompt: '%s'\n", stopprompt.c_str()); + } + } if (!params.input_prefix.empty()) { - fprintf(stderr, "Input prefix: '%s'\n", params.input_prefix.c_str()); + fprintf(stderr, "Input prefix %s: '%s'\n", params.input_prefix_bos ? "(with bos token)" : "", params.input_prefix.c_str()); + } + if (!params.input_suffix.empty()) { + fprintf(stderr, "Input suffix %s: '%s'\n", params.input_suffix_bos ? "(with bos token)" : "", params.input_suffix.c_str()); } } fprintf(stderr, "sampling: temp = %f, top_k = %d, top_p = %f, repeat_last_n = %i, repeat_penalty = %f\n", @@ -229,7 +233,13 @@ int main(int argc, char ** argv) { is_interacting = params.interactive_start; } - bool is_antiprompt = false; + struct Antiprompt { + bool any = false; + bool trailing_space = false; + size_t len; + bool is_stop_prompt = false; + } antiprompt; + bool input_noecho = false; int n_past = 0; @@ -299,7 +309,7 @@ int main(int argc, char ** argv) { } // replace end of text token with newline token when in interactive mode - if (id == llama_token_eos() && params.interactive && !params.instruct) { + if (id == llama_token_eos() && params.interactive && params.input_prefix.empty()) { id = llama_token_newline.front(); if (params.antiprompt.size() != 0) { // tokenize and inject first reverse prompt @@ -345,27 +355,63 @@ int main(int argc, char ** argv) { // check if we should prompt the user for more if (params.interactive && (int) embd_inp.size() <= n_consumed) { - // check for reverse prompt - if (params.antiprompt.size()) { + // check for reverse prompt or stop prompt + if (params.antiprompt.size() || params.stopprompt.size()) { std::string last_output; for (auto id : last_n_tokens) { last_output += llama_token_to_str(ctx, id); } - is_antiprompt = false; + antiprompt.any = false; + antiprompt.is_stop_prompt = false; // Check if each of the reverse prompts appears at the end of the output. - for (std::string & antiprompt : params.antiprompt) { - if (last_output.find(antiprompt.c_str(), last_output.length() - antiprompt.length(), antiprompt.length()) != std::string::npos) { + for (std::string & prompt : params.antiprompt) { + antiprompt.trailing_space = prompt.back() == ' '; + antiprompt.len = prompt.length() - (antiprompt.trailing_space ? 1 : 0); + if (last_output.find(prompt.c_str(), last_output.length() - antiprompt.len, antiprompt.len) != std::string::npos) { is_interacting = true; - is_antiprompt = true; + antiprompt.any = true; set_console_color(con_st, CONSOLE_COLOR_USER_INPUT); fflush(stdout); break; } } + if (!antiprompt.any) { + for (std::string & prompt : params.stopprompt) { + antiprompt.trailing_space = prompt.back() == ' '; + antiprompt.len = prompt.length() - (antiprompt.trailing_space ? 1 : 0); + if (last_output.find(prompt.c_str(), last_output.length() - antiprompt.len, antiprompt.len) != std::string::npos) { + is_interacting = true; + antiprompt.any = true; + antiprompt.is_stop_prompt = true; + set_console_color(con_st, CONSOLE_COLOR_USER_INPUT); + fflush(stdout); + break; + } + } + } } - if (n_past > 0 && is_interacting) { + if (n_past > 0 && is_interacting) + { + std::string buffer; + if (!params.clean_interface && !params.input_prefix.empty() && !antiprompt.any) { + // avoid printing again user's new line (TODO: try to revert enter press and print newline) + int i = params.input_prefix.front() == '\n' ? 1 : 0; + for (; i < inp_pfx.size(); i++) { + printf("%s", llama_token_to_str(ctx, inp_pfx[i])); + } + fflush(stdout); + } + if (antiprompt.any && antiprompt.trailing_space) { + // add back removed trailing space to buffer(workaround) + buffer += ' '; + if (!params.clean_interface) { + printf("%s", buffer.c_str()); + } + fflush(stdout); + } + // potentially set color to indicate we are taking user input set_console_color(con_st, CONSOLE_COLOR_USER_INPUT); @@ -374,19 +420,14 @@ int main(int argc, char ** argv) { signal(SIGINT, sigint_handler); #endif - if (params.instruct) { + if (params.clean_interface) { printf("\n> "); } - std::string buffer; - if (!params.input_prefix.empty()) { - buffer += params.input_prefix; - printf("%s", buffer.c_str()); - } - std::string line; bool another_line = true; do { + // TODO: try to revert going to new line after enter (to enable in-line text writing) #if defined(_WIN32) std::wstring wline; if (!std::getline(std::wcin, wline)) { @@ -405,18 +446,31 @@ int main(int argc, char ** argv) { } else { line.pop_back(); // Remove the continue character } - buffer += line + '\n'; // Append the line to the result + buffer += line; + if (another_line || !antiprompt.is_stop_prompt) { + buffer += '\n'; // Append the line to the result + } } while (another_line); // done taking input, reset color set_console_color(con_st, CONSOLE_COLOR_DEFAULT); + if (!params.clean_interface && !params.input_suffix.empty() && !antiprompt.is_stop_prompt) { + // avoid printing again user's new line (TODO: try to revert enter press and print newline) + int i = params.input_suffix.front() == '\n' ? 1 : 0; + for (; i < inp_sfx.size(); i++) { + printf("%s", llama_token_to_str(ctx, inp_sfx[i])); + } + // we won't add back removed trailing space here (workaround) + fflush(stdout); + } + // Add tokens to embd only if the input buffer is non-empty // Entering a empty line lets the user pass control back if (buffer.length() > 1) { - // instruct mode: insert instruction prefix - if (params.instruct && !is_antiprompt) { + // insert input prefix + if (!params.input_prefix.empty() && !antiprompt.any) { n_consumed = embd_inp.size(); embd_inp.insert(embd_inp.end(), inp_pfx.begin(), inp_pfx.end()); } @@ -424,8 +478,8 @@ int main(int argc, char ** argv) { auto line_inp = ::llama_tokenize(ctx, buffer, false); embd_inp.insert(embd_inp.end(), line_inp.begin(), line_inp.end()); - // instruct mode: insert response suffix - if (params.instruct) { + // insert response suffix + if (!params.input_suffix.empty() && !antiprompt.is_stop_prompt) { embd_inp.insert(embd_inp.end(), inp_sfx.begin(), inp_sfx.end()); } @@ -442,7 +496,7 @@ int main(int argc, char ** argv) { // end of text token if (!embd.empty() && embd.back() == llama_token_eos()) { - if (params.instruct) { + if (params.interactive && !params.input_prefix.empty()) { is_interacting = true; } else { fprintf(stderr, " [end of text]\n"); diff --git a/prompts/alpaca.txt b/prompts/alpaca.txt deleted file mode 100644 index 2224bdeb0bcd4..0000000000000 --- a/prompts/alpaca.txt +++ /dev/null @@ -1 +0,0 @@ -Below is an instruction that describes a task. Write a response that appropriately completes the request. diff --git a/prompts/chat-with-bob.txt b/prompts/chat-with-bob.txt deleted file mode 100644 index 009da39aee066..0000000000000 --- a/prompts/chat-with-bob.txt +++ /dev/null @@ -1,7 +0,0 @@ -Transcript of a dialog, where the User interacts with an Assistant named Bob. Bob is helpful, kind, honest, good at writing, and never fails to answer the User's requests immediately and with precision. - -User: Hello, Bob. -Bob: Hello. How may I help you today? -User: Please tell me the largest city in Europe. -Bob: Sure. The largest city in Europe is Moscow, the capital of Russia. -User: From ddf6bd6497fae1cbdea6aad3d6dcdc7292a52815 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Pazdiora?= Date: Sun, 9 Apr 2023 18:36:23 +0200 Subject: [PATCH 02/12] Add multiline mode, update text input. --- examples/common.cpp | 52 ++++++++++++++++++++++++++++++++++++++++++ examples/common.h | 3 +++ examples/main/main.cpp | 51 +++++++++++++++++------------------------ 3 files changed, 76 insertions(+), 30 deletions(-) diff --git a/examples/common.cpp b/examples/common.cpp index f149206cdc64b..212974481ad9c 100644 --- a/examples/common.cpp +++ b/examples/common.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -245,6 +246,8 @@ bool gpt_params_parse(int argc, char ** argv, gpt_params & params) { params.antiprompt.push_back("### Instruction:\n\n"); } else if (arg == "--color") { params.use_color = true; + } else if (arg == "--multiline") { + params.multiline_mode = true; } else if (arg == "--mlock") { params.use_mlock = true; } else if (arg == "--no-mmap") { @@ -323,6 +326,7 @@ void gpt_print_usage(char * argv_0, const gpt_params & params) { fprintf(stderr, " run in interactive mode and poll user input upon seeing PROMPT (can be\n"); fprintf(stderr, " specified more than once for multiple prompts).\n"); fprintf(stderr, " --color colorise output to distinguish prompt and user input from generations\n"); + fprintf(stderr, " --multiline multiline mode (use Ctrl+D on Linux/Mac and Ctrl+Z on Windpws to send input)\n"); fprintf(stderr, " -s SEED, --seed SEED RNG seed (default: -1, use random seed for <= 0)\n"); fprintf(stderr, " -t N, --threads N number of threads to use during computation (default: %d)\n", params.n_threads); fprintf(stderr, " -p PROMPT, --prompt PROMPT\n"); @@ -441,3 +445,51 @@ void win32_utf8_encode(const std::wstring & wstr, std::string & str) { str = strTo; } #endif + +bool get_input_text(std::string & input_text, bool escape_newline_mode) { + bool another_line = true; + do { + std::string line; +#if defined(_WIN32) + std::wstring wline; + if (!std::getline(std::wcin, wline)) { + // input stream is bad or EOF received + if (std::wcin.bad()) { + fprintf(stderr, "%s: error: input stream bad\n", __func__); + return 1; + } + } + if (std::wcin.eof()) { + another_line = false; + std::wcin.clear(); + std::wcin.seekg(0, std::ios::beg); + } + win32_utf8_encode(wline, line); +#else + if (!std::getline(std::cin, line)) { + // input stream is bad or EOF received + if (std::wcin.bad()) { + fprintf(stderr, "%s: error: input stream bad\n", __func__); + return 1; + } + } + if (std::ccin.eof()) { + another_line = false; + std::cin.clear(); + std::cin.seekg(0, std::ios::beg); + } +#endif + if (escape_newline_mode) { + if (line.empty() || line.back() != '\\') { + another_line = false; + } else { + line.pop_back(); // Remove the continue character + } + } + input_text += line; + if (another_line) { + input_text += '\n'; // Append the line to the result + } + } while (another_line); + return true; +} diff --git a/examples/common.h b/examples/common.h index 3a7f103e5060c..2c4632b41649a 100644 --- a/examples/common.h +++ b/examples/common.h @@ -56,6 +56,7 @@ struct gpt_params { bool verbose_prompt = false; // print prompt tokens before generation bool clean_interface = false; // hides input prefix & suffix and displays '>' + bool multiline_mode = false; // enables multi-line mode, to send input press CTRL+D on Linux/Max, CTRL+Z on Windows }; bool gpt_params_parse(int argc, char ** argv, gpt_params & params); @@ -100,3 +101,5 @@ void set_console_color(console_state & con_st, console_color_t color); void win32_console_init(bool enable_color); void win32_utf8_encode(const std::wstring & wstr, std::string & str); #endif + +bool get_input_text(std::string & input_text, bool escape_newline_mode); diff --git a/examples/main/main.cpp b/examples/main/main.cpp index 744c968392d80..7ea9c3654f64b 100644 --- a/examples/main/main.cpp +++ b/examples/main/main.cpp @@ -25,7 +25,8 @@ static bool is_interacting = false; #if defined (__unix__) || (defined (__APPLE__) && defined (__MACH__)) || defined (_WIN32) void sigint_handler(int signo) { set_console_color(con_st, CONSOLE_COLOR_DEFAULT); - printf("\n"); // this also force flush stdout. + fflush(stdout); + fflush(stderr); if (signo == SIGINT) { if (!is_interacting) { is_interacting=true; @@ -228,8 +229,18 @@ int main(int argc, char ** argv) { #if defined (__unix__) || (defined (__APPLE__) && defined (__MACH__)) || defined (_WIN32) " - Press Ctrl+C to interject at any time.\n" #endif - " - Press Return to return control to LLaMa.\n" - " - If you want to submit another line, end your input in '\\'.\n\n"); + ); + if (params.multiline_mode) { +#if defined (_WIN32) + fprintf(stderr, " - Press Ctrl+Z and Return (EOF) to return control to LLaMa.\n\n"); +#else + fprintf(stderr, " - Press Ctrl+D (EOF) to return control to LLaMa.\n\n"); +#endif + } + else { + fprintf(stderr, " - Press Return to return control to LLaMa.\n" + " - If you want to submit another line, end your input in '\\'.\n\n"); + } is_interacting = params.interactive_start; } @@ -424,33 +435,13 @@ int main(int argc, char ** argv) { printf("\n> "); } - std::string line; - bool another_line = true; - do { - // TODO: try to revert going to new line after enter (to enable in-line text writing) -#if defined(_WIN32) - std::wstring wline; - if (!std::getline(std::wcin, wline)) { - // input stream is bad or EOF received - return 0; - } - win32_utf8_encode(wline, line); -#else - if (!std::getline(std::cin, line)) { - // input stream is bad or EOF received - return 0; - } -#endif - if (line.empty() || line.back() != '\\') { - another_line = false; - } else { - line.pop_back(); // Remove the continue character - } - buffer += line; - if (another_line || !antiprompt.is_stop_prompt) { - buffer += '\n'; // Append the line to the result - } - } while (another_line); + if (!get_input_text(buffer, !params.multiline_mode)) { + // input stream is bad + return 1; + } + if (!antiprompt.is_stop_prompt) { + buffer += "\n"; + } // done taking input, reset color set_console_color(con_st, CONSOLE_COLOR_DEFAULT); From 0dc9dcff130c798b5cdd55d836ef8cdf6e10ae27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Pazdiora?= Date: Sun, 9 Apr 2023 18:38:23 +0200 Subject: [PATCH 03/12] bugfix --- examples/common.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/common.cpp b/examples/common.cpp index 212974481ad9c..59b0fadfcfb6d 100644 --- a/examples/common.cpp +++ b/examples/common.cpp @@ -473,7 +473,7 @@ bool get_input_text(std::string & input_text, bool escape_newline_mode) { return 1; } } - if (std::ccin.eof()) { + if (std::cin.eof()) { another_line = false; std::cin.clear(); std::cin.seekg(0, std::ios::beg); From 7c60721217d59cced92647a6594760bc58a7721d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Pazdiora?= Date: Sun, 9 Apr 2023 19:49:33 +0200 Subject: [PATCH 04/12] update implementation --- configs/alpaca.txt | 10 +++-- configs/chat-with-bob.txt | 11 ++++-- configs/vicuna-simple.txt | 11 ++++-- configs/vicuna-stop.txt | 12 ++++-- configs/vicuna.txt | 10 +++-- examples/common.cpp | 39 +++++++++++-------- examples/common.h | 12 ++++-- examples/main/main.cpp | 80 +++++++++++++++++++++++++-------------- 8 files changed, 120 insertions(+), 65 deletions(-) diff --git a/configs/alpaca.txt b/configs/alpaca.txt index 06158cdb97b55..5e1558d18bd1d 100644 --- a/configs/alpaca.txt +++ b/configs/alpaca.txt @@ -1,5 +1,9 @@ ---clean-interface --interactive-first --keep -1 ---in-prefix-bos --in-prefix "\n\n### Instruction:\n\n" -r "### Instruction:\n\n" ---in-suffix "\n\n### Response:\n\n" +--clean-interface +--interactive-first +--keep -1 +--ins-prefix-bos +--ins-prefix "\n\n### Instruction:\n\n" +--ins-suffix "\n\n### Response:\n\n" +--reverse-prompt "### Instruction:\n\n" -p "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n" diff --git a/configs/chat-with-bob.txt b/configs/chat-with-bob.txt index 9019753d084d8..ede63d35aa078 100644 --- a/configs/chat-with-bob.txt +++ b/configs/chat-with-bob.txt @@ -1,7 +1,10 @@ ---interactive-first --keep -1 ---in-prefix-bos ---in-prefix "\nUser: " -r "User: " ---in-suffix "\nBob: " +--interactive-first +--keep -1 +--ins-prefix-bos +--ins-prefix "\nUser: " +--ins-suffix "\nBob: " +--reverse-prompt "User: " +--rm-trailing-space-workaround -p "Transcript of a dialog, where the User interacts with an Assistant named Bob. Bob is helpful, kind, honest, good at writing, and never fails to answer the User's requests immediately and with precision. User: Hello, Bob. diff --git a/configs/vicuna-simple.txt b/configs/vicuna-simple.txt index 053391479e0d1..a87c70082583a 100644 --- a/configs/vicuna-simple.txt +++ b/configs/vicuna-simple.txt @@ -1,4 +1,7 @@ ---interactive-first --keep -1 ---in-prefix-bos ---in-prefix "\n### Human: " --reverse-prompt "### Human: " ---in-suffix "\n### Assistant: " \ No newline at end of file +--interactive-first +--keep -1 +--ins-prefix-bos +--ins-prefix "\n### Human: " +--ins-suffix "\n### Assistant: " +--reverse-prompt "### Human: " +--rm-trailing-space-workaround diff --git a/configs/vicuna-stop.txt b/configs/vicuna-stop.txt index e88a9fd053b79..a8d8907244256 100644 --- a/configs/vicuna-stop.txt +++ b/configs/vicuna-stop.txt @@ -1,4 +1,8 @@ ---interactive-first --keep -1 ---in-prefix-bos ---in-prefix "\n### Human: " --reverse-prompt "### Human: " ---in-suffix "\n### Assistant: " --stop-prompt "### Assistant: " \ No newline at end of file +--interactive-first +--keep -1 +--ins-prefix-bos +--ins-prefix "\n### Human: " +--ins-suffix "\n### Assistant: " +--reverse-prompt "### Human: " +--stop-prompt "### Assistant: " +--rm-trailing-space-workaround diff --git a/configs/vicuna.txt b/configs/vicuna.txt index bf3b966c7fe64..f4d0d2d742a6c 100644 --- a/configs/vicuna.txt +++ b/configs/vicuna.txt @@ -1,5 +1,9 @@ ---interactive-first --keep -1 ---in-prefix-bos --in-prefix "\n### Human: " -r "### Human: " ---in-suffix "\n### Assistant: " +--interactive-first +--keep -1 +--ins-prefix-bos +--ins-prefix "\n### Human: " +--ins-suffix "\n### Assistant: " +--reverse-prompt "### Human: " +--rm-trailing-space-workaround -p "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions." diff --git a/examples/common.cpp b/examples/common.cpp index 59b0fadfcfb6d..9bf5152feb9f5 100644 --- a/examples/common.cpp +++ b/examples/common.cpp @@ -231,18 +231,18 @@ bool gpt_params_parse(int argc, char ** argv, gpt_params & params) { "--clean-interface " "--interactive-first " "--keep -1 " - "--in-prefix-bos " - "--in-prefix \"\\n\\n### Instruction:\\n\\n\" " - "--in-suffix \"\\n\\n### Response:\\n\\n\" " + "--ins-prefix-bos " + "--ins-prefix \"\\n\\n### Instruction:\\n\\n\" " + "--ins-suffix \"\\n\\n### Response:\\n\\n\" " "-r \"### Instruction:\\n\\n\" " "\n\n"); // params.instruct = true; params.clean_interface = true; params.interactive_start = true; params.n_keep = -1; - params.input_prefix_bos = true; - params.input_prefix = "\n\n### Instruction:\n\n"; - params.input_suffix = "\n\n### Response:\n\n"; + params.instruct_prefix_bos = true; + params.instruct_prefix = "\n\n### Instruction:\n\n"; + params.instruct_suffix = "\n\n### Response:\n\n"; params.antiprompt.push_back("### Instruction:\n\n"); } else if (arg == "--color") { params.use_color = true; @@ -268,6 +268,8 @@ bool gpt_params_parse(int argc, char ** argv, gpt_params & params) { break; } params.stopprompt.push_back(args[i]); + } else if (arg == "--rm-trailing-space-workaround") { + params.rm_trailing_space_workaround = true; } else if (arg == "--perplexity") { params.perplexity = true; } else if (arg == "--ignore-eos") { @@ -283,22 +285,28 @@ bool gpt_params_parse(int argc, char ** argv, gpt_params & params) { exit(0); } else if (arg == "--random-prompt") { params.random_prompt = true; - } else if (arg == "--in-prefix-bos") { - params.input_prefix_bos = true; } else if (arg == "--in-prefix") { if (++i >= args_c) { invalid_param = true; break; } params.input_prefix = args[i]; - } else if (arg == "--in-suffix-bos") { - params.input_suffix_bos = true; - } else if (arg == "--in-suffix") { + } else if (arg == "--ins-prefix-bos") { + params.instruct_prefix_bos = true; + } else if (arg == "--ins-prefix") { if (++i >= args_c) { invalid_param = true; break; } - params.input_suffix = args[i]; + params.instruct_prefix = args[i]; + } else if (arg == "--ins-suffix-bos") { + params.instruct_suffix_bos = true; + } else if (arg == "--ins-suffix") { + if (++i >= args_c) { + invalid_param = true; + break; + } + params.instruct_suffix = args[i]; } else { fprintf(stderr, "error: unknown argument: %s\n", arg.c_str()); gpt_print_usage(argv[0], default_params); @@ -332,10 +340,11 @@ void gpt_print_usage(char * argv_0, const gpt_params & params) { fprintf(stderr, " -p PROMPT, --prompt PROMPT\n"); fprintf(stderr, " prompt to start generation with (default: empty)\n"); fprintf(stderr, " --random-prompt start with a randomized prompt.\n"); - fprintf(stderr, " --in-prefix-bos append bos token to input prefix.\n"); fprintf(stderr, " --in-prefix STRING string to prefix user inputs with (default: empty)\n"); - fprintf(stderr, " --in-suffix-bos append bos token to input suffix.\n"); - fprintf(stderr, " --in-suffix STRING string to suffix user inputs with (default: empty)\n"); + fprintf(stderr, " --ins-prefix STRING (instruct) prefix user inputs with tokenized string (default: empty)\n"); + fprintf(stderr, " --ins-prefix-bos (instruct) prepend bos token to instruct prefix.\n"); + fprintf(stderr, " --ins-suffix STRING (instruct) suffix user inputs with tokenized string (default: empty)\n"); + fprintf(stderr, " --ins-suffix-bos (instruct) prepend bos token to instruct suffix.\n"); fprintf(stderr, " -f FNAME, --file FNAME\n"); fprintf(stderr, " prompt file to start generation.\n"); fprintf(stderr, " -n N, --n_predict N number of tokens to predict (default: %d, -1 = infinity)\n", params.n_predict); diff --git a/examples/common.h b/examples/common.h index 2c4632b41649a..413604600f3b9 100644 --- a/examples/common.h +++ b/examples/common.h @@ -32,12 +32,16 @@ struct gpt_params { std::string model = "models/lamma-7B/ggml-model.bin"; // model path std::string prompt = ""; std::string input_prefix = ""; // string to prefix user inputs with - bool input_prefix_bos = false; // append bos token to input prefix - std::string input_suffix = ""; // string to suffix user inputs with - bool input_suffix_bos = false; // append bos token to input suffix + + std::string instruct_prefix = ""; // prefix user inputs with tokenized string + bool instruct_prefix_bos = false; // prepend bos token to instruct prefix + std::string instruct_suffix = ""; // suffix user inputs with tokenized string + bool instruct_suffix_bos = false; // prepend bos token to instruct suffix std::vector antiprompt; // string upon seeing which more user input is prompted - std::vector stopprompt; // string upon seeing which more user input is prompted (without adding prefixes and suffixes) + std::vector stopprompt; // string upon seeing which more user input is prompted (without adding instruct prefixes and suffixes) + + bool rm_trailing_space_workaround = false; // workaround for removing trailing space from reverse/stop prompts bool memory_f16 = true; // use f16 instead of f32 for memory kv bool random_prompt = false; // do not randomize prompt if none provided diff --git a/examples/main/main.cpp b/examples/main/main.cpp index 7ea9c3654f64b..47969d2182890 100644 --- a/examples/main/main.cpp +++ b/examples/main/main.cpp @@ -85,6 +85,8 @@ int main(int argc, char ** argv) { params.prompt = gpt_random_prompt(rng); } + bool instruct_mode = params.instruct_prefix.empty() && params.instruct_suffix.empty(); + // params.prompt = R"(// this function checks if the number n is prime //bool is_prime(int n) {)"; @@ -154,10 +156,12 @@ int main(int argc, char ** argv) { } // prefix & suffix for instruct mode - const auto inp_pfx = ::llama_tokenize(ctx, params.input_prefix, params.input_prefix_bos); - std::string input_suffix = params.input_suffix; - if (input_suffix.back() == ' ') { input_suffix.pop_back(); } // (remove trailing space workaround) - const auto inp_sfx = ::llama_tokenize(ctx, input_suffix, params.input_suffix_bos); + const auto inp_pfx = ::llama_tokenize(ctx, params.instruct_prefix, params.instruct_prefix_bos); + std::string instruct_suffix = params.instruct_suffix; + if (params.rm_trailing_space_workaround) { + if (instruct_suffix.back() == ' ') { instruct_suffix.pop_back(); } + } + const auto inp_sfx = ::llama_tokenize(ctx, instruct_suffix, params.instruct_suffix_bos); // enable interactive mode if reverse prompt or interactive start is specified if (params.antiprompt.size() != 0 || params.stopprompt.size() != 0 || params.interactive_start) { @@ -209,10 +213,13 @@ int main(int argc, char ** argv) { } if (!params.input_prefix.empty()) { - fprintf(stderr, "Input prefix %s: '%s'\n", params.input_prefix_bos ? "(with bos token)" : "", params.input_prefix.c_str()); + fprintf(stderr, "Input prefix: '%s'\n", params.input_prefix.c_str()); } - if (!params.input_suffix.empty()) { - fprintf(stderr, "Input suffix %s: '%s'\n", params.input_suffix_bos ? "(with bos token)" : "", params.input_suffix.c_str()); + if (!params.instruct_prefix.empty()) { + fprintf(stderr, "Instruct prefix %s: '%s'\n", params.instruct_prefix_bos ? "(with bos token)" : "", params.instruct_prefix.c_str()); + } + if (!params.instruct_suffix.empty()) { + fprintf(stderr, "Instruct suffix %s: '%s'\n", params.instruct_suffix_bos ? "(with bos token)" : "", params.instruct_suffix.c_str()); } } fprintf(stderr, "sampling: temp = %f, top_k = %d, top_p = %f, repeat_last_n = %i, repeat_penalty = %f\n", @@ -232,9 +239,9 @@ int main(int argc, char ** argv) { ); if (params.multiline_mode) { #if defined (_WIN32) - fprintf(stderr, " - Press Ctrl+Z and Return (EOF) to return control to LLaMa.\n\n"); + fprintf(stderr, " - [MULTILINE MODE] Press Ctrl+Z and Return (EOF) to return control to LLaMa.\n\n"); #else - fprintf(stderr, " - Press Ctrl+D (EOF) to return control to LLaMa.\n\n"); + fprintf(stderr, " - [MULTILINE MODE] Press Ctrl+D (EOF) to return control to LLaMa.\n\n"); #endif } else { @@ -320,7 +327,7 @@ int main(int argc, char ** argv) { } // replace end of text token with newline token when in interactive mode - if (id == llama_token_eos() && params.interactive && params.input_prefix.empty()) { + if (id == llama_token_eos() && params.interactive && instruct_mode) { id = llama_token_newline.front(); if (params.antiprompt.size() != 0) { // tokenize and inject first reverse prompt @@ -377,8 +384,10 @@ int main(int argc, char ** argv) { antiprompt.is_stop_prompt = false; // Check if each of the reverse prompts appears at the end of the output. for (std::string & prompt : params.antiprompt) { - antiprompt.trailing_space = prompt.back() == ' '; - antiprompt.len = prompt.length() - (antiprompt.trailing_space ? 1 : 0); + if (params.rm_trailing_space_workaround) { + antiprompt.trailing_space = prompt.back() == ' '; + antiprompt.len = prompt.length() - (antiprompt.trailing_space ? 1 : 0); + } if (last_output.find(prompt.c_str(), last_output.length() - antiprompt.len, antiprompt.len) != std::string::npos) { is_interacting = true; antiprompt.any = true; @@ -389,8 +398,10 @@ int main(int argc, char ** argv) { } if (!antiprompt.any) { for (std::string & prompt : params.stopprompt) { - antiprompt.trailing_space = prompt.back() == ' '; - antiprompt.len = prompt.length() - (antiprompt.trailing_space ? 1 : 0); + if (params.rm_trailing_space_workaround) { + antiprompt.trailing_space = prompt.back() == ' '; + antiprompt.len = prompt.length() - (antiprompt.trailing_space ? 1 : 0); + } if (last_output.find(prompt.c_str(), last_output.length() - antiprompt.len, antiprompt.len) != std::string::npos) { is_interacting = true; antiprompt.any = true; @@ -406,21 +417,26 @@ int main(int argc, char ** argv) { if (n_past > 0 && is_interacting) { std::string buffer; - if (!params.clean_interface && !params.input_prefix.empty() && !antiprompt.any) { + if (!params.clean_interface && !params.instruct_prefix.empty() && !antiprompt.any) { // avoid printing again user's new line (TODO: try to revert enter press and print newline) - int i = params.input_prefix.front() == '\n' ? 1 : 0; + int i = params.instruct_prefix.front() == '\n' ? 1 : 0; for (; i < inp_pfx.size(); i++) { printf("%s", llama_token_to_str(ctx, inp_pfx[i])); } fflush(stdout); } - if (antiprompt.any && antiprompt.trailing_space) { - // add back removed trailing space to buffer(workaround) - buffer += ' '; - if (!params.clean_interface) { - printf("%s", buffer.c_str()); + if (params.rm_trailing_space_workaround) { + // add only if not stopprompt (as stopprompt could be used to pause + // assistant and then continue without input - adding back trailing + // space may mess it up.) + if (!antiprompt.is_stop_prompt && antiprompt.any && antiprompt.trailing_space) { + // add back removed trailing space to buffer(workaround) + buffer += ' '; + if (!params.clean_interface) { + printf("%s", buffer.c_str()); + } + fflush(stdout); } - fflush(stdout); } // potentially set color to indicate we are taking user input @@ -435,6 +451,11 @@ int main(int argc, char ** argv) { printf("\n> "); } + if (!params.input_prefix.empty()) { + buffer += params.input_prefix; + printf("%s", buffer.c_str()); + } + if (!get_input_text(buffer, !params.multiline_mode)) { // input stream is bad return 1; @@ -446,13 +467,16 @@ int main(int argc, char ** argv) { // done taking input, reset color set_console_color(con_st, CONSOLE_COLOR_DEFAULT); - if (!params.clean_interface && !params.input_suffix.empty() && !antiprompt.is_stop_prompt) { + if (!params.clean_interface && !params.instruct_suffix.empty() && !antiprompt.is_stop_prompt) { // avoid printing again user's new line (TODO: try to revert enter press and print newline) - int i = params.input_suffix.front() == '\n' ? 1 : 0; + int i = params.instruct_suffix.front() == '\n' ? 1 : 0; for (; i < inp_sfx.size(); i++) { printf("%s", llama_token_to_str(ctx, inp_sfx[i])); } - // we won't add back removed trailing space here (workaround) + // if (remove trailing space workaround) { + // We won't add back removed trailing space here, because assistant continues here, + // and it may mess up it's output (remove trailing space workaround). + // } fflush(stdout); } @@ -461,7 +485,7 @@ int main(int argc, char ** argv) { if (buffer.length() > 1) { // insert input prefix - if (!params.input_prefix.empty() && !antiprompt.any) { + if (!params.instruct_prefix.empty() && !antiprompt.any) { n_consumed = embd_inp.size(); embd_inp.insert(embd_inp.end(), inp_pfx.begin(), inp_pfx.end()); } @@ -470,7 +494,7 @@ int main(int argc, char ** argv) { embd_inp.insert(embd_inp.end(), line_inp.begin(), line_inp.end()); // insert response suffix - if (!params.input_suffix.empty() && !antiprompt.is_stop_prompt) { + if (!params.instruct_suffix.empty() && !antiprompt.is_stop_prompt) { embd_inp.insert(embd_inp.end(), inp_sfx.begin(), inp_sfx.end()); } @@ -487,7 +511,7 @@ int main(int argc, char ** argv) { // end of text token if (!embd.empty() && embd.back() == llama_token_eos()) { - if (params.interactive && !params.input_prefix.empty()) { + if (instruct_mode) { is_interacting = true; } else { fprintf(stderr, " [end of text]\n"); From d98f613cc41a196a7e287246b10ab16a5a8b4b46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Pazdiora?= Date: Sun, 9 Apr 2023 20:19:57 +0200 Subject: [PATCH 05/12] typos --- examples/common.cpp | 2 +- examples/common.h | 2 +- examples/main/main.cpp | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/common.cpp b/examples/common.cpp index 9bf5152feb9f5..ebaa97374f0b5 100644 --- a/examples/common.cpp +++ b/examples/common.cpp @@ -334,7 +334,7 @@ void gpt_print_usage(char * argv_0, const gpt_params & params) { fprintf(stderr, " run in interactive mode and poll user input upon seeing PROMPT (can be\n"); fprintf(stderr, " specified more than once for multiple prompts).\n"); fprintf(stderr, " --color colorise output to distinguish prompt and user input from generations\n"); - fprintf(stderr, " --multiline multiline mode (use Ctrl+D on Linux/Mac and Ctrl+Z on Windpws to send input)\n"); + fprintf(stderr, " --multiline multiline mode (use Ctrl+D on Linux/Mac and Ctrl+Z then Return on Windows to send input)\n"); fprintf(stderr, " -s SEED, --seed SEED RNG seed (default: -1, use random seed for <= 0)\n"); fprintf(stderr, " -t N, --threads N number of threads to use during computation (default: %d)\n", params.n_threads); fprintf(stderr, " -p PROMPT, --prompt PROMPT\n"); diff --git a/examples/common.h b/examples/common.h index 413604600f3b9..edbacb697e4b7 100644 --- a/examples/common.h +++ b/examples/common.h @@ -60,7 +60,7 @@ struct gpt_params { bool verbose_prompt = false; // print prompt tokens before generation bool clean_interface = false; // hides input prefix & suffix and displays '>' - bool multiline_mode = false; // enables multi-line mode, to send input press CTRL+D on Linux/Max, CTRL+Z on Windows + bool multiline_mode = false; // enables multi-line mode, to send input press CTRL+D on Linux/Max, Ctrl+Z then Return on Windows }; bool gpt_params_parse(int argc, char ** argv, gpt_params & params); diff --git a/examples/main/main.cpp b/examples/main/main.cpp index 47969d2182890..5da2326d49156 100644 --- a/examples/main/main.cpp +++ b/examples/main/main.cpp @@ -239,7 +239,7 @@ int main(int argc, char ** argv) { ); if (params.multiline_mode) { #if defined (_WIN32) - fprintf(stderr, " - [MULTILINE MODE] Press Ctrl+Z and Return (EOF) to return control to LLaMa.\n\n"); + fprintf(stderr, " - [MULTILINE MODE] Press Ctrl+Z then Return (EOF) to return control to LLaMa.\n\n"); #else fprintf(stderr, " - [MULTILINE MODE] Press Ctrl+D (EOF) to return control to LLaMa.\n\n"); #endif From 73966bc983024bc38256426f0610f378f5a78063 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Pazdiora?= Date: Sun, 9 Apr 2023 21:05:11 +0200 Subject: [PATCH 06/12] Change --multiline implementation to be toggled by EOF. --- examples/common.cpp | 44 ++++++++++++++++++++++++++---------------- examples/main/main.cpp | 7 ++++--- 2 files changed, 31 insertions(+), 20 deletions(-) diff --git a/examples/common.cpp b/examples/common.cpp index ebaa97374f0b5..737b15d97d29d 100644 --- a/examples/common.cpp +++ b/examples/common.cpp @@ -334,7 +334,7 @@ void gpt_print_usage(char * argv_0, const gpt_params & params) { fprintf(stderr, " run in interactive mode and poll user input upon seeing PROMPT (can be\n"); fprintf(stderr, " specified more than once for multiple prompts).\n"); fprintf(stderr, " --color colorise output to distinguish prompt and user input from generations\n"); - fprintf(stderr, " --multiline multiline mode (use Ctrl+D on Linux/Mac and Ctrl+Z then Return on Windows to send input)\n"); + fprintf(stderr, " --multiline multiline mode (use Ctrl+D on Linux/Mac and Ctrl+Z then Return on Windows to toggle multiline)\n"); fprintf(stderr, " -s SEED, --seed SEED RNG seed (default: -1, use random seed for <= 0)\n"); fprintf(stderr, " -t N, --threads N number of threads to use during computation (default: %d)\n", params.n_threads); fprintf(stderr, " -p PROMPT, --prompt PROMPT\n"); @@ -455,45 +455,55 @@ void win32_utf8_encode(const std::wstring & wstr, std::string & str) { } #endif -bool get_input_text(std::string & input_text, bool escape_newline_mode) { +bool get_input_text(std::string & input_text, bool eof_toggled_multiline_mode) { bool another_line = true; + bool is_eof_multiline_toggled = false; do { std::string line; #if defined(_WIN32) + auto & stdcin = std::wcin; std::wstring wline; - if (!std::getline(std::wcin, wline)) { + if (!std::getline(stdcin, wline)) { // input stream is bad or EOF received - if (std::wcin.bad()) { + if (stdcin.bad()) { fprintf(stderr, "%s: error: input stream bad\n", __func__); return 1; } } - if (std::wcin.eof()) { - another_line = false; - std::wcin.clear(); - std::wcin.seekg(0, std::ios::beg); - } win32_utf8_encode(wline, line); #else - if (!std::getline(std::cin, line)) { + auto & stdcin = std::cin; + if (!std::getline(stdcin, line)) { // input stream is bad or EOF received - if (std::wcin.bad()) { + if (stdcin.bad()) { fprintf(stderr, "%s: error: input stream bad\n", __func__); return 1; } } - if (std::cin.eof()) { - another_line = false; - std::cin.clear(); - std::cin.seekg(0, std::ios::beg); - } #endif - if (escape_newline_mode) { + if (stdcin.eof()) { + stdcin.clear(); + stdcin.seekg(0, std::ios::beg); + if (!eof_toggled_multiline_mode) { + another_line = false; + } else { + is_eof_multiline_toggled = !is_eof_multiline_toggled; + if (is_eof_multiline_toggled) { + input_text += line; + continue; + } + } + } + if (!eof_toggled_multiline_mode) { if (line.empty() || line.back() != '\\') { another_line = false; } else { line.pop_back(); // Remove the continue character } + } else { + if (!is_eof_multiline_toggled) { + another_line = false; + } } input_text += line; if (another_line) { diff --git a/examples/main/main.cpp b/examples/main/main.cpp index 5da2326d49156..0f8ef88cb7379 100644 --- a/examples/main/main.cpp +++ b/examples/main/main.cpp @@ -238,10 +238,11 @@ int main(int argc, char ** argv) { #endif ); if (params.multiline_mode) { + fprintf(stderr, " - Press Return to return control to LLaMa.\n" #if defined (_WIN32) - fprintf(stderr, " - [MULTILINE MODE] Press Ctrl+Z then Return (EOF) to return control to LLaMa.\n\n"); + " - [MULTILINE MODE] Press Ctrl+Z then Return (EOF) to toggle.\n\n"); #else - fprintf(stderr, " - [MULTILINE MODE] Press Ctrl+D (EOF) to return control to LLaMa.\n\n"); + " - [MULTILINE MODE] Press Ctrl+D (EOF) to toggle.\n\n"); #endif } else { @@ -456,7 +457,7 @@ int main(int argc, char ** argv) { printf("%s", buffer.c_str()); } - if (!get_input_text(buffer, !params.multiline_mode)) { + if (!get_input_text(buffer, params.multiline_mode)) { // input stream is bad return 1; } From c1be1ee073a3306780978b03194409f05c1e13e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Pazdiora?= Date: Sun, 9 Apr 2023 21:20:39 +0200 Subject: [PATCH 07/12] bugfix --- examples/main/main.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/main/main.cpp b/examples/main/main.cpp index 0f8ef88cb7379..c4090d99f29d1 100644 --- a/examples/main/main.cpp +++ b/examples/main/main.cpp @@ -85,7 +85,7 @@ int main(int argc, char ** argv) { params.prompt = gpt_random_prompt(rng); } - bool instruct_mode = params.instruct_prefix.empty() && params.instruct_suffix.empty(); + bool instruct_mode = !params.instruct_prefix.empty() || !params.instruct_suffix.empty(); // params.prompt = R"(// this function checks if the number n is prime //bool is_prime(int n) {)"; @@ -328,7 +328,7 @@ int main(int argc, char ** argv) { } // replace end of text token with newline token when in interactive mode - if (id == llama_token_eos() && params.interactive && instruct_mode) { + if (id == llama_token_eos() && params.interactive && !instruct_mode) { id = llama_token_newline.front(); if (params.antiprompt.size() != 0) { // tokenize and inject first reverse prompt From 3d2bf47f2499a1983cc438263b34ab849f4a5893 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Pazdiora?= Date: Sun, 9 Apr 2023 21:56:16 +0200 Subject: [PATCH 08/12] default multiline mode --- examples/common.cpp | 6 +++--- examples/common.h | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/common.cpp b/examples/common.cpp index 737b15d97d29d..859253b580fd6 100644 --- a/examples/common.cpp +++ b/examples/common.cpp @@ -246,8 +246,8 @@ bool gpt_params_parse(int argc, char ** argv, gpt_params & params) { params.antiprompt.push_back("### Instruction:\n\n"); } else if (arg == "--color") { params.use_color = true; - } else if (arg == "--multiline") { - params.multiline_mode = true; + } else if (arg == "--disable-multiline") { + params.multiline_mode = false; } else if (arg == "--mlock") { params.use_mlock = true; } else if (arg == "--no-mmap") { @@ -334,7 +334,7 @@ void gpt_print_usage(char * argv_0, const gpt_params & params) { fprintf(stderr, " run in interactive mode and poll user input upon seeing PROMPT (can be\n"); fprintf(stderr, " specified more than once for multiple prompts).\n"); fprintf(stderr, " --color colorise output to distinguish prompt and user input from generations\n"); - fprintf(stderr, " --multiline multiline mode (use Ctrl+D on Linux/Mac and Ctrl+Z then Return on Windows to toggle multiline)\n"); + fprintf(stderr, " --disable-multiline disable multiline mode (use Ctrl+D on Linux/Mac and Ctrl+Z then Return on Windows to toggle multiline)\n"); fprintf(stderr, " -s SEED, --seed SEED RNG seed (default: -1, use random seed for <= 0)\n"); fprintf(stderr, " -t N, --threads N number of threads to use during computation (default: %d)\n", params.n_threads); fprintf(stderr, " -p PROMPT, --prompt PROMPT\n"); diff --git a/examples/common.h b/examples/common.h index edbacb697e4b7..ae3401b8acd81 100644 --- a/examples/common.h +++ b/examples/common.h @@ -60,7 +60,7 @@ struct gpt_params { bool verbose_prompt = false; // print prompt tokens before generation bool clean_interface = false; // hides input prefix & suffix and displays '>' - bool multiline_mode = false; // enables multi-line mode, to send input press CTRL+D on Linux/Max, Ctrl+Z then Return on Windows + bool multiline_mode = true; // enables multi-line mode, to send input press CTRL+D on Linux/Max, Ctrl+Z then Return on Windows }; bool gpt_params_parse(int argc, char ** argv, gpt_params & params); From e163b7375fe14284bc139c863c3c707ec21ba779 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Pazdiora?= Date: Mon, 10 Apr 2023 00:33:35 +0200 Subject: [PATCH 09/12] add more configs --- configs/alpaca-native-enhanced.txt | 21 +++++++++++++++++++++ configs/llama.txt | 3 +++ 2 files changed, 24 insertions(+) create mode 100644 configs/alpaca-native-enhanced.txt create mode 100644 configs/llama.txt diff --git a/configs/alpaca-native-enhanced.txt b/configs/alpaca-native-enhanced.txt new file mode 100644 index 0000000000000..41e7e2c1d7dae --- /dev/null +++ b/configs/alpaca-native-enhanced.txt @@ -0,0 +1,21 @@ +--ctx_size 2048 +--batch_size 16 +--repeat_penalty 1.15 +--temp 0.4 +--top_k 30 +--top_p 0.18 + +--interactive-first +--keep -1 + +--ins-prefix-bos +--ins-prefix "\n\nUser: " +--ins-suffix "\n\nAssistant: " +--reverse-prompt "User: " + +-p "You are an AI language model designed to assist the User by answering their questions, offering advice, and engaging in casual conversation in a friendly, helpful, and informative manner. You respond clearly, coherently, and you consider the conversation history. + +User: Hey, how's it going? + +Assistant: Hey there! I'm doing great, thank you. What can I help you with today? Let's have a fun chat!" + diff --git a/configs/llama.txt b/configs/llama.txt new file mode 100644 index 0000000000000..5abebb41e0db5 --- /dev/null +++ b/configs/llama.txt @@ -0,0 +1,3 @@ +--interactive-first +--keep -1 +--temp 0.1 \ No newline at end of file From c4c7ead0bdad365c6db673cf0df95abaf37d0fe1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Pazdiora?= Date: Tue, 11 Apr 2023 23:06:38 +0200 Subject: [PATCH 10/12] update formating --- configs/alpaca-native-enhanced.txt | 14 +++++++------- configs/alpaca.txt | 6 +++--- configs/chat-with-bob.txt | 6 +++--- configs/llama.txt | 4 ++-- configs/vicuna-simple.txt | 2 +- configs/vicuna-stop.txt | 2 +- configs/vicuna.txt | 2 +- 7 files changed, 18 insertions(+), 18 deletions(-) diff --git a/configs/alpaca-native-enhanced.txt b/configs/alpaca-native-enhanced.txt index 41e7e2c1d7dae..109d315924d85 100644 --- a/configs/alpaca-native-enhanced.txt +++ b/configs/alpaca-native-enhanced.txt @@ -1,14 +1,14 @@ ---ctx_size 2048 ---batch_size 16 +--ctx_size 2048 +--batch_size 16 --repeat_penalty 1.15 ---temp 0.4 ---top_k 30 ---top_p 0.18 +--temp 0.4 +--top_k 30 +--top_p 0.18 ---interactive-first +--interactive-first --keep -1 ---ins-prefix-bos +--ins-prefix-bos --ins-prefix "\n\nUser: " --ins-suffix "\n\nAssistant: " --reverse-prompt "User: " diff --git a/configs/alpaca.txt b/configs/alpaca.txt index 5e1558d18bd1d..99a3ab47efc45 100644 --- a/configs/alpaca.txt +++ b/configs/alpaca.txt @@ -1,7 +1,7 @@ ---clean-interface ---interactive-first +--clean-interface +--interactive-first --keep -1 ---ins-prefix-bos +--ins-prefix-bos --ins-prefix "\n\n### Instruction:\n\n" --ins-suffix "\n\n### Response:\n\n" --reverse-prompt "### Instruction:\n\n" diff --git a/configs/chat-with-bob.txt b/configs/chat-with-bob.txt index ede63d35aa078..198c69e2b7eae 100644 --- a/configs/chat-with-bob.txt +++ b/configs/chat-with-bob.txt @@ -1,7 +1,7 @@ ---interactive-first +--interactive-first --keep -1 ---ins-prefix-bos ---ins-prefix "\nUser: " +--ins-prefix-bos +--ins-prefix "\nUser: " --ins-suffix "\nBob: " --reverse-prompt "User: " --rm-trailing-space-workaround diff --git a/configs/llama.txt b/configs/llama.txt index 5abebb41e0db5..9d23e75aceb50 100644 --- a/configs/llama.txt +++ b/configs/llama.txt @@ -1,3 +1,3 @@ ---interactive-first +--interactive-first --keep -1 ---temp 0.1 \ No newline at end of file +--temp 0.1 diff --git a/configs/vicuna-simple.txt b/configs/vicuna-simple.txt index a87c70082583a..efa60d96a1fbf 100644 --- a/configs/vicuna-simple.txt +++ b/configs/vicuna-simple.txt @@ -1,4 +1,4 @@ ---interactive-first +--interactive-first --keep -1 --ins-prefix-bos --ins-prefix "\n### Human: " diff --git a/configs/vicuna-stop.txt b/configs/vicuna-stop.txt index a8d8907244256..911d067efd5b8 100644 --- a/configs/vicuna-stop.txt +++ b/configs/vicuna-stop.txt @@ -1,4 +1,4 @@ ---interactive-first +--interactive-first --keep -1 --ins-prefix-bos --ins-prefix "\n### Human: " diff --git a/configs/vicuna.txt b/configs/vicuna.txt index f4d0d2d742a6c..6d811410ab3b6 100644 --- a/configs/vicuna.txt +++ b/configs/vicuna.txt @@ -1,4 +1,4 @@ ---interactive-first +--interactive-first --keep -1 --ins-prefix-bos --ins-prefix "\n### Human: " From ab73745993d05adb229988015812a8ef509771d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Pazdiora?= Date: Tue, 11 Apr 2023 23:08:25 +0200 Subject: [PATCH 11/12] update formatting --- examples/common.cpp | 2 +- examples/common.h | 2 +- examples/main/main.cpp | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/examples/common.cpp b/examples/common.cpp index cb7b583d5d997..57b4854e11415 100644 --- a/examples/common.cpp +++ b/examples/common.cpp @@ -32,7 +32,7 @@ extern "C" __declspec(dllimport) int __stdcall WideCharToMultiByte(unsigned int #define CP_UTF8 65001 #endif -void split_args(std::string args_string, std::vector & output_args) +void split_args(std::string args_string, std::vector & output_args) { std::string current_arg = ""; bool in_quotes = false; diff --git a/examples/common.h b/examples/common.h index ae3401b8acd81..df8e4c6ccb990 100644 --- a/examples/common.h +++ b/examples/common.h @@ -40,7 +40,7 @@ struct gpt_params { std::vector antiprompt; // string upon seeing which more user input is prompted std::vector stopprompt; // string upon seeing which more user input is prompted (without adding instruct prefixes and suffixes) - + bool rm_trailing_space_workaround = false; // workaround for removing trailing space from reverse/stop prompts bool memory_f16 = true; // use f16 instead of f32 for memory kv diff --git a/examples/main/main.cpp b/examples/main/main.cpp index 9252ef3bb0efc..68b4b2840858e 100644 --- a/examples/main/main.cpp +++ b/examples/main/main.cpp @@ -420,7 +420,7 @@ int main(int argc, char ** argv) { } } - if (n_past > 0 && is_interacting) + if (n_past > 0 && is_interacting) { std::string buffer; if (!params.clean_interface && !params.instruct_prefix.empty() && !antiprompt.any) { @@ -430,10 +430,10 @@ int main(int argc, char ** argv) { printf("%s", llama_token_to_str(ctx, inp_pfx[i])); } fflush(stdout); - } + } if (params.rm_trailing_space_workaround) { // add only if not stopprompt (as stopprompt could be used to pause - // assistant and then continue without input - adding back trailing + // assistant and then continue without input - adding back trailing // space may mess it up.) if (!antiprompt.is_stop_prompt && antiprompt.any && antiprompt.trailing_space) { // add back removed trailing space to buffer(workaround) From c59009a835b14cf21c655b91f2735f7627eb44aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Pazdiora?= Date: Tue, 11 Apr 2023 23:19:27 +0200 Subject: [PATCH 12/12] apply suggestions --- examples/common.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/common.cpp b/examples/common.cpp index 57b4854e11415..b7a378a721cf8 100644 --- a/examples/common.cpp +++ b/examples/common.cpp @@ -32,7 +32,7 @@ extern "C" __declspec(dllimport) int __stdcall WideCharToMultiByte(unsigned int #define CP_UTF8 65001 #endif -void split_args(std::string args_string, std::vector & output_args) +void split_args(const std::string & args_string, std::vector & output_args) { std::string current_arg = ""; bool in_quotes = false; @@ -65,7 +65,7 @@ void split_args(std::string args_string, std::vector & output_args) } } -std::string unescape(std::string str) { +std::string unescape(const std::string & str) { return std::regex_replace(str, std::regex("\\\\n"), "\n"); }