From 80181c0712e85bd5edd045fcfc80a4a5a8f71f8a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Pazdiora?= <tomas.pazdiora@gmail.com>
Date: Sun, 9 Apr 2023 06:11:29 +0200
Subject: [PATCH 01/12] Add support for configs, add configurable prefixes /
 suffixes, deprecate instruct mode, add stop prompt

---
 configs/alpaca.txt        |   5 +
 configs/chat-with-bob.txt |  10 ++
 configs/vicuna-simple.txt |   4 +
 configs/vicuna-stop.txt   |   4 +
 configs/vicuna.txt        |   5 +
 examples/common.cpp       | 191 +++++++++++++++++++++++++++++---------
 examples/common.h         |  25 +++--
 examples/main/main.cpp    | 120 +++++++++++++++++-------
 prompts/alpaca.txt        |   1 -
 prompts/chat-with-bob.txt |   7 --
 10 files changed, 279 insertions(+), 93 deletions(-)
 create mode 100644 configs/alpaca.txt
 create mode 100644 configs/chat-with-bob.txt
 create mode 100644 configs/vicuna-simple.txt
 create mode 100644 configs/vicuna-stop.txt
 create mode 100644 configs/vicuna.txt
 delete mode 100644 prompts/alpaca.txt
 delete mode 100644 prompts/chat-with-bob.txt

diff --git a/configs/alpaca.txt b/configs/alpaca.txt
new file mode 100644
index 0000000000000..06158cdb97b55
--- /dev/null
+++ b/configs/alpaca.txt
@@ -0,0 +1,5 @@
+--clean-interface --interactive-first --keep -1
+--in-prefix-bos --in-prefix "\n\n### Instruction:\n\n" -r "### Instruction:\n\n"
+--in-suffix "\n\n### Response:\n\n"
+
+-p "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n"
diff --git a/configs/chat-with-bob.txt b/configs/chat-with-bob.txt
new file mode 100644
index 0000000000000..9019753d084d8
--- /dev/null
+++ b/configs/chat-with-bob.txt
@@ -0,0 +1,10 @@
+--interactive-first --keep -1
+--in-prefix-bos 
+--in-prefix "\nUser: " -r "User: "
+--in-suffix "\nBob: " 
+
+-p "Transcript of a dialog, where the User interacts with an Assistant named Bob. Bob is helpful, kind, honest, good at writing, and never fails to answer the User's requests immediately and with precision.
+User: Hello, Bob.
+Bob: Hello. How may I help you today?
+User: Please tell me the largest city in Europe.
+Bob: Sure. The largest city in Europe is Moscow, the capital of Russia."
diff --git a/configs/vicuna-simple.txt b/configs/vicuna-simple.txt
new file mode 100644
index 0000000000000..053391479e0d1
--- /dev/null
+++ b/configs/vicuna-simple.txt
@@ -0,0 +1,4 @@
+--interactive-first --keep -1
+--in-prefix-bos 
+--in-prefix "\n### Human: " --reverse-prompt "### Human: "
+--in-suffix "\n### Assistant: " 
\ No newline at end of file
diff --git a/configs/vicuna-stop.txt b/configs/vicuna-stop.txt
new file mode 100644
index 0000000000000..e88a9fd053b79
--- /dev/null
+++ b/configs/vicuna-stop.txt
@@ -0,0 +1,4 @@
+--interactive-first --keep -1
+--in-prefix-bos 
+--in-prefix "\n### Human: " --reverse-prompt "### Human: "
+--in-suffix "\n### Assistant: " --stop-prompt "### Assistant: "
\ No newline at end of file
diff --git a/configs/vicuna.txt b/configs/vicuna.txt
new file mode 100644
index 0000000000000..bf3b966c7fe64
--- /dev/null
+++ b/configs/vicuna.txt
@@ -0,0 +1,5 @@
+--interactive-first --keep -1
+--in-prefix-bos --in-prefix "\n### Human: " -r "### Human: "
+--in-suffix "\n### Assistant: "
+
+-p "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions."
diff --git a/examples/common.cpp b/examples/common.cpp
index f909eed24b325..f149206cdc64b 100644
--- a/examples/common.cpp
+++ b/examples/common.cpp
@@ -3,9 +3,11 @@
 #include <cassert>
 #include <cstring>
 #include <fstream>
+#include <sstream>
 #include <string>
 #include <iterator>
 #include <algorithm>
+#include <regex>
 
 #if defined(_MSC_VER) || defined(__MINGW32__)
 #include <malloc.h> // using malloc.h with MSC/MINGW
@@ -29,6 +31,43 @@ extern "C" __declspec(dllimport) int __stdcall WideCharToMultiByte(unsigned int
 #define CP_UTF8 65001
 #endif
 
+void split_args(std::string args_string, std::vector<std::string> & output_args) 
+{
+    std::string current_arg = "";
+    bool in_quotes = false;
+    char quote_type;
+
+    for (char c : args_string) {
+        if (c == '"' || c == '\'') {
+            if (!in_quotes) {
+                in_quotes = true;
+                quote_type = c;
+            } else if (quote_type == c) {
+                in_quotes = false;
+            } else {
+                current_arg += c;
+            }
+        } else if (in_quotes) {
+            current_arg += c;
+        } else if (std::isspace(c)) {
+            if (current_arg != "") {
+                output_args.push_back(current_arg);
+                current_arg = "";
+            }
+        } else {
+            current_arg += c;
+        }
+    }
+
+    if (current_arg != "") {
+        output_args.push_back(current_arg);
+    }
+}
+
+std::string unescape(std::string str) {
+    return std::regex_replace(str, std::regex("\\\\n"), "\n");
+}
+
 bool gpt_params_parse(int argc, char ** argv, gpt_params & params) {
     // determine sensible default number of threads.
     // std::thread::hardware_concurrency may not be equal to the number of cores, or may return 0.
@@ -46,35 +85,66 @@ bool gpt_params_parse(int argc, char ** argv, gpt_params & params) {
     std::string arg;
     gpt_params default_params;
 
+    // get additional arguments from config files
+    std::vector<std::string> args;
     for (int i = 1; i < argc; i++) {
         arg = argv[i];
+        if (arg == "--config") {
+            if (++i >= argc) {
+                invalid_param = true;
+                break;
+            }
+            std::ifstream file(argv[i]);
+            if (!file) {
+                fprintf(stderr, "error: failed to open file '%s'\n", argv[i]);
+                invalid_param = true;
+                break;
+            }
+            std::string args_string;
+            std::copy(std::istreambuf_iterator<char>(file), std::istreambuf_iterator<char>(), back_inserter(args_string));
+            if (args_string.back() == '\n') {
+                args_string.pop_back();
+            }
+            split_args(args_string, args);
+            for (int j = 0; j < args.size(); j++) {
+                args[j] = unescape(args[j]);
+            }
+        } else {
+            args.emplace_back(argv[i]);
+        }
+    }
+
+    // parse args
+    int args_c = static_cast<int>(args.size());
+    for (int i = 0; i < args_c && !invalid_param; i++) {
+        arg = args[i];
 
         if (arg == "-s" || arg == "--seed") {
-            if (++i >= argc) {
+            if (++i >= args_c) {
                 invalid_param = true;
                 break;
             }
-            params.seed = std::stoi(argv[i]);
+            params.seed = std::stoi(args[i]);
         } else if (arg == "-t" || arg == "--threads") {
-            if (++i >= argc) {
+            if (++i >= args_c) {
                 invalid_param = true;
                 break;
             }
-            params.n_threads = std::stoi(argv[i]);
+            params.n_threads = std::stoi(args[i]);
         } else if (arg == "-p" || arg == "--prompt") {
-            if (++i >= argc) {
+            if (++i >= args_c) {
                 invalid_param = true;
                 break;
             }
-            params.prompt = argv[i];
+            params.prompt = args[i];
         } else if (arg == "-f" || arg == "--file") {
-            if (++i >= argc) {
+            if (++i >= args_c) {
                 invalid_param = true;
                 break;
             }
-            std::ifstream file(argv[i]);
+            std::ifstream file(args[i]);
             if (!file) {
-                fprintf(stderr, "error: failed to open file '%s'\n", argv[i]);
+                fprintf(stderr, "error: failed to open file '%s'\n", args[i].c_str());
                 invalid_param = true;
                 break;
             }
@@ -83,78 +153,96 @@ bool gpt_params_parse(int argc, char ** argv, gpt_params & params) {
                 params.prompt.pop_back();
             }
         } else if (arg == "-n" || arg == "--n_predict") {
-            if (++i >= argc) {
+            if (++i >= args_c) {
                 invalid_param = true;
                 break;
             }
-            params.n_predict = std::stoi(argv[i]);
+            params.n_predict = std::stoi(args[i]);
         } else if (arg == "--top_k") {
-            if (++i >= argc) {
+            if (++i >= args_c) {
                 invalid_param = true;
                 break;
             }
-            params.top_k = std::stoi(argv[i]);
+            params.top_k = std::stoi(args[i]);
         } else if (arg == "-c" || arg == "--ctx_size") {
-            if (++i >= argc) {
+            if (++i >= args_c) {
                 invalid_param = true;
                 break;
             }
-            params.n_ctx = std::stoi(argv[i]);
+            params.n_ctx = std::stoi(args[i]);
         } else if (arg == "--memory_f32") {
             params.memory_f16 = false;
         } else if (arg == "--top_p") {
-            if (++i >= argc) {
+            if (++i >= args_c) {
                 invalid_param = true;
                 break;
             }
-            params.top_p = std::stof(argv[i]);
+            params.top_p = std::stof(args[i]);
         } else if (arg == "--temp") {
-            if (++i >= argc) {
+            if (++i >= args_c) {
                 invalid_param = true;
                 break;
             }
-            params.temp = std::stof(argv[i]);
+            params.temp = std::stof(args[i]);
         } else if (arg == "--repeat_last_n") {
-            if (++i >= argc) {
+            if (++i >= args_c) {
                 invalid_param = true;
                 break;
             }
-            params.repeat_last_n = std::stoi(argv[i]);
+            params.repeat_last_n = std::stoi(args[i]);
         } else if (arg == "--repeat_penalty") {
-            if (++i >= argc) {
+            if (++i >= args_c) {
                 invalid_param = true;
                 break;
             }
-            params.repeat_penalty = std::stof(argv[i]);
+            params.repeat_penalty = std::stof(args[i]);
         } else if (arg == "-b" || arg == "--batch_size") {
-            if (++i >= argc) {
+            if (++i >= args_c) {
                 invalid_param = true;
                 break;
             }
-            params.n_batch = std::stoi(argv[i]);
+            params.n_batch = std::stoi(args[i]);
             params.n_batch = std::min(512, params.n_batch);
         } else if (arg == "--keep") {
-            if (++i >= argc) {
+            if (++i >= args_c) {
                 invalid_param = true;
                 break;
             }
-            params.n_keep = std::stoi(argv[i]);
+            params.n_keep = std::stoi(args[i]);
         } else if (arg == "-m" || arg == "--model") {
-            if (++i >= argc) {
+            if (++i >= args_c) {
                 invalid_param = true;
                 break;
             }
-            params.model = argv[i];
+            params.model = args[i];
         } else if (arg == "-i" || arg == "--interactive") {
             params.interactive = true;
         } else if (arg == "--embedding") {
             params.embedding = true;
+        } else if (arg == "--clean-interface") {
+            params.clean_interface = true;
         } else if (arg == "--interactive-start") {
             params.interactive = true;
         } else if (arg == "--interactive-first") {
             params.interactive_start = true;
         } else if (arg == "-ins" || arg == "--instruct") {
-            params.instruct = true;
+            fprintf(stderr, "\n\nWarning: instruct mode is deprecated! Use: \n"
+                "--clean-interface "
+                "--interactive-first "
+                "--keep -1 "
+                "--in-prefix-bos "
+                "--in-prefix \"\\n\\n### Instruction:\\n\\n\" "
+                "--in-suffix \"\\n\\n### Response:\\n\\n\" "
+                "-r \"### Instruction:\\n\\n\" "
+            "\n\n");
+            // params.instruct = true;
+            params.clean_interface = true;
+            params.interactive_start = true;
+            params.n_keep = -1;
+            params.input_prefix_bos = true;
+            params.input_prefix = "\n\n### Instruction:\n\n";
+            params.input_suffix = "\n\n### Response:\n\n";
+            params.antiprompt.push_back("### Instruction:\n\n");
         } else if (arg == "--color") {
             params.use_color = true;
         } else if (arg == "--mlock") {
@@ -166,55 +254,71 @@ bool gpt_params_parse(int argc, char ** argv, gpt_params & params) {
         } else if (arg == "--verbose-prompt") {
             params.verbose_prompt = true;
         } else if (arg == "-r" || arg == "--reverse-prompt") {
-            if (++i >= argc) {
+            if (++i >= args_c) {
                 invalid_param = true;
                 break;
             }
-            params.antiprompt.push_back(argv[i]);
+            params.antiprompt.push_back(args[i]);
+        } else if (arg == "--stop-prompt") {
+            if (++i >= args_c) {
+                invalid_param = true;
+                break;
+            }
+            params.stopprompt.push_back(args[i]);
         } else if (arg == "--perplexity") {
             params.perplexity = true;
         } else if (arg == "--ignore-eos") {
             params.ignore_eos = true;
         } else if (arg == "--n_parts") {
-            if (++i >= argc) {
+            if (++i >= args_c) {
                 invalid_param = true;
                 break;
             }
-            params.n_parts = std::stoi(argv[i]);
+            params.n_parts = std::stoi(args[i]);
         } else if (arg == "-h" || arg == "--help") {
-            gpt_print_usage(argc, argv, default_params);
+            gpt_print_usage(argv[0], default_params);
             exit(0);
         } else if (arg == "--random-prompt") {
             params.random_prompt = true;
+        } else if (arg == "--in-prefix-bos") {
+            params.input_prefix_bos = true;
         } else if (arg == "--in-prefix") {
-            if (++i >= argc) {
+            if (++i >= args_c) {
+                invalid_param = true;
+                break;
+            }
+            params.input_prefix = args[i];
+        } else if (arg == "--in-suffix-bos") {
+            params.input_suffix_bos = true;
+        } else if (arg == "--in-suffix") {
+            if (++i >= args_c) {
                 invalid_param = true;
                 break;
             }
-            params.input_prefix = argv[i];
+            params.input_suffix = args[i];
         } else {
             fprintf(stderr, "error: unknown argument: %s\n", arg.c_str());
-            gpt_print_usage(argc, argv, default_params);
+            gpt_print_usage(argv[0], default_params);
             exit(1);
         }
     }
     if (invalid_param) {
         fprintf(stderr, "error: invalid parameter for argument: %s\n", arg.c_str());
-        gpt_print_usage(argc, argv, default_params);
+        gpt_print_usage(argv[0], default_params);
         exit(1);
     }
 
     return true;
 }
 
-void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) {
-    fprintf(stderr, "usage: %s [options]\n", argv[0]);
+void gpt_print_usage(char * argv_0, const gpt_params & params) {
+    fprintf(stderr, "usage: %s [options]\n", argv_0);
     fprintf(stderr, "\n");
     fprintf(stderr, "options:\n");
     fprintf(stderr, "  -h, --help            show this help message and exit\n");
     fprintf(stderr, "  -i, --interactive     run in interactive mode\n");
     fprintf(stderr, "  --interactive-first   run in interactive mode and wait for input right away\n");
-    fprintf(stderr, "  -ins, --instruct      run in instruction mode (use with Alpaca models)\n");
+    fprintf(stderr, "  --clean-interface     hides input prefix & suffix and displays '>' instead\n");
     fprintf(stderr, "  -r PROMPT, --reverse-prompt PROMPT\n");
     fprintf(stderr, "                        run in interactive mode and poll user input upon seeing PROMPT (can be\n");
     fprintf(stderr, "                        specified more than once for multiple prompts).\n");
@@ -224,7 +328,10 @@ void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) {
     fprintf(stderr, "  -p PROMPT, --prompt PROMPT\n");
     fprintf(stderr, "                        prompt to start generation with (default: empty)\n");
     fprintf(stderr, "  --random-prompt       start with a randomized prompt.\n");
+    fprintf(stderr, "  --in-prefix-bos       append bos token to input prefix.\n");
     fprintf(stderr, "  --in-prefix STRING    string to prefix user inputs with (default: empty)\n");
+    fprintf(stderr, "  --in-suffix-bos       append bos token to input suffix.\n");
+    fprintf(stderr, "  --in-suffix STRING    string to suffix user inputs with (default: empty)\n");
     fprintf(stderr, "  -f FNAME, --file FNAME\n");
     fprintf(stderr, "                        prompt file to start generation.\n");
     fprintf(stderr, "  -n N, --n_predict N   number of tokens to predict (default: %d, -1 = infinity)\n", params.n_predict);
diff --git a/examples/common.h b/examples/common.h
index 1ea6f74451811..3a7f103e5060c 100644
--- a/examples/common.h
+++ b/examples/common.h
@@ -14,14 +14,14 @@
 //
 
 struct gpt_params {
-    int32_t seed          = -1;   // RNG seed
-    int32_t n_threads     = std::min(4, (int32_t) std::thread::hardware_concurrency());
-    int32_t n_predict     = 128;  // new tokens to predict
-    int32_t repeat_last_n = 64;   // last n tokens to penalize
-    int32_t n_parts       = -1;   // amount of model parts (-1 = determine from model dimensions)
-    int32_t n_ctx         = 512;  // context size
-    int32_t n_batch       = 8;    // batch size for prompt processing
-    int32_t n_keep        = 0;    // number of tokens to keep from initial prompt
+    int32_t seed          = -1;    // RNG seed
+    int32_t n_threads     = std::min(4, (int32_t) std::thread::hardware_concurrency()); // max 4 threads (default)
+    int32_t n_predict     = 128;   // new tokens to predict
+    int32_t repeat_last_n = 64;    // last n tokens to penalize
+    int32_t n_parts       = -1;    // amount of model parts (-1 = determine from model dimensions)
+    int32_t n_ctx         = 512;   // context size
+    int32_t n_batch       = 8;     // batch size for prompt processing
+    int32_t n_keep        = 0;     // number of tokens to keep from initial prompt (-1 for all)
 
     // sampling parameters
     int32_t top_k = 40;
@@ -32,9 +32,12 @@ struct gpt_params {
     std::string model  = "models/lamma-7B/ggml-model.bin"; // model path
     std::string prompt = "";
     std::string input_prefix = ""; // string to prefix user inputs with
-
+    bool input_prefix_bos = false; // append bos token to input prefix
+    std::string input_suffix = ""; // string to suffix user inputs with
+    bool input_suffix_bos = false; // append bos token to input suffix
 
     std::vector<std::string> antiprompt; // string upon seeing which more user input is prompted
+    std::vector<std::string> stopprompt; // string upon seeing which more user input is prompted (without adding prefixes and suffixes)
 
     bool memory_f16        = true;  // use f16 instead of f32 for memory kv
     bool random_prompt     = false; // do not randomize prompt if none provided
@@ -51,11 +54,13 @@ struct gpt_params {
     bool use_mlock         = false; // use mlock to keep model in memory
     bool mem_test          = false; // compute maximum memory usage
     bool verbose_prompt    = false; // print prompt tokens before generation
+
+    bool clean_interface   = false; // hides input prefix & suffix and displays '>'
 };
 
 bool gpt_params_parse(int argc, char ** argv, gpt_params & params);
 
-void gpt_print_usage(int argc, char ** argv, const gpt_params & params);
+void gpt_print_usage(char * argv_0, const gpt_params & params);
 
 std::string gpt_random_prompt(std::mt19937 & rng);
 
diff --git a/examples/main/main.cpp b/examples/main/main.cpp
index d333d0dbacd20..744c968392d80 100644
--- a/examples/main/main.cpp
+++ b/examples/main/main.cpp
@@ -148,22 +148,18 @@ int main(int argc, char ** argv) {
     }
 
     // number of tokens to keep when resetting context
-    if (params.n_keep < 0 || params.n_keep > (int)embd_inp.size() || params.instruct) {
+    if (params.n_keep < 0 || params.n_keep > (int)embd_inp.size()) {
         params.n_keep = (int)embd_inp.size();
     }
 
     // prefix & suffix for instruct mode
-    const auto inp_pfx = ::llama_tokenize(ctx, "\n\n### Instruction:\n\n", true);
-    const auto inp_sfx = ::llama_tokenize(ctx, "\n\n### Response:\n\n", false);
-
-    // in instruct mode, we inject a prefix and a suffix to each input by the user
-    if (params.instruct) {
-        params.interactive_start = true;
-        params.antiprompt.push_back("### Instruction:\n\n");
-    }
+    const auto inp_pfx = ::llama_tokenize(ctx, params.input_prefix, params.input_prefix_bos);
+    std::string input_suffix = params.input_suffix;
+    if (input_suffix.back() == ' ') { input_suffix.pop_back(); } // (remove trailing space workaround)
+    const auto inp_sfx = ::llama_tokenize(ctx, input_suffix, params.input_suffix_bos);
 
     // enable interactive mode if reverse prompt or interactive start is specified
-    if (params.antiprompt.size() != 0 || params.interactive_start) { 
+    if (params.antiprompt.size() != 0 || params.stopprompt.size() != 0 || params.interactive_start) { 
         params.interactive = true;
     }
 
@@ -205,9 +201,17 @@ int main(int argc, char ** argv) {
                 fprintf(stderr, "Reverse prompt: '%s'\n", antiprompt.c_str());
             }
         }
+        if (params.stopprompt.size()) {
+            for (auto stopprompt : params.stopprompt) {
+                fprintf(stderr, "Stop prompt: '%s'\n", stopprompt.c_str());
+            }
+        }
 
         if (!params.input_prefix.empty()) {
-            fprintf(stderr, "Input prefix: '%s'\n", params.input_prefix.c_str());
+            fprintf(stderr, "Input prefix %s: '%s'\n", params.input_prefix_bos ? "(with bos token)" : "", params.input_prefix.c_str());
+        }
+        if (!params.input_suffix.empty()) {
+            fprintf(stderr, "Input suffix %s: '%s'\n", params.input_suffix_bos ? "(with bos token)" : "", params.input_suffix.c_str());
         }
     }
     fprintf(stderr, "sampling: temp = %f, top_k = %d, top_p = %f, repeat_last_n = %i, repeat_penalty = %f\n",
@@ -229,7 +233,13 @@ int main(int argc, char ** argv) {
         is_interacting = params.interactive_start;
     }
 
-    bool is_antiprompt = false;
+    struct Antiprompt {
+        bool any = false;
+        bool trailing_space = false;
+        size_t len;
+        bool is_stop_prompt = false;
+    } antiprompt;
+
     bool input_noecho  = false;
 
     int n_past     = 0;
@@ -299,7 +309,7 @@ int main(int argc, char ** argv) {
             }
 
             // replace end of text token with newline token when in interactive mode
-            if (id == llama_token_eos() && params.interactive && !params.instruct) {
+            if (id == llama_token_eos() && params.interactive && params.input_prefix.empty()) {
                 id = llama_token_newline.front();
                 if (params.antiprompt.size() != 0) {
                     // tokenize and inject first reverse prompt
@@ -345,27 +355,63 @@ int main(int argc, char ** argv) {
         // check if we should prompt the user for more
         if (params.interactive && (int) embd_inp.size() <= n_consumed) {
 
-            // check for reverse prompt
-            if (params.antiprompt.size()) {
+            // check for reverse prompt or stop prompt
+            if (params.antiprompt.size() || params.stopprompt.size()) {
                 std::string last_output;
                 for (auto id : last_n_tokens) {
                     last_output += llama_token_to_str(ctx, id);
                 }
 
-                is_antiprompt = false;
+                antiprompt.any = false;
+                antiprompt.is_stop_prompt = false;
                 // Check if each of the reverse prompts appears at the end of the output.
-                for (std::string & antiprompt : params.antiprompt) {
-                    if (last_output.find(antiprompt.c_str(), last_output.length() - antiprompt.length(), antiprompt.length()) != std::string::npos) {
+                for (std::string & prompt : params.antiprompt) {
+                    antiprompt.trailing_space = prompt.back() == ' ';
+                    antiprompt.len = prompt.length() - (antiprompt.trailing_space ? 1 : 0);
+                    if (last_output.find(prompt.c_str(), last_output.length() - antiprompt.len, antiprompt.len) != std::string::npos) {
                         is_interacting = true;
-                        is_antiprompt = true;
+                        antiprompt.any = true;
                         set_console_color(con_st, CONSOLE_COLOR_USER_INPUT);
                         fflush(stdout);
                         break;
                     }
                 }
+                if (!antiprompt.any) {
+                    for (std::string & prompt : params.stopprompt) {
+                        antiprompt.trailing_space = prompt.back() == ' ';
+                        antiprompt.len = prompt.length() - (antiprompt.trailing_space ? 1 : 0);
+                        if (last_output.find(prompt.c_str(), last_output.length() - antiprompt.len, antiprompt.len) != std::string::npos) {
+                            is_interacting = true;
+                            antiprompt.any = true;
+                            antiprompt.is_stop_prompt = true;
+                            set_console_color(con_st, CONSOLE_COLOR_USER_INPUT);
+                            fflush(stdout);
+                            break;
+                        }
+                    }
+                }
             }
 
-            if (n_past > 0 && is_interacting) {
+            if (n_past > 0 && is_interacting) 
+            {
+                std::string buffer;
+                if (!params.clean_interface && !params.input_prefix.empty() && !antiprompt.any) {
+                    // avoid printing again user's new line (TODO: try to revert enter press and print newline)
+                    int i = params.input_prefix.front() == '\n' ? 1 : 0;
+                    for (; i < inp_pfx.size(); i++) {
+                        printf("%s", llama_token_to_str(ctx, inp_pfx[i]));
+                    }
+                    fflush(stdout);
+                } 
+                if (antiprompt.any && antiprompt.trailing_space) {
+                    // add back removed trailing space to buffer(workaround)
+                    buffer += ' ';
+                    if (!params.clean_interface) {
+                        printf("%s", buffer.c_str());
+                    }
+                    fflush(stdout);
+                }
+
                 // potentially set color to indicate we are taking user input
                 set_console_color(con_st, CONSOLE_COLOR_USER_INPUT);
 
@@ -374,19 +420,14 @@ int main(int argc, char ** argv) {
                 signal(SIGINT, sigint_handler);
 #endif
 
-                if (params.instruct) {
+                if (params.clean_interface) {
                     printf("\n> ");
                 }
 
-                std::string buffer;
-                if (!params.input_prefix.empty()) {
-                    buffer += params.input_prefix;
-                    printf("%s", buffer.c_str());
-                }
-
                 std::string line;
                 bool another_line = true;
                 do {
+                    // TODO: try to revert going to new line after enter (to enable in-line text writing)
 #if defined(_WIN32)
                     std::wstring wline;
                     if (!std::getline(std::wcin, wline)) {
@@ -405,18 +446,31 @@ int main(int argc, char ** argv) {
                     } else {
                         line.pop_back(); // Remove the continue character
                     }
-                    buffer += line + '\n'; // Append the line to the result
+                    buffer += line;
+                    if (another_line || !antiprompt.is_stop_prompt) {
+                        buffer += '\n'; // Append the line to the result
+                    }
                 } while (another_line);
 
                 // done taking input, reset color
                 set_console_color(con_st, CONSOLE_COLOR_DEFAULT);
 
+                if (!params.clean_interface && !params.input_suffix.empty() && !antiprompt.is_stop_prompt) {
+                    // avoid printing again user's new line (TODO: try to revert enter press and print newline)
+                    int i = params.input_suffix.front() == '\n' ? 1 : 0;
+                    for (; i < inp_sfx.size(); i++) {
+                        printf("%s", llama_token_to_str(ctx, inp_sfx[i]));
+                    }
+                    // we won't add back removed trailing space here (workaround)
+                    fflush(stdout);
+                }
+
                 // Add tokens to embd only if the input buffer is non-empty
                 // Entering a empty line lets the user pass control back
                 if (buffer.length() > 1) {
 
-                    // instruct mode: insert instruction prefix
-                    if (params.instruct && !is_antiprompt) {
+                    // insert input prefix
+                    if (!params.input_prefix.empty() && !antiprompt.any) {
                         n_consumed = embd_inp.size();
                         embd_inp.insert(embd_inp.end(), inp_pfx.begin(), inp_pfx.end());
                     }
@@ -424,8 +478,8 @@ int main(int argc, char ** argv) {
                     auto line_inp = ::llama_tokenize(ctx, buffer, false);
                     embd_inp.insert(embd_inp.end(), line_inp.begin(), line_inp.end());
 
-                    // instruct mode: insert response suffix
-                    if (params.instruct) {
+                    // insert response suffix
+                    if (!params.input_suffix.empty() && !antiprompt.is_stop_prompt) {
                         embd_inp.insert(embd_inp.end(), inp_sfx.begin(), inp_sfx.end());
                     }
 
@@ -442,7 +496,7 @@ int main(int argc, char ** argv) {
 
         // end of text token
         if (!embd.empty() && embd.back() == llama_token_eos()) {
-            if (params.instruct) {
+            if (params.interactive && !params.input_prefix.empty()) {
                 is_interacting = true;
             } else {
                 fprintf(stderr, " [end of text]\n");
diff --git a/prompts/alpaca.txt b/prompts/alpaca.txt
deleted file mode 100644
index 2224bdeb0bcd4..0000000000000
--- a/prompts/alpaca.txt
+++ /dev/null
@@ -1 +0,0 @@
-Below is an instruction that describes a task. Write a response that appropriately completes the request.
diff --git a/prompts/chat-with-bob.txt b/prompts/chat-with-bob.txt
deleted file mode 100644
index 009da39aee066..0000000000000
--- a/prompts/chat-with-bob.txt
+++ /dev/null
@@ -1,7 +0,0 @@
-Transcript of a dialog, where the User interacts with an Assistant named Bob. Bob is helpful, kind, honest, good at writing, and never fails to answer the User's requests immediately and with precision.
-
-User: Hello, Bob.
-Bob: Hello. How may I help you today?
-User: Please tell me the largest city in Europe.
-Bob: Sure. The largest city in Europe is Moscow, the capital of Russia.
-User:

From ddf6bd6497fae1cbdea6aad3d6dcdc7292a52815 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Pazdiora?= <tomas.pazdiora@gmail.com>
Date: Sun, 9 Apr 2023 18:36:23 +0200
Subject: [PATCH 02/12] Add multiline mode, update text input.

---
 examples/common.cpp    | 52 ++++++++++++++++++++++++++++++++++++++++++
 examples/common.h      |  3 +++
 examples/main/main.cpp | 51 +++++++++++++++++------------------------
 3 files changed, 76 insertions(+), 30 deletions(-)

diff --git a/examples/common.cpp b/examples/common.cpp
index f149206cdc64b..212974481ad9c 100644
--- a/examples/common.cpp
+++ b/examples/common.cpp
@@ -2,6 +2,7 @@
 
 #include <cassert>
 #include <cstring>
+#include <iostream>
 #include <fstream>
 #include <sstream>
 #include <string>
@@ -245,6 +246,8 @@ bool gpt_params_parse(int argc, char ** argv, gpt_params & params) {
             params.antiprompt.push_back("### Instruction:\n\n");
         } else if (arg == "--color") {
             params.use_color = true;
+        } else if (arg == "--multiline") {
+            params.multiline_mode = true;
         } else if (arg == "--mlock") {
             params.use_mlock = true;
         } else if (arg == "--no-mmap") {
@@ -323,6 +326,7 @@ void gpt_print_usage(char * argv_0, const gpt_params & params) {
     fprintf(stderr, "                        run in interactive mode and poll user input upon seeing PROMPT (can be\n");
     fprintf(stderr, "                        specified more than once for multiple prompts).\n");
     fprintf(stderr, "  --color               colorise output to distinguish prompt and user input from generations\n");
+    fprintf(stderr, "  --multiline           multiline mode (use Ctrl+D on Linux/Mac and Ctrl+Z on Windpws to send input)\n");
     fprintf(stderr, "  -s SEED, --seed SEED  RNG seed (default: -1, use random seed for <= 0)\n");
     fprintf(stderr, "  -t N, --threads N     number of threads to use during computation (default: %d)\n", params.n_threads);
     fprintf(stderr, "  -p PROMPT, --prompt PROMPT\n");
@@ -441,3 +445,51 @@ void win32_utf8_encode(const std::wstring & wstr, std::string & str) {
 	str = strTo;
 }
 #endif
+
+bool get_input_text(std::string & input_text, bool escape_newline_mode) {
+    bool another_line = true;
+    do {
+        std::string line;
+#if defined(_WIN32)
+        std::wstring wline;
+        if (!std::getline(std::wcin, wline)) {
+            // input stream is bad or EOF received
+            if (std::wcin.bad()) {
+                fprintf(stderr, "%s: error: input stream bad\n", __func__);
+                return 1;
+            }
+        }
+        if (std::wcin.eof()) {
+            another_line = false;
+            std::wcin.clear();
+            std::wcin.seekg(0, std::ios::beg);
+        }
+        win32_utf8_encode(wline, line);
+#else
+        if (!std::getline(std::cin, line)) {
+            // input stream is bad or EOF received
+            if (std::wcin.bad()) {
+                fprintf(stderr, "%s: error: input stream bad\n", __func__);
+                return 1;
+            }
+        }
+        if (std::ccin.eof()) {
+            another_line = false;
+            std::cin.clear();
+            std::cin.seekg(0, std::ios::beg);
+        }
+#endif
+        if (escape_newline_mode) {
+            if (line.empty() || line.back() != '\\') {
+                another_line = false;
+            } else {
+                line.pop_back(); // Remove the continue character
+            }
+        }
+        input_text += line;
+        if (another_line) {
+            input_text += '\n'; // Append the line to the result
+        }
+    } while (another_line);
+    return true;
+}
diff --git a/examples/common.h b/examples/common.h
index 3a7f103e5060c..2c4632b41649a 100644
--- a/examples/common.h
+++ b/examples/common.h
@@ -56,6 +56,7 @@ struct gpt_params {
     bool verbose_prompt    = false; // print prompt tokens before generation
 
     bool clean_interface   = false; // hides input prefix & suffix and displays '>'
+    bool multiline_mode    = false; // enables multi-line mode, to send input press CTRL+D on Linux/Max, CTRL+Z on Windows
 };
 
 bool gpt_params_parse(int argc, char ** argv, gpt_params & params);
@@ -100,3 +101,5 @@ void set_console_color(console_state & con_st, console_color_t color);
 void win32_console_init(bool enable_color);
 void win32_utf8_encode(const std::wstring & wstr, std::string & str);
 #endif
+
+bool get_input_text(std::string & input_text, bool escape_newline_mode);
diff --git a/examples/main/main.cpp b/examples/main/main.cpp
index 744c968392d80..7ea9c3654f64b 100644
--- a/examples/main/main.cpp
+++ b/examples/main/main.cpp
@@ -25,7 +25,8 @@ static bool is_interacting = false;
 #if defined (__unix__) || (defined (__APPLE__) && defined (__MACH__)) || defined (_WIN32)
 void sigint_handler(int signo) {
     set_console_color(con_st, CONSOLE_COLOR_DEFAULT);
-    printf("\n"); // this also force flush stdout.
+    fflush(stdout);
+    fflush(stderr);
     if (signo == SIGINT) {
         if (!is_interacting) {
             is_interacting=true;
@@ -228,8 +229,18 @@ int main(int argc, char ** argv) {
 #if defined (__unix__) || (defined (__APPLE__) && defined (__MACH__)) || defined (_WIN32)
                " - Press Ctrl+C to interject at any time.\n"
 #endif
-               " - Press Return to return control to LLaMa.\n"
-               " - If you want to submit another line, end your input in '\\'.\n\n");
+        );
+        if (params.multiline_mode) {
+#if defined (_WIN32)
+            fprintf(stderr, " - Press Ctrl+Z and Return (EOF) to return control to LLaMa.\n\n");
+#else
+            fprintf(stderr, " - Press Ctrl+D (EOF) to return control to LLaMa.\n\n");
+#endif
+        }
+        else {
+            fprintf(stderr, " - Press Return to return control to LLaMa.\n"
+                            " - If you want to submit another line, end your input in '\\'.\n\n");
+        }
         is_interacting = params.interactive_start;
     }
 
@@ -424,33 +435,13 @@ int main(int argc, char ** argv) {
                     printf("\n> ");
                 }
 
-                std::string line;
-                bool another_line = true;
-                do {
-                    // TODO: try to revert going to new line after enter (to enable in-line text writing)
-#if defined(_WIN32)
-                    std::wstring wline;
-                    if (!std::getline(std::wcin, wline)) {
-                        // input stream is bad or EOF received
-                        return 0;
-                    }
-                    win32_utf8_encode(wline, line);
-#else
-                    if (!std::getline(std::cin, line)) {
-                        // input stream is bad or EOF received
-                        return 0;
-                    }
-#endif
-                    if (line.empty() || line.back() != '\\') {
-                        another_line = false;
-                    } else {
-                        line.pop_back(); // Remove the continue character
-                    }
-                    buffer += line;
-                    if (another_line || !antiprompt.is_stop_prompt) {
-                        buffer += '\n'; // Append the line to the result
-                    }
-                } while (another_line);
+                if (!get_input_text(buffer, !params.multiline_mode)) {
+                    // input stream is bad
+                    return 1;
+                }
+                if (!antiprompt.is_stop_prompt) {
+                    buffer += "\n";
+                }
 
                 // done taking input, reset color
                 set_console_color(con_st, CONSOLE_COLOR_DEFAULT);

From 0dc9dcff130c798b5cdd55d836ef8cdf6e10ae27 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Pazdiora?= <tomas.pazdiora@gmail.com>
Date: Sun, 9 Apr 2023 18:38:23 +0200
Subject: [PATCH 03/12] bugfix

---
 examples/common.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/common.cpp b/examples/common.cpp
index 212974481ad9c..59b0fadfcfb6d 100644
--- a/examples/common.cpp
+++ b/examples/common.cpp
@@ -473,7 +473,7 @@ bool get_input_text(std::string & input_text, bool escape_newline_mode) {
                 return 1;
             }
         }
-        if (std::ccin.eof()) {
+        if (std::cin.eof()) {
             another_line = false;
             std::cin.clear();
             std::cin.seekg(0, std::ios::beg);

From 7c60721217d59cced92647a6594760bc58a7721d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Pazdiora?= <tomas.pazdiora@gmail.com>
Date: Sun, 9 Apr 2023 19:49:33 +0200
Subject: [PATCH 04/12] update implementation

---
 configs/alpaca.txt        | 10 +++--
 configs/chat-with-bob.txt | 11 ++++--
 configs/vicuna-simple.txt | 11 ++++--
 configs/vicuna-stop.txt   | 12 ++++--
 configs/vicuna.txt        | 10 +++--
 examples/common.cpp       | 39 +++++++++++--------
 examples/common.h         | 12 ++++--
 examples/main/main.cpp    | 80 +++++++++++++++++++++++++--------------
 8 files changed, 120 insertions(+), 65 deletions(-)

diff --git a/configs/alpaca.txt b/configs/alpaca.txt
index 06158cdb97b55..5e1558d18bd1d 100644
--- a/configs/alpaca.txt
+++ b/configs/alpaca.txt
@@ -1,5 +1,9 @@
---clean-interface --interactive-first --keep -1
---in-prefix-bos --in-prefix "\n\n### Instruction:\n\n" -r "### Instruction:\n\n"
---in-suffix "\n\n### Response:\n\n"
+--clean-interface 
+--interactive-first 
+--keep -1
+--ins-prefix-bos 
+--ins-prefix "\n\n### Instruction:\n\n"
+--ins-suffix "\n\n### Response:\n\n"
+--reverse-prompt "### Instruction:\n\n"
 
 -p "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n"
diff --git a/configs/chat-with-bob.txt b/configs/chat-with-bob.txt
index 9019753d084d8..ede63d35aa078 100644
--- a/configs/chat-with-bob.txt
+++ b/configs/chat-with-bob.txt
@@ -1,7 +1,10 @@
---interactive-first --keep -1
---in-prefix-bos 
---in-prefix "\nUser: " -r "User: "
---in-suffix "\nBob: " 
+--interactive-first 
+--keep -1
+--ins-prefix-bos 
+--ins-prefix "\nUser: " 
+--ins-suffix "\nBob: "
+--reverse-prompt "User: "
+--rm-trailing-space-workaround
 
 -p "Transcript of a dialog, where the User interacts with an Assistant named Bob. Bob is helpful, kind, honest, good at writing, and never fails to answer the User's requests immediately and with precision.
 User: Hello, Bob.
diff --git a/configs/vicuna-simple.txt b/configs/vicuna-simple.txt
index 053391479e0d1..a87c70082583a 100644
--- a/configs/vicuna-simple.txt
+++ b/configs/vicuna-simple.txt
@@ -1,4 +1,7 @@
---interactive-first --keep -1
---in-prefix-bos 
---in-prefix "\n### Human: " --reverse-prompt "### Human: "
---in-suffix "\n### Assistant: " 
\ No newline at end of file
+--interactive-first 
+--keep -1
+--ins-prefix-bos
+--ins-prefix "\n### Human: "
+--ins-suffix "\n### Assistant: "
+--reverse-prompt "### Human: "
+--rm-trailing-space-workaround
diff --git a/configs/vicuna-stop.txt b/configs/vicuna-stop.txt
index e88a9fd053b79..a8d8907244256 100644
--- a/configs/vicuna-stop.txt
+++ b/configs/vicuna-stop.txt
@@ -1,4 +1,8 @@
---interactive-first --keep -1
---in-prefix-bos 
---in-prefix "\n### Human: " --reverse-prompt "### Human: "
---in-suffix "\n### Assistant: " --stop-prompt "### Assistant: "
\ No newline at end of file
+--interactive-first 
+--keep -1
+--ins-prefix-bos
+--ins-prefix "\n### Human: "
+--ins-suffix "\n### Assistant: "
+--reverse-prompt "### Human: "
+--stop-prompt "### Assistant: "
+--rm-trailing-space-workaround
diff --git a/configs/vicuna.txt b/configs/vicuna.txt
index bf3b966c7fe64..f4d0d2d742a6c 100644
--- a/configs/vicuna.txt
+++ b/configs/vicuna.txt
@@ -1,5 +1,9 @@
---interactive-first --keep -1
---in-prefix-bos --in-prefix "\n### Human: " -r "### Human: "
---in-suffix "\n### Assistant: "
+--interactive-first 
+--keep -1
+--ins-prefix-bos
+--ins-prefix "\n### Human: "
+--ins-suffix "\n### Assistant: "
+--reverse-prompt "### Human: "
+--rm-trailing-space-workaround
 
 -p "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions."
diff --git a/examples/common.cpp b/examples/common.cpp
index 59b0fadfcfb6d..9bf5152feb9f5 100644
--- a/examples/common.cpp
+++ b/examples/common.cpp
@@ -231,18 +231,18 @@ bool gpt_params_parse(int argc, char ** argv, gpt_params & params) {
                 "--clean-interface "
                 "--interactive-first "
                 "--keep -1 "
-                "--in-prefix-bos "
-                "--in-prefix \"\\n\\n### Instruction:\\n\\n\" "
-                "--in-suffix \"\\n\\n### Response:\\n\\n\" "
+                "--ins-prefix-bos "
+                "--ins-prefix \"\\n\\n### Instruction:\\n\\n\" "
+                "--ins-suffix \"\\n\\n### Response:\\n\\n\" "
                 "-r \"### Instruction:\\n\\n\" "
             "\n\n");
             // params.instruct = true;
             params.clean_interface = true;
             params.interactive_start = true;
             params.n_keep = -1;
-            params.input_prefix_bos = true;
-            params.input_prefix = "\n\n### Instruction:\n\n";
-            params.input_suffix = "\n\n### Response:\n\n";
+            params.instruct_prefix_bos = true;
+            params.instruct_prefix = "\n\n### Instruction:\n\n";
+            params.instruct_suffix = "\n\n### Response:\n\n";
             params.antiprompt.push_back("### Instruction:\n\n");
         } else if (arg == "--color") {
             params.use_color = true;
@@ -268,6 +268,8 @@ bool gpt_params_parse(int argc, char ** argv, gpt_params & params) {
                 break;
             }
             params.stopprompt.push_back(args[i]);
+        } else if (arg == "--rm-trailing-space-workaround") {
+            params.rm_trailing_space_workaround = true;
         } else if (arg == "--perplexity") {
             params.perplexity = true;
         } else if (arg == "--ignore-eos") {
@@ -283,22 +285,28 @@ bool gpt_params_parse(int argc, char ** argv, gpt_params & params) {
             exit(0);
         } else if (arg == "--random-prompt") {
             params.random_prompt = true;
-        } else if (arg == "--in-prefix-bos") {
-            params.input_prefix_bos = true;
         } else if (arg == "--in-prefix") {
             if (++i >= args_c) {
                 invalid_param = true;
                 break;
             }
             params.input_prefix = args[i];
-        } else if (arg == "--in-suffix-bos") {
-            params.input_suffix_bos = true;
-        } else if (arg == "--in-suffix") {
+        } else if (arg == "--ins-prefix-bos") {
+            params.instruct_prefix_bos = true;
+        } else if (arg == "--ins-prefix") {
             if (++i >= args_c) {
                 invalid_param = true;
                 break;
             }
-            params.input_suffix = args[i];
+            params.instruct_prefix = args[i];
+        } else if (arg == "--ins-suffix-bos") {
+            params.instruct_suffix_bos = true;
+        } else if (arg == "--ins-suffix") {
+            if (++i >= args_c) {
+                invalid_param = true;
+                break;
+            }
+            params.instruct_suffix = args[i];
         } else {
             fprintf(stderr, "error: unknown argument: %s\n", arg.c_str());
             gpt_print_usage(argv[0], default_params);
@@ -332,10 +340,11 @@ void gpt_print_usage(char * argv_0, const gpt_params & params) {
     fprintf(stderr, "  -p PROMPT, --prompt PROMPT\n");
     fprintf(stderr, "                        prompt to start generation with (default: empty)\n");
     fprintf(stderr, "  --random-prompt       start with a randomized prompt.\n");
-    fprintf(stderr, "  --in-prefix-bos       append bos token to input prefix.\n");
     fprintf(stderr, "  --in-prefix STRING    string to prefix user inputs with (default: empty)\n");
-    fprintf(stderr, "  --in-suffix-bos       append bos token to input suffix.\n");
-    fprintf(stderr, "  --in-suffix STRING    string to suffix user inputs with (default: empty)\n");
+    fprintf(stderr, "  --ins-prefix STRING   (instruct) prefix user inputs with tokenized string (default: empty)\n");
+    fprintf(stderr, "  --ins-prefix-bos      (instruct) prepend bos token to instruct prefix.\n");
+    fprintf(stderr, "  --ins-suffix STRING   (instruct) suffix user inputs with tokenized string (default: empty)\n");
+    fprintf(stderr, "  --ins-suffix-bos      (instruct) prepend bos token to instruct suffix.\n");
     fprintf(stderr, "  -f FNAME, --file FNAME\n");
     fprintf(stderr, "                        prompt file to start generation.\n");
     fprintf(stderr, "  -n N, --n_predict N   number of tokens to predict (default: %d, -1 = infinity)\n", params.n_predict);
diff --git a/examples/common.h b/examples/common.h
index 2c4632b41649a..413604600f3b9 100644
--- a/examples/common.h
+++ b/examples/common.h
@@ -32,12 +32,16 @@ struct gpt_params {
     std::string model  = "models/lamma-7B/ggml-model.bin"; // model path
     std::string prompt = "";
     std::string input_prefix = ""; // string to prefix user inputs with
-    bool input_prefix_bos = false; // append bos token to input prefix
-    std::string input_suffix = ""; // string to suffix user inputs with
-    bool input_suffix_bos = false; // append bos token to input suffix
+
+    std::string instruct_prefix = ""; // prefix user inputs with tokenized string
+    bool instruct_prefix_bos = false; // prepend bos token to instruct prefix
+    std::string instruct_suffix = ""; // suffix user inputs with tokenized string
+    bool instruct_suffix_bos = false; // prepend bos token to instruct suffix
 
     std::vector<std::string> antiprompt; // string upon seeing which more user input is prompted
-    std::vector<std::string> stopprompt; // string upon seeing which more user input is prompted (without adding prefixes and suffixes)
+    std::vector<std::string> stopprompt; // string upon seeing which more user input is prompted (without adding instruct prefixes and suffixes)
+    
+    bool rm_trailing_space_workaround = false; // workaround for removing trailing space from reverse/stop prompts
 
     bool memory_f16        = true;  // use f16 instead of f32 for memory kv
     bool random_prompt     = false; // do not randomize prompt if none provided
diff --git a/examples/main/main.cpp b/examples/main/main.cpp
index 7ea9c3654f64b..47969d2182890 100644
--- a/examples/main/main.cpp
+++ b/examples/main/main.cpp
@@ -85,6 +85,8 @@ int main(int argc, char ** argv) {
         params.prompt = gpt_random_prompt(rng);
     }
 
+    bool instruct_mode = params.instruct_prefix.empty() && params.instruct_suffix.empty();
+
 //    params.prompt = R"(// this function checks if the number n is prime
 //bool is_prime(int n) {)";
 
@@ -154,10 +156,12 @@ int main(int argc, char ** argv) {
     }
 
     // prefix & suffix for instruct mode
-    const auto inp_pfx = ::llama_tokenize(ctx, params.input_prefix, params.input_prefix_bos);
-    std::string input_suffix = params.input_suffix;
-    if (input_suffix.back() == ' ') { input_suffix.pop_back(); } // (remove trailing space workaround)
-    const auto inp_sfx = ::llama_tokenize(ctx, input_suffix, params.input_suffix_bos);
+    const auto inp_pfx = ::llama_tokenize(ctx, params.instruct_prefix, params.instruct_prefix_bos);
+    std::string instruct_suffix = params.instruct_suffix;
+    if (params.rm_trailing_space_workaround) {
+        if (instruct_suffix.back() == ' ') { instruct_suffix.pop_back(); }
+    }
+    const auto inp_sfx = ::llama_tokenize(ctx, instruct_suffix, params.instruct_suffix_bos);
 
     // enable interactive mode if reverse prompt or interactive start is specified
     if (params.antiprompt.size() != 0 || params.stopprompt.size() != 0 || params.interactive_start) { 
@@ -209,10 +213,13 @@ int main(int argc, char ** argv) {
         }
 
         if (!params.input_prefix.empty()) {
-            fprintf(stderr, "Input prefix %s: '%s'\n", params.input_prefix_bos ? "(with bos token)" : "", params.input_prefix.c_str());
+            fprintf(stderr, "Input prefix: '%s'\n", params.input_prefix.c_str());
         }
-        if (!params.input_suffix.empty()) {
-            fprintf(stderr, "Input suffix %s: '%s'\n", params.input_suffix_bos ? "(with bos token)" : "", params.input_suffix.c_str());
+        if (!params.instruct_prefix.empty()) {
+            fprintf(stderr, "Instruct prefix %s: '%s'\n", params.instruct_prefix_bos ? "(with bos token)" : "", params.instruct_prefix.c_str());
+        }
+        if (!params.instruct_suffix.empty()) {
+            fprintf(stderr, "Instruct suffix %s: '%s'\n", params.instruct_suffix_bos ? "(with bos token)" : "", params.instruct_suffix.c_str());
         }
     }
     fprintf(stderr, "sampling: temp = %f, top_k = %d, top_p = %f, repeat_last_n = %i, repeat_penalty = %f\n",
@@ -232,9 +239,9 @@ int main(int argc, char ** argv) {
         );
         if (params.multiline_mode) {
 #if defined (_WIN32)
-            fprintf(stderr, " - Press Ctrl+Z and Return (EOF) to return control to LLaMa.\n\n");
+            fprintf(stderr, " - [MULTILINE MODE] Press Ctrl+Z and Return (EOF) to return control to LLaMa.\n\n");
 #else
-            fprintf(stderr, " - Press Ctrl+D (EOF) to return control to LLaMa.\n\n");
+            fprintf(stderr, " - [MULTILINE MODE] Press Ctrl+D (EOF) to return control to LLaMa.\n\n");
 #endif
         }
         else {
@@ -320,7 +327,7 @@ int main(int argc, char ** argv) {
             }
 
             // replace end of text token with newline token when in interactive mode
-            if (id == llama_token_eos() && params.interactive && params.input_prefix.empty()) {
+            if (id == llama_token_eos() && params.interactive && instruct_mode) {
                 id = llama_token_newline.front();
                 if (params.antiprompt.size() != 0) {
                     // tokenize and inject first reverse prompt
@@ -377,8 +384,10 @@ int main(int argc, char ** argv) {
                 antiprompt.is_stop_prompt = false;
                 // Check if each of the reverse prompts appears at the end of the output.
                 for (std::string & prompt : params.antiprompt) {
-                    antiprompt.trailing_space = prompt.back() == ' ';
-                    antiprompt.len = prompt.length() - (antiprompt.trailing_space ? 1 : 0);
+                    if (params.rm_trailing_space_workaround) {
+                        antiprompt.trailing_space = prompt.back() == ' ';
+                        antiprompt.len = prompt.length() - (antiprompt.trailing_space ? 1 : 0);
+                    }
                     if (last_output.find(prompt.c_str(), last_output.length() - antiprompt.len, antiprompt.len) != std::string::npos) {
                         is_interacting = true;
                         antiprompt.any = true;
@@ -389,8 +398,10 @@ int main(int argc, char ** argv) {
                 }
                 if (!antiprompt.any) {
                     for (std::string & prompt : params.stopprompt) {
-                        antiprompt.trailing_space = prompt.back() == ' ';
-                        antiprompt.len = prompt.length() - (antiprompt.trailing_space ? 1 : 0);
+                        if (params.rm_trailing_space_workaround) {
+                            antiprompt.trailing_space = prompt.back() == ' ';
+                            antiprompt.len = prompt.length() - (antiprompt.trailing_space ? 1 : 0);
+                        }
                         if (last_output.find(prompt.c_str(), last_output.length() - antiprompt.len, antiprompt.len) != std::string::npos) {
                             is_interacting = true;
                             antiprompt.any = true;
@@ -406,21 +417,26 @@ int main(int argc, char ** argv) {
             if (n_past > 0 && is_interacting) 
             {
                 std::string buffer;
-                if (!params.clean_interface && !params.input_prefix.empty() && !antiprompt.any) {
+                if (!params.clean_interface && !params.instruct_prefix.empty() && !antiprompt.any) {
                     // avoid printing again user's new line (TODO: try to revert enter press and print newline)
-                    int i = params.input_prefix.front() == '\n' ? 1 : 0;
+                    int i = params.instruct_prefix.front() == '\n' ? 1 : 0;
                     for (; i < inp_pfx.size(); i++) {
                         printf("%s", llama_token_to_str(ctx, inp_pfx[i]));
                     }
                     fflush(stdout);
                 } 
-                if (antiprompt.any && antiprompt.trailing_space) {
-                    // add back removed trailing space to buffer(workaround)
-                    buffer += ' ';
-                    if (!params.clean_interface) {
-                        printf("%s", buffer.c_str());
+                if (params.rm_trailing_space_workaround) {
+                    // add only if not stopprompt (as stopprompt could be used to pause
+                        //     assistant and then continue without input - adding back trailing 
+                        //     space may mess it up.)
+                    if (!antiprompt.is_stop_prompt && antiprompt.any && antiprompt.trailing_space) {
+                        // add back removed trailing space to buffer(workaround)
+                        buffer += ' ';
+                        if (!params.clean_interface) {
+                            printf("%s", buffer.c_str());
+                        }
+                        fflush(stdout);
                     }
-                    fflush(stdout);
                 }
 
                 // potentially set color to indicate we are taking user input
@@ -435,6 +451,11 @@ int main(int argc, char ** argv) {
                     printf("\n> ");
                 }
 
+                if (!params.input_prefix.empty()) {
+                    buffer += params.input_prefix;
+                    printf("%s", buffer.c_str());
+                }
+
                 if (!get_input_text(buffer, !params.multiline_mode)) {
                     // input stream is bad
                     return 1;
@@ -446,13 +467,16 @@ int main(int argc, char ** argv) {
                 // done taking input, reset color
                 set_console_color(con_st, CONSOLE_COLOR_DEFAULT);
 
-                if (!params.clean_interface && !params.input_suffix.empty() && !antiprompt.is_stop_prompt) {
+                if (!params.clean_interface && !params.instruct_suffix.empty() && !antiprompt.is_stop_prompt) {
                     // avoid printing again user's new line (TODO: try to revert enter press and print newline)
-                    int i = params.input_suffix.front() == '\n' ? 1 : 0;
+                    int i = params.instruct_suffix.front() == '\n' ? 1 : 0;
                     for (; i < inp_sfx.size(); i++) {
                         printf("%s", llama_token_to_str(ctx, inp_sfx[i]));
                     }
-                    // we won't add back removed trailing space here (workaround)
+                    // if (remove trailing space workaround) {
+                    //     We won't add back removed trailing space here, because assistant continues here,
+                    //         and it may mess up it's output (remove trailing space workaround).
+                    // }
                     fflush(stdout);
                 }
 
@@ -461,7 +485,7 @@ int main(int argc, char ** argv) {
                 if (buffer.length() > 1) {
 
                     // insert input prefix
-                    if (!params.input_prefix.empty() && !antiprompt.any) {
+                    if (!params.instruct_prefix.empty() && !antiprompt.any) {
                         n_consumed = embd_inp.size();
                         embd_inp.insert(embd_inp.end(), inp_pfx.begin(), inp_pfx.end());
                     }
@@ -470,7 +494,7 @@ int main(int argc, char ** argv) {
                     embd_inp.insert(embd_inp.end(), line_inp.begin(), line_inp.end());
 
                     // insert response suffix
-                    if (!params.input_suffix.empty() && !antiprompt.is_stop_prompt) {
+                    if (!params.instruct_suffix.empty() && !antiprompt.is_stop_prompt) {
                         embd_inp.insert(embd_inp.end(), inp_sfx.begin(), inp_sfx.end());
                     }
 
@@ -487,7 +511,7 @@ int main(int argc, char ** argv) {
 
         // end of text token
         if (!embd.empty() && embd.back() == llama_token_eos()) {
-            if (params.interactive && !params.input_prefix.empty()) {
+            if (instruct_mode) {
                 is_interacting = true;
             } else {
                 fprintf(stderr, " [end of text]\n");

From d98f613cc41a196a7e287246b10ab16a5a8b4b46 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Pazdiora?= <tomas.pazdiora@gmail.com>
Date: Sun, 9 Apr 2023 20:19:57 +0200
Subject: [PATCH 05/12] typos

---
 examples/common.cpp    | 2 +-
 examples/common.h      | 2 +-
 examples/main/main.cpp | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/examples/common.cpp b/examples/common.cpp
index 9bf5152feb9f5..ebaa97374f0b5 100644
--- a/examples/common.cpp
+++ b/examples/common.cpp
@@ -334,7 +334,7 @@ void gpt_print_usage(char * argv_0, const gpt_params & params) {
     fprintf(stderr, "                        run in interactive mode and poll user input upon seeing PROMPT (can be\n");
     fprintf(stderr, "                        specified more than once for multiple prompts).\n");
     fprintf(stderr, "  --color               colorise output to distinguish prompt and user input from generations\n");
-    fprintf(stderr, "  --multiline           multiline mode (use Ctrl+D on Linux/Mac and Ctrl+Z on Windpws to send input)\n");
+    fprintf(stderr, "  --multiline           multiline mode (use Ctrl+D on Linux/Mac and Ctrl+Z then Return on Windows to send input)\n");
     fprintf(stderr, "  -s SEED, --seed SEED  RNG seed (default: -1, use random seed for <= 0)\n");
     fprintf(stderr, "  -t N, --threads N     number of threads to use during computation (default: %d)\n", params.n_threads);
     fprintf(stderr, "  -p PROMPT, --prompt PROMPT\n");
diff --git a/examples/common.h b/examples/common.h
index 413604600f3b9..edbacb697e4b7 100644
--- a/examples/common.h
+++ b/examples/common.h
@@ -60,7 +60,7 @@ struct gpt_params {
     bool verbose_prompt    = false; // print prompt tokens before generation
 
     bool clean_interface   = false; // hides input prefix & suffix and displays '>'
-    bool multiline_mode    = false; // enables multi-line mode, to send input press CTRL+D on Linux/Max, CTRL+Z on Windows
+    bool multiline_mode    = false; // enables multi-line mode, to send input press CTRL+D on Linux/Max, Ctrl+Z then Return on Windows
 };
 
 bool gpt_params_parse(int argc, char ** argv, gpt_params & params);
diff --git a/examples/main/main.cpp b/examples/main/main.cpp
index 47969d2182890..5da2326d49156 100644
--- a/examples/main/main.cpp
+++ b/examples/main/main.cpp
@@ -239,7 +239,7 @@ int main(int argc, char ** argv) {
         );
         if (params.multiline_mode) {
 #if defined (_WIN32)
-            fprintf(stderr, " - [MULTILINE MODE] Press Ctrl+Z and Return (EOF) to return control to LLaMa.\n\n");
+            fprintf(stderr, " - [MULTILINE MODE] Press Ctrl+Z then Return (EOF) to return control to LLaMa.\n\n");
 #else
             fprintf(stderr, " - [MULTILINE MODE] Press Ctrl+D (EOF) to return control to LLaMa.\n\n");
 #endif

From 73966bc983024bc38256426f0610f378f5a78063 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Pazdiora?= <tomas.pazdiora@gmail.com>
Date: Sun, 9 Apr 2023 21:05:11 +0200
Subject: [PATCH 06/12] Change --multiline implementation to be toggled by EOF.

---
 examples/common.cpp    | 44 ++++++++++++++++++++++++++----------------
 examples/main/main.cpp |  7 ++++---
 2 files changed, 31 insertions(+), 20 deletions(-)

diff --git a/examples/common.cpp b/examples/common.cpp
index ebaa97374f0b5..737b15d97d29d 100644
--- a/examples/common.cpp
+++ b/examples/common.cpp
@@ -334,7 +334,7 @@ void gpt_print_usage(char * argv_0, const gpt_params & params) {
     fprintf(stderr, "                        run in interactive mode and poll user input upon seeing PROMPT (can be\n");
     fprintf(stderr, "                        specified more than once for multiple prompts).\n");
     fprintf(stderr, "  --color               colorise output to distinguish prompt and user input from generations\n");
-    fprintf(stderr, "  --multiline           multiline mode (use Ctrl+D on Linux/Mac and Ctrl+Z then Return on Windows to send input)\n");
+    fprintf(stderr, "  --multiline           multiline mode (use Ctrl+D on Linux/Mac and Ctrl+Z then Return on Windows to toggle multiline)\n");
     fprintf(stderr, "  -s SEED, --seed SEED  RNG seed (default: -1, use random seed for <= 0)\n");
     fprintf(stderr, "  -t N, --threads N     number of threads to use during computation (default: %d)\n", params.n_threads);
     fprintf(stderr, "  -p PROMPT, --prompt PROMPT\n");
@@ -455,45 +455,55 @@ void win32_utf8_encode(const std::wstring & wstr, std::string & str) {
 }
 #endif
 
-bool get_input_text(std::string & input_text, bool escape_newline_mode) {
+bool get_input_text(std::string & input_text, bool eof_toggled_multiline_mode) {
     bool another_line = true;
+    bool is_eof_multiline_toggled = false;
     do {
         std::string line;
 #if defined(_WIN32)
+        auto & stdcin = std::wcin;
         std::wstring wline;
-        if (!std::getline(std::wcin, wline)) {
+        if (!std::getline(stdcin, wline)) {
             // input stream is bad or EOF received
-            if (std::wcin.bad()) {
+            if (stdcin.bad()) {
                 fprintf(stderr, "%s: error: input stream bad\n", __func__);
                 return 1;
             }
         }
-        if (std::wcin.eof()) {
-            another_line = false;
-            std::wcin.clear();
-            std::wcin.seekg(0, std::ios::beg);
-        }
         win32_utf8_encode(wline, line);
 #else
-        if (!std::getline(std::cin, line)) {
+        auto & stdcin = std::cin;
+        if (!std::getline(stdcin, line)) {
             // input stream is bad or EOF received
-            if (std::wcin.bad()) {
+            if (stdcin.bad()) {
                 fprintf(stderr, "%s: error: input stream bad\n", __func__);
                 return 1;
             }
         }
-        if (std::cin.eof()) {
-            another_line = false;
-            std::cin.clear();
-            std::cin.seekg(0, std::ios::beg);
-        }
 #endif
-        if (escape_newline_mode) {
+        if (stdcin.eof()) {
+            stdcin.clear();
+            stdcin.seekg(0, std::ios::beg);
+            if (!eof_toggled_multiline_mode) {
+                another_line = false;
+            } else {
+                is_eof_multiline_toggled = !is_eof_multiline_toggled;
+                if (is_eof_multiline_toggled) {
+                    input_text += line;
+                    continue;
+                }
+            }
+        }
+        if (!eof_toggled_multiline_mode) {
             if (line.empty() || line.back() != '\\') {
                 another_line = false;
             } else {
                 line.pop_back(); // Remove the continue character
             }
+        } else {
+            if (!is_eof_multiline_toggled) {
+                another_line = false;
+            }
         }
         input_text += line;
         if (another_line) {
diff --git a/examples/main/main.cpp b/examples/main/main.cpp
index 5da2326d49156..0f8ef88cb7379 100644
--- a/examples/main/main.cpp
+++ b/examples/main/main.cpp
@@ -238,10 +238,11 @@ int main(int argc, char ** argv) {
 #endif
         );
         if (params.multiline_mode) {
+            fprintf(stderr, " - Press Return to return control to LLaMa.\n"
 #if defined (_WIN32)
-            fprintf(stderr, " - [MULTILINE MODE] Press Ctrl+Z then Return (EOF) to return control to LLaMa.\n\n");
+                            " - [MULTILINE MODE] Press Ctrl+Z then Return (EOF) to toggle.\n\n");
 #else
-            fprintf(stderr, " - [MULTILINE MODE] Press Ctrl+D (EOF) to return control to LLaMa.\n\n");
+                            " - [MULTILINE MODE] Press Ctrl+D (EOF) to toggle.\n\n");
 #endif
         }
         else {
@@ -456,7 +457,7 @@ int main(int argc, char ** argv) {
                     printf("%s", buffer.c_str());
                 }
 
-                if (!get_input_text(buffer, !params.multiline_mode)) {
+                if (!get_input_text(buffer, params.multiline_mode)) {
                     // input stream is bad
                     return 1;
                 }

From c1be1ee073a3306780978b03194409f05c1e13e0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Pazdiora?= <tomas.pazdiora@gmail.com>
Date: Sun, 9 Apr 2023 21:20:39 +0200
Subject: [PATCH 07/12] bugfix

---
 examples/main/main.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/examples/main/main.cpp b/examples/main/main.cpp
index 0f8ef88cb7379..c4090d99f29d1 100644
--- a/examples/main/main.cpp
+++ b/examples/main/main.cpp
@@ -85,7 +85,7 @@ int main(int argc, char ** argv) {
         params.prompt = gpt_random_prompt(rng);
     }
 
-    bool instruct_mode = params.instruct_prefix.empty() && params.instruct_suffix.empty();
+    bool instruct_mode = !params.instruct_prefix.empty() || !params.instruct_suffix.empty();
 
 //    params.prompt = R"(// this function checks if the number n is prime
 //bool is_prime(int n) {)";
@@ -328,7 +328,7 @@ int main(int argc, char ** argv) {
             }
 
             // replace end of text token with newline token when in interactive mode
-            if (id == llama_token_eos() && params.interactive && instruct_mode) {
+            if (id == llama_token_eos() && params.interactive && !instruct_mode) {
                 id = llama_token_newline.front();
                 if (params.antiprompt.size() != 0) {
                     // tokenize and inject first reverse prompt

From 3d2bf47f2499a1983cc438263b34ab849f4a5893 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Pazdiora?= <tomas.pazdiora@gmail.com>
Date: Sun, 9 Apr 2023 21:56:16 +0200
Subject: [PATCH 08/12] default multiline mode

---
 examples/common.cpp | 6 +++---
 examples/common.h   | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/examples/common.cpp b/examples/common.cpp
index 737b15d97d29d..859253b580fd6 100644
--- a/examples/common.cpp
+++ b/examples/common.cpp
@@ -246,8 +246,8 @@ bool gpt_params_parse(int argc, char ** argv, gpt_params & params) {
             params.antiprompt.push_back("### Instruction:\n\n");
         } else if (arg == "--color") {
             params.use_color = true;
-        } else if (arg == "--multiline") {
-            params.multiline_mode = true;
+        } else if (arg == "--disable-multiline") {
+            params.multiline_mode = false;
         } else if (arg == "--mlock") {
             params.use_mlock = true;
         } else if (arg == "--no-mmap") {
@@ -334,7 +334,7 @@ void gpt_print_usage(char * argv_0, const gpt_params & params) {
     fprintf(stderr, "                        run in interactive mode and poll user input upon seeing PROMPT (can be\n");
     fprintf(stderr, "                        specified more than once for multiple prompts).\n");
     fprintf(stderr, "  --color               colorise output to distinguish prompt and user input from generations\n");
-    fprintf(stderr, "  --multiline           multiline mode (use Ctrl+D on Linux/Mac and Ctrl+Z then Return on Windows to toggle multiline)\n");
+    fprintf(stderr, "  --disable-multiline   disable multiline mode (use Ctrl+D on Linux/Mac and Ctrl+Z then Return on Windows to toggle multiline)\n");
     fprintf(stderr, "  -s SEED, --seed SEED  RNG seed (default: -1, use random seed for <= 0)\n");
     fprintf(stderr, "  -t N, --threads N     number of threads to use during computation (default: %d)\n", params.n_threads);
     fprintf(stderr, "  -p PROMPT, --prompt PROMPT\n");
diff --git a/examples/common.h b/examples/common.h
index edbacb697e4b7..ae3401b8acd81 100644
--- a/examples/common.h
+++ b/examples/common.h
@@ -60,7 +60,7 @@ struct gpt_params {
     bool verbose_prompt    = false; // print prompt tokens before generation
 
     bool clean_interface   = false; // hides input prefix & suffix and displays '>'
-    bool multiline_mode    = false; // enables multi-line mode, to send input press CTRL+D on Linux/Max, Ctrl+Z then Return on Windows
+    bool multiline_mode    = true; // enables multi-line mode, to send input press CTRL+D on Linux/Max, Ctrl+Z then Return on Windows
 };
 
 bool gpt_params_parse(int argc, char ** argv, gpt_params & params);

From e163b7375fe14284bc139c863c3c707ec21ba779 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Pazdiora?= <tomas.pazdiora@gmail.com>
Date: Mon, 10 Apr 2023 00:33:35 +0200
Subject: [PATCH 09/12] add more configs

---
 configs/alpaca-native-enhanced.txt | 21 +++++++++++++++++++++
 configs/llama.txt                  |  3 +++
 2 files changed, 24 insertions(+)
 create mode 100644 configs/alpaca-native-enhanced.txt
 create mode 100644 configs/llama.txt

diff --git a/configs/alpaca-native-enhanced.txt b/configs/alpaca-native-enhanced.txt
new file mode 100644
index 0000000000000..41e7e2c1d7dae
--- /dev/null
+++ b/configs/alpaca-native-enhanced.txt
@@ -0,0 +1,21 @@
+--ctx_size 2048 
+--batch_size 16 
+--repeat_penalty 1.15
+--temp 0.4 
+--top_k 30 
+--top_p 0.18 
+
+--interactive-first 
+--keep -1
+
+--ins-prefix-bos 
+--ins-prefix "\n\nUser: "
+--ins-suffix "\n\nAssistant: "
+--reverse-prompt "User: "
+
+-p "You are an AI language model designed to assist the User by answering their questions, offering advice, and engaging in casual conversation in a friendly, helpful, and informative manner. You respond clearly, coherently, and you consider the conversation history.
+
+User: Hey, how's it going?
+
+Assistant: Hey there! I'm doing great, thank you. What can I help you with today? Let's have a fun chat!"
+
diff --git a/configs/llama.txt b/configs/llama.txt
new file mode 100644
index 0000000000000..5abebb41e0db5
--- /dev/null
+++ b/configs/llama.txt
@@ -0,0 +1,3 @@
+--interactive-first 
+--keep -1
+--temp 0.1
\ No newline at end of file

From c4c7ead0bdad365c6db673cf0df95abaf37d0fe1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Pazdiora?= <tomas.pazdiora@gmail.com>
Date: Tue, 11 Apr 2023 23:06:38 +0200
Subject: [PATCH 10/12] update formating

---
 configs/alpaca-native-enhanced.txt | 14 +++++++-------
 configs/alpaca.txt                 |  6 +++---
 configs/chat-with-bob.txt          |  6 +++---
 configs/llama.txt                  |  4 ++--
 configs/vicuna-simple.txt          |  2 +-
 configs/vicuna-stop.txt            |  2 +-
 configs/vicuna.txt                 |  2 +-
 7 files changed, 18 insertions(+), 18 deletions(-)

diff --git a/configs/alpaca-native-enhanced.txt b/configs/alpaca-native-enhanced.txt
index 41e7e2c1d7dae..109d315924d85 100644
--- a/configs/alpaca-native-enhanced.txt
+++ b/configs/alpaca-native-enhanced.txt
@@ -1,14 +1,14 @@
---ctx_size 2048 
---batch_size 16 
+--ctx_size 2048
+--batch_size 16
 --repeat_penalty 1.15
---temp 0.4 
---top_k 30 
---top_p 0.18 
+--temp 0.4
+--top_k 30
+--top_p 0.18
 
---interactive-first 
+--interactive-first
 --keep -1
 
---ins-prefix-bos 
+--ins-prefix-bos
 --ins-prefix "\n\nUser: "
 --ins-suffix "\n\nAssistant: "
 --reverse-prompt "User: "
diff --git a/configs/alpaca.txt b/configs/alpaca.txt
index 5e1558d18bd1d..99a3ab47efc45 100644
--- a/configs/alpaca.txt
+++ b/configs/alpaca.txt
@@ -1,7 +1,7 @@
---clean-interface 
---interactive-first 
+--clean-interface
+--interactive-first
 --keep -1
---ins-prefix-bos 
+--ins-prefix-bos
 --ins-prefix "\n\n### Instruction:\n\n"
 --ins-suffix "\n\n### Response:\n\n"
 --reverse-prompt "### Instruction:\n\n"
diff --git a/configs/chat-with-bob.txt b/configs/chat-with-bob.txt
index ede63d35aa078..198c69e2b7eae 100644
--- a/configs/chat-with-bob.txt
+++ b/configs/chat-with-bob.txt
@@ -1,7 +1,7 @@
---interactive-first 
+--interactive-first
 --keep -1
---ins-prefix-bos 
---ins-prefix "\nUser: " 
+--ins-prefix-bos
+--ins-prefix "\nUser: "
 --ins-suffix "\nBob: "
 --reverse-prompt "User: "
 --rm-trailing-space-workaround
diff --git a/configs/llama.txt b/configs/llama.txt
index 5abebb41e0db5..9d23e75aceb50 100644
--- a/configs/llama.txt
+++ b/configs/llama.txt
@@ -1,3 +1,3 @@
---interactive-first 
+--interactive-first
 --keep -1
---temp 0.1
\ No newline at end of file
+--temp 0.1
diff --git a/configs/vicuna-simple.txt b/configs/vicuna-simple.txt
index a87c70082583a..efa60d96a1fbf 100644
--- a/configs/vicuna-simple.txt
+++ b/configs/vicuna-simple.txt
@@ -1,4 +1,4 @@
---interactive-first 
+--interactive-first
 --keep -1
 --ins-prefix-bos
 --ins-prefix "\n### Human: "
diff --git a/configs/vicuna-stop.txt b/configs/vicuna-stop.txt
index a8d8907244256..911d067efd5b8 100644
--- a/configs/vicuna-stop.txt
+++ b/configs/vicuna-stop.txt
@@ -1,4 +1,4 @@
---interactive-first 
+--interactive-first
 --keep -1
 --ins-prefix-bos
 --ins-prefix "\n### Human: "
diff --git a/configs/vicuna.txt b/configs/vicuna.txt
index f4d0d2d742a6c..6d811410ab3b6 100644
--- a/configs/vicuna.txt
+++ b/configs/vicuna.txt
@@ -1,4 +1,4 @@
---interactive-first 
+--interactive-first
 --keep -1
 --ins-prefix-bos
 --ins-prefix "\n### Human: "

From ab73745993d05adb229988015812a8ef509771d3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Pazdiora?= <tomas.pazdiora@gmail.com>
Date: Tue, 11 Apr 2023 23:08:25 +0200
Subject: [PATCH 11/12] update formatting

---
 examples/common.cpp    | 2 +-
 examples/common.h      | 2 +-
 examples/main/main.cpp | 6 +++---
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/examples/common.cpp b/examples/common.cpp
index cb7b583d5d997..57b4854e11415 100644
--- a/examples/common.cpp
+++ b/examples/common.cpp
@@ -32,7 +32,7 @@ extern "C" __declspec(dllimport) int __stdcall WideCharToMultiByte(unsigned int
 #define CP_UTF8 65001
 #endif
 
-void split_args(std::string args_string, std::vector<std::string> & output_args) 
+void split_args(std::string args_string, std::vector<std::string> & output_args)
 {
     std::string current_arg = "";
     bool in_quotes = false;
diff --git a/examples/common.h b/examples/common.h
index ae3401b8acd81..df8e4c6ccb990 100644
--- a/examples/common.h
+++ b/examples/common.h
@@ -40,7 +40,7 @@ struct gpt_params {
 
     std::vector<std::string> antiprompt; // string upon seeing which more user input is prompted
     std::vector<std::string> stopprompt; // string upon seeing which more user input is prompted (without adding instruct prefixes and suffixes)
-    
+
     bool rm_trailing_space_workaround = false; // workaround for removing trailing space from reverse/stop prompts
 
     bool memory_f16        = true;  // use f16 instead of f32 for memory kv
diff --git a/examples/main/main.cpp b/examples/main/main.cpp
index 9252ef3bb0efc..68b4b2840858e 100644
--- a/examples/main/main.cpp
+++ b/examples/main/main.cpp
@@ -420,7 +420,7 @@ int main(int argc, char ** argv) {
                 }
             }
 
-            if (n_past > 0 && is_interacting) 
+            if (n_past > 0 && is_interacting)
             {
                 std::string buffer;
                 if (!params.clean_interface && !params.instruct_prefix.empty() && !antiprompt.any) {
@@ -430,10 +430,10 @@ int main(int argc, char ** argv) {
                         printf("%s", llama_token_to_str(ctx, inp_pfx[i]));
                     }
                     fflush(stdout);
-                } 
+                }
                 if (params.rm_trailing_space_workaround) {
                     // add only if not stopprompt (as stopprompt could be used to pause
-                        //     assistant and then continue without input - adding back trailing 
+                        //     assistant and then continue without input - adding back trailing
                         //     space may mess it up.)
                     if (!antiprompt.is_stop_prompt && antiprompt.any && antiprompt.trailing_space) {
                         // add back removed trailing space to buffer(workaround)

From c59009a835b14cf21c655b91f2735f7627eb44aa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Pazdiora?= <tomas.pazdiora@gmail.com>
Date: Tue, 11 Apr 2023 23:19:27 +0200
Subject: [PATCH 12/12] apply suggestions

---
 examples/common.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/examples/common.cpp b/examples/common.cpp
index 57b4854e11415..b7a378a721cf8 100644
--- a/examples/common.cpp
+++ b/examples/common.cpp
@@ -32,7 +32,7 @@ extern "C" __declspec(dllimport) int __stdcall WideCharToMultiByte(unsigned int
 #define CP_UTF8 65001
 #endif
 
-void split_args(std::string args_string, std::vector<std::string> & output_args)
+void split_args(const std::string & args_string, std::vector<std::string> & output_args)
 {
     std::string current_arg = "";
     bool in_quotes = false;
@@ -65,7 +65,7 @@ void split_args(std::string args_string, std::vector<std::string> & output_args)
     }
 }
 
-std::string unescape(std::string str) {
+std::string unescape(const std::string & str) {
     return std::regex_replace(str, std::regex("\\\\n"), "\n");
 }