optimiztions for serving

Signed-off-by: Tsai, Louie <[email protected]>

Author: louie-tsai

.buildkite/nightly-benchmarks/tests/serving-tests-cpu.json

@@ -6,6 +6,7 @@
             "VLLM_RPC_TIMEOUT": 100000,
 	    "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
 	    "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
+	    "VLLM_CPU_NUM_OF_RESERVED_CPU": 0,
 	    "VLLM_CPU_KVCACHE_SPACE": 40
         },
         "server_parameters": {
@@ -18,6 +19,7 @@
 	    "trust_remote_code": "",
             "disable_log_stats": "",
             "disable_log_requests": "",
+	    "enforce_eager": "",
             "load_format": "dummy"
         },
         "client_parameters": {
@@ -35,6 +37,7 @@
             "VLLM_RPC_TIMEOUT": 100000,
 	    "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
 	    "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
+	    "VLLM_CPU_NUM_OF_RESERVED_CPU": 0,
 	    "VLLM_CPU_KVCACHE_SPACE": 40
         },
         "server_parameters": {
@@ -47,6 +50,7 @@
 	    "trust_remote_code": "",
             "disable_log_stats": "",
             "disable_log_requests": "",
+	    "enforce_eager": "",
             "load_format": "dummy"
         },
         "client_parameters": {
@@ -64,6 +68,7 @@
             "VLLM_RPC_TIMEOUT": 100000,
 	    "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
 	    "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
+	    "VLLM_CPU_NUM_OF_RESERVED_CPU": 0,
 	    "VLLM_CPU_KVCACHE_SPACE": 40
         },
         "server_parameters": {
@@ -76,6 +81,7 @@
 	    "trust_remote_code": "",
             "disable_log_stats": "",
             "disable_log_requests": "",
+	    "enforce_eager": "",
             "load_format": "dummy"
         },
         "client_parameters": {
@@ -93,6 +99,7 @@
             "VLLM_RPC_TIMEOUT": 100000,
 	    "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
 	    "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
+	    "VLLM_CPU_NUM_OF_RESERVED_CPU": 0,
 	    "VLLM_CPU_KVCACHE_SPACE": 40
         },
         "server_parameters": {
@@ -106,6 +113,7 @@
 	    "enable_chunked_prefill": "",
             "disable_log_stats": "",
             "disable_log_requests": "",
+	    "enforce_eager": "",
             "load_format": "dummy"
         },
         "client_parameters": {
@@ -125,6 +133,7 @@
             "VLLM_RPC_TIMEOUT": 100000,
 	    "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
 	    "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
+	    "VLLM_CPU_NUM_OF_RESERVED_CPU": 0,
 	    "VLLM_CPU_KVCACHE_SPACE": 40
         },
         "server_parameters": {
@@ -138,6 +147,7 @@
 	    "enable_chunked_prefill": "",
             "disable_log_stats": "",
             "disable_log_requests": "",
+	    "enforce_eager": "",
             "load_format": "dummy"
         },
         "client_parameters": {