323 lines
9.8 KiB
JSON
323 lines
9.8 KiB
JSON
[
|
|
{
|
|
"test_name": "llama8B_tp1_sharegpt",
|
|
"qps_list": [4,8,16,32,"inf"],
|
|
"common_parameters": {
|
|
"model": "meta-llama/Meta-Llama-3-8B-Instruct",
|
|
"tp": 1,
|
|
"dataset_name": "sharegpt",
|
|
"dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json",
|
|
"num_prompts": 500,
|
|
"port": 8000,
|
|
"reuse_server": false
|
|
},
|
|
"lmdeploy_server_parameters": {
|
|
"dtype": "bfloat16"
|
|
},
|
|
"lmdeploy_client_parameters": {
|
|
},
|
|
"tgi_server_parameters": {
|
|
},
|
|
"tgi_client_parameters": {
|
|
"endpoint": "/generate_stream"
|
|
},
|
|
"trt_server_parameters": {
|
|
"model_type": "llama",
|
|
"model_dtype": "bfloat16",
|
|
"max_batch_size": 2048,
|
|
"max_input_len": 4096,
|
|
"max_seq_len": 6144,
|
|
"max_num_tokens": 16384,
|
|
"trt_llm_version": "v0.11.0"
|
|
},
|
|
"trt_client_parameters": {
|
|
"endpoint": "/v2/models/ensemble/generate_stream"
|
|
},
|
|
"vllm_server_parameters": {
|
|
"disable_log_stats": "",
|
|
"disable_log_requests": "",
|
|
"gpu_memory_utilization": 0.9,
|
|
"num_scheduler_steps": 10,
|
|
"max_num_seqs": 512,
|
|
"dtype": "bfloat16"
|
|
},
|
|
"vllm_client_parameters": {
|
|
},
|
|
"sglang_server_parameters": {
|
|
"disable_radix_cache": "",
|
|
"enable_torch_compile": "",
|
|
"dtype": "bfloat16"
|
|
},
|
|
"sglang_client_parameters": {
|
|
}
|
|
},
|
|
{
|
|
"test_name": "llama8B_tp1_sonnet_512_16",
|
|
"qps_list": [4,8,16,32,"inf"],
|
|
"common_parameters": {
|
|
"model": "meta-llama/Meta-Llama-3-8B-Instruct",
|
|
"tp": 1,
|
|
"dataset_name": "sonnet",
|
|
"dataset_path": "./sonnet_4x.txt",
|
|
"num_prompts": 500,
|
|
"port": 8000,
|
|
"sonnet_input_len": 512,
|
|
"sonnet_output_len": 16,
|
|
"sonnet_prefix_len": 50,
|
|
"reuse_server": true
|
|
},
|
|
"lmdeploy_server_parameters": {
|
|
"dtype": "bfloat16"
|
|
},
|
|
"lmdeploy_client_parameters": {
|
|
},
|
|
"tgi_server_parameters": {
|
|
},
|
|
"tgi_client_parameters": {
|
|
"endpoint": "/generate_stream"
|
|
},
|
|
"trt_server_parameters": {
|
|
"model_type": "llama",
|
|
"model_dtype": "bfloat16",
|
|
"max_batch_size": 2048,
|
|
"max_input_len": 4096,
|
|
"max_seq_len": 6144,
|
|
"max_num_tokens": 16384,
|
|
"trt_llm_version": "v0.11.0"
|
|
},
|
|
"trt_client_parameters": {
|
|
"endpoint": "/v2/models/ensemble/generate_stream"
|
|
},
|
|
"vllm_server_parameters": {
|
|
"disable_log_stats": "",
|
|
"disable_log_requests": "",
|
|
"gpu_memory_utilization": 0.9,
|
|
"num_scheduler_steps": 10,
|
|
"max_num_seqs": 512,
|
|
"dtype": "bfloat16"
|
|
},
|
|
"vllm_client_parameters": {
|
|
},
|
|
"sglang_server_parameters": {
|
|
"disable_radix_cache": "",
|
|
"enable_torch_compile": "",
|
|
"dtype": "bfloat16"
|
|
},
|
|
"sglang_client_parameters": {
|
|
}
|
|
},
|
|
{
|
|
"test_name": "llama8B_tp1_sonnet_512_256",
|
|
"qps_list": [4,8,16,32,"inf"],
|
|
"common_parameters": {
|
|
"model": "meta-llama/Meta-Llama-3-8B-Instruct",
|
|
"tp": 1,
|
|
"dataset_name": "sonnet",
|
|
"dataset_path": "./sonnet_4x.txt",
|
|
"num_prompts": 500,
|
|
"port": 8000,
|
|
"sonnet_input_len": 512,
|
|
"sonnet_output_len": 256,
|
|
"sonnet_prefix_len": 50,
|
|
"reuse_server": true
|
|
},
|
|
"lmdeploy_server_parameters": {
|
|
"dtype": "bfloat16"
|
|
},
|
|
"lmdeploy_client_parameters": {
|
|
},
|
|
"tgi_server_parameters": {
|
|
},
|
|
"tgi_client_parameters": {
|
|
"endpoint": "/generate_stream"
|
|
},
|
|
"trt_server_parameters": {
|
|
"model_type": "llama",
|
|
"model_dtype": "bfloat16",
|
|
"max_batch_size": 2048,
|
|
"max_input_len": 4096,
|
|
"max_seq_len": 6144,
|
|
"max_num_tokens": 16384,
|
|
"trt_llm_version": "v0.11.0"
|
|
},
|
|
"trt_client_parameters": {
|
|
"endpoint": "/v2/models/ensemble/generate_stream"
|
|
},
|
|
"vllm_server_parameters": {
|
|
"disable_log_stats": "",
|
|
"disable_log_requests": "",
|
|
"gpu_memory_utilization": 0.9,
|
|
"num_scheduler_steps": 10,
|
|
"max_num_seqs": 512,
|
|
"dtype": "bfloat16"
|
|
},
|
|
"vllm_client_parameters": {
|
|
},
|
|
"sglang_server_parameters": {
|
|
"disable_radix_cache": "",
|
|
"enable_torch_compile": "",
|
|
"dtype": "bfloat16"
|
|
},
|
|
"sglang_client_parameters": {
|
|
}
|
|
},
|
|
{
|
|
"test_name": "llama70B_tp4_sharegpt",
|
|
"qps_list": [4,8,16,32,"inf"],
|
|
"common_parameters": {
|
|
"model": "meta-llama/Meta-Llama-3-70B-Instruct",
|
|
"tp": 4,
|
|
"dataset_name": "sharegpt",
|
|
"dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json",
|
|
"num_prompts": 500,
|
|
"port": 8000,
|
|
"reuse_server": false
|
|
},
|
|
"lmdeploy_server_parameters": {
|
|
"dtype": "bfloat16"
|
|
},
|
|
"lmdeploy_client_parameters": {
|
|
},
|
|
"tgi_server_parameters": {
|
|
},
|
|
"tgi_client_parameters": {
|
|
"endpoint": "/generate_stream"
|
|
},
|
|
"trt_server_parameters": {
|
|
"model_type": "llama",
|
|
"model_dtype": "bfloat16",
|
|
"max_batch_size": 2048,
|
|
"max_input_len": 4096,
|
|
"max_seq_len": 6144,
|
|
"max_num_tokens": 16384,
|
|
"trt_llm_version": "v0.11.0"
|
|
},
|
|
"trt_client_parameters": {
|
|
"endpoint": "/v2/models/ensemble/generate_stream"
|
|
},
|
|
"vllm_server_parameters": {
|
|
"disable_log_stats": "",
|
|
"disable_log_requests": "",
|
|
"gpu_memory_utilization": 0.9,
|
|
"num_scheduler_steps": 10,
|
|
"max_num_seqs": 512,
|
|
"dtype": "bfloat16"
|
|
},
|
|
"vllm_client_parameters": {
|
|
},
|
|
"sglang_server_parameters": {
|
|
"disable_radix_cache": "",
|
|
"dtype": "bfloat16"
|
|
},
|
|
"sglang_client_parameters": {
|
|
}
|
|
},
|
|
{
|
|
"test_name": "llama70B_tp4_sonnet_512_16",
|
|
"qps_list": [4,8,16,32,"inf"],
|
|
"common_parameters": {
|
|
"model": "meta-llama/Meta-Llama-3-70B-Instruct",
|
|
"tp": 4,
|
|
"dataset_name": "sonnet",
|
|
"dataset_path": "./sonnet_4x.txt",
|
|
"num_prompts": 500,
|
|
"port": 8000,
|
|
"sonnet_input_len": 512,
|
|
"sonnet_output_len": 16,
|
|
"sonnet_prefix_len": 50,
|
|
"reuse_server": true
|
|
},
|
|
"lmdeploy_server_parameters": {
|
|
"dtype": "bfloat16"
|
|
},
|
|
"lmdeploy_client_parameters": {
|
|
},
|
|
"tgi_server_parameters": {
|
|
},
|
|
"tgi_client_parameters": {
|
|
"endpoint": "/generate_stream"
|
|
},
|
|
"trt_server_parameters": {
|
|
"model_type": "llama",
|
|
"model_dtype": "bfloat16",
|
|
"max_batch_size": 2048,
|
|
"max_input_len": 4096,
|
|
"max_seq_len": 6144,
|
|
"max_num_tokens": 16384,
|
|
"trt_llm_version": "v0.11.0"
|
|
},
|
|
"trt_client_parameters": {
|
|
"endpoint": "/v2/models/ensemble/generate_stream"
|
|
},
|
|
"vllm_server_parameters": {
|
|
"disable_log_stats": "",
|
|
"disable_log_requests": "",
|
|
"gpu_memory_utilization": 0.9,
|
|
"num_scheduler_steps": 10,
|
|
"max_num_seqs": 512,
|
|
"dtype": "bfloat16"
|
|
},
|
|
"vllm_client_parameters": {
|
|
},
|
|
"sglang_server_parameters": {
|
|
"disable_radix_cache": "",
|
|
"dtype": "bfloat16"
|
|
},
|
|
"sglang_client_parameters": {
|
|
}
|
|
},
|
|
{
|
|
"test_name": "llama70B_tp4_sonnet_512_256",
|
|
"qps_list": [4,8,16,32,"inf"],
|
|
"common_parameters": {
|
|
"model": "meta-llama/Meta-Llama-3-70B-Instruct",
|
|
"tp": 4,
|
|
"dataset_name": "sonnet",
|
|
"dataset_path": "./sonnet_4x.txt",
|
|
"num_prompts": 500,
|
|
"port": 8000,
|
|
"sonnet_input_len": 512,
|
|
"sonnet_output_len": 256,
|
|
"sonnet_prefix_len": 50,
|
|
"reuse_server": true
|
|
},
|
|
"lmdeploy_server_parameters": {
|
|
"dtype": "bfloat16"
|
|
},
|
|
"lmdeploy_client_parameters": {
|
|
},
|
|
"tgi_server_parameters": {
|
|
},
|
|
"tgi_client_parameters": {
|
|
"endpoint": "/generate_stream"
|
|
},
|
|
"trt_server_parameters": {
|
|
"model_type": "llama",
|
|
"model_dtype": "bfloat16",
|
|
"max_batch_size": 2048,
|
|
"max_input_len": 4096,
|
|
"max_seq_len": 6144,
|
|
"max_num_tokens": 16384,
|
|
"trt_llm_version": "v0.11.0"
|
|
},
|
|
"trt_client_parameters": {
|
|
"endpoint": "/v2/models/ensemble/generate_stream"
|
|
},
|
|
"vllm_server_parameters": {
|
|
"disable_log_stats": "",
|
|
"disable_log_requests": "",
|
|
"gpu_memory_utilization": 0.9,
|
|
"num_scheduler_steps": 10,
|
|
"max_num_seqs": 512,
|
|
"dtype": "bfloat16"
|
|
},
|
|
"vllm_client_parameters": {
|
|
},
|
|
"sglang_server_parameters": {
|
|
"disable_radix_cache": "",
|
|
"dtype": "bfloat16"
|
|
},
|
|
"sglang_client_parameters": {
|
|
}
|
|
}
|
|
] |