# 1. Start the etcd server
etcd --listen-client-urls http://{IP}:2379 --advertise-client-urls http://{IP}:2379

# 2. Start the mooncake_master server
LD_LIBRARY_PATH=/usr/local/lib/:$LD_LIBRARY_PATH \
mooncake_master --port 50001

# 3. Run multiple vllm instances
# kv_producer role
CUDA_VISIBLE_DEVICES=1 \
LD_LIBRARY_PATH=/usr/local/lib/:$LD_LIBRARY_PATH \
MOONCAKE_CONFIG_PATH=./mooncake.json \
vllm serve deepseek-ai/DeepSeek-V2-Lite \
		--port 8100 \
		--trust-remote-code \
		--max-model-len 10000 \
		--gpu-memory-utilization 0.8 \
		--kv-transfer-config '{"kv_connector":"MooncakeStoreConnector", "kv_role":"kv_producer", "kv_rank":0, "kv_parallel_size":2}'


# kv_consumer role
CUDA_VISIBLE_DEVICES=2 \
LD_LIBRARY_PATH=/usr/local/lib/:$LD_LIBRARY_PATH \
MOONCAKE_CONFIG_PATH=./mooncake.json \
vllm serve deepseek-ai/DeepSeek-V2-Lite \
		--port 8200 \
		--trust-remote-code \
		--max-model-len 10000 \
		--gpu-memory-utilization 0.8 \
		--kv-transfer-config '{"kv_connector":"MooncakeStoreConnector", "kv_rank":1, "kv_role":"kv_consumer", "kv_rank":1, "kv_parallel_size":2}'

CUDA_VISIBLE_DEVICES=3 \
LD_LIBRARY_PATH=/usr/local/lib/:$LD_LIBRARY_PATH \
MOONCAKE_CONFIG_PATH=./mooncake.json \
vllm serve deepseek-ai/DeepSeek-V2-Lite \
		--port 8201 \
		--trust-remote-code \
		--max-model-len 10000 \
		--gpu-memory-utilization 0.8 \
		--kv-transfer-config '{"kv_connector":"MooncakeStoreConnector", "kv_rank":1, "kv_role":"kv_consumer", "kv_rank":1, "kv_parallel_size":2}'

# 4. Run round robin proxy server
python3 examples/online_serving/disagg_examples/disagg_proxy_demo.py --model deepseek-ai/DeepSeek-V2-Lite --prefill localhost:8100  --decode localhost:8200 localhost:8201  --port 8000

#5. Send request 
curl -s http://localhost:8000/v1/completions -H "Content-Type: application/json" -d '{
  "model": "deepseek-ai/DeepSeek-V2-Lite",
  "prompt": "San Francisco is a",
  "max_tokens": 100}'

{"id":"cmpl-163f1756892949d1b914d0756cd98eab","object":"text_completion","created":1743687181,"model":"deepseek-ai/DeepSeek-V2-Lite","choices":[{"index":0,"text":" city that is known for its diversity, and the same can be said for the city’s food scene. From the iconic clam chowder to the deliciously sweet and sour Chinese food, San Francisco has a wide variety of cuisines to choose from.\nOne of the most popular dishes in San Francisco is the clam chowder. This creamy soup is made with clams, potatoes, and bacon, and is typically served with a side of sourdough bread. The clam ch","logprobs":null,"finish_reason":"length","stop_reason":null,"prompt_logprobs":null}],"usage":{"prompt_tokens":5,"total_tokens":105,"completion_tokens":100,"prompt_tokens_details":null}}

# 1. Run prefill instance
CUDA_VISIBLE_DEVICES=1 vllm serve deepseek-ai/DeepSeek-V2-Lite \
    --port 8100 \
    --max-model-len 10000 \
    --gpu-memory-utilization 0.8 \
    --trust-remote-code \
    --kv-transfer-config \
    '{"kv_connector":"PyNcclConnector","kv_role":"kv_producer","kv_rank":0,"kv_parallel_size":2}'

# 2. Run decode instance
CUDA_VISIBLE_DEVICES=2 vllm serve deepseek-ai/DeepSeek-V2-Lite \
    --port 8200 \
    --max-model-len 10000 \
    --gpu-memory-utilization 0.8 \
    --trust-remote-code \
    --kv-transfer-config \
    '{"kv_connector":"PyNcclConnector","kv_role":"kv_consumer","kv_rank":1,"kv_parallel_size":2}'
	
# 3. Run simple proxy server	
python3 benchmarks/disagg_benchmarks/disagg_prefill_proxy_server.py

# 4. Send request 
curl -X POST -s http://localhost:8000/v1/completions \
-H "Content-Type: application/json" \
-d '{
"model": "deepseek-ai/DeepSeek-V2-Lite",
"prompt": "San Francisco is a",
"max_tokens": 100
}'
{"id":"cmpl-959481be2f694efca9e574693c965753","object":"text_completion","created":1743688337,"model":"deepseek-ai/DeepSeek-V2-Lite","choices":[{"index":0,"text":" city of many neighborhoods, each with its own unique character and charm. From the bustling streets of Chinatown to the quiet streets of the Haight-Ashbury district, there is something for everyone in this vibrant city. One of the most popular neighborhoods in San Francisco is the Castro district, which is known for its vibrant LGBTQ+ community and its many bars and restaurants. Another popular neighborhood is the Mission district, which is known for its diverse population and its many restaurants and shops. No matter what","logprobs":null,"finish_reason":"length","stop_reason":null,"prompt_logprobs":null}],"usage":{"prompt_tokens":5,"total_tokens":105,"completion_tokens":100,"prompt_tokens_details":null}}