CUDA_VISIBLE_DEVICES=0 VLLM_NIXL_SIDE_CHANNEL_PORT=5567 vllm serve /workspace/models/qwen2.5_7B \
  --port 20001 \
  --tensor-parallel-size 1 \
  --enforce-eager \
  --block-size 16 \
  --enable-log-requests \
  --kv-transfer-config '{"kv_connector":"NixlConnector","kv_role":"kv_both"}'

CUDA_VISIBLE_DEVICES=1 VLLM_NIXL_SIDE_CHANNEL_PORT=5667 viztracer -o decode.json vllm serve /workspace/models/qwen2.5_7B \
  --port 20002 \
  --tensor-parallel-size 1 \
  --enforce-eager \
  --block-size 16 \
  --enable-log-requests \
  --kv-transfer-config '{"kv_connector":"NixlConnector","kv_role":"kv_both"}'

python ./tests/v1/kv_connector/nixl_integration/toy_proxy_server.py \
      --port 40000 \
      --prefiller-port 20001 \
      --decoder-port 20002

python benchmarks/benchmark_serving.py \
    --backend vllm \
    --endpoint /v1/completions \
    --model /workspace/models/qwen2.5_7B \
    --dataset-name random \
    --random-input 800 \
    --random-output 100 \
    --num-prompts 100 \
    --request-rate 5 \
    --host localhost \
    --port 40000 \