[Bugfix]: fix issue with n>1 sampling on v1 requests overriding each other by jeffrey-dot-li · Pull Request #16863 · vllm-project/vllm

jeffrey-dot-li · 2025-04-18T17:33:09Z

Fixes issue with n>1 sampling on v1 requests overriding each other
FIX #12584 #14280
MRE:

import os
os.environ["VLLM_USE_V1"] = "1"
import ray
from vllm.sampling_params import SamplingParams, RequestOutputKind
from vllm.v1.engine.async_llm import AsyncLLM


@ray.remote
class VllmApp:
    def __init__(self):
        import vllm

        engine_args = vllm.AsyncEngineArgs(
            model="Qwen/Qwen2.5-0.5B-Instruct",
            distributed_executor_backend="ray",
            disable_log_requests=True,
        )
        self.llm = AsyncLLM.from_engine_args(engine_args)

    async def run(self):
        sampling_params = SamplingParams(
            temperature=1.0,
            max_tokens=32,
            n=2,
        )
        stream = self.llm.generate(
            prompt="The capital of France is",
            sampling_params=sampling_params,
            request_id="1",
        )
        from vllm.utils import collect_from_async_generator

        return await collect_from_async_generator(stream)


if __name__ == "__main__":
    pg_resources = [
        {"CPU": 1},
        {"CPU": 1, "GPU": 1},
    ]
    pg = ray.util.placement_group(pg_resources)
    app = VllmApp.options(placement_group=pg).remote()
    result = app.run.remote()
    print(ray.get(result))

Before: (See how CompletionOutput.index is always 1 after the first token)

[RequestOutput(request_id=1, prompt='The capital of France is', prompt_token_ids=[785, 6722, 315, 9625, 374], encoder_prompt=None, encoder_prompt_token_ids=None, prompt_logprobs=None, outputs=[CompletionOutput(index=0, text=' also', token_ids=[1083], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None)], finished=False, metrics=None, lora_request=None, num_cached_tokens=None, multi_modal_placeholders={}), RequestOutput(request_id=1, prompt=None, prompt_token_ids=[785, 6722, 315, 9625, 374], encoder_prompt=None, encoder_prompt_token_ids=None, prompt_logprobs=None, outputs=[CompletionOutput(index=1, text=' ______', token_ids=[32671], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None)], finished=False, metrics=None, lora_request=None, num_cached_tokens=None, multi_modal_placeholders={}), RequestOutput(request_id=1, prompt=None, prompt_token_ids=[785, 6722, 315, 9625, 374], encoder_prompt=None, encoder_prompt_token_ids=None, prompt_logprobs=None, outputs=[CompletionOutput(index=1, text=' _______', token_ids=[32671, 62], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None)], finished=False, metrics=None, lora_request=None, num_cached_tokens=None, multi_modal_placeholders={}), RequestOutput(request_id=1, prompt=None, prompt_token_ids=[785, 6722, 315, 9625, 374], encoder_prompt=None, encoder_prompt_token_ids=None, prompt_logprobs=None, outputs=[CompletionOutput(index=1, text=' _______ ____', token_ids=[32671, 62, 30743], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None)], finished=False, metrics=None, lora_request=None, num_cached_tokens=None, multi_modal_placeholders={}), RequestOutput(request_id=1, prompt=None, prompt_token_ids=[785, 6722, 315, 9625, 374], encoder_prompt=None, encoder_prompt_token_ids=None, prompt_logprobs=None, outputs=[CompletionOutput(index=1, text=' _______ ____.\n', token_ids=[32671, 62, 30743, 624], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None)], finished=False, metrics=None, lora_request=None, num_cached_tokens=None, multi_modal_placeholders={}), RequestOutput(request_id=1, prompt=None, prompt_token_ids=[785, 6722, 315, 9625, 374], encoder_prompt=None, encoder_prompt_token_ids=None, prompt_logprobs=None, outputs=[CompletionOutput(index=1, text=' _______ ____.\nIn', token_ids=[32671, 62, 30743, 624, 641], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None)], finished=False, metrics=None, lora_request=None, num_cached_tokens=None, multi_modal_placeholders={}), RequestOutput(request_id=1, prompt=None, prompt_token_ids=[785, 6722, 315, 9625, 374], encoder_prompt=None, encoder_prompt_token_ids=None, prompt_logprobs=None, outputs=[CompletionOutput(index=1, text=' _______ ____.\nIn Bordeaux', token_ids=[32671, 62, 30743, 624, 641, 83957], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None)], finished=False, metrics=None, lora_request=None, num_cached_tokens=None, multi_modal_placeholders={}), RequestOutput(request_id=1, prompt=None, prompt_token_ids=[785, 6722, 315, 9625, 374], encoder_prompt=None, encoder_prompt_token_ids=None, prompt_logprobs=None, outputs=[CompletionOutput(index=1, text=' _______ ____.\nIn Bordeaux,', token_ids=[32671, 62, 30743, 624, 641, 83957, 11], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None)], finished=False, metrics=None, lora_request=None, num_cached_tokens=None, multi_modal_placeholders={}), RequestOutput(request_id=1, prompt=None, prompt_token_ids=[785, 6722, 315, 9625, 374], encoder_prompt=None, encoder_prompt_token_ids=None, prompt_logprobs=None, outputs=[CompletionOutput(index=1, text=' _______ ____.\nIn Bordeaux, Switzerland', token_ids=[32671, 62, 30743, 624, 641, 83957, 11, 29121], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None)], finished=False, metrics=None, lora_request=None, num_cached_tokens=None, multi_modal_placeholders={}), RequestOutput(request_id=1, prompt=None, prompt_token_ids=[785, 6722, 315, 9625, 374], encoder_prompt=None, encoder_prompt_token_ids=None, prompt_logprobs=None, outputs=[CompletionOutput(index=1, text=' _______ ____.\nIn Bordeaux, Switzerland there', token_ids=[32671, 62, 30743, 624, 641, 83957, 11, 29121, 1052], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None)], finished=False, metrics=None, lora_request=None, num_cached_tokens=None, multi_modal_placeholders={}), RequestOutput(request_id=1, prompt=None, prompt_token_ids=[785, 6722, 315, 9625, 374], encoder_prompt=None, encoder_prompt_token_ids=None, prompt_logprobs=None, outputs=[CompletionOutput(index=1, text=" _______ ____.\nIn Bordeaux, Switzerland there's", token_ids=[32671, 62, 30743, 624, 641, 83957, 11, 29121, 1052, 594], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None)], finished=False, metrics=None, lora_request=None, num_cached_tokens=None, multi_modal_placeholders={}), RequestOutput(request_id=1, prompt=None, prompt_token_ids=[785, 6722, 315, 9625, 374], encoder_prompt=None, encoder_prompt_token_ids=None, prompt_logprobs=None, outputs=[CompletionOutput(index=1, text=" _______ ____.\nIn Bordeaux, Switzerland there's a", token_ids=[32671, 62, 30743, 624, 641, 83957, 11, 29121, 1052, 594, 264], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None)], finished=False, metrics=None, lora_request=None, num_cached_tokens=None, multi_modal_placeholders={}), RequestOutput(request_id=1, prompt=None, prompt_token_ids=[785, 6722, 315, 9625, 374], encoder_prompt=None, encoder_prompt_token_ids=None, prompt_logprobs=None, outputs=[CompletionOutput(index=1, text=" _______ ____.\nIn Bordeaux, Switzerland there's a train", token_ids=[32671, 62, 30743, 624, 641, 83957, 11, 29121, 1052, 594, 264, 5426], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None)], finished=False, metrics=None, lora_request=None, num_cached_tokens=None, multi_modal_placeholders={}), RequestOutput(request_id=1, prompt=None, prompt_token_ids=[785, 6722, 315, 9625, 374], encoder_prompt=None, encoder_prompt_token_ids=None, prompt_logprobs=None, outputs=[CompletionOutput(index=1, text=" _______ ____.\nIn Bordeaux, Switzerland there's a train company", token_ids=[32671, 62, 30743, 624, 641, 83957, 11, 29121, 1052, 594, 264, 5426, 2813], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None)], finished=False, metrics=None, lora_request=None, num_cached_tokens=None, multi_modal_placeholders={}), RequestOutput(request_id=1, prompt=None, prompt_token_ids=[785, 6722, 315, 9625, 374], encoder_prompt=None, encoder_prompt_token_ids=None, prompt_logprobs=None, outputs=[CompletionOutput(index=1, text=" _______ ____.\nIn Bordeaux, Switzerland there's a train company named", token_ids=[32671, 62, 30743, 624, 641, 83957, 11, 29121, 1052, 594, 264, 5426, 2813, 6941], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None)], finished=False, metrics=None, lora_request=None, num_cached_tokens=None, multi_modal_placeholders={}), RequestOutput(request_id=1, prompt=None, prompt_token_ids=[785, 6722, 315, 9625, 374], encoder_prompt=None, encoder_prompt_token_ids=None, prompt_logprobs=None, outputs=[CompletionOutput(index=1, text=" _______ ____.\nIn Bordeaux, Switzerland there's a train company named ______", token_ids=[32671, 62, 30743, 624, 641, 83957, 11, 29121, 1052, 594, 264, 5426, 2813, 6941, 32671], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None)], finished=False, metrics=None, lora_request=None, num_cached_tokens=None, multi_modal_placeholders={}), RequestOutput(request_id=1, prompt=None, prompt_token_ids=[785, 6722, 315, 9625, 374], encoder_prompt=None, encoder_prompt_token_ids=None, prompt_logprobs=None, outputs=[CompletionOutput(index=1, text=" _______ ____.\nIn Bordeaux, Switzerland there's a train company named _______", token_ids=[32671, 62, 30743, 624, 641, 83957, 11, 29121, 1052, 594, 264, 5426, 2813, 6941, 32671, 62], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None)], finished=False, metrics=None, lora_request=None, num_cached_tokens=None, multi_modal_placeholders={}), RequestOutput(request_id=1, prompt=None, prompt_token_ids=[785, 6722, 315, 9625, 374], encoder_prompt=None, encoder_prompt_token_ids=None, prompt_logprobs=None, outputs=[CompletionOutput(index=1, text=" _______ ____.\nIn Bordeaux, Switzerland there's a train company named _______.\n\n", token_ids=[32671, 62, 30743, 624, 641, 83957, 11, 29121, 1052, 594, 264, 5426, 2813, 6941, 32671, 62, 382], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None)], finished=False, metrics=None, lora_request=None, num_cached_tokens=None, multi_modal_placeholders={}), RequestOutput(request_id=1, prompt=None, prompt_token_ids=[785, 6722, 315, 9625, 374], encoder_prompt=None, encoder_prompt_token_ids=None, prompt_logprobs=None, outputs=[CompletionOutput(index=1, text=" _______ ____.\nIn Bordeaux, Switzerland there's a train company named _______.\n\nWhich", token_ids=[32671, 62, 30743, 624, 641, 83957, 11, 29121, 1052, 594, 264, 5426, 2813, 6941, 32671, 62, 382, 23085], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None)], finished=False, metrics=None, lora_request=None, num_cached_tokens=None, multi_modal_placeholders={}), RequestOutput(request_id=1, prompt=None, prompt_token_ids=[785, 6722, 315, 9625, 374], encoder_prompt=None, encoder_prompt_token_ids=None, prompt_logprobs=None, outputs=[CompletionOutput(index=1, text=" _______ ____.\nIn Bordeaux, Switzerland there's a train company named _______.\n\nWhich French", token_ids=[32671, 62, 30743, 624, 641, 83957, 11, 29121, 1052, 594, 264, 5426, 2813, 6941, 32671, 62, 382, 23085, 8585], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None)], finished=False, metrics=None, lora_request=None, num_cached_tokens=None, multi_modal_placeholders={}), RequestOutput(request_id=1, prompt=None, prompt_token_ids=[785, 6722, 315, 9625, 374], encoder_prompt=None, encoder_prompt_token_ids=None, prompt_logprobs=None, outputs=[CompletionOutput(index=1, text=" _______ ____.\nIn Bordeaux, Switzerland there's a train company named _______.\n\nWhich French town", token_ids=[32671, 62, 30743, 624, 641, 83957, 11, 29121, 1052, 594, 264, 5426, 2813, 6941, 32671, 62, 382, 23085, 8585, 6290], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None)], finished=False, metrics=None, lora_request=None, num_cached_tokens=None, multi_modal_placeholders={}), RequestOutput(request_id=1, prompt=None, prompt_token_ids=[785, 6722, 315, 9625, 374], encoder_prompt=None, encoder_prompt_token_ids=None, prompt_logprobs=None, outputs=[CompletionOutput(index=1, text=" _______ ____.\nIn Bordeaux, Switzerland there's a train company named _______.\n\nWhich French town is", token_ids=[32671, 62, 30743, 624, 641, 83957, 11, 29121, 1052, 594, 264, 5426, 2813, 6941, 32671, 62, 382, 23085, 8585, 6290, 374], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None)], finished=False, metrics=None, lora_request=None, num_cached_tokens=None, multi_modal_placeholders={}), RequestOutput(request_id=1, prompt=None, prompt_token_ids=[785, 6722, 315, 9625, 374], encoder_prompt=None, encoder_prompt_token_ids=None, prompt_logprobs=None, outputs=[CompletionOutput(index=1, text=" _______ ____.\nIn Bordeaux, Switzerland there's a train company named _______.\n\nWhich French town is entirely", token_ids=[32671, 62, 30743, 624, 641, 83957, 11, 29121, 1052, 594, 264, 5426, 2813, 6941, 32671, 62, 382, 23085, 8585, 6290, 374, 11368], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None)], finished=False, metrics=None, lora_request=None, num_cached_tokens=None, multi_modal_placeholders={}), RequestOutput(request_id=1, prompt=None, prompt_token_ids=[785, 6722, 315, 9625, 374], encoder_prompt=None, encoder_prompt_token_ids=None, prompt_logprobs=None, outputs=[CompletionOutput(index=1, text=" _______ ____.\nIn Bordeaux, Switzerland there's a train company named _______.\n\nWhich French town is entirely surrounded", token_ids=[32671, 62, 30743, 624, 641, 83957, 11, 29121, 1052, 594, 264, 5426, 2813, 6941, 32671, 62, 382, 23085, 8585, 6290, 374, 11368, 22865], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None)], finished=False, metrics=None, lora_request=None, num_cached_tokens=None, multi_modal_placeholders={}), RequestOutput(request_id=1, prompt=None, prompt_token_ids=[785, 6722, 315, 9625, 374], encoder_prompt=None, encoder_prompt_token_ids=None, prompt_logprobs=None, outputs=[CompletionOutput(index=1, text=" _______ ____.\nIn Bordeaux, Switzerland there's a train company named _______.\n\nWhich French town is entirely surrounded by", token_ids=[32671, 62, 30743, 624, 641, 83957, 11, 29121, 1052, 594, 264, 5426, 2813, 6941, 32671, 62, 382, 23085, 8585, 6290, 374, 11368, 22865, 553], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None)], finished=False, metrics=None, lora_request=None, num_cached_tokens=None, multi_modal_placeholders={}), RequestOutput(request_id=1, prompt=None, prompt_token_ids=[785, 6722, 315, 9625, 374], encoder_prompt=None, encoder_prompt_token_ids=None, prompt_logprobs=None, outputs=[CompletionOutput(index=1, text=" _______ ____.\nIn Bordeaux, Switzerland there's a train company named _______.\n\nWhich French town is entirely surrounded by water", token_ids=[32671, 62, 30743, 624, 641, 83957, 11, 29121, 1052, 594, 264, 5426, 2813, 6941, 32671, 62, 382, 23085, 8585, 6290, 374, 11368, 22865, 553, 3015], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None)], finished=False, metrics=None, lora_request=None, num_cached_tokens=None, multi_modal_placeholders={}), RequestOutput(request_id=1, prompt=None, prompt_token_ids=[785, 6722, 315, 9625, 374], encoder_prompt=None, encoder_prompt_token_ids=None, prompt_logprobs=None, outputs=[CompletionOutput(index=1, text=" _______ ____.\nIn Bordeaux, Switzerland there's a train company named _______.\n\nWhich French town is entirely surrounded by water,", token_ids=[32671, 62, 30743, 624, 641, 83957, 11, 29121, 1052, 594, 264, 5426, 2813, 6941, 32671, 62, 382, 23085, 8585, 6290, 374, 11368, 22865, 553, 3015, 11], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None)], finished=False, metrics=None, lora_request=None, num_cached_tokens=None, multi_modal_placeholders={}), RequestOutput(request_id=1, prompt=None, prompt_token_ids=[785, 6722, 315, 9625, 374], encoder_prompt=None, encoder_prompt_token_ids=None, prompt_logprobs=None, outputs=[CompletionOutput(index=1, text=" _______ ____.\nIn Bordeaux, Switzerland there's a train company named _______.\n\nWhich French town is entirely surrounded by water, with", token_ids=[32671, 62, 30743, 624, 641, 83957, 11, 29121, 1052, 594, 264, 5426, 2813, 6941, 32671, 62, 382, 23085, 8585, 6290, 374, 11368, 22865, 553, 3015, 11, 448], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None)], finished=False, metrics=None, lora_request=None, num_cached_tokens=None, multi_modal_placeholders={}), RequestOutput(request_id=1, prompt=None, prompt_token_ids=[785, 6722, 315, 9625, 374], encoder_prompt=None, encoder_prompt_token_ids=None, prompt_logprobs=None, outputs=[CompletionOutput(index=1, text=" _______ ____.\nIn Bordeaux, Switzerland there's a train company named _______.\n\nWhich French town is entirely surrounded by water, with no", token_ids=[32671, 62, 30743, 624, 641, 83957, 11, 29121, 1052, 594, 264, 5426, 2813, 6941, 32671, 62, 382, 23085, 8585, 6290, 374, 11368, 22865, 553, 3015, 11, 448, 902], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None)], finished=False, metrics=None, lora_request=None, num_cached_tokens=None, multi_modal_placeholders={}), RequestOutput(request_id=1, prompt=None, prompt_token_ids=[785, 6722, 315, 9625, 374], encoder_prompt=None, encoder_prompt_token_ids=None, prompt_logprobs=None, outputs=[CompletionOutput(index=1, text=" _______ ____.\nIn Bordeaux, Switzerland there's a train company named _______.\n\nWhich French town is entirely surrounded by water, with no permanent", token_ids=[32671, 62, 30743, 624, 641, 83957, 11, 29121, 1052, 594, 264, 5426, 2813, 6941, 32671, 62, 382, 23085, 8585, 6290, 374, 11368, 22865, 553, 3015, 11, 448, 902, 15330], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None)], finished=False, metrics=None, lora_request=None, num_cached_tokens=None, multi_modal_placeholders={}), RequestOutput(request_id=1, prompt=None, prompt_token_ids=[785, 6722, 315, 9625, 374], encoder_prompt=None, encoder_prompt_token_ids=None, prompt_logprobs=None, outputs=[CompletionOutput(index=1, text=" _______ ____.\nIn Bordeaux, Switzerland there's a train company named _______.\n\nWhich French town is entirely surrounded by water, with no permanent structures", token_ids=[32671, 62, 30743, 624, 641, 83957, 11, 29121, 1052, 594, 264, 5426, 2813, 6941, 32671, 62, 382, 23085, 8585, 6290, 374, 11368, 22865, 553, 3015, 11, 448, 902, 15330, 14389], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None)], finished=False, metrics=None, lora_request=None, num_cached_tokens=None, multi_modal_placeholders={}), RequestOutput(request_id=1, prompt=None, prompt_token_ids=[785, 6722, 315, 9625, 374], encoder_prompt=None, encoder_prompt_token_ids=None, prompt_logprobs=None, outputs=[CompletionOutput(index=1, text=" _______ ____.\nIn Bordeaux, Switzerland there's a train company named _______.\n\nWhich French town is entirely surrounded by water, with no permanent structures?", token_ids=[32671, 62, 30743, 624, 641, 83957, 11, 29121, 1052, 594, 264, 5426, 2813, 6941, 32671, 62, 382, 23085, 8585, 6290, 374, 11368, 22865, 553, 3015, 11, 448, 902, 15330, 14389, 30], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None)], finished=False, metrics=None, lora_request=None, num_cached_tokens=None, multi_modal_placeholders={}), RequestOutput(request_id=1, prompt=None, prompt_token_ids=[785, 6722, 315, 9625, 374], encoder_prompt=None, encoder_prompt_token_ids=None, prompt_logprobs=None, outputs=[CompletionOutput(index=1, text=" _______ ____.\nIn Bordeaux, Switzerland there's a train company named _______.\n\nWhich French town is entirely surrounded by water, with no permanent structures? Co", token_ids=[32671, 62, 30743, 624, 641, 83957, 11, 29121, 1052, 594, 264, 5426, 2813, 6941, 32671, 62, 382, 23085, 8585, 6290, 374, 11368, 22865, 553, 3015, 11, 448, 902, 15330, 14389, 30, 3539], cumulative_logprob=None, logprobs=None, finish_reason=length, stop_reason=None)], finished=True, metrics=None, lora_request=None, num_cached_tokens=None, multi_modal_placeholders={})]

After (See how we get both index=0 and index=1 all the way to the end):

[RequestOutput(request_id=1, prompt='The capital of France is', prompt_token_ids=[785, 6722, 315, 9625, 374], encoder_prompt=None, encoder_prompt_token_ids=None, prompt_logprobs=None, outputs=[CompletionOutput(index=0, text=' ______', token_ids=[32671], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None)], finished=False, metrics=None, lora_request=None, num_cached_tokens=None, multi_modal_placeholders={}), RequestOutput(request_id=1, prompt='The capital of France is', prompt_token_ids=[785, 6722, 315, 9625, 374], encoder_prompt=None, encoder_prompt_token_ids=None, prompt_logprobs=None, outputs=[CompletionOutput(index=0, text=' ______.', token_ids=[32671, 13], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None), CompletionOutput(index=1, text=' Paris', token_ids=[12095], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None)], finished=False, metrics=None, lora_request=None, num_cached_tokens=None, multi_modal_placeholders={}), RequestOutput(request_id=1, prompt='The capital of France is', prompt_token_ids=[785, 6722, 315, 9625, 374], encoder_prompt=None, encoder_prompt_token_ids=None, prompt_logprobs=None, outputs=[CompletionOutput(index=0, text=' ______. __', token_ids=[32671, 13, 1304], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None), CompletionOutput(index=1, text=' Paris.', token_ids=[12095, 13], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None)], finished=False, metrics=None, lora_request=None, num_cached_tokens=None, multi_modal_placeholders={}), RequestOutput(request_id=1, prompt='The capital of France is', prompt_token_ids=[785, 6722, 315, 9625, 374], encoder_prompt=None, encoder_prompt_token_ids=None, prompt_logprobs=None, outputs=[CompletionOutput(index=0, text=' ______. ____\n', token_ids=[32671, 13, 1304, 12148], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None), CompletionOutput(index=1, text=' Paris. But', token_ids=[12095, 13, 1988], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None)], finished=False, metrics=None, lora_request=None, num_cached_tokens=None, multi_modal_placeholders={}), RequestOutput(request_id=1, prompt='The capital of France is', prompt_token_ids=[785, 6722, 315, 9625, 374], encoder_prompt=None, encoder_prompt_token_ids=None, prompt_logprobs=None, outputs=[CompletionOutput(index=0, text=' ______. ____\nR', token_ids=[32671, 13, 1304, 12148, 49], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None), CompletionOutput(index=1, text=' Paris. But Paris', token_ids=[12095, 13, 1988, 12095], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None)], finished=False, metrics=None, lora_request=None, num_cached_tokens=None, multi_modal_placeholders={}), RequestOutput(request_id=1, prompt='The capital of France is', prompt_token_ids=[785, 6722, 315, 9625, 374], encoder_prompt=None, encoder_prompt_token_ids=None, prompt_logprobs=None, outputs=[CompletionOutput(index=0, text=' ______. ____\nRome', token_ids=[32671, 13, 1304, 12148, 49, 635], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None), CompletionOutput(index=1, text=' Paris. But Paris doesn', token_ids=[12095, 13, 1988, 12095, 3171], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None)], finished=False, metrics=None, lora_request=None, num_cached_tokens=None, multi_modal_placeholders={}), RequestOutput(request_id=1, prompt='The capital of France is', prompt_token_ids=[785, 6722, 315, 9625, 374], encoder_prompt=None, encoder_prompt_token_ids=None, prompt_logprobs=None, outputs=[CompletionOutput(index=0, text=' ______. ____\nRome\n\n', token_ids=[32671, 13, 1304, 12148, 49, 635, 271], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None), CompletionOutput(index=1, text=' Paris. But Paris doesn’t', token_ids=[12095, 13, 1988, 12095, 3171, 1405], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None)], finished=False, metrics=None, lora_request=None, num_cached_tokens=None, multi_modal_placeholders={}), RequestOutput(request_id=1, prompt='The capital of France is', prompt_token_ids=[785, 6722, 315, 9625, 374], encoder_prompt=None, encoder_prompt_token_ids=None, prompt_logprobs=None, outputs=[CompletionOutput(index=0, text=' ______. ____\nRome\n\nChoose', token_ids=[32671, 13, 1304, 12148, 49, 635, 271, 24051], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None), CompletionOutput(index=1, text=' Paris. But Paris doesn’t like', token_ids=[12095, 13, 1988, 12095, 3171, 1405, 1075], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None)], finished=False, metrics=None, lora_request=None, num_cached_tokens=None, multi_modal_placeholders={}), RequestOutput(request_id=1, prompt='The capital of France is', prompt_token_ids=[785, 6722, 315, 9625, 374], encoder_prompt=None, encoder_prompt_token_ids=None, prompt_logprobs=None, outputs=[CompletionOutput(index=0, text=' ______. ____\nRome\n\nChoose your', token_ids=[32671, 13, 1304, 12148, 49, 635, 271, 24051, 697], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None), CompletionOutput(index=1, text=' Paris. But Paris doesn’t like to', token_ids=[12095, 13, 1988, 12095, 3171, 1405, 1075, 311], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None)], finished=False, metrics=None, lora_request=None, num_cached_tokens=None, multi_modal_placeholders={}), RequestOutput(request_id=1, prompt='The capital of France is', prompt_token_ids=[785, 6722, 315, 9625, 374], encoder_prompt=None, encoder_prompt_token_ids=None, prompt_logprobs=None, outputs=[CompletionOutput(index=0, text=' ______. ____\nRome\n\nChoose your answer', token_ids=[32671, 13, 1304, 12148, 49, 635, 271, 24051, 697, 4226], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None), CompletionOutput(index=1, text=' Paris. But Paris doesn’t like to call', token_ids=[12095, 13, 1988, 12095, 3171, 1405, 1075, 311, 1618], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None)], finished=False, metrics=None, lora_request=None, num_cached_tokens=None, multi_modal_placeholders={}), RequestOutput(request_id=1, prompt='The capital of France is', prompt_token_ids=[785, 6722, 315, 9625, 374], encoder_prompt=None, encoder_prompt_token_ids=None, prompt_logprobs=None, outputs=[CompletionOutput(index=0, text=' ______. ____\nRome\n\nChoose your answer from', token_ids=[32671, 13, 1304, 12148, 49, 635, 271, 24051, 697, 4226, 504], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None), CompletionOutput(index=1, text=' Paris. But Paris doesn’t like to call itself', token_ids=[12095, 13, 1988, 12095, 3171, 1405, 1075, 311, 1618, 5086], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None)], finished=False, metrics=None, lora_request=None, num_cached_tokens=None, multi_modal_placeholders={}), RequestOutput(request_id=1, prompt='The capital of France is', prompt_token_ids=[785, 6722, 315, 9625, 374], encoder_prompt=None, encoder_prompt_token_ids=None, prompt_logprobs=None, outputs=[CompletionOutput(index=0, text=' ______. ____\nRome\n\nChoose your answer from:\n', token_ids=[32671, 13, 1304, 12148, 49, 635, 271, 24051, 697, 4226, 504, 510], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None), CompletionOutput(index=1, text=' Paris. But Paris doesn’t like to call itself by', token_ids=[12095, 13, 1988, 12095, 3171, 1405, 1075, 311, 1618, 5086, 553], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None)], finished=False, metrics=None, lora_request=None, num_cached_tokens=None, multi_modal_placeholders={}), RequestOutput(request_id=1, prompt='The capital of France is', prompt_token_ids=[785, 6722, 315, 9625, 374], encoder_prompt=None, encoder_prompt_token_ids=None, prompt_logprobs=None, outputs=[CompletionOutput(index=0, text=' ______. ____\nRome\n\nChoose your answer from:\ni', token_ids=[32671, 13, 1304, 12148, 49, 635, 271, 24051, 697, 4226, 504, 510, 72], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None), CompletionOutput(index=1, text=' Paris. But Paris doesn’t like to call itself by its', token_ids=[12095, 13, 1988, 12095, 3171, 1405, 1075, 311, 1618, 5086, 553, 1181], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None)], finished=False, metrics=None, lora_request=None, num_cached_tokens=None, multi_modal_placeholders={}), RequestOutput(request_id=1, prompt='The capital of France is', prompt_token_ids=[785, 6722, 315, 9625, 374], encoder_prompt=None, encoder_prompt_token_ids=None, prompt_logprobs=None, outputs=[CompletionOutput(index=0, text=' ______. ____\nRome\n\nChoose your answer from:\ni.', token_ids=[32671, 13, 1304, 12148, 49, 635, 271, 24051, 697, 4226, 504, 510, 72, 13], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None), CompletionOutput(index=1, text=' Paris. But Paris doesn’t like to call itself by its name', token_ids=[12095, 13, 1988, 12095, 3171, 1405, 1075, 311, 1618, 5086, 553, 1181, 829], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None)], finished=False, metrics=None, lora_request=None, num_cached_tokens=None, multi_modal_placeholders={}), RequestOutput(request_id=1, prompt='The capital of France is', prompt_token_ids=[785, 6722, 315, 9625, 374], encoder_prompt=None, encoder_prompt_token_ids=None, prompt_logprobs=None, outputs=[CompletionOutput(index=0, text=' ______. ____\nRome\n\nChoose your answer from:\ni. I', token_ids=[32671, 13, 1304, 12148, 49, 635, 271, 24051, 697, 4226, 504, 510, 72, 13, 358], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None), CompletionOutput(index=1, text=' Paris. But Paris doesn’t like to call itself by its name because', token_ids=[12095, 13, 1988, 12095, 3171, 1405, 1075, 311, 1618, 5086, 553, 1181, 829, 1576], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None)], finished=False, metrics=None, lora_request=None, num_cached_tokens=None, multi_modal_placeholders={}), RequestOutput(request_id=1, prompt='The capital of France is', prompt_token_ids=[785, 6722, 315, 9625, 374], encoder_prompt=None, encoder_prompt_token_ids=None, prompt_logprobs=None, outputs=[CompletionOutput(index=0, text=' ______. ____\nRome\n\nChoose your answer from:\ni. I know', token_ids=[32671, 13, 1304, 12148, 49, 635, 271, 24051, 697, 4226, 504, 510, 72, 13, 358, 1414], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None), CompletionOutput(index=1, text=' Paris. But Paris doesn’t like to call itself by its name because of', token_ids=[12095, 13, 1988, 12095, 3171, 1405, 1075, 311, 1618, 5086, 553, 1181, 829, 1576, 315], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None)], finished=False, metrics=None, lora_request=None, num_cached_tokens=None, multi_modal_placeholders={}), RequestOutput(request_id=1, prompt='The capital of France is', prompt_token_ids=[785, 6722, 315, 9625, 374], encoder_prompt=None, encoder_prompt_token_ids=None, prompt_logprobs=None, outputs=[CompletionOutput(index=0, text=' ______. ____\nRome\n\nChoose your answer from:\ni. I know.;\n', token_ids=[32671, 13, 1304, 12148, 49, 635, 271, 24051, 697, 4226, 504, 510, 72, 13, 358, 1414, 37403], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None), CompletionOutput(index=1, text=' Paris. But Paris doesn’t like to call itself by its name because of a', token_ids=[12095, 13, 1988, 12095, 3171, 1405, 1075, 311, 1618, 5086, 553, 1181, 829, 1576, 315, 264], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None)], finished=False, metrics=None, lora_request=None, num_cached_tokens=None, multi_modal_placeholders={}), RequestOutput(request_id=1, prompt='The capital of France is', prompt_token_ids=[785, 6722, 315, 9625, 374], encoder_prompt=None, encoder_prompt_token_ids=None, prompt_logprobs=None, outputs=[CompletionOutput(index=0, text=' ______. ____\nRome\n\nChoose your answer from:\ni. I know.;\nii', token_ids=[32671, 13, 1304, 12148, 49, 635, 271, 24051, 697, 4226, 504, 510, 72, 13, 358, 1414, 37403, 3808], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None), CompletionOutput(index=1, text=' Paris. But Paris doesn’t like to call itself by its name because of a conflict', token_ids=[12095, 13, 1988, 12095, 3171, 1405, 1075, 311, 1618, 5086, 553, 1181, 829, 1576, 315, 264, 12055], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None)], finished=False, metrics=None, lora_request=None, num_cached_tokens=None, multi_modal_placeholders={}), RequestOutput(request_id=1, prompt='The capital of France is', prompt_token_ids=[785, 6722, 315, 9625, 374], encoder_prompt=None, encoder_prompt_token_ids=None, prompt_logprobs=None, outputs=[CompletionOutput(index=0, text=' ______. ____\nRome\n\nChoose your answer from:\ni. I know.;\nii.', token_ids=[32671, 13, 1304, 12148, 49, 635, 271, 24051, 697, 4226, 504, 510, 72, 13, 358, 1414, 37403, 3808, 13], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None), CompletionOutput(index=1, text=' Paris. But Paris doesn’t like to call itself by its name because of a conflict of', token_ids=[12095, 13, 1988, 12095, 3171, 1405, 1075, 311, 1618, 5086, 553, 1181, 829, 1576, 315, 264, 12055, 315], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None)], finished=False, metrics=None, lora_request=None, num_cached_tokens=None, multi_modal_placeholders={}), RequestOutput(request_id=1, prompt='The capital of France is', prompt_token_ids=[785, 6722, 315, 9625, 374], encoder_prompt=None, encoder_prompt_token_ids=None, prompt_logprobs=None, outputs=[CompletionOutput(index=0, text=' ______. ____\nRome\n\nChoose your answer from:\ni. I know.;\nii. No', token_ids=[32671, 13, 1304, 12148, 49, 635, 271, 24051, 697, 4226, 504, 510, 72, 13, 358, 1414, 37403, 3808, 13, 2308], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None), CompletionOutput(index=1, text=' Paris. But Paris doesn’t like to call itself by its name because of a conflict of meaning', token_ids=[12095, 13, 1988, 12095, 3171, 1405, 1075, 311, 1618, 5086, 553, 1181, 829, 1576, 315, 264, 12055, 315, 7290], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None)], finished=False, metrics=None, lora_request=None, num_cached_tokens=None, multi_modal_placeholders={}), RequestOutput(request_id=1, prompt='The capital of France is', prompt_token_ids=[785, 6722, 315, 9625, 374], encoder_prompt=None, encoder_prompt_token_ids=None, prompt_logprobs=None, outputs=[CompletionOutput(index=0, text=' ______. ____\nRome\n\nChoose your answer from:\ni. I know.;\nii. No;\n', token_ids=[32671, 13, 1304, 12148, 49, 635, 271, 24051, 697, 4226, 504, 510, 72, 13, 358, 1414, 37403, 3808, 13, 2308, 280], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None), CompletionOutput(index=1, text=' Paris. But Paris doesn’t like to call itself by its name because of a conflict of meaning between', token_ids=[12095, 13, 1988, 12095, 3171, 1405, 1075, 311, 1618, 5086, 553, 1181, 829, 1576, 315, 264, 12055, 315, 7290, 1948], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None)], finished=False, metrics=None, lora_request=None, num_cached_tokens=None, multi_modal_placeholders={}), RequestOutput(request_id=1, prompt='The capital of France is', prompt_token_ids=[785, 6722, 315, 9625, 374], encoder_prompt=None, encoder_prompt_token_ids=None, prompt_logprobs=None, outputs=[CompletionOutput(index=0, text=' ______. ____\nRome\n\nChoose your answer from:\ni. I know.;\nii. No;\nIII', token_ids=[32671, 13, 1304, 12148, 49, 635, 271, 24051, 697, 4226, 504, 510, 72, 13, 358, 1414, 37403, 3808, 13, 2308, 280, 22615], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None), CompletionOutput(index=1, text=' Paris. But Paris doesn’t like to call itself by its name because of a conflict of meaning between its', token_ids=[12095, 13, 1988, 12095, 3171, 1405, 1075, 311, 1618, 5086, 553, 1181, 829, 1576, 315, 264, 12055, 315, 7290, 1948, 1181], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None)], finished=False, metrics=None, lora_request=None, num_cached_tokens=None, multi_modal_placeholders={}), RequestOutput(request_id=1, prompt='The capital of France is', prompt_token_ids=[785, 6722, 315, 9625, 374], encoder_prompt=None, encoder_prompt_token_ids=None, prompt_logprobs=None, outputs=[CompletionOutput(index=0, text=' ______. ____\nRome\n\nChoose your answer from:\ni. I know.;\nii. No;\nIII.', token_ids=[32671, 13, 1304, 12148, 49, 635, 271, 24051, 697, 4226, 504, 510, 72, 13, 358, 1414, 37403, 3808, 13, 2308, 280, 22615, 13], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None), CompletionOutput(index=1, text=' Paris. But Paris doesn’t like to call itself by its name because of a conflict of meaning between its famous', token_ids=[12095, 13, 1988, 12095, 3171, 1405, 1075, 311, 1618, 5086, 553, 1181, 829, 1576, 315, 264, 12055, 315, 7290, 1948, 1181, 11245], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None)], finished=False, metrics=None, lora_request=None, num_cached_tokens=None, multi_modal_placeholders={}), RequestOutput(request_id=1, prompt='The capital of France is', prompt_token_ids=[785, 6722, 315, 9625, 374], encoder_prompt=None, encoder_prompt_token_ids=None, prompt_logprobs=None, outputs=[CompletionOutput(index=0, text=' ______. ____\nRome\n\nChoose your answer from:\ni. I know.;\nii. No;\nIII. Yes', token_ids=[32671, 13, 1304, 12148, 49, 635, 271, 24051, 697, 4226, 504, 510, 72, 13, 358, 1414, 37403, 3808, 13, 2308, 280, 22615, 13, 7414], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None), CompletionOutput(index=1, text=' Paris. But Paris doesn’t like to call itself by its name because of a conflict of meaning between its famous architecture', token_ids=[12095, 13, 1988, 12095, 3171, 1405, 1075, 311, 1618, 5086, 553, 1181, 829, 1576, 315, 264, 12055, 315, 7290, 1948, 1181, 11245, 17646], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None)], finished=False, metrics=None, lora_request=None, num_cached_tokens=None, multi_modal_placeholders={}), RequestOutput(request_id=1, prompt='The capital of France is', prompt_token_ids=[785, 6722, 315, 9625, 374], encoder_prompt=None, encoder_prompt_token_ids=None, prompt_logprobs=None, outputs=[CompletionOutput(index=0, text=' ______. ____\nRome\n\nChoose your answer from:\ni. I know.;\nii. No;\nIII. Yes;\n', token_ids=[32671, 13, 1304, 12148, 49, 635, 271, 24051, 697, 4226, 504, 510, 72, 13, 358, 1414, 37403, 3808, 13, 2308, 280, 22615, 13, 7414, 280], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None), CompletionOutput(index=1, text=' Paris. But Paris doesn’t like to call itself by its name because of a conflict of meaning between its famous architecture and', token_ids=[12095, 13, 1988, 12095, 3171, 1405, 1075, 311, 1618, 5086, 553, 1181, 829, 1576, 315, 264, 12055, 315, 7290, 1948, 1181, 11245, 17646, 323], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None)], finished=False, metrics=None, lora_request=None, num_cached_tokens=None, multi_modal_placeholders={}), RequestOutput(request_id=1, prompt='The capital of France is', prompt_token_ids=[785, 6722, 315, 9625, 374], encoder_prompt=None, encoder_prompt_token_ids=None, prompt_logprobs=None, outputs=[CompletionOutput(index=0, text=' ______. ____\nRome\n\nChoose your answer from:\ni. I know.;\nii. No;\nIII. Yes;\nIs', token_ids=[32671, 13, 1304, 12148, 49, 635, 271, 24051, 697, 4226, 504, 510, 72, 13, 358, 1414, 37403, 3808, 13, 2308, 280, 22615, 13, 7414, 280, 3872], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None), CompletionOutput(index=1, text=' Paris. But Paris doesn’t like to call itself by its name because of a conflict of meaning between its famous architecture and the', token_ids=[12095, 13, 1988, 12095, 3171, 1405, 1075, 311, 1618, 5086, 553, 1181, 829, 1576, 315, 264, 12055, 315, 7290, 1948, 1181, 11245, 17646, 323, 279], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None)], finished=False, metrics=None, lora_request=None, num_cached_tokens=None, multi_modal_placeholders={}), RequestOutput(request_id=1, prompt='The capital of France is', prompt_token_ids=[785, 6722, 315, 9625, 374], encoder_prompt=None, encoder_prompt_token_ids=None, prompt_logprobs=None, outputs=[CompletionOutput(index=0, text=' ______. ____\nRome\n\nChoose your answer from:\ni. I know.;\nii. No;\nIII. Yes;\nIs the', token_ids=[32671, 13, 1304, 12148, 49, 635, 271, 24051, 697, 4226, 504, 510, 72, 13, 358, 1414, 37403, 3808, 13, 2308, 280, 22615, 13, 7414, 280, 3872, 279], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None), CompletionOutput(index=1, text=' Paris. But Paris doesn’t like to call itself by its name because of a conflict of meaning between its famous architecture and the seas', token_ids=[12095, 13, 1988, 12095, 3171, 1405, 1075, 311, 1618, 5086, 553, 1181, 829, 1576, 315, 264, 12055, 315, 7290, 1948, 1181, 11245, 17646, 323, 279, 51740], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None)], finished=False, metrics=None, lora_request=None, num_cached_tokens=None, multi_modal_placeholders={}), RequestOutput(request_id=1, prompt='The capital of France is', prompt_token_ids=[785, 6722, 315, 9625, 374], encoder_prompt=None, encoder_prompt_token_ids=None, prompt_logprobs=None, outputs=[CompletionOutput(index=0, text=' ______. ____\nRome\n\nChoose your answer from:\ni. I know.;\nii. No;\nIII. Yes;\nIs the following', token_ids=[32671, 13, 1304, 12148, 49, 635, 271, 24051, 697, 4226, 504, 510, 72, 13, 358, 1414, 37403, 3808, 13, 2308, 280, 22615, 13, 7414, 280, 3872, 279, 2701], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None), CompletionOutput(index=1, text=' Paris. But Paris doesn’t like to call itself by its name because of a conflict of meaning between its famous architecture and the seas around', token_ids=[12095, 13, 1988, 12095, 3171, 1405, 1075, 311, 1618, 5086, 553, 1181, 829, 1576, 315, 264, 12055, 315, 7290, 1948, 1181, 11245, 17646, 323, 279, 51740, 2163], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None)], finished=False, metrics=None, lora_request=None, num_cached_tokens=None, multi_modal_placeholders={}), RequestOutput(request_id=1, prompt='The capital of France is', prompt_token_ids=[785, 6722, 315, 9625, 374], encoder_prompt=None, encoder_prompt_token_ids=None, prompt_logprobs=None, outputs=[CompletionOutput(index=0, text=' ______. ____\nRome\n\nChoose your answer from:\ni. I know.;\nii. No;\nIII. Yes;\nIs the following a', token_ids=[32671, 13, 1304, 12148, 49, 635, 271, 24051, 697, 4226, 504, 510, 72, 13, 358, 1414, 37403, 3808, 13, 2308, 280, 22615, 13, 7414, 280, 3872, 279, 2701, 264], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None), CompletionOutput(index=1, text=' Paris. But Paris doesn’t like to call itself by its name because of a conflict of meaning between its famous architecture and the seas around it', token_ids=[12095, 13, 1988, 12095, 3171, 1405, 1075, 311, 1618, 5086, 553, 1181, 829, 1576, 315, 264, 12055, 315, 7290, 1948, 1181, 11245, 17646, 323, 279, 51740, 2163, 432], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None)], finished=False, metrics=None, lora_request=None, num_cached_tokens=None, multi_modal_placeholders={}), RequestOutput(request_id=1, prompt='The capital of France is', prompt_token_ids=[785, 6722, 315, 9625, 374], encoder_prompt=None, encoder_prompt_token_ids=None, prompt_logprobs=None, outputs=[CompletionOutput(index=0, text=' ______. ____\nRome\n\nChoose your answer from:\ni. I know.;\nii. No;\nIII. Yes;\nIs the following a factual', token_ids=[32671, 13, 1304, 12148, 49, 635, 271, 24051, 697, 4226, 504, 510, 72, 13, 358, 1414, 37403, 3808, 13, 2308, 280, 22615, 13, 7414, 280, 3872, 279, 2701, 264, 59901], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None), CompletionOutput(index=1, text=' Paris. But Paris doesn’t like to call itself by its name because of a conflict of meaning between its famous architecture and the seas around it.\n\n', token_ids=[12095, 13, 1988, 12095, 3171, 1405, 1075, 311, 1618, 5086, 553, 1181, 829, 1576, 315, 264, 12055, 315, 7290, 1948, 1181, 11245, 17646, 323, 279, 51740, 2163, 432, 382], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None)], finished=False, metrics=None, lora_request=None, num_cached_tokens=None, multi_modal_placeholders={}), RequestOutput(request_id=1, prompt='The capital of France is', prompt_token_ids=[785, 6722, 315, 9625, 374], encoder_prompt=None, encoder_prompt_token_ids=None, prompt_logprobs=None, outputs=[CompletionOutput(index=0, text=' ______. ____\nRome\n\nChoose your answer from:\ni. I know.;\nii. No;\nIII. Yes;\nIs the following a factual question', token_ids=[32671, 13, 1304, 12148, 49, 635, 271, 24051, 697, 4226, 504, 510, 72, 13, 358, 1414, 37403, 3808, 13, 2308, 280, 22615, 13, 7414, 280, 3872, 279, 2701, 264, 59901, 3405], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None), CompletionOutput(index=1, text=' Paris. But Paris doesn’t like to call itself by its name because of a conflict of meaning between its famous architecture and the seas around it.\n\n1', token_ids=[12095, 13, 1988, 12095, 3171, 1405, 1075, 311, 1618, 5086, 553, 1181, 829, 1576, 315, 264, 12055, 315, 7290, 1948, 1181, 11245, 17646, 323, 279, 51740, 2163, 432, 382, 16], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None)], finished=False, metrics=None, lora_request=None, num_cached_tokens=None, multi_modal_placeholders={}), RequestOutput(request_id=1, prompt='The capital of France is', prompt_token_ids=[785, 6722, 315, 9625, 374], encoder_prompt=None, encoder_prompt_token_ids=None, prompt_logprobs=None, outputs=[CompletionOutput(index=0, text=' ______. ____\nRome\n\nChoose your answer from:\ni. I know.;\nii. No;\nIII. Yes;\nIs the following a factual question or', token_ids=[32671, 13, 1304, 12148, 49, 635, 271, 24051, 697, 4226, 504, 510, 72, 13, 358, 1414, 37403, 3808, 13, 2308, 280, 22615, 13, 7414, 280, 3872, 279, 2701, 264, 59901, 3405, 476], cumulative_logprob=None, logprobs=None, finish_reason=length, stop_reason=None), CompletionOutput(index=1, text=' Paris. But Paris doesn’t like to call itself by its name because of a conflict of meaning between its famous architecture and the seas around it.\n\n1.', token_ids=[12095, 13, 1988, 12095, 3171, 1405, 1075, 311, 1618, 5086, 553, 1181, 829, 1576, 315, 264, 12055, 315, 7290, 1948, 1181, 11245, 17646, 323, 279, 51740, 2163, 432, 382, 16, 13], cumulative_logprob=None, logprobs=None, finish_reason=None, stop_reason=None)], finished=False, metrics=None, lora_request=None, num_cached_tokens=None, multi_modal_placeholders={}), RequestOutput(request_id=1, prompt=None, prompt_token_ids=[785, 6722, 315, 9625, 374], encoder_prompt=None, encoder_prompt_token_ids=None, prompt_logprobs=None, outputs=[CompletionOutput(index=1, text=' Paris. But Paris doesn’t like to call itself by its name because of a conflict of meaning between its famous architecture and the seas around it.\n\n1. In', token_ids=[12095, 13, 1988, 12095, 3171, 1405, 1075, 311, 1618, 5086, 553, 1181, 829, 1576, 315, 264, 12055, 315, 7290, 1948, 1181, 11245, 17646, 323, 279, 51740, 2163, 432, 382, 16, 13, 758], cumulative_logprob=None, logprobs=None, finish_reason=length, stop_reason=None)], finished=True, metrics=None, lora_request=None, num_cached_tokens=None, multi_modal_placeholders={})]

github-actions · 2025-04-18T17:33:19Z

👋 Hi! Thank you for contributing to the vLLM project.

💬 Join our developer Slack at https://slack.vllm.ai to discuss your PR in #pr-reviews, coordinate on features in #feat- channels, or join special interest groups in #sig- channels.

Just a reminder: PRs would not trigger full CI run by default. Instead, it would only run fastcheck CI which starts running only a small and essential subset of CI tests to quickly catch errors. You can run other CI tests on top of those by going to your fastcheck build on Buildkite UI (linked in the PR checks section) and unblock them. If you do not have permission to unblock, ping simon-mo or khluu to add you in our Buildkite org.

Once the PR is approved and ready to go, your PR reviewer(s) can run CI to test the changes comprehensively before merging.

To run CI, PR reviewers can either: Add ready label to the PR or enable auto-merge.

🚀

Signed-off-by: Jeffrey Li <jeffrey.dot.li@gmail.com>

njhill

Thanks for catching this @jeffrey-dot-li!

Would you be willing to also update the unit test to cover this case?

vllm/tests/v1/engine/test_output_processor.py

Line 842 in 3d3ab36

async def test_request_output_collector():

vllm/outputs.py

njhill · 2025-04-18T21:08:16Z

Thanks @jeffrey-dot-li! Could you revert the unrelated formatting changes?

Signed-off-by: Jeffrey Li <jeffrey.dot.li@gmail.com>

jeffrey-dot-li · 2025-04-21T14:51:59Z

Thanks @jeffrey-dot-li! Could you revert the unrelated formatting changes?

Fixed sorry about that thought it would be caught in precommit

…other (vllm-project#16863) Signed-off-by: Jeffrey Li <jeffrey.dot.li@gmail.com> Signed-off-by: Frieda (Jingying) Huang <jingyingfhuang@gmail.com>

…other (vllm-project#16863) Signed-off-by: Jeffrey Li <jeffrey.dot.li@gmail.com>

…other (vllm-project#16863) Signed-off-by: Jeffrey Li <jeffrey.dot.li@gmail.com> Signed-off-by: Agata Dobrzyniewicz <adobrzyniewicz@habana.ai>

…other (vllm-project#16863) Signed-off-by: Jeffrey Li <jeffrey.dot.li@gmail.com> Signed-off-by: Mu Huai <tianbowen.tbw@antgroup.com>

jeffrey-dot-li requested review from WoosukKwon, alexm-redhat, comaniac, njhill, robertgshaw2-redhat and ywang96 as code owners April 18, 2025 17:33

mergify bot added the v1 label Apr 18, 2025

jeffrey-dot-li force-pushed the upstream-main branch from 6d81806 to c9e7c0c Compare April 18, 2025 17:37

fix: fix issue with n>1 sampling on v1 requests overriding each other

640db51

Signed-off-by: Jeffrey Li <jeffrey.dot.li@gmail.com>

jeffrey-dot-li force-pushed the upstream-main branch from c9e7c0c to 640db51 Compare April 18, 2025 17:38

njhill reviewed Apr 18, 2025

View reviewed changes

vllm/outputs.py Outdated Show resolved Hide resolved

vllm/outputs.py Outdated Show resolved Hide resolved

vllm/outputs.py Outdated Show resolved Hide resolved

njhill added the bug Something isn't working label Apr 18, 2025

jeffrey-dot-li requested a review from njhill April 18, 2025 20:32

review: add tests, refactor

1f562ef

Signed-off-by: Jeffrey Li <jeffrey.dot.li@gmail.com>

jeffrey-dot-li force-pushed the upstream-main branch from 74306de to 1f562ef Compare April 21, 2025 14:51

njhill approved these changes Apr 21, 2025

View reviewed changes

njhill added the ready ONLY add when PR is ready to merge/full CI is needed label Apr 21, 2025

DarkLight1337 merged commit 0e42544 into vllm-project:main Apr 22, 2025
59 checks passed

njhill mentioned this pull request Apr 25, 2025

[Bug]: API Returns Only Single Result Despite n=8 Parameter Setting #17173

Closed

1 task

jikunshang pushed a commit to jikunshang/vllm that referenced this pull request Apr 29, 2025

[Bugfix]: fix issue with n>1 sampling on v1 requests overriding each …

9d2e118

…other (vllm-project#16863) Signed-off-by: Jeffrey Li <jeffrey.dot.li@gmail.com>

lk-chen pushed a commit to lk-chen/vllm that referenced this pull request Apr 29, 2025

[Bugfix]: fix issue with n>1 sampling on v1 requests overriding each …

705f5f1

…other (vllm-project#16863) Signed-off-by: Jeffrey Li <jeffrey.dot.li@gmail.com>

ckhordiasma mentioned this pull request May 14, 2025

nm vllm ent 0.8.5 sync red-hat-data-services/vllm#139

Merged

Zephyr271828 mentioned this pull request Dec 15, 2025

Only a single filtered_resps is logged for repeat > 1 for each sample EleutherAI/lm-evaluation-harness#1232

Open

simonucl mentioned this pull request Jan 27, 2026

feat: migrate to vllm AsyncLLMEngine with async generation support sail-sg/oat#77

Merged

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Uh oh!

[Bugfix]: fix issue with n>1 sampling on v1 requests overriding each other#16863

[Bugfix]: fix issue with n>1 sampling on v1 requests overriding each other#16863
DarkLight1337 merged 2 commits intovllm-project:mainfrom
jeffrey-dot-li:upstream-main

jeffrey-dot-li commented Apr 18, 2025 •

edited by github-actions bot

Loading

Uh oh!

github-actions bot commented Apr 18, 2025

Uh oh!

njhill left a comment •

edited

Loading

Uh oh!

Uh oh!

Uh oh!

Uh oh!

njhill commented Apr 18, 2025 •

edited

Loading

Uh oh!

jeffrey-dot-li commented Apr 21, 2025

Uh oh!

Uh oh!

Reviewers

Assignees

Labels

Projects

Milestone

Development

Uh oh!

3 participants

Uh oh!

Conversation

jeffrey-dot-li commented Apr 18, 2025 • edited by github-actions bot Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Uh oh!

github-actions bot commented Apr 18, 2025

Uh oh!

njhill left a comment • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Choose a reason for hiding this comment

Uh oh!

Uh oh!

Uh oh!

Uh oh!

njhill commented Apr 18, 2025 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Uh oh!

jeffrey-dot-li commented Apr 21, 2025

Uh oh!

Uh oh!

Reviewers

Assignees

Labels

Projects

Milestone

Development

Uh oh!

3 participants

jeffrey-dot-li commented Apr 18, 2025 •

edited by github-actions bot

Loading

njhill left a comment •

edited

Loading

njhill commented Apr 18, 2025 •

edited

Loading