[bug] 'CompletionOutput' object has no attribute 'tool_call_content'

[weedge]

  File "/root/offline_inference.py", line 347, in llm_engine_generate
    for result in llm_engine.generate(prompts, True):
  File "/usr/local/lib/python3.10/site-packages/fastdeploy/engine/engine.py", line 1124, in generate
    processed = self.data_processor.process_response(result)
  File "/usr/local/lib/python3.10/site-packages/fastdeploy/input/ernie_processor.py", line 218, in process_response
    response_dict.outputs.tool_call_content == []:
AttributeError: 'CompletionOutput' object has no attribute 'tool_call_content'

from self.scheduler.get_results() touch bug result: (text=None, token_ids=[100282])

18ae43ca-e9d9-4e26-8c2d-4844f6aedd14 RequestOutput(request_id=18ae43ca-e9d9-4e26-8c2d-4844f6aedd14, prompt=None, prompt_token_ids=None, outputs=CompletionOutput(index=0, send_idx=350, text=None, token_ids=[100282], draft_token_ids=[], reasoning_content=None, metrics=RequestMetrics(arrival_time=1752141460.6052523, inference_start_time=None, first_token_time=None, time_in_queue=None, preprocess_cost_time=None, model_forward_time=None, model_execute_time=None, request_start_time=1752141455.6731944), num_cached_tokens=0)

[LiqinruiG]

Thank you for your feedback. This is indeed a bug. It will be triggered when both the model's thought process content and the response content are empty. Could you please provide the model you are using and the specific case that triggered the bug?

[weedge]

extend LLMEngine impl _get_generated_tokens method, want use LLMEngine generate method to stream generate token, case like this code:

    import io
    import os
    import requests
    from PIL import Image

    from fastdeploy.entrypoints.llm import LLM
    from fastdeploy.engine.sampling_params import SamplingParams
    from fastdeploy.input.ernie_tokenizer import ErnieBotTokenizer
    from fastdeploy.utils import FlexibleArgumentParser, api_server_logger, is_port_available
    from fastdeploy.engine.args_utils import EngineArgs
    from fastdeploy.utils import llm_logger, retrive_model_from_server
    from fastdeploy.engine.engine import LLMEngine
    from fastdeploy.engine.request import RequestOutput

    class LLMEngineMonkey(LLMEngine):
        def _get_generated_tokens(self, request_id) -> Generator[RequestOutput, None, None]:
            """
            Get generated tokens for a specific request ID.
            This is a generator function that yields results until the generation is complete.

            Args:
                request_id (str): The ID of the request to get tokens for.

            Yields:
                RequestOutput: The generated tokens for the request.
            """
            finished = False
            while not finished and self.running:
                try:
                    results = self.scheduler.get_results()
                    if request_id in results:
                        contents = results[request_id]
                        for result in contents:
                            print(request_id, result)
                            yield result
                            if result.finished:
                                finished = True
                                break
                    if not finished:
                        time.sleep(0.001)  # Small sleep to avoid busy waiting
                except Exception as e:
                    llm_logger.error(
                        f"Error in _get_generated_tokens: {e}, {str(traceback.format_exc())}"
                    )
                    break

    LLM_MODEL = os.getenv("LLM_MODEL")
    PATH = os.path.join(HF_MODEL_DIR, LLM_MODEL)
    llm_engine = LLMEngineMonkey.from_engine_args(
        EngineArgs(
            model=PATH,
            tensor_parallel_size=int(os.getenv("TP", 1)),
            quantization=os.getenv("QUANTIZATION", "wint4"),
            max_model_len=32768,
            enable_mm=True,
            limit_mm_per_prompt={"image": 100},
            reasoning_parser="ernie-45-vl",
        )
    )
    if not llm_engine.start():
        api_server_logger.error("Failed to initialize FastDeploy LLM engine, service exit now!")
        return
    api_server_logger.info(f"FastDeploy LLM engine initialized!")

    tokenizer = ErnieBotTokenizer.from_pretrained(PATH)
    messages = [
        {
            "role": "user",
            "content": [
                {
                    "type": "image_url",
                    "image_url": {
                        "url": "https://paddlenlp.bj.bcebos.com/datasets/paddlemix/demo_images/example2.jpg"
                    },
                },
                {"type": "text", "text": "这张图片的内容是什么"},
            ],
        }
    ]
    prompt = tokenizer.apply_chat_template(messages, tokenize=False)
    images, videos = [], []
    for message in messages:
        content = message["content"]
        if not isinstance(content, list):
            continue
        for part in content:
            if part["type"] == "image_url":
                url = part["image_url"]["url"]
                image_bytes = requests.get(url).content
                img = Image.open(io.BytesIO(image_bytes))
                images.append(img)
            elif part["type"] == "video_url":
                url = part["video_url"]["url"]
                video_bytes = requests.get(url).content
                videos.append({"video": video_bytes, "max_frames": 30})

    prompts = {"prompt": prompt, "multimodal_data": {"image": images, "video": videos}}
    print(prompts)
    for result in llm_engine.generate(prompts, True):
        print(result)

[weedge]

skip use llm_engine generate method, just use _get_generated_tokens to get token_id , use ErnieBotTokenizer to decode, this code is OK!

    import io
    import os
    import requests
    from PIL import Image

    from fastdeploy.entrypoints.llm import LLM
    from fastdeploy.engine.sampling_params import SamplingParams
    from fastdeploy.input.ernie_tokenizer import ErnieBotTokenizer
    from fastdeploy.utils import FlexibleArgumentParser, api_server_logger, is_port_available
    from fastdeploy.engine.args_utils import EngineArgs
    from fastdeploy.utils import llm_logger, retrive_model_from_server
    from fastdeploy.engine.engine import LLMEngine
    from fastdeploy.engine.request import RequestOutput

    class LLMEngineMonkey(LLMEngine):
        def _get_generated_tokens(self, request_id) -> Generator[RequestOutput, None, None]:
            """
            Get generated tokens for a specific request ID.
            This is a generator function that yields results until the generation is complete.

            Args:
                request_id (str): The ID of the request to get tokens for.

            Yields:
                RequestOutput: The generated tokens for the request.
            """
            finished = False
            while not finished and self.running:
                try:
                    results = self.scheduler.get_results()
                    if request_id in results:
                        contents = results[request_id]
                        for result in contents:
                            # print(request_id, result)
                            yield result
                            if result.finished:
                                finished = True
                                break
                    if not finished:
                        time.sleep(0.001)  # Small sleep to avoid busy waiting
                except Exception as e:
                    llm_logger.error(
                        f"Error in _get_generated_tokens: {e}, {str(traceback.format_exc())}"
                    )
                    break

    LLM_MODEL = os.getenv("LLM_MODEL")
    PATH = os.path.join(HF_MODEL_DIR, LLM_MODEL)
    llm_engine = LLMEngineMonkey.from_engine_args(
        EngineArgs(
            model=PATH,
            tensor_parallel_size=int(os.getenv("TP", 1)),
            quantization=os.getenv("QUANTIZATION", "wint4"),
            max_model_len=32768,
            enable_mm=True,
            limit_mm_per_prompt={"image": 100},
            reasoning_parser="ernie-45-vl",
        )
    )
    if not llm_engine.start():
        api_server_logger.error("Failed to initialize FastDeploy LLM engine, service exit now!")
        return
    api_server_logger.info(f"FastDeploy LLM engine initialized!")

    vocab_file_names = ["tokenizer.model", "spm.model", "ernie_token_100k.model"]
    for i in range(len(vocab_file_names)):
        if os.path.exists(os.path.join(PATH, vocab_file_names[i])):
            ErnieBotTokenizer.resource_files_names["vocab_file"] = vocab_file_names[i]
            break
    tokenizer = ErnieBotTokenizer.from_pretrained(PATH)
    messages = [
        {
            "role": "user",
            "content": [
                {
                    "type": "image_url",
                    "image_url": {
                        "url": "https://paddlenlp.bj.bcebos.com/datasets/paddlemix/demo_images/example2.jpg"
                    },
                },
                {"type": "text", "text": "这张图片的内容是什么"},
            ],
        }
    ]
    prompt = tokenizer.apply_chat_template(messages, tokenize=False)
    images, videos = [], []
    for message in messages:
        content = message["content"]
        if not isinstance(content, list):
            continue
        for part in content:
            if part["type"] == "image_url":
                url = part["image_url"]["url"]
                image_bytes = requests.get(url).content
                img = Image.open(io.BytesIO(image_bytes))
                images.append(img)
            elif part["type"] == "video_url":
                url = part["video_url"]["url"]
                video_bytes = requests.get(url).content
                videos.append({"video": video_bytes, "max_frames": 30})

    prompts = {"prompt": prompt, "multimodal_data": {"image": images, "video": videos}}
    prompts["request_id"] = str(uuid.uuid4())
    prompts["max_tokens"] = llm_engine.cfg.max_model_len
    print(f"{prompts=}")
    sampling_params = SamplingParams(temperature=0.1, max_tokens=6400)
    print(f"{sampling_params=}")
    llm_engine.add_requests(prompts, sampling_params, enable_thinking=thinking)

    for result in llm_engine._get_generated_tokens(prompts["request_id"]):
        # print(result)
        if result.outputs and result.outputs.token_ids and len(result.outputs.token_ids) > 0:
            # print(result.outputs.token_ids)
            tokens = tokenizer.decode(result.outputs.token_ids)
            print(tokens, flush=True, end="")

        if result.finished:
            print("\n")
            print(prompts["request_id"], "finished")

[weedge]

use ErnieBotTokenizer to decode tokenizer.decode , sometime have Garbled characters (Wint4 ERNIE-4.5-VL-28B-A3B-Paddle)

4. **文化背景**：此类造像常见于北魏至隋唐时期的佛教艺术，风格融合印度���陀罗与本土元素，注重神韵表达而非写实细节，体现“以形写神”的审美理念。

[LiqinruiG]

You're doing an amazing job! The offline interface really needs to support streaming output capability. For multimodal models（ERNIE-4.5-VL-28B-A3B-Paddle）, their output includes the thinking process, and dynamic parsing is required during streaming. You can refer to reasoning.ernie_vl_reasoning_parsers.

As for the solution to the Chinese garbled characters issue, check out the code below:
https://github.com/PaddlePaddle/FastDeploy/blob/develop/fastdeploy/input/ernie_processor.py#L334

[weedge]

thanks, use llm_engine.data_processor.tokenizer ErnieProcessor :: ErnieBotTokenizer

    import io
    import os
    import requests
    from PIL import Image

    from fastdeploy.entrypoints.llm import LLM
    from fastdeploy.engine.sampling_params import SamplingParams
    from fastdeploy.input.ernie_tokenizer import ErnieBotTokenizer
    from fastdeploy.utils import FlexibleArgumentParser, api_server_logger, is_port_available
    from fastdeploy.engine.args_utils import EngineArgs
    from fastdeploy.utils import llm_logger, retrive_model_from_server
    from fastdeploy.engine.engine import LLMEngine
    from fastdeploy.engine.request import RequestOutput

    class LLMEngineMonkey(LLMEngine):
        def _get_generated_tokens(self, request_id) -> Generator[RequestOutput, None, None]:
            """
            Get generated tokens for a specific request ID.
            This is a generator function that yields results until the generation is complete.

            Args:
                request_id (str): The ID of the request to get tokens for.

            Yields:
                RequestOutput: The generated tokens for the request.
            """
            finished = False
            while not finished and self.running:
                try:
                    results = self.scheduler.get_results()
                    if request_id in results:
                        contents = results[request_id]
                        for result in contents:
                            # print(request_id, result)
                            yield result
                            if result.finished:
                                finished = True
                                break
                    if not finished:
                        time.sleep(0.001)  # Small sleep to avoid busy waiting
                except Exception as e:
                    llm_logger.error(
                        f"Error in _get_generated_tokens: {e}, {str(traceback.format_exc())}"
                    )
                    break

    LLM_MODEL = os.getenv("LLM_MODEL")
    PATH = os.path.join(HF_MODEL_DIR, LLM_MODEL)
    llm_engine = LLMEngineMonkey.from_engine_args(
        EngineArgs(
            model=PATH,
            tensor_parallel_size=int(os.getenv("TP", 1)),
            quantization=os.getenv("QUANTIZATION", "wint4"),
            max_model_len=32768,
            enable_mm=True,
            limit_mm_per_prompt={"image": 10},
            reasoning_parser="ernie-45-vl",
        )
    )
    if not llm_engine.start():
        api_server_logger.error("Failed to initialize FastDeploy LLM engine, service exit now!")
        return
    api_server_logger.info(f"FastDeploy LLM engine initialized!")

    messages = [
        {
            "role": "user",
            "content": [
                {
                    "type": "image_url",
                    "image_url": {
                        "url": "https://paddlenlp.bj.bcebos.com/datasets/paddlemix/demo_images/example2.jpg"
                    },
                },
                {"type": "text", "text": "这张图片的内容是什么"},
            ],
        }
    ]
    prompt = llm_engine.data_processor.tokenizer.apply_chat_template(messages, tokenize=False)
    images, videos = [], []
    for message in messages:
        content = message["content"]
        if not isinstance(content, list):
            continue
        for part in content:
            if part["type"] == "image_url":
                url = part["image_url"]["url"]
                image_bytes = requests.get(url).content
                img = Image.open(io.BytesIO(image_bytes))
                images.append(img)
            elif part["type"] == "video_url":
                url = part["video_url"]["url"]
                video_bytes = requests.get(url).content
                videos.append({"video": video_bytes, "max_frames": 30})

    prompts = {"prompt": prompt, "multimodal_data": {"image": images, "video": videos}}
    prompts["request_id"] = str(uuid.uuid4())
    prompts["max_tokens"] = llm_engine.cfg.max_model_len
    print(f"{prompts=}")
    sampling_params = SamplingParams(temperature=0.1, max_tokens=6400)
    print(f"{sampling_params=} {thinking=}")
    # https://paddlepaddle.github.io/FastDeploy/offline_inference/#text-completion-interface-llmgenerate
    # The generate interface does not currently support passing parameters to control the thinking function (on/off). It always uses the model's default parameters.
    llm_engine.add_requests(prompts, sampling_params, enable_thinking=True)

    start = perf_counter()
    times = []
    decode_texts_1 = ""
    decode_texts_thinking_2 = ""
    decode_texts_answer_2 = ""

    for item in llm_engine._get_generated_tokens(prompts["request_id"]):
        times.append(perf_counter() - start)
        result = item.to_dict()
        #print(result)
        if (
            result.get("outputs")
            and result.get("outputs").get("token_ids")
            and len(result.get("outputs").get("token_ids")) > 0
        ):
            tokens = llm_engine.data_processor.tokenizer.decode(
                result.get("outputs").get("token_ids")
            )
            decode_texts_1 += tokens

            # result = llm_engine.data_processor.process_response(result)
            result = llm_engine.data_processor.process_response_dict_streaming(
                result, enable_thinking=True
            )
            # print(result)

            if result.get("outputs").get("reasoning_content"):
                # print(result.get("outputs").get("reasoning_content"), flush=True, end="")
                decode_texts_thinking_2 += result.get("outputs").get("reasoning_content")

            if result.get("outputs").get("text"):
                # print(result.get("outputs").get("text"), flush=True, end="")
                decode_texts_answer_2 += result.get("outputs").get("text")

        if result.get("finished") is True:
            print(f"{decode_texts_1=}")
            print("\n============\n")
            print(f"{decode_texts_thinking_2=}")
            print(f"{decode_texts_answer_2=}")
            print("\n")
            print(prompts["request_id"], "finished")

        start = perf_counter()
    print(f"TTFT: {times[0]:.4f}s total time: {sum(times):.4f}s")```

[weedge]

use replace("<think>", "<think>\n</think>\n") can close thinking mode (like qwen3)

prompts["prompt"] = prompts["prompt"].replace("<think>", "<think>\n</think>\n")