fix t2i (#4163)

Co-authored-by: Yuanle Liu <[email protected]>

Author: RichardWooSJTU

fastdeploy/config.py

@@ -890,7 +890,9 @@ def __init__(self, args):
             self.kv_cache_ratio = 1.0
         else:
             self.kv_cache_ratio = 0.75
-        self.enc_dec_block_num = 0 if current_platform.is_iluvatar() or current_platform.is_maca() else 2
+        self.enc_dec_block_num = (
+            0 if current_platform.is_iluvatar() or current_platform.is_maca() else envs.FD_ENC_DEC_BLOCK_NUM
+        )
         self.prealloc_dec_block_slot_num_threshold = 12
         self.cache_dtype = "bfloat16"
         self.model_cfg = None

fastdeploy/engine/args_utils.py

@@ -704,7 +704,7 @@ def add_cli_args(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
         cache_group.add_argument(
             "--prealloc-dec-block-slot-num-threshold",
             type=int,
-            default=12,
+            default=EngineArgs.prealloc_dec_block_slot_num_threshold,
             help="Number of token slot threadshold to allocate next blocks for decoding.",
         )
 

fastdeploy/engine/request.py

@@ -304,6 +304,7 @@ def to_dict(self):
             "index": self.index,
             "send_idx": self.send_idx,
             "token_ids": self.token_ids,
+            "decode_type": self.decode_type,
             "logprob": self.logprob,
             "top_logprobs": self.top_logprobs,
             "logprobs": self.logprobs,

fastdeploy/envs.py

@@ -82,6 +82,8 @@
     "EXPORTER_OTLP_HEADERS": lambda: os.getenv("EXPORTER_OTLP_HEADERS"),
     # enable kv cache block scheduler v1 (no need for kv_cache_ratio)
     "ENABLE_V1_KVCACHE_SCHEDULER": lambda: int(os.getenv("ENABLE_V1_KVCACHE_SCHEDULER", "1")),
+    # set prealloc block num for decoder
+    "FD_ENC_DEC_BLOCK_NUM": lambda: int(os.getenv("FD_ENC_DEC_BLOCK_NUM", "2")),
     # Whether to use PLUGINS.
     "FD_PLUGINS": lambda: None if "FD_PLUGINS" not in os.environ else os.environ["FD_PLUGINS"].split(","),
     # set trace attribute job_id.

fastdeploy/scheduler/local_scheduler.py

@@ -306,6 +306,7 @@ def put_results(self, results: List[RequestOutput]):
                 if response.request_id not in self.responses:
                     self.responses[response.request_id] = [response]
                     continue
+                scheduler_logger.debug(f"append response {response.raw}")
                 self.responses[response.request_id].append(response)
             self.responses_not_empty.notify_all()