[opt]opti-qwen2-vl-vit (#1004)

Co-authored-by: hiworldwzj <[email protected]>
Co-authored-by: none <none>
Co-authored-by: liujiacheng <[email protected]>

Author: 3 people

lightllm/models/qwen2_5_vl/qwen2_5_visual.py

@@ -31,6 +31,10 @@ class QWen2VLTokenizer(BaseMultiModalTokenizer):
     def __init__(self, tokenizer=None, image_processor=None, **kwargs):
         super().__init__(tokenizer)
         self.image_processor = image_processor
+        self.min_pixel = self.image_processor.min_pixels
+        self.max_pixel = self.image_processor.max_pixels
+        self.patch_size = self.image_processor.patch_size
+        self.merge_size = self.image_processor.merge_size
         self.image_start_id = kwargs["model_cfg"]["vision_start_token_id"]
         self.image_end_id = kwargs["model_cfg"]["vision_end_token_id"]
         self.image_token_id = kwargs["model_cfg"]["image_token_id"]
@@ -46,17 +50,13 @@ def init_audioitem_extral_params(
         raise NotImplementedError
 
     def get_image_token_length(self, img: ImageItem):
-        width = img.image_w
-        height = img.image_h
-        resized_height, resized_width = smart_resize(height=height, width=width)
-        self.patch_size = self.image_processor.image_processor.patch_size
-        self.merge_size = self.image_processor.image_processor.merge_size
-        grid_t = 1
+        width, height = img.image_w, img.image_h
+        resized_height, resized_width = smart_resize(
+            height=height, width=width, min_pixels=self.min_pixel, max_pixels=self.max_pixel
+        )
         grid_h, grid_w = resized_height // self.patch_size, resized_width // self.patch_size
-        merge_length = self.merge_size ** 2
-        self.token_num = (grid_t * grid_h * grid_w) // merge_length
-        self.image_length = self.token_num
-        return self.image_length
+        token_num = (grid_h * grid_w) // (self.merge_size ** 2)
+        return token_num
 
     def get_audio_token_length(self, audio: AudioItem):
         raise NotImplementedError