|
1 | 1 | import os |
2 | | -from collections.abc import Iterable, Sequence |
| 2 | +from collections.abc import Iterable |
3 | 3 | from multiprocessing import get_all_start_methods |
4 | 4 | from typing import Any |
5 | 5 |
|
|
10 | 10 | EmbeddingWorker, |
11 | 11 | OnnxModel, |
12 | 12 | OnnxOutputContext, |
13 | | - OnnxProvider, |
14 | 13 | ) |
15 | 14 | from qwen3_embed.common.types import Device, NumpyArray |
16 | 15 | from qwen3_embed.common.utils import iter_batch |
@@ -79,61 +78,83 @@ def _rerank_pairs( |
79 | 78 | pairs: Iterable[tuple[str, str]], |
80 | 79 | batch_size: int, |
81 | 80 | parallel: int | None = None, |
82 | | - providers: Sequence[OnnxProvider] | None = None, |
83 | | - cuda: bool | Device = Device.AUTO, |
84 | | - device_ids: list[int] | None = None, |
85 | | - local_files_only: bool = False, |
86 | | - specific_model_path: str | None = None, |
87 | | - extra_session_options: dict[str, Any] | None = None, |
88 | 81 | **kwargs: Any, |
89 | 82 | ) -> Iterable[float]: |
90 | | - is_small = False |
91 | | - |
92 | | - if isinstance(pairs, tuple): |
93 | | - pairs = [pairs] # type: ignore[invalid-assignment] |
94 | | - is_small = True |
95 | | - |
96 | | - if isinstance(pairs, list) and len(pairs) < batch_size: |
97 | | - is_small = True |
| 83 | + pairs, is_small = self._prepare_pairs(pairs, batch_size) |
98 | 84 |
|
99 | 85 | if parallel is None or is_small: |
100 | 86 | if not hasattr(self, "model") or self.model is None: |
101 | 87 | self.load_onnx_model() |
102 | 88 | for batch in iter_batch(pairs, batch_size): |
103 | 89 | yield from self._post_process_onnx_output(self.onnx_embed_pairs(batch, **kwargs)) |
104 | 90 | else: |
105 | | - if parallel == 0: |
106 | | - parallel = os.cpu_count() |
107 | | - |
108 | | - start_method = "forkserver" if "forkserver" in get_all_start_methods() else "spawn" |
109 | | - params = { |
110 | | - "model_name": model_name, |
111 | | - "cache_dir": cache_dir, |
112 | | - "providers": providers, |
113 | | - "local_files_only": local_files_only, |
114 | | - "specific_model_path": specific_model_path, |
| 91 | + yield from self._rerank_pairs_parallel( |
| 92 | + model_name=model_name, |
| 93 | + cache_dir=cache_dir, |
| 94 | + pairs=pairs, |
| 95 | + batch_size=batch_size, |
| 96 | + parallel=parallel, |
115 | 97 | **kwargs, |
116 | | - } |
117 | | - |
118 | | - if extra_session_options is not None: |
119 | | - params.update(extra_session_options) |
120 | | - |
121 | | - # Carry per-instance worker state (e.g. a runtime custom-model |
122 | | - # registry) into the spawned workers, which start with a fresh |
123 | | - # interpreter + empty registry. Mirrors the embedding pool path. |
124 | | - params.update(self._extra_worker_params()) |
125 | | - |
126 | | - pool = ParallelWorkerPool( |
127 | | - worker=self._get_worker_class(), |
128 | | - config=PoolConfig( |
129 | | - num_workers=parallel or 1, |
130 | | - cuda=cuda, |
131 | | - device_ids=device_ids, |
132 | | - start_method=start_method, |
133 | | - ), |
134 | 98 | ) |
135 | | - for batch in pool.ordered_map(iter_batch(pairs, batch_size), **params): |
136 | | - yield from self._post_process_onnx_output(batch) |
| 99 | + |
| 100 | + def _prepare_pairs( |
| 101 | + self, pairs: Iterable[tuple[str, str]], batch_size: int |
| 102 | + ) -> tuple[Iterable[tuple[str, str]], bool]: |
| 103 | + is_small = False |
| 104 | + |
| 105 | + if isinstance(pairs, tuple): |
| 106 | + pairs = [pairs] # type: ignore[invalid-assignment] |
| 107 | + is_small = True |
| 108 | + |
| 109 | + if isinstance(pairs, list) and len(pairs) < batch_size: |
| 110 | + is_small = True |
| 111 | + |
| 112 | + return pairs, is_small |
| 113 | + |
| 114 | + def _rerank_pairs_parallel( |
| 115 | + self, |
| 116 | + model_name: str, |
| 117 | + cache_dir: str, |
| 118 | + pairs: Iterable[tuple[str, str]], |
| 119 | + batch_size: int, |
| 120 | + parallel: int, |
| 121 | + **kwargs: Any, |
| 122 | + ) -> Iterable[float]: |
| 123 | + if parallel == 0: |
| 124 | + parallel = os.cpu_count() or 1 |
| 125 | + |
| 126 | + start_method = "forkserver" if "forkserver" in get_all_start_methods() else "spawn" |
| 127 | + |
| 128 | + # Extract parallel-specific arguments from kwargs |
| 129 | + cuda = kwargs.pop("cuda", Device.AUTO) |
| 130 | + device_ids = kwargs.pop("device_ids", None) |
| 131 | + extra_session_options = kwargs.pop("extra_session_options", None) |
| 132 | + |
| 133 | + params = { |
| 134 | + "model_name": model_name, |
| 135 | + "cache_dir": cache_dir, |
| 136 | + **kwargs, |
| 137 | + } |
| 138 | + |
| 139 | + if extra_session_options is not None: |
| 140 | + params.update(extra_session_options) |
| 141 | + |
| 142 | + # Carry per-instance worker state (e.g. a runtime custom-model |
| 143 | + # registry) into the spawned workers, which start with a fresh |
| 144 | + # interpreter + empty registry. Mirrors the embedding pool path. |
| 145 | + params.update(self._extra_worker_params()) |
| 146 | + |
| 147 | + pool = ParallelWorkerPool( |
| 148 | + worker=self._get_worker_class(), |
| 149 | + config=PoolConfig( |
| 150 | + num_workers=parallel, |
| 151 | + cuda=cuda, |
| 152 | + device_ids=device_ids, |
| 153 | + start_method=start_method, |
| 154 | + ), |
| 155 | + ) |
| 156 | + for batch in pool.ordered_map(iter_batch(pairs, batch_size), **params): |
| 157 | + yield from self._post_process_onnx_output(batch) |
137 | 158 |
|
138 | 159 | def _post_process_onnx_output( |
139 | 160 | self, output: OnnxOutputContext, **kwargs: Any |
|
0 commit comments