|
| 1 | +from typing import List, Literal, Optional, Type, Union |
| 2 | + |
| 3 | +from pydantic import ConfigDict, Field |
| 4 | + |
| 5 | +from inference.core.entities.requests.inference import LMMInferenceRequest |
| 6 | +from inference.core.env import ( |
| 7 | + HOSTED_CORE_MODEL_URL, |
| 8 | + LOCAL_INFERENCE_API_URL, |
| 9 | + WORKFLOWS_REMOTE_API_TARGET, |
| 10 | +) |
| 11 | +from inference.core.managers.base import ModelManager |
| 12 | +from inference.core.workflows.core_steps.common.entities import StepExecutionMode |
| 13 | +from inference.core.workflows.execution_engine.entities.base import ( |
| 14 | + Batch, |
| 15 | + OutputDefinition, |
| 16 | + WorkflowImageData, |
| 17 | +) |
| 18 | +from inference.core.workflows.execution_engine.entities.types import ( |
| 19 | + DICTIONARY_KIND, |
| 20 | + IMAGE_KIND, |
| 21 | + ROBOFLOW_MODEL_ID_KIND, |
| 22 | + ImageInputField, |
| 23 | + Selector, |
| 24 | +) |
| 25 | +from inference.core.workflows.prototypes.block import ( |
| 26 | + BlockResult, |
| 27 | + WorkflowBlock, |
| 28 | + WorkflowBlockManifest, |
| 29 | +) |
| 30 | +from inference_sdk import InferenceHTTPClient |
| 31 | + |
| 32 | + |
| 33 | +########################################################################## |
| 34 | +# Qwen3.5 Workflow Block Manifest |
| 35 | +########################################################################## |
| 36 | +class BlockManifest(WorkflowBlockManifest): |
| 37 | + model_config = ConfigDict( |
| 38 | + json_schema_extra={ |
| 39 | + "name": "Qwen3.5", |
| 40 | + "version": "v2", |
| 41 | + "short_description": "Run Qwen3.5 on an image.", |
| 42 | + "long_description": ( |
| 43 | + "This workflow block runs Qwen3.5—a vision language model that accepts an image " |
| 44 | + "and an optional text prompt—and returns a text answer based on a conversation template." |
| 45 | + ), |
| 46 | + "license": "Apache-2.0", |
| 47 | + "block_type": "model", |
| 48 | + "search_keywords": [ |
| 49 | + "Qwen3.5", |
| 50 | + "qwen3.5", |
| 51 | + "vision language model", |
| 52 | + "VLM", |
| 53 | + "Alibaba", |
| 54 | + ], |
| 55 | + "is_vlm_block": True, |
| 56 | + "ui_manifest": { |
| 57 | + "section": "model", |
| 58 | + "icon": "fal fa-atom", |
| 59 | + "blockPriority": 5.7, |
| 60 | + }, |
| 61 | + }, |
| 62 | + protected_namespaces=(), |
| 63 | + ) |
| 64 | + type: Literal["roboflow_core/qwen3_5vl@v2"] |
| 65 | + |
| 66 | + images: Selector(kind=[IMAGE_KIND]) = ImageInputField |
| 67 | + prompt: Optional[str] = Field( |
| 68 | + default=None, |
| 69 | + description="Optional text prompt to provide additional context to Qwen3.5. Otherwise it will just be a default one, which may affect the desired model behavior.", |
| 70 | + examples=["What is in this image?"], |
| 71 | + ) |
| 72 | + model_version: Union[ |
| 73 | + Literal["qwen3_5-0.8b", "qwen3_5-2b", "qwen3_5-4b"], |
| 74 | + Selector(kind=[ROBOFLOW_MODEL_ID_KIND]), |
| 75 | + str, |
| 76 | + ] = Field( |
| 77 | + default="qwen3_5-0.8b", |
| 78 | + description="The Qwen3.5 model to be used for inference.", |
| 79 | + examples=["qwen3_5-0.8b", "qwen3_5-2b", "qwen3_5-4b"], |
| 80 | + ) |
| 81 | + |
| 82 | + system_prompt: Optional[str] = Field( |
| 83 | + default=None, |
| 84 | + description="Optional system prompt to provide additional context to Qwen3.5.", |
| 85 | + examples=["You are a helpful assistant."], |
| 86 | + ) |
| 87 | + |
| 88 | + max_new_tokens: Optional[int] = Field( |
| 89 | + default=None, |
| 90 | + description="Maximum number of tokens to generate. If not set, the model's default will be used.", |
| 91 | + ) |
| 92 | + |
| 93 | + @classmethod |
| 94 | + def describe_outputs(cls) -> List[OutputDefinition]: |
| 95 | + return [ |
| 96 | + OutputDefinition( |
| 97 | + name="parsed_output", |
| 98 | + kind=[DICTIONARY_KIND], |
| 99 | + description="A parsed version of the output, provided as a dictionary containing the text.", |
| 100 | + ), |
| 101 | + ] |
| 102 | + |
| 103 | + @classmethod |
| 104 | + def get_parameters_accepting_batches(cls) -> List[str]: |
| 105 | + return ["images"] |
| 106 | + |
| 107 | + @classmethod |
| 108 | + def get_execution_engine_compatibility(cls) -> Optional[str]: |
| 109 | + return ">=1.3.0,<2.0.0" |
| 110 | + |
| 111 | + @classmethod |
| 112 | + def get_supported_model_variants(cls) -> Optional[List[str]]: |
| 113 | + return ["qwen3_5-0.8b", "qwen3_5-2b", "qwen3_5-4b"] |
| 114 | + |
| 115 | + |
| 116 | +########################################################################## |
| 117 | +# Qwen3.5 Workflow Block |
| 118 | +########################################################################## |
| 119 | +class Qwen35VLBlockV2(WorkflowBlock): |
| 120 | + def __init__( |
| 121 | + self, |
| 122 | + model_manager: ModelManager, |
| 123 | + api_key: Optional[str], |
| 124 | + step_execution_mode: StepExecutionMode, |
| 125 | + ): |
| 126 | + self._model_manager = model_manager |
| 127 | + self._api_key = api_key |
| 128 | + self._step_execution_mode = step_execution_mode |
| 129 | + |
| 130 | + @classmethod |
| 131 | + def get_init_parameters(cls) -> List[str]: |
| 132 | + return ["model_manager", "api_key", "step_execution_mode"] |
| 133 | + |
| 134 | + @classmethod |
| 135 | + def get_manifest(cls) -> Type[WorkflowBlockManifest]: |
| 136 | + return BlockManifest |
| 137 | + |
| 138 | + def run( |
| 139 | + self, |
| 140 | + images: Batch[WorkflowImageData], |
| 141 | + model_version: str, |
| 142 | + prompt: Optional[str], |
| 143 | + system_prompt: Optional[str], |
| 144 | + max_new_tokens: Optional[int] = None, |
| 145 | + ) -> BlockResult: |
| 146 | + if self._step_execution_mode == StepExecutionMode.LOCAL: |
| 147 | + return self.run_locally( |
| 148 | + images=images, |
| 149 | + model_version=model_version, |
| 150 | + prompt=prompt, |
| 151 | + system_prompt=system_prompt, |
| 152 | + max_new_tokens=max_new_tokens, |
| 153 | + ) |
| 154 | + elif self._step_execution_mode == StepExecutionMode.REMOTE: |
| 155 | + return self.run_remotely( |
| 156 | + images=images, |
| 157 | + model_version=model_version, |
| 158 | + prompt=prompt, |
| 159 | + system_prompt=system_prompt, |
| 160 | + max_new_tokens=max_new_tokens, |
| 161 | + ) |
| 162 | + else: |
| 163 | + raise ValueError( |
| 164 | + f"Unknown step execution mode: {self._step_execution_mode}" |
| 165 | + ) |
| 166 | + |
| 167 | + def run_remotely( |
| 168 | + self, |
| 169 | + images: Batch[WorkflowImageData], |
| 170 | + model_version: str, |
| 171 | + prompt: Optional[str], |
| 172 | + system_prompt: Optional[str], |
| 173 | + max_new_tokens: Optional[int] = None, |
| 174 | + ) -> BlockResult: |
| 175 | + api_url = ( |
| 176 | + LOCAL_INFERENCE_API_URL |
| 177 | + if WORKFLOWS_REMOTE_API_TARGET != "hosted" |
| 178 | + else HOSTED_CORE_MODEL_URL |
| 179 | + ) |
| 180 | + client = InferenceHTTPClient( |
| 181 | + api_url=api_url, |
| 182 | + api_key=self._api_key, |
| 183 | + ) |
| 184 | + if WORKFLOWS_REMOTE_API_TARGET == "hosted": |
| 185 | + client.select_api_v0() |
| 186 | + |
| 187 | + prompt = prompt or "Describe what's in this image." |
| 188 | + system_prompt = ( |
| 189 | + system_prompt |
| 190 | + or "You are a Qwen3.5 model that can answer questions about any image." |
| 191 | + ) |
| 192 | + combined_prompt = prompt + "<system_prompt>" + system_prompt |
| 193 | + |
| 194 | + predictions = [] |
| 195 | + for image in images: |
| 196 | + result = client.infer_lmm( |
| 197 | + inference_input=image.base64_image, |
| 198 | + model_id=model_version, |
| 199 | + prompt=combined_prompt, |
| 200 | + model_id_in_path=True, |
| 201 | + enable_thinking=False, |
| 202 | + max_new_tokens=max_new_tokens, |
| 203 | + ) |
| 204 | + response_text = result.get("response", result) |
| 205 | + predictions.append({"parsed_output": response_text}) |
| 206 | + |
| 207 | + return predictions |
| 208 | + |
| 209 | + def run_locally( |
| 210 | + self, |
| 211 | + images: Batch[WorkflowImageData], |
| 212 | + model_version: str, |
| 213 | + prompt: Optional[str], |
| 214 | + system_prompt: Optional[str], |
| 215 | + max_new_tokens: Optional[int] = None, |
| 216 | + ) -> BlockResult: |
| 217 | + inference_images = [ |
| 218 | + i.to_inference_format(numpy_preferred=False) for i in images |
| 219 | + ] |
| 220 | + prompt = prompt or "Describe what's in this image." |
| 221 | + system_prompt = system_prompt or "You are a helpful assistant." |
| 222 | + prompts = [prompt + "<system_prompt>" + system_prompt] * len(inference_images) |
| 223 | + self._model_manager.add_model(model_id=model_version, api_key=self._api_key) |
| 224 | + |
| 225 | + predictions = [] |
| 226 | + for image, single_prompt in zip(inference_images, prompts): |
| 227 | + request_kwargs = dict( |
| 228 | + api_key=self._api_key, |
| 229 | + model_id=model_version, |
| 230 | + image=image, |
| 231 | + source="workflow-execution", |
| 232 | + prompt=single_prompt, |
| 233 | + enable_thinking=False, |
| 234 | + ) |
| 235 | + if max_new_tokens is not None: |
| 236 | + request_kwargs["max_new_tokens"] = max_new_tokens |
| 237 | + request = LMMInferenceRequest(**request_kwargs) |
| 238 | + prediction = self._model_manager.infer_from_request_sync( |
| 239 | + model_id=model_version, request=request |
| 240 | + ) |
| 241 | + response_text = prediction.response |
| 242 | + predictions.append({"parsed_output": response_text}) |
| 243 | + return predictions |
0 commit comments