|
| 1 | +# <complete_code> |
| 2 | +# <imports> |
| 3 | +import json |
| 4 | +from typing import Any |
| 5 | + |
| 6 | +from openai import OpenAI |
| 7 | + |
| 8 | +from foundry_local_sdk import Configuration, FoundryLocalManager |
| 9 | +# </imports> |
| 10 | + |
| 11 | + |
| 12 | +def get_response_text(response: Any) -> str: |
| 13 | + if isinstance(getattr(response, "output_text", None), str): |
| 14 | + return response.output_text |
| 15 | + return "".join( |
| 16 | + getattr(part, "text", "") |
| 17 | + for item in getattr(response, "output", []) or [] |
| 18 | + for part in getattr(item, "content", []) or [] |
| 19 | + if getattr(part, "type", None) == "output_text" |
| 20 | + ) |
| 21 | + |
| 22 | + |
| 23 | +# <init> |
| 24 | +# Initialize the Foundry Local SDK |
| 25 | +config = Configuration(app_name="foundry_local_samples") |
| 26 | +FoundryLocalManager.initialize(config) |
| 27 | +manager = FoundryLocalManager.instance |
| 28 | + |
| 29 | +# Download and register all execution providers. |
| 30 | +current_ep = "" |
| 31 | + |
| 32 | + |
| 33 | +def _ep_progress(ep_name: str, percent: float): |
| 34 | + global current_ep |
| 35 | + if ep_name != current_ep: |
| 36 | + if current_ep: |
| 37 | + print() |
| 38 | + current_ep = ep_name |
| 39 | + print(f"\r {ep_name:<30} {percent:5.1f}%", end="", flush=True) |
| 40 | + |
| 41 | + |
| 42 | +manager.download_and_register_eps(progress_callback=_ep_progress) |
| 43 | +if current_ep: |
| 44 | + print() |
| 45 | +# </init> |
| 46 | + |
| 47 | +# <model_setup> |
| 48 | +model_alias = "qwen2.5-0.5b" |
| 49 | +model = manager.catalog.get_model(model_alias) |
| 50 | + |
| 51 | +print(f"\nDownloading model {model_alias}...") |
| 52 | +model.download( |
| 53 | + lambda progress: print( |
| 54 | + f"\rDownloading model: {progress:.2f}%", |
| 55 | + end="", |
| 56 | + flush=True, |
| 57 | + ) |
| 58 | +) |
| 59 | +print("\nModel downloaded") |
| 60 | + |
| 61 | +print("\nLoading model...") |
| 62 | +model.load() |
| 63 | +print("Model loaded") |
| 64 | +# </model_setup> |
| 65 | + |
| 66 | +# <server_setup> |
| 67 | +print("\nStarting web service...") |
| 68 | +manager.start_web_service() |
| 69 | +base_url = manager.urls[0].rstrip("/") + "/v1" |
| 70 | +print("Web service started") |
| 71 | + |
| 72 | +# <<<<<< OPENAI SDK USAGE >>>>>> |
| 73 | +# Use the OpenAI SDK to call the local Foundry web service Responses API |
| 74 | +openai = OpenAI( |
| 75 | + base_url=base_url, |
| 76 | + api_key="notneeded", |
| 77 | +) |
| 78 | +# </server_setup> |
| 79 | + |
| 80 | +try: |
| 81 | + print("\nTesting a non-streaming Responses call...") |
| 82 | + response = openai.responses.create( |
| 83 | + model=model.id, |
| 84 | + input="Reply with one short sentence about local AI.", |
| 85 | + ) |
| 86 | + print(f"[ASSISTANT]: {get_response_text(response)}") |
| 87 | + |
| 88 | + print("\nTesting a streaming Responses call...") |
| 89 | + stream = openai.responses.create( |
| 90 | + model=model.id, |
| 91 | + input="Count from one to three.", |
| 92 | + stream=True, |
| 93 | + ) |
| 94 | + |
| 95 | + print("[ASSISTANT STREAM]: ", end="", flush=True) |
| 96 | + for event in stream: |
| 97 | + if getattr(event, "type", None) == "response.output_text.delta": |
| 98 | + print(getattr(event, "delta", ""), end="", flush=True) |
| 99 | + print() |
| 100 | + |
| 101 | + print("\nTesting Responses tool calling...") |
| 102 | + tools = [ |
| 103 | + { |
| 104 | + "type": "function", |
| 105 | + "name": "get_weather", |
| 106 | + "description": "Get the current weather. This sample always returns Seattle weather.", |
| 107 | + "parameters": { |
| 108 | + "type": "object", |
| 109 | + "properties": {}, |
| 110 | + "additionalProperties": False, |
| 111 | + }, |
| 112 | + }, |
| 113 | + ] |
| 114 | + |
| 115 | + tool_response = openai.responses.create( |
| 116 | + model=model.id, |
| 117 | + input="Use the get_weather tool and then answer with the weather.", |
| 118 | + tools=tools, |
| 119 | + tool_choice="required", |
| 120 | + store=True, |
| 121 | + ) |
| 122 | + |
| 123 | + function_call = next( |
| 124 | + (item for item in getattr(tool_response, "output", []) or [] if getattr(item, "type", None) == "function_call"), |
| 125 | + None, |
| 126 | + ) |
| 127 | + if function_call is None: |
| 128 | + raise RuntimeError("Expected the model to call get_weather.") |
| 129 | + |
| 130 | + print(f"[TOOL CALL]: {function_call.name}({function_call.arguments})") |
| 131 | + |
| 132 | + final_response = openai.responses.create( |
| 133 | + model=model.id, |
| 134 | + previous_response_id=tool_response.id, |
| 135 | + input=[ |
| 136 | + { |
| 137 | + "type": "function_call_output", |
| 138 | + "call_id": function_call.call_id, |
| 139 | + "output": json.dumps({"location": "Seattle", "weather": "72 degrees F and sunny"}), |
| 140 | + } |
| 141 | + ], |
| 142 | + tools=tools, |
| 143 | + ) |
| 144 | + |
| 145 | + print(f"[ASSISTANT FINAL]: {get_response_text(final_response)}") |
| 146 | + # <<<<<< END OPENAI SDK USAGE >>>>>> |
| 147 | +finally: |
| 148 | + # Tidy up |
| 149 | + openai.close() |
| 150 | + manager.stop_web_service() |
| 151 | + model.unload() |
| 152 | +# </complete_code> |
0 commit comments