|
| 1 | +# FunASR + LangChain Integration |
| 2 | + |
| 3 | +Use FunASR as a speech-to-text tool in your LangChain agents. Since FunASR exposes an OpenAI-compatible API, integration is straightforward. |
| 4 | + |
| 5 | +## Setup |
| 6 | + |
| 7 | +```bash |
| 8 | +# Start FunASR server |
| 9 | +pip install torch torchaudio |
| 10 | +pip install funasr vllm fastapi uvicorn python-multipart |
| 11 | +funasr-server --device cuda |
| 12 | + |
| 13 | +# Install LangChain |
| 14 | +pip install langchain langchain-openai |
| 15 | +``` |
| 16 | + |
| 17 | +## As a LangChain Tool |
| 18 | + |
| 19 | +```python |
| 20 | +from langchain.tools import tool |
| 21 | +from openai import OpenAI |
| 22 | + |
| 23 | +asr_client = OpenAI(base_url="http://localhost:8000/v1", api_key="unused") |
| 24 | + |
| 25 | +@tool |
| 26 | +def speech_to_text(audio_path: str) -> str: |
| 27 | + """Transcribe an audio file to text using local FunASR. |
| 28 | + Supports wav, mp3, flac. Returns transcribed text with speaker IDs.""" |
| 29 | + result = asr_client.audio.transcriptions.create( |
| 30 | + model="fun-asr-nano", |
| 31 | + file=open(audio_path, "rb"), |
| 32 | + response_format="verbose_json" |
| 33 | + ) |
| 34 | + return result.text |
| 35 | + |
| 36 | + |
| 37 | +# Use with any LangChain agent |
| 38 | +from langchain_openai import ChatOpenAI |
| 39 | +from langchain.agents import AgentExecutor, create_tool_calling_agent |
| 40 | +from langchain_core.prompts import ChatPromptTemplate |
| 41 | + |
| 42 | +llm = ChatOpenAI(model="gpt-4o") |
| 43 | +tools = [speech_to_text] |
| 44 | +prompt = ChatPromptTemplate.from_messages([ |
| 45 | + ("system", "You are a helpful assistant that can transcribe audio files."), |
| 46 | + ("human", "{input}"), |
| 47 | + ("placeholder", "{agent_scratchpad}"), |
| 48 | +]) |
| 49 | + |
| 50 | +agent = create_tool_calling_agent(llm, tools, prompt) |
| 51 | +executor = AgentExecutor(agent=agent, tools=tools) |
| 52 | +result = executor.invoke({"input": "Please transcribe meeting.wav"}) |
| 53 | +``` |
| 54 | + |
| 55 | +## With Dify / AutoGen / CrewAI |
| 56 | + |
| 57 | +Any framework supporting OpenAI audio API connects directly: |
| 58 | + |
| 59 | +```python |
| 60 | +from openai import OpenAI |
| 61 | +client = OpenAI( |
| 62 | + base_url="http://localhost:8000/v1", # FunASR server |
| 63 | + api_key="unused" |
| 64 | +) |
| 65 | +result = client.audio.transcriptions.create( |
| 66 | + model="fun-asr-nano", |
| 67 | + file=open("audio.wav", "rb") |
| 68 | +) |
| 69 | +``` |
| 70 | + |
| 71 | +## Features |
| 72 | + |
| 73 | +- 50+ languages (Chinese dialects, English, Japanese, Korean...) |
| 74 | +- Speaker diarization (`spk=true`) |
| 75 | +- Word-level timestamps (`response_format="verbose_json"`) |
| 76 | +- Hotword boosting |
| 77 | +- 170x realtime, fully local, MIT license |
| 78 | + |
| 79 | +## Links |
| 80 | + |
| 81 | +- [FunASR GitHub](https://github.com/modelscope/FunASR) |
| 82 | +- [OpenAI API examples](../openai_api/) |
| 83 | +- [Website](https://www.funasr.com) |
0 commit comments