Skip to content

feat: add LiteLLM as AI gateway provider #130

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
RheagalFire wants to merge 1 commit into
base: main
Choose a base branch
from
Open

feat: add LiteLLM as AI gateway provider #130

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 35 additions & 0 deletions config.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,9 @@
"clientConfig": {
"compatibility": "strict"
}
},
"litellm": {
"createClient": "createOpenAI"
}
},
"models": {
Expand Down Expand Up @@ -100,6 +103,38 @@
"maxTokens": 16000
}
}
},
"litellm": {
"default": {
"model": "openai/gpt-4o-mini",
"temperature": 0,
"maxTokens": 8000
},
"tools": {
"coder": {
"temperature": 0.7
},
"researchPlanner": {},
"evaluator": {},
"errorAnalyzer": {},
"queryRewriter": {
"temperature": 0.1
},
"serpCluster": {},
"agent": {
"temperature": 0.7
},
"agentBeastMode": {
"temperature": 0.7
},
"fallback": {
"temperature": 0
},
"finalizer": {},
"reducer": {
"maxTokens": 16000
}
}
}
}
}
13 changes: 11 additions & 2 deletions src/config.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import { logInfo, logError, logDebug, logWarning } from './logging';
dotenv.config();

// Types
export type LLMProvider = 'openai' | 'gemini' | 'vertex';
export type LLMProvider = 'openai' | 'gemini' | 'vertex' | 'litellm';
export type ToolName = keyof typeof configJson.models.gemini.tools;

// Type definitions for our config structure
Expand Down Expand Up @@ -42,6 +42,8 @@ if (env.https_proxy) {
export const OPENAI_BASE_URL = env.OPENAI_BASE_URL;
export const GEMINI_API_KEY = env.GEMINI_API_KEY;
export const OPENAI_API_KEY = env.OPENAI_API_KEY;
export const LITELLM_API_KEY = process.env.LITELLM_API_KEY || '';
export const LITELLM_BASE_URL = process.env.LITELLM_BASE_URL || 'http://localhost:4000/v1';
export const JINA_API_KEY = env.JINA_API_KEY;
export const BRAVE_API_KEY = env.BRAVE_API_KEY;
export const SERPER_API_KEY = env.SERPER_API_KEY;
Expand All @@ -58,7 +60,7 @@ export const LLM_PROVIDER: LLMProvider = (() => {
})();

function isValidProvider(provider: string): provider is LLMProvider {
return provider === 'openai' || provider === 'gemini' || provider === 'vertex';
return provider === 'openai' || provider === 'gemini' || provider === 'vertex' || provider === 'litellm';
}

interface ToolConfig {
Expand Down Expand Up @@ -112,6 +114,13 @@ export function getModel(toolName: ToolName) {
return createOpenAI(opt)(config.model);
}

if (LLM_PROVIDER === 'litellm') {
return createOpenAI({
apiKey: LITELLM_API_KEY || 'sk-1234',
baseURL: LITELLM_BASE_URL,
})(config.model);
}

if (LLM_PROVIDER === 'vertex') {
const createVertex = require('@ai-sdk/google-vertex').createVertex;
return createVertex({ project: process.env.GCLOUD_PROJECT, ...providerConfig?.clientConfig })(config.model);
Expand Down