Skip to content

Commit 9d928fd

Browse files
mythzmythz
committed
remove expensive opus model from openrouter
1 parent 8fb988c commit 9d928fd

File tree

2 files changed

+162
-1
lines changed

2 files changed

+162
-1
lines changed

‎.github/workflows/test-providers.yml‎

Lines changed: 162 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,162 @@
1+
name: Test LLM Providers
2+
3+
on:
4+
# Run on manual trigger
5+
workflow_dispatch:
6+
# Run on schedule (daily at 00:00 UTC)
7+
schedule:
8+
- cron: '0 0 * * *'
9+
# Run on push to main branch
10+
push:
11+
branches:
12+
- main
13+
# Run on pull requests
14+
pull_request:
15+
branches:
16+
- main
17+
18+
permissions:
19+
contents: read
20+
21+
jobs:
22+
test-providers:
23+
runs-on: ubuntu-latest
24+
25+
env:
26+
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
27+
CODESTRAL_API_KEY: ${{ secrets.CODESTRAL_API_KEY }}
28+
DASHSCOPE_API_KEY: ${{ secrets.DASHSCOPE_API_KEY }}
29+
DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}
30+
GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
31+
GOOGLE_FREE_API_KEY: ${{ secrets.GOOGLE_FREE_API_KEY }}
32+
GROK_API_KEY: ${{ secrets.GROK_API_KEY }}
33+
GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }}
34+
MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }}
35+
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
36+
OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
37+
ZAI_API_KEY: ${{ secrets.ZAI_API_KEY }}
38+
39+
steps:
40+
- name: Set up Python
41+
uses: actions/setup-python@v5
42+
with:
43+
python-version: "3.x"
44+
45+
- name: Install llms-py package
46+
run: |
47+
python -m pip install --upgrade pip
48+
pip install llms-py
49+
50+
- name: Verify installation
51+
run: |
52+
llms --version
53+
54+
- name: Test providers
55+
id: test-providers
56+
continue-on-error: true
57+
run: |
58+
# Create output directory
59+
mkdir -p test-results
60+
61+
# Initialize results files
62+
RESULTS_TXT="test-results/provider-test-results.txt"
63+
RESULTS_MD="test-results/provider-test-results.md"
64+
65+
echo "LLM Provider Test Results" > $RESULTS_TXT
66+
echo "=========================" >> $RESULTS_TXT
67+
echo "Date: $(date -u '+%Y-%m-%d %H:%M:%S UTC')" >> $RESULTS_TXT
68+
echo "llms-py version: $(pip show llms-py | grep Version | cut -d' ' -f2)" >> $RESULTS_TXT
69+
echo "" >> $RESULTS_TXT
70+
71+
echo "# LLM Provider Test Results" > $RESULTS_MD
72+
echo "" >> $RESULTS_MD
73+
echo "**Date:** $(date -u '+%Y-%m-%d %H:%M:%S UTC') " >> $RESULTS_MD
74+
echo "**llms-py version:** \`$(pip show llms-py | grep Version | cut -d' ' -f2)\`" >> $RESULTS_MD
75+
echo "" >> $RESULTS_MD
76+
echo "## Test Results" >> $RESULTS_MD
77+
echo "" >> $RESULTS_MD
78+
echo "| Provider | Status | Details |" >> $RESULTS_MD
79+
echo "|----------|--------|---------|" >> $RESULTS_MD
80+
81+
# List of providers to test
82+
PROVIDERS=(
83+
"openrouter_free"
84+
"groq"
85+
"codestral"
86+
"openrouter"
87+
"google"
88+
"anthropic"
89+
"openai"
90+
"grok"
91+
"qwen"
92+
"z.ai"
93+
"qwen"
94+
"mistral"
95+
)
96+
97+
# Test each provider
98+
for provider in "${PROVIDERS[@]}"; do
99+
echo "Testing provider: $provider" >> $RESULTS_TXT
100+
echo "-----------------------------------" >> $RESULTS_TXT
101+
102+
# Run the check command and capture output
103+
if output=$(llms --check "$provider" 2>&1); then
104+
status="✅ PASS"
105+
echo "Status: PASS" >> $RESULTS_TXT
106+
echo "$output" >> $RESULTS_TXT
107+
108+
# Escape pipe characters and newlines for markdown table
109+
details=$(echo "$output" | head -n 3 | tr '\n' ' ' | sed 's/|/\\|/g' | cut -c1-100)
110+
echo "| \`$provider\` | ✅ PASS | $details |" >> $RESULTS_MD
111+
else
112+
status="❌ FAIL"
113+
echo "Status: FAIL" >> $RESULTS_TXT
114+
echo "$output" >> $RESULTS_TXT
115+
116+
# Escape pipe characters and newlines for markdown table
117+
details=$(echo "$output" | head -n 3 | tr '\n' ' ' | sed 's/|/\\|/g' | cut -c1-100)
118+
echo "| \`$provider\` | ❌ FAIL | $details |" >> $RESULTS_MD
119+
fi
120+
121+
echo "" >> $RESULTS_TXT
122+
echo "" >> $RESULTS_TXT
123+
done
124+
125+
echo "=========================" >> $RESULTS_TXT
126+
echo "Test run completed" >> $RESULTS_TXT
127+
128+
echo "" >> $RESULTS_MD
129+
echo "---" >> $RESULTS_MD
130+
echo "" >> $RESULTS_MD
131+
echo "Test run completed at $(date -u '+%Y-%m-%d %H:%M:%S UTC')" >> $RESULTS_MD
132+
133+
# Display results in console
134+
echo ""
135+
echo "=========================================="
136+
echo "Test Results Summary"
137+
echo "=========================================="
138+
cat $RESULTS_TXT
139+
echo "=========================================="
140+
141+
- name: Upload test results (TXT)
142+
uses: actions/upload-artifact@v4
143+
if: always()
144+
with:
145+
name: provider-test-results-txt
146+
path: test-results/provider-test-results.txt
147+
retention-days: 90
148+
149+
- name: Upload test results (Markdown)
150+
uses: actions/upload-artifact@v4
151+
if: always()
152+
with:
153+
name: provider-test-results-md
154+
path: test-results/provider-test-results.md
155+
retention-days: 90
156+
157+
- name: Display results summary
158+
if: always()
159+
run: |
160+
echo "## Test Results Summary" >> $GITHUB_STEP_SUMMARY
161+
cat test-results/provider-test-results.md >> $GITHUB_STEP_SUMMARY
162+

‎llms/llms.json‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -261,7 +261,6 @@
261261
"nova-micro": "amazon/nova-micro-v1",
262262
"nova-lite": "amazon/nova-lite-v1",
263263
"nova-pro": "amazon/nova-pro-v1",
264-
"claude-opus-4-1": "anthropic/claude-opus-4.1",
265264
"claude-sonnet-4-5": "anthropic/claude-sonnet-4.5",
266265
"claude-sonnet-4-0": "anthropic/claude-sonnet-4",
267266
"gpt-5": "openai/gpt-5",

0 commit comments

Comments
 (0)