Skip to content

Commit 02979dc

Browse files
author
Sebastian BURGIN-FIX (ext)
committed
wip
2 parents ddcc06b + be5c9d4 commit 02979dc

112 files changed

Lines changed: 2600 additions & 1602 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

.env.ci

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -14,12 +14,12 @@ LOG_LEVEL=debug
1414

1515
FLARE_KEY=
1616

17-
DB_CONNECTION=mysql
17+
DB_CONNECTION=pgsql
1818
DB_HOST=127.0.0.1
19-
DB_PORT=3306
20-
DB_DATABASE=laravel
21-
DB_USERNAME=root
22-
DB_PASSWORD=root
19+
DB_PORT=5432
20+
DB_DATABASE=testing
21+
DB_USERNAME=postgres
22+
DB_PASSWORD=postgres
2323

2424
FPH_ENABLED=true
2525
CSP_ENABLED=true

.env.example

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,4 +57,6 @@ CLOUDINARY_CLOUD_NAME=my-cloud-name
5757
CLOUDINARY_API_KEY=my-api-key
5858
CLOUDINARY_API_SECRET=my-api-secret
5959

60-
LARAVEL_DEFAULT_FATHOM_SITE_ID=WECFOADW
60+
LARAVEL_DEFAULT_FATHOM_SITE_ID=WECFOADW
61+
LITELLM_URL=https://llm.codebar.net
62+
LITELLM_MASTER_KEY=

.github/workflows/pest_coverage_pull_request.yml

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ jobs:
2121
uses: shivammathur/setup-php@v2
2222
with:
2323
php-version: ${{ env.PHP_VERSION }}
24-
extensions: dom, curl, fileinfo, mysql, libxml, mbstring, zip, pcntl, pdo, sqlite, pdo_sqlite, bcmath, soap, intl, gd, exif, iconv, imagick, xdebug
24+
extensions: dom, curl, fileinfo, pgsql, pdo_pgsql, libxml, mbstring, zip, pcntl, pdo, sqlite, pdo_sqlite, bcmath, soap, intl, gd, exif, iconv, imagick, xdebug
2525
coverage: xdebug
2626

2727
- name: Prepare the .env file
@@ -45,9 +45,9 @@ jobs:
4545

4646
- name: Setup Database
4747
run: |
48-
sudo systemctl start mysql
49-
mysql --user="root" --password="root" -e "CREATE DATABASE laravel character set UTF8mb4 collate utf8mb4_bin;"
50-
mysql --user="root" --password="root" -e "CREATE DATABASE logs character set UTF8mb4 collate utf8mb4_bin;"
48+
sudo systemctl start postgresql.service
49+
sudo -u postgres psql -c "ALTER USER postgres PASSWORD 'postgres';"
50+
sudo -u postgres createdb testing
5151
5252
- name: Generate App Key & Run Migrations
5353
run: |

.github/workflows/pest_pull_request.yml

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ jobs:
1919
uses: shivammathur/setup-php@v2
2020
with:
2121
php-version: ${{ env.PHP_VERSION }}
22-
extensions: dom, curl, fileinfo, mysql, libxml, mbstring, zip, pcntl, pdo, sqlite, pdo_sqlite, bcmath, soap, intl, gd, exif, iconv, imagick
22+
extensions: dom, curl, fileinfo, pgsql, pdo_pgsql, libxml, mbstring, zip, pcntl, pdo, sqlite, pdo_sqlite, bcmath, soap, intl, gd, exif, iconv, imagick
2323

2424
- name: Prepare the .env file
2525
run: cp .env.ci .env
@@ -42,8 +42,9 @@ jobs:
4242

4343
- name: Setup Database
4444
run: |
45-
sudo systemctl start mysql
46-
mysql --user="root" --password="root" -e "CREATE DATABASE laravel character set UTF8mb4 collate utf8mb4_bin;"
45+
sudo systemctl start postgresql.service
46+
sudo -u postgres psql -c "ALTER USER postgres PASSWORD 'postgres';"
47+
sudo -u postgres createdb testing
4748
4849
- name: Generate App Key & Run Migrations
4950
run: |
Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
<?php
2+
3+
namespace App\Actions;
4+
5+
use Carbon\CarbonImmutable;
6+
use Illuminate\Http\Client\PendingRequest;
7+
use Illuminate\Support\Collection;
8+
use Illuminate\Support\Facades\Http;
9+
10+
class FetchLlmUsageAction
11+
{
12+
/**
13+
* Fetch the per-model usage aggregates for a single day from the LiteLLM proxy.
14+
*
15+
* @return Collection<int, array{date: string, model: string, prompt_tokens: int, completion_tokens: int, total_tokens: int, requests: int, spend: float}>
16+
*/
17+
public function fetchDay(CarbonImmutable $date): Collection
18+
{
19+
$response = $this->client()->get('/spend/logs', [
20+
'start_date' => $date->toDateString(),
21+
'end_date' => $date->addDay()->toDateString(),
22+
'summarize' => 'false',
23+
])->throw();
24+
25+
$logs = $response->json();
26+
27+
return $this->parseSpendLogs($date, is_array($logs) ? $logs : []);
28+
}
29+
30+
private function client(): PendingRequest
31+
{
32+
return Http::baseUrl(config()->string('services.litellm.url'))
33+
->withToken(config()->string('services.litellm.master_key'))
34+
->acceptJson()
35+
->connectTimeout(10)
36+
->timeout(120)
37+
->retry(times: 2, sleepMilliseconds: 1000, throw: false);
38+
}
39+
40+
/**
41+
* @param array<mixed> $logs
42+
* @return Collection<int, array{date: string, model: string, prompt_tokens: int, completion_tokens: int, total_tokens: int, requests: int, spend: float}>
43+
*/
44+
private function parseSpendLogs(CarbonImmutable $date, array $logs): Collection
45+
{
46+
// The end_date bound is inclusive, so the response can contain rows of
47+
// the following day — keep only rows that started on the requested day.
48+
return collect($logs)
49+
->filter(fn (mixed $log): bool => filled($this->stringValue($log, 'model_group')))
50+
->filter(fn (mixed $log): bool => str_starts_with($this->stringValue($log, 'startTime'), $date->toDateString()))
51+
->groupBy(fn (mixed $log): string => $this->stringValue($log, 'model_group'))
52+
->map(fn (Collection $rows, string $model): array => [
53+
'date' => $date->toDateString(),
54+
'model' => $model,
55+
'prompt_tokens' => $rows->sum(fn (mixed $log): int => $this->intValue($log, 'prompt_tokens')),
56+
'completion_tokens' => $rows->sum(fn (mixed $log): int => $this->intValue($log, 'completion_tokens')),
57+
'total_tokens' => $rows->sum(fn (mixed $log): int => $this->intValue($log, 'total_tokens')),
58+
'requests' => $rows->count(),
59+
'spend' => round($rows->sum(fn (mixed $log): float => $this->floatValue($log, 'spend')), 6),
60+
])
61+
->values();
62+
}
63+
64+
private function stringValue(mixed $log, string $key): string
65+
{
66+
$value = data_get($log, $key);
67+
68+
return is_string($value) ? $value : '';
69+
}
70+
71+
private function intValue(mixed $log, string $key): int
72+
{
73+
$value = data_get($log, $key);
74+
75+
return is_numeric($value) ? (int) $value : 0;
76+
}
77+
78+
private function floatValue(mixed $log, string $key): float
79+
{
80+
$value = data_get($log, $key);
81+
82+
return is_numeric($value) ? (float) $value : 0.0;
83+
}
84+
}
Lines changed: 146 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,146 @@
1+
<?php
2+
3+
namespace App\Actions;
4+
5+
use App\Models\AiModelDailyUsage;
6+
use Carbon\CarbonImmutable;
7+
use Closure;
8+
use Illuminate\Database\Eloquent\Builder;
9+
use Illuminate\Database\Eloquent\Collection as EloquentCollection;
10+
use Illuminate\Support\Collection;
11+
use Illuminate\Support\Facades\Cache;
12+
use Illuminate\Support\Str;
13+
14+
class LlmUsageStatsAction
15+
{
16+
public const string VERSION_CACHE_KEY = 'llm_usage_version';
17+
18+
public const string OTHER_MODEL = 'other';
19+
20+
/**
21+
* @return Collection<int, string>
22+
*/
23+
public function models(): Collection
24+
{
25+
return $this->remember('models', function () {
26+
return AiModelDailyUsage::query()
27+
->whereNotNull('ai_model_id')
28+
->distinct()
29+
->orderBy('model')
30+
->get(['model'])
31+
->map(fn (AiModelDailyUsage $row): string => $row->model);
32+
});
33+
}
34+
35+
public function hasOtherModels(): bool
36+
{
37+
return $this->remember('has_other_models', function () {
38+
return AiModelDailyUsage::query()->whereNull('ai_model_id')->exists();
39+
});
40+
}
41+
42+
/**
43+
* @return Collection<int, numeric-string>
44+
*/
45+
public function years(): Collection
46+
{
47+
return $this->remember('years', function () {
48+
return AiModelDailyUsage::query()
49+
->orderBy('date')
50+
->get(['date'])
51+
->map(fn (AiModelDailyUsage $row): string => $row->date->format('Y'))
52+
->unique()
53+
->values();
54+
});
55+
}
56+
57+
/**
58+
* @return Collection<int, array{label: string, prompt_tokens: int<0, max>, completion_tokens: int<0, max>, total_tokens: int<0, max>, requests: int<0, max>}>
59+
*/
60+
public function monthlyBreakdown(?string $year, ?string $month, ?string $model): Collection
61+
{
62+
$suffix = 'breakdown_'.($year ?? 'all').'_'.($month ?? 'all').'_'.($model ?? 'all');
63+
64+
return $this->remember($suffix, function () use ($year, $month, $model) {
65+
return AiModelDailyUsage::query()
66+
->when($model === self::OTHER_MODEL, fn (Builder $query) => $query->whereNull('ai_model_id'))
67+
->when($model && $model !== self::OTHER_MODEL, fn (Builder $query) => $query->where('model', $model))
68+
->when($year, fn (Builder $query) => $query->whereYear('date', $year))
69+
->when($month, fn (Builder $query) => $query->whereMonth('date', $month))
70+
->orderBy('date')
71+
->get()
72+
->groupBy(fn (AiModelDailyUsage $row) => $row->date->format('Y-m'))
73+
->map(
74+
/** @param EloquentCollection<int, AiModelDailyUsage> $rows */
75+
fn (EloquentCollection $rows, string $label) => [
76+
'label' => $label,
77+
'prompt_tokens' => $rows->sum(fn (AiModelDailyUsage $row): int => $row->prompt_tokens),
78+
'completion_tokens' => $rows->sum(fn (AiModelDailyUsage $row): int => $row->completion_tokens),
79+
'total_tokens' => $rows->sum(fn (AiModelDailyUsage $row): int => $row->total_tokens),
80+
'requests' => $rows->sum(fn (AiModelDailyUsage $row): int => $row->requests),
81+
]
82+
)
83+
->values();
84+
});
85+
}
86+
87+
/**
88+
* @return array{prompt_tokens: int, completion_tokens: int, total_tokens: int, requests: int}
89+
*/
90+
public function currentMonthSummary(?string $model = null): array
91+
{
92+
return $this->summary('month_summary', CarbonImmutable::now()->startOfMonth(), $model);
93+
}
94+
95+
/**
96+
* @return array{prompt_tokens: int, completion_tokens: int, total_tokens: int, requests: int}
97+
*/
98+
public function currentYearSummary(?string $model = null): array
99+
{
100+
return $this->summary('year_summary', CarbonImmutable::now()->startOfYear(), $model);
101+
}
102+
103+
/**
104+
* @return array{prompt_tokens: int, completion_tokens: int, total_tokens: int, requests: int}
105+
*/
106+
public function totalSummary(?string $model = null): array
107+
{
108+
return $this->summary('total_summary', null, $model);
109+
}
110+
111+
/**
112+
* @return array{prompt_tokens: int, completion_tokens: int, total_tokens: int, requests: int}
113+
*/
114+
private function summary(string $suffix, ?CarbonImmutable $from, ?string $model = null): array
115+
{
116+
return $this->remember($suffix.'_'.($model ?? 'all'), function () use ($from, $model) {
117+
$rows = AiModelDailyUsage::query()
118+
->when($from, fn (Builder $query) => $query->where('date', '>=', $from))
119+
->when($model === self::OTHER_MODEL, fn (Builder $query) => $query->whereNull('ai_model_id'))
120+
->when($model && $model !== self::OTHER_MODEL, fn (Builder $query) => $query->where('model', $model))
121+
->get();
122+
123+
return [
124+
'prompt_tokens' => $rows->sum(fn (AiModelDailyUsage $row): int => $row->prompt_tokens),
125+
'completion_tokens' => $rows->sum(fn (AiModelDailyUsage $row): int => $row->completion_tokens),
126+
'total_tokens' => $rows->sum(fn (AiModelDailyUsage $row): int => $row->total_tokens),
127+
'requests' => $rows->sum(fn (AiModelDailyUsage $row): int => $row->requests),
128+
];
129+
});
130+
}
131+
132+
/**
133+
* @template TValue
134+
*
135+
* @param Closure(): TValue $callback
136+
* @return TValue
137+
*/
138+
private function remember(string $suffix, Closure $callback): mixed
139+
{
140+
$version = Cache::get(self::VERSION_CACHE_KEY, 0);
141+
$version = is_int($version) ? $version : 0;
142+
$key = Str::slug("llm_usage_{$version}_{$suffix}", '_');
143+
144+
return Cache::remember($key, now()->addHour(), $callback);
145+
}
146+
}
Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
<?php
2+
3+
namespace App\Actions;
4+
5+
use App\Models\AiModel;
6+
use App\Models\AiModelDailyUsage;
7+
use Illuminate\Support\Collection;
8+
use Illuminate\Support\Facades\Artisan;
9+
use Illuminate\Support\Facades\Cache;
10+
use Spatie\ResponseCache\Commands\ClearCommand;
11+
12+
class StoreLlmUsageAction
13+
{
14+
/**
15+
* @param Collection<int, array{date: string, model: string, prompt_tokens: int, completion_tokens: int, total_tokens: int, requests: int, spend: float}> $rows
16+
*/
17+
public function store(Collection $rows): int
18+
{
19+
if ($rows->isEmpty()) {
20+
return 0;
21+
}
22+
23+
$aiModelIds = AiModel::pluck('id', 'name');
24+
25+
$rows = $rows->map(fn (array $row) => [
26+
...$row,
27+
'ai_model_id' => $aiModelIds->get($row['model']),
28+
]);
29+
30+
$count = AiModelDailyUsage::upsert(
31+
$rows->all(),
32+
uniqueBy: ['date', 'model'],
33+
update: ['ai_model_id', 'prompt_tokens', 'completion_tokens', 'total_tokens', 'requests', 'spend'],
34+
);
35+
36+
Cache::increment(LlmUsageStatsAction::VERSION_CACHE_KEY);
37+
38+
Artisan::call(ClearCommand::class);
39+
40+
return $count;
41+
}
42+
}

0 commit comments

Comments
 (0)