Skip to content

Commit 26fdbd4

Browse files
author
Sebastian BURGIN-FIX (ext)
committed
KI & AI Usage
1 parent 9b24f75 commit 26fdbd4

40 files changed

Lines changed: 1187 additions & 38 deletions

‎.env.ci‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -14,12 +14,12 @@ LOG_LEVEL=debug
1414

1515
FLARE_KEY=
1616

17-
DB_CONNECTION=mysql
17+
DB_CONNECTION=pgsql
1818
DB_HOST=127.0.0.1
19-
DB_PORT=3306
20-
DB_DATABASE=laravel
21-
DB_USERNAME=root
22-
DB_PASSWORD=root
19+
DB_PORT=5432
20+
DB_DATABASE=testing
21+
DB_USERNAME=postgres
22+
DB_PASSWORD=postgres
2323

2424
FPH_ENABLED=true
2525
CSP_ENABLED=true

‎.env.example‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,4 +57,6 @@ CLOUDINARY_CLOUD_NAME=my-cloud-name
5757
CLOUDINARY_API_KEY=my-api-key
5858
CLOUDINARY_API_SECRET=my-api-secret
5959

60-
LARAVEL_DEFAULT_FATHOM_SITE_ID=WECFOADW
60+
LARAVEL_DEFAULT_FATHOM_SITE_ID=WECFOADW
61+
LITELLM_URL=https://llm.codebar.net
62+
LITELLM_MASTER_KEY=

‎.github/workflows/pest_coverage_pull_request.yml‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ jobs:
2121
uses: shivammathur/setup-php@v2
2222
with:
2323
php-version: ${{ env.PHP_VERSION }}
24-
extensions: dom, curl, fileinfo, mysql, libxml, mbstring, zip, pcntl, pdo, sqlite, pdo_sqlite, bcmath, soap, intl, gd, exif, iconv, imagick, xdebug
24+
extensions: dom, curl, fileinfo, pgsql, pdo_pgsql, libxml, mbstring, zip, pcntl, pdo, sqlite, pdo_sqlite, bcmath, soap, intl, gd, exif, iconv, imagick, xdebug
2525
coverage: xdebug
2626

2727
- name: Prepare the .env file
@@ -45,9 +45,9 @@ jobs:
4545

4646
- name: Setup Database
4747
run: |
48-
sudo systemctl start mysql
49-
mysql --user="root" --password="root" -e "CREATE DATABASE laravel character set UTF8mb4 collate utf8mb4_bin;"
50-
mysql --user="root" --password="root" -e "CREATE DATABASE logs character set UTF8mb4 collate utf8mb4_bin;"
48+
sudo systemctl start postgresql.service
49+
sudo -u postgres psql -c "ALTER USER postgres PASSWORD 'postgres';"
50+
sudo -u postgres createdb testing
5151
5252
- name: Generate App Key & Run Migrations
5353
run: |

‎.github/workflows/pest_pull_request.yml‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ jobs:
1919
uses: shivammathur/setup-php@v2
2020
with:
2121
php-version: ${{ env.PHP_VERSION }}
22-
extensions: dom, curl, fileinfo, mysql, libxml, mbstring, zip, pcntl, pdo, sqlite, pdo_sqlite, bcmath, soap, intl, gd, exif, iconv, imagick
22+
extensions: dom, curl, fileinfo, pgsql, pdo_pgsql, libxml, mbstring, zip, pcntl, pdo, sqlite, pdo_sqlite, bcmath, soap, intl, gd, exif, iconv, imagick
2323

2424
- name: Prepare the .env file
2525
run: cp .env.ci .env
@@ -42,8 +42,9 @@ jobs:
4242

4343
- name: Setup Database
4444
run: |
45-
sudo systemctl start mysql
46-
mysql --user="root" --password="root" -e "CREATE DATABASE laravel character set UTF8mb4 collate utf8mb4_bin;"
45+
sudo systemctl start postgresql.service
46+
sudo -u postgres psql -c "ALTER USER postgres PASSWORD 'postgres';"
47+
sudo -u postgres createdb testing
4748
4849
- name: Generate App Key & Run Migrations
4950
run: |

‎app/Actions/FetchLlmUsageAction.php‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
<?php
2+
3+
namespace App\Actions;
4+
5+
use Carbon\CarbonImmutable;
6+
use Illuminate\Http\Client\PendingRequest;
7+
use Illuminate\Support\Collection;
8+
use Illuminate\Support\Facades\Http;
9+
10+
class FetchLlmUsageAction
11+
{
12+
/**
13+
* Fetch the per-model usage aggregates for a single day from the LiteLLM proxy.
14+
*
15+
* @return Collection<int, array{date: string, model: string, prompt_tokens: int, completion_tokens: int, total_tokens: int, requests: int, spend: float}>
16+
*/
17+
public function fetchDay(CarbonImmutable $date): Collection
18+
{
19+
$response = $this->client()->get('/spend/logs', [
20+
'start_date' => $date->toDateString(),
21+
'end_date' => $date->addDay()->toDateString(),
22+
'summarize' => 'false',
23+
])->throw();
24+
25+
return $this->parseSpendLogs($date, $response->json() ?? []);
26+
}
27+
28+
private function client(): PendingRequest
29+
{
30+
return Http::baseUrl(config('services.litellm.url'))
31+
->withToken(config('services.litellm.master_key'))
32+
->acceptJson()
33+
->connectTimeout(10)
34+
->timeout(120)
35+
->retry(times: 2, sleepMilliseconds: 1000, throw: false);
36+
}
37+
38+
/**
39+
* @return Collection<int, array{date: string, model: string, prompt_tokens: int, completion_tokens: int, total_tokens: int, requests: int, spend: float}>
40+
*/
41+
private function parseSpendLogs(CarbonImmutable $date, array $logs): Collection
42+
{
43+
// The end_date bound is inclusive, so the response can contain rows of
44+
// the following day — keep only rows that started on the requested day.
45+
return collect($logs)
46+
->filter(fn (mixed $log): bool => filled(data_get($log, 'model_group')))
47+
->filter(fn (mixed $log): bool => str_starts_with((string) data_get($log, 'startTime'), $date->toDateString()))
48+
->groupBy(fn (mixed $log): string => data_get($log, 'model_group'))
49+
->map(fn (Collection $rows, string $model): array => [
50+
'date' => $date->toDateString(),
51+
'model' => $model,
52+
'prompt_tokens' => (int) $rows->sum(fn (mixed $log): int => (int) data_get($log, 'prompt_tokens', 0)),
53+
'completion_tokens' => (int) $rows->sum(fn (mixed $log): int => (int) data_get($log, 'completion_tokens', 0)),
54+
'total_tokens' => (int) $rows->sum(fn (mixed $log): int => (int) data_get($log, 'total_tokens', 0)),
55+
'requests' => $rows->count(),
56+
'spend' => round($rows->sum(fn (mixed $log): float => (float) data_get($log, 'spend', 0)), 6),
57+
])
58+
->values();
59+
}
60+
}

‎app/Actions/LlmUsageStatsAction.php‎

Lines changed: 132 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,132 @@
1+
<?php
2+
3+
namespace App\Actions;
4+
5+
use App\Models\AiModelDailyUsage;
6+
use Carbon\CarbonImmutable;
7+
use Illuminate\Support\Collection;
8+
use Illuminate\Support\Facades\Cache;
9+
use Illuminate\Support\Str;
10+
11+
class LlmUsageStatsAction
12+
{
13+
public const VERSION_CACHE_KEY = 'llm_usage_version';
14+
15+
public const OTHER_MODEL = 'other';
16+
17+
/**
18+
* @return Collection<int, string>
19+
*/
20+
public function models(): Collection
21+
{
22+
return $this->remember('models', function () {
23+
return AiModelDailyUsage::query()
24+
->whereNotNull('ai_model_id')
25+
->distinct()
26+
->orderBy('model')
27+
->pluck('model');
28+
});
29+
}
30+
31+
public function hasOtherModels(): bool
32+
{
33+
return $this->remember('has_other_models', function () {
34+
return AiModelDailyUsage::query()->whereNull('ai_model_id')->exists();
35+
});
36+
}
37+
38+
/**
39+
* @return Collection<int, string>
40+
*/
41+
public function years(): Collection
42+
{
43+
return $this->remember('years', function () {
44+
return AiModelDailyUsage::query()
45+
->orderBy('date')
46+
->pluck('date')
47+
->map(fn ($date) => $date->format('Y'))
48+
->unique()
49+
->values();
50+
});
51+
}
52+
53+
/**
54+
* @return Collection<int, array{label: string, prompt_tokens: int, completion_tokens: int, total_tokens: int, requests: int}>
55+
*/
56+
public function monthlyBreakdown(?string $year, ?string $month, ?string $model): Collection
57+
{
58+
$suffix = 'breakdown_'.($year ?? 'all').'_'.($month ?? 'all').'_'.($model ?? 'all');
59+
60+
return $this->remember($suffix, function () use ($year, $month, $model) {
61+
return AiModelDailyUsage::query()
62+
->when($model === self::OTHER_MODEL, fn ($query) => $query->whereNull('ai_model_id'))
63+
->when($model && $model !== self::OTHER_MODEL, fn ($query) => $query->where('model', $model))
64+
->when($year, fn ($query) => $query->whereYear('date', $year))
65+
->when($month, fn ($query) => $query->whereMonth('date', $month))
66+
->orderBy('date')
67+
->get()
68+
->groupBy(fn (AiModelDailyUsage $row) => $row->date->format('Y-m'))
69+
->map(fn (Collection $rows, string $label) => [
70+
'label' => $label,
71+
'prompt_tokens' => (int) $rows->sum('prompt_tokens'),
72+
'completion_tokens' => (int) $rows->sum('completion_tokens'),
73+
'total_tokens' => (int) $rows->sum('total_tokens'),
74+
'requests' => (int) $rows->sum('requests'),
75+
])
76+
->values();
77+
});
78+
}
79+
80+
/**
81+
* @return array{prompt_tokens: int, completion_tokens: int, total_tokens: int, requests: int}
82+
*/
83+
public function currentMonthSummary(?string $model = null): array
84+
{
85+
return $this->summary('month_summary', CarbonImmutable::now()->startOfMonth(), $model);
86+
}
87+
88+
/**
89+
* @return array{prompt_tokens: int, completion_tokens: int, total_tokens: int, requests: int}
90+
*/
91+
public function currentYearSummary(?string $model = null): array
92+
{
93+
return $this->summary('year_summary', CarbonImmutable::now()->startOfYear(), $model);
94+
}
95+
96+
/**
97+
* @return array{prompt_tokens: int, completion_tokens: int, total_tokens: int, requests: int}
98+
*/
99+
public function totalSummary(?string $model = null): array
100+
{
101+
return $this->summary('total_summary', null, $model);
102+
}
103+
104+
/**
105+
* @return array{prompt_tokens: int, completion_tokens: int, total_tokens: int, requests: int}
106+
*/
107+
private function summary(string $suffix, ?CarbonImmutable $from, ?string $model = null): array
108+
{
109+
return $this->remember($suffix.'_'.($model ?? 'all'), function () use ($from, $model) {
110+
$rows = AiModelDailyUsage::query()
111+
->when($from, fn ($query) => $query->where('date', '>=', $from))
112+
->when($model === self::OTHER_MODEL, fn ($query) => $query->whereNull('ai_model_id'))
113+
->when($model && $model !== self::OTHER_MODEL, fn ($query) => $query->where('model', $model))
114+
->get();
115+
116+
return [
117+
'prompt_tokens' => (int) $rows->sum('prompt_tokens'),
118+
'completion_tokens' => (int) $rows->sum('completion_tokens'),
119+
'total_tokens' => (int) $rows->sum('total_tokens'),
120+
'requests' => (int) $rows->sum('requests'),
121+
];
122+
});
123+
}
124+
125+
private function remember(string $suffix, callable $callback): mixed
126+
{
127+
$version = Cache::get(self::VERSION_CACHE_KEY, 0);
128+
$key = Str::slug("llm_usage_{$version}_{$suffix}", '_');
129+
130+
return Cache::remember($key, now()->addHour(), $callback);
131+
}
132+
}

‎app/Actions/StoreLlmUsageAction.php‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
<?php
2+
3+
namespace App\Actions;
4+
5+
use App\Models\AiModel;
6+
use App\Models\AiModelDailyUsage;
7+
use Illuminate\Support\Collection;
8+
use Illuminate\Support\Facades\Artisan;
9+
use Illuminate\Support\Facades\Cache;
10+
use Spatie\ResponseCache\Commands\ClearCommand;
11+
12+
class StoreLlmUsageAction
13+
{
14+
/**
15+
* @param Collection<int, array{date: string, model: string, prompt_tokens: int, completion_tokens: int, total_tokens: int, requests: int, spend: float}> $rows
16+
*/
17+
public function store(Collection $rows): int
18+
{
19+
if ($rows->isEmpty()) {
20+
return 0;
21+
}
22+
23+
$aiModelIds = AiModel::pluck('id', 'name');
24+
25+
$rows = $rows->map(fn (array $row) => [
26+
...$row,
27+
'ai_model_id' => $aiModelIds->get($row['model']),
28+
]);
29+
30+
$count = AiModelDailyUsage::upsert(
31+
$rows->all(),
32+
uniqueBy: ['date', 'model'],
33+
update: ['ai_model_id', 'prompt_tokens', 'completion_tokens', 'total_tokens', 'requests', 'spend'],
34+
);
35+
36+
Cache::increment(LlmUsageStatsAction::VERSION_CACHE_KEY);
37+
38+
Artisan::call(ClearCommand::class);
39+
40+
return $count;
41+
}
42+
}

‎app/Console/Commands/FetchLlmAnalyticsCommand.php‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
<?php
2+
3+
namespace App\Console\Commands;
4+
5+
use App\Jobs\FetchLlmUsageJob;
6+
use Carbon\CarbonImmutable;
7+
use Carbon\CarbonPeriod;
8+
use Carbon\Exceptions\InvalidFormatException;
9+
use Illuminate\Console\Command;
10+
11+
class FetchLlmAnalyticsCommand extends Command
12+
{
13+
protected $signature = 'llm:fetch-analytics
14+
{--from= : Start date (YYYY-MM-DD), defaults to 3 days ago}
15+
{--to= : End date (YYYY-MM-DD), defaults to today}';
16+
17+
protected $description = 'Fetch daily per-model LLM usage from the LiteLLM proxy and store it locally';
18+
19+
public function handle(): int
20+
{
21+
try {
22+
$to = $this->date($this->option('to')) ?? CarbonImmutable::today();
23+
$from = $this->date($this->option('from')) ?? $to->subDays(3);
24+
} catch (InvalidFormatException) {
25+
$this->error('Invalid date format, expected YYYY-MM-DD.');
26+
27+
return self::FAILURE;
28+
}
29+
30+
if ($from->greaterThan($to)) {
31+
$this->error('The --from date must not be after the --to date.');
32+
33+
return self::FAILURE;
34+
}
35+
36+
$days = collect(CarbonPeriod::create($from, $to));
37+
38+
$days->each(fn (mixed $day) => FetchLlmUsageJob::dispatch(CarbonImmutable::instance($day)));
39+
40+
$this->info("Dispatched {$days->count()} job(s) for {$from->toDateString()} to {$to->toDateString()}.");
41+
42+
return self::SUCCESS;
43+
}
44+
45+
private function date(?string $value): ?CarbonImmutable
46+
{
47+
return $value === null
48+
? null
49+
: CarbonImmutable::createFromFormat('!Y-m-d', $value);
50+
}
51+
}

‎app/Helpers/HelperDate.php‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,4 +19,18 @@ public function formatDate(Carbon $date): string
1919
{
2020
return $date->format('d.m.Y');
2121
}
22+
23+
public function monthLabel(string $yearMonth): string
24+
{
25+
return Carbon::createFromFormat('!Y-m', $yearMonth)
26+
->locale(app()->getLocale())
27+
->translatedFormat('F Y');
28+
}
29+
30+
public function monthName(int $month): string
31+
{
32+
return Carbon::create(month: $month)
33+
->locale(app()->getLocale())
34+
->translatedFormat('F');
35+
}
2236
}

0 commit comments

Comments
 (0)