Skip to content

Commit d1d9889

Browse files
jorgeantonio21Cifko
jorgeantonio21
and
Cifko
authored
feat: release v0.1.13 (#640)
* feat: add usage per user/model/day (#639) * feat: add user_id to fiat payments (#638) * add user_id to fiat payments * add sql migration * comments * fix: proper user_id (#641) * fix: usage per day values (#642) * feat: shuffle jobs for homogeneous load balancing (#644) * introduce shuffling to uniformly load balance * fmt --------- Co-authored-by: Martin Stefcek <35243812+Cifko@users.noreply.github.com>
1 parent bd31f9b commit d1d9889

13 files changed

Lines changed: 269 additions & 36 deletions

‎atoma-service/src/handlers/chat_completions.rs‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -213,6 +213,7 @@ pub async fn chat_completions_handler(
213213
stack_small_id,
214214
estimated_output_tokens,
215215
price_per_one_million_tokens,
216+
user_id,
216217
user_address,
217218
num_input_tokens,
218219
payload_hash,
@@ -243,6 +244,7 @@ pub async fn chat_completions_handler(
243244
payload_hash,
244245
stack_small_id,
245246
price_per_one_million_tokens,
247+
user_id,
246248
user_address.clone(),
247249
is_stream,
248250
payload.clone(),
@@ -289,6 +291,7 @@ pub async fn chat_completions_handler(
289291
} else {
290292
update_fiat_amount(
291293
&state.state_manager_sender,
294+
user_id,
292295
user_address,
293296
model.to_string(),
294297
num_input_tokens,
@@ -415,6 +418,7 @@ pub async fn confidential_chat_completions_handler(
415418
stack_small_id,
416419
num_input_tokens,
417420
estimated_output_tokens,
421+
user_id,
418422
user_address,
419423
price_per_one_million_tokens,
420424
payload_hash,
@@ -450,6 +454,7 @@ pub async fn confidential_chat_completions_handler(
450454
payload_hash,
451455
stack_small_id,
452456
price_per_one_million_tokens,
457+
user_id,
453458
user_address.clone(),
454459
is_stream,
455460
payload.clone(),
@@ -497,6 +502,7 @@ pub async fn confidential_chat_completions_handler(
497502
} else {
498503
update_fiat_amount(
499504
&state.state_manager_sender,
505+
user_id,
500506
user_address,
501507
model.to_string(),
502508
num_input_tokens,
@@ -576,6 +582,7 @@ async fn handle_response(
576582
payload_hash: [u8; PAYLOAD_HASH_SIZE],
577583
stack_small_id: Option<i64>,
578584
price_per_one_million_tokens: i64,
585+
user_id: Option<i64>,
579586
user_address: String,
580587
is_stream: bool,
581588
payload: Value,
@@ -601,6 +608,7 @@ async fn handle_response(
601608
num_input_tokens,
602609
estimated_output_tokens,
603610
price_per_one_million_tokens,
611+
user_id,
604612
user_address,
605613
payload_hash,
606614
streaming_encryption_metadata,
@@ -616,6 +624,7 @@ async fn handle_response(
616624
num_input_tokens,
617625
estimated_output_tokens,
618626
price_per_one_million_tokens,
627+
user_id,
619628
user_address,
620629
payload_hash,
621630
client_encryption_metadata,
@@ -694,6 +703,7 @@ async fn handle_non_streaming_response(
694703
num_input_tokens: i64,
695704
estimated_output_tokens: i64,
696705
price_per_one_million_compute_units: i64,
706+
user_id: Option<i64>,
697707
user_address: String,
698708
payload_hash: [u8; PAYLOAD_HASH_SIZE],
699709
client_encryption_metadata: Option<EncryptionMetadata>,
@@ -734,6 +744,7 @@ async fn handle_non_streaming_response(
734744
num_input_tokens,
735745
estimated_output_tokens,
736746
price_per_one_million_compute_units,
747+
user_id,
737748
user_address,
738749
input_tokens,
739750
output_tokens,
@@ -802,6 +813,7 @@ async fn handle_streaming_response(
802813
num_input_tokens: i64,
803814
estimated_output_tokens: i64,
804815
price_per_one_million_compute_units: i64,
816+
user_id: Option<i64>,
805817
user_address: String,
806818
payload_hash: [u8; 32],
807819
streaming_encryption_metadata: Option<StreamingEncryptionMetadata>,
@@ -944,6 +956,7 @@ async fn handle_streaming_response(
944956
request_id,
945957
timer,
946958
price_per_one_million_compute_units,
959+
user_id,
947960
user_address,
948961
Arc::clone(&state.running_num_requests),
949962
chat_completions_service_url,
@@ -1504,6 +1517,7 @@ pub mod utils {
15041517
estimated_input_tokens: i64,
15051518
estimated_output_tokens: i64,
15061519
price_per_one_million_tokens: i64,
1520+
user_id: Option<i64>,
15071521
user_address: String,
15081522
input_tokens: i64,
15091523
output_tokens: i64,
@@ -1608,6 +1622,7 @@ pub mod utils {
16081622
} else {
16091623
update_fiat_amount(
16101624
&state.state_manager_sender,
1625+
user_id,
16111626
user_address,
16121627
model.to_string(),
16131628
estimated_input_tokens,

‎atoma-service/src/handlers/completions.rs‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -189,6 +189,7 @@ pub async fn completions_handler(
189189
num_input_tokens,
190190
payload_hash,
191191
client_encryption_metadata,
192+
user_id,
192193
user_address,
193194
price_per_one_million_tokens,
194195
..
@@ -217,6 +218,7 @@ pub async fn completions_handler(
217218
payload_hash,
218219
stack_small_id,
219220
price_per_one_million_tokens,
221+
user_id,
220222
user_address.clone(),
221223
is_stream,
222224
payload.clone(),
@@ -263,6 +265,7 @@ pub async fn completions_handler(
263265
} else {
264266
update_fiat_amount(
265267
&state.state_manager_sender,
268+
user_id,
266269
user_address,
267270
model.to_string(),
268271
num_input_tokens,
@@ -393,6 +396,7 @@ pub async fn confidential_completions_handler(
393396
client_encryption_metadata,
394397
user_address,
395398
price_per_one_million_tokens,
399+
user_id,
396400
..
397401
} = request_metadata;
398402
info!(
@@ -424,6 +428,7 @@ pub async fn confidential_completions_handler(
424428
payload_hash,
425429
stack_small_id,
426430
price_per_one_million_tokens,
431+
user_id,
427432
user_address.clone(),
428433
is_stream,
429434
payload.clone(),
@@ -472,6 +477,7 @@ pub async fn confidential_completions_handler(
472477
} else {
473478
update_fiat_amount(
474479
&state.state_manager_sender,
480+
user_id,
475481
user_address,
476482
model.to_string(),
477483
num_input_tokens,
@@ -551,6 +557,7 @@ async fn handle_response(
551557
payload_hash: [u8; PAYLOAD_HASH_SIZE],
552558
stack_small_id: Option<i64>,
553559
price_per_one_million_compute_units: i64,
560+
user_id: Option<i64>,
554561
user_address: String,
555562
is_stream: bool,
556563
payload: Value,
@@ -576,6 +583,7 @@ async fn handle_response(
576583
num_input_tokens,
577584
estimated_output_tokens,
578585
price_per_one_million_compute_units,
586+
user_id,
579587
user_address,
580588
payload_hash,
581589
streaming_encryption_metadata,
@@ -591,6 +599,7 @@ async fn handle_response(
591599
num_input_tokens,
592600
estimated_output_tokens,
593601
price_per_one_million_compute_units,
602+
user_id,
594603
user_address,
595604
payload_hash,
596605
client_encryption_metadata,
@@ -669,6 +678,7 @@ async fn handle_non_streaming_response(
669678
num_input_tokens: i64,
670679
estimated_total_tokens: i64,
671680
price_per_one_million_compute_units: i64,
681+
user_id: Option<i64>,
672682
user_address: String,
673683
payload_hash: [u8; PAYLOAD_HASH_SIZE],
674684
client_encryption_metadata: Option<EncryptionMetadata>,
@@ -709,6 +719,7 @@ async fn handle_non_streaming_response(
709719
num_input_tokens,
710720
estimated_total_tokens,
711721
price_per_one_million_compute_units,
722+
user_id,
712723
user_address,
713724
input_tokens,
714725
output_tokens,
@@ -777,6 +788,7 @@ async fn handle_streaming_response(
777788
num_input_tokens: i64,
778789
estimated_output_tokens: i64,
779790
price_per_one_million_tokens: i64,
791+
user_id: Option<i64>,
780792
user_address: String,
781793
payload_hash: [u8; 32],
782794
streaming_encryption_metadata: Option<StreamingEncryptionMetadata>,
@@ -914,6 +926,7 @@ async fn handle_streaming_response(
914926
request_id,
915927
timer,
916928
price_per_one_million_tokens,
929+
user_id,
917930
user_address,
918931
Arc::clone(&state.running_num_requests),
919932
completions_service_url,
@@ -1470,6 +1483,7 @@ pub mod utils {
14701483
num_input_tokens: i64,
14711484
estimated_output_tokens: i64,
14721485
price_per_one_million_compute_units: i64,
1486+
user_id: Option<i64>,
14731487
user_address: String,
14741488
input_tokens: i64,
14751489
output_tokens: i64,
@@ -1574,6 +1588,7 @@ pub mod utils {
15741588
} else {
15751589
update_fiat_amount(
15761590
&state.state_manager_sender,
1591+
user_id,
15771592
user_address,
15781593
model.to_string(),
15791594
num_input_tokens,

‎atoma-service/src/handlers/embeddings.rs‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,7 @@ pub async fn embeddings_handler(
106106
client_encryption_metadata,
107107
endpoint_path: endpoint,
108108
price_per_one_million_tokens,
109+
user_id,
109110
user_address,
110111
..
111112
} = request_metadata;
@@ -157,6 +158,7 @@ pub async fn embeddings_handler(
157158
} else {
158159
update_fiat_amount(
159160
&state.state_manager_sender,
161+
user_id,
160162
user_address,
161163
model.to_string(),
162164
num_input_tokens,
@@ -253,6 +255,7 @@ pub async fn confidential_embeddings_handler(
253255
client_encryption_metadata,
254256
endpoint_path: endpoint,
255257
price_per_one_million_tokens,
258+
user_id,
256259
user_address,
257260
..
258261
} = request_metadata;
@@ -295,6 +298,7 @@ pub async fn confidential_embeddings_handler(
295298
} else {
296299
update_fiat_amount(
297300
&state.state_manager_sender,
301+
user_id,
298302
user_address,
299303
model.to_string(),
300304
num_input_tokens,
@@ -328,6 +332,7 @@ pub async fn confidential_embeddings_handler(
328332
} else {
329333
update_fiat_amount(
330334
&state.state_manager_sender,
335+
user_id,
331336
user_address,
332337
model.to_string(),
333338
num_input_tokens,

‎atoma-service/src/handlers/image_generations.rs‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,7 @@ pub async fn image_generations_handler(
111111
endpoint_path: endpoint,
112112
price_per_one_million_tokens,
113113
user_address,
114+
user_id,
114115
..
115116
} = request_metadata;
116117

@@ -156,6 +157,7 @@ pub async fn image_generations_handler(
156157
} else {
157158
update_fiat_amount(
158159
&state.state_manager_sender,
160+
user_id,
159161
user_address,
160162
model.to_string(),
161163
num_input_tokens,
@@ -255,6 +257,7 @@ pub async fn confidential_image_generations_handler(
255257
endpoint_path: endpoint,
256258
price_per_one_million_tokens,
257259
user_address,
260+
user_id,
258261
..
259262
} = request_metadata;
260263

@@ -289,6 +292,7 @@ pub async fn confidential_image_generations_handler(
289292
} else {
290293
update_fiat_amount(
291294
&state.state_manager_sender,
295+
user_id,
292296
user_address,
293297
model.to_string(),
294298
num_input_tokens,
@@ -322,6 +326,7 @@ pub async fn confidential_image_generations_handler(
322326
} else {
323327
update_fiat_amount(
324328
&state.state_manager_sender,
329+
user_id,
325330
user_address,
326331
model.to_string(),
327332
num_input_tokens,

0 commit comments

Comments
 (0)