16397 llm proxy custom providers (#7)

Author: sergei-bronnikov

.gitignore

@@ -1,4 +1,5 @@
 release_notes.md
 target
 .DS_STORE
-.vscode/launch.json
+.vscode/launch.json
+.env

CHANGELOG.md

@@ -1,3 +1,60 @@
+## 1.39.0 - 2024-11-15
+### Added
+- Added encryption integration
+
+### Changed
+- Removed support for Redis TLS config
+
+
+## 1.38.0 - 2024-11-09
+### Added
+- Added support for `claude-3-5-haiku`
+- Added support for Redis TLS config
+- Added support for `gpt-4o-2024-08-06`
+
+## 1.37.0 - 2024-10-23
+### Added
+- Added request level timeout with HTTP header `x-request-timeout`
+
+## 1.36.5 - 2024-10-16
+### Changed
+- Updated `gpt-4o` pricing according to OpenAI updates
+
+## 1.36.4 - 2024-10-15
+### Added
+- Added support for `gpt-4o-mini` in routes
+
+## 1.36.3 - 2024-09-16
+### Added
+- Added support for OpenAI o1 models
+
+## 1.36.2 - 2024-09-13
+### Fixed
+- Fixed compatibility issues between Anthropic SDK and AWS Bedrock
+  
+## 1.36.1 - 2024-09-10
+### Fixed
+- Fixed provider selection issue when a key is associated with multiple providers
+
+## 1.36.0 - 2024-09-09
+### Added
+- Added Amazon Bedrock integration for Claude models
+
+## 1.35.2 - 2024-08-10
+### Changed
+- Changed aggregated table column data types from `INT` to `BIGINT`
+
+## 1.35.1 - 2024-08-10
+### Changed
+- Changed cache TTL from `1h` to `24h` for keys and provider settings
+
+## 1.35.0 - 2024-08-10
+### Added
+- Added cost tracking for `gpt-4o-2024-08-06`
+
+### Changed
+- Changed default read time out for PostgreSQL
+
 ## 1.34.0 - 2024-07-29
 ### Added
 - Added cost tracking for `gpt-4o-mini`

Dockerfile.dev

@@ -1,12 +1,12 @@
-FROM golang:1.22.1 AS build
+FROM golang:1.23.2 AS build
 ENV CGO_ENABLED=0
 ENV GOOS=linux
 
 WORKDIR /go/src/github.com/bricks-cloud/bricksllm/
 COPY . /go/src/github.com/bricks-cloud/bricksllm/
 RUN go build -ldflags="-s -w" -o ./bin/bricksllm ./cmd/bricksllm/main.go
 
-FROM alpine:3.17
+FROM alpine:3.20
 RUN apk --no-cache add ca-certificates
 WORKDIR /usr/bin
 COPY --from=build /go/src/github.com/bricks-cloud/bricksllm/bin /go/bin

Dockerfile.prod

@@ -1,12 +1,12 @@
-FROM golang:1.22.1 AS build
+FROM golang:1.23.2 AS build
 ENV CGO_ENABLED=0
 ENV GOOS=linux
 
 WORKDIR /go/src/github.com/bricks-cloud/bricksllm/
 COPY . /go/src/github.com/bricks-cloud/bricksllm/
 RUN go build -ldflags="-s -w" -o ./bin/bricksllm ./cmd/bricksllm/main.go
 
-FROM alpine:3.17
+FROM alpine:3.20
 RUN apk --no-cache add ca-certificates
 WORKDIR /usr/bin
 COPY --from=build /go/src/github.com/bricks-cloud/bricksllm/bin /go/bin

cmd/bricksllm/main.go

@@ -12,6 +12,7 @@ import (
 	auth "github.com/bricks-cloud/bricksllm/internal/authenticator"
 	"github.com/bricks-cloud/bricksllm/internal/cache"
 	"github.com/bricks-cloud/bricksllm/internal/config"
+	"github.com/bricks-cloud/bricksllm/internal/encryptor"
 	"github.com/bricks-cloud/bricksllm/internal/logger/zap"
 	"github.com/bricks-cloud/bricksllm/internal/manager"
 	"github.com/bricks-cloud/bricksllm/internal/message"
@@ -173,137 +174,112 @@ func main() {
 	}
 	rMemStore.Listen()
 
-	rateLimitRedisCache := redis.NewClient(&redis.Options{
-		Addr:     fmt.Sprintf("%s:%s", cfg.RedisHosts, cfg.RedisPort),
-		Password: cfg.RedisPassword,
-		DB:       0,
-	})
+	defaultRedisOption := func(cfg *config.Config, dbIndex int) *redis.Options {
+
+		options := &redis.Options{
+			Addr:     fmt.Sprintf("%s:%s", cfg.RedisHosts, cfg.RedisPort),
+			Password: cfg.RedisPassword,
+			DB:       cfg.RedisDBStartIndex + dbIndex,
+		}
+
+		return options
+	}
+
+	rateLimitRedisCache := redis.NewClient(defaultRedisOption(cfg, 0))
 	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
 	defer cancel()
 	if err := rateLimitRedisCache.Ping(ctx).Err(); err != nil {
 		log.Sugar().Fatalf("error connecting to rate limit redis cache: %v", err)
 	}
 
-	costLimitRedisCache := redis.NewClient(&redis.Options{
-		Addr:     fmt.Sprintf("%s:%s", cfg.RedisHosts, cfg.RedisPort),
-		Password: cfg.RedisPassword,
-		DB:       1,
-	})
+	costLimitRedisCache := redis.NewClient(defaultRedisOption(cfg, 1))
 
 	ctx, cancel = context.WithTimeout(context.Background(), 2*time.Second)
 	defer cancel()
 	if err := costLimitRedisCache.Ping(ctx).Err(); err != nil {
 		log.Sugar().Fatalf("error connecting to cost limit redis cache: %v", err)
 	}
 
-	costRedisStorage := redis.NewClient(&redis.Options{
-		Addr:     fmt.Sprintf("%s:%s", cfg.RedisHosts, cfg.RedisPort),
-		Password: cfg.RedisPassword,
-		DB:       2,
-	})
+	costRedisStorage := redis.NewClient(defaultRedisOption(cfg, 2))
 
 	ctx, cancel = context.WithTimeout(context.Background(), 2*time.Second)
 	defer cancel()
 	if err := costRedisStorage.Ping(ctx).Err(); err != nil {
 		log.Sugar().Fatalf("error connecting to cost limit redis storage: %v", err)
 	}
 
-	apiRedisCache := redis.NewClient(&redis.Options{
-		Addr:     fmt.Sprintf("%s:%s", cfg.RedisHosts, cfg.RedisPort),
-		Password: cfg.RedisPassword,
-		DB:       3,
-	})
+	apiRedisCache := redis.NewClient(defaultRedisOption(cfg, 3))
 
 	ctx, cancel = context.WithTimeout(context.Background(), 2*time.Second)
 	defer cancel()
 	if err := apiRedisCache.Ping(ctx).Err(); err != nil {
 		log.Sugar().Fatalf("error connecting to api redis cache: %v", err)
 	}
 
-	accessRedisCache := redis.NewClient(&redis.Options{
-		Addr:     fmt.Sprintf("%s:%s", cfg.RedisHosts, cfg.RedisPort),
-		Password: cfg.RedisPassword,
-		DB:       4,
-	})
+	accessRedisCache := redis.NewClient(defaultRedisOption(cfg, 4))
 
 	ctx, cancel = context.WithTimeout(context.Background(), 2*time.Second)
 	defer cancel()
 	if err := accessRedisCache.Ping(ctx).Err(); err != nil {
 		log.Sugar().Fatalf("error connecting to api redis cache: %v", err)
 	}
 
-	userRateLimitRedisCache := redis.NewClient(&redis.Options{
-		Addr:     fmt.Sprintf("%s:%s", cfg.RedisHosts, cfg.RedisPort),
-		Password: cfg.RedisPassword,
-		DB:       5,
-	})
+	userRateLimitRedisCache := redis.NewClient(defaultRedisOption(cfg, 5))
 
 	ctx, cancel = context.WithTimeout(context.Background(), 2*time.Second)
 	defer cancel()
 	if err := userRateLimitRedisCache.Ping(ctx).Err(); err != nil {
 		log.Sugar().Fatalf("error connecting to user rate limit redis cache: %v", err)
 	}
 
-	userCostLimitRedisCache := redis.NewClient(&redis.Options{
-		Addr:     fmt.Sprintf("%s:%s", cfg.RedisHosts, cfg.RedisPort),
-		Password: cfg.RedisPassword,
-		DB:       6,
-	})
+	userCostLimitRedisCache := redis.NewClient(defaultRedisOption(cfg, 6))
 
 	ctx, cancel = context.WithTimeout(context.Background(), 2*time.Second)
 	defer cancel()
 	if err := userCostLimitRedisCache.Ping(ctx).Err(); err != nil {
 		log.Sugar().Fatalf("error connecting to user cost limit redis cache: %v", err)
 	}
 
-	userCostRedisStorage := redis.NewClient(&redis.Options{
-		Addr:     fmt.Sprintf("%s:%s", cfg.RedisHosts, cfg.RedisPort),
-		Password: cfg.RedisPassword,
-		DB:       7,
-	})
+	userCostRedisStorage := redis.NewClient(defaultRedisOption(cfg, 7))
 
 	ctx, cancel = context.WithTimeout(context.Background(), 2*time.Second)
 	defer cancel()
 	if err := userCostRedisStorage.Ping(ctx).Err(); err != nil {
 		log.Sugar().Fatalf("error connecting to user cost redis cache: %v", err)
 	}
 
-	userAccessRedisCache := redis.NewClient(&redis.Options{
-		Addr:     fmt.Sprintf("%s:%s", cfg.RedisHosts, cfg.RedisPort),
-		Password: cfg.RedisPassword,
-		DB:       8,
-	})
+	userAccessRedisCache := redis.NewClient(defaultRedisOption(cfg, 8))
 
 	ctx, cancel = context.WithTimeout(context.Background(), 2*time.Second)
 	defer cancel()
 	if err := userAccessRedisCache.Ping(ctx).Err(); err != nil {
 		log.Sugar().Fatalf("error connecting to user access redis storage: %v", err)
 	}
 
-	providerSettingsRedisCache := redis.NewClient(&redis.Options{
-		Addr:     fmt.Sprintf("%s:%s", cfg.RedisHosts, cfg.RedisPort),
-		Password: cfg.RedisPassword,
-		DB:       9,
-	})
+	providerSettingsRedisCache := redis.NewClient(defaultRedisOption(cfg, 9))
 
 	ctx, cancel = context.WithTimeout(context.Background(), 2*time.Second)
 	defer cancel()
 	if err := providerSettingsRedisCache.Ping(ctx).Err(); err != nil {
 		log.Sugar().Fatalf("error connecting to provider settings redis storage: %v", err)
 	}
 
-	keysRedisCache := redis.NewClient(&redis.Options{
-		Addr:     fmt.Sprintf("%s:%s", cfg.RedisHosts, cfg.RedisPort),
-		Password: cfg.RedisPassword,
-		DB:       10,
-	})
+	keysRedisCache := redis.NewClient(defaultRedisOption(cfg, 10))
 
 	ctx, cancel = context.WithTimeout(context.Background(), 2*time.Second)
 	defer cancel()
 	if err := keysRedisCache.Ping(ctx).Err(); err != nil {
 		log.Sugar().Fatalf("error connecting to keys redis storage: %v", err)
 	}
 
+	requestsLimitRedisStorage := redis.NewClient(defaultRedisOption(cfg, 11))
+
+	ctx, cancel = context.WithTimeout(context.Background(), 2*time.Second)
+	defer cancel()
+	if err := requestsLimitRedisStorage.Ping(ctx).Err(); err != nil {
+		log.Sugar().Fatalf("error connecting to requests limit redis storage: %v", err)
+	}
+
 	rateLimitCache := redisStorage.NewCache(rateLimitRedisCache, cfg.RedisWriteTimeout, cfg.RedisReadTimeout)
 	costLimitCache := redisStorage.NewCache(costLimitRedisCache, cfg.RedisWriteTimeout, cfg.RedisReadTimeout)
 	costStorage := redisStorage.NewStore(costRedisStorage, cfg.RedisWriteTimeout, cfg.RedisReadTimeout)
@@ -317,12 +293,17 @@ func main() {
 
 	psCache := redisStorage.NewProviderSettingsCache(providerSettingsRedisCache, cfg.RedisWriteTimeout, cfg.RedisReadTimeout)
 	keysCache := redisStorage.NewKeysCache(keysRedisCache, cfg.RedisWriteTimeout, cfg.RedisReadTimeout)
+	requestsLimitStorage := redisStorage.NewStore(requestsLimitRedisStorage, cfg.RedisWriteTimeout, cfg.RedisReadTimeout)
 
-	v := validator.NewValidator(costLimitCache, rateLimitCache, costStorage)
+	encryptor, err := encryptor.NewEncryptor(cfg.DecryptionEndpoint, cfg.EncryptionEndpoint, cfg.EnableEncrytion, cfg.EncryptionTimeout, cfg.Audience)
+	if cfg.EnableEncrytion && err != nil {
+		log.Sugar().Fatalf("error creating encryption client: %v", err)
+	}
+	v := validator.NewValidator(costLimitCache, rateLimitCache, costStorage, requestsLimitStorage)
 
-	m := manager.NewManager(store, costLimitCache, rateLimitCache, accessCache, keysCache)
+	m := manager.NewManager(store, costLimitCache, rateLimitCache, accessCache, keysCache, requestsLimitStorage)
 	krm := manager.NewReportingManager(costStorage, store, store, v)
-	psm := manager.NewProviderSettingsManager(store, psCache)
+	psm := manager.NewProviderSettingsManager(store, psCache, encryptor)
 	cpm := manager.NewCustomProvidersManager(store, cpMemStore)
 	rm := manager.NewRouteManager(store, store, rMemStore, psm)
 	pm := manager.NewPolicyManager(store, rMemStore)
@@ -357,9 +338,9 @@ func main() {
 
 	uv := validator.NewUserValidator(userCostLimitCache, userRateLimitCache, userCostStorage)
 
-	rec := recorder.NewRecorder(costStorage, userCostStorage, costLimitCache, userCostLimitCache, ce, store)
+	rec := recorder.NewRecorder(costStorage, userCostStorage, costLimitCache, userCostLimitCache, ce, store, requestsLimitStorage)
 	rlm := manager.NewRateLimitManager(rateLimitCache, userRateLimitCache)
-	a := auth.NewAuthenticator(psm, m, rm, store)
+	a := auth.NewAuthenticator(psm, m, rm, store, encryptor)
 
 	c := cache.NewCache(apiCache)
 

docker-compose.yml

@@ -1,12 +1,11 @@
-version: '3.8'
 services:
   redis:
     image: redis:6.2-alpine
     restart: always
     ports:
       - '6379:6379'
     command: redis-server --save 20 1 --loglevel warning --requirepass eYVX7EwVmmxKPCDmwMtyKVge8oLd2t81
-    volumes: 
+    volumes:
       - redis:/data
   postgresql:
     image: postgres:14.1-alpine
@@ -16,10 +15,10 @@ services:
       - POSTGRES_PASSWORD=postgres
     ports:
       - '5432:5432'
-    volumes: 
+    volumes:
       - postgresql:/var/lib/postgresql/data
   # bricksllm:
-  #   depends_on: 
+  #   depends_on:
   #     - redis
   #     - postgresql
   #   image: luyuanxin1995/bricksllm
@@ -38,4 +37,4 @@ volumes:
   redis:
     driver: local
   postgresql:
-    driver: local
+    driver: local

docs/admin.yaml

@@ -1423,8 +1423,6 @@ components:
       type: object
       description: API Credentials associated with different providers.
       example: { "apikey": "MY_OPENAI_API_KEY" }
-      required:
-        - apikey
       properties:
         apikey:
           type: string
@@ -1438,6 +1436,18 @@ components:
           type: string
           example: MY_AZURE_OPENAI_RESOURCE_NAME
           description: Required for Azure OpenAI integrations.
+        awsAccessKeyId:
+          type: string
+          example: MY_AWS_ACCESS_KEY_ID
+          description: Required for Bedrock Anthropic integrations.
+        awsSecretAccessKey:
+          type: string
+          example: MY_AWS_SECRET_ACCESS_KEY
+          description: Required for Bedrock Anthropic integrations.
+        awsRegion:
+          type: string
+          example: MY_AWS_REGION
+          description: Required for Bedrock Anthropic integrations.
 
     ReportingEventsRequest:
       type: object