Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1,824 changes: 802 additions & 1,022 deletions rust/cloud-storage/Cargo.lock

Large diffs are not rendered by default.

40 changes: 26 additions & 14 deletions rust/cloud-storage/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ members = [
"delete_chat_handler",
"deleted_item_poller",
"document_cognition_service",
"document_cognition_service/load-test",
"document_cognition_service_client",
"document_storage_service",
"document_storage_service_client",
Expand Down Expand Up @@ -151,15 +152,15 @@ askama = "0.14"
async-openai = { version = "0.36.1", features = ["byot", "chat-completion", "completions"] }
async-stream = "0.3.6"
async-trait = "0.1.86"
aws-config = { version = "1.8.12", features = ["behavior-version-latest"] }
aws-sdk-dynamodb = "1.98.0"
aws-sdk-lambda = "1.104.0"
aws-sdk-s3 = "1.121.0"
aws-sdk-secretsmanager = "1.93.0"
aws-sdk-sesv2 = "1.103.0"
aws-sdk-sns = "1.90.0"
aws-sdk-sqs = "1.91.0"
aws-smithy-types = "1.2.13"
aws-config = { version = "1.8.16", features = ["behavior-version-latest"] }
aws-sdk-dynamodb = "1.111.0"
aws-sdk-lambda = "1.122.0"
aws-sdk-s3 = "1.132.0"
aws-sdk-secretsmanager = "1.104.0"
aws-sdk-sesv2 = "1.118.0"
aws-sdk-sns = "1.99.0"
aws-sdk-sqs = "1.98.0"
aws-smithy-types = "1.4.7"
aws_lambda_events = { version = "0.16.0", default-features = false }
axum = { version = "0.8", features = ["tower-log"] }
axum-extra = { version = "0.10" }
Expand Down Expand Up @@ -188,14 +189,24 @@ jsonwebtoken = { git = "https://github.com/whutchinson98/jsonwebtoken", features
] }
lambda_runtime = "0.13.0"
lazy_static = "1.5.0"
livekit-api = "0.4.16"
livekit-api = { version = "0.4.16", default-features = false, features = [
"services-tokio",
"access-token",
"webhooks",
"rustls-tls-webpki-roots",
] }
livekit-protocol = "0.7.2"
log = "0.4.26"
mockall = "0.13"
nom = "8"
openssl = { version = "0.10.71", features = ["vendored"] }
opentelemetry = "0.31.0"
opentelemetry-otlp = { version = "0.31.0", features = ["grpc-tonic"] }
opentelemetry-otlp = { version = "0.31.0", default-features = false, features = [
"grpc-tonic",
"trace",
"metrics",
"logs",
"internal-logs",
] }
opentelemetry_sdk = { version = "0.31.0", features = ["rt-tokio"] }
ordered-float = "5"
paste = "1"
Expand All @@ -206,7 +217,7 @@ recursion = "0.5.4"
redis = "1.0.3"
regex = "1.11.1"
rmcp = "1.6.0"
reqwest = { version = "0.13", features = ["form", "hickory-dns", "json"] }
reqwest = { version = "0.13", default-features = false, features = ["rustls", "form", "hickory-dns", "json"] }
rootcause = { version = "0.12" }
schemars = { version = "1.0.4", features = ["chrono04", "uuid1"] }
semver = { version = "1", features = ["serde"] }
Expand All @@ -230,6 +241,7 @@ time = "0.3.37"
tokio = { version = "1.43.0", features = ["macros", "rt-multi-thread"] }
tokio-stream = { version = "0.1.17" }
tokio-util = "0.7.13"
tungstenite = { version = "0.28", features = ["url"] }
tower = { version = "0.5.1", features = ["util"] }
tower-http = { version = "0.6.1", features = [
"compression-gzip",
Expand All @@ -254,7 +266,7 @@ uuid = { version = "1.15.1", features = ["fast-rng", "serde", "v4", "v7"] }
async-stripe = { version = "0.41", default-features = false, features = [
"billing",
"checkout",
"runtime-tokio-hyper",
"runtime-tokio-hyper-rustls-webpki",
] }
axum-macros = "0.5"
bytes = "1.11.1"
Expand Down
1 change: 0 additions & 1 deletion rust/cloud-storage/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,6 @@ RUN : \
&& apt-get update \
&& apt-get install -y --no-install-recommends \
ca-certificates \
openssl \
dumb-init \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
Expand Down
1 change: 0 additions & 1 deletion rust/cloud-storage/Dockerfile.convert_service
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,6 @@ RUN : \
libreoffice \
libreofficekit-dev \
ca-certificates \
openssl \
dumb-init \
libkrb5-3 \
ttf-mscorefonts-installer \
Expand Down
1 change: 0 additions & 1 deletion rust/cloud-storage/Dockerfile.convert_service.dev
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
libreoffice \
libreofficekit-dev \
ca-certificates \
openssl \
dumb-init \
libkrb5-3 \
ttf-mscorefonts-installer \
Expand Down
1 change: 0 additions & 1 deletion rust/cloud-storage/Dockerfile.convert_service.prebuilt
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@ RUN : \
libreoffice \
libreofficekit-dev \
ca-certificates \
openssl \
dumb-init \
libkrb5-3 \
ttf-mscorefonts-installer \
Expand Down
2 changes: 1 addition & 1 deletion rust/cloud-storage/Dockerfile.dev
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry,sharing=shared \
# Base runtime image
FROM debian:trixie-slim AS base
RUN apt-get update && apt-get install -y --no-install-recommends \
ca-certificates openssl dumb-init curl \
ca-certificates dumb-init curl \
&& apt-get clean && rm -rf /var/lib/apt/lists/*
WORKDIR /app
ENV PORT=8080
Expand Down
1 change: 0 additions & 1 deletion rust/cloud-storage/Dockerfile.prebuilt
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ WORKDIR /app
RUN apt-get update \
&& apt-get install -y --no-install-recommends \
ca-certificates \
openssl \
dumb-init \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
Expand Down
1 change: 0 additions & 1 deletion rust/cloud-storage/Dockerfile.search_processing_service
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@ RUN : \
&& apt-get update \
&& apt-get install -y --no-install-recommends \
ca-certificates \
openssl \
dumb-init \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ FROM debian:trixie-slim AS search_processing_service
WORKDIR /app

RUN apt-get update && apt-get install -y --no-install-recommends \
ca-certificates openssl dumb-init curl \
ca-certificates dumb-init curl \
&& apt-get clean && rm -rf /var/lib/apt/lists/*

COPY --from=builder /app/svc /app/svc
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ RUN : \
&& apt-get update \
&& apt-get install -y --no-install-recommends \
ca-certificates \
openssl \
dumb-init \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
Expand Down
2 changes: 1 addition & 1 deletion rust/cloud-storage/Dockerfile.sha-cleanup-worker
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ RUN --mount=type=cache,id=cargo_build,target=/usr/local/cargo/registry cargo bui
FROM debian:trixie-slim AS runner

# Install ssl certs so we can connect to the DB with ssl true
RUN apt-get update && apt install -y ca-certificates openssl
RUN apt-get update && apt install -y ca-certificates

COPY --from=builder /app/target/release/sha_cleanup_worker .

Expand Down
5 changes: 2 additions & 3 deletions rust/cloud-storage/authentication_service/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ anyhow = { workspace = true }
async-stripe = { workspace = true, default-features = false, features = [
"billing",
"checkout",
"runtime-tokio-hyper",
"runtime-tokio-hyper-rustls-webpki",
] }
async-stripe-webhook = { version = "1.0.0-alpha.4", default-features = false }
aws-sdk-secretsmanager = { workspace = true }
Expand Down Expand Up @@ -92,15 +92,14 @@ models_team = { path = "../models_team" }
native_app_service = { path = "../native_app_service" }
non_empty = { path = "../non_empty" }
notification = { path = "../notification" }
openssl = { workspace = true }
rand = { workspace = true }
rate_limit = { path = "../rate_limit" }
redis = { workspace = true }
referral = { path = "../referral" }
remote_env_var = { path = "../remote_env_var" }
reqwest = { workspace = true }
roles_and_permissions = { path = "../roles_and_permissions" }
rsa = "0.9.10"
rsa = { version = "0.9.10", features = ["getrandom"] }
secretsmanager_client = { path = "../secretsmanager_client" }
serde = { workspace = true }
serde_json = { workspace = true }
Expand Down
30 changes: 18 additions & 12 deletions rust/cloud-storage/authentication_service/src/bin/generate_keys.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,12 @@
use anyhow::Context;
use base64::{Engine, engine::general_purpose::URL_SAFE_NO_PAD};
use openssl::{pkey::PKey, rsa::Rsa};
use rsa::{RsaPrivateKey, pkcs1::DecodeRsaPrivateKey, traits::PublicKeyParts};
use rsa::{
RsaPrivateKey,
pkcs1::{DecodeRsaPrivateKey, EncodeRsaPrivateKey, LineEnding as Pkcs1LineEnding},
pkcs8::{EncodePublicKey, LineEnding as Pkcs8LineEnding},
rand_core::OsRng,
traits::PublicKeyParts,
};
use serde_json::json;
use std::fs;

Expand All @@ -10,26 +15,27 @@ fn main() -> anyhow::Result<()> {
println!("Generating keys for environment {env}");

// Generate 2048-bit RSA private key
let rsa = Rsa::generate(2048).context("unable to generate rsa key")?;
let private_key = PKey::from_rsa(rsa).context("unable to convert rsa key to pkey")?;
let mut rng = OsRng;
let private_key = RsaPrivateKey::new(&mut rng, 2048).context("unable to generate rsa key")?;

// Convert private key to PEM format (PKCS#1 format to match openssl genrsa)
// Convert private key to PEM format
let private_pem = private_key
.rsa()?
.private_key_to_pem()
.context("unable to convert pkey to pem")?;
.to_pkcs1_pem(Pkcs1LineEnding::LF)
.context("unable to convert private key to pem")?;

// Write private key to file
fs::write("./output/private.pem", &private_pem)
fs::write("./output/private.pem", private_pem.as_bytes())
.context("unable to write private key to file")?;

// Extract public key from private key
let public_key = private_key
.public_key_to_pem()
.context("unable to convert pkey to pem")?;
.to_public_key()
.to_public_key_pem(Pkcs8LineEnding::LF)
.context("unable to convert public key to pem")?;

// Write public key to file
fs::write("./output/public.pem", &public_key).context("unable to write public key to file")?;
fs::write("./output/public.pem", public_key.as_bytes())
.context("unable to write public key to file")?;

// Read the private key PEM file
let private_key_pem =
Expand Down
4 changes: 2 additions & 2 deletions rust/cloud-storage/connection_gateway/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ models_bulk_upload = { path = "../models_bulk_upload" }
redis = { workspace = true, features = [
"aio",
"tokio-comp",
"tokio-native-tls-comp",
"tokio-rustls-comp",
] }
secretsmanager_client = { path = "../secretsmanager_client" }
serde = { workspace = true }
Expand All @@ -69,7 +69,7 @@ tokio = { workspace = true }
tower = { workspace = true }
tower-http = { workspace = true }
tracing = { workspace = true, features = ["attributes"] }
tungstenite = "0.28"
tungstenite = { workspace = true }
url = { workspace = true }
utoipa = { workspace = true, features = ["axum_extras"] }
utoipa-swagger-ui = { workspace = true, features = ["axum"] }
Expand Down
3 changes: 0 additions & 3 deletions rust/cloud-storage/delete_chat_handler/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,7 @@ aws_lambda_events = { workspace = true, features = ["sqs"] }
lambda_runtime = { workspace = true }
macro_entrypoint = { path = "../macro_entrypoint" }
macro_uuid = { path = "../macro_uuid" }
openssl = { workspace = true }
sqlx = { workspace = true }
tokio = { workspace = true }
tracing = { workspace = true }

[package.metadata.cargo-machete]
ignored = ["openssl"]
3 changes: 0 additions & 3 deletions rust/cloud-storage/deleted_item_poller/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@ macro_entrypoint = { path = "../macro_entrypoint" }
macro_env = { path = "../macro_env" }
macro_uuid = { path = "../macro_uuid" }
model = { path = "../model" }
openssl = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
sqlx = { workspace = true }
Expand All @@ -31,5 +30,3 @@ tokio = { workspace = true }
tracing = { workspace = true }
urlencoding = { workspace = true }

[package.metadata.cargo-machete]
ignored = ["openssl"]
1 change: 0 additions & 1 deletion rust/cloud-storage/deny.toml
Original file line number Diff line number Diff line change
Expand Up @@ -258,7 +258,6 @@ skip = [
{ crate = "http-body", reason = "Hyper and AWS SDK version mismatch" },
{ crate = "hyper", reason = "AWS SDK uses older hyper version" },
{ crate = "hyper-rustls", reason = "TLS dependencies version mismatch" },
{ crate = "hyper-tls", reason = "TLS dependencies version mismatch" },
{ crate = "reqwest", reason = "livekit-api pins reqwest 0.12 via services-tokio feature; workspace uses 0.13" },
{ crate = "infer", reason = "File type detection dependencies" },
{ crate = "itertools", reason = "Various dependencies require different versions" },
Expand Down
2 changes: 1 addition & 1 deletion rust/cloud-storage/document_cognition_service/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ redis = { workspace = true, features = [
"aio",
"streams",
"tokio-comp",
"tokio-native-tls-comp",
"tokio-rustls-comp",
] }
regex = { workspace = true }
reqwest.workspace = true
Expand Down
Loading
Loading