Allow specifying internal S3 endpoint (#685)

* start supporting internal s3 endpoint

* completely rip out createS3Client and the like

* fix endpoints

* fix icon uploading

* handle already loaded videos

* cleanup

* remove loadedmedtadata dispatchevent

* put md5 hash back

* fix file key parsing

* fix complete upload

* remove leading slash

* clippy

Author: Brendonovich

apps/desktop/src-tauri/src/lib.rs

@@ -1106,6 +1106,8 @@ async fn upload_exported_video(
             Ok(UploadResult::Success(uploaded_video.link))
         }
         Err(e) => {
+            error!("Failed to upload video: {e}");
+
             NotificationType::UploadFailed.send(&app);
             Err(e)
         }

apps/desktop/src-tauri/src/upload.rs

@@ -35,13 +35,6 @@ use specta::Type;
 #[derive(Deserialize, Serialize, Clone, Type, Debug)]
 pub struct S3UploadMeta {
     id: String,
-    user_id: String,
-    #[serde(default)]
-    aws_region: String,
-    #[serde(default, deserialize_with = "deserialize_empty_object_as_string")]
-    aws_bucket: String,
-    #[serde(default)]
-    aws_endpoint: String,
 }
 
 fn deserialize_empty_object_as_string<'de, D>(deserializer: D) -> Result<String, D::Error>
@@ -88,62 +81,16 @@ impl S3UploadMeta {
         &self.id
     }
 
-    pub fn user_id(&self) -> &str {
-        &self.user_id
-    }
-
-    pub fn aws_region(&self) -> &str {
-        &self.aws_region
-    }
-
-    pub fn aws_bucket(&self) -> &str {
-        &self.aws_bucket
-    }
-
-    pub fn aws_endpoint(&self) -> &str {
-        &self.aws_endpoint
-    }
-
-    pub fn new(
-        id: String,
-        user_id: String,
-        aws_region: String,
-        aws_bucket: String,
-        aws_endpoint: String,
-    ) -> Self {
-        Self {
-            id,
-            user_id,
-            aws_region,
-            aws_bucket,
-            aws_endpoint,
-        }
-    }
-
-    pub fn ensure_defaults(&mut self) {
-        if self.aws_region.is_empty() {
-            self.aws_region = std::env::var("NEXT_PUBLIC_CAP_AWS_REGION")
-                .unwrap_or_else(|_| "us-east-1".to_string());
-        }
-        if self.aws_bucket.is_empty() {
-            self.aws_bucket =
-                std::env::var("NEXT_PUBLIC_CAP_AWS_BUCKET").unwrap_or_else(|_| "capso".to_string());
-        }
-        if self.aws_endpoint.is_empty() {
-            self.aws_endpoint = std::env::var("NEXT_PUBLIC_CAP_AWS_ENDPOINT")
-                .unwrap_or_else(|_| "https://s3.amazonaws.com".to_string());
-        }
+    pub fn new(id: String) -> Self {
+        Self { id }
     }
 }
 
 #[derive(serde::Serialize)]
 #[serde(rename_all = "camelCase")]
 struct S3UploadBody {
-    user_id: String,
-    file_key: String,
-    aws_bucket: String,
-    aws_region: String,
-    aws_endpoint: String,
+    video_id: String,
+    subpath: String,
 }
 
 #[derive(serde::Serialize)]
@@ -201,33 +148,17 @@ pub async fn upload_video(
 ) -> Result<UploadedVideo, String> {
     println!("Uploading video {video_id}...");
 
-    let file_name = file_path
-        .file_name()
-        .and_then(|name| name.to_str())
-        .ok_or("Invalid file path")?
-        .to_string();
-
     let client = reqwest::Client::new();
     let s3_config = match existing_config {
         Some(config) => config,
-        None => create_or_get_video(app, false, Some(video_id), None).await?,
+        None => create_or_get_video(app, false, Some(video_id.clone()), None).await?,
     };
 
-    let file_key = format!(
-        "{}/{}/{}",
-        s3_config.user_id(),
-        s3_config.id(),
-        "result.mp4"
-    );
-
     let body = build_video_upload_body(
         &file_path,
         S3UploadBody {
-            user_id: s3_config.user_id().to_string(),
-            file_key: file_key.clone(),
-            aws_bucket: s3_config.aws_bucket().to_string(),
-            aws_region: s3_config.aws_region().to_string(),
-            aws_endpoint: s3_config.aws_endpoint().to_string(),
+            video_id: video_id.clone(),
+            subpath: "result.mp4".to_string(),
         },
     )?;
 
@@ -336,17 +267,10 @@ pub async fn upload_image(app: &AppHandle, file_path: PathBuf) -> Result<Uploade
     let client = reqwest::Client::new();
     let s3_config = create_or_get_video(app, true, None, None).await?;
 
-    let file_key = format!("{}/{}/{}", s3_config.user_id, s3_config.id, file_name);
-
-    println!("File key: {file_key}");
-
     let body = S3ImageUploadBody {
         base: S3UploadBody {
-            user_id: s3_config.user_id,
-            file_key: file_key.clone(),
-            aws_bucket: s3_config.aws_bucket,
-            aws_region: s3_config.aws_region,
-            aws_endpoint: s3_config.aws_endpoint,
+            video_id: s3_config.id.clone(),
+            subpath: file_name,
         },
     };
 
@@ -427,7 +351,6 @@ pub async fn create_or_get_video(
         )
     })?;
 
-    config.ensure_defaults();
     Ok(config)
 }
 
@@ -532,18 +455,10 @@ pub async fn prepare_screenshot_upload(
     s3_config: &S3UploadMeta,
     screenshot_path: PathBuf,
 ) -> Result<reqwest::Response, String> {
-    let file_key = format!(
-        "{}/{}/screenshot/screen-capture.jpg",
-        s3_config.user_id, s3_config.id
-    );
-
     let body = S3ImageUploadBody {
         base: S3UploadBody {
-            user_id: s3_config.user_id.clone(),
-            file_key: file_key.clone(),
-            aws_bucket: s3_config.aws_bucket.clone(),
-            aws_region: s3_config.aws_region.clone(),
-            aws_endpoint: s3_config.aws_endpoint.clone(),
+            video_id: s3_config.id.clone(),
+            subpath: "screenshot/screen-capture.jpg".to_string(),
         },
     };
 
@@ -632,10 +547,8 @@ impl InstantMultipartUpload {
         // --------------------------------------------
         // basic constants and info for chunk approach
         // --------------------------------------------
-        let file_name = "result.mp4";
         let client = reqwest::Client::new();
         let s3_config = pre_created_video.config;
-        let file_key = format!("{}/{}/{}", s3_config.user_id(), s3_config.id(), file_name);
 
         let mut uploaded_parts = Vec::new();
         let mut part_number = 1;
@@ -652,7 +565,7 @@ impl InstantMultipartUpload {
                 c.post(url)
                     .header("Content-Type", "application/json")
                     .json(&serde_json::json!({
-                        "fileKey": file_key,
+                        "videoId": s3_config.id(),
                         "contentType": "video/mp4"
                     }))
             })
@@ -689,6 +602,7 @@ impl InstantMultipartUpload {
                 return Err("No uploadId returned from initiate endpoint".to_string());
             }
         };
+
         if upload_id.is_empty() {
             return Err("Empty uploadId returned from initiate endpoint".to_string());
         }
@@ -747,7 +661,7 @@ impl InstantMultipartUpload {
                     &app,
                     &client,
                     &file_path,
-                    &file_key,
+                    s3_config.id(),
                     &upload_id,
                     &mut part_number,
                     &mut last_uploaded_position,
@@ -774,7 +688,7 @@ impl InstantMultipartUpload {
                         &app,
                         &client,
                         &file_path,
-                        &file_key,
+                        s3_config.id(),
                         &upload_id,
                         &mut 1,
                         &mut 0,
@@ -800,10 +714,9 @@ impl InstantMultipartUpload {
                 Self::finalize_upload(
                     &app,
                     &file_path,
-                    &file_key,
+                    &s3_config.id(),
                     &upload_id,
                     &uploaded_parts,
-                    &video_id,
                 )
                 .await?;
 
@@ -825,7 +738,7 @@ impl InstantMultipartUpload {
         app: &AppHandle,
         client: &reqwest::Client,
         file_path: &PathBuf,
-        file_key: &str,
+        video_id: &str,
         upload_id: &str,
         part_number: &mut i32,
         last_uploaded_position: &mut u64,
@@ -929,7 +842,7 @@ impl InstantMultipartUpload {
                 c.post(url)
                     .header("Content-Type", "application/json")
                     .json(&serde_json::json!({
-                        "fileKey": file_key,
+                        "videoId": video_id,
                         "uploadId": upload_id,
                         "partNumber": *part_number,
                         "md5Sum": &md5_sum
@@ -1069,10 +982,9 @@ impl InstantMultipartUpload {
     async fn finalize_upload(
         app: &AppHandle,
         file_path: &PathBuf,
-        file_key: &str,
+        video_id: &str,
         upload_id: &str,
         uploaded_parts: &[UploadedPart],
-        video_id: &str,
     ) -> Result<(), String> {
         println!(
             "Completing multipart upload with {} parts",
@@ -1106,7 +1018,7 @@ impl InstantMultipartUpload {
                 c.post(url)
                     .header("Content-Type", "application/json")
                     .json(&serde_json::json!({
-                        "fileKey": file_key,
+                        "videoId": video_id,
                         "uploadId": upload_id,
                         "parts": uploaded_parts
                     }))

apps/desktop/src-tauri/src/web_api.rs

@@ -14,16 +14,18 @@ async fn do_authed_request(
 ) -> Result<reqwest::Response, reqwest::Error> {
     let client = reqwest::Client::new();
 
-    let mut req = build(client, url).header(
-        "Authorization",
-        format!(
-            "Bearer {}",
-            match &auth.secret {
-                AuthSecret::ApiKey { api_key } => api_key,
-                AuthSecret::Session { token, .. } => token,
-            }
-        ),
-    );
+    let mut req = build(client, url)
+        .header(
+            "Authorization",
+            format!(
+                "Bearer {}",
+                match &auth.secret {
+                    AuthSecret::ApiKey { api_key } => api_key,
+                    AuthSecret::Session { token, .. } => token,
+                }
+            ),
+        )
+        .header("X-Desktop-Version", env!("CARGO_PKG_VERSION"));
 
     if let Some(s) = std::option_env!("VITE_VERCEL_AUTOMATION_BYPASS_SECRET") {
         req = req.header("x-vercel-protection-bypass", s);

apps/desktop/src/utils/tauri.ts

@@ -363,7 +363,7 @@ export type RequestOpenSettings = { page: string }
 export type RequestRestartRecording = null
 export type RequestStartRecording = null
 export type RequestStopRecording = null
-export type S3UploadMeta = { id: string; user_id: string; aws_region?: string; aws_bucket?: string; aws_endpoint?: string }
+export type S3UploadMeta = { id: string }
 export type ScreenCaptureTarget = { variant: "window"; id: number } | { variant: "screen"; id: number } | { variant: "area"; screen: number; bounds: Bounds }
 export type SegmentRecordings = { display: Video; camera: Video | null; mic: Audio | null; system_audio: Audio | null }
 export type SerializedEditorInstance = { framesSocketUrl: string; recordingDuration: number; savedProjectConfig: ProjectConfiguration; recordings: ProjectRecordings; path: string }

apps/web/actions/organization/create-space.ts

@@ -6,8 +6,7 @@ import { spaces, users, spaceMembers } from "@cap/database/schema";
 import { inArray, eq, and } from "drizzle-orm";
 import { nanoId, nanoIdLength } from "@cap/database/helpers";
 import { revalidatePath } from "next/cache";
-import { createS3Client, getS3Bucket } from "@/utils/s3";
-import { createPresignedPost } from "@aws-sdk/s3-presigned-post";
+import { createBucketProvider } from "@/utils/s3";
 import { serverEnv } from "@cap/env";
 import { v4 as uuidv4 } from "uuid";
 
@@ -52,6 +51,7 @@ export async function createSpace(
         )
       )
       .limit(1);
+
     if (existingSpace.length > 0) {
       return {
         success: false,
@@ -89,46 +89,22 @@ export async function createSpace(
           user.activeOrganizationId
         }/spaces/${spaceId}/icon-${Date.now()}.${fileExtension}`;
 
-        // Get S3 client
-        const [s3Client] = await createS3Client();
-        const bucketName = await getS3Bucket();
-
-        // Create presigned post
-        const presignedPostData = await createPresignedPost(s3Client, {
-          Bucket: bucketName,
-          Key: fileKey,
-          Fields: {
-            "Content-Type": iconFile.type,
-          },
-          Expires: 600, // 10 minutes
-        });
-
-        // Upload file to S3
-        const formDataForS3 = new FormData();
-        Object.entries(presignedPostData.fields).forEach(([key, value]) => {
-          formDataForS3.append(key, value as string);
-        });
-        formDataForS3.append("file", iconFile);
+        const bucket = await createBucketProvider();
 
-        const uploadResponse = await fetch(presignedPostData.url, {
-          method: "POST",
-          body: formDataForS3,
+        await bucket.putObject(fileKey, await iconFile.bytes(), {
+          contentType: iconFile.type,
         });
 
-        if (!uploadResponse.ok) {
-          throw new Error("Failed to upload file to S3");
-        }
-
         // Construct the icon URL
         if (serverEnv().CAP_AWS_BUCKET_URL) {
           // If a custom bucket URL is defined, use it
           iconUrl = `${serverEnv().CAP_AWS_BUCKET_URL}/${fileKey}`;
         } else if (serverEnv().CAP_AWS_ENDPOINT) {
           // For custom endpoints like MinIO
-          iconUrl = `${serverEnv().CAP_AWS_ENDPOINT}/${bucketName}/${fileKey}`;
+          iconUrl = `${serverEnv().CAP_AWS_ENDPOINT}/${bucket.name}/${fileKey}`;
         } else {
           // Default AWS S3 URL format
-          iconUrl = `https://${bucketName}.s3.${
+          iconUrl = `https://${bucket.name}.s3.${
             serverEnv().CAP_AWS_REGION || "us-east-1"
           }.amazonaws.com/${fileKey}`;
         }