refactor: rename sync core and export s3 boundaries (#15)

Author: anandpant

Cargo.lock

@@ -1,8 +1,8 @@
 [workspace]
 members = [
   "apps/convex-sync",
-  "crates/convex-cdc-core",
-  "crates/convex-target-s3",
+  "crates/convex-sync-core",
+  "crates/convex-export-s3",
 ]
 resolver = "2"
 

Cargo.toml

@@ -24,8 +24,8 @@ extraction model:
 flowchart TD
   Root[convex-streaming-olap-export]
   CLI[apps/convex-sync]
-  Core[crates/convex-cdc-core]
-  S3[crates/convex-target-s3]
+  Core[crates/convex-sync-core]
+  S3[crates/convex-export-s3]
   AWS[platform/aws]
   DBS3[platform/databricks/s3]
   DBN[platform/databricks/native]
@@ -40,8 +40,8 @@ flowchart TD
 Read the repo by layer:
 
 - [`apps/convex-sync/README.md`](apps/convex-sync/README.md): CLI surface and S3/export runtime commands
-- `crates/convex-cdc-core/`: shared Convex client, checkpoint FSM, event normalization, sync engine
-- `crates/convex-target-s3/`: raw parquet sink, staging materialization, S3 publish flow
+- `crates/convex-sync-core/`: shared Convex client, checkpoint FSM, event normalization, sync engine
+- `crates/convex-export-s3/`: raw parquet sink, staging materialization, S3 publish flow
 - [`platform/aws/README.md`](platform/aws/README.md): AWS assets for publishing and downstream readers
 - [`platform/databricks/README.md`](platform/databricks/README.md): Databricks target family overview
 - [`platform/databricks/s3/README.md`](platform/databricks/s3/README.md): Databricks consuming the S3 export path

README.md

@@ -11,8 +11,8 @@ path = "src/main.rs"
 
 [dependencies]
 clap.workspace = true
-convex-cdc-core = { path = "../../crates/convex-cdc-core" }
-convex-target-s3 = { path = "../../crates/convex-target-s3" }
+convex-sync-core = { path = "../../crates/convex-sync-core" }
+convex-export-s3 = { path = "../../crates/convex-export-s3" }
 dotenvy.workspace = true
 tokio.workspace = true
 url.workspace = true

apps/convex-sync/Cargo.toml

@@ -5,11 +5,18 @@ use std::{
 };
 
 use clap::{Args, Parser, Subcommand};
-use convex_cdc_core::{
+use convex_export_s3::{
+    publish::{publish_staging_to_s3, PublishS3Options},
+    service::{run_service, RunOptions},
+    sink::parquet::ParquetRawChangeLogWriter,
+    staging::materialize::{MaterializeStagingOptions, StagingMaterializer},
+};
+use convex_sync_core::{
     config::{ConvexConnectionConfig, OutputConfig, OutputFormat},
     convex::{client::ConvexClient, schemas::JsonSchemasQuery},
     errors::AppResult,
     model::schema::SchemaCatalog,
+    output::{write_jsonl_stream, write_value},
     state::checkpoint_store::FileCheckpointStore,
     sync::{
         delta_sync::{fetch_delta_events, DeltaSyncOptions},
@@ -18,15 +25,6 @@ use convex_cdc_core::{
     },
     telemetry::{logging, metrics},
 };
-use convex_target_s3::{
-    publish::{publish_staging_to_s3, PublishS3Options},
-    service::{run_service, RunOptions},
-    sink::{
-        jsonl::{write_jsonl_stream, write_value},
-        parquet::ParquetRawChangeLogWriter,
-    },
-    staging::materialize::{MaterializeStagingOptions, StagingMaterializer},
-};
 use url::Url;
 
 const CLI_VERSION: &str = match option_env!("CONVEX_SYNC_VERSION") {
@@ -246,7 +244,7 @@ async fn handle_snapshot(client: &ConvexClient, args: SnapshotArgs) -> AppResult
     let mut writer = open_writer(&output)?;
     if args.raw {
         let response = client
-            .list_snapshot(&convex_cdc_core::convex::snapshot::ListSnapshotQuery {
+            .list_snapshot(&convex_sync_core::convex::snapshot::ListSnapshotQuery {
                 snapshot: args.snapshot,
                 cursor: args.cursor,
                 table_name: args.table_name,
@@ -287,7 +285,7 @@ async fn handle_deltas(client: &ConvexClient, args: DeltasArgs) -> AppResult<()>
     let mut writer = open_writer(&output)?;
     if args.raw {
         let response = client
-            .document_deltas(&convex_cdc_core::convex::deltas::DocumentDeltasQuery {
+            .document_deltas(&convex_sync_core::convex::deltas::DocumentDeltasQuery {
                 cursor: args.cursor,
                 table_name: args.table_name,
             })
@@ -394,10 +392,10 @@ async fn load_schema_catalog(client: &ConvexClient) -> AppResult<SchemaCatalog>
 
 fn build_client(connection: &ConnectionArgs) -> AppResult<ConvexClient> {
     let deployment_url = connection.deployment_url.clone().ok_or(
-        convex_cdc_core::errors::AppError::MissingRequiredConfig("CONVEX_DEPLOYMENT_URL"),
+        convex_sync_core::errors::AppError::MissingRequiredConfig("CONVEX_DEPLOYMENT_URL"),
     )?;
     let deploy_key = connection.deploy_key.clone().ok_or(
-        convex_cdc_core::errors::AppError::MissingRequiredConfig("CONVEX_DEPLOY_KEY"),
+        convex_sync_core::errors::AppError::MissingRequiredConfig("CONVEX_DEPLOY_KEY"),
     )?;
     ConvexClient::new(ConvexConnectionConfig::new(deployment_url, deploy_key)?)
 }

apps/convex-sync/src/main.rs

@@ -0,0 +1,26 @@
+[package]
+name = "convex-export-s3"
+edition.workspace = true
+license.workspace = true
+publish.workspace = true
+version = "0.0.1"
+
+[lib]
+name = "convex_export_s3"
+path = "src/lib.rs"
+
+[dependencies]
+arrow-array.workspace = true
+arrow-schema.workspace = true
+aws-config.workspace = true
+aws-sdk-s3.workspace = true
+convex-sync-core = { path = "../convex-sync-core" }
+hex.workspace = true
+parquet.workspace = true
+serde.workspace = true
+serde_json.workspace = true
+sha2.workspace = true
+tokio.workspace = true
+tracing.workspace = true
+url.workspace = true
+walkdir.workspace = true

crates/convex-export-s3/Cargo.toml

@@ -0,0 +1,4 @@
+pub mod publish;
+pub mod service;
+pub mod sink;
+pub mod staging;