Merge pull request #76 from ClickHouse/issue-74-download-sources

Use builds.clickhouse.com as primary download source

Author: sdairs

.github/workflows/test-install.yml

@@ -0,0 +1,139 @@
+name: Test Install
+
+on:
+  pull_request:
+    paths:
+      - "src/version_manager/**"
+      - "src/main.rs"
+      - "src/cli.rs"
+      - "Cargo.toml"
+      - ".github/workflows/test-install.yml"
+
+jobs:
+  unit-tests:
+    name: Unit Tests
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@stable
+      - run: cargo test
+
+  url-probing:
+    name: URL Probing (${{ matrix.platform }})
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - platform: amd64
+            desc: Linux x86_64
+          - platform: aarch64
+            desc: Linux ARM64
+          - platform: macos
+            desc: macOS x86_64
+          - platform: macos-aarch64
+            desc: macOS ARM64
+    steps:
+      - name: Verify builds.clickhouse.com URLs
+        run: |
+          echo "Testing builds.clickhouse.com for ${{ matrix.desc }}..."
+
+          # master should always be available
+          echo -n "  master: "
+          curl -sf -o /dev/null -w "%{http_code}" \
+            "https://builds.clickhouse.com/master/${{ matrix.platform }}/clickhouse"
+          echo ""
+
+          # A known recent version should be available
+          echo -n "  25.12: "
+          curl -sf -o /dev/null -w "%{http_code}" \
+            "https://builds.clickhouse.com/25.12/${{ matrix.platform }}/clickhouse"
+          echo ""
+
+          # A version too old should return 403
+          echo -n "  24.1 (expect 403): "
+          code=$(curl -s -o /dev/null -w "%{http_code}" \
+            "https://builds.clickhouse.com/24.1/${{ matrix.platform }}/clickhouse")
+          echo "$code"
+          if [ "$code" != "403" ]; then
+            echo "Expected 403 for old version, got $code"
+            exit 1
+          fi
+
+      - name: Verify packages.clickhouse.com URLs (Linux only)
+        if: matrix.platform == 'amd64' || matrix.platform == 'aarch64'
+        run: |
+          arch=${{ matrix.platform }}
+          # packages uses arm64 not aarch64
+          if [ "$arch" = "aarch64" ]; then arch="arm64"; fi
+
+          echo "Testing packages.clickhouse.com for ${{ matrix.desc }}..."
+
+          # Check a known stable version exists
+          echo -n "  stable tarball: "
+          curl -sf -o /dev/null -w "%{http_code}" -I \
+            "https://packages.clickhouse.com/tgz/stable/clickhouse-common-static-25.3.2.39-${arch}.tgz"
+          echo ""
+
+          # Check client-only tarball exists
+          echo -n "  client-only tarball: "
+          curl -sf -o /dev/null -w "%{http_code}" -I \
+            "https://packages.clickhouse.com/tgz/stable/clickhouse-client-25.3.2.39-${arch}.tgz"
+          echo ""
+
+  install-test:
+    name: Install Test (${{ matrix.os }})
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: dtolnay/rust-toolchain@stable
+
+      - name: Build CLI
+        run: cargo build
+
+      - name: Test install latest
+        run: |
+          ./target/debug/clickhousectl local install latest
+          # Verify a version was installed
+          ./target/debug/clickhousectl local list | grep -E "^  [0-9]"
+
+      - name: Test install minor version
+        run: |
+          ./target/debug/clickhousectl local install 25.12
+          ./target/debug/clickhousectl local list | grep "25.12"
+
+      - name: Test local use
+        run: |
+          ./target/debug/clickhousectl local use 25.12
+          ./target/debug/clickhousectl local which | grep "25.12"
+
+      - name: Test already installed (skips download)
+        run: |
+          ./target/debug/clickhousectl local install 25.12 2>&1 | grep "already installed"
+
+      - name: Test force re-install
+        run: |
+          ./target/debug/clickhousectl local install 25.12 --force
+
+      - name: Test invalid version (expect error)
+        run: |
+          if ./target/debug/clickhousectl local install foo 2>&1; then
+            echo "Should have errored for invalid version"
+            exit 1
+          fi
+
+      - name: Test 3-part version rejected (expect error)
+        run: |
+          if ./target/debug/clickhousectl local install 25.12.9 2>&1; then
+            echo "Should have errored for 3-part version"
+            exit 1
+          fi
+
+      - name: Verify installed binary works
+        run: |
+          version_dir=$(ls -d ~/.clickhouse/versions/*/clickhouse | head -1)
+          $version_dir --version

src/cli.rs

@@ -114,21 +114,21 @@ pub enum LocalCommands {
     /// Install a ClickHouse version
     #[command(after_help = "\
 CONTEXT FOR AGENTS:
-  Downloads a ClickHouse binary to ~/.clickhouse/versions/{version}/.
-  Accepts version specs: \"stable\", \"lts\", partial like \"25.12\", or exact like \"25.12.5.44\".
-  Optionally set as default with `clickhousectl local use <version>`.
-  `clickhousectl local use <version>` will auto-install if the version is missing and set as default.
-  Related: `clickhousectl local list --remote` to see downloadable versions.")]
+  `clickhousectl local use <version>` will auto-install if the version is missing and set as default.")]
     Install {
-        /// Version to install (e.g., 25.1.2.3, 25.1, stable, lts)
+        /// Version to install (e.g., latest, stable, lts, 25, 25.12, 25.12.9.61)
         version: String,
+
+        /// Force re-install even if version is already installed
+        #[arg(long)]
+        force: bool,
     },
 
     /// List installed versions
     #[command(after_help = "\
 CONTEXT FOR AGENTS:
   Without flags: shows locally installed versions (exact version strings).
-  With --remote: shows versions available for download from GitHub releases.
+  With --remote: shows versions available for download from builds.clickhouse.com.
   Use the exact version strings from this output with `clickhousectl local remove` or `clickhousectl local use`.
   Related: `clickhousectl local install <version>` to install, `clickhousectl local which` to see current default.")]
     List {

src/cloud/commands.rs

@@ -1713,22 +1713,25 @@ async fn ensure_version_installed(
 
     let version_spec = service_version.ok_or("service has no clickhouse_version set")?;
 
-    // Resolve the spec to an exact version first (e.g. "25.12" -> "25.12.9.61")
-    let resolved = version_manager::resolve_version(version_spec).await?;
+    let spec = version_manager::parse_version_spec(version_spec)?;
+    let platform = version_manager::platform::Platform::detect()?;
+    let resolved = version_manager::resolve::resolve(&spec, &platform).await?;
 
-    // Check if already installed
-    let version_dir = paths::version_dir(&resolved.version)?;
-    if version_dir.exists() {
-        return Ok(resolved.version);
+    // If exact version is known, check if already installed
+    if let Some(ref version) = resolved.exact_version {
+        let version_dir = paths::version_dir(version)?;
+        if version_dir.exists() {
+            return Ok(version.clone());
+        }
     }
 
-    // Install
     eprintln!(
         "Service requires ClickHouse {} — downloading...",
-        resolved.version
+        resolved.display_version
     );
-    version_manager::install_version(&resolved.version, resolved.channel).await?;
-    Ok(resolved.version)
+    let version =
+        version_manager::install::install_resolved(&resolved, &platform, false).await?;
+    Ok(version)
 }
 
 #[cfg(test)]

src/main.rs

@@ -46,7 +46,7 @@ async fn run_skills(args: SkillsArgs) -> Result<()> {
 
 async fn run_local(cmd: LocalCommands) -> Result<()> {
     match cmd {
-        LocalCommands::Install { version } => install(&version).await,
+        LocalCommands::Install { version, force } => install(&version, force).await,
         LocalCommands::List { remote } => {
             if remote {
                 list_available().await
@@ -73,12 +73,21 @@ async fn run_local(cmd: LocalCommands) -> Result<()> {
     }
 }
 
-async fn install(version_spec: &str) -> Result<()> {
-    println!("Resolving version {}...", version_spec);
-    let entry = version_manager::resolve_version(version_spec).await?;
-    println!("Resolved to version {} ({})", entry.version, entry.channel);
+async fn install(version_spec: &str, force: bool) -> Result<()> {
+    let spec = version_manager::parse_version_spec(version_spec)?;
+    let platform = version_manager::platform::Platform::detect()?;
+
+    eprintln!("Resolving {}...", spec);
+    let resolved = version_manager::resolve::resolve(&spec, &platform).await?;
+
+    let version = version_manager::install::install_resolved(&resolved, &platform, force).await?;
+
+    // If this is the first installed version, set it as default
+    if version_manager::get_default_version().is_err() {
+        version_manager::set_default_version(&version)?;
+        eprintln!("Set as default version");
+    }
 
-    version_manager::install_version(&entry.version, entry.channel).await?;
     Ok(())
 }
 
@@ -105,8 +114,8 @@ fn list_installed() -> Result<()> {
 }
 
 async fn list_available() -> Result<()> {
-    println!("Fetching available versions...");
-    let versions = version_manager::list_available_versions().await?;
+    eprintln!("Checking available versions on builds.clickhouse.com...");
+    let versions = version_manager::list_available_versions_from_builds().await?;
 
     if versions.is_empty() {
         println!("No versions available");
@@ -115,36 +124,47 @@ async fn list_available() -> Result<()> {
 
     let installed = version_manager::list_installed_versions().unwrap_or_default();
 
-    println!("Available versions:");
-    for entry in versions.iter().take(20) {
-        if installed.contains(&entry.version) {
-            println!("  {} [{}] (installed)", entry.version, entry.channel);
+    println!("Available versions (builds.clickhouse.com):");
+    for v in &versions {
+        // Check if any installed version matches this minor
+        let prefix = format!("{}.", v);
+        let is_installed = installed.iter().any(|iv| iv.starts_with(&prefix) || iv == v);
+        if is_installed {
+            println!("  {} (installed)", v);
         } else {
-            println!("  {} [{}]", entry.version, entry.channel);
+            println!("  {}", v);
         }
     }
-
-    if versions.len() > 20 {
-        println!("  ... and {} more", versions.len() - 20);
-    }
+    println!();
+    println!("Install with: clickhousectl local install <version>");
+    println!("For exact patch versions, use: clickhousectl local install 25.12.9.61");
 
     Ok(())
 }
 
 async fn use_version(version_spec: &str) -> Result<()> {
-    println!("Resolving version {}...", version_spec);
-    let entry = version_manager::resolve_version(version_spec).await?;
-    let version = &entry.version;
-
-    // Install if not already installed
-    let installed = version_manager::list_installed_versions()?;
-    if !installed.contains(version) {
-        println!("Version {} not installed, installing...", version);
-        version_manager::install_version(version, entry.channel).await?;
-    }
+    let spec = version_manager::parse_version_spec(version_spec)?;
+    let platform = version_manager::platform::Platform::detect()?;
+
+    eprintln!("Resolving {}...", spec);
+    let resolved = version_manager::resolve::resolve(&spec, &platform).await?;
+
+    // If exact version is known, check if already installed
+    let version = if let Some(ref v) = resolved.exact_version {
+        let installed = version_manager::list_installed_versions()?;
+        if installed.contains(v) {
+            v.clone()
+        } else {
+            eprintln!("Version {} not installed, installing...", v);
+            version_manager::install::install_resolved(&resolved, &platform, false).await?
+        }
+    } else {
+        // Version not known upfront (builds source) — install will detect it
+        version_manager::install::install_resolved(&resolved, &platform, false).await?
+    };
 
-    version_manager::set_default_version(version)?;
-    println!("Default version set to {}", version);
+    version_manager::set_default_version(&version)?;
+    eprintln!("Default version set to {}", version);
     Ok(())
 }
 

src/version_manager/download.rs

@@ -1,18 +1,28 @@
 use crate::error::{Error, Result};
-use crate::version_manager::list::Channel;
-use crate::version_manager::resolve::build_download_url;
+use crate::version_manager::platform::{DownloadSource, Platform};
 use futures_util::StreamExt;
 use indicatif::{ProgressBar, ProgressStyle};
 use std::path::Path;
 use tokio::io::AsyncWriteExt;
 
-/// Downloads a ClickHouse version to the specified path
-pub async fn download_version(version: &str, channel: Channel, dest_path: &Path) -> Result<()> {
-    let url = build_download_url(version, channel)?;
+/// Downloads from a DownloadSource to the specified path
+pub async fn download_from_source(
+    source: &DownloadSource,
+    platform: &Platform,
+    dest_path: &Path,
+) -> Result<()> {
+    let url = source.url(platform);
+    download_url(&url, dest_path).await
+}
+
+/// Downloads a file from a URL to the specified path, with progress bar
+pub async fn download_url(url: &str, dest_path: &Path) -> Result<()> {
+    let client = reqwest::Client::builder()
+        .user_agent("clickhousectl")
+        .build()?;
 
-    let client = reqwest::Client::new();
     let response = client
-        .get(&url)
+        .get(url)
         .send()
         .await?
         .error_for_status()