feature: clone qdrant from offset script

Author: cdxker

.github/workflows/push-server.yml

@@ -1032,3 +1032,53 @@ jobs:
           push: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
+
+  clone-qdrant-from-offset:
+    name: Push Clone From Offset
+    runs-on: ${{ matrix.runner }}
+    strategy:
+      matrix:
+        include:
+          - runner: blacksmith-8vcpu-ubuntu-2204
+            platform: linux/amd64
+            tag: latest
+          # - runner: blacksmith-8vcpu-ubuntu-2204
+          #   platform: linux/arm64
+          #   tag: latest-arm
+          #   suffix: -arm
+    steps:
+      - name: Checkout the repo
+        uses: actions/checkout@v4
+
+      # - name: Set up QEMU
+      #   uses: docker/setup-qemu-action@v3
+
+      - name: Setup buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          # list of Docker images to use as base name for tags
+          images: |
+            trieve/clone-qdrant-from-offset
+          tags: |
+            type=raw,${{matrix.tag}}
+            type=sha,suffix=${{matrix.suffix}}
+
+      - name: Build and push Docker image
+        uses: useblacksmith/build-push-action@v1.0.0-beta
+        with:
+          platforms: ${{ matrix.platform }}
+          context: server/
+          file: ./server/Dockerfile.clone-qdrant-from-offset
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

server/Cargo.toml

@@ -89,6 +89,10 @@ path = "src/bin/backfill-qdrant-from-pg.rs"
 name = "send-stripe-billing"
 path = "src/bin/send-stripe-billing.rs"
 
+[[bin]]
+name = "clone-qdrant-from-offset"
+path = "src/bin/clone-qdrant-from-offset.rs"
+
 [dependencies]
 actix-identity = { version = "0.7.1" }
 actix-session = { version = "0.9.0", features = [

server/Dockerfile.backfill-qdrant-from-pg

@@ -3,7 +3,7 @@ FROM rust:1.81-slim-bookworm AS chef
 # it will be cached from the second build onwards
 RUN apt-get update -y && apt-get -y install pkg-config libssl-dev libpq-dev g++ curl
 RUN cargo install cargo-chef 
-WORKDIR app
+WORKDIR /app
 
 FROM chef AS planner
 COPY . .

server/Dockerfile.clone-qdrant-from-offset

@@ -0,0 +1,28 @@
+FROM rust:1.81-slim-bookworm AS chef
+# We only pay the installation cost once, 
+# it will be cached from the second build onwards
+RUN apt-get update -y && apt-get -y install pkg-config libssl-dev libpq-dev g++ curl
+RUN cargo install cargo-chef 
+WORKDIR /app
+
+FROM chef AS planner
+COPY . .
+RUN cargo chef prepare  --recipe-path recipe.json
+
+FROM chef AS builder
+COPY --from=planner /app/recipe.json recipe.json
+# Build dependencies - this is the caching Docker layer!
+RUN cargo chef cook --release --recipe-path recipe.json --bin "clone-qdrant-from-offset"
+# Build application
+COPY . .
+RUN cargo build --release --features "runtime-env" --bin "clone-qdrant-from-offset"
+
+FROM debian:bookworm-slim as runtime
+RUN apt-get update -y && apt-get -y install pkg-config libssl-dev libpq-dev ca-certificates 
+WORKDIR /app
+COPY ./migrations/ /app/migrations
+COPY --from=builder /app/target/release/backfill-qdrant-from-offset /app/backfill-qdrant-from-offset
+
+
+EXPOSE 8090
+ENTRYPOINT ["/app/clone-qdrant-from-offset"]