Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
223 changes: 173 additions & 50 deletions .github/workflows/release-build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ permissions:

env:
REGISTRY_GHCR: ghcr.io
IMAGE_NAME: basekick-labs/arc-go
IMAGE_NAME: basekick-labs/arc

jobs:
# Extract version from branch name or input
Expand Down Expand Up @@ -118,43 +118,44 @@ jobs:
if-no-files-found: error
retention-days: 1

# Build Go binaries for all platforms
# Build Go binaries using native runners (CGO required for SQLite)
build-binaries:
name: Build Binaries
runs-on: ubuntu-latest
needs: prepare
strategy:
matrix:
include:
- goos: linux
goarch: amd64
- runner: ubuntu-latest
suffix: linux-amd64
- goos: linux
goarch: arm64
- runner: ubuntu-24.04-arm
suffix: linux-arm64
runs-on: ${{ matrix.runner }}
steps:
- name: Checkout code
uses: actions/checkout@v4

- name: Set up Go
uses: actions/setup-go@v5
with:
go-version: '1.22'
go-version: '1.23'
cache: true

- name: Build binary
env:
GOOS: ${{ matrix.goos }}
GOARCH: ${{ matrix.goarch }}
CGO_ENABLED: 0
CGO_ENABLED: 1
run: |
VERSION=${{ needs.prepare.outputs.version }}

# Build without CGO for static binary (no Arrow support in packages)
go build -v \
-ldflags="-s -w -X main.Version=${VERSION}" \
go build -ldflags "-s -w -X main.Version=${VERSION}" \
-o arc-${{ matrix.suffix }} \
./cmd/arc

chmod +x arc-${{ matrix.suffix }}

# Show binary info
file arc-${{ matrix.suffix }}
ls -lh arc-${{ matrix.suffix }}

- name: Upload binary
uses: actions/upload-artifact@v4
with:
Expand Down Expand Up @@ -241,7 +242,7 @@ jobs:
.
Features: 9.47M records/sec ingestion, DuckDB SQL queries,
Parquet storage, S3/MinIO support.
Homepage: https://github.com/basekick-labs/arc-go
Homepage: https://github.com/basekick-labs/arc
EOF

# Create postinst script
Expand Down Expand Up @@ -365,10 +366,9 @@ jobs:
Release: 1%{?dist}
Summary: High-Performance Time-Series Database
License: AGPL-3.0
URL: https://github.com/basekick-labs/arc-go
URL: https://github.com/basekick-labs/arc
Source0: arc-${VERSION}.tar.gz
Source1: arc.service
BuildArch: ${ARCH}

%description
Arc is a high-performance time-series database built on DuckDB,
Expand Down Expand Up @@ -421,7 +421,7 @@ jobs:
EOF

# Build RPM
rpmbuild -ba ~/rpmbuild/SPECS/arc.spec
rpmbuild -ba --target ${ARCH} ~/rpmbuild/SPECS/arc.spec

# Copy to current directory
cp ~/rpmbuild/RPMS/${ARCH}/arc-${VERSION}-1.*.rpm .
Expand Down Expand Up @@ -467,14 +467,114 @@ jobs:
- name: Create manifest list and push
working-directory: /tmp/digests
run: |
# Create multi-arch manifest
# Create multi-arch manifest with version, short_version, and latest tags
docker buildx imagetools create \
-t ${{ env.REGISTRY_GHCR }}/${{ env.IMAGE_NAME }}:${{ needs.prepare.outputs.version }} \
-t ${{ env.REGISTRY_GHCR }}/${{ env.IMAGE_NAME }}:${{ needs.prepare.outputs.short_version }} \
-t ${{ env.REGISTRY_GHCR }}/${{ env.IMAGE_NAME }}:latest \
$(printf '${{ env.REGISTRY_GHCR }}/${{ env.IMAGE_NAME }}@sha256:%s ' *)

echo "✅ Multi-arch manifest created and pushed" >> $GITHUB_STEP_SUMMARY
echo "Tags: ${{ needs.prepare.outputs.version }}, ${{ needs.prepare.outputs.short_version }}" >> $GITHUB_STEP_SUMMARY
echo "Tags: ${{ needs.prepare.outputs.version }}, ${{ needs.prepare.outputs.short_version }}, latest" >> $GITHUB_STEP_SUMMARY

# Test Docker image health
test-docker:
name: Test Docker Image
runs-on: ubuntu-latest
needs: [prepare, docker-merge]
steps:
- name: Pull and run Docker image
run: |
VERSION=${{ needs.prepare.outputs.version }}
IMAGE="${{ env.REGISTRY_GHCR }}/${{ env.IMAGE_NAME }}:${VERSION}"

echo "Pulling image: ${IMAGE}"
docker pull ${IMAGE}

echo "Starting Arc container..."
docker run -d --name arc-test -p 8000:8000 ${IMAGE}

# Wait for startup
echo "Waiting for Arc to start..."
sleep 10

# Test health endpoint
echo "Testing health endpoint..."
for i in {1..30}; do
if curl -sf http://localhost:8000/health; then
echo ""
echo "✅ Health check passed!"
docker logs arc-test
exit 0
fi
echo "Attempt $i/30 - waiting..."
sleep 2
done

echo "❌ Health check failed after 60 seconds"
docker logs arc-test
exit 1

- name: Test version endpoint
run: |
VERSION=${{ needs.prepare.outputs.version }}

# Check if version endpoint returns expected version
RESPONSE=$(curl -sf http://localhost:8000/health || echo "{}")
echo "Health response: ${RESPONSE}"

# Verify version is not "dev"
if echo "${RESPONSE}" | grep -q '"version"'; then
echo "✅ Version info present in health response"
fi

- name: Cleanup
if: always()
run: |
docker stop arc-test || true
docker rm arc-test || true

# Test binary execution
test-binary:
name: Test Binary
runs-on: ubuntu-latest
needs: [prepare, build-binaries]
steps:
- name: Download amd64 binary
uses: actions/download-artifact@v4
with:
name: arc-binary-linux-amd64

- name: Test binary execution
run: |
chmod +x arc-linux-amd64

# Test --version or basic startup
echo "Testing binary execution..."

# Start Arc in background
./arc-linux-amd64 &
ARC_PID=$!

# Wait for startup
sleep 5

# Test health endpoint
echo "Testing health endpoint..."
for i in {1..20}; do
if curl -sf http://localhost:8000/health; then
echo ""
echo "✅ Binary health check passed!"
kill $ARC_PID || true
exit 0
fi
echo "Attempt $i/20 - waiting..."
sleep 2
done

echo "❌ Binary health check failed"
kill $ARC_PID || true
exit 1

# Package Helm chart
helm-package:
Expand Down Expand Up @@ -601,7 +701,7 @@ jobs:
create-draft-release:
name: Create Draft Release
runs-on: ubuntu-latest
needs: [prepare, docker-merge, helm-package, test-helm, debian-build, rpm-build]
needs: [prepare, docker-merge, test-docker, test-binary, helm-package, test-helm, debian-build, rpm-build]
permissions:
contents: write
steps:
Expand All @@ -623,9 +723,35 @@ jobs:
VERSION=${{ needs.prepare.outputs.version }}

cat > release-notes.md << 'EOF'
# Arc ${{ needs.prepare.outputs.version }} (Go)
# Arc v${{ needs.prepare.outputs.version }} - Go Implementation

**Major Release: Complete rewrite from Python to Go**

Arc is a high-performance time-series database built on DuckDB, optimized for IoT, observability, and analytics workloads.

## 🚀 Migration Highlights

High-performance time-series database built on DuckDB. Go implementation.
This release marks the complete migration from Python to Go, delivering:

### Performance Improvements
- **9.47M records/sec** MessagePack ingestion (125% faster than Python's 4.21M)
- **1.92M records/sec** Line Protocol ingestion (76% faster than Python's 1.09M)
- **2.88M rows/sec** Arrow query throughput

### Reliability
- **Memory stable**: No memory leaks (Python leaked 372MB per 500 queries)
- **Single binary**: No Python dependencies, pip, or virtual environments
- **Type-safe**: Strong typing catches bugs at compile time

### Full Feature Parity
- ✅ Authentication (user/password)
- ✅ Automatic Compaction (Parquet optimization)
- ✅ Write-Ahead Log (WAL for durability)
- ✅ Retention Policies (automatic data expiration)
- ✅ Continuous Queries (real-time aggregations)
- ✅ Delete API (selective data removal)
- ✅ S3/MinIO storage backend
- ✅ Arrow IPC query responses

## Quick Start

Expand All @@ -635,30 +761,29 @@ jobs:
docker run -d \
-p 8000:8000 \
-v arc-data:/app/data \
ghcr.io/basekick-labs/arc-go:${{ needs.prepare.outputs.version }}
ghcr.io/basekick-labs/arc:${{ needs.prepare.outputs.version }}
```

### Debian/Ubuntu (amd64, arm64)

```bash
# Download and install
wget https://github.com/basekick-labs/arc-go/releases/download/v${{ needs.prepare.outputs.version }}/arc_${{ needs.prepare.outputs.version }}_amd64.deb
wget https://github.com/basekick-labs/arc/releases/download/v${{ needs.prepare.outputs.version }}/arc_${{ needs.prepare.outputs.version }}_amd64.deb
sudo dpkg -i arc_${{ needs.prepare.outputs.version }}_amd64.deb

# Start and enable
sudo systemctl enable arc
sudo systemctl start arc

# Check status
sudo systemctl status arc
curl http://localhost:8000/health
```

### RHEL/Fedora/Rocky (x86_64, aarch64)

```bash
# Download and install
wget https://github.com/basekick-labs/arc-go/releases/download/v${{ needs.prepare.outputs.version }}/arc-${{ needs.prepare.outputs.version }}-1.x86_64.rpm
wget https://github.com/basekick-labs/arc/releases/download/v${{ needs.prepare.outputs.version }}/arc-${{ needs.prepare.outputs.version }}-1.x86_64.rpm
sudo rpm -i arc-${{ needs.prepare.outputs.version }}-1.x86_64.rpm

# Start and enable
Expand All @@ -669,47 +794,41 @@ jobs:
### Kubernetes (Helm)

```bash
helm install arc https://github.com/basekick-labs/arc-go/releases/download/v${{ needs.prepare.outputs.version }}/arc-${{ needs.prepare.outputs.version }}.tgz
helm install arc https://github.com/basekick-labs/arc/releases/download/v${{ needs.prepare.outputs.version }}/arc-${{ needs.prepare.outputs.version }}.tgz
```

## Download Artifacts

| Platform | Architecture | Package |
|----------|--------------|---------|
| Docker | amd64, arm64 | `ghcr.io/basekick-labs/arc-go:${{ needs.prepare.outputs.version }}` |
| Docker | amd64, arm64 | `ghcr.io/basekick-labs/arc:${{ needs.prepare.outputs.version }}` |
| Debian/Ubuntu | amd64 | `arc_${{ needs.prepare.outputs.version }}_amd64.deb` |
| Debian/Ubuntu | arm64 | `arc_${{ needs.prepare.outputs.version }}_arm64.deb` |
| RHEL/Fedora | x86_64 | `arc-${{ needs.prepare.outputs.version }}-1.x86_64.rpm` |
| RHEL/Fedora | aarch64 | `arc-${{ needs.prepare.outputs.version }}-1.aarch64.rpm` |
| Kubernetes | - | `arc-${{ needs.prepare.outputs.version }}.tgz` (Helm) |

## Performance
## Breaking Changes

Benchmarked on Apple MacBook Pro M3 Max (14 cores, 36GB RAM):
- **Python version**: The Python implementation is preserved in the `python-legacy` branch
- **Configuration**: TOML config format (unchanged, but verify your `arc.toml`)

| Metric | Throughput |
|--------|------------|
| MessagePack Ingestion | **9.47M rec/s** |
| Line Protocol Ingestion | 1.92M rec/s |
| Arrow Query | 2.88M rows/s |
## Upgrading from Python

## What's New

- Go implementation with 125% faster ingestion than Python
- Stable memory (no leaks)
- Single binary deployment
- Multi-arch support (amd64, arm64)
- Native packages for Debian/Ubuntu and RHEL/Fedora
- Systemd integration (auto-start on boot)
1. Stop existing Arc service
2. Backup your data directory
3. Install the new Go binary (same config format)
4. Start Arc - data is automatically migrated

## Documentation

[docs.basekick.net/arc](https://docs.basekick.net/arc)
- [Full Documentation](https://docs.basekick.net/arc)
- [Migration Guide](https://github.com/basekick-labs/arc/blob/main/MIGRATION_PLAN.md)

## Community

- **Discord**: [Join](https://discord.gg/nxnWfUxsdm)
- **Issues**: [GitHub](https://github.com/basekick-labs/arc-go/issues)
- **Issues**: [GitHub](https://github.com/basekick-labs/arc/issues)
EOF

cat release-notes.md
Expand Down Expand Up @@ -744,14 +863,18 @@ jobs:
echo "" >> $GITHUB_STEP_SUMMARY
echo "| Type | Package |" >> $GITHUB_STEP_SUMMARY
echo "|------|---------|" >> $GITHUB_STEP_SUMMARY
echo "| Docker | \`ghcr.io/basekick-labs/arc-go:${VERSION}\` |" >> $GITHUB_STEP_SUMMARY
echo "| Docker | \`ghcr.io/basekick-labs/arc:${VERSION}\` |" >> $GITHUB_STEP_SUMMARY
echo "| Helm | \`arc-${VERSION}.tgz\` |" >> $GITHUB_STEP_SUMMARY
echo "| Debian amd64 | \`arc_${VERSION}_amd64.deb\` |" >> $GITHUB_STEP_SUMMARY
echo "| Debian arm64 | \`arc_${VERSION}_arm64.deb\` |" >> $GITHUB_STEP_SUMMARY
echo "| RPM x86_64 | \`arc-${VERSION}-1.x86_64.rpm\` |" >> $GITHUB_STEP_SUMMARY
echo "| RPM aarch64 | \`arc-${VERSION}-1.aarch64.rpm\` |" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "### Tests Passed" >> $GITHUB_STEP_SUMMARY
echo "- ✅ Docker image health check" >> $GITHUB_STEP_SUMMARY
echo "- ✅ Binary execution test" >> $GITHUB_STEP_SUMMARY
echo "- ✅ Helm chart deployment" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "### Next Steps" >> $GITHUB_STEP_SUMMARY
echo "1. Download and test artifacts" >> $GITHUB_STEP_SUMMARY
echo "2. Review release notes" >> $GITHUB_STEP_SUMMARY
echo "3. Publish when ready" >> $GITHUB_STEP_SUMMARY
echo "1. Review release notes" >> $GITHUB_STEP_SUMMARY
echo "2. Publish when ready" >> $GITHUB_STEP_SUMMARY
Loading
Loading