Merge pull request #3 from lupodevelop/ci/workflow

lupodevelop · web-flow · commit 25c52f6e91ac · 2025-11-09T18:02:00.000+01:00
Ci/workflow

After some edits, all is green!
diff --git a/.gitattributes b/.gitattributes
@@ -1,2 +1,3 @@
 # Auto detect text files and perform LF normalization
-* text=auto
+* text=auto eol=lf
+*.gleam text eol=lf
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -0,0 +1,69 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  test:
+    name: Build and Test
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Start ClickHouse
+        run: |
+          docker compose up -d clickhouse
+          echo "Waiting for ClickHouse to be ready..."
+          for i in $(seq 1 60); do
+            if curl -sSf http://localhost:8123/ >/dev/null 2>&1; then
+              echo "ClickHouse is ready"
+              break
+            fi
+            sleep 2
+          done
+
+      - name: Setup Erlang/OTP and Gleam
+        uses: erlef/setup-beam@v1
+        with:
+          otp-version: "27.0"
+          gleam-version: "1.13.0"
+
+      - name: Show Gleam version
+        run: gleam --version
+
+      - name: Cache Gleam deps and build
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/gleam
+            ~/.gleam
+            ./_gleam_deps
+            ./build
+          key: ${{ runner.os }}-gleam-1.13.0-otp27-cache-v1-${{ hashFiles('**/gleam.toml') }}
+          restore-keys: |
+            ${{ runner.os }}-gleam-1.13.0-otp27-cache-v1-
+
+      - name: Install dependencies
+        run: gleam deps download
+
+      - name: Check formatting
+        run: gleam format --check src test
+
+      - name: Build project
+        run: gleam build
+
+      - name: Run all tests
+        env:
+          CLICKHOUSE_USER: test_user
+          CLICKHOUSE_PASSWORD: test_password
+          CLICKHOUSE_DB: test_db
+          CLICKHOUSE_URL: http://localhost:8123
+        run: gleam test
+
+      - name: Cleanup ClickHouse
+        if: always()
+        run: docker compose down -v
diff --git a/.github/workflows/smoke-integration.yml b/.github/workflows/smoke-integration.yml
@@ -0,0 +1,65 @@
+name: Smoke Integration (PR)
+
+on:
+  pull_request:
+    branches: ["main"]
+
+jobs:
+  smoke-integration:
+    name: Smoke Integration (ClickHouse)
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Start ClickHouse (docker-compose)
+        env:
+          CLICKHOUSE_IMAGE: clickhouse/clickhouse-server:23.7
+        run: |
+          echo "Using CLICKHOUSE_IMAGE=$CLICKHOUSE_IMAGE"
+          docker compose pull clickhouse || true
+          docker compose up -d --remove-orphans clickhouse
+
+      - name: Wait for ClickHouse HTTP
+        run: |
+          echo "Waiting for ClickHouse HTTP on localhost:8123..."
+          for i in $(seq 1 40); do
+            if curl -sSf http://localhost:8123/ >/dev/null 2>&1; then
+              echo "ClickHouse is up"
+              break
+            fi
+            sleep 2
+          done
+
+      - name: Smoke test - SELECT 1
+        run: |
+          set -e
+          OUT=$(curl -sS -u test_user:test_password "http://localhost:8123/?query=SELECT%201%20as%20result%20FORMAT%20JSONEachRow&database=test_db")
+          echo "Got: $OUT"
+          if [ "$OUT" != '{"result":1}' ] && [ "$OUT" != '{"result":1}\n' ]; then
+            echo "Unexpected response: $OUT"
+            exit 2
+          fi
+
+      - name: Smoke test - create table, insert and select
+        run: |
+          set -e
+          DDL="CREATE TABLE IF NOT EXISTS smoke_test (id UInt32, name String) ENGINE = MergeTree() ORDER BY id"
+          curl -sS -u test_user:test_password -d "$DDL" "http://localhost:8123/?database=test_db"
+
+          INSERT="INSERT INTO smoke_test (id,name) VALUES (1,'smoke')"
+          curl -sS -u test_user:test_password -d "$INSERT" "http://localhost:8123/?database=test_db"
+
+          OUT=$(curl -sS -u test_user:test_password "http://localhost:8123/?query=SELECT%20*%20FROM%20smoke_test%20WHERE%20id%3D1%20FORMAT%20JSONEachRow&database=test_db")
+          echo "Select returned: $OUT"
+          if [ -z "$OUT" ]; then
+            echo "Select returned empty"
+            exit 3
+          fi
+
+      - name: Cleanup ClickHouse
+        if: always()
+        run: |
+          docker compose down -v || true
diff --git a/README.md b/README.md
@@ -4,14 +4,45 @@
   <img src="assets/image.png" alt="Sparkling logo" width="240" />
 </p>
 
-[![License](https://img.shields.io/badge/license-Apache%202.0-yellow.svg)](LICENSE) [![Built with Gleam](https://img.shields.io/badge/Built%20with-Gleam-ffaff3)](https://gleam.run)
+[![CI](https://github.com/lupodevelop/sparkling/actions/workflows/ci.yml/badge.svg)](https://github.com/lupodevelop/sparkling/actions/workflows/ci.yml) [![License](https://img.shields.io/badge/license-Apache%202.0-yellow.svg)](LICENSE) [![Built with Gleam](https://img.shields.io/badge/Built%20with-Gleam-ffaff3)](https://gleam.run) [![Gleam Version](https://img.shields.io/badge/gleam-%3E%3D1.13.0-ffaff3)](https://gleam.run)
 
 **Sparkling** is a *lightweight*, **type-safe** data layer for **ClickHouse** written in Gleam. It provides a small, focused API for defining schemas, building queries, and encoding/decoding ClickHouse formats.
 
 *No magic*, just small, composable functions that play nicely in Gleam apps.
 
 > Why "Sparkling"? One rainy Tuesday a tiny inflatable rubber duck stole a shooting pink star and decided to become a freelance data wrangler, and it now guides queries through the night, humming 8-bit lullabies. Totally plausible.
 
+## Quick start
+
+See the extracted quick start example: `docs/quickstart.md` it contains a short walkthrough (define schema, build a query, execute it with a repo).
+
+Minimal example:
+
+```gleam
+import sparkling/repo
+
+let r = repo.new("http://localhost:8123")
+  |> repo.with_database("mydb")
+
+case r.execute_sql(r, "SELECT 1 as result FORMAT JSONEachRow") {
+  Ok(body) -> io.println(body)
+  Error(_) -> io.println("query failed")
+}
+```
+
+## What you'll find here
+
+- `sparkling/schema` — typed table & column definitions
+- `sparkling/query` — immutable query builder (to_sql)
+- `sparkling/repo` — HTTP executor with retry hooks
+- `sparkling/encode` / `sparkling/decode` — format handlers (JSONEachRow default)
+- `sparkling/types` — helpers for Decimal, DateTime64, UUID, LowCardinality
+
+For more examples see `docs/examples/` and `docs/quickstart.md`.
+
+**Design note:** Sparkling's API and composable query builder were partly inspired by *Ecto*; 
+many ideas about schema definition and query composition borrow from its approach while keeping a small, Gleam-friendly surface.
+
 ## Development
 
 Run tests and format/check locally:
diff --git a/test/TESTS.md b/test/TESTS.md
@@ -0,0 +1,62 @@
+# Tests — Quick reference
+
+This document explains how tests are organised in the repository and how to run them locally and in CI.
+
+Directory layout
+
+- `test/sparkling/` — unit and component tests that do not require external services (fast).
+- `test/smoke/` — smoke/sanity tests. Quick checks that the test runner and a minimal API behave as expected.
+- `test/integration/` — integration tests that require external services (e.g. ClickHouse). These are slower and should not run on every PR.
+
+Running tests locally
+
+- Run the full test suite (if you have required services available):
+
+```bash
+# from the project root
+gleam test
+```
+
+- Run unit tests only (recommended for PRs):
+
+```bash
+# if your test runner accepts paths:
+gleam test test/sparkling
+```
+
+If your runner does not accept paths, run `gleam test` locally and ensure integration tests are not executed by default (integration tests should be placed under `test/integration/`).
+
+- Run integration tests (requires ClickHouse or other external services):
+
+```bash
+# Start ClickHouse (example using Docker)
+docker run -d --name clickhouse-server -p 8123:8123 -p 9000:9000 clickhouse/clickhouse-server:latest
+
+# Run integration tests
+gleam test test/integration
+
+# When finished, stop/remove the container
+docker stop clickhouse-server && docker rm clickhouse-server
+```
+
+Environment variables
+
+- Use environment variables to configure integration endpoints (example):
+
+```bash
+export CLICKHOUSE_URL=http://localhost:8123
+```
+
+Document required variables in `test/integration/README.md`.
+
+CI guidance
+
+- Pull Requests: run unit tests only.
+- Integration tests: run in separate CI jobs (manual trigger, nightly, or on release tags). Ensure the CI environment has access to required services and secrets before enabling integration jobs.
+
+Best practices and PR checklist
+
+- [ ] Unit tests pass locally (`gleam test`).
+- [ ] Integration tests are documented in `test/integration/README.md` with clear prerequisites and run instructions.
+- [ ] Do not add CI workflows that perform sensitive operations (publishing) in the import PR. Add automation in a separate PR after the code is stable.
+- [ ] Keep a lightweight smoke test under `test/smoke/` to validate the test runner and minimal API surface.
diff --git a/test/integration/README.md b/test/integration/README.md
@@ -0,0 +1,98 @@
+# Integration tests
+
+Integration tests that exercise the code against a real ClickHouse instance.
+
+This document explains how to run integration tests locally and how to wire them into CI in a safe way.
+
+Prerequisites
+
+- Docker (used for the example below).
+- docker-compose (optional, if you prefer a compose-based setup).
+
+Recommended local setup (Docker Compose)
+
+1. Start ClickHouse with docker-compose (example):
+
+```bash
+# from the repository root, if you have a docker-compose.yml configured
+docker-compose up -d
+```
+
+2. Wait for the server to be ready:
+
+```bash
+docker-compose ps
+```
+
+3. Run only the integration tests:
+
+```bash
+# from the repository root
+gleam test test/integration
+```
+
+4. Tear down the environment when finished:
+
+```bash
+docker-compose down -v
+```
+
+Quick single-container alternative (no compose)
+
+```bash
+docker run -d --name clickhouse-server -p 8123:8123 -p 9000:9000 clickhouse/clickhouse-server:latest
+# wait for readiness, then run tests
+gleam test test/integration
+docker stop clickhouse-server && docker rm clickhouse-server
+```
+
+Connection details (local defaults)
+
+- URL: http://localhost:8123
+- Database: test_db (tests may create/drop their own DBs)
+- User: test_user
+- Password: test_password
+
+If your tests require different credentials or endpoints, set the appropriate environment variables (see the `Environment variables` section below).
+
+Implemented integration tests
+
+- `basic_connectivity_test.gleam` — verifies connection and basic queries
+- `complex_types_roundtrip_test.gleam` — round-trip tests for complex ClickHouse types
+- `format_compatibility_test.gleam` — tests different I/O formats (JSONEachRow, CSV, TabSeparated)
+- `performance_test.gleam` — optional performance/load checks (may be slow)
+
+Environment variables
+
+Use environment variables to configure endpoints and credentials for integration tests. Example:
+
+```bash
+export CLICKHOUSE_URL=http://localhost:8123
+export CLICKHOUSE_USER=test_user
+export CLICKHOUSE_PASSWORD=test_password
+```
+
+Document required environment variables and secrets clearly before enabling integration jobs in CI.
+
+CI guidance
+
+- Integration tests should not run by default on every PR. Run them in separate CI jobs, e.g. on manual dispatch, nightly builds, or release tags.
+- Example GitHub Actions job (run in a dedicated workflow or as a separate job):
+
+```yaml
+- name: Integration tests (optional)
+  run: |
+    docker-compose up -d
+    # optionally wait / healthcheck
+    sleep 10
+    gleam test test/integration
+    docker-compose down -v
+  # do not fail the whole pipeline if integration env is missing
+  continue-on-error: true
+```
+
+Notes and recommendations
+
+- Keep integration tests isolated and idempotent: tests should create and drop any resources they need.
+- Mark expensive or flaky tests (e.g. `performance_test.gleam`) so CI can skip them unless explicitly requested.
+- Before enabling integration tests in CI, ensure any required secrets or credentials are set in the CI environment.
diff --git a/test/sparkling/encode_decode_roundtrip_test.gleam b/test/sparkling/encode_decode_roundtrip_test.gleam
