Release/1.2.0

.github/workflows/lint.yml

@@ -12,22 +12,22 @@ jobs:
   ruff-lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
-      - uses: astral-sh/ruff-action@v3
-        with:
-          version: "0.12.12"
-          args: check
-      - uses: astral-sh/ruff-action@v3
-        with:
-          args: format --check
+      - uses: actions/checkout@v6
       - name: Install uv
-        uses: astral-sh/setup-uv@v5
+        uses: astral-sh/setup-uv@v7
         with:
           version: "0.7.3"
       - name: Set up Python
         run: uv python install
       - name: Sync project, including dev dependencies
         run: uv sync --all-extras
+      - uses: astral-sh/ruff-action@v3
+        with:
+          version: "0.12.12"
+          args: check
+      - uses: astral-sh/ruff-action@v3
+        with:
+          args: format --check
       - name: Build static content
         run: |
           uv run python ./scripts/gen_api_docs.py

.github/workflows/publish.yml

@@ -18,9 +18,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v6
       - name: Install uv
-        uses: astral-sh/setup-uv@v5
+        uses: astral-sh/setup-uv@v7
         with:
           version: "0.7.3"
       - name: Set up Python

.github/workflows/static.yml

@@ -20,9 +20,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v6
       - name: Install uv
-        uses: astral-sh/setup-uv@v5
+        uses: astral-sh/setup-uv@v7
         with:
           version: "0.7.3"
       - name: Set up Python
@@ -36,7 +36,7 @@ jobs:
       - name: Setup Pages
         uses: actions/configure-pages@v5
       - name: Upload artifact
-        uses: actions/upload-pages-artifact@v3
+        uses: actions/upload-pages-artifact@v4
         with:
           path: 'site'
       - name: Deploy to GitHub Pages

.github/workflows/tests.yml

@@ -14,13 +14,19 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
 
       - name: Install uv
-        uses: astral-sh/setup-uv@v6
+        uses: astral-sh/setup-uv@v7
+
+      - name: Install PostgreSQL and PostGIS tooling
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y postgresql-16 postgresql-client-16 postgresql-16-postgis-3 postgresql-16-postgis-3-scripts
+          echo "/usr/lib/postgresql/16/bin" >> "$GITHUB_PATH"
 
       - name: Install the project
         run: uv sync --all-extras --dev
 
       - name: Run tests
-        run: uv run pytest tests/
+        run: uv run pytest tests/

.gitignore

@@ -1,3 +1,6 @@
+# OpenSAMPL data paths
+archive/
+ntp-snapshots/
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]

CHANGELOG.md

@@ -37,6 +37,31 @@ This project adheres to [Semantic Versioning](https://semver.org/).
 *Unreleased* versions radiate potential—-and dread. Once you merge an infernal PR, move its bullet under a new version heading with the actual release date.*
 
 -->
+## [1.2.0] - 2026-04-29
+### Added
+- 🔥 First-class NTP vendor and probe support using the existing OpenSAMPL extension model
+- 🔥 Local and remote NTP collection paths, including `ntp_metadata` loading behavior
+- 🔥 NTP-focused metrics such as jitter, delay, stratum, reachability, root delay, root dispersion, poll interval, and sync health
+- 🔥 Additional NTP metadata handling for collector/target probe relationships and reference-backed loading
+- 🔥 Compact reference/source metadata views in dashboards to improve interpretation of NTP-backed timing data
+- 🔥 Documentation covering the NTP extension path, collection semantics, and geolocation behavior
+- 🔥 Additional unit and integration-style tests for NTP collection, loading, geolocation helpers, and seeded database defaults
+- 🔥 Moved alembic migration code into openSAMPL along with Docker image information
+- 🔥 Moved backend api code into openSAMPL along with Docker image information
+- 🔥 Docker-compose for developers which installs openSAMPL as editable on backend image
+
+### Changed
+- ⚡ Hardened dashboard queries and variables to avoid brittle empty-filter handling and varchar-versus-UUID failures
+- ⚡ Updated timing dashboards and wording to use reference-safe terminology for NTP-backed demo paths
+- ⚡ Reworked integration-style tests to use the project MockDB harness instead of requiring a locally spawned PostgreSQL instance
+- ⚡ Updated CI to install PostgreSQL/PostGIS tooling so the workflow can support `pytest-postgresql`-style environments when needed
+
+### Fixed
+- 🩹 Seeded default metric UUID handling in the MockDB test harness now points to the UNKNOWN metric as intended
+- 🩹 Bug which caused random data duration to always be 1 hour
+- 🩹 Issue with CI order causing linting to fail
+- 🩹 Fixed variable assignment for mkdocs-click integration
+
 ## [1.1.5] - 2025-09-22
 ### Fixed
 - 🩹 More durable timestamp extrapolation in time data insertion

README.md

@@ -20,143 +20,153 @@
 </div>
 
 
-OpenSAMPL was created to provide a set of Python tools for managing clock data in a TimescaleDB database, specifically designed for synchronization analytics and monitoring.
-This project came out of [**CAST**](https://cast.ornl.gov), the **C**enter for **A**lternative **S**yncrhonization and **T**iming, a research group at Oak Ridge National Laboratory (ORNL).
-The name OpenSAMPL stands for **O**pen **S**ynchronization **A**nalytics and **M**onitoring **PL**atform, and provides the code and logic for uploading, managing, and visualizing clock data from various sources, including ADVA probes and Microchip TWST data files,
-with the goal of this project being to provide a comprehensive and open-source solution for clock data management and analysis. 
-Visualizations are provided via [grafana](https://grafana.com/), and the data is stored in a [TimescaleDB](https://www.timescale.com/) database, which is a time-series database built on PostgreSQL.
+OpenSAMPL provides Python tools for collecting, loading, and visualizing clock data in a
+TimescaleDB-backed synchronization analytics stack.
+This project came out of [**CAST**](https://cast.ornl.gov), the **C**enter for
+**A**lternative **S**ynchronization and **T**iming at Oak Ridge National Laboratory (ORNL).
+The name OpenSAMPL stands for **O**pen **S**ynchronization **A**nalytics and
+**M**onitoring **PL**atform.
+
+The current codebase supports loading and analysis workflows for ADVA, Microchip TWST,
+Microchip TP4100, and NTP-derived probe data. Visualization is provided through
+[Grafana](https://grafana.com/), and the data is stored in
+[TimescaleDB](https://www.timescale.com/), which is built on PostgreSQL.
 
 
 ### (**O**pen **S**ynchronization **A**nalytics and **M**onitoring **PL**atform)
 
-python tools for adding clock data to a timescale db. 
+Python tools for adding clock and timing data to a TimescaleDB database.
 
-## CLI TOOL
+## Installation
 
-### Installation
+1. Ensure you have Python 3.10 or higher installed.
+2. Install the latest release:
 
-1. Ensure you have Python 3.9 or higher installed
-2. Pip install the latest version of opensampl: 
 ```bash
 pip install opensampl
 ```
 
 ### Development Setup
+
 ```bash
 uv venv
-uv sync --extra all
+uv sync --all-extras --dev
 source .venv/bin/activate
 ```
-This will create a virtual environment and install the development dependencies.
+This creates a virtual environment and installs the development dependencies.
 
 ### Environment Setup
 
-The tool requires several environment variables. Create a `.env` file in your project root:
+The CLI reads configuration from environment variables or a local `.env` file.
 
-When routing through a backend:
+When routing through a backend service:
 ```bash
-ROUTE_TO_BACKEND=true  # Set to true if using backend service
-BACKEND_URL=http://localhost:8000  # Only needed if ROUTE_TO_BACKEND is true
+ROUTE_TO_BACKEND=true
+BACKEND_URL=http://localhost:8000
 
-# Archive configuration
-ARCHIVE_PATH=/path/to/archive  # Where processed files are stored
+ARCHIVE_PATH=/path/to/archive
 ```
-When directly accessing db: 
+
+When connecting directly to PostgreSQL / TimescaleDB:
 ```bash
-# Database connection
 DATABASE_URL=postgresql://<user>:<password>@<host>:<port>/<database>
-
-# Archive configuration
-ARCHIVE_PATH=/path/to/archive  # Where processed files are stored
+ARCHIVE_PATH=/path/to/archive
 ```
 
-### Basic Usage
+Use `opensampl config show` to inspect the current resolved configuration.
 
-The CLI tool provides several commands. You can use `opensampl --help` (or, any deeper `opensampl [command] --help`) to get details
+## CLI
 
-#### Load Probe Data
+The main CLI exposes `collect`, `config`, `create`, `init`, and `load`.
+Use `opensampl --help` and `opensampl <command> --help` for current options.
 
-Load data from ADVA probes:
+If you plan to use the NTP, Microchip TWST, or Microchip TP4100 collectors, install the optional collection dependencies:
 
 ```bash
-# Load single file
-opensampl load probe adva path/to/file.txt.gz
-
-# Load directory of files
-opensampl load probe adva path/to/directory/
+pip install "opensampl[collect]"
 ```
-ADVA probes have all their metadata and their time data in each file, so no need to use the `-m` or `-t` options, though if you want to skip loading one or the other it becomes useful! 
 
-options:
-- `--metadata` (`-m`): Only load probe metadata
-- `--time-data` (`-t`): Only load time series data
-- `--no-archive` (`-n`): Don't archive processed files
-- `--archive-path` (`-a`): Override default archive directory
-- `--max-workers` (`-w`): Maximum number of worker threads (default: 4)
-- `--chunk-size` (`-c`): Number of time data entries per batch (default: 10000)
+### Load Probe Data
 
-#### Load Direct Table Data
+Load data with the probe type name directly:
 
-Load data directly into a database table. Format can be yaml or json. Can be a list of dictionaries or a single dictionary.
+```bash
+opensampl load ADVA path/to/file.txt.gz
+opensampl load ADVA path/to/directory/
+```
 
-you do not have to specify schema, is assumed to be castdb. 
+ADVA files bundle metadata and time-series data in a single file, so the split flags are
+usually not needed.
 
-The --if-exists option controls how to handle conflicts:
-  - update: Only update fields that are provided and non-default (default)
-  - error: Raise an error if entry exists
-  - replace: Replace all non-primary-key fields with new values
-  - ignore: Skip if entry exists
+```bash
+opensampl load MicrochipTWST path/to/twst-output
+opensampl load MicrochipTP4100 path/to/tp4100-output
+```
+
+NTP data is collected first and then loaded from the output directory:
 
 ```bash
-opensampl load table table_name path/to/data.yaml
+opensampl collect ntp --mode remote --server pool.ntp.org --output-path ./ntp-out
+opensampl load NTP ./ntp-out
 ```
 
-So, you can do things like the following  
+Load options:
+
+- `--metadata` / `-m`: load only probe metadata
+- `--time-data` / `-t`: load only time-series data
+- `--no-archive` / `-n`: skip archiving processed files
+- `--archive-path` / `-a`: override the archive directory
+- `--max-workers` / `-w`: set the worker count
+- `--chunk-size` / `-c`: set the batch size for time-series inserts
+
+### Load Direct Table Data
+
+Load YAML or JSON directly into a table:
+
 ```bash
-opensampl load table locations --if-exists replace updated_location.yaml
+opensampl load table locations updated_location.yaml
 ```
-Where this is the updated_location
+
+Conflict handling is controlled by `--if-exists`:
+
+- `update`: fill null fields in an existing row
+- `error`: raise if the row exists
+- `replace`: replace non-primary-key values
+- `ignore`: skip existing rows
+
+Example input:
+
 ```yaml
 name: EPB Chattanooga
 lat: 35.9311256
 lon: -84.3292469
 ```
-And it will overwrite the existing entry for EPB Chattanooga, or create a new one if it doesn't exist yet.
-
 
 ### View Configuration
 
-Display current environment configuration:
-
 ```bash
-# Show all variables
-poetry run opensampl config show
-
-# Show with descriptions
-poetry run opensampl config show --explain
-
-# Show specific variable
-poetry run opensampl config show --var DATABASE_URL
+opensampl config show
+opensampl config show --explain
+opensampl config show --var DATABASE_URL
 ```
 
 ### Set Configuration
 
-Update environment variables:
-
 ```bash
-poetry run opensampl config set VARIABLE_NAME value
+opensampl config set VARIABLE_NAME value
 ```
 
 ## File Format Support
 
-The tool currently supports:
-
-ADVA probe data files with the following naming convention:
-`<ip_address>CLOCK_PROBE-<probe_id>-YYYY-MM-DD-HH-MM-SS.txt.gz`
+The loaders currently support:
 
-Example: `10.0.0.121CLOCK_PROBE-1-1-2024-01-02-18-24-56.txt.gz`
+- ADVA probe files named like
+  `<ip_address>CLOCK_PROBE-<probe_id>-YYYY-MM-DD-HH-MM-SS.txt.gz>`
+- Microchip TWST and TP4100 output produced by the collector tooling
+- NTP snapshot output produced by `opensampl collect ntp`
 
-Microchip TWST Data Files as generated by the script available. 
+Example ADVA file:
+`10.0.0.121CLOCK_PROBE-1-1-2024-01-02-18-24-56.txt.gz`
 
 # Contributing
 We welcome contributions! Please see our [CONTRIBUTING.md](CONTRIBUTING.md) for details on how to get started.
@@ -234,4 +244,3 @@ adva_mask_margin: 0  # Mask margin
 - Table relationships are maintained through UUID references
 - Geographic coordinates use WGS84 projection (SRID 4326) by default
 - Boolean fields (public) are optional and can be null
-

docs/api/collect/cli.md

@@ -0,0 +1,8 @@
+# CLI Reference
+
+This page provides documentation for our command line tools.
+
+::: mkdocs-click
+    :module: opensampl.collect.cli
+    :command: cli
+    :prog_name: opensampl-collect

docs/api/collect/microchip/tp4100/collect_4100.md

@@ -0,0 +1,7 @@
+# `opensampl.collect.microchip.tp4100.collect_4100`
+
+::: opensampl.collect.microchip.tp4100.collect_4100
+    options:
+      show_root_heading: false
+      show_submodules: true
+      show_source: true

docs/api/collect/microchip/twst/context.md

@@ -0,0 +1,7 @@
+# `opensampl.collect.microchip.twst.context`
+
+::: opensampl.collect.microchip.twst.context
+    options:
+      show_root_heading: false
+      show_submodules: true
+      show_source: true

docs/api/collect/microchip/twst/generate_twst_files.md

@@ -0,0 +1,7 @@
+# `opensampl.collect.microchip.twst.generate_twst_files`
+
+::: opensampl.collect.microchip.twst.generate_twst_files
+    options:
+      show_root_heading: false
+      show_submodules: true
+      show_source: true