Documentation & Simplified Install

* drop scipy optional dependency and [apps] entirely
* add documentation for converters
* update converter entry points

Author: Teque5

.github/workflows/main.yml

@@ -22,7 +22,7 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install .[test,apps]
+          pip install .[test]
       - name: Test with pytest
         run: |
           coverage run

.readthedocs.yaml

@@ -18,7 +18,6 @@ python:
       path: .
       extra_requirements:
         - test
-        - apps
     - requirements: docs/requirements.txt
 
 # Build documentation in the "docs/" directory with Sphinx

docs/source/api.rst

@@ -7,9 +7,10 @@ SigMF API
    :template: custom-module-template.rst
    :recursive:
 
-   sigmf.apps.convert_wav
    sigmf.archive
    sigmf.archivereader
+   sigmf.convert.blue
+   sigmf.convert.wav
    sigmf.error
    sigmf.schema
    sigmf.sigmf_hash

docs/source/converters.rst

@@ -0,0 +1,84 @@
+=================
+Format Converters
+=================
+
+The SigMF Python library includes converters to import data from various file formats into SigMF format.
+These converters make it easy to migrate existing RF recordings to the standardized SigMF format while preserving metadata when possible.
+
+Overview
+--------
+
+Converters are available for:
+
+* **BLUE files** - MIDAS Blue and Platinum BLUE RF recordings (``.cdif``)
+* **WAV files** - Audio recordings (``.wav``)
+
+All converters return a :class:`~sigmf.SigMFFile` object that can be used immediately or saved to disk.
+Converters preserve datatypes and metadata where possible.
+
+
+Command Line Usage
+~~~~~~~~~~~~~~~~~~
+
+Converters can be used from the command line after ``pip install sigmf``:
+
+.. code-block:: bash
+
+    sigmf_convert_blue input.cdif
+    sigmf_convert_wav input.wav
+
+or by using module syntax:
+
+.. code-block:: bash
+
+    python3 -m sigmf.convert.blue input.cdif
+    python3 -m sigmf.convert.wav input.wav
+
+
+BLUE Converter
+--------------
+
+The BLUE converter handles CDIF (.cdif) recordings while placing BLUE header information into the following global fields:
+
+* ``blue:fixed`` - fixed header information (at start of file)
+* ``blue:adjunct`` - adjunct header information (after fixed header)
+* ``blue:extended`` - extended header information (at end of file)
+* ``blue:keywords`` - user-defined key-value pairs
+
+.. autofunction:: sigmf.convert.blue.blue_to_sigmf
+
+
+.. code-block:: python
+
+    from sigmf.convert.blue import blue_to_sigmf
+
+    # read BLUE, write SigMF, and return SigMFFile object
+    meta = blue_to_sigmf(blue_path="recording.cdif", out_path="recording.sigmf")
+
+    # access converted data
+    samples = meta.read_samples()
+    sample_rate_hz = meta.sample_rate
+
+    # access BLUE-specific metadata
+    blue_type = meta.get_global_field("blue:fixed")["type"] # e.g., 1000
+    blue_version = meta.get_global_field("blue:keywords")["IO"] # e.g., "X-Midas"
+
+
+WAV Converter
+-------------
+
+This is useful when working with audio datasets.
+
+.. autofunction:: sigmf.convert.wav.wav_to_sigmf
+
+
+.. code-block:: python
+
+    from sigmf.convert.wav import wav_to_sigmf
+
+    # read WAV, write SigMF, and return SigMFFile object
+    meta = wav_to_sigmf(wav_path="recording.wav", out_path="recording.sigmf")
+
+    # access converted data
+    samples = meta.read_samples()
+    sample_rate_hz = meta.sample_rate

docs/source/index.rst

@@ -23,6 +23,7 @@ To get started, see the :doc:`quickstart` section or learn how to :ref:`install`
 
    quickstart
    advanced
+   converters
    developers
 
 .. toctree::

pyproject.toml

@@ -33,18 +33,15 @@ dependencies = [
 
     [project.scripts]
     sigmf_validate = "sigmf.validate:main"
-    sigmf_convert_wav = "sigmf.apps.convert_wav:main [apps]"
-    sigmf_convert_blue = "sigmf.apps.convert_blue:main [apps]"
+    sigmf_convert_wav = "sigmf.convert.wav:main"
+    sigmf_convert_blue = "sigmf.convert.blue:main"
     [project.optional-dependencies]
     test = [
         "pylint",
         "pytest",
         "pytest-cov",
         "hypothesis",   # next-gen testing framework
     ]
-    apps = [
-        "scipy",        # for wav i/o
-    ]
 
 [tool.setuptools]
 packages = ["sigmf"]
@@ -107,6 +104,6 @@ legacy_tox_ini = '''
 
     [testenv]
     usedevelop = True
-    deps = .[test,apps]
+    deps = .[test]
     commands = coverage run
 '''

sigmf/apps/__init__.py sigmf/convert/__init__.pysigmf/apps/__init__.py renamed to sigmf/convert/__init__.py

@@ -10,12 +10,12 @@
 import getpass
 import logging
 import tempfile
+import wave
 from datetime import datetime, timezone
-from os import PathLike
 from pathlib import Path
 from typing import Optional
 
-from scipy.io import wavfile
+import numpy as np
 
 from .. import SigMFFile
 from .. import __version__ as toolversion
@@ -25,19 +25,27 @@
 log = logging.getLogger()
 
 
-def convert_wav(
+def wav_to_sigmf(
     wav_path: str,
     out_path: Optional[str] = None,
     to_archive: bool = True,
     author: Optional[str] = None,
-) -> PathLike:
+) -> SigMFFile:
     """
-    Read a wav and write a sigmf archive.
+    Read a wav, write a sigmf, return SigMFFile object.
+
+    Note: Can only read PCM wav files. Use scipy.io.wavefile for broader support.
     """
     wav_path = Path(wav_path)
     wav_stem = wav_path.stem
-    samp_rate, wav_data = wavfile.read(wav_path)
-
+    with wave.open(str(wav_path), "rb") as wav_reader:
+        n_channels = wav_reader.getnchannels()
+        samp_width = wav_reader.getsampwidth()
+        samp_rate = wav_reader.getframerate()
+        n_frames = wav_reader.getnframes()
+        raw_data = wav_reader.readframes(n_frames)
+    np_dtype = f"int{samp_width * 8}"
+    wav_data = np.frombuffer(raw_data, dtype=np_dtype).reshape(-1, n_channels)
     global_info = {
         SigMFFile.AUTHOR_KEY: getpass.getuser() if author is None else author,
         SigMFFile.DATATYPE_KEY: get_data_type_str(wav_data),
@@ -93,7 +101,11 @@ def main() -> None:
     }
     logging.basicConfig(level=level_lut[min(args.verbose, 2)])
 
-    _ = convert_wav(
+    _ = wav_to_sigmf(
         wav_path=args.input,
         author=args.author,
     )
+
+
+if __name__ == "__main__":
+    main()

sigmf/apps/convert_blue.py sigmf/convert/blue.pysigmf/apps/convert_blue.py renamed to sigmf/convert/blue.py

@@ -9,20 +9,14 @@
 import os
 import tempfile
 import unittest
+import wave
 from pathlib import Path
 
 import numpy as np
 
-try:
-    from scipy.io import wavfile
-
-    SCIPY_AVAILABLE = True
-except ImportError:
-    SCIPY_AVAILABLE = False
-
 import sigmf
-from sigmf.apps.convert_blue import convert_blue
-from sigmf.apps.convert_wav import convert_wav
+from sigmf.convert.blue import blue_to_sigmf
+from sigmf.convert.wav import wav_to_sigmf
 
 BLUE_ENV_VAR = "NONSIGMF_RECORDINGS_PATH"
 
@@ -32,8 +26,6 @@ class TestWAVConverter(unittest.TestCase):
 
     def setUp(self) -> None:
         """create temp wav file for testing"""
-        if not SCIPY_AVAILABLE:
-            self.skipTest("scipy is required for WAV file tests")
         self.tmp_dir = tempfile.TemporaryDirectory()
         self.tmp_path = Path(self.tmp_dir.name)
         self.wav_path = self.tmp_path / "foo.wav"
@@ -42,19 +34,28 @@ def setUp(self) -> None:
         ttt = np.linspace(0, duration_s, int(samp_rate * duration_s), endpoint=False)
         freq = 440  # A4 note
         self.audio_data = 0.5 * np.sin(2 * np.pi * freq * ttt)
-        wavfile.write(self.wav_path, samp_rate, self.audio_data.astype(np.float32))
+        # note scipy could write float wav files directly,
+        # but to avoid adding scipy as a dependency for sigmf-python,
+        # convert float audio to 16-bit PCM integer format
+        audio_int16 = (self.audio_data * 32767).astype(np.int16)
+
+        # write wav file using built-in wave module
+        with wave.open(str(self.wav_path), "wb") as wav_file:
+            wav_file.setnchannels(1)  # mono
+            wav_file.setsampwidth(2)  # 16-bit = 2 bytes
+            wav_file.setframerate(samp_rate)
+            wav_file.writeframes(audio_int16.tobytes())
 
     def tearDown(self) -> None:
         """clean up temporary directory"""
         self.tmp_dir.cleanup()
 
     def test_wav_to_sigmf(self):
         sigmf_path = self.tmp_path / "bar"
-        _ = convert_wav(wav_path=self.wav_path, out_path=sigmf_path)
-        meta = sigmf.fromfile(sigmf_path)
+        meta = wav_to_sigmf(wav_path=self.wav_path, out_path=sigmf_path)
         data = meta.read_samples()
-        # allow small numerical differences due to data type conversions
-        self.assertTrue(np.allclose(self.audio_data, data, atol=1e-8))
+        # allow numerical differences due to PCM conversion
+        self.assertTrue(np.allclose(self.audio_data, data, atol=1e-4))
 
 
 class TestBlueConverter(unittest.TestCase):
@@ -79,18 +80,24 @@ def tearDown(self) -> None:
 
     def test_blue_to_sigmf(self):
         for bdx, bluefile in enumerate(self.bluefiles):
-            sigmf_path = self.tmp_path / f"converted_{bdx}"
-            _ = convert_blue(blue_path=bluefile, out_path=sigmf_path)
-            meta = sigmf.fromfile(sigmf_path)
+            sigmf_path = self.tmp_path / bluefile.stem
+            meta = blue_to_sigmf(blue_path=bluefile, out_path=sigmf_path)
 
             ### EVERYTHING BELOW HERE IS FOR DEBUGGING ONLY _ REMOVE LATER ###
             # plot stft of RF data for visual inspection
+            import matplotlib.pyplot as plt
             from scipy.signal import spectrogram
+            from swiftfox import summary
 
             samples = meta.read_samples()
+            plt.figure(figsize=(10, 10))
+            summary(samples, detail=0.1, samp_rate=meta.get_global_field("core:sample_rate"))
+            plt.figure()
+            plt.plot(samples.real)
+            plt.plot(samples.imag)
+
             freqs, times, spec = spectrogram(samples, fs=meta.get_global_field("core:sample_rate"), nperseg=1024)
             # use imshow to plot spectrogram
-            import matplotlib.pyplot as plt
 
             plt.figure()
             plt.imshow(