Compression Support & New sigmf.fromarray method (#146)

Adds read/write support for compressed SigMF archives (`.sigmf.gz`, `.sigmf.xz`, `.sigmf.zip`) using only stdlib so no new dependencies are needed. Compression uses `tarfile` for gzip/xz and `zipfile` for zip.

Also fixed performance issue where reading uncompressed `.sigmf` archives was extracting data to RAM — these are now numpy memmapped directly into the tar.

Also added a `sigmf.fromarray()` convenience function to mirror `sigmf.fromfile()`. It infers the SigMF datatype from the numpy dtype. This can then be used with existing tofile() method:

```python
data = np.array([<SOME_STUFF>], dtype=<SOME_DTYPE>)
meta = sigmf.fromarray(data, sample_rate=48000)
meta.tofile("x") # create `x.sigmf-data` and `x.sigmf-meta` pair
meta.tofile("x.sigmf.gz") # create `x.sigmf.gz` compressed archive
```

Author: Teque5

README.md

@@ -28,13 +28,29 @@ meta = sigmf.fromfile("recording.sigmf-meta")
 samples = meta[0:1024]  # get first 1024 samples
 sample_rate = meta.sample_rate  # get sample rate
 
+# read compressed SigMF archives
+meta = sigmf.fromfile("recording.sigmf.gz")   # gzip-compressed
+meta = sigmf.fromfile("recording.sigmf.xz")   # xz-compressed
+meta = sigmf.fromfile("recording.sigmf.zip")  # zip archive
 
 # read other formats containing RF time series as SigMF
 meta = sigmf.fromfile("recording.wav")   # WAV
 meta = sigmf.fromfile("recording.cdif")  # BLUE / Platinum
 meta = sigmf.fromfile("recording.xml")   # Signal Hound Spike
 ```
 
+### Write SigMF
+
+```python
+import numpy as np
+import sigmf
+
+data = np.array([0.1 + 0.2j, 0.3 + 0.4j], dtype=np.complex64)
+meta = sigmf.fromarray(data, sample_rate=48000)
+# creates recording.sigmf-data and recording.sigmf-meta
+meta.tofile("recording")
+```
+
 ### Docs
 
 **[Please visit our documentation for full API reference and more info.](https://sigmf.readthedocs.io/en/latest/)**

docs/source/advanced.rst

@@ -18,7 +18,7 @@ the recording of the SigMF logo used in this example `from the specification
     from sigmf import SigMFFile, sigmffile
 
     # Load a dataset
-    path = 'logo/sigmf_logo' # extension is optional
+    path = "logo/sigmf_logo"  # extension is optional
     signal = sigmffile.fromfile(path)
 
     # Get some metadata and all annotations
@@ -31,13 +31,15 @@ the recording of the SigMF logo used in this example `from the specification
     for adx, annotation in enumerate(annotations):
         annotation_start_idx = annotation[SigMFFile.START_INDEX_KEY]
         annotation_length = annotation[SigMFFile.LENGTH_INDEX_KEY]
-        annotation_comment = annotation.get(SigMFFile.COMMENT_KEY, "[annotation {}]".format(adx))
+        annotation_comment = annotation.get(
+            SigMFFile.COMMENT_KEY, "[annotation {}]".format(adx)
+        )
 
         # Get capture info associated with the start of annotation
         capture = signal.get_capture_info(annotation_start_idx)
         freq_center = capture.get(SigMFFile.FREQUENCY_KEY, 0)
-        freq_min = freq_center - 0.5*sample_rate
-        freq_max = freq_center + 0.5*sample_rate
+        freq_min = freq_center - 0.5 * sample_rate
+        freq_max = freq_center + 0.5 * sample_rate
 
         # Get frequency edges of annotation (default to edges of capture)
         freq_start = annotation.get(SigMFFile.FLO_KEY)
@@ -66,34 +68,41 @@ First, create a single SigMF Recording and save it to disk:
     data = np.zeros(1024, dtype=np.complex64)
 
     # write those samples to file in cf32_le
-    data.tofile('example_cf32.sigmf-data')
+    data.tofile("example_cf32.sigmf-data")
 
     # create the metadata
     meta = SigMFFile(
-        data_file='example_cf32.sigmf-data', # extension is optional
-        global_info = {
+        data_file="example_cf32.sigmf-data",  # extension is optional
+        global_info={
             SigMFFile.DATATYPE_KEY: get_data_type_str(data),  # in this case, 'cf32_le'
             SigMFFile.SAMPLE_RATE_KEY: 48000,
-            SigMFFile.AUTHOR_KEY: 'jane.doe@domain.org',
-            SigMFFile.DESCRIPTION_KEY: 'All zero complex float32 example file.',
-        }
+            SigMFFile.AUTHOR_KEY: "jane.doe@domain.org",
+            SigMFFile.DESCRIPTION_KEY: "All zero complex float32 example file.",
+        },
     )
 
     # create a capture key at time index 0
-    meta.add_capture(0, metadata={
-        SigMFFile.FREQUENCY_KEY: 915000000,
-        SigMFFile.DATETIME_KEY: get_sigmf_iso8601_datetime_now(),
-    })
+    meta.add_capture(
+        0,
+        metadata={
+            SigMFFile.FREQUENCY_KEY: 915000000,
+            SigMFFile.DATETIME_KEY: get_sigmf_iso8601_datetime_now(),
+        },
+    )
 
     # add an annotation at sample 100 with length 200 & 10 KHz width
-    meta.add_annotation(100, 200, metadata = {
-        SigMFFile.FLO_KEY: 914995000.0,
-        SigMFFile.FHI_KEY: 915005000.0,
-        SigMFFile.COMMENT_KEY: 'example annotation',
-    })
+    meta.add_annotation(
+        100,
+        200,
+        metadata={
+            SigMFFile.FLO_KEY: 914995000.0,
+            SigMFFile.FHI_KEY: 915005000.0,
+            SigMFFile.COMMENT_KEY: "example annotation",
+        },
+    )
 
     # check for mistakes & write to disk
-    meta.tofile('example_cf32.sigmf-meta') # extension is optional
+    meta.tofile("example_cf32.sigmf-meta")  # extension is optional
 
 Now lets add another SigMF Recording and associate them with a SigMF Collection:
 
@@ -103,47 +112,50 @@ Now lets add another SigMF Recording and associate them with a SigMF Collection:
 
     data_ci16 = np.zeros(1024, dtype=np.complex64)
 
-    #rescale and save as a complex int16 file:
+    # rescale and save as a complex int16 file:
     data_ci16 *= pow(2, 15)
-    data_ci16.view(np.float32).astype(np.int16).tofile('example_ci16.sigmf-data')
+    data_ci16.view(np.float32).astype(np.int16).tofile("example_ci16.sigmf-data")
 
     # create the metadata for the second file
     meta_ci16 = SigMFFile(
-        data_file='example_ci16.sigmf-data', # extension is optional
-        global_info = {
-            SigMFFile.DATATYPE_KEY: 'ci16_le', # get_data_type_str() is only valid for numpy types
+        data_file="example_ci16.sigmf-data",  # extension is optional
+        global_info={
+            SigMFFile.DATATYPE_KEY: "ci16_le",  # get_data_type_str() is only valid for numpy types
             SigMFFile.SAMPLE_RATE_KEY: 48000,
-            SigMFFile.DESCRIPTION_KEY: 'All zero complex int16 file.',
-        }
+            SigMFFile.DESCRIPTION_KEY: "All zero complex int16 file.",
+        },
     )
     meta_ci16.add_capture(0, metadata=meta.get_capture_info(0))
-    meta_ci16.tofile('example_ci16.sigmf-meta')
-
-    collection = SigMFCollection(['example_cf32.sigmf-meta', 'example_ci16.sigmf-meta'],
-            metadata = {'collection': {
-                SigMFCollection.AUTHOR_KEY: 'sigmf@sigmf.org',
-                SigMFCollection.DESCRIPTION_KEY: 'Collection of two all zero files.',
+    meta_ci16.tofile("example_ci16.sigmf-meta")
+
+    collection = SigMFCollection(
+        ["example_cf32.sigmf-meta", "example_ci16.sigmf-meta"],
+        metadata={
+            "collection": {
+                SigMFCollection.AUTHOR_KEY: "sigmf@sigmf.org",
+                SigMFCollection.DESCRIPTION_KEY: "Collection of two all zero files.",
             }
-        }
+        },
     )
     streams = collection.get_stream_names()
     sigmf = [collection.get_SigMFFile(stream) for stream in streams]
-    collection.tofile('example_zeros.sigmf-collection')
+    collection.tofile("example_zeros.sigmf-collection")
 
 The SigMF Collection and its associated Recordings can now be loaded like this:
 
 .. code-block:: python
 
     import sigmf
-    collection = sigmf.fromfile('example_zeros')
-    ci16_sigmffile = collection.get_SigMFFile(stream_name='example_ci16')
-    cf32_sigmffile = collection.get_SigMFFile(stream_name='example_cf32')
+
+    collection = sigmf.fromfile("example_zeros")
+    ci16_sigmffile = collection.get_SigMFFile(stream_name="example_ci16")
+    cf32_sigmffile = collection.get_SigMFFile(stream_name="example_cf32")
 
 -----------------------------------------------
 Load a SigMF Archive and slice without untaring
 -----------------------------------------------
 
-Since an *archive* is merely a tarball (uncompressed), and since there any many
+Since an *archive* is a tarball (uncompressed by default), and since there are many
 excellent tools for manipulating tar files, it's fairly straightforward to
 access the *data* part of a SigMF archive without un-taring it. This is a
 compelling feature because **1** archives make it harder for the ``-data`` and
@@ -195,3 +207,50 @@ read it, this can be done "in mid air" or "without touching the ground (disk)".
     >>> arc[:10]
     array([-20.+11.j, -21. -6.j, -17.-20.j, -13.-52.j,   0.-75.j,  22.-58.j,
             48.-44.j,  49.-60.j,  31.-56.j,  23.-47.j], dtype=complex64)
+
+------------------------------
+Compressed SigMF Archives
+------------------------------
+
+SigMF archives can be compressed using gzip, xz, or zip.
+The file extension determines the archive format:
+
++---------------------+-------------+
+| Extension           | Format      |
++=====================+=============+
+| ``.sigmf``          | uncompressed|
++---------------------+-------------+
+| ``.sigmf.gz``       | gzip tar    |
++---------------------+-------------+
+| ``.sigmf.xz``       | xz tar      |
++---------------------+-------------+
+| ``.sigmf.zip``      | zip archive |
++---------------------+-------------+
+
+**Writing compressed archives:**
+
+::
+
+    >>> import sigmf
+    >>> signal = sigmf.sigmffile.fromfile('recording.sigmf-meta')
+
+    # extension determines format
+    >>> signal.tofile('recording.sigmf.xz')
+    >>> signal.archive('recording.sigmf.gz')
+
+    # compression parameter creates archive with correct extension
+    >>> signal.tofile('recording', compression='xz')  # → recording.sigmf.xz
+    >>> signal.archive('recording', compression='gz') # → recording.sigmf.gz
+
+**Reading compressed archives:**
+
+::
+
+    >>> signal = sigmf.fromfile('recording.sigmf.xz')
+    >>> signal[:10]
+    array([-20.+11.j, ...], dtype=complex64)
+
+**Memory behavior:**
+
+Uncompressed ``.sigmf`` archives use ``numpy.memmap`` for zero-copy access.
+Compressed archives must decompress into RAM before access.

docs/source/converters.rst

@@ -29,8 +29,8 @@ formats and reads without writing any output files:
 
     # auto-detect and create NCD for any supported format
     meta = sigmf.fromfile("recording.cdif")  # BLUE file
-    meta = sigmf.fromfile("recording.wav")   # WAV file
-    meta = sigmf.fromfile("recording.xml")   # Signal Hound Spike file
+    meta = sigmf.fromfile("recording.wav")  # WAV file
+    meta = sigmf.fromfile("recording.xml")  # Signal Hound Spike file
     meta = sigmf.fromfile("recording.sigmf")  # SigMF archive
 
     all_samples = meta.read_samples()

docs/source/quickstart.rst

@@ -23,15 +23,16 @@ Read a SigMF Recording
 .. code-block:: python
 
     import sigmf
+
     handle = sigmf.fromfile("example.sigmf")
     # reading data
-    handle.read_samples() # read all timeseries data
-    handle[10:50] # read memory slice of samples 10 through 50
+    handle.read_samples()  # read all timeseries data
+    handle[10:50]  # read memory slice of samples 10 through 50
     # accessing metadata
-    handle.sample_rate # get sample rate attribute
-    handle.get_global_info() # returns 'global' dictionary
-    handle.get_captures() # returns list of 'captures' dictionaries
-    handle.get_annotations() # returns list of all annotations
+    handle.sample_rate  # get sample rate attribute
+    handle.get_global_info()  # returns 'global' dictionary
+    handle.get_captures()  # returns list of 'captures' dictionaries
+    handle.get_annotations()  # returns list of all annotations
 
 -----------------------------------
 Verify SigMF Integrity & Compliance
@@ -45,6 +46,35 @@ Verify SigMF Integrity & Compliance
 Save a Numpy array as a SigMF Recording
 ---------------------------------------
 
+.. code-block:: python
+
+    import numpy as np
+    import sigmf
+
+    # suppose we have a complex timeseries signal
+    data = np.zeros(1024, dtype=np.complex64)
+
+    # create SigMFFile from array — datatype is inferred from the numpy array
+    meta = sigmf.fromarray(data, sample_rate=48000, frequency=915e6)
+
+    # write to separate .sigmf-meta and .sigmf-data files
+    meta.tofile("example")
+
+    # or write to a SigMF archive (example.sigmf)
+    meta.tofile("example.sigmf")
+
+    # or write to a compressed archive (example.sigmf.xz)
+    meta.tofile("example.sigmf.xz")
+
+The ``SigMFFile`` object can be modified before writing to add additional
+captures, annotations, or global metadata fields.
+
+---------------------------------------------------
+Save a Numpy array with Full Metadata (Advanced)
+---------------------------------------------------
+
+For full control over global fields, captures, and annotations:
+
 .. code-block:: python
 
     import numpy as np
@@ -59,30 +89,37 @@ Save a Numpy array as a SigMF Recording
 
     # create the metadata
     meta = SigMFFile(
-        data_file="example.sigmf-data", # extension is optional
-        global_info = {
+        data_file="example.sigmf-data",  # extension is optional
+        global_info={
             SigMFFile.DATATYPE_KEY: get_data_type_str(data),  # in this case, "cf32_le"
             SigMFFile.SAMPLE_RATE_KEY: 48000,
             SigMFFile.AUTHOR_KEY: "jane.doe@domain.org",
             SigMFFile.DESCRIPTION_KEY: "All zero complex float32 example file.",
-        }
+        },
     )
 
     # create a capture key at time index 0
-    meta.add_capture(0, metadata={
-        SigMFFile.FREQUENCY_KEY: 915000000,
-        SigMFFile.DATETIME_KEY: get_sigmf_iso8601_datetime_now(),
-    })
+    meta.add_capture(
+        0,
+        metadata={
+            SigMFFile.FREQUENCY_KEY: 915000000,
+            SigMFFile.DATETIME_KEY: get_sigmf_iso8601_datetime_now(),
+        },
+    )
 
     # add an annotation at sample 100 with length 200 & 10 KHz width
-    meta.add_annotation(100, 200, metadata = {
-        SigMFFile.FLO_KEY: 914995000.0,
-        SigMFFile.FHI_KEY: 915005000.0,
-        SigMFFile.COMMENT_KEY: "example annotation",
-    })
+    meta.add_annotation(
+        100,
+        200,
+        metadata={
+            SigMFFile.FLO_KEY: 914995000.0,
+            SigMFFile.FHI_KEY: 915005000.0,
+            SigMFFile.COMMENT_KEY: "example annotation",
+        },
+    )
 
     # validate & write to disk
-    meta.tofile("example.sigmf-meta") # extension is optional
+    meta.tofile("example.sigmf-meta")  # extension is optional
 
 ----------------------------------
 Attribute Access for Global Fields

docs/source/siggen.rst

@@ -64,8 +64,8 @@ A seed ensures reproducibility across runs.
     signal = SigMFGenerator(seed=0xDEADBEEF).generate()
 
     # the number and type of components are randomly chosen
-    print(signal.description)       # e.g. "synthetic signal with 3 tones and 2 sweeps"
-    print(signal.get_annotations()) # one annotation per component
+    print(signal.description)  # e.g. "synthetic signal with 3 tones and 2 sweeps"
+    print(signal.get_annotations())  # one annotation per component
 
 Without a seed, each call produces a different signal.
 

sigmf/__init__.py

@@ -5,7 +5,7 @@
 # SPDX-License-Identifier: LGPL-3.0-or-later
 
 # version of this python module
-__version__ = "1.9.1"
+__version__ = "1.10.0"
 # matching version of the SigMF specification
 __specification__ = "1.2.6"
 
@@ -22,4 +22,4 @@
 from .archive import SigMFArchive
 from .archivereader import SigMFArchiveReader
 from .siggen import SigMFGenerator
-from .sigmffile import SigMFCollection, SigMFFile, fromarchive, fromfile
+from .sigmffile import SigMFCollection, SigMFFile, fromarchive, fromarray, fromfile