|
30 | 30 |
|
31 | 31 | # cast-value.py |
32 | 32 |
|
33 | | -Python implementation of the `cast_value` codec for Zarr. |
| 33 | +A python implementation of the `cast_value` codec for [Zarr](https://zarr.dev/), |
| 34 | +with [zarr-python](https://zarr.readthedocs.io/en/stable/) integration. |
34 | 35 |
|
35 | | -## `cast_value` codec |
| 36 | +## what |
36 | 37 |
|
37 | | -The `cast_value` codec defines an operation for safely converting an array from |
38 | | -one numeric data type to another. |
| 38 | +The `cast_value` codec defines how to _safely_ convert arrays between integer |
| 39 | +and float data types. In Zarr terminology, this codec is an "array -> array" |
| 40 | +codec, which means its input and output are both arrays. |
| 41 | + |
| 42 | +You can find the |
| 43 | +[specification for this codec](https://github.com/zarr-developers/zarr-extensions/tree/main/codecs/cast_value) |
| 44 | +in the |
| 45 | +[zarr-extensions repository](https://github.com/zarr-developers/zarr-extensions). |
| 46 | + |
| 47 | +## why |
| 48 | + |
| 49 | +This codec is commonly used to for lossy data compression: when decoded data |
| 50 | +should be high-precision floats, but the absolute range of the values fits |
| 51 | +within the range of a smaller integer data type, then encoding the floats as |
| 52 | +ints before writing data can vastly shrink the stored values. |
| 53 | + |
| 54 | +For example, if your data is a sequence of `float64` values like |
| 55 | +`[100.1, 120.3, 125.5]`, storing those values as `uint8`, e.g. |
| 56 | +`[100, 120, 125]`, offers 8-fold reduction in storage size, provided the |
| 57 | +precision lost due to rounding is acceptable. |
| 58 | + |
| 59 | +## how |
| 60 | + |
| 61 | +```python |
| 62 | +# import the codec that uses the rust backend |
| 63 | +from cast_value import CastValueRustV1 |
| 64 | + |
| 65 | +# Create an in-memory zarr array with float64 dtype, stored as uint8. |
| 66 | +# The cast_value codec handles the conversion: float64 -> uint8 on write, |
| 67 | +# uint8 -> float64 on read. |
| 68 | + |
| 69 | +codec = CastValueRustV1( |
| 70 | + data_type="uint8", |
| 71 | + rounding="nearest-even", |
| 72 | + out_of_range="clamp", |
| 73 | + scalar_map={ |
| 74 | + "encode": [(np.nan, 0), (np.inf, 1), (-np.inf, 2)], |
| 75 | + "decode": [(0, np.nan), (1, np.inf), (2, -np.inf)], |
| 76 | + }, |
| 77 | +) |
| 78 | +# Create array and write float64 data — values are rounded and clamped to [0, 255] |
| 79 | +data = np.array([np.nan, np.inf, -np.inf, 3.3, 4]) |
| 80 | +arr = zarr.create_array(data=data, store=zarr.storage.MemoryStore(), filters=codec) |
| 81 | + |
| 82 | +# Read it back — comes back as float64, but with uint8 precision |
| 83 | +result = arr[:] |
| 84 | + |
| 85 | +print(f"Array dtype: {arr.dtype}") |
| 86 | +print(f"Values written: {data}") |
| 87 | +print(f"Values read: {result}") |
| 88 | + |
| 89 | +""" |
| 90 | +Array dtype: float64 |
| 91 | +Values written: [ nan inf -inf 3.3 4. ] |
| 92 | +Values read: [ nan inf -inf 3. 4.] |
| 93 | +""" |
| 94 | +``` |
| 95 | + |
| 96 | +# who |
| 97 | + |
| 98 | +Davis Bennett (@d-v-b) |
0 commit comments