Skip to content

Commit 93affa8

Browse files
committed
refactor: deduplicate echodata storage/plot APIs and fix integration tests
- Remove export_interactive_echogram from echogram.py (use create_interactive_echogram) - Update plot_and_upload_echograms to call existing create_interactive_echogram - Add connection_string param to get_azure_credentials and get_azure_filesystem - Propagate connection_string through all downstream storage functions - Delete duplicate get_azure_blob_filesystem, open_zarr_store, save_zarr_store - Standardize env var order: AZURE_STORAGE_CONNECTION_STRING -> AZURE_CONNECTION_STRING - Fix integration tests to use platforms array format instead of legacy flat fields
1 parent 64003b3 commit 93affa8

4 files changed

Lines changed: 428 additions & 57 deletions

File tree

oceanstream/echodata/plot/echogram.py

Lines changed: 172 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1104,3 +1104,175 @@ def plot_sv_with_seabed(
11041104
logger.info(f"Echogram with seabed overlay saved to {out_path}")
11051105
return out_path
11061106

1107+
1108+
# ── Blob upload helpers (ported from saildrone.process.plot) ────────────
1109+
1110+
1111+
def plot_and_upload_echograms(
1112+
sv_dataset: "xr.Dataset",
1113+
*,
1114+
cruise_id: str | None = None,
1115+
file_base_name: str | None = None,
1116+
save_to_blobstorage: bool = False,
1117+
output_path: str | None = None,
1118+
upload_path: str | None = None,
1119+
container_name: str | None = None,
1120+
export_filename=None,
1121+
create_interactive_pages: bool = False,
1122+
channel: int | None = None,
1123+
cmap: str = "ocean_r",
1124+
plot_var: str = "Sv",
1125+
title_template: str = "{channel_label}",
1126+
depth_var: str | None = None,
1127+
connection_string: str | None = None,
1128+
) -> list[str]:
1129+
"""Generate echograms and optionally upload them to Azure Blob Storage.
1130+
1131+
When *save_to_blobstorage* is True, PNGs are written to a temp directory,
1132+
the directory is uploaded to Azure Blob, and a list of blob-relative paths
1133+
is returned.
1134+
1135+
Parameters
1136+
----------
1137+
sv_dataset : xr.Dataset
1138+
Dataset containing Sv data.
1139+
cruise_id : str, optional
1140+
Cruise identifier (used for path construction).
1141+
file_base_name : str
1142+
Base name for the echogram files.
1143+
save_to_blobstorage : bool
1144+
Upload to Azure Blob after generation.
1145+
output_path : str, optional
1146+
Local directory for non-blob output.
1147+
upload_path : str, optional
1148+
Target path inside the blob container.
1149+
container_name : str, optional
1150+
Azure Blob container name.
1151+
export_filename : callable, optional
1152+
Transform echogram filenames for the returned list.
1153+
create_interactive_pages : bool
1154+
Generate interactive HTML echograms.
1155+
channel : int, optional
1156+
Plot only a specific channel index.
1157+
cmap : str
1158+
Matplotlib colormap name.
1159+
plot_var : str
1160+
Variable to plot (default ``"Sv"``).
1161+
title_template : str
1162+
Title template with ``{channel_label}`` placeholder.
1163+
depth_var : str, optional
1164+
Depth variable name override.
1165+
connection_string : str, optional
1166+
Azure connection string override.
1167+
1168+
Returns
1169+
-------
1170+
list[str]
1171+
Paths of generated/uploaded echogram files.
1172+
"""
1173+
import shutil
1174+
1175+
if save_to_blobstorage:
1176+
echograms_dir = f"/tmp/osechograms/{cruise_id or 'default'}/{file_base_name}"
1177+
else:
1178+
if output_path is None:
1179+
raise ValueError("output_path is required when save_to_blobstorage is False")
1180+
echograms_dir = f"{output_path}/echograms/{file_base_name}"
1181+
1182+
os.makedirs(echograms_dir, exist_ok=True)
1183+
1184+
echogram_files = plot_sv_data(
1185+
sv_dataset,
1186+
file_base_name=file_base_name or "echogram",
1187+
output_path=echograms_dir,
1188+
plot_var=plot_var,
1189+
cmap=cmap,
1190+
channel=channel,
1191+
title_template=title_template,
1192+
)
1193+
1194+
if create_interactive_pages:
1195+
try:
1196+
n_ch = sv_dataset.sizes.get("channel", 1)
1197+
for ch in range(n_ch):
1198+
html_path = f"{echograms_dir}/{file_base_name}_{ch}.html"
1199+
create_interactive_echogram(
1200+
sv_dataset, channel=ch, out_html=html_path,
1201+
var=plot_var, cmap=cmap,
1202+
)
1203+
except Exception as exc:
1204+
logger.warning("Interactive echogram generation failed: %s", exc)
1205+
1206+
if save_to_blobstorage:
1207+
from oceanstream.echodata.storage import upload_file_to_blob
1208+
1209+
upload_dest = upload_path or f"{cruise_id}/{file_base_name}"
1210+
1211+
# Upload all files in the echograms directory
1212+
echograms_path = Path(echograms_dir)
1213+
for fpath in echograms_path.rglob("*"):
1214+
if fpath.is_file():
1215+
blob_name = f"{upload_dest}/{fpath.name}"
1216+
upload_file_to_blob(
1217+
str(fpath), blob_name, container_name,
1218+
connection_string=connection_string,
1219+
)
1220+
1221+
shutil.rmtree(echograms_dir, ignore_errors=True)
1222+
1223+
if export_filename is not None:
1224+
uploaded_files = [export_filename(str(e)) for e in echogram_files]
1225+
else:
1226+
uploaded_files = [
1227+
f"{cruise_id}/{file_base_name}/{Path(e).name}" for e in echogram_files
1228+
]
1229+
else:
1230+
uploaded_files = [str(Path(e).name) for e in echogram_files]
1231+
1232+
return uploaded_files
1233+
1234+
1235+
def plot_and_upload_masks(
1236+
ds: "xr.Dataset",
1237+
*,
1238+
file_base_name: str | None = None,
1239+
upload_path: str | None = None,
1240+
container_name: str | None = None,
1241+
title_template: str = "{channel_label} – {cube_name}",
1242+
connection_string: str | None = None,
1243+
) -> list[str]:
1244+
"""Plot mask cubes and upload to Azure Blob Storage.
1245+
1246+
Returns
1247+
-------
1248+
list[str]
1249+
Blob-relative paths of the uploaded images.
1250+
"""
1251+
import shutil
1252+
1253+
local_dir = f"/tmp/osechograms/{file_base_name}"
1254+
os.makedirs(local_dir, exist_ok=True)
1255+
1256+
try:
1257+
paths = plot_masks_vertical(
1258+
ds, file_base_name=file_base_name, output_path=local_dir,
1259+
title_template=title_template,
1260+
)
1261+
except Exception as exc:
1262+
logger.error("plot_masks_vertical failed: %s", exc)
1263+
return []
1264+
1265+
from oceanstream.echodata.storage import upload_file_to_blob
1266+
1267+
for fpath in Path(local_dir).rglob("*"):
1268+
if fpath.is_file():
1269+
blob_name = f"{upload_path}/{fpath.name}"
1270+
upload_file_to_blob(
1271+
str(fpath), blob_name, container_name,
1272+
connection_string=connection_string,
1273+
)
1274+
1275+
shutil.rmtree(local_dir, ignore_errors=True)
1276+
1277+
return [f"{file_base_name}/{p.name}" for p in paths.values()]
1278+

0 commit comments

Comments
 (0)