Skip to content

Optimize extract #37

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jourdain merged 8 commits into from
Apr 8, 2026
Merged

Optimize extract #37

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
9249e96
fix: check self MTime (projection was modified) as well
danlipsa Mar 30, 2026
b7a17c9
fix: cache output for EAMExtract
danlipsa Apr 2, 2026
dd6ec62
fix: allow trimming beyond mid point
danlipsa Apr 6, 2026
dadc5ae
fix: fix crash when: trim_lon = 200 then trim_lon = 0
danlipsa Apr 7, 2026
cf1ad82
fix: remove vtkPVGeometryFilter as it is applied in EAMExtract
danlipsa Apr 8, 2026
cf7558a
fix: no need to do something different for latlon projection
danlipsa Apr 8, 2026
669576a
fix: fix the filter for polydata input
danlipsa Apr 8, 2026
2144611
fix: better fix for MTime() issue
danlipsa Apr 8, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 3 additions & 13 deletions src/e3sm_quickview/pipeline.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@
from pathlib import Path

from paraview import simple
from paraview.modules.vtkPVVTKExtensionsFiltersRendering import vtkPVGeometryFilter
from vtkmodules.vtkCommonCore import vtkLogger
from vtkmodules.vtkRenderingCore import vtkActor, vtkPolyDataMapper

Expand Down Expand Up @@ -153,17 +152,8 @@ def __init__(self, projection="Mollweide"):
Projection=projection,
Translate=0,
)
vtk_geometry = self.proj.GetClientSideObject()
self.vtk_geometry = vtkPVGeometryFilter(
use_outline=0,
block_colors_distinct_values=0,
generate_cell_normals=0,
generate_point_normals=0,
generate_feature_edges=0,
splitting=False,
triangulate=0,
input_connection=vtk_geometry.output_port,
)
self.geometry = simple.ExtractSurface(Input=self.proj)
self.vtk_geometry = self.geometry.GetClientSideObject()

# Add observer to
vtk_obj = self.reader.GetClientSideObject()
Expand Down Expand Up @@ -250,7 +240,7 @@ def update(self, time=0.0):
if not self.valid:
return

self.proj.UpdatePipeline(time)
self.geometry.UpdatePipeline(time)

def crop(self, longitude_min_max, latitude_min_max):
self._crop.TrimLongitude = range_to_trim(longitude_min_max, 180)
Expand Down
131 changes: 75 additions & 56 deletions src/e3sm_quickview/plugins/eam_projection.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
from paraview.util.vtkAlgorithm import *
from vtkmodules.vtkCommonCore import (
vtkPoints,
vtkUnsignedCharArray,
)
from vtkmodules.vtkCommonDataModel import (
vtkCellArray,
Expand Down Expand Up @@ -61,6 +62,44 @@ def ProcessPoint(point, radius):
return [x, y, z]


def add_cell_arrays(inData, outData, cached_output):
"""
Adds arrays not modified in inData to outData.
New arrays (or arrays modified) values are
set using the PedigreeIds because the number of values
in the new array (just read from the file) is different
than the number of values in the arrays already processed through he
pipeline.
"""
pedigreeIds = cached_output.cell_data["PedigreeIds"]
if pedigreeIds is None:
print_error("Error: no PedigreeIds array")
return
cached_cell_data = cached_output.GetCellData()
in_cell_data = inData.GetCellData()
outData.ShallowCopy(cached_output)
out_cell_data = outData.GetCellData()

out_cell_data.Initialize()
for i in range(in_cell_data.GetNumberOfArrays()):
in_array = in_cell_data.GetArray(i)
cached_array = cached_cell_data.GetArray(in_array.GetName())
if cached_array and cached_array.GetMTime() >= in_array.GetMTime():
# this scalar has been seen before
# simply add a reference in the outData
out_cell_data.AddArray(cached_array)
else:
# this scalar is new
# we have to fill in the additional cells resulted from the clip
out_array = in_array.NewInstance()
array0 = cached_cell_data.GetArray(0)
out_array.SetNumberOfComponents(array0.GetNumberOfComponents())
out_array.SetNumberOfTuples(array0.GetNumberOfTuples())
out_array.SetName(in_array.GetName())
out_cell_data.AddArray(out_array)
outData.cell_data[out_array.GetName()] = inData.cell_data[i][pedigreeIds]


@smproxy.filter()
@smproperty.input(name="Input")
@smdomain.datatype(
Expand Down Expand Up @@ -271,24 +310,14 @@ def RequestData(self, request, inInfo, outInfo):
else:
outData.ShallowCopy(inData)

if self.project == 0:
# Use cache to move mtime forward when needed
if self.cached_points is None:
self.cached_points = vtkPoints()
self.cached_points.ShallowCopy(inData.GetPoints())

outData.SetPoints(self.cached_points)
return 1

if (
self.cached_points
and self.cached_points.GetMTime() >= inData.GetPoints().GetMTime()
if self.cached_points and self.cached_points.GetMTime() >= max(
inData.GetPoints().GetMTime(), self.GetMTime()
):
outData.SetPoints(self.cached_points)
else:
# we modify the points, so copy them
out_points_vtk = vtkPoints()
out_points_vtk.DeepCopy(inData.GetPoints())
out_points_vtk.DeepCopy(outData.GetPoints())
outData.SetPoints(out_points_vtk)
out_points_np = outData.points

Expand Down Expand Up @@ -442,22 +471,22 @@ def __init__(self):
self.trim_lon = [0, 0]
self.trim_lat = [0, 0]
self.cached_cell_centers = None
self.cached_ghosts = None
self._cached_output = None

def SetTrimLongitude(self, left, right):
if left < 0 or left > 180 or right < 0 or right > 180:
if left < 0 or left > 360 or right < 0 or right > 360 or left > (360 - right):
print_error(
f"SetTrimLongitude called with parameters outside [0, 180]: {left=}, {right=}"
f"SetTrimLongitude called with invalid parameters: {left=}, {right=}"
)
return
if self.trim_lon[0] != left or self.trim_lon[1] != right:
self.trim_lon = [left, right]
self.Modified()

def SetTrimLatitude(self, left, right):
if left < 0 or left > 90 or right < 0 or right > 90:
if left < 0 or left > 180 or right < 0 or right > 180 or left > (180 - right):
print_error(
f"SetTrimLatitude called with parameters outside [0, 180]: {left=}, {right=}"
f"SetTrimLatitude called with invalid parameters: {left=}, {right=}"
)
return
if self.trim_lat[0] != left or self.trim_lat[1] != right:
Expand All @@ -469,9 +498,12 @@ def RequestData(self, request, inInfo, outInfo):
outData = self.GetOutputData(outInfo, 0)
if self.trim_lon == [0, 0] and self.trim_lat == [0, 0]:
outData.ShallowCopy(inData)
# if the filter execution follows an another execution that trims the
# number of points, the downstream filter could think that
# the trimmed points are still valid which results in a crash
outData.GetPoints().Modified()
return 1

outData.ShallowCopy(inData)
if self.cached_cell_centers and self.cached_cell_centers.GetMTime() >= max(
inData.GetPoints().GetMTime(), inData.GetCells().GetMTime()
):
Expand All @@ -494,27 +526,41 @@ def RequestData(self, request, inInfo, outInfo):
# get the numpy array for cell centers
cc = numpy_support.vtk_to_numpy(cell_centers)

if self.cached_ghosts and self.cached_ghosts.GetMTime() >= max(
if self._cached_output and self._cached_output.GetMTime() >= max(
self.GetMTime(), inData.GetPoints().GetMTime(), cell_centers.GetMTime()
):
ghost = self.cached_ghosts
outData.ShallowCopy(self._cached_output)
add_cell_arrays(inData, outData, self._cached_output)
else:
# import pdb;pdb.set_trace()
# add PedigreeIds
generate_ids = vtkGenerateIds()
generate_ids.SetInputData(inData)
generate_ids.PointIdsOff()
generate_ids.SetCellIdsArrayName("PedigreeIds")
generate_ids.Update()
outData.ShallowCopy(generate_ids.GetOutput())
# we have to deep copy the cell array because we modify it
# with RemoveGhostCells
cells = vtkCellArray()
cell_types = vtkUnsignedCharArray()
cells.DeepCopy(outData.GetCells())
cell_types.DeepCopy(outData.GetCellTypesArray())
outData.SetCells(cell_types, cells)

# compute the new bounds by trimming the inData bounds
bounds = list(inData.GetBounds())
bounds[0] = bounds[0] + self.trim_lon[0]
bounds[1] = bounds[1] - self.trim_lon[1]
bounds[2] = bounds[2] + self.trim_lat[0]
bounds[3] = bounds[3] - self.trim_lat[1]

# add hidden cells based on bounds
# add HIDDENCELL based on bounds
outside_mask = (
(cc[:, 0] < bounds[0])
| (cc[:, 0] > bounds[1])
| (cc[:, 1] < bounds[2])
| (cc[:, 1] > bounds[3])
)

# Create ghost array (0 = visible, HIDDENCELL = invisible)
ghost_np = np.where(
outside_mask, vtkDataSetAttributes.HIDDENCELL, 0
Expand All @@ -523,11 +569,11 @@ def RequestData(self, request, inInfo, outInfo):
# Convert to VTK and add to output
ghost = numpy_support.numpy_to_vtk(ghost_np)
ghost.SetName(vtkDataSetAttributes.GhostArrayName())
# the previous cached_ghosts, if any,
# is available for garbage collection after this assignment
self.cached_ghosts = ghost
outData.GetCellData().AddArray(ghost)
outData.GetCellData().AddArray(ghost)
outData.RemoveGhostCells()

self._cached_output = outData.NewInstance()
self._cached_output.ShallowCopy(outData)
return 1


Expand Down Expand Up @@ -602,34 +648,7 @@ def RequestData(self, request, inInfo, outInfo):
and self._cached_output.GetCells().GetMTime()
>= inData.GetCells().GetMTime()
):
# only scalars have been added or removed
cached_cell_data = self._cached_output.GetCellData()

in_cell_data = inData.GetCellData()

outData.ShallowCopy(self._cached_output)
out_cell_data = outData.GetCellData()

out_cell_data.Initialize()
for i in range(in_cell_data.GetNumberOfArrays()):
in_array = in_cell_data.GetArray(i)
cached_array = cached_cell_data.GetArray(in_array.GetName())
if cached_array and cached_array.GetMTime() >= in_array.GetMTime():
# this scalar has been seen before
# simply add a reference in the outData
out_cell_data.AddArray(cached_array)
else:
# this scalar is new
# we have to fill in the additional cells resulted from the clip
out_array = in_array.NewInstance()
array0 = cached_cell_data.GetArray(0)
out_array.SetNumberOfComponents(array0.GetNumberOfComponents())
out_array.SetNumberOfTuples(array0.GetNumberOfTuples())
out_array.SetName(in_array.GetName())
out_cell_data.AddArray(out_array)
outData.cell_data[out_array.GetName()] = inData.cell_data[i][
self._cached_output.cell_data["PedigreeIds"]
]
add_cell_arrays(inData, outData, self._cached_output)
else:
generate_ids = vtkGenerateIds()
generate_ids.SetInputData(inData)
Expand Down
Loading