Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 13 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -182,7 +182,7 @@ providers:

The third configuration option `tile_limit` can be specified to enforce a maximum number of features in a single tile.
This will apply to all tiles regardless of if the other filters are enabled by `disable_at_z`. Features will be ordered
by the size of there bounding box, pruning the smallest feature until the feature limit is met for the tile.
by the size of there bounding box, pruning the smallest feature until the feature limit is met for the tile. Point-based geometry are pruned at random.

```yaml
providers:
Expand Down Expand Up @@ -216,6 +216,18 @@ providers:
...
```

The fifth configuration option `tile_limit` can be specified to enforce a maximum size (in Mb) for a single tile.
This will apply to all tiles regardless of if the other filters are enabled by `disable_at_z`.

```yaml
providers:
- type: tile
name: pygeoapi_plugins.provider.mvt_postgresql.MVTPostgreSQLProvider_
...
disable_at_z: 0 # Apply no CQL or Pixel Size filter
tile_size: 10 # Tile size shall be no more than 10Mb
```

### MVT PostgreSQL with Caching

There are two additional Postgres based MVT providers with caching of tiles to prevent significant server load
Expand Down
57 changes: 50 additions & 7 deletions pygeoapi_plugins/provider/mvt_postgresql.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@
from pygeoapi.provider.sql import PostgreSQLProvider
from pygeoapi.crs import get_srid

from pygeoapi.util import url_join
from pygeoapi.util import url_join, human_size

LOGGER = logging.getLogger(__name__)

Expand Down Expand Up @@ -117,6 +117,17 @@ def __init__(self, provider_def):

# Maximum number of features in a tile
self.tile_limit = provider_def.get('tile_limit', 0)
geom_column = getattr(self.table_model, self.geom)
with Session(self._engine) as session:
(geom_type,) = session.query(
func.ST_GeometryType(geom_column).label('geom_type')
).first() # type: ignore
self.tile_limit_order = (
func.random() if 'point' in geom_type.lower()
else ST_Area(Box2D(geom_column)).desc()
)
# Maximum tile size (in MB)
self.tile_size = provider_def.get('tile_size', 0) * 1024 * 1024

self.min_zoom = provider_def['options']['zoom']['min']
self.max_zoom = provider_def['options']['zoom']['max']
Expand Down Expand Up @@ -160,7 +171,8 @@ def get_tiles(
LOGGER.debug(f'Querying {self.table} for MVT tile {z}/{x}/{y}')
envelope = self.get_envelope(z, y, x, tileset_schema.tileMatrixSet)
envelope_srid = get_srid(tileset_schema.crs)
mvt_cte = self._get_mvt_cte(envelope, envelope_srid, z)
mvt_cte = self._get_mvt_cte(
envelope, envelope_srid, z, self.tile_limit)
mvt_query = select(ST_AsMVT(mvt_cte, self.layer))

# Log the compiled query
Expand All @@ -172,8 +184,38 @@ def get_tiles(
# Execute the query
with Session(self._engine) as session:
result = session.execute(mvt_query).scalar()

return bytes(result) if result else None
if result is None:
return

result_bytes = bytes(result)
result_size = len(result_bytes)
if self.tile_size and self.tile_size < result_size:
LOGGER.debug(
'Tile exceeds configured size\n'
f'Provider maximum size: {human_size(self.tile_size)}\n'
f'Tile size: {human_size(result_size)}'
)

matched = session.query(func.count(mvt_cte)).scalar()
i_size = int(
matched - matched * (self.tile_size / result_size)
)

while self.tile_size < result_size:
matched -= i_size
new_mvt_cte = self._get_mvt_cte(
envelope, envelope_srid, z, matched)

new_mvt_query = select(ST_AsMVT(new_mvt_cte, self.layer))
new_result = session.execute(new_mvt_query).scalar()
if new_result is None:
return

result_bytes = bytes(new_result)
result_size = len(result_bytes)

LOGGER.debug(f'Returning tile of size: {human_size(result_size)}')
return result_bytes

def get_vendor_metadata(
self,
Expand Down Expand Up @@ -246,13 +288,14 @@ def get_metadata(self, *args, **kwargs):

return metadata

def _get_mvt_cte(self, envelope, envelope_srid, z):
def _get_mvt_cte(self, envelope, envelope_srid, z, tile_limit):
"""
Gets tile MVT Query

:param envelope: the tile envelope
:param envelope_srid: the SRID of the tile envelope
:param z: the zoom level
:param tile_limit: limit tiles based on number of features

:returns: a SQLAlchemy CTE query that returns the MVT tile features
"""
Expand Down Expand Up @@ -294,8 +337,8 @@ def _get_mvt_cte(self, envelope, envelope_srid, z):
).filter(*filters)

# Apply tile limit if set
if self.tile_limit:
query = query.order_by(func.random()).limit(self.tile_limit)
if tile_limit:
query = query.order_by(self.tile_limit_order).limit(tile_limit)

# Return as CTE
return query.cte('mvtcte').table_valued()
Expand Down
27 changes: 26 additions & 1 deletion tests/test_mvt_postgresql_provider.py
Original file line number Diff line number Diff line change
Expand Up @@ -170,7 +170,6 @@ def test_tile_limit(config):
)
assert tile2 is not None
assert len(tile2) == pytest.approx(74047, 0.5)
assert len(tile2) <= len(tile)

config['tile_limit'] = 500
p = MVTPostgreSQLProvider_(config)
Expand Down Expand Up @@ -200,6 +199,32 @@ def test_tile_limit(config):
assert len(tile4) < len(tile3)


def test_tile_size(config):
tileset = 'WebMercatorQuad'
z, x, y = 10, 595, 521

p = MVTPostgreSQLProvider_(config)
tile = p.get_tiles(
tileset=tileset,
z=z,
x=x,
y=y,
)
assert tile is not None
assert len(tile) == pytest.approx(74047, 0.1)

config['tile_size'] = 0.05
p = MVTPostgreSQLProvider_(config)
tile = p.get_tiles(
tileset=tileset,
z=z,
x=x,
y=y,
)
assert tile is not None
assert len(tile) == pytest.approx(0.05 * 1024 * 1024, 0.1)


def test_tile_simplify(config):
tileset = 'WebMercatorQuad'
z, x, y = 10, 595, 521
Expand Down
Loading