TilesRenderer: Add concept of "virtual" tiles, support additive tile splitting (#1498)

* Add "isVirtual" flag

* Adjust handling of virtual children in QuantizedMeshPlugin

* ImageOverlayPlugin: use isVirtual

* Update imageoverlayplugin

* Update ImageOverlayPlugin

* Cleanup

* Add virtual support to "optimizedTraverseFunctions"

* Add types

* Use internal virtual properties

* Use any pre-seeded fields

* Fixed

* Fix "processed" check

* Small updates

* fix race condition for quantized mesh splitting

* Update comment

* Update ImageOverlayPlugin

Author: gkjohnson

src/core/renderer/tiles/Tile.d.ts

@@ -11,6 +11,14 @@ export interface TileInternalData {
 	basePath: string;
 	depth: number;
 	depthFromRenderedParent: number;
+	/**
+	 * Whether this tile was synthetically based on loaded parent tile state.
+	 */
+	isVirtual: boolean;
+	/**
+	 * The number of virtual children appended to this tile's children array by plugins.
+	 */
+	virtualChildCount: number;
 }
 
 /**

src/core/renderer/tiles/TilesRendererBase.js

@@ -812,6 +812,11 @@ export class TilesRendererBase {
 			basePath: tilesetDir,
 			depth: - 1,
 			depthFromRenderedParent: - 1,
+			isVirtual: false,
+			virtualChildCount: 0,
+
+			// preserve any pre-seeded fields
+			...tile.internal,
 		};
 
 		if ( tile.content?.uri ) {
@@ -983,7 +988,7 @@ export class TilesRendererBase {
 	ensureChildrenArePreprocessed( tile, forceImmediate = this.stats.tilesProcessed < this.maxTilesProcessed ) {
 
 		const children = tile.children;
-		if ( children.length === 0 || children[ 0 ].internal ) {
+		if ( children.length === 0 || children[ 0 ].traversal ) {
 
 			return;
 
@@ -995,7 +1000,6 @@ export class TilesRendererBase {
 
 				this.preprocessNode( children[ i ], tile.internal.basePath, tile );
 
-
 			}
 
 		};

src/core/renderer/tiles/optimizedTraverseFunctions.js

@@ -21,6 +21,8 @@ function isUsedThisFrame( tile, frameCount ) {
 
 function isProcessed( tile ) {
 
+	// TODO: here "traversal" is used to determine whether a tile has been initialized. We should
+	// move these to tiles utils
 	return Boolean( tile.traversal );
 
 }
@@ -29,7 +31,7 @@ function isProcessed( tile ) {
 function areChildrenProcessed( tile ) {
 
 	// all children are processed at once
-	const childrenReady = tile.children.length === 0 || Boolean( tile.children[ 0 ].internal );
+	const childrenReady = tile.children.length === 0 || isProcessed( tile.children[ 0 ] );
 	const contentReady = ! tile.internal.hasUnrenderableContent || isDownloadFinished( tile.internal.loadingState );
 	return childrenReady && contentReady;
 
@@ -437,6 +439,14 @@ function toggleTiles( tile, renderer ) {
 
 		}
 
+		// keep tiles with virtual children retained in the LRU cache so the content is
+		// available to regenerate virtual children if the overlay configuration changes.
+		if ( tile.internal.virtualChildCount > 0 && tile.internal.hasContent ) {
+
+			renderer.markTileUsed( tile );
+
+		}
+
 		// if the tile is loaded and in frustum we can mark it as visible
 		tile.traversal.visible = tile.internal.hasRenderableContent && tile.traversal.active && tile.traversal.inFrustum && tile.internal.loadingState === LOADED;
 		renderer.stats.used ++;

src/core/renderer/tiles/traverseFunctions.js

@@ -34,7 +34,7 @@ function isProcessed( tile ) {
 function areChildrenProcessed( tile ) {
 
 	// all children are processed at once
-	return tile.children.length === 0 || Boolean( tile.children[ 0 ].internal );
+	return tile.children.length === 0 || isProcessed( tile.children[ 0 ] );
 
 }
 

src/three/plugins/QuantizedMeshPlugin.js

@@ -314,6 +314,8 @@ export class QuantizedMeshPlugin {
 		// data is ready for clipping. It's possible that this child data gets to the parse stage
 		// first, otherwise, while the parent is still downloading.
 		// Ideally we would be able to guarantee parents are loaded first but this is an odd case.
+		// Assign the scene value preemptively to ensure it's available for splitting.
+		tile.engineData.scene = result;
 		this.expandChildren( tile );
 
 		return result;
@@ -406,8 +408,11 @@ export class QuantizedMeshPlugin {
 
 				} else {
 
-					tile.children.push( child );
+					// mark the child as "virtual" since it relies on the parent geometry
 					child.content = { uri: `tile.quantized_tile_split?bottom=${ cy === 0 }&left=${ cx === 0 }` };
+					child.internal = { isVirtual: true };
+					tile.internal.virtualChildCount ++;
+					tile.children.push( child );
 
 				}
 
@@ -417,7 +422,8 @@ export class QuantizedMeshPlugin {
 
 		if ( ! hasChildren ) {
 
-			tile.children.length = 0;
+			tile.children.length -= tile.internal.virtualChildCount;
+			tile.internal.virtualChildCount = 0;
 
 		}
 
@@ -436,27 +442,31 @@ export class QuantizedMeshPlugin {
 
 	disposeTile( tile ) {
 
+		const { tiles, layer } = this;
+
 		// dispose of the available array since we will get it again if this tile is loaded
-		if ( getTileHasMetadata( tile, this.layer ) ) {
+		if ( getTileHasMetadata( tile, layer ) ) {
 
 			tile[ TILE_AVAILABLE ] = null;
 
 		}
 
-		// Note: we remove all children always because child tiles can rely on splitting parent tiles
-		// and we can find ourselves in a situation where a child tile is ready first but the parent tile
-		// hasn't loaded, causing a stall / race condition in the parsing queue. To avoid this dependency
-		// we just remove all children and generate them again one the parent is loaded.
-		// Only get rid of the children if this plugin was responsible for them.
+		// Remove virtual children when the parent is disposed since they depend on the parent's
+		// loaded scene for clipping and cannot be rendered or re-generated without it. They will
+		// be re-created once the parent is loaded again.
 		if ( TILE_AVAILABLE in tile ) {
 
-			tile.children.forEach( child => {
+			const { virtualChildCount } = tile.internal;
+			const len = tile.children.length;
+			const start = len - virtualChildCount;
+			for ( let i = start; i < len; i ++ ) {
 
-				// TODO: there should be a reliable way for removing children like this.
-				this.tiles.processNodeQueue.remove( child );
+				tiles.processNodeQueue.remove( tile.children[ i ] );
 
-			} );
-			tile.children.length = 0;
+			}
+
+			tile.children.length -= virtualChildCount;
+			tile.internal.virtualChildCount = 0;
 
 		}
 

src/three/plugins/images/ImageOverlayPlugin.js

@@ -22,6 +22,7 @@ const _normal = /* @__PURE__ */ new Vector3();
 const _box = /* @__PURE__ */ new Box3();
 const SPLIT_TILE_DATA = Symbol( 'SPLIT_TILE_DATA' );
 const SPLIT_HASH = Symbol( 'SPLIT_HASH' );
+const ORIGINAL_REFINE = Symbol( 'ORIGINAL_REFINE' );
 
 // Plugin for overlaying tiled image data on top of 3d tiles geometry.
 export class ImageOverlayPlugin {
@@ -221,12 +222,46 @@ export class ImageOverlayPlugin {
 
 	}
 
+	_removeVirtualChildren( tile ) {
+
+		if ( ! ( ORIGINAL_REFINE in tile ) ) {
+
+			return;
+
+		}
+
+		// remove the virtual children associated with the given tile
+		const { tiles } = this;
+		const { virtualChildCount } = tile.internal;
+		const len = tile.children.length;
+		const start = len - virtualChildCount;
+		for ( let i = start; i < len; i ++ ) {
+
+			const child = tile.children[ i ];
+			tiles.processNodeQueue.remove( child );
+			tiles.lruCache.remove( child );
+			child.parent = null;
+
+		}
+
+		tile.children.length -= virtualChildCount;
+		tile.internal.virtualChildCount = 0;
+		tile.refine = tile[ ORIGINAL_REFINE ];
+		delete tile[ ORIGINAL_REFINE ];
+		delete tile[ SPLIT_HASH ];
+
+	}
+
 	disposeTile( tile ) {
 
 		const { overlayInfo, tileControllers, processQueue, pendingTiles, processedTiles } = this;
 
 		processedTiles.delete( tile );
 
+		// remove any virtual children since they depend on this tile being loaded for regeneration.
+		// they will be recreated with fresh split configuration when the tile is reloaded.
+		this._removeVirtualChildren( tile );
+
 		// Cancel any ongoing tasks. If a tile is cancelled while downloading
 		// this will not have been created, yet.
 		if ( tileControllers.has( tile ) ) {
@@ -354,8 +389,6 @@ export class ImageOverlayPlugin {
 			this._updateLayers( tile );
 			this.disposeTile( tile );
 
-			delete tile[ SPLIT_HASH ];
-
 		} );
 
 		tiles.removeEventListener( 'update-after', this._onUpdateAfter );
@@ -403,49 +436,37 @@ export class ImageOverlayPlugin {
 
 		}
 
-		// collect the tiles split into virtual tiles
+		// collect the tiles split into virtual tiles, sorted deepest-first so nested virtual tiles
+		// are cleaned up before their parents when iterating
 		const { tiles } = this;
-		const parents = new Set();
+		const splitTiles = [];
 		this.processedTiles.forEach( tile => {
 
 			if ( SPLIT_HASH in tile ) {
 
-				parents.add( tile );
+				splitTiles.push( tile );
 
 			}
 
 		} );
 
-		// dispose of the virtual children if this tile would not be split or the spilt could change
-		// under the current overlays used.
-		parents.forEach( parent => {
-
-			if ( parent.parent === null ) {
-
-				return;
+		// ensure we clean depth first
+		splitTiles.sort( ( a, b ) => b.internal.depth - a.internal.depth );
 
-			}
+		// dispose of the virtual children if this tile would not be split or the split could change
+		// under the current overlays used.
+		splitTiles.forEach( tile => {
 
-			const clone = parent.engineData.scene.clone();
+			const clone = tile.engineData.scene.clone();
 			clone.updateMatrixWorld();
 
-			if ( fullDispose || parent[ SPLIT_HASH ] !== this._getSplitVectors( clone, parent ).hash ) {
-
-				// TODO: if are parent tile is forcibly remove then we should make sure that all the children are, too?
-				const children = collectChildren( parent );
-				children.sort( ( a, b ) => ( b.internal.depth || 0 ) - ( a.internal.depth || 0 ) );
+			if ( fullDispose || tile[ SPLIT_HASH ] !== this._getSplitVectors( clone, tile ).hash ) {
 
 				// note that we need to remove children from the processing queue in this case
-				// because we are forcibly evicting them from the cache.
-				children.forEach( child => {
-
-					tiles.processNodeQueue.remove( child );
-					tiles.lruCache.remove( child );
-					child.parent = null;
-
-				} );
-
-				parent.children.length = 0;
+				// because we are forcibly evicting them from the cache. Since parents is sorted
+				// deepest-first, nested virtual tiles are already cleaned up before we reach
+				// their parent here.
+				this._removeVirtualChildren( tile );
 
 			}
 
@@ -462,18 +483,6 @@ export class ImageOverlayPlugin {
 
 		}
 
-		function collectChildren( root, target = [] ) {
-
-			root.children.forEach( child => {
-
-				target.push( child );
-				collectChildren( child, target );
-
-			} );
-			return target;
-
-		}
-
 	}
 
 	_getSplitVectors( scene, tile, centerTarget = _center ) {
@@ -567,7 +576,16 @@ export class ImageOverlayPlugin {
 
 	async expandVirtualChildren( scene, tile ) {
 
-		if ( tile.children.length !== 0 || this.enableTileSplitting === false ) {
+		const { refine } = tile;
+
+		// Only split tiles that would benefit from it:
+		// - REPLACE tiles with no children are leaf tiles where splitting improves overlay UV projection quality. REPLACE tiles
+		// that already have children are already refined by their children so splitting is unnecessary.
+		// - ADD tiles always need splitting since their content is rendered alongside children at all levels.
+		// Also skip any tiles that already have virtual children to avoid interfering with other plugins.
+		const shouldSplit = ( refine === 'REPLACE' && tile.children.length === 0 ) || refine === 'ADD';
+		const alreadySplit = tile.internal.virtualChildCount !== 0;
+		if ( this.enableTileSplitting === false || ! shouldSplit || alreadySplit ) {
 
 			return;
 
@@ -577,17 +595,16 @@ export class ImageOverlayPlugin {
 		const clone = scene.clone();
 		clone.updateMatrixWorld();
 
-		// get the directions to split on
+		// get the directions to split on & if there are no directions to split on then exit early
 		const { directions, hash } = this._getSplitVectors( clone, tile, _center );
-		tile[ SPLIT_HASH ] = hash;
-
-		// if there are no directions to split on then exit early
 		if ( directions.length === 0 ) {
 
 			return;
 
 		}
 
+		tile[ SPLIT_HASH ] = hash;
+
 		// set up the splitter to ignore overlay uvs
 		const clipper = new GeometryClipper();
 		clipper.attributeList = key => ! /^layer_uv_\d+/.test( key );
@@ -604,7 +621,7 @@ export class ImageOverlayPlugin {
 
 		// run the clipping operations by performing every permutation of sides
 		// defined by the split directions
-		const children = [];
+		const splitChildren = [];
 		clipper.forEachSplitPermutation( () => {
 
 			// clip the object itself
@@ -720,7 +737,8 @@ export class ImageOverlayPlugin {
 
 			}
 
-			children.push( {
+			splitChildren.push( {
+				internal: { isVirtual: true },
 				refine: 'REPLACE',
 				geometricError: tile.geometricError * 0.5,
 				boundingVolume: boundingVolume,
@@ -731,12 +749,13 @@ export class ImageOverlayPlugin {
 
 		} );
 
-		// force the tile "refine" mode to be set to "REPLACE" if we're splitting tiles
-		// TODO: If a tile is of type "ADD" refine and it has children then it will not be split
-		// as expected since only geometry tiles with no children are split. Instead we'd want
-		// to split this tiles geometry in addition to adding the child tiles.
+		// force the tile "refine" mode to be set to "REPLACE" so that the virtual children
+		// replace this tile's geometry display. Save the original mode so it can be restored
+		// if virtual children are later removed.
+		tile[ ORIGINAL_REFINE ] = tile.refine;
 		tile.refine = 'REPLACE';
-		tile.children.push( ...children );
+		tile.children.push( ...splitChildren );
+		tile.internal.virtualChildCount += splitChildren.length;
 
 	}