NUKE surface sorting by fog

#1408 had removed the hacky way of drawing fog surfaces and made them into actual stages. This change removes the sorting by fog index since it now has no positive effect (and potentially has a negative one, preventing surfaces from being merged). The fognum bits are redistributed towards drawsurf indexes and entity count.

Author: VReaperV

src/engine/renderer/Material.cpp

@@ -2036,7 +2036,7 @@ void MaterialSystem::AddAutospriteSurfaces() {
 	for ( const drawSurf_t &drawSurf : autospriteSurfaces )
 	{
 		R_AddDrawSurf( drawSurf.surface, drawSurf.shader,
-		               drawSurf.lightmapNum(), drawSurf.fogNum(), drawSurf.bspSurface );
+		               drawSurf.lightmapNum(), drawSurf.fog, drawSurf.bspSurface );
 	}
 }
 

src/engine/renderer/ShadeCommon.h

@@ -87,16 +87,6 @@ template<typename Obj> static bool hasExplicitelyDisabledLightMap( Obj* obj )
 	return GetSurfaceShader( obj )->surfaceFlags & SURF_NOLIGHTMAP;
 }
 
-inline size_t GetFogNum( shaderCommands_t* tess )
-{
-	return tess->fogNum;
-}
-
-inline size_t GetFogNum( drawSurf_t* drawSurf )
-{
-	return drawSurf->fogNum();
-}
-
 inline shaderStage_t* GetSurfaceLastStage( shaderCommands_t* tess )
 {
 	return tess->surfaceLastStage;

src/engine/renderer/tr_backend.cpp

@@ -5446,7 +5446,7 @@ const RenderCommand *FinalisePortalCommand::ExecuteSelf( ) const
 	glStencilOp( GL_KEEP, GL_KEEP, GL_KEEP );
 
 	Tess_Begin( Tess_StageIteratorColor, shader,
-		nullptr, false, surface->lightmapNum(), surface->fogNum(), surface->bspSurface );
+		nullptr, false, surface->lightmapNum(), surface->fog, surface->bspSurface );
 	rb_surfaceTable[Util::ordinal( *( surface->surface ) )]( surface->surface );
 	Tess_End();
 

src/engine/renderer/tr_bsp.cpp

@@ -2789,15 +2789,6 @@ static int LeafSurfaceCompare( const void *a, const void *b )
 		return 1;
 	}
 
-	if ( aa->fogIndex < bb->fogIndex )
-	{
-		return -1;
-	}
-	else if ( aa->fogIndex > bb->fogIndex )
-	{
-		return 1;
-	}
-
 	// sort by leaf
 	if ( aa->interactionBits < bb->interactionBits )
 	{

src/engine/renderer/tr_local.h

@@ -1605,35 +1605,30 @@ enum class ssaoMode {
 	// 4. fogNum
 	// 5. index
 
-	static const uint64_t SORT_INDEX_BITS = 16;
-	static const uint64_t SORT_FOGNUM_BITS = 13;
-	static const uint64_t SORT_ENTITYNUM_BITS = 10;
+	static const uint64_t SORT_INDEX_BITS = 20;
+	static const uint64_t SORT_ENTITYNUM_BITS = 13;
 	static const uint64_t SORT_LIGHTMAP_BITS = 9;
 	static const uint64_t SORT_SHADER_BITS = 16;
+	static const uint64_t SORT_UNUSED_BITS = 6;
 
 	static_assert( SORT_SHADER_BITS +
 		SORT_LIGHTMAP_BITS +
 		SORT_ENTITYNUM_BITS +
-		SORT_FOGNUM_BITS +
-		SORT_INDEX_BITS == 64, "invalid number of drawSurface sort bits" );
+		SORT_INDEX_BITS +
+		SORT_UNUSED_BITS == 64, "invalid number of drawSurface sort bits" );
 
-	static const uint64_t SORT_FOGNUM_SHIFT = SORT_INDEX_BITS;
-	static const uint64_t SORT_ENTITYNUM_SHIFT = SORT_FOGNUM_BITS + SORT_FOGNUM_SHIFT;
+	static const uint64_t SORT_ENTITYNUM_SHIFT = SORT_INDEX_BITS;
 	static const uint64_t SORT_LIGHTMAP_SHIFT = SORT_ENTITYNUM_BITS + SORT_ENTITYNUM_SHIFT;
 	static const uint64_t SORT_SHADER_SHIFT = SORT_LIGHTMAP_BITS + SORT_LIGHTMAP_SHIFT;
 
 #define MASKBITS( b ) ( 1 << (b) ) - 1
 	static const uint32_t SORT_INDEX_MASK = MASKBITS( SORT_INDEX_BITS );
-	static const uint32_t SORT_FOGNUM_MASK = MASKBITS( SORT_FOGNUM_BITS );
 	static const uint32_t SORT_ENTITYNUM_MASK = MASKBITS( SORT_ENTITYNUM_BITS );
 	static const uint32_t SORT_LIGHTMAP_MASK = MASKBITS( SORT_LIGHTMAP_BITS );
 	static const uint32_t SORT_SHADER_MASK = MASKBITS( SORT_SHADER_BITS );
 
 	static_assert( SORT_INDEX_MASK >= MAX_DRAWSURFS - 1, "not enough index bits" );
 
-	// need space for 0 fog (no fog), in addition to MAX_MAP_FOGS
-	static_assert( SORT_FOGNUM_MASK >= MAX_MAP_FOGS, "not enough fognum bits" );
-
 	// need space for tr.worldEntity, in addition to MAX_REF_ENTITIES
 	static_assert( SORT_ENTITYNUM_MASK >= MAX_REF_ENTITIES, "not enough entity bits" );
 
@@ -1652,7 +1647,6 @@ enum class ssaoMode {
 		int fog;
 		int portalNum = -1;
 
-
 		uint32_t materialPackIDs[MAX_SHADER_STAGES];
 		uint32_t materialIDs[MAX_SHADER_STAGES];
 
@@ -1671,21 +1665,17 @@ enum class ssaoMode {
 		inline int entityNum() const {
 			return int( ( sort >> SORT_ENTITYNUM_SHIFT ) & SORT_ENTITYNUM_MASK ) - 1;
 		}
-		inline int fogNum() const {
-			return int( ( sort >> SORT_FOGNUM_SHIFT ) & SORT_FOGNUM_MASK );
-		}
 		inline int lightmapNum() const {
 			return int( ( sort >> SORT_LIGHTMAP_SHIFT ) & SORT_LIGHTMAP_MASK ) - 1;
 		}
 		inline int shaderNum() const {
 			return int( sort >> SORT_SHADER_SHIFT );
 		}
 
-		inline void setSort( int shaderNum, int lightmapNum, int entityNum, int fogNum, int index ) {
+		inline void setSort( int shaderNum, int lightmapNum, int entityNum, int index ) {
 			entityNum = entityNum + 1; //world entity is -1
 			lightmapNum = lightmapNum + 1; //no lightmap is -1
 			sort = uint64_t( index & SORT_INDEX_MASK ) |
-				( uint64_t( fogNum & SORT_FOGNUM_MASK ) << SORT_FOGNUM_SHIFT ) |
 				( uint64_t( entityNum & SORT_ENTITYNUM_MASK ) << SORT_ENTITYNUM_SHIFT ) |
 				( uint64_t( lightmapNum & SORT_LIGHTMAP_MASK ) << SORT_LIGHTMAP_SHIFT ) |
 				( uint64_t( shaderNum & SORT_SHADER_MASK ) << SORT_SHADER_SHIFT );

src/engine/renderer/tr_main.cpp

@@ -1882,7 +1882,7 @@ int R_AddDrawSurf( surfaceType_t *surface, shader_t *shader, int lightmapNum, in
 		index = MAX_DRAWSURFS - index; // reverse the sorting (front:back -> back:front)
 	}
 
-	drawSurf->setSort( shader->sortedIndex, lightmapNum, entityNum, fogNum, index );
+	drawSurf->setSort( shader->sortedIndex, lightmapNum, entityNum, index );
 
 	tr.refdef.numDrawSurfs++;