Use depth-based fog shader for all fogs

Add a new macro variant of the 'fogGlobal' shader which can be used for
fogs that the viewer is outside of. Now when material system is not
active, this shader is used for all fogs. The "outside" version of the
shader is more complex than the inside one, since it is necessary to
trace against all planes of the fog to find the length in fog.

This change removes the requirement that a fog be enclosed by opaque
walls on all sides except one (the "surface plane"). Now fogs can be
rendered (almost) correctly from any side. The exception is that if eye
position is very close but not inside the fog, within about 5 qu, the
inside fog shader has to be used instead of the outside one to avoid
near plane culling. This causes an artifact when passing into the fog
through a non-surface side which can be noticed if you look closely.

Author: slipher

src/engine/renderer/VertexSpecification.h

@@ -44,6 +44,10 @@ enum
 
 	ATTR_INDEX_FOG_SURFACE,
 
+	// This occupies 5 attribute slots
+	ATTR_INDEX_FOG_PLANES_0,
+	ATTR_INDEX_FOG_PLANES_LAST = ATTR_INDEX_FOG_PLANES_0 + 4,
+
 	ATTR_INDEX_MAX
 };
 
@@ -58,6 +62,7 @@ static const char* const attributeNames[] =
 	"attr_Position2",
 	"attr_QTangent2",
 	"attr_FogSurface",
+	"attr_FogPlanes", nullptr, nullptr, nullptr, nullptr,
 };
 
 enum
@@ -74,6 +79,7 @@ enum
   ATTR_QTANGENT2      = BIT( ATTR_INDEX_QTANGENT2 ),
 
   ATTR_FOG_SURFACE    = BIT( ATTR_INDEX_FOG_SURFACE ),
+  ATTR_FOG_PLANES     = BIT( ATTR_INDEX_FOG_PLANES_0 ) * ( BIT( 5 ) - 1 ),
 
   ATTR_INTERP_BITS = ATTR_POSITION2 | ATTR_QTANGENT2,
 };

src/engine/renderer/gl_shader.cpp

@@ -2007,7 +2007,10 @@ void GLShaderManager::BindAttribLocations( GLuint program ) const
 {
 	for ( uint32_t i = 0; i < ATTR_INDEX_MAX; i++ )
 	{
-		glBindAttribLocation( program, i, attributeNames[ i ] );
+		if ( attributeNames[ i ] != nullptr )
+		{
+			glBindAttribLocation( program, i, attributeNames[ i ] );
+		}
 	}
 }
 
@@ -2729,7 +2732,8 @@ GLShader_fogGlobal::GLShader_fogGlobal() :
 	u_Color_Float( this ),
 	u_Color_Uint( this ),
 	u_ViewOrigin( this ),
-	u_FogGradient( this )
+	u_FogGradient( this ),
+	GLCompileMacro_OUTSIDE_FOG( this )
 {
 }
 

src/engine/renderer/gl_shader.h

@@ -919,6 +919,7 @@ class GLCompileMacro
 // It also works regardless of RTTI is enabled or not.
 	enum EGLCompileMacro : unsigned
 	{
+	  OUTSIDE_FOG,
 	  USE_BSP_SURFACE,
 	  USE_VERTEX_SKINNING,
 	  USE_VERTEX_ANIMATION,
@@ -985,6 +986,40 @@ class GLCompileMacro
 	virtual ~GLCompileMacro() = default;
 };
 
+class GLCompileMacro_OUTSIDE_FOG :
+	GLCompileMacro
+{
+public:
+	GLCompileMacro_OUTSIDE_FOG( GLShader *shader ) :
+		GLCompileMacro( shader )
+	{
+	}
+
+	const char *GetName() const override
+	{
+		return "OUTSIDE_FOG";
+	}
+
+	EGLCompileMacro GetType() const override
+	{
+		return EGLCompileMacro::OUTSIDE_FOG;
+	}
+
+	int GetShaderTypes() const override {
+		return ShaderType::VERTEX | ShaderType::FRAGMENT;
+	}
+
+	void SetOutsideFog( bool enable )
+	{
+		SetMacro( enable );
+	}
+
+	uint32_t GetRequiredVertexAttributes() const override
+	{
+		return ATTR_FOG_PLANES;
+	}
+};
+
 class GLCompileMacro_USE_BSP_SURFACE :
 	GLCompileMacro
 {
@@ -3215,7 +3250,8 @@ class GLShader_fogGlobal :
 	public u_Color_Float,
 	public u_Color_Uint,
 	public u_ViewOrigin,
-	public u_FogGradient
+	public u_FogGradient,
+	public GLCompileMacro_OUTSIDE_FOG
 {
 public:
 	GLShader_fogGlobal();

src/engine/renderer/glsl_source/fogGlobal_fp.glsl

@@ -30,6 +30,11 @@ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 IN(smooth) vec3 var_Position;
 IN(flat) vec4 var_FogSurface;
 
+#ifdef OUTSIDE_FOG
+#define NUM_PLANES 5
+IN(flat) vec4 var_FogPlanes[NUM_PLANES];
+#endif
+
 uniform sampler2D	u_DepthMap;
 
 uniform colorPack u_Color;
@@ -40,7 +45,24 @@ uniform mat4		u_UnprojectMatrix;
 
 DECLARE_OUTPUT(vec4)
 
-// This shader can be used to draw a fog volume the viewer is inside of.
+#ifdef OUTSIDE_FOG
+// Trace against the inner sides of the fog brush
+float FogDistance(vec3 start, vec3 dir)
+{
+	vec4 start4 = vec4(-start, 1.0);
+	float minDist = 1.0e20;
+	for (int i = 0; i < NUM_PLANES; i++)
+	{
+		float dist = dot(start4, var_FogPlanes[i]) / dot(dir, var_FogPlanes[i].xyz) ;
+		if (dist >= 0.0)
+		{
+			minDist = min(minDist, dist);
+		}
+	}
+	return minDist < 1.0e20 ? minDist : 0.0;
+}
+#endif
+
 void	main()
 {
 	#insert material_fp
@@ -53,9 +75,17 @@ void	main()
 	vec4 P = u_UnprojectMatrix * vec4(gl_FragCoord.xy, depth, 1.0);
 	P.xyz /= P.w;
 
-	// calculate the length in fog
-	float depthDist = distance(u_ViewOrigin, P.xyz);
-	float fogBoundaryDist = distance(u_ViewOrigin, var_Position);
+	#ifdef OUTSIDE_FOG
+		vec3 startPoint = var_Position;
+		vec3 viewDir = normalize(var_Position - u_ViewOrigin);
+		float fogBoundaryDist = FogDistance(var_Position, viewDir);
+	#else
+		vec3 startPoint = u_ViewOrigin;
+		float fogBoundaryDist = distance(u_ViewOrigin, var_Position);
+	#endif
+
+	float depthDist = distance(startPoint, P.xyz);
+
 	vec3 endPoint;
 	float distInFog;
 	if ( depthDist < fogBoundaryDist )
@@ -65,11 +95,16 @@ void	main()
 	}
 	else
 	{
-		endPoint = var_Position;
+		#ifdef OUTSIDE_FOG
+			endPoint = var_Position + fogBoundaryDist * viewDir;
+		#else
+			endPoint = var_Position;
+		#endif
+
 		distInFog = fogBoundaryDist;
 	}
 
-	float t0 = dot(var_FogSurface.xyz, u_ViewOrigin) + var_FogSurface.w;
+	float t0 = dot(var_FogSurface.xyz, startPoint) + var_FogSurface.w;
 	float t1 = dot(var_FogSurface.xyz, endPoint) + var_FogSurface.w;
 
 	float s = distInFog * GetFogGradientModifier(u_FogGradient.y, t0, t1) * u_FogGradient.x;

src/engine/renderer/glsl_source/fogGlobal_vp.glsl

@@ -40,10 +40,23 @@ IN vec4 attr_FogSurface;
 OUT(smooth) vec3 var_Position;
 OUT(flat) vec4 var_FogSurface;
 
+#ifdef OUTSIDE_FOG
+#define NUM_PLANES 5
+IN vec4 attr_FogPlanes[NUM_PLANES];
+OUT(flat) vec4 var_FogPlanes[NUM_PLANES];
+#endif
+
 void main()
 {
 	vec4 position = vec4(attr_Position, 1.0);
 	gl_Position = u_ModelViewProjectionMatrix * position;
 	var_Position = attr_Position;
 	var_FogSurface = attr_FogSurface;
+
+	#ifdef OUTSIDE_FOG
+		for (int i = 0; i < NUM_PLANES; i++)
+		{
+			var_FogPlanes[i] = attr_FogPlanes[i];
+		}
+	#endif
 }

src/engine/renderer/tr_backend.cpp

@@ -1441,7 +1441,9 @@ void Render_fogGlobal( shaderStage_t *stage )
 
 	RB_PrepareForSamplingDepthMap();
 
-	GL_Cull( cullType_t::CT_FRONT_SIDED );
+	GL_Cull( stage->shader->cullType );
+
+	gl_fogGlobalShader->SetOutsideFog( stage->type == stageType_t::ST_FOGMAP_OUTER );
 
 	gl_fogGlobalShader->BindProgram();
 
@@ -1454,12 +1456,26 @@ void Render_fogGlobal( shaderStage_t *stage )
 		SetUniform_Color( gl_fogGlobalShader, stage->shader->fogParms.color );
 	}
 
-	// It's important to avoid far plane clipping
-	matrix_t projection, mvp;
-	MatrixPerspectiveProjectionFovXYInfiniteRH( projection, tr.refdef.fov_x, tr.refdef.fov_y, 1.0f );
-	MatrixMultiply( projection, glState.modelViewMatrix[ glState.stackIndex ], mvp );
+	switch ( stage->type )
+	{
+	case stageType_t::ST_FOGMAP_INNER:
+	{
+		// It's important to avoid far plane clipping
+		matrix_t projection, mvp;
+		MatrixPerspectiveProjectionFovXYInfiniteRH( projection, tr.refdef.fov_x, tr.refdef.fov_y, 1.0f );
+		MatrixMultiply( projection, glState.modelViewMatrix[ glState.stackIndex ], mvp );
+		gl_fogGlobalShader->SetUniform_ModelViewProjectionMatrix( mvp );
+		break;
+	}
+	case stageType_t::ST_FOGMAP_OUTER:
+	{
+		gl_fogGlobalShader->SetUniform_ModelViewProjectionMatrix( glState.modelViewProjectionMatrix[ glState.stackIndex ] );
+		break;
+	}
+	default:
+		ASSERT_UNREACHABLE();
+	}
 
-	gl_fogGlobalShader->SetUniform_ModelViewProjectionMatrix( mvp );
 	gl_fogGlobalShader->SetUniform_UnprojectMatrix( backEnd.viewParms.unprojectionMatrix );
 
 	// bind u_DepthMap

src/engine/renderer/tr_bsp.cpp

@@ -3264,10 +3264,11 @@ static void R_LoadFogs( lump_t *l, lump_t *brushesLump, lump_t *sidesLump )
 
 	struct FogVert
 	{
-		float position[ 3 ];
+		vec3_t position;
 		vec4_t surface;
+		vec4_t planes[ 5 ]; // faces other than the current triangle's
 	};
-	std::vector<FogVert> fogVerts(8 * count);
+	std::vector<FogVert> fogVerts(24 * count);
 	std::vector<glIndex_t> fogIndexes(36 * count);
 
 	for ( i = 0; i < count; i++, fogs++ )
@@ -3349,22 +3350,50 @@ static void R_LoadFogs( lump_t *l, lump_t *brushesLump, lump_t *sidesLump )
 			out->surface[ 3 ] = -s_worldData.planes[ planeNum ].dist;
 		}
 
-		// add faces of fog brush for drawing fog from inside
-		for ( int p = 0; p < 8; p++ )
+		// add faces of fog brush with information about the other planes
+		// TODO: allow non-axis-aligned boxes. The GLSL can draw any brush with up to 6 faces.
+		constexpr int faces[ 6 ][ 4 ]
 		{
-			fogVerts[ i * 8 + p ].position[ 0 ] = out->bounds[ p & 1 ][ 0 ];
-			fogVerts[ i * 8 + p ].position[ 1 ] = out->bounds[ ( p >> 1 ) & 1 ][ 1 ];
-			fogVerts[ i * 8 + p ].position[ 2 ] = out->bounds[ p >> 2 ][ 2 ];
+			{ 2, 0, 6, 4 },
+			{ 1, 3, 5, 7 },
+			{ 0, 1, 4, 5 },
+			{ 3, 2, 7, 6 },
+			{ 2, 3, 0, 1 },
+			{ 7, 6, 5, 4 }
+		};
+
+		for ( int face = 0; face < 6; face++ )
+		{
+			for ( int v = 0; v < 4; v++ )
+			{
+				FogVert &vert = fogVerts[ i * 24 + face * 4 + v ];
+				int p = faces[ face ][ v ];
+				vert.position[ 0 ] = out->bounds[ p & 1 ][ 0 ];
+				vert.position[ 1 ] = out->bounds[ ( p >> 1 ) & 1 ][ 1 ];
+				vert.position[ 2 ] = out->bounds[ p >> 2 ][ 2 ];
 
-			VectorCopy( out->surface, fogVerts[ i * 8 + p ].surface );
-			fogVerts[ i * 8 + p ].surface[ 3 ] = -out->surface[ 3 ];
-		}
+				VectorCopy( out->surface, vert.surface );
+				vert.surface[ 3 ] = -out->surface[ 3 ];
 
-		constexpr int box[ 36 ] = { 2, 3, 0, 0, 3, 1, 0, 1, 4, 4, 1, 5, 2, 0, 6, 6, 0, 4,
-		                            1, 3, 5, 5, 3, 7, 3, 2, 7, 7, 2, 6, 7, 6, 5, 5, 6, 4 };
-		for ( int p = 0; p < 36; p++ )
-		{
-			fogIndexes[ 36 * i + p ] = 8 * i + box[ p ];
+				vec4_t *plane = vert.planes;
+				for ( int otherFace = 0; otherFace < 6; otherFace++ )
+				{
+					if ( face != otherFace )
+					{
+						planeNum = LittleLong( sides[ firstSide + otherFace ].planeNum );
+						VectorCopy( s_worldData.planes[ planeNum ].normal, *plane );
+						( *plane )[ 3 ] = s_worldData.planes[ planeNum ].dist;
+						++plane;
+					}
+				}
+			}
+
+			fogIndexes[ 36 * i + 6 * face + 0 ] = i * 24 + face * 4 + 0;
+			fogIndexes[ 36 * i + 6 * face + 1 ] = i * 24 + face * 4 + 1;
+			fogIndexes[ 36 * i + 6 * face + 2 ] = i * 24 + face * 4 + 2;
+			fogIndexes[ 36 * i + 6 * face + 3 ] = i * 24 + face * 4 + 2;
+			fogIndexes[ 36 * i + 6 * face + 4 ] = i * 24 + face * 4 + 1;
+			fogIndexes[ 36 * i + 6 * face + 5 ] = i * 24 + face * 4 + 3;
 		}
 
 		// add draw surf for fog brush faces
@@ -3378,11 +3407,16 @@ static void R_LoadFogs( lump_t *l, lump_t *brushesLump, lump_t *sidesLump )
 	vertexAttributeSpec_t attributes[] {
 		{ ATTR_INDEX_POSITION, GL_FLOAT, GL_FLOAT, &fogVerts[ 0 ].position, 3, sizeof( fogVerts[ 0 ] ), 0 },
 		{ ATTR_INDEX_FOG_SURFACE, GL_FLOAT, GL_FLOAT, &fogVerts[ 0 ].surface, 4, sizeof( fogVerts[ 0 ] ), 0 },
+		{ ATTR_INDEX_FOG_PLANES_0 + 0, GL_FLOAT, GL_FLOAT, &fogVerts[ 0 ].planes[ 0 ], 4, sizeof( FogVert ), 0 },
+		{ ATTR_INDEX_FOG_PLANES_0 + 1, GL_FLOAT, GL_FLOAT, &fogVerts[ 0 ].planes[ 1 ], 4, sizeof( FogVert ), 0 },
+		{ ATTR_INDEX_FOG_PLANES_0 + 2, GL_FLOAT, GL_FLOAT, &fogVerts[ 0 ].planes[ 2 ], 4, sizeof( FogVert ), 0 },
+		{ ATTR_INDEX_FOG_PLANES_0 + 3, GL_FLOAT, GL_FLOAT, &fogVerts[ 0 ].planes[ 3 ], 4, sizeof( FogVert ), 0 },
+		{ ATTR_INDEX_FOG_PLANES_0 + 4, GL_FLOAT, GL_FLOAT, &fogVerts[ 0 ].planes[ 4 ], 4, sizeof( FogVert ), 0 },
 	};
 	VBO_t *fogVBO = R_CreateStaticVBO(
 		"fogs VBO", std::begin( attributes ), std::end( attributes ), fogVerts.size() );
 	IBO_t *fogIBO = R_CreateStaticIBO( "fogs IBO", &fogIndexes[ 0 ], fogIndexes.size() );
-	SetupVAOBuffers( fogVBO, fogIBO, ATTR_POSITION | ATTR_FOG_SURFACE, &fogVBO->VAO );
+	SetupVAOBuffers( fogVBO, fogIBO, ATTR_POSITION | ATTR_FOG_SURFACE | ATTR_FOG_PLANES, &fogVBO->VAO );
 
 	for ( int j = 1; j < s_worldData.numFogs; j++ )
 	{

src/engine/renderer/tr_init.cpp

@@ -1559,7 +1559,11 @@ ScreenshotCmd screenshotPNGRegistration("screenshotPNG", ssFormat_t::SSF_PNG, "p
 					gl_fogQuake3Shader->MarkProgramForBuilding();
 				}
 
-				gl_fogGlobalShader->MarkProgramForBuilding();
+				for ( bool outer : { false, true } )
+				{
+					gl_fogGlobalShader->SetOutsideFog( outer );
+					gl_fogGlobalShader->MarkProgramForBuilding();
+				}
 			}
 
 			for ( int i = 0; i < tr.numModels; i++ ) {

src/engine/renderer/tr_local.h

@@ -942,6 +942,7 @@ enum
 	  ST_LIQUIDMAP,
 	  ST_FOGMAP,
 	  ST_FOGMAP_INNER, // a fog seen from inside
+	  ST_FOGMAP_OUTER, // a fog seen from outside
 	  ST_LIGHTMAP,
 	  ST_STYLELIGHTMAP,
 	  ST_STYLECOLORMAP,
@@ -1206,6 +1207,7 @@ enum
 		struct shader_t *depthShader;
 		struct shader_t *fogShader;
 		struct shader_t *fogInnerShader;
+		struct shader_t *fogOuterShader;
 		struct shader_t *next;
 	};
 
@@ -3355,8 +3357,7 @@ void GLimp_LogComment_( std::string comment );
 	void RE_AddPolyToSceneET( qhandle_t hShader, int numVerts, const polyVert_t *verts );
 	void RE_AddPolysToScene( qhandle_t hShader, int numVerts, const polyVert_t *verts, int numPolys );
 
-	bool R_InsideFog( int fognum );
-	void R_AddInnerFogSurfaces();
+	void R_AddFogBrushSurfaces();
 
 	void RE_AddDynamicLightToSceneET( const vec3_t org, float radius, float intensity, float r, float g, float b, qhandle_t hShader, int flags );
 	void RE_AddDynamicLightToSceneQ3A( const vec3_t org, float intensity, float r, float g, float b );