Move alphaGen portal support to "generic" shader

For alphaGen portal, a "portal" shader was used, which was like the
"generic" shader but with most features removed. Instead of that, move
alphaGen portal support to the "generic" shader so that it can be
combined with other features. This fixes the ancient-remains foliage
which uses alphaGen portal to fade out when the camera is close, but
also needs alpha testing which was not supported in the "portal" shader.
(Also it's vertex-lit so it should use overbright.)

I added back the capability for the "generic" shader to do vertex
lighting, but the implementation is simpler than before.

Author: slipher

src/engine/renderer/Material.cpp

@@ -142,7 +142,7 @@ void UpdateSurfaceDataNOP( uint32_t*, shaderStage_t*, bool, bool ) {
 }
 
 void UpdateSurfaceDataGeneric3D( uint32_t* materials, shaderStage_t* pStage, bool, bool ) {
-	// shader_t* shader = pStage->shader;
+	shader_t* shader = pStage->shader;
 
 	materials += pStage->bufferOffset;
 
@@ -154,11 +154,13 @@ void UpdateSurfaceDataGeneric3D( uint32_t* materials, shaderStage_t* pStage, boo
 	alphaGen_t alphaGen = SetAlphaGen( pStage );
 
 	const bool styleLightMap = pStage->type == stageType_t::ST_STYLELIGHTMAP || pStage->type == stageType_t::ST_STYLECOLORMAP;
-	gl_genericShaderMaterial->SetUniform_ColorModulateColorGen_Uint( rgbGen, alphaGen, styleLightMap );
+	gl_genericShaderMaterial->SetUniform_ColorModulateColorGen_Uint( rgbGen, alphaGen, styleLightMap || pStage->forceVertexLighting );
 
 	Tess_ComputeColor( pStage );
 	gl_genericShaderMaterial->SetUniform_Color_Uint( tess.svars.color );
 
+	gl_genericShaderMaterial->SetUniform_InversePortalRange( 1.0f / shader->portalRange );
+
 	gl_genericShaderMaterial->SetUniform_DepthScale( pStage->depthFadeValue );
 
 	gl_genericShaderMaterial->WriteUniformsToBuffer( materials, GLShader::MATERIAL );
@@ -879,6 +881,7 @@ void BindShaderGeneric3D( Material* material ) {
 	// Select shader permutation.
 	gl_genericShaderMaterial->SetTCGenEnvironment( material->tcGenEnvironment );
 	gl_genericShaderMaterial->SetTCGenLightmap( material->tcGen_Lightmap );
+	gl_genericShaderMaterial->SetAlphaGenPortal( material->alphaGenPortal );
 	gl_genericShaderMaterial->SetDepthFade( material->hasDepthFade );
 	gl_genericShaderMaterial->SetDeform( material->deformIndex );
 
@@ -892,6 +895,14 @@ void BindShaderGeneric3D( Material* material ) {
 
 	gl_genericShaderMaterial->SetUniform_ModelMatrix( backEnd.orientation.transformMatrix );
 	gl_genericShaderMaterial->SetUniform_ModelViewProjectionMatrix( glState.modelViewProjectionMatrix[glState.stackIndex] );
+	{
+		matrix_t unprojectMatrix;
+		MatrixCopy( backEnd.viewParms.projectionMatrix, unprojectMatrix );
+		MatrixInverse( unprojectMatrix );
+		MatrixMultiplyTranslation( unprojectMatrix, -1.0f, -1.0f, -1.0f );
+		MatrixMultiplyScale( unprojectMatrix, 2.0f / windowConfig.vidWidth, 2.0f / windowConfig.vidHeight, 2.0f );
+		gl_genericShaderMaterial->SetUniform_UnprojectMatrix( unprojectMatrix );
+	}
 
 	gl_genericShaderMaterial->SetUniform_DepthMapBindless( GL_BindToTMU( 1, tr.depthSamplerImage ) );
 
@@ -1097,6 +1108,10 @@ void ProcessMaterialGeneric3D( Material* material, shaderStage_t* pStage, Materi
 	gl_genericShaderMaterial->SetTCGenEnvironment( pStage->tcGen_Environment );
 	gl_genericShaderMaterial->SetTCGenLightmap( pStage->tcGen_Lightmap );
 
+	bool alphaGenPortal = pStage->alphaGen == alphaGen_t::AGEN_PORTAL;
+	material->alphaGenPortal = alphaGenPortal;
+	gl_genericShaderMaterial->SetAlphaGenPortal( alphaGenPortal );
+
 	bool hasDepthFade = pStage->hasDepthFade;
 	material->hasDepthFade = hasDepthFade;
 	gl_genericShaderMaterial->SetDepthFade( hasDepthFade );
@@ -1299,7 +1314,8 @@ void MaterialSystem::AddStage( MaterialSurface* surface, shaderStage_t* pStage,
 		}
 
 		if ( pStage->shader->reliefDepthScale != pStage2->shader->reliefDepthScale
-		     || pStage->shader->reliefOffsetBias != pStage2->shader->reliefOffsetBias )
+		     || pStage->shader->reliefOffsetBias != pStage2->shader->reliefOffsetBias
+		     || pStage->shader->portalRange != pStage2->shader->portalRange )
 		{
 			continue;
 		}

src/engine/renderer/Material.h

@@ -129,6 +129,7 @@ struct Material {
 	int deformIndex;
 	bool tcGenEnvironment;
 	bool tcGen_Lightmap;
+	bool alphaGenPortal;
 	bool hasDepthFade;
 
 	bool bspSurface;

src/engine/renderer/gl_shader.cpp

@@ -2446,12 +2446,14 @@ GLShader_generic::GLShader_generic() :
 	u_AlphaThreshold( this ),
 	u_ModelMatrix( this ),
 	u_ModelViewProjectionMatrix( this ),
+	u_UnprojectMatrix( this ),
 	u_ColorModulateColorGen_Float( this ),
 	u_ColorModulateColorGen_Uint( this ),
 	u_Color_Float( this ),
 	u_Color_Uint( this ),
 	u_Bones( this ),
 	u_VertexInterpolation( this ),
+	u_InversePortalRange( this ),
 	u_DepthScale( this ),
 	u_ProfilerZero( this ),
 	u_ProfilerRenderSubGroups( this ),
@@ -2460,6 +2462,7 @@ GLShader_generic::GLShader_generic() :
 	GLCompileMacro_USE_VERTEX_ANIMATION( this ),
 	GLCompileMacro_USE_TCGEN_ENVIRONMENT( this ),
 	GLCompileMacro_USE_TCGEN_LIGHTMAP( this ),
+	GLCompileMacro_USE_ALPHAGEN_PORTAL( this ),
 	GLCompileMacro_USE_DEPTH_FADE( this )
 {
 }
@@ -2480,8 +2483,10 @@ GLShader_genericMaterial::GLShader_genericMaterial() :
 	u_AlphaThreshold( this ),
 	u_ModelMatrix( this ),
 	u_ModelViewProjectionMatrix( this ),
+	u_UnprojectMatrix( this ),
 	u_ColorModulateColorGen_Uint( this ),
 	u_Color_Uint( this ),
+	u_InversePortalRange( this ),
 	u_DepthScale( this ),
 	u_ShowTris( this ),
 	u_MaterialColour( this ),
@@ -2490,6 +2495,7 @@ GLShader_genericMaterial::GLShader_genericMaterial() :
 	GLDeformStage( this ),
 	GLCompileMacro_USE_TCGEN_ENVIRONMENT( this ),
 	GLCompileMacro_USE_TCGEN_LIGHTMAP( this ),
+	GLCompileMacro_USE_ALPHAGEN_PORTAL( this ),
 	GLCompileMacro_USE_DEPTH_FADE( this ) {
 }
 

src/engine/renderer/gl_shader.h

@@ -939,6 +939,7 @@ class GLCompileMacro
 	  USE_REFLECTIVE_SPECULAR,
 	  LIGHT_DIRECTIONAL,
 	  USE_DEPTH_FADE,
+	  USE_ALPHAGEN_PORTAL,
 	  USE_PHYSICAL_MAPPING,
 	};
 
@@ -1425,6 +1426,35 @@ class GLCompileMacro_USE_DEPTH_FADE :
 	}
 };
 
+class GLCompileMacro_USE_ALPHAGEN_PORTAL :
+	GLCompileMacro
+{
+public:
+	GLCompileMacro_USE_ALPHAGEN_PORTAL( GLShader *shader ) :
+		GLCompileMacro( shader )
+	{
+	}
+
+	const char *GetName() const override
+	{
+		return "USE_ALPHAGEN_PORTAL";
+	}
+
+	EGLCompileMacro GetType() const override
+	{
+		return EGLCompileMacro::USE_ALPHAGEN_PORTAL;
+	}
+
+	int GetShaderTypes() const override {
+		return ShaderType::FRAGMENT;
+	}
+
+	void SetAlphaGenPortal( bool enable )
+	{
+		SetMacro( enable );
+	}
+};
+
 class GLCompileMacro_USE_PHYSICAL_MAPPING :
 	GLCompileMacro
 {
@@ -2401,7 +2431,7 @@ class u_InversePortalRange :
 {
 public:
 	u_InversePortalRange( GLShader *shader ) :
-		GLUniform1f( shader, "u_InversePortalRange", PUSH )
+		GLUniform1f( shader, "u_InversePortalRange", MATERIAL_OR_PUSH )
 	{
 	}
 
@@ -2939,12 +2969,14 @@ class GLShader_generic :
 	public u_AlphaThreshold,
 	public u_ModelMatrix,
 	public u_ModelViewProjectionMatrix,
+	public u_UnprojectMatrix,
 	public u_ColorModulateColorGen_Float,
 	public u_ColorModulateColorGen_Uint,
 	public u_Color_Float,
 	public u_Color_Uint,
 	public u_Bones,
 	public u_VertexInterpolation,
+	public u_InversePortalRange,
 	public u_DepthScale,
 	public u_ProfilerZero,
 	public u_ProfilerRenderSubGroups,
@@ -2953,6 +2985,7 @@ class GLShader_generic :
 	public GLCompileMacro_USE_VERTEX_ANIMATION,
 	public GLCompileMacro_USE_TCGEN_ENVIRONMENT,
 	public GLCompileMacro_USE_TCGEN_LIGHTMAP,
+	public GLCompileMacro_USE_ALPHAGEN_PORTAL,
 	public GLCompileMacro_USE_DEPTH_FADE
 {
 public:
@@ -2969,8 +3002,10 @@ class GLShader_genericMaterial :
 	public u_AlphaThreshold,
 	public u_ModelMatrix,
 	public u_ModelViewProjectionMatrix,
+	public u_UnprojectMatrix,
 	public u_ColorModulateColorGen_Uint,
 	public u_Color_Uint,
+	public u_InversePortalRange,
 	public u_DepthScale,
 	public u_ShowTris,
 	public u_MaterialColour,
@@ -2979,6 +3014,7 @@ class GLShader_genericMaterial :
 	public GLDeformStage,
 	public GLCompileMacro_USE_TCGEN_ENVIRONMENT,
 	public GLCompileMacro_USE_TCGEN_LIGHTMAP,
+	public GLCompileMacro_USE_ALPHAGEN_PORTAL,
 	public GLCompileMacro_USE_DEPTH_FADE {
 	public:
 	GLShader_genericMaterial();

src/engine/renderer/glsl_source/common.glsl

@@ -135,7 +135,6 @@ void ColorModulateColor_lightFactor(
 
 	vec4 unpackedColor = UnpackColor( packedColor );
 
-	unpackedColor.rgb *= lightFactor;
-
 	ModulateColor( colorModulation, unpackedColor, color );
+	color.rgb *= lightFactor;
 }

src/engine/renderer/glsl_source/generic_fp.glsl

@@ -42,6 +42,11 @@ IN(smooth) float        var_FadeDepth;
 uniform sampler2D       u_DepthMap;
 #endif
 
+#if defined(USE_ALPHAGEN_PORTAL)
+uniform float u_InversePortalRange;
+uniform mat4 u_UnprojectMatrix;
+#endif
+
 #insert shaderProfiler_fp
 
 DECLARE_OUTPUT(vec4)
@@ -67,6 +72,16 @@ void	main()
 		return;
 	}
 
+	// Normally alpha modulation is supposed to be done before the alpha test, but this looks
+	// better for the ancient-remains foliage combining alphagen portal with an alpha-tested
+	// plant image: fades out to 0 smoothly instead of cutting off at 0.5.
+#if defined(USE_ALPHAGEN_PORTAL)
+	vec4 position = u_UnprojectMatrix * vec4(gl_FragCoord.xyz, 1.0);
+	float portalAlpha = length(position.xyz / position.w) * u_InversePortalRange;
+portalAlpha = length(position.xyz / position.w) * (1.0/256.0);
+	color.a *= clamp(portalAlpha, 0.0, 1.0);
+#endif
+
 #if defined(USE_DEPTH_FADE)
 	float depth = texture2D(u_DepthMap, gl_FragCoord.xy / r_FBufSize).x;
 

src/engine/renderer/tr_shade.cpp

@@ -814,6 +814,7 @@ void ProcessShaderGeneric3D( const shaderStage_t* pStage ) {
 	gl_genericShader->SetDeform( pStage->deformIndex );
 	gl_genericShader->SetTCGenEnvironment( pStage->tcGen_Environment );
 	gl_genericShader->SetTCGenLightmap( pStage->tcGen_Lightmap );
+	gl_genericShader->SetAlphaGenPortal( pStage->alphaGen == alphaGen_t::AGEN_PORTAL );
 	gl_genericShader->SetDepthFade( pStage->hasDepthFade );
 }
 
@@ -926,7 +927,7 @@ void Render_generic3D( shaderStage_t *pStage )
 	alphaGen_t alphaGen = SetAlphaGen( pStage );
 
 	const bool styleLightMap = pStage->type == stageType_t::ST_STYLELIGHTMAP || pStage->type == stageType_t::ST_STYLECOLORMAP;
-	SetUniform_ColorModulateColorGen( gl_genericShader, rgbGen, alphaGen, styleLightMap );
+	SetUniform_ColorModulateColorGen( gl_genericShader, rgbGen, alphaGen, styleLightMap || pStage->forceVertexLighting );
 
 	// u_Color
 	SetUniform_Color( gl_genericShader, tess.svars.color );
@@ -963,6 +964,18 @@ void Render_generic3D( shaderStage_t *pStage )
 
 	gl_genericShader->SetUniform_TextureMatrix( tess.svars.texMatrices[ TB_COLORMAP ] );
 
+	if ( pStage->alphaGen == alphaGen_t::AGEN_PORTAL )
+	{
+		matrix_t unprojectMatrix;
+		MatrixCopy( backEnd.viewParms.projectionMatrix, unprojectMatrix );
+		MatrixInverse( unprojectMatrix );
+		MatrixMultiplyTranslation( unprojectMatrix, -1.0f, -1.0f, -1.0f );
+		MatrixMultiplyScale( unprojectMatrix, 2.0f / windowConfig.vidWidth, 2.0f / windowConfig.vidHeight, 2.0f );
+
+		gl_genericShader->SetUniform_InversePortalRange( 1.0f / tess.surfaceShader->portalRange );
+		gl_genericShader->SetUniform_UnprojectMatrix( unprojectMatrix );
+	}
+
 	if ( hasDepthFade )
 	{
 		gl_genericShader->SetUniform_DepthScale( pStage->depthFadeValue );

src/engine/renderer/tr_shader.cpp

@@ -2928,7 +2928,6 @@ static bool ParseStage( shaderStage_t *stage, const char **text )
 			}
 			else if ( !Q_stricmp( token, "portal" ) )
 			{
-				stage->type = stageType_t::ST_PORTALMAP;
 				stage->alphaGen = alphaGen_t::AGEN_PORTAL;
 
 				token = COM_ParseExt2( text, false );
@@ -5431,9 +5430,15 @@ static void FinishStages()
 			case stageType_t::ST_COLORMAP:
 				if ( stage->rgbGen == colorGen_t::CGEN_VERTEX && shader.registerFlags & RSF_BSP )
 				{
-					// vertex colors used as lighting detected. Enable overbright and realtime lights
-					stage->type = stageType_t::ST_DIFFUSEMAP;
+					// Vertex colors used as lighting detected. Enable overbright
+
 					stage->forceVertexLighting = true; // use vertex lighting even if there is a lightmap
+
+					if ( stage->alphaGen != alphaGen_t::AGEN_PORTAL ) // AGEN_PORTAL not supported in lightMapping
+					{
+						// use lightMapping shader to get realtime lights
+						stage->type = stageType_t::ST_DIFFUSEMAP;
+					}
 				}
 
 			default: