Shadow Overhaul (scp-fs2open#7529)

* Extract shadows into own shader and render pass

# Conflicts:
#	code/def_files/data/effects/main-g.sdr

* Consolidate render queue code

* More cleanup

* Fix missing clear of shadow buffer inter frame

* Request capability for viewport layer rendering

* instanced shadows

* Fix issues

* Switch to PCF+PCSS and proper textures for it

* Smooth PCSS toggle

* Fix for windows

* Fix OGL error

* Cleanup of old flag

* Expose shadow smoothness

* Fix missing uniform

* Make number of shadow cascades dynamic

* Progress on cockpit shadow integration

* Wrangle Cockpit and show-ship shadows into correct behaviour

* Cleanup

* make player ship shadow table options more intuitive

* Cleanup

* Only read from correct set of shadow maps, also stretch first main-scene map to 0

* Remove now non-necessary validation

* Make shadow cascade buffer more typesafe and less wasteful

* Don't allocate cockpit cascades if not used

* Cleanup

* Final fixes

* Fix Warnings

* clang tidy

* Fix missing fallback for shader precaching

* That weird FOV multiplier that always ruins your day

* Avoid shadow geometry artifacts by using hardware depth clamping

* Correctly handle shadow cascade overrides with hardcoded length

* GCC9 and its sad state of C++17 support.

* Address Feedback

# Conflicts:
#	code/graphics/shadows.cpp

Author: BMagnu

code/def_files/data/effects/deferred-f.sdr

@@ -10,30 +10,31 @@ uniform sampler2D ColorBuffer;
 uniform sampler2D NormalBuffer;
 uniform sampler2D PositionBuffer;
 uniform sampler2D SpecBuffer;
-uniform sampler2DArray shadow_map;
+uniform sampler2DArrayShadow shadow_map;
 
 #ifdef ENV_MAP
 uniform samplerCube sEnvmap;
 uniform samplerCube sIrrmap;
 #endif
 
 layout (std140) uniform globalDeferredData {
-	mat4 shadow_mv_matrix;
-	mat4 shadow_proj_matrix[4];
-
 	mat4 inv_view_matrix;
 
-	float veryneardist;
-	float neardist;
-	float middist;
-	float fardist;
-
 	float invScreenWidth;
 	float invScreenHeight;
-
 	float nearPlane;
 };
 
+layout (std140) uniform shadowCascadeParams {
+	int cascade_offset;
+	int cascade_count;
+
+	mat4 shadow_mv_matrix;
+	mat4 shadow_proj_matrix[NUM_SHADOW_CASCADES];
+	vec4 cascade_distances[(NUM_SHADOW_CASCADES + 4 - 1) / 4];
+	vec4 smoothness_factors[(NUM_SHADOW_CASCADES + 4 - 1) / 4];
+};
+
 layout (std140) uniform matrixData {
 	mat4 modelViewMatrix;
 	mat4 projMatrix;
@@ -259,14 +260,12 @@ void main()
 
 		if (enable_shadows) {
 			vec4 fragShadowPos = shadow_mv_matrix * inv_view_matrix * vec4(position, 1.0);
-			vec4 fragShadowUV[4];
-			fragShadowUV[0] = transformToShadowMap(shadow_proj_matrix[0], 0, fragShadowPos);
-			fragShadowUV[1] = transformToShadowMap(shadow_proj_matrix[1], 1, fragShadowPos);
-			fragShadowUV[2] = transformToShadowMap(shadow_proj_matrix[2], 2, fragShadowPos);
-			fragShadowUV[3] = transformToShadowMap(shadow_proj_matrix[3], 3, fragShadowPos);
-
-			attenuation *= getShadowValue(shadow_map, -position.z, fragShadowPos.z, fragShadowUV, fardist, middist,
-									neardist, veryneardist);
+			vec4 fragShadowUV[NUM_SHADOW_CASCADES];
+			for (int i = 0; i < NUM_SHADOW_CASCADES; i++) {
+				fragShadowUV[i] = transformToShadowMap(shadow_proj_matrix[i], i, fragShadowPos);
+			}
+
+			attenuation *= getShadowValue(shadow_map, -position.z, fragShadowUV, cascade_distances, smoothness_factors, cascade_offset, cascade_count);
 		}
 
 		vec3 halfVec = normalize(lightDir + eyeDir);

code/def_files/data/effects/main-f.sdr

@@ -31,8 +31,6 @@ layout (std140) uniform modelData {
 	mat4 viewMatrix;
 	mat4 projMatrix;
 	mat4 textureMatrix;
-	mat4 shadow_mv_matrix;
-	mat4 shadow_proj_matrix[4];
 
 	vec4 color;
 
@@ -50,7 +48,6 @@ layout (std140) uniform modelData {
 	int n_lights;
 	float defaultGloss;
 
-	//EXCLUSIVELY used for non-deferred rendering
 	vec3 ambientFactor;
 	int desaturate;
 
@@ -66,7 +63,6 @@ layout (std140) uniform modelData {
 	int effect_num;
 	int sBasemapIndex;
 
-
 	vec4 fogColor;
 
 	vec3 base_color;
@@ -80,11 +76,6 @@ layout (std140) uniform modelData {
 	float znear;
 	float zfar;
 
-	float veryneardist;
-	float neardist;
-	float middist;
-	float fardist;
-
 	int sGlowmapIndex;
 	int sSpecmapIndex;
 	int sNormalmapIndex;
@@ -95,6 +86,16 @@ layout (std140) uniform modelData {
 	int flags;
 };
 
+layout (std140) uniform shadowCascadeParams {
+	int cascade_offset;
+	int cascade_count;
+
+	mat4 shadow_mv_matrix;
+	mat4 shadow_proj_matrix[NUM_SHADOW_CASCADES];
+	vec4 cascade_distances[(NUM_SHADOW_CASCADES + 4 - 1) / 4];
+	vec4 smoothness_factors[(NUM_SHADOW_CASCADES + 4 - 1) / 4];
+};
+
 in VertexOutput {
 	mat3 tangentMatrix;
 
@@ -107,7 +108,7 @@ in VertexOutput {
 	vec4 texCoord;
 
 #prereplace IF_FLAG_COMPILED MODEL_SDR_FLAG_SHADOWS
-	vec4 shadowUV[4];
+	vec4 shadowUV[NUM_SHADOW_CASCADES];
 	vec4 shadowPos;
 #prereplace ENDIF_FLAG_COMPILED MODEL_SDR_FLAG_SHADOWS
 } vertIn;
@@ -132,17 +133,15 @@ uniform sampler2D sFramebuffer;
 uniform sampler2DArray sMiscmap;
 #prereplace ENDIF_FLAG_COMPILED MODEL_SDR_FLAG_MISC
 #prereplace IF_FLAG_COMPILED MODEL_SDR_FLAG_SHADOWS
-uniform sampler2DArray shadow_map;
+uniform sampler2DArrayShadow shadow_map;
 #prereplace ENDIF_FLAG_COMPILED MODEL_SDR_FLAG_SHADOWS
 
 out vec4 fragOut0;
 
-#ifndef MODEL_SDR_FLAG_SHADOW_MAP
 out vec4 fragOut1;
 out vec4 fragOut2;
 out vec4 fragOut3;
 out vec4 fragOut4;
-#endif
 
 vec3 FresnelLazarovEnv(vec3 specColor, vec3 view, vec3 normal, float gloss)
 {
@@ -203,12 +202,6 @@ vec3 CalculateLighting(vec3 normal, vec3 diffuseMaterial, vec3 specularMaterial,
 
 void main()
 {
-	#ifdef MODEL_SDR_FLAG_SHADOW_MAP
-		// need depth and depth squared for variance shadow maps
-		fragOut0 = vec4(vertIn.position.z, vertIn.position.z * vertIn.position.z * VARIANCE_SHADOW_SCALE_INV, 0.0, 1.0);
-
-		return;
-	#else
 	vec3 eyeDir = vec3(normalize(-vertIn.position).xyz);
 	vec2 texCoord = vertIn.texCoord.xy;
 
@@ -337,7 +330,7 @@ void main()
 		#prereplace IF_FLAG MODEL_SDR_FLAG_LIGHT
 			float shadow = 1.0;
 			#prereplace IF_FLAG MODEL_SDR_FLAG_SHADOWS
-				shadow = getShadowValue(shadow_map, -vertIn.position.z, vertIn.shadowPos.z, vertIn.shadowUV, fardist, middist, neardist, veryneardist);
+				shadow = getShadowValue(shadow_map, -vertIn.position.z, vertIn.shadowUV, cascade_distances, smoothness_factors, cascade_offset, cascade_count);
 			#prereplace ENDIF_FLAG //MODEL_SDR_FLAG_SHADOWS
 			baseColor.rgb = CalculateLighting(normal, baseColor.rgb, specColor.rgb, glossData, fresnelFactor, shadow, aoFactors.x);
 		#prereplace ELSE_FLAG //MODEL_SDR_FLAG_LIGHT
@@ -435,6 +428,4 @@ void main()
 		fragOut3 = vec4(specColor.rgb, fresnelFactor);
 		fragOut4 = emissiveColor;
 	#prereplace ENDIF_FLAG //MODEL_SDR_FLAG_DEFERRED
-
-	#endif
 }

code/def_files/data/effects/main-g.sdr

@@ -7,15 +7,6 @@
 layout (triangles) in;
 // For every triangle line we generate 2 triangles which can be done with 4 vertices so in total we will need 12 vertices
 layout (triangle_strip, max_vertices = 12) out;
-#elif defined(MODEL_SDR_FLAG_SHADOW_MAP)
-layout (triangles) in;
-layout (triangle_strip, max_vertices = 3) out;
-#endif
-
-#ifdef MODEL_SDR_FLAG_SHADOW_MAP
- #ifdef GL_ARB_gpu_shader5
-layout(invocations = 4) in;
- #endif
 #endif
 
 #define MAX_LIGHTS 8
@@ -37,8 +28,6 @@ layout (std140) uniform modelData {
 	mat4 viewMatrix;
 	mat4 projMatrix;
 	mat4 textureMatrix;
-	mat4 shadow_mv_matrix;
-	mat4 shadow_proj_matrix[4];
 
 	vec4 color;
 
@@ -84,11 +73,6 @@ layout (std140) uniform modelData {
 	float znear;
 	float zfar;
 
-	float veryneardist;
-	float neardist;
-	float middist;
-	float fardist;
-
 	int sGlowmapIndex;
 
 	int sSpecmapIndex;
@@ -108,19 +92,12 @@ in VertexOutput {
 	float fogDist;
 #prereplace ENDIF_FLAG_COMPILED MODEL_SDR_FLAG_FOG
 
-#ifdef MODEL_SDR_FLAG_SHADOW_MAP
- #if !defined(GL_ARB_gpu_shader5)
-	float instance;
- #endif
-	float clipModel;
-#else
 	vec4 position;
-#endif
 	vec3 normal;
 	vec4 texCoord;
 
 #prereplace IF_FLAG_COMPILED MODEL_SDR_FLAG_SHADOWS
-	vec4 shadowUV[4];
+	vec4 shadowUV[NUM_SHADOW_CASCADES];
 	vec4 shadowPos;
 #prereplace ENDIF_FLAG_COMPILED MODEL_SDR_FLAG_SHADOWS
 } vertIn[];
@@ -137,62 +114,12 @@ out VertexOutput {
 	vec4 texCoord;
 
 #prereplace IF_FLAG_COMPILED MODEL_SDR_FLAG_SHADOWS
-	vec4 shadowUV[4];
+	vec4 shadowUV[NUM_SHADOW_CASCADES];
 	vec4 shadowPos;
 #prereplace ENDIF_FLAG_COMPILED MODEL_SDR_FLAG_SHADOWS
 } vertOut;
 
-#ifdef MODEL_SDR_FLAG_SHADOW_MAP
-	void main(void)
-	{
-		#ifdef GL_ARB_gpu_shader5
-			int instanceID = gl_InvocationID;
-		#else
-			int instanceID = int(vertIn[0].instance);
-		#endif
-		for(int vert = 0; vert < gl_in.length(); vert++)
-		{
-			#prereplace IF_FLAG MODEL_SDR_FLAG_TRANSFORM
-				if (vertIn[vert].clipModel > 0.9) {
-					// If the model was clipped in the vertex shader then we do not apply the shadow projection matrix here
-					gl_Position = gl_in[vert].gl_Position;
-				} else {
-					gl_Position = shadow_proj_matrix[instanceID] * gl_in[vert].gl_Position;
-				}
-			#prereplace ELSE_FLAG //MODEL_SDR_FLAG_TRANSFORM
-				gl_Position = shadow_proj_matrix[instanceID] * gl_in[vert].gl_Position;
-			#prereplace ENDIF_FLAG //MODEL_SDR_FLAG_TRANSFORM
-
-			vertOut.position = gl_in[vert].gl_Position;
-			vertOut.normal = vertIn[vert].normal;
-			vertOut.texCoord = vertIn[vert].texCoord;
-
-			gl_Layer = instanceID;
-
-			#prereplace IF_FLAG MODEL_SDR_FLAG_NORMAL
-				vertOut.tangentMatrix = vertIn[vert].tangentMatrix;
-			#prereplace ENDIF_FLAG //MODEL_SDR_FLAG_NORMAL
-
-			#prereplace IF_FLAG MODEL_SDR_FLAG_FOG
-				vertOut.fogDist = vertIn[vert].fogDist;
-			#prereplace ENDIF_FLAG //MODEL_SDR_FLAG_FOG
-
-			#prereplace IF_FLAG MODEL_SDR_FLAG_SHADOWS
-				vertOut.shadowUV[0] = vertIn[vert].shadowUV[0];
-				vertOut.shadowUV[1] = vertIn[vert].shadowUV[1];
-				vertOut.shadowUV[2] = vertIn[vert].shadowUV[2];
-				vertOut.shadowUV[3] = vertIn[vert].shadowUV[3];
-				vertOut.shadowPos = vertIn[vert].shadowPos;
-			#prereplace ENDIF_FLAG //MODEL_SDR_FLAG_SHADOWS
-
-			if (use_clip_plane) {
-				gl_ClipDistance[0] = gl_in[vert].gl_ClipDistance[0];
-			}
-			EmitVertex();
-		}
-		EndPrimitive();
-	}
-#elif defined(MODEL_SDR_FLAG_THICK_OUTLINES)
+#ifdef MODEL_SDR_FLAG_THICK_OUTLINES
 	const vec2 pixelOffsetDir[4] = vec2[](
 		vec2(0.0, 1.0),
 		vec2(1.0, 1.0),
@@ -232,10 +159,9 @@ out VertexOutput {
                 #prereplace ENDIF_FLAG //MODEL_SDR_FLAG_FOG
 
                 #prereplace IF_FLAG MODEL_SDR_FLAG_SHADOWS
-                	vertOut.shadowUV[0] = vertIn[vert].shadowUV[0];
-                	vertOut.shadowUV[1] = vertIn[vert].shadowUV[1];
-                	vertOut.shadowUV[2] = vertIn[vert].shadowUV[2];
-                	vertOut.shadowUV[3] = vertIn[vert].shadowUV[3];
+                	for (int s = 0; s < NUM_SHADOW_CASCADES; s++) {
+                		vertOut.shadowUV[s] = vertIn[vert].shadowUV[s];
+                	}
                 	vertOut.shadowPos = vertIn[vert].shadowPos;
                 #prereplace ENDIF_FLAG //MODEL_SDR_FLAG_SHADOWS