GPU Stencil Shadow Volume generation - help!

I want to be able to generate stencil shadow volumes for a model made up of GL_TRIANGLE_ADJACENCY on the GPU inside a geometry shader. I had the idea that I could pass through the non-adjacency vertices to the fragment shader for rendering, and use transform feedback to capture the shadow polygons generated in the geometry shader to render later with the help of the stencil buffer.

At the moment i’m just trying to see whether my shader extrusion code works, which it doesn’t seem to do properly, and there’s obviously something I don’t understand properly, but I don’t kmow if it’s my code and/or limitations with what i’m trying to accomplish inside the shaders. For test purposes i’ve hardcoded the position of a single light in the shader as vec4(0.0, 1.0, 0.0, 1.0).

So, here are my questions/problems:

1: I pass in my model, view and projection matrices into the vertex shader. From there I can do something like:


mat4 MVMatrix = ViewMatrix * ModelMatrix;
mat4 PMVMatrix = ProjectionMatrix * MVMatrix;
gl_Position = PMVMatrix * VertexPosition;

However, if I just pass through the untransformed vertex to the geometry shader, along with the projection, view and model matrices, and try to do a similar thing there (obviously using arrays of mat4 in the geometry shader),
the object doesn’t get transformed properly. However, if I calculate the modelview matrix and pass that to the geometry shader along with the projection matrix, that works fine! That seems strange to me, but any idea why this should be? Surely I should be able to recreate the modelview matrix or the projection-modelview matrix in the geometry shader just as well as in the vertex shader?

2: I have my normals as part of the model, which are defined as always perpendicular to the face they are a part of. The other tutorials on stencil shadow volumes use a cross product to recompute the normals of each face. Can’t I just rotate the normals by the modelview matrix to get the same effect, and save on having to do the cross product? It “seems” to work sort-of-OK when it comes to the cheesy flat shading code in the vertex shader (though i suspect that it may depend on the view matrix somehow that I can’t put my finger on… the light is always at (0.0, 1.0, 0.0) but sometimes seems to over/undersaturate the colours when viewed from different angles, and rotating the lightsource by the modelview matrix seems to make things worse?)

3: My extrusion code does not extrude geometry away from the lightsource, and I can’t figure out why. It always seems to want to point towards (0, 0, 0). I’ve tried all sorts of rotations with the lightsource with the modelview matrix but to no avail, and i’m lost at this point. If the light is always at (0.0, 1.0, 0.0) then do I need to transform it, and can calculate the ray as (vertex - lightsource)? Can anyone give me any pointers as to where i’m going wrong? I’m sure the issue lies entirely within the shader somewhere, but i’m new to shader coding and I can’t figure it out, and i’m just tripping over my own feet at this point.

Thanks for any insight into where i’m going wrong!

Andy

I have this definition of a cube, seems overkill with the amount of vertices but hey…


/**
 * Cube model with adjacency information
 *
 * Each face is made up of 2 triangles:
 *
 * 	Triangle 1				Triangle 2
 * 	----------				----------
 *
 *               5
 *                .
 *            0 . . 4 			            2
 *	   .   o---o     		       3....o.
 *	  .	|  /.    			.  /| .
 *	1.	| / .    			. / |  .1
 *	   .	|/  .   			./  | .
 *	     .  o....3     			o---o.
 *             2     			        4. .0
 *                                                .
 *                                                5
 *
 * Even numbered vertices are vertices belonging to the model.
 * Odd numbered vertices are vertices describing adjacency information.
 *
 * Vertex data is ordered so that the 3rd vertex of each adjacency triangle describes
 *   the other triangle making up the face, so we can ignore it in the geometry shader.
 *
 */
static const Vertex cubeTriangleVerticesAdjacency[] =
{
		//two triangles for front face F1 and F2
		{vec3(-0.5f, 0.5f, 0.5f), 	vec3(0.0f, 0.0f, 1.0f),	vec4(0.25f, 0.0f, 0.75f, 1.0f)},
		{vec3(-0.5f, -0.5f, -0.5f),	vec3(-1.0f, 0.0f, 0.0f), vec4(0.35f, 0.0f, 0.85f, 1.0f)},									//L2.4
		{vec3(-0.5f, -0.5f, 0.5f), 	vec3(0.0f, 0.0f, 1.0f),	vec4(0.25f, 0.0f, 0.75f, 1.0f)},
		{vec3(0.0f, 0.0f, 0.0f), 	vec3(0.0f, 0.0f, 0.0f),	vec4(0.0f, 0.0f, 0.0f, 0.0f)},										//X - never extrude from vertex 3 since it is always the diagonal of a square
		{vec3(0.5f, 0.5f, 0.5f), 	vec3(0.0f, 0.0f, 1.0f),	vec4(0.25f, 0.0f, 0.75f, 1.0f)},
		{vec3(0.5f, 0.5f, -0.5f),	vec3(0.0f, 1.0f, 0.0f),	vec4(0.45f, 0.0f, 0.95f, 1.0f)},									//T2.2

		{vec3(0.5f, -0.5f, 0.5f),	vec3(0.0f, 0.0f, 1.0f),	vec4(0.25f, 0.0f, 0.75f, 1.0f)},
		{vec3(0.5f, 0.5f, -0.5f),	vec3(1.0f, 0.0f, 0.0f),	vec4(0.15f, 0.0f, 0.65f, 1.0f)},									//R1.4
		{vec3(0.5f, 0.5f, 0.5f),	vec3(0.0f, 0.0f, 1.0f),	vec4(0.25f, 0.0f, 0.75f, 1.0f)},
		{vec3(0.0f, 0.0f, 0.0f), 	vec3(0.0f, 0.0f, 0.0f),	vec4(0.0f, 0.0f, 0.0f, 0.0f)},										//X
		{vec3(-0.5f, -0.5f, 0.5f),	vec3(0.0f, 0.0f, 1.0f),	vec4(0.25f, 0.0f, 0.75f, 1.0f)},
		{vec3(-0.5f, -0.5f, -0.5f),	vec3(0.0f, -1.0f, 0.0f), vec4(0.10f, 0.0f, 0.60f, 1.0f)},									//U1.2


		//2 triangles for left face L1 and L2
		{vec3(-0.5f, 0.5f, -0.5f),	vec3(-1.0f, 0.0f, 0.0f), vec4(0.35f, 0.0f, 0.85f, 1.0f)},
		{vec3(0.5f, -0.5f, -0.5f),	vec3(0.0f, 0.0f, -1.0f), vec4(0.05f, 0.0f, 0.50f, 1.0f)},									//B2.4
		{vec3(-0.5f, -0.5f, -0.5f),	vec3(-1.0f, 0.0f, 0.0f), vec4(0.35f, 0.0f, 0.85f, 1.0f)},
		{vec3(0.0f, 0.0f, 0.0f), 	vec3(0.0f, 0.0f, 0.0f),	vec4(0.0f, 0.0f, 0.0f, 0.0f)},										//X
		{vec3(-0.5f, 0.5f, 0.5f),	vec3(-1.0f, 0.0f, 0.0f), vec4(0.35f, 0.0f, 0.85f, 1.0f)},
		{vec3(0.5f, -0.5f, 0.5f),	vec3(0.0f, -1.0f, 0.0f), vec4(0.10f, 0.0f, 0.60f, 1.0f)},									//U1.4

		{vec3(-0.5f, -0.5f, 0.5f),	vec3(-1.0f, 0.0f, 0.0f), vec4(0.35f, 0.0f, 0.85f, 1.0f)},
		{vec3(0.5f, 0.5f, 0.5f), 	vec3(0.0f, 0.0f, 1.0f),	vec4(0.25f, 0.0f, 0.75f, 1.0f)},									//F1.4
		{vec3(-0.5f, 0.5f, 0.5f),	vec3(-1.0f, 0.0f, 0.0f), vec4(0.35f, 0.0f, 0.85f, 1.0f)},
		{vec3(0.0f, 0.0f, 0.0f), 	vec3(0.0f, 0.0f, 0.0f),	vec4(0.0f, 0.0f, 0.0f, 0.0f)},										//X
		{vec3(-0.5f, -0.5f, -0.5f),	vec3(-1.0f, 0.0f, 0.0f), vec4(0.35f, 0.0f, 0.85f, 1.0f)},
		{vec3(0.5f, -0.5f, 0.5f),	vec3(0.0f, -1.0f, 0.0f), vec4(0.10f, 0.0f, 0.60f, 1.0f)},									//U1.4


		//2 triangles for right face R1 and R2
		{vec3(0.5f, 0.5f, 0.5f),	vec3(1.0f, 0.0f, 0.0f),	vec4(0.15f, 0.0f, 0.65f, 1.0f)},
		{vec3(-0.5f, -0.5f, 0.5f),	vec3(0.0f, 0.0f, 1.0f),	vec4(0.25f, 0.0f, 0.75f, 1.0f)},									//F2.4
		{vec3(0.5f, -0.5f, 0.5f),	vec3(1.0f, 0.0f, 0.0f),	vec4(0.15f, 0.0f, 0.65f, 1.0f)},
		{vec3(0.0f, 0.0f, 0.0f), 	vec3(0.0f, 0.0f, 0.0f),	vec4(0.0f, 0.0f, 0.0f, 0.0f)},										//X
		{vec3(0.5f, 0.5f, -0.5f),	vec3(1.0f, 0.0f, 0.0f),	vec4(0.15f, 0.0f, 0.65f, 1.0f)},
		{vec3(-0.5f, 0.5f, 0.5f),	vec3(0.0f, 1.0f, 0.0f),	vec4(0.45f, 0.0f, 0.95f, 1.0f)},									//T2.4

		{vec3(0.5f, -0.5f, -0.5f),	vec3(1.0f, 0.0f, 0.0f),	vec4(0.15f, 0.0f, 0.65f, 1.0f)},
		{vec3(-0.5, 0.5, -0.5),		vec3(0.0f, 0.0f, -1.0f), vec4(0.05f, 0.0f, 0.50f, 1.0f)},									//B1.4
		{vec3(0.5f, 0.5f, -0.5f),	vec3(1.0f, 0.0f, 0.0f),	vec4(0.15f, 0.0f, 0.65f, 1.0f)},
		{vec3(0.0f, 0.0f, 0.0f), 	vec3(0.0f, 0.0f, 0.0f),	vec4(0.0f, 0.0f, 0.0f, 0.0f)},										//X
		{vec3(0.5f, -0.5f, 0.5f),	vec3(1.0f, 0.0f, 0.0f),	vec4(0.15f, 0.0f, 0.65f, 1.0f)},
		{vec3(-0.5f, -0.5f, -0.5f),	vec3(0.0f, -1.0f, 0.0f), vec4(0.10f, 0.0f, 0.60f, 1.0f)},									//U2.4


		//2 triangles for top face, T1 and T2
		{vec3(-0.5f, 0.5f, -0.5f),	vec3(0.0f, 1.0f, 0.0f),	vec4(0.45f, 0.0f, 0.95f, 1.0f)},
		{vec3(-0.5f, -0.5f, -0.5f),	vec3(-1.0f, 0.0f, 0.0f), vec4(0.35f, 0.0f, 0.85f, 1.0f)},									//L1.2
		{vec3(-0.5f, 0.5f, 0.5f),	vec3(0.0f, 1.0f, 0.0f),	vec4(0.45f, 0.0f, 0.95f, 1.0f)},
		{vec3(0.0f, 0.0f, 0.0f), 	vec3(0.0f, 0.0f, 0.0f),	vec4(0.0f, 0.0f, 0.0f, 0.0f)},										//X
		{vec3(0.5f, 0.5f, -0.5f),	vec3(0.0f, 1.0f, 0.0f),	vec4(0.45f, 0.0f, 0.95f, 1.0f)},
		{vec3(0.5, -0.5, -0.5),		vec3(0.0f, 0.0f, -1.0f), vec4(0.05f, 0.0f, 0.50f, 1.0f)},									//B1.2

		{vec3(0.5f, 0.5f, 0.5f),	vec3(0.0f, 1.0f, 0.0f),	vec4(0.45f, 0.0f, 0.95f, 1.0f)},
		{vec3(0.5f, -0.5f, 0.5f),	vec3(1.0f, 0.0f, 0.0f),	vec4(0.15f, 0.0f, 0.65f, 1.0f)},									//R1.2
		{vec3(0.5f, 0.5f, -0.5f),	vec3(0.0f, 1.0f, 0.0f),	vec4(0.45f, 0.0f, 0.95f, 1.0f)},
		{vec3(0.0f, 0.0f, 0.0f), 	vec3(0.0f, 0.0f, 0.0f),	vec4(0.0f, 0.0f, 0.0f, 0.0f)},										//X
		{vec3(-0.5f, 0.5f, 0.5f),	vec3(0.0f, 1.0f, 0.0f),	vec4(0.45f, 0.0f, 0.95f, 1.0f)},
		{vec3(-0.5f, -0.5f, 0.5f), 	vec3(0.0f, 0.0f, 1.0f),	vec4(0.25f, 0.0f, 0.75f, 1.0f)},									//F1.2


		//2 triangles for bottom face U1 and U2
		{vec3(-0.5f, -0.5f, 0.5f),	vec3(0.0f, -1.0f, 0.0f), vec4(0.10f, 0.0f, 0.60f, 1.0f)},
		{vec3(-0.5f, -0.5f, -0.5f),	vec3(-1.0f, 0.0f, 0.0f), vec4(0.35f, 0.0f, 0.85f, 1.0f)},									//L2.2
		{vec3(-0.5f, -0.5f, -0.5f),	vec3(0.0f, -1.0f, 0.0f), vec4(0.10f, 0.0f, 0.60f, 1.0f)},
		{vec3(0.0f, 0.0f, 0.0f), 	vec3(0.0f, 0.0f, 0.0f),	vec4(0.0f, 0.0f, 0.0f, 0.0f)},										//X
		{vec3(0.5f, -0.5f, 0.5f),	vec3(0.0f, -1.0f, 0.0f), vec4(0.10f, 0.0f, 0.60f, 1.0f)},
		{vec3(0.5f, 0.5f, 0.5f),	vec3(0.0f, 0.0f, 1.0f),	vec4(0.25f, 0.0f, 0.75f, 1.0f)},									//F2.2

		{vec3(0.5f, -0.5f, -0.5f),	vec3(0.0f, -1.0f, 0.0f), vec4(0.10f, 0.0f, 0.60f, 1.0f)},
		{vec3(0.5f, 0.5f, -0.5f),	vec3(1.0f, 0.0f, 0.0f),	vec4(0.15f, 0.0f, 0.65f, 1.0f)},									//R2.2
		{vec3(0.5f, -0.5f, 0.5f),	vec3(0.0f, -1.0f, 0.0f), vec4(0.10f, 0.0f, 0.60f, 1.0f)},
		{vec3(0.0f, 0.0f, 0.0f), 	vec3(0.0f, 0.0f, 0.0f),	vec4(0.0f, 0.0f, 0.0f, 0.0f)},										//X
		{vec3(-0.5f, -0.5f, -0.5f),	vec3(0.0f, -1.0f, 0.0f), vec4(0.10f, 0.0f, 0.60f, 1.0f)},
		{vec3(-0.5f, 0.5f, -0.5f),	vec3(0.0f, 0.0f, -1.0f), vec4(0.05f, 0.0f, 0.50f, 1.0f)},									//B2.2


		//2 triangles for back face B1 and B2
		{vec3(0.5, 0.5, -0.5),		vec3(0.0f, 0.0f, -1.0f), vec4(0.05f, 0.0f, 0.50f, 1.0f)},
		{vec3(0.5f, -0.5f, 0.5f),	vec3(1.0f, 0.0f, 0.0f),	vec4(0.15f, 0.0f, 0.65f, 1.0f)},									//R2.4
		{vec3(0.5, -0.5, -0.5),		vec3(0.0f, 0.0f, -1.0f), vec4(0.05f, 0.0f, 0.50f, 1.0f)},
		{vec3(0.0f, 0.0f, 0.0f), 	vec3(0.0f, 0.0f, 0.0f),	vec4(0.0f, 0.0f, 0.0f, 0.0f)},										//X
		{vec3(-0.5, 0.5, -0.5),		vec3(0.0f, 0.0f, -1.0f), vec4(0.05f, 0.0f, 0.50f, 1.0f)},
		{vec3(-0.5f, 0.5f, 0.5f),	vec3(0.0f, 1.0f, 0.0f),	vec4(0.45f, 0.0f, 0.95f, 1.0f)},									//T1.2

		{vec3(-0.5f, -0.5f, -0.5f),	vec3(0.0f, 0.0f, -1.0f), vec4(0.05f, 0.0f, 0.50f, 1.0f)},
		{vec3(-0.5f, 0.5f, 0.5f),	vec3(-1.0f, 0.0f, 0.0f), vec4(0.35f, 0.0f, 0.85f, 1.0f)},									//L1.4
		{vec3(-0.5f, 0.5f, -0.5f),	vec3(0.0f, 0.0f, -1.0f), vec4(0.05f, 0.0f, 0.50f, 1.0f)},
		{vec3(0.0f, 0.0f, 0.0f), 	vec3(0.0f, 0.0f, 0.0f),	vec4(0.0f, 0.0f, 0.0f, 0.0f)},										//X
		{vec3(0.5f, -0.5f, -0.5f),	vec3(0.0f, 0.0f, -1.0f), vec4(0.05f, 0.0f, 0.50f, 1.0f)},
		{vec3(0.5f, -0.5f, 0.5f),	vec3(0.0f, -1.0f, 0.0f), vec4(0.10f, 0.0f, 0.60f, 1.0f)}									//U2.2
};


Here is my vertex shader:


const GLchar* SimpleFlatVertexShader = STRINGIFY(

//escape hash characters in stringify by pre-and-postfixing the line with newline characters...


#version 120 


attribute vec4 VertexPosition;
attribute vec4 SourceNormal;
attribute vec4 SourceColour;
      
uniform mat4 Projection;						//how we're transforming 3D into 2D (calcualted when setting view frustum)
uniform mat4 ViewMatrix;						//camera view onto the world
uniform mat4 ModelMatrix;						//a model's scale, rotation and translation matrix with respect to its own coords

varying vec4 GSDestinationColour;
varying vec4 GSLightPosition;
varying vec4 GSNormal;
varying mat4 GSProjectionMatrix;
varying mat4 GSViewMatrix;
varying mat4 GSModelMatrix;                                         


void main() 
{ 	

	mat4 ModelViewMatrix = ViewMatrix * ModelMatrix;

  	gl_Position =  ModelViewMatrix * VertexPosition; 

	vec4 norm = ModelViewMatrix * SourceNormal;
	vec4 lightpos = vec4(0.0, 1.0, 0.0, 1.0);

	GSDestinationColour.xyz = SourceColour.xyz * dot(lightpos, norm);
		
	GSDestinationColour.w = 1.0;
	
	GSNormal = norm;			//SourceNormal;						
	GSProjectionMatrix = Projection;
	GSViewMatrix = ViewMatrix;
	GSModelMatrix = ModelMatrix;
}


);


Here is my geometry shader:



const GLchar* StencilShadowVolumeGeometryShader = STRINGIFY(

//escape hash characters in stringify by pre-and-postfixing the line with newline characters...


#version 120  


#extension GL_EXT_geometry_shader4: enable 


 varying in vec4 GSNormal[6];
 varying in mat4 GSProjectionMatrix[6];
 varying in mat4 GSViewMatrix[6];
 varying in mat4 GSModelMatrix[6];
 varying in vec4 GSDestinationColour[6];
 varying out vec4 DestinationColour;
 
  
void main()
{
	
	int i;

	//Pass-through shader
	for(i = 0; i < gl_VerticesIn; i += 2)
  	{	 		 
	 	gl_Position = GSProjectionMatrix[0] * gl_PositionIn[i];		
	 	
 		DestinationColour = GSDestinationColour[i];	
        
    	EmitVertex();
  	}
  	
  	EndPrimitive();


	vec4 lightPos = vec4(0.0, 1.0, 0.0, 1.0);				//GSLightPosition[0];

	vec3 lightDirection = normalize(lightPos.xyz - gl_PositionIn[0].xyz);
	

	/*
		Step 1:  Check if the triangle faces the light, ignore otherwise
		Step 2:  Determine if the edges of the triangle are shadow casters
				   by checking the dot product of the triangle and its 
				   adjoining face.
		Step 3:	Extrude from this edge if the triangle faces the light and
				  the adjoining face does not.
	
	*/

	vec3 faceNormal = GSNormal[0].xyz;			//cross(gl_PositionIn[2].xyz - gl_PositionIn[0].xyz, gl_PositionIn[4].xyz - gl_PositionIn[0].xyz);

	if(dot(lightDirection, faceNormal) < 0.0)
	{
		if(dot(faceNormal, GSNormal[1].xyz) > 0.0)
		{
			gl_Position =  GSProjectionMatrix[0] * gl_PositionIn[0];
			DestinationColour = vec4(0.5, 0.5, 0.5, 1.0);
			EmitVertex();
			
			gl_Position = GSProjectionMatrix[0] * gl_PositionIn[0]  + vec4(lightDirection, 1.0);
			DestinationColour = vec4(0.5, 0.5, 0.5, 1.0);
			EmitVertex();
			
			gl_Position =  GSProjectionMatrix[0] * gl_PositionIn[2];		
			DestinationColour = vec4(0.5, 0.5, 0.5, 1.0);
			EmitVertex();
			
			EndPrimitive();

			lightDirection = normalize(lightPos.xyz - gl_PositionIn[2].xyz);
	
			gl_Position = GSProjectionMatrix[0] * gl_PositionIn[0] + vec4(lightDirection, 1.0) * 2.0;
			DestinationColour = vec4(0.75, 0.75, 0.75, 1.0);
			EmitVertex();			

			
			gl_Position = GSProjectionMatrix[0] * gl_PositionIn[2]  + vec4(lightDirection, 1.0) * 2.0;
			DestinationColour = vec4(0.75, 0.75, 0.75, 1.0);
			EmitVertex();
			
						
			gl_Position = GSProjectionMatrix[0] * gl_PositionIn[2];	
			DestinationColour = vec4(0.75, 0.75, 0.75, 1.0);
			EmitVertex();
			
			EndPrimitive();
		}
		
		
		if(dot(faceNormal, GSNormal[5].xyz) > 0.0)
		{
		
		lightDirection = normalize(lightPos.xyz - gl_PositionIn[0].xyz);
		
			gl_Position =  GSProjectionMatrix[0] * gl_PositionIn[0];
			DestinationColour = vec4(0.35, 0.35, 0.35, 1.0);
			EmitVertex();
			
			gl_Position = GSProjectionMatrix[0] * gl_PositionIn[0]  + vec4(lightDirection, 1.0) * 2.0;
			DestinationColour = vec4(0.35, 0.35, 0.35, 1.0);
			EmitVertex();
			
			gl_Position =  GSProjectionMatrix[0] * gl_PositionIn[4];		
			DestinationColour = vec4(0.35, 0.35, 0.35, 1.0);
			EmitVertex();
			
			EndPrimitive();

			lightDirection = normalize(lightPos.xyz - gl_PositionIn[4].xyz);
	
			gl_Position = GSProjectionMatrix[0] * gl_PositionIn[0] + vec4(lightDirection, 1.0) * 2.0;
			DestinationColour = vec4(0.25, 0.25, 0.25, 1.0);
			EmitVertex();			

			
			gl_Position = GSProjectionMatrix[0] * gl_PositionIn[4]  + vec4(lightDirection, 1.0) * 2.0;
			DestinationColour = vec4(0.25, 0.25, 0.25, 1.0);
			EmitVertex();
			
						
			gl_Position = GSProjectionMatrix[0] * gl_PositionIn[4];	
			DestinationColour = vec4(0.25, 0.25, 0.25, 1.0);
			EmitVertex();
			
			EndPrimitive();
		}
	
	}

}


);


And the fragment shader is dirt simple:


const GLchar* SimpleFlatFragmentShader = STRINGIFY(


#version 120 


varying vec4 DestinationColour;

void main() 
{ 
    gl_FragColor = DestinationColour;                                        
}

);