[solved]My tessellation pipeline that draws nothing.

Started by
27 comments, last by Jason Z 10 years, 12 months ago

Ok, I believe that I am using the right blend state, and I believe that my ProjectionView matrix is ok, and I've eliminated relevant DX11 warnings and errors, but I still can't get anything to appear on screen. I'm greatly hampered by the fact that there is no way to debug my tessellation shaders with SharpDX and VS2012 express. This is my first stab at tessellation so I probably am doing something obviously wrong, so I would like to post my shaders to see if anything jumps out at anyone. What I am trying to do is draw instanced 3rd order bezier triangles. Besides the above, what other things might I check when nothing is drawn?

Here is the VS:


cbuffer PerFrameBuffer : register(b0)
{
	//float4x4 World; This is just the identity matrix, so not needed
	float4x4 ViewProjection;
	float4 vecEye; 
	float4 LightDirection;
	float4 LightColor; 
};

struct VertexInput
{
	//imediate data
	float3 PositionLocal		: POSITION0;
	int index					: BLENDINDICES;
	//instance data
	float3 Translation			: POSITION1;
	float Altitudes[4]			: PSIZE;
};

struct VertexOutput
{
	float3 PositionWorld	: POSITION;
	float2 MainTexCoord		: TEXCOORD;
};

VertexOutput VS(VertexInput vin)
{
	VertexOutput vout;

	//Add altitudes
	float3 transPos = vin.PositionLocal + vin.Translation;
	transPos.y += vin.Altitudes[vin.index];

	vout.MainTexCoord = transPos.xz;
	vout.PositionWorld = transPos;

	return vout;
}

Here is the pass thru Hull shader:



struct VertexOutput
{
	float3 PositionWorld	: POSITION;
	float2 MainTexCoord		: TEXCOORD;
};

struct PatchTess
{
	float EdgeTess[3]	: SV_TessFactor;
	float InsideTess	: SV_InsideTessFactor;
};

PatchTess PatchHS(InputPatch<VertexOutput,10> patch, uint patchID : SV_PrimitiveID)
{
	PatchTess pt;

	pt.EdgeTess[0] = 6;
	pt.EdgeTess[1] = 6;
	pt.EdgeTess[2] = 6;
	pt.InsideTess = 6;

	return pt;
}

struct HullOut
{
	float3 PositionWorld	: POSITION;
	float2 MainTexCoord		: TEXCOORD;
};

[domain("tri")]
[partitioning("fractional_even")]
[outputtopology("triangle_cw")]
[outputcontrolpoints(10)]
[patchconstantfunc("PatchHS")]
HullOut HS(InputPatch<VertexOutput,10> p, uint i : SV_OutputControlPointID, uint patchId : SV_PrimitiveID)
{
	HullOut hout;

	hout = p;

	return hout;
}

Here is the domain shader, where the meat of the intended operation is. The math here might be wrong, but should still produce visible geometry:


#define p300 0
#define p030 1
#define p003 2
#define p210 3
#define p201 4
#define p120 5
#define p021 6
#define p012 7
#define p102 8
#define p111 9

#define U bary.x
#define V bary.y
#define W bary.z

cbuffer PerFrameBuffer : register(b0)
{
	//float4x4 World; This is just the identity matrix, so not needed
	float4x4 ViewProjection;
	float4 vecEye; 
	float4 LightDirection;
	float4 LightColor; 
};

struct PatchTess
{
	float EdgeTess[3]	: SV_TessFactor;
	float InsideTess	: SV_InsideTessFactor;
};

struct HullOut
{
	float3 PositionWorld	: POSITION;
	float2 MainTexCoord		: TEXCOORD;
};

struct DomainOut
{
	float4 PositionH		: SV_Position;
	float3 Normal			: NORMAL0;
	float3 Tangent			: TANGENT0;
	float3 Bitangent		: BITANGENT0;
	float3 View				: NORMAL1;
	float2 MainTexCoord		: TEXCOORD0;
};

[domain("tri")]
DomainOut DS(PatchTess patchTess, float3 bary : SV_DomainLocation, const OutputPatch<HullOut,10> cp)
{
	DomainOut dout;

	float3 position =
		cp[p300].PositionWorld * pow(U,3) +
		cp[p030].PositionWorld * pow(V,3) +
		cp[p003].PositionWorld * pow(W,3) +
		cp[p210].PositionWorld * 3 * pow(U,2) * V +
		cp[p201].PositionWorld * 3 * pow(U,2) * W +
		cp[p120].PositionWorld * 3 * U * pow(V,2) +
		cp[p021].PositionWorld * 3 * pow(V,2) * W +
		cp[p012].PositionWorld * 3 * V * pow(W,2) +
		cp[p102].PositionWorld * 3 * U * pow(W,2) +
		cp[p111].PositionWorld * 6 * U * V * W;

	float3 tangent =
		cp[p300].PositionWorld * pow(U,2) +
		cp[p120].PositionWorld * pow(V,2) +
		cp[p102].PositionWorld * pow(W,2) +
		cp[p201].PositionWorld * 2 * U * W +
		cp[p210].PositionWorld * 2 * U * V +
		cp[p111].PositionWorld * 2 * V * W;

	float3 bitangent = 
		cp[p030].PositionWorld * pow(V,2) +
		cp[p012].PositionWorld * pow(W,2) +
		cp[p210].PositionWorld * pow(U,2) +
		cp[p120].PositionWorld * 2 * U * V +
		cp[p021].PositionWorld * 2 * V * W +
		cp[p111].PositionWorld * 2 * U * W;

	tangent = normalize(tangent);
	bitangent = normalize(bitangent);
	float3 normal = normalize(cross(tangent,bitangent));

	dout.View = vecEye.xyz - position.xyz;
	dout.PositionH = mul(float4(position,1.0f),ViewProjection);
	dout.Normal = normal;
	dout.Tangent = tangent;
	dout.Bitangent = bitangent;

	dout.MainTexCoord = cp[p300].MainTexCoord * U + cp[p030].MainTexCoord * V + cp[p003].MainTexCoord * W;

	return dout;
}

Last is the pixel shader, which has a simple shading algorithm that has been disabled to merely produce one color. I'm really hoping to see that color 'Reddish' but it's not happening:


cbuffer PerFrameBuffer : register(b0)
{
	//float4x4 World; This is just the identity matrix, so not needed
	float4x4 ViewProjection;
	float4 vecEye; 
	float4 LightDirection;
	float4 LightColor; 
};

struct DomainOut
{
	float4 PositionH		: SV_Position;
	float3 Normal			: NORMAL0;
	float3 Tangent			: TANGENT0;
	float3 Bitangent		: BITANGENT0;
	float3 View				: NORMAL1;
	float2 MainTexCoord		: TEXCOORD0;
};

float4 PS(DomainOut input) : SV_Target
{
    float4 Color = float4 (0, 0, 0, 1); 
	float4 Reddish = float4 (0.8,0.5,0.8,1.0);
	float4 specularColor = float4(1,1,1,1);

    float4 diffuse = {LightColor.rgb, 1.0f};   
    float4 ambient = { 0.03f, 0.03f, 0.03f, 1.0f};   
  
    float3 LightDir = normalize(LightDirection.xyz);   
    float3 ViewDir = normalize(input.View);    
       
    float4 diff = saturate(dot(input.Normal, LightDir)); // diffuse component   
  
    // R = 2 * (N.L) * N - L   
    float3 Reflect = normalize(2 * diff * input.Normal - LightDir);    
    float specular = pow(saturate(dot(Reflect, ViewDir)), 15); // R.V^n   
  
    // I = Acolor + Dcolor * N.L + (R.V)n   
    Color += ambient + diffuse  * diff + specular * (specularColor * LightColor);  

	float4 Final = Color * Reddish;
	Final.a = 1.0;

	//return Final;
	return Reddish;
       
}

Advertisement

When I started out with tessellation, I found it helpful to really simplify things and then build up the complexity step by step. I would suggest making your pixel shader just output a solid white color - then you can eliminate any shading errors or missing constant buffers from the pixel shader right off the bat.

Next I would actually eliminate the hull and domain shaders, and just pass your geometry from vertex shader to pixel shader. Verify that your vertices are appearing in the location that you expect (this will likely require moving some transformation code from the DS to the VS, but you can move it back fairly easily later). Instead of doing instanced rendering, try with just standard rendering instead (that will eliminate issues with your instance buffers).

Only once you have the geometry input to the tessellation stages verified, and then that the output stages from the tessellation stages verified, would I try to debug the hull and domain shaders. Have you been able to verify these things yet? If so, then we can move a little deeper into the investigation - if we know it is a tessellation issue or not!

Thanks Jason. That sounds like a good idea. It's time to start paring things down. One thing I forgot to mention above was the setup of the depth buffer, as if maybe it was always failing the depth test or something, I've got that set up how it should be, I had forgotten to set the DepthStencilState, but fixing that didn't help. I'll do what you suggest, although I'll have to change alot of stuff, because my inputs are just a series of control points. I'll have to decide how to turn them into triangles, or just draw points.

About the control points - just output them as regular points, and ensure their positions are as expected. Don't expend too much energy on the verification - you are just making sure it is set up the way you expect. That should really only require the changes in the shaders, plus the primitive topology.

As this is my first dx11 app, I do have a related question. Is there anything else I am supposed to do to turn on the different pipeline stages other than setting them in the immediate context? In SharpDX you would say context.HullShader.Set(hexCornerHS).

For instance let's say I want to use tessellation on one drawn element, and then only a vertex/pixel shader on another drawn element. Would I then have to set the hull/domain shaders to 'null' in order to disable them?

1) Well for the tesselation you need hull and domain shader. Other than that: constant buffers, views etc for the correct stages.
2) Yes. Whenever you issue a draw call, everything that is currently set goes into it. It won't change until you change it.

Oh, and cephalo, I found something useful. With a pass-through geometry shader like this...

[maxvertexcount(3)]
void PassThroughTriangleGS(triangle DomainOut gin[3], inout TriangleStream<DomainOut> stream)
{
    for(int i = 0; i < 3; i++)
    {
        stream.Append(gin);
    }
    stream.RestartStrip();
}
... you can actually force PIX to show what the tesselator spits out. It even highlights the whole patch biggrin.png
715781249054966.jpg
b98a4c249054972.jpg

interesting tip unbird. I thought that I couldn't have a geometry shader when the tessellation stages were active.

Ok, I tried unbird's trick, and kindof unfortunately I got almost the exact geometry I expected in the post GS window. However, I don't understand why the viewport shows a mere dot. In actual rendering, I don't see a dot even though my pixel shader currently puts out one color.

[attachment=14875:PIXpic.png]

To see how this geometry looks right to me, here is a screenshot of the DirectX 9 version of this project: It's basically a hex terrain map.

[attachment=14876:HexMap.jpg]

interesting tip unbird. I thought that I couldn't have a geometry shader when the tessellation stages were active.

There isn't any restriction like that. You have to have a vertex shader and a pixel shader, but outside of that there is no hard rule except that the last stage before the rasterizer has to output the SV_Position semantic.

The w-components of your post GS vertices is very large - that means that your geometry is very far away, and will appear small in your viewport. Can you double check your view and projection matrices to make sure they are properly set? Have you tried these matrices with a normal rendering setup (i.e. sans tessellation)?

This topic is closed to new replies.

Advertisement