Stencil Shadow doesn' work

Started by
7 comments, last by bushmanpaul 13 years, 5 months ago
Hi

I am having some trouble getting stencil shadows to work. Instead of creating a shadow it creates a distortion effect over the entire window where there is drawn.I haven't included any pictures because I don't think they're necessary, if you think they might help just tell me and I'll post em.

The Scene is drawn as follows
        float clear[] = {0.0f, 0.0f, 0.0f, 1.0f }; //rgba	pContext->ClearRenderTargetView(pRenderTargetView, clear);	pContext->ClearDepthStencilView(pDepthStencilView,1,1,0);	pContext->ClearDepthStencilView(pLight->pDepthStencil,1,1,0);        RenderShadowMap(pLight);	pContext->OMSetRenderTargets(1,&pRenderTargetView,pDepthStencilView);		D3D11_MAPPED_SUBRESOURCE mappedResource;	pContext->Map(pConstantBuffer, 0, D3D11_MAP_WRITE_DISCARD, 0, &mappedResource);	ConstantBufferType* constantBuffer;	constantBuffer = (ConstantBufferType*)mappedResource.pData;	//Set Matrices	D3DXMatrixTranspose(&WorldMatrix, &WorldMatrix);	D3DXMatrixTranspose(&ViewMatrix , &ViewMatrix);	constantBuffer->World = WorldMatrix;	constantBuffer->View = ViewMatrix;	constantBuffer->Projection = ProjectionMatrix;	constantBuffer->CamPos = D3DXVECTOR4(pCamera->getPosition(),1.0f);	constantBuffer->LightView = pLights[0].LightView;        pContext->Unmap(pConstantBuffer, 0);        pContext->VSSetConstantBuffers(0 ,1 ,&pConstantBuffer);	pContext->PSSetConstantBuffers(0 ,1 ,&pConstantBuffer); 	//Init Shaders	UINT stride = sizeof(SimpleVert);	UINT offset = 0;	pContext->IASetVertexBuffers(0,1,&pVertexBuffer,&stride,&offset);	pContext->IASetIndexBuffer(pIndexBuffer, DXGI_FORMAT_R32_UINT, 0);	pContext->IASetPrimitiveTopology(D3D11_PRIMITIVE_TOPOLOGY_TRIANGLELIST);	pContext->IASetInputLayout(pInputLayout);	//Set Shaders	pContext->VSSetShader(pVertexShader,NULL,0);	pContext->PSSetShader(pPixelShader,NULL,0);	pContext->PSSetShaderResources(0,1,&pTexture);	pContext->PSSetShaderResources(1,1,&pLights[0].pDepthMap);	//Draw	pContext->DrawIndexed(numIndices,0,0);


The pLight is a Light struct some of the relevant methods is as follows

void Light::RenderShaderResourceView(){ //Initialize everything needed// if I don't use a null tex it gives an error Still bound as Input	ID3D11ShaderResourceView* pNULLTex = NULL;	pContext->PSSetShaderResources(2,1,&pNULLTex);        ID3D11RenderTargetView *pNullRTView = NULL;        pContext->OMSetRenderTargets(1, &pNullRTView, pDepthStencil);}//The render shadowMap functionvoid Scene::RenderShadowMap(Light light){		D3D11_MAPPED_SUBRESOURCE mappedResource;	pContext->Map(pDepthShaderBuffer, 0, D3D11_MAP_WRITE_DISCARD, 0, &mappedResource);	ShadowMapBuffer* constantBuffer;	constantBuffer = (ShadowMapBuffer*)mappedResource.pData;	//Create View Matrix from light Point of view	D3DXMatrixLookAtLH(&light.LightView,&D3DXVECTOR3(5.0f,15.0f,5.0f),&D3DXVECTOR3(0.0f,0.0f,0.0f),&D3DXVECTOR3(0.0f,1.0f,0.0f));	D3DXMatrixTranspose(&light.LightView,&light.LightView);	D3DXMatrixTranspose(&ProjectionMatrix,&ProjectionMatrix);	constantBuffer->Projection = ProjectionMatrix;	constantBuffer->LightView = light.LightView;	pContext->Unmap(pDepthShaderBuffer, 0);	pContext->IASetInputLayout(pInputLayout);	UINT stride = sizeof(SimpleVert);	UINT offset = 0;	pContext->IASetVertexBuffers(0,1,&pVertexBuffer,&stride,&offset);	pContext->IASetIndexBuffer(pIndexBuffer, DXGI_FORMAT_R32_UINT, 0);	pContext->IASetPrimitiveTopology(D3D11_PRIMITIVE_TOPOLOGY_TRIANGLELIST);	//Set Shaders	pContext->VSSetShader(pDepthVertexShader,NULL,0);	pContext->PSSetShader(NULL,NULL,0);	light.RenderShaderResourceView();	pContext->DrawIndexed(numIndices,0,0);}


I know the code isn't very well organized but I am still learning DX

the shader code is as follows

PS_INPUT VS( VS_INPUT input ){	PS_INPUT output;		input.pos.w = 1.0f;		//Position	output.pos = mul(input.pos, View);	output.pos = mul(output.pos, Projection);		//Lighting		//Pass Along Values;	output.norm = input.norm;	output.tex = input.tex;		output.lightTex = mul(input.pos,LightView);	output.lightTex = mul(output.lightTex,Projection);		return output;}float4 PS( PS_INPUT input ) : SV_TARGET{	float4 color = Texture.Sample(TextureSample, input.tex);		//Shadow DEBUG	//float2 shadowCol = NormalShader(input.pos, input.norm, CamPos, 20.0f, LightPos);		float4 finalColor = color + 0.1f;		//Simple is or isn't lit if I change biger than to smaller than the        //distortion stays mostly the same, I added the float value because I        //thought it might simply see everything in the shadow if more than          //that no shadows if less everywhere shadows	if(DepthMap.Sample(DepthSample,input.lightTex.xy) > input.lightTex.z/input.lightTex.w + 0.0028723f) finalColor = 0.1f;		return finalColor;};PS_INPUT_SHADOWPASS ShadowMapVS( VS_INPUT_SHADOWPASS input ){	PS_INPUT_SHADOWPASS output;		input.pos.w = 1.0f;		output.pos = mul(input.pos,LightView);	output.pos = mul(output.pos,Projection);		return output;};//dont use the pixel shader but I did write one anywayfloat ShadowMapPS( PS_INPUT_SHADOWPASS input ) : SV_DEPTH{ 	return input.pos.z; }


The data structures in the shader


cbuffer PerFrameBuffer{	matrix World;	matrix View;	matrix Projection;	matrix LightView;	float4 CamPos;};struct PS_INPUT{	float4 pos	: SV_POSITION;	float2 tex	: TEXCOORD0;	float3 norm	: NORMAL0;	float4 lightTex : TEXCOORD1;};struct VS_INPUT{	float4 pos		: POSITION;	float2 tex		: TEXCOORD0;	float3 norm		: NORMAL;};struct PS_INPUT_SHADOWPASS{	float4 pos	: SV_POSITION;};struct VS_INPUT_SHADOWPASS{	float4 pos	: POSITION;};Texture2D Texture;Texture2D<float> DepthMap;

the rastersate and depthstencil state
	depthStencilDesc.DepthEnable = true;	depthStencilDesc.DepthWriteMask = D3D11_DEPTH_WRITE_MASK_ALL;	depthStencilDesc.DepthFunc = D3D11_COMPARISON_LESS;	depthStencilDesc.StencilEnable = true;	depthStencilDesc.StencilReadMask = 0xFF;	depthStencilDesc.StencilWriteMask = 0xFF;	// Stencil operations if pixel is front-facing.	depthStencilDesc.FrontFace.StencilFailOp = D3D11_STENCIL_OP_KEEP;	depthStencilDesc.FrontFace.StencilDepthFailOp = D3D11_STENCIL_OP_INCR;	depthStencilDesc.FrontFace.StencilPassOp = D3D11_STENCIL_OP_KEEP;	depthStencilDesc.FrontFace.StencilFunc = D3D11_COMPARISON_ALWAYS;	depthStencilDesc.BackFace.StencilFailOp = D3D11_STENCIL_OP_KEEP;	depthStencilDesc.BackFace.StencilDepthFailOp = D3D11_STENCIL_OP_INCR;	depthStencilDesc.BackFace.StencilPassOp = D3D11_STENCIL_OP_KEEP;	depthStencilDesc.BackFace.StencilFunc = D3D11_COMPARISON_ALWAYS;	rasterDesc.FillMode = D3D11_FILL_SOLID;	rasterDesc.CullMode = D3D11_CULL_FRONT;	rasterDesc.FrontCounterClockwise = true;	rasterDesc.DepthBias = true;	rasterDesc.DepthBiasClamp = 1.e5;	rasterDesc.SlopeScaledDepthBias = 8.0f;	rasterDesc.DepthClipEnable = true;	rasterDesc.ScissorEnable = false;	rasterDesc.MultisampleEnable = false;	rasterDesc.AntialiasedLineEnable = false;


The dephtStencil is gotten from the depth render and share a texture.

Thanks in advance, any advice is welcome.

[Edited by - bushmanpaul on October 20, 2010 9:32:11 AM]
Advertisement
Heres the pictures anyway

Programming Problem 1

Programming Problem 2
I added tessalation to the program and problem only got worse it only makes total shadow or no shadow. I decided to check if my shadowmap shaders worked by setting them as my main shaders ,adding a Pixel Shader and chaning the lightView to View matrix so that I can move around.

Then when I did this I got a black screen so I am asuming that the shaders doesn't work properly.

Here are they

//TessalatorTexture2D Texture : register(t0);Texture2D NormalMap : register(t1);Texture2D BumpMap : register(t2);struct VS_INPUT_SHADOW{	float4 pos	: POSITION;	float2 tex	: TEXCOORD;};struct VS_OUTPUT_SHADOW{	float4 pos	: SV_POSITION;	float2 tex	: TEXCOORD;};struct CONSTANT_HS_OUTPUT{	float Edges[3] : SV_TessFactor;	float Inside : SV_InsideTessFactor;};Texture2D BumpMap : register(t2);VS_OUTPUT_SHADOW ShadowMapVS( VS_INPUT_SHADOW input ){	VS_OUTPUT_SHADOW output;	input.pos.w = 1.0f;	output.pos = input.pos;		output.tex = input.tex;		return output;};// patch constant functionCONSTANT_HS_OUTPUT ConstantsHS( InputPatch&lt;VS_OUTPUT_SHADOW, 3&gt; ip,uint PatchID : SV_PrimitiveID ){		CONSTANT_HS_OUTPUT	Output;		float TessAmount = 64.0f;		Output.Edges[0] = TessAmount;	Output.Edges[1] = TessAmount;	Output.Edges[2] = TessAmount;    Output.Inside = TessAmount;	    return Output;}// hull shader[domain("tri")][partitioning("integer")][outputtopology("triangle_cw")][outputcontrolpoints(3)][patchconstantfunc("ConstantsHS")]VS_OUTPUT_SHADOW ShadowMapHS( InputPatch&lt;VS_OUTPUT_SHADOW, 3 &gt; ip, uint i : SV_OutputControlPointID ){    VS_OUTPUT_SHADOW Output = ip;        return Output;}// domain shader[domain("tri")]VS_OUTPUT_SHADOW ShadowMapDS( CONSTANT_HS_OUTPUT input, float3 BarycentricCoordinates : SV_DomainLocation,const OutputPatch&lt;VS_OUTPUT_SHADOW, 3&gt; TrianglePatch ){	VS_OUTPUT_SHADOW output;	float4 pos = BarycentricCoordinates.x * TrianglePatch[0].pos +                  BarycentricCoordinates.y * TrianglePatch[1].pos +                  BarycentricCoordinates.z * TrianglePatch[2].pos ;                 	output.tex = BarycentricCoordinates.x * TrianglePatch[0].tex +                  BarycentricCoordinates.y * TrianglePatch[1].tex +                  BarycentricCoordinates.z * TrianglePatch[2].tex ;         float bumpHeight = 1.0f;            if(pos.y &lt; 2.0f) //So that the floating cube doesn't use a bump map    { 		pos.y = pos.y + 1.0f - BumpMap.SampleLevel(SampleState,output.tex,1).x * bumpHeight;    }			pos = mul( pos, View );    output.pos = mul( pos, Projection );	    return output;    }float ShadowMapPS( VS_OUTPUT_SHADOW input ) : SV_DEPTH{	return (float4)1.0f;        //input.pos.z/input.pos.w;}


[Edited by - bushmanpaul on October 19, 2010 10:55:45 AM]
First Read this please: http://www.gamedev.net/community/forums/faq.asp#tags

Its hard for me to read source code when its not inside source boxes...

Second, from your images I guess you should try other Bias values...
Changing the bias didn't work and noise appeared everywhere and there should've been a shadow even if the bias was of (I think).

Thanks for the tip about source boxes. The problem seems to lie in the shadowmap shader or at least most of the problem.

Any help or advice will be aprecianted.Thanks in advance.
OK, I fixed the problem about not rendering the shadowMap.It now renders but it still doesn't work.

Heres the code used to calculate the shadow

float4 posTex = input.lightTex;    posTex /= posTex.w;    posTex += float4(1,1,1,0);    posTex *= 0.5;    posTex.y = 1.0f - posTex.y;    //Simple is or isn't lit    float D = DepthMap.Sample(DepthSample,posTex.xy);    if( posTex.z < D ) return color/3.0f;


Here is the picture

Programming prob

Thanks in advance.
The shadowMap now renders correctly( Looks like it does) but the interpretation from the pixel sahder still gives a few problem.

Shadow Map not placed correctly

As you can see the shadow is projected but according to me it needs to move a bit more to the corner if I look at the depthMap(simply use it instead of a texture). The shadows on the cube itself is really odd, I subtracted 0.002 becuase otherwise more stripes appear or shadows start to dissapear.I find this very odd becuase I am using cull_back when rendering from the light view.

Here is how the texcoord is calculated

    //Domain Shader    Output.lightTex = mul(Output.pos,LightView);    Output.lightTex = mul(Output.lightTex,Projection);    //Pixel Shader    float4 posTex = input.lightTex;    posTex /= posTex.w;    posTex += float4(1,1,1,0);    posTex *= 0.5;    posTex.y = 1.0f - posTex.y;    //Simple is or isn't lit    float D = DepthMap.Sample(DepthSample,posTex.xy);    if( !(posTex.z -0.002 < D) ) return color/3.0f;


I can easily solve this problem by simply making everything that is not pointing to the light dark (simple dot product) but there will most probebly be other problems with close (to the caster)front facing facing that will still give the problem

Any advice will be greatly appreciated ( I am getting bored talking with myself ). Thanks in advance
Shadow now renders in the correct position but something seems to be wrong with the bias (or projection). The backfaces(shadowmap) of the cube(from light view) are not parralel to the actual faces and the shadow is skew I tried removing bias but it didn't work.I subtracted 0.0015 to show this problem more clearly(if I just leave it the shadow completely covers the box.

getting the shadow

float4 posTex = input.lightTex;        posTex /= posTex.w;        posTex += float4(1,1,1,0);	posTex *= 0.5;	posTex.y = 1.0f - posTex.y;        //Simple is or isn't lit	float D = DepthMap.Sample(DepthSample,posTex.xy);	if( (posTex.z -0.0015> D) || dot(input.norm,lightVec) < 0.0f) return color/3.0f;


the raster and stencilstate

D3D11_RASTERIZER_DESC drasterDesc;	drasterDesc.FillMode = D3D11_FILL_SOLID;	drasterDesc.CullMode = D3D11_CULL_BACK;	drasterDesc.FrontCounterClockwise = true;	drasterDesc.DepthBias = false;	drasterDesc.DepthBiasClamp = 0.0;	drasterDesc.SlopeScaledDepthBias = 0.0f;	drasterDesc.DepthClipEnable = true;	drasterDesc.ScissorEnable = false;	drasterDesc.MultisampleEnable = false;	drasterDesc.AntialiasedLineEnable = false;D3D11_DEPTH_STENCIL_DESC depthStencilDesc;	// Set up the description of the stencil state.	depthStencilDesc.DepthEnable = true;	depthStencilDesc.DepthWriteMask = D3D11_DEPTH_WRITE_MASK_ALL;	depthStencilDesc.DepthFunc = D3D11_COMPARISON_LESS;	depthStencilDesc.StencilEnable = true;	depthStencilDesc.StencilReadMask = 0xFF;	depthStencilDesc.StencilWriteMask = 0xFF;	// Stencil operations if pixel is front-facing.	depthStencilDesc.FrontFace.StencilFailOp = D3D11_STENCIL_OP_KEEP;	depthStencilDesc.FrontFace.StencilDepthFailOp = D3D11_STENCIL_OP_INCR;	depthStencilDesc.FrontFace.StencilPassOp = D3D11_STENCIL_OP_KEEP;	depthStencilDesc.FrontFace.StencilFunc = D3D11_COMPARISON_ALWAYS;	depthStencilDesc.BackFace.StencilFailOp = D3D11_STENCIL_OP_KEEP;	depthStencilDesc.BackFace.StencilDepthFailOp = D3D11_STENCIL_OP_INCR;	depthStencilDesc.BackFace.StencilPassOp = D3D11_STENCIL_OP_KEEP;	depthStencilDesc.BackFace.StencilFunc = D3D11_COMPARISON_ALWAYS;


Incorrect position
I solved it! Simply changed how I get the tex coord to

    float2 posTex = 0.5 * input.lightTex.xy / input.lightTex.w + float2( 0.5, 0.5 );    posTex.y = 1.0f - posTex.y;        float z = input.lightTex.z / input.lightTex.w ;


Thanks to Aqua Costa for trying to help.

This topic is closed to new replies.

Advertisement