Sign in to follow this  
bushmanpaul

Stencil Shadow doesn' work

Recommended Posts

bushmanpaul    100
Hi

I am having some trouble getting stencil shadows to work. Instead of creating a shadow it creates a distortion effect over the entire window where there is drawn.I haven't included any pictures because I don't think they're necessary, if you think they might help just tell me and I'll post em.

The Scene is drawn as follows


float clear[] = {0.0f, 0.0f, 0.0f, 1.0f }; //rgba

pContext->ClearRenderTargetView(pRenderTargetView, clear);
pContext->ClearDepthStencilView(pDepthStencilView,1,1,0);
pContext->ClearDepthStencilView(pLight->pDepthStencil,1,1,0);

RenderShadowMap(pLight);

pContext->OMSetRenderTargets(1,&pRenderTargetView,pDepthStencilView);

D3D11_MAPPED_SUBRESOURCE mappedResource;

pContext->Map(pConstantBuffer, 0, D3D11_MAP_WRITE_DISCARD, 0, &mappedResource);

ConstantBufferType* constantBuffer;

constantBuffer = (ConstantBufferType*)mappedResource.pData;

//Set Matrices
D3DXMatrixTranspose(&WorldMatrix, &WorldMatrix);
D3DXMatrixTranspose(&ViewMatrix , &ViewMatrix);

constantBuffer->World = WorldMatrix;
constantBuffer->View = ViewMatrix;
constantBuffer->Projection = ProjectionMatrix;
constantBuffer->CamPos = D3DXVECTOR4(pCamera->getPosition(),1.0f);
constantBuffer->LightView = pLights[0].LightView;

pContext->Unmap(pConstantBuffer, 0);

pContext->VSSetConstantBuffers(0 ,1 ,&pConstantBuffer);
pContext->PSSetConstantBuffers(0 ,1 ,&pConstantBuffer);

//Init Shaders
UINT stride = sizeof(SimpleVert);
UINT offset = 0;

pContext->IASetVertexBuffers(0,1,&pVertexBuffer,&stride,&offset);

pContext->IASetIndexBuffer(pIndexBuffer, DXGI_FORMAT_R32_UINT, 0);

pContext->IASetPrimitiveTopology(D3D11_PRIMITIVE_TOPOLOGY_TRIANGLELIST);

pContext->IASetInputLayout(pInputLayout);

//Set Shaders
pContext->VSSetShader(pVertexShader,NULL,0);
pContext->PSSetShader(pPixelShader,NULL,0);

pContext->PSSetShaderResources(0,1,&pTexture);
pContext->PSSetShaderResources(1,1,&pLights[0].pDepthMap);

//Draw
pContext->DrawIndexed(numIndices,0,0);




The pLight is a Light struct some of the relevant methods is as follows


void Light::RenderShaderResourceView()
{
//Initialize everything needed
// if I don't use a null tex it gives an error Still bound as Input
ID3D11ShaderResourceView* pNULLTex = NULL;
pContext->PSSetShaderResources(2,1,&pNULLTex);

ID3D11RenderTargetView *pNullRTView = NULL;

pContext->OMSetRenderTargets(1, &pNullRTView, pDepthStencil);
}

//The render shadowMap function

void Scene::RenderShadowMap(Light light)
{

D3D11_MAPPED_SUBRESOURCE mappedResource;

pContext->Map(pDepthShaderBuffer, 0, D3D11_MAP_WRITE_DISCARD, 0, &mappedResource);

ShadowMapBuffer* constantBuffer;

constantBuffer = (ShadowMapBuffer*)mappedResource.pData;

//Create View Matrix from light Point of view
D3DXMatrixLookAtLH(&light.LightView,&D3DXVECTOR3(5.0f,15.0f,5.0f),&D3DXVECTOR3(0.0f,0.0f,0.0f),&D3DXVECTOR3(0.0f,1.0f,0.0f));

D3DXMatrixTranspose(&light.LightView,&light.LightView);
D3DXMatrixTranspose(&ProjectionMatrix,&ProjectionMatrix);

constantBuffer->Projection = ProjectionMatrix;
constantBuffer->LightView = light.LightView;

pContext->Unmap(pDepthShaderBuffer, 0);

pContext->IASetInputLayout(pInputLayout);

UINT stride = sizeof(SimpleVert);
UINT offset = 0;

pContext->IASetVertexBuffers(0,1,&pVertexBuffer,&stride,&offset);

pContext->IASetIndexBuffer(pIndexBuffer, DXGI_FORMAT_R32_UINT, 0);

pContext->IASetPrimitiveTopology(D3D11_PRIMITIVE_TOPOLOGY_TRIANGLELIST);

//Set Shaders
pContext->VSSetShader(pDepthVertexShader,NULL,0);
pContext->PSSetShader(NULL,NULL,0);

light.RenderShaderResourceView();

pContext->DrawIndexed(numIndices,0,0);
}



I know the code isn't very well organized but I am still learning DX

the shader code is as follows


PS_INPUT VS( VS_INPUT input )
{
PS_INPUT output;

input.pos.w = 1.0f;

//Position
output.pos = mul(input.pos, View);
output.pos = mul(output.pos, Projection);

//Lighting

//Pass Along Values;
output.norm = input.norm;
output.tex = input.tex;

output.lightTex = mul(input.pos,LightView);
output.lightTex = mul(output.lightTex,Projection);

return output;
}

float4 PS( PS_INPUT input ) : SV_TARGET
{
float4 color = Texture.Sample(TextureSample, input.tex);

//Shadow DEBUG
//float2 shadowCol = NormalShader(input.pos, input.norm, CamPos, 20.0f, LightPos);

float4 finalColor = color + 0.1f;

//Simple is or isn't lit if I change biger than to smaller than the
//distortion stays mostly the same, I added the float value because I
//thought it might simply see everything in the shadow if more than
//that no shadows if less everywhere shadows
if(DepthMap.Sample(DepthSample,input.lightTex.xy) > input.lightTex.z/input.lightTex.w + 0.0028723f) finalColor = 0.1f;

return finalColor;
};

PS_INPUT_SHADOWPASS ShadowMapVS( VS_INPUT_SHADOWPASS input )
{
PS_INPUT_SHADOWPASS output;

input.pos.w = 1.0f;

output.pos = mul(input.pos,LightView);
output.pos = mul(output.pos,Projection);

return output;
};

//dont use the pixel shader but I did write one anyway
float ShadowMapPS( PS_INPUT_SHADOWPASS input ) : SV_DEPTH
{
return input.pos.z;
}



The data structures in the shader



cbuffer PerFrameBuffer
{
matrix World;
matrix View;
matrix Projection;
matrix LightView;
float4 CamPos;
};

struct PS_INPUT
{
float4 pos : SV_POSITION;
float2 tex : TEXCOORD0;
float3 norm : NORMAL0;
float4 lightTex : TEXCOORD1;
};

struct VS_INPUT
{
float4 pos : POSITION;
float2 tex : TEXCOORD0;
float3 norm : NORMAL;
};

struct PS_INPUT_SHADOWPASS
{
float4 pos : SV_POSITION;
};

struct VS_INPUT_SHADOWPASS
{
float4 pos : POSITION;
};

Texture2D Texture;
Texture2D<float> DepthMap;



the rastersate and depthstencil state

depthStencilDesc.DepthEnable = true;
depthStencilDesc.DepthWriteMask = D3D11_DEPTH_WRITE_MASK_ALL;
depthStencilDesc.DepthFunc = D3D11_COMPARISON_LESS;

depthStencilDesc.StencilEnable = true;
depthStencilDesc.StencilReadMask = 0xFF;
depthStencilDesc.StencilWriteMask = 0xFF;

// Stencil operations if pixel is front-facing.
depthStencilDesc.FrontFace.StencilFailOp = D3D11_STENCIL_OP_KEEP;
depthStencilDesc.FrontFace.StencilDepthFailOp = D3D11_STENCIL_OP_INCR;
depthStencilDesc.FrontFace.StencilPassOp = D3D11_STENCIL_OP_KEEP;
depthStencilDesc.FrontFace.StencilFunc = D3D11_COMPARISON_ALWAYS;

depthStencilDesc.BackFace.StencilFailOp = D3D11_STENCIL_OP_KEEP;
depthStencilDesc.BackFace.StencilDepthFailOp = D3D11_STENCIL_OP_INCR;
depthStencilDesc.BackFace.StencilPassOp = D3D11_STENCIL_OP_KEEP;
depthStencilDesc.BackFace.StencilFunc = D3D11_COMPARISON_ALWAYS;

rasterDesc.FillMode = D3D11_FILL_SOLID;
rasterDesc.CullMode = D3D11_CULL_FRONT;
rasterDesc.FrontCounterClockwise = true;
rasterDesc.DepthBias = true;
rasterDesc.DepthBiasClamp = 1.e5;
rasterDesc.SlopeScaledDepthBias = 8.0f;
rasterDesc.DepthClipEnable = true;
rasterDesc.ScissorEnable = false;
rasterDesc.MultisampleEnable = false;
rasterDesc.AntialiasedLineEnable = false;





The dephtStencil is gotten from the depth render and share a texture.

Thanks in advance, any advice is welcome.

[Edited by - bushmanpaul on October 20, 2010 9:32:11 AM]

Share this post


Link to post
Share on other sites
bushmanpaul    100
I added tessalation to the program and problem only got worse it only makes total shadow or no shadow. I decided to check if my shadowmap shaders worked by setting them as my main shaders ,adding a Pixel Shader and chaning the lightView to View matrix so that I can move around.

Then when I did this I got a black screen so I am asuming that the shaders doesn't work properly.

Here are they



//Tessalator
Texture2D Texture : register(t0);
Texture2D NormalMap : register(t1);
Texture2D BumpMap : register(t2);

struct VS_INPUT_SHADOW
{
float4 pos : POSITION;
float2 tex : TEXCOORD;
};

struct VS_OUTPUT_SHADOW
{
float4 pos : SV_POSITION;
float2 tex : TEXCOORD;
};

struct CONSTANT_HS_OUTPUT
{
float Edges[3] : SV_TessFactor;
float Inside : SV_InsideTessFactor;
};

Texture2D BumpMap : register(t2);

VS_OUTPUT_SHADOW ShadowMapVS( VS_INPUT_SHADOW input )
{
VS_OUTPUT_SHADOW output;

input.pos.w = 1.0f;
output.pos = input.pos;

output.tex = input.tex;

return output;
};

// patch constant function
CONSTANT_HS_OUTPUT ConstantsHS( InputPatch&lt;VS_OUTPUT_SHADOW, 3&gt; ip,uint PatchID : SV_PrimitiveID )
{
CONSTANT_HS_OUTPUT Output;

float TessAmount = 64.0f;

Output.Edges[0] = TessAmount;
Output.Edges[1] = TessAmount;
Output.Edges[2] = TessAmount;
Output.Inside = TessAmount;

return Output;
}

// hull shader
[domain("tri")]
[partitioning("integer")]
[outputtopology("triangle_cw")]
[outputcontrolpoints(3)]
[patchconstantfunc("ConstantsHS")]
VS_OUTPUT_SHADOW ShadowMapHS( InputPatch&lt;VS_OUTPUT_SHADOW, 3 &gt; ip, uint i : SV_OutputControlPointID )
{
VS_OUTPUT_SHADOW Output = ip[i];

return Output;
}

// domain shader
[domain("tri")]
VS_OUTPUT_SHADOW ShadowMapDS( CONSTANT_HS_OUTPUT input, float3 BarycentricCoordinates : SV_DomainLocation,const OutputPatch&lt;VS_OUTPUT_SHADOW, 3&gt; TrianglePatch )
{
VS_OUTPUT_SHADOW output;

float4 pos = BarycentricCoordinates.x * TrianglePatch[0].pos +
BarycentricCoordinates.y * TrianglePatch[1].pos +
BarycentricCoordinates.z * TrianglePatch[2].pos ;

output.tex = BarycentricCoordinates.x * TrianglePatch[0].tex +
BarycentricCoordinates.y * TrianglePatch[1].tex +
BarycentricCoordinates.z * TrianglePatch[2].tex ;

float bumpHeight = 1.0f;


if(pos.y &lt; 2.0f) //So that the floating cube doesn't use a bump map
{
pos.y = pos.y + 1.0f - BumpMap.SampleLevel(SampleState,output.tex,1).x * bumpHeight;
}


pos = mul( pos, View );
output.pos = mul( pos, Projection );

return output;
}

float ShadowMapPS( VS_OUTPUT_SHADOW input ) : SV_DEPTH
{
return (float4)1.0f;
//input.pos.z/input.pos.w;
}



[Edited by - bushmanpaul on October 19, 2010 10:55:45 AM]

Share this post


Link to post
Share on other sites
bushmanpaul    100
Changing the bias didn't work and noise appeared everywhere and there should've been a shadow even if the bias was of (I think).

Thanks for the tip about source boxes. The problem seems to lie in the shadowmap shader or at least most of the problem.

Any help or advice will be aprecianted.Thanks in advance.

Share this post


Link to post
Share on other sites
bushmanpaul    100
OK, I fixed the problem about not rendering the shadowMap.It now renders but it still doesn't work.

Heres the code used to calculate the shadow



float4 posTex = input.lightTex;
posTex /= posTex.w;
posTex += float4(1,1,1,0);
posTex *= 0.5;
posTex.y = 1.0f - posTex.y;

//Simple is or isn't lit
float D = DepthMap.Sample(DepthSample,posTex.xy);
if( posTex.z < D ) return color/3.0f;




Here is the picture

Programming prob

Thanks in advance.

Share this post


Link to post
Share on other sites
bushmanpaul    100
The shadowMap now renders correctly( Looks like it does) but the interpretation from the pixel sahder still gives a few problem.

Shadow Map not placed correctly

As you can see the shadow is projected but according to me it needs to move a bit more to the corner if I look at the depthMap(simply use it instead of a texture). The shadows on the cube itself is really odd, I subtracted 0.002 becuase otherwise more stripes appear or shadows start to dissapear.I find this very odd becuase I am using cull_back when rendering from the light view.

Here is how the texcoord is calculated


//Domain Shader

Output.lightTex = mul(Output.pos,LightView);
Output.lightTex = mul(Output.lightTex,Projection);

//Pixel Shader

float4 posTex = input.lightTex;
posTex /= posTex.w;
posTex += float4(1,1,1,0);
posTex *= 0.5;
posTex.y = 1.0f - posTex.y;

//Simple is or isn't lit
float D = DepthMap.Sample(DepthSample,posTex.xy);
if( !(posTex.z -0.002 < D) ) return color/3.0f;




I can easily solve this problem by simply making everything that is not pointing to the light dark (simple dot product) but there will most probebly be other problems with close (to the caster)front facing facing that will still give the problem

Any advice will be greatly appreciated ( I am getting bored talking with myself ). Thanks in advance

Share this post


Link to post
Share on other sites
bushmanpaul    100
Shadow now renders in the correct position but something seems to be wrong with the bias (or projection). The backfaces(shadowmap) of the cube(from light view) are not parralel to the actual faces and the shadow is skew I tried removing bias but it didn't work.I subtracted 0.0015 to show this problem more clearly(if I just leave it the shadow completely covers the box.

getting the shadow



float4 posTex = input.lightTex;
posTex /= posTex.w;
posTex += float4(1,1,1,0);
posTex *= 0.5;
posTex.y = 1.0f - posTex.y;

//Simple is or isn't lit
float D = DepthMap.Sample(DepthSample,posTex.xy);
if( (posTex.z -0.0015> D) || dot(input.norm,lightVec) < 0.0f) return color/3.0f;




the raster and stencilstate



D3D11_RASTERIZER_DESC drasterDesc;
drasterDesc.FillMode = D3D11_FILL_SOLID;
drasterDesc.CullMode = D3D11_CULL_BACK;
drasterDesc.FrontCounterClockwise = true;
drasterDesc.DepthBias = false;
drasterDesc.DepthBiasClamp = 0.0;
drasterDesc.SlopeScaledDepthBias = 0.0f;
drasterDesc.DepthClipEnable = true;
drasterDesc.ScissorEnable = false;
drasterDesc.MultisampleEnable = false;
drasterDesc.AntialiasedLineEnable = false;

D3D11_DEPTH_STENCIL_DESC depthStencilDesc;

// Set up the description of the stencil state.
depthStencilDesc.DepthEnable = true;
depthStencilDesc.DepthWriteMask = D3D11_DEPTH_WRITE_MASK_ALL;
depthStencilDesc.DepthFunc = D3D11_COMPARISON_LESS;

depthStencilDesc.StencilEnable = true;
depthStencilDesc.StencilReadMask = 0xFF;
depthStencilDesc.StencilWriteMask = 0xFF;

// Stencil operations if pixel is front-facing.
depthStencilDesc.FrontFace.StencilFailOp = D3D11_STENCIL_OP_KEEP;
depthStencilDesc.FrontFace.StencilDepthFailOp = D3D11_STENCIL_OP_INCR;
depthStencilDesc.FrontFace.StencilPassOp = D3D11_STENCIL_OP_KEEP;
depthStencilDesc.FrontFace.StencilFunc = D3D11_COMPARISON_ALWAYS;

depthStencilDesc.BackFace.StencilFailOp = D3D11_STENCIL_OP_KEEP;
depthStencilDesc.BackFace.StencilDepthFailOp = D3D11_STENCIL_OP_INCR;
depthStencilDesc.BackFace.StencilPassOp = D3D11_STENCIL_OP_KEEP;
depthStencilDesc.BackFace.StencilFunc = D3D11_COMPARISON_ALWAYS;




Incorrect position

Share this post


Link to post
Share on other sites
bushmanpaul    100
I solved it! Simply changed how I get the tex coord to


float2 posTex = 0.5 * input.lightTex.xy / input.lightTex.w + float2( 0.5, 0.5 );
posTex.y = 1.0f - posTex.y;

float z = input.lightTex.z / input.lightTex.w ;



Thanks to Aqua Costa for trying to help.

Share this post


Link to post
Share on other sites

Create an account or sign in to comment

You need to be a member in order to leave a comment

Create an account

Sign up for a new account in our community. It's easy!

Register a new account

Sign in

Already have an account? Sign in here.

Sign In Now

Sign in to follow this