stencil shadow problem

Started by
6 comments, last by wangzhonnew 17 years, 2 months ago
hi guys, i'm current implementing stencil shadow for my scene. in the scene, all object got good shadow until i added some trees into it. for those trees, it draw the shadow volumn out for some parts and other parts just doesnt cast any shadow... i'm so confused why rest of my objects can cast good shadow but only some parts of the trees will draw out the shadow volumn... here is the code to generate shadow volumn:

		HRESULT RSD3DShadowVolumn::Render( LPDIRECT3DDEVICE9 pd3dDevice )
		{
			HRESULT hr=S_OK;
			//if we do not support stencil, just return;
			if(!m_pD3D->GetStencilSupported())
			{
				Log(RS_WARNING, "Stencil is not supported\n");
				return E_FAIL;
			}
			// Disable z-buffer writes (note: z-testing still occurs), and enable the
			// stencil-buffer
			m_pDevice->SetRenderState( D3DRS_ZWRITEENABLE,  FALSE );
			m_pDevice->SetRenderState( D3DRS_STENCILENABLE, TRUE );

			// don't need to draw color
			m_pDevice->SetRenderState( D3DRS_SHADEMODE,     D3DSHADE_FLAT );

			// Set up stencil compare fuction, reference value, and masks.
			// Stencil test passes if ((ref & mask) cmpfn (stencil & mask)) is true.
			// Note: since we set up the stencil-test to always pass, the STENCILFAIL
			// renderstate is really not needed.
			m_pDevice->SetRenderState( D3DRS_STENCILFUNC,  D3DCMP_ALWAYS );
			m_pDevice->SetRenderState( D3DRS_STENCILZFAIL, D3DSTENCILOP_KEEP );
			m_pDevice->SetRenderState( D3DRS_STENCILFAIL,  D3DSTENCILOP_KEEP );

			// If ztest passes, inc/decrement stencil buffer value
			m_pDevice->SetRenderState( D3DRS_STENCILREF,       0x1 );
			m_pDevice->SetRenderState( D3DRS_STENCILMASK,      0xffffffff );
			m_pDevice->SetRenderState( D3DRS_STENCILWRITEMASK, 0xffffffff );
			m_pDevice->SetRenderState( D3DRS_STENCILPASS,      D3DSTENCILOP_INCR );

			// Make sure that no pixels get drawn to the frame buffer
			m_pDevice->SetRenderState( D3DRS_ALPHABLENDENABLE, TRUE );
			m_pDevice->SetRenderState( D3DRS_SRCBLEND,  D3DBLEND_ZERO );
			m_pDevice->SetRenderState( D3DRS_DESTBLEND, D3DBLEND_ONE );

			if(m_pD3D->GetTwoStencilSupported())
			{
				// With 2-sided stencil, we can avoid rendering twice:
				m_pDevice->SetRenderState( D3DRS_TWOSIDEDSTENCILMODE, TRUE );
				m_pDevice->SetRenderState( D3DRS_CCW_STENCILFUNC,  D3DCMP_ALWAYS );
				m_pDevice->SetRenderState( D3DRS_CCW_STENCILZFAIL, D3DSTENCILOP_KEEP );
				m_pDevice->SetRenderState( D3DRS_CCW_STENCILFAIL,  D3DSTENCILOP_KEEP );
				m_pDevice->SetRenderState( D3DRS_CCW_STENCILPASS, D3DSTENCILOP_DECR );

				m_pDevice->SetRenderState( D3DRS_CULLMODE,  D3DCULL_NONE );

				// Draw both sides of shadow volume in stencil/z only
				m_pDevice->SetTransform( D3DTS_WORLD, m_pD3D->GetWorldMatrix(m_pD3D->ActivatedWorldMatrix()));

				//render
				m_pDevice->SetFVF( RS_PVERTEXFVF );
				hr=m_pDevice->DrawPrimitiveUP( D3DPT_TRIANGLELIST, m_dwNumVertices/3,m_pVertices, sizeof(D3DXVECTOR3) );

				m_pDevice->SetRenderState( D3DRS_TWOSIDEDSTENCILMODE, FALSE );
			}
			else
			{
				// Draw front-side of shadow volume in stencil/z only
				m_pDevice->SetTransform( D3DTS_WORLD, m_pD3D->GetWorldMatrix(m_pD3D->ActivatedWorldMatrix()));
				//render
				m_pDevice->SetFVF( RS_PVERTEXFVF );
				hr=m_pDevice->DrawPrimitiveUP( D3DPT_TRIANGLELIST, m_dwNumVertices/3,m_pVertices, sizeof(D3DXVECTOR3) );

				// Now reverse cull order so back sides of shadow volume are written.
				m_pDevice->SetRenderState( D3DRS_CULLMODE,   D3DCULL_CW );

				// Decrement stencil buffer value
				m_pDevice->SetRenderState( D3DRS_STENCILPASS, D3DSTENCILOP_DECR );

				// Draw back-side of shadow volume in stencil/z only
				m_pDevice->SetTransform( D3DTS_WORLD, m_pD3D->GetWorldMatrix(m_pD3D->ActivatedWorldMatrix()));
				//render
				m_pDevice->SetFVF( RS_PVERTEXFVF );
				hr=m_pDevice->DrawPrimitiveUP( D3DPT_TRIANGLELIST, m_dwNumVertices/3,m_pVertices, sizeof(D3DXVECTOR3) );
			}

			// Restore render states
			m_pDevice->SetRenderState( D3DRS_SHADEMODE, D3DSHADE_GOURAUD );
			m_pDevice->SetRenderState( D3DRS_CULLMODE,  D3DCULL_CCW );
			m_pDevice->SetRenderState( D3DRS_ZWRITEENABLE,     TRUE );
			m_pDevice->SetRenderState( D3DRS_STENCILENABLE,    FALSE );
			m_pDevice->SetRenderState( D3DRS_ALPHABLENDENABLE, FALSE );

			return hr;
		}







and here is the code to draw shadow

		void RSD3D::DrawShadow()
		{
			// Set renderstates (disable z-buffering, enable stencil, disable fog, and
			// turn on alphablending)
			DWORD dwFogEnabled;
			m_pDevice->GetRenderState(D3DRS_FOGENABLE, &dwFogEnabled);
			m_pDevice->SetRenderState( D3DRS_ZENABLE,          FALSE );
			m_pDevice->SetRenderState( D3DRS_STENCILENABLE,    TRUE );
			m_pDevice->SetRenderState( D3DRS_FOGENABLE,        FALSE );
			m_pDevice->SetRenderState( D3DRS_ALPHABLENDENABLE, TRUE );
			m_pDevice->SetRenderState( D3DRS_SRCBLEND,  D3DBLEND_SRCALPHA );
			m_pDevice->SetRenderState( D3DRS_DESTBLEND, D3DBLEND_INVSRCALPHA );

			m_pDevice->SetTextureStageState( 0, D3DTSS_COLORARG1, D3DTA_TEXTURE );
			m_pDevice->SetTextureStageState( 0, D3DTSS_COLORARG2, D3DTA_DIFFUSE );
			m_pDevice->SetTextureStageState( 0, D3DTSS_COLOROP,   D3DTOP_MODULATE );
			m_pDevice->SetTextureStageState( 0, D3DTSS_ALPHAARG1, D3DTA_TEXTURE );
			m_pDevice->SetTextureStageState( 0, D3DTSS_ALPHAARG2, D3DTA_DIFFUSE );
			m_pDevice->SetTextureStageState( 0, D3DTSS_ALPHAOP,   D3DTOP_MODULATE );

			// Only write where stencil val >= 1 (count indicates # of shadows that
			// overlap that pixel)
			m_pDevice->SetRenderState( D3DRS_STENCILREF,  0x1 );
			m_pDevice->SetRenderState( D3DRS_STENCILFUNC, D3DCMP_LESSEQUAL );
			m_pDevice->SetRenderState( D3DRS_STENCILPASS, D3DSTENCILOP_KEEP );

			// Draw a big, gray square
			m_pDevice->SetFVF( SHADOWVERTEX::FVF );
			m_pDevice->SetStreamSource( 0, m_pBigSquareVB, 0, sizeof(SHADOWVERTEX) );
			m_pDevice->DrawPrimitive( D3DPT_TRIANGLESTRIP, 0, 2 );

			// Restore render states
			m_pDevice->SetRenderState( D3DRS_ZENABLE,          TRUE );
			m_pDevice->SetRenderState( D3DRS_STENCILENABLE,    FALSE );
			if(EnableFog())
				m_pDevice->SetRenderState( D3DRS_FOGENABLE,        TRUE );
			m_pDevice->SetRenderState( D3DRS_ALPHABLENDENABLE, FALSE );
			m_pDevice->SetRenderState(D3DRS_FOGENABLE, dwFogEnabled);

		}







the part to generate stencil shape is:

		//-----------------------------------------------------------------------------
		// Name: BuildFromMesh()
		// Desc: Takes a mesh as input, and uses it to build a shadowvolume. The
		//       technique used considers each triangle of the mesh, and adds it's
		//       edges to a temporary list. The edge list is maintained, such that
		//       only silohuette edges are kept. Finally, the silohuette edges are
		//       extruded to make the shadow volume vertex list.
		//-----------------------------------------------------------------------------
		HRESULT RSD3DShadowVolumn::BuildFromMesh( LPD3DXMESH pMesh, D3DXVECTOR3 vLight )
		{
			Reset();
			// Note: the MESHVERTEX format depends on the FVF of the mesh
			struct MESHVERTEX { D3DXVECTOR3 p, n; FLOAT tu, tv; };

			MESHVERTEX* pVertices;
			WORD*       pIndices;

			// Lock the geometry buffers
			pMesh->LockVertexBuffer( 0L, (LPVOID*)&pVertices );
			pMesh->LockIndexBuffer( 0L, (LPVOID*)&pIndices );
			DWORD dwNumFaces    = pMesh->GetNumFaces();

			// Allocate a temporary edge list
			WORD* pEdges = new WORD[dwNumFaces*6];
			if( pEdges == NULL )
			{
				pMesh->UnlockVertexBuffer();
				pMesh->UnlockIndexBuffer();
				return E_OUTOFMEMORY;
			}
			DWORD dwNumEdges = 0;

			// For each face
			for( DWORD i=0; i<dwNumFaces; i++ )
			{
				WORD wFace0 = pIndices[3*i+0];
				WORD wFace1 = pIndices[3*i+1];
				WORD wFace2 = pIndices[3*i+2];

				D3DXVECTOR3 v0 = pVertices[wFace0].p;
				D3DXVECTOR3 v1 = pVertices[wFace1].p;
				D3DXVECTOR3 v2 = pVertices[wFace2].p;

				// Transform vertices or transform light?
				D3DXVECTOR3 vCross1(v2-v1);
				D3DXVECTOR3 vCross2(v1-v0);
				D3DXVECTOR3 vNormal;
				D3DXVec3Cross( &vNormal, &vCross1, &vCross2 );

				if( D3DXVec3Dot( &vNormal, &vLight ) >= 0.0f )
				{
					AddEdge( pEdges, dwNumEdges, wFace0, wFace1 );
					AddEdge( pEdges, dwNumEdges, wFace1, wFace2 );
					AddEdge( pEdges, dwNumEdges, wFace2, wFace0 );
				}
			}

			for(UINT i=0; i<dwNumEdges; i++ )
			{
				D3DXVECTOR3 v1 = pVertices[pEdges[2*i+0]].p;
				D3DXVECTOR3 v2 = pVertices[pEdges[2*i+1]].p;
				D3DXVECTOR3 v3 = v1 - vLight*10;
				D3DXVECTOR3 v4 = v2 - vLight*10;

				// Add a quad (two triangles) to the vertex list
				m_pVertices[m_dwNumVertices++] = v1;
				m_pVertices[m_dwNumVertices++] = v2;
				m_pVertices[m_dwNumVertices++] = v3;

				m_pVertices[m_dwNumVertices++] = v2;
				m_pVertices[m_dwNumVertices++] = v4;
				m_pVertices[m_dwNumVertices++] = v3;
			}
			// Delete the temporary edge list
			delete[] pEdges;

			// Unlock the geometry buffers
			pMesh->UnlockVertexBuffer();
			pMesh->UnlockIndexBuffer();
			m_bUpdated=true;
			return S_OK;
		}

		//-----------------------------------------------------------------------------
		// Name: AddEdge()
		// Desc: Adds an edge to a list of silohuette edges of a shadow volume.
		//-----------------------------------------------------------------------------
		void RSD3DShadowVolumn::AddEdge( WORD* pEdges, DWORD& dwNumEdges, WORD v0, WORD v1 )
		{
			// Remove interior edges (which appear in the list twice)
			for( DWORD i=0; i < dwNumEdges; i++ )
			{
				if( ( pEdges[2*i+0] == v0 && pEdges[2*i+1] == v1 ) ||
					( pEdges[2*i+0] == v1 && pEdges[2*i+1] == v0 ) )
				{
					if( dwNumEdges > 1 )
					{
						pEdges[2*i+0] = pEdges[2*(dwNumEdges-1)+0];
						pEdges[2*i+1] = pEdges[2*(dwNumEdges-1)+1];
					}
					dwNumEdges--;
					return;
				}
			}

			pEdges[2*dwNumEdges+0] = v0;
			pEdges[2*dwNumEdges+1] = v1;
			dwNumEdges++;
		}




any advise are appreciated...
Advertisement
i tried to load some other model that created by myself in 3dmax and they looks fine, only some part on the tree model that i got online got something displayed wrong... i'm not sure if my code has problem or the model is corrupted.
and if it is model's problem, in which senario it will generate the shadow volumn like the screenshot?
Ah! I had a similar problem once. It turns out that stencil shadow algorithm requires the occluders be closed triangle meshes. Meaning every edge in the model must only be shared by 2 triangles thus disallowing any holes that would expose the interior of the model.

I got similar artifacts as the ones I see in your pic. This might be it, or you might be calculating the silhouette wrong.

[EDIT] Some models have three triangles sharing an edge, that is bad too.
++ My::Game ++
thanks for your reply.
after some debugging in AddEdge, i found that if i remove some code inside AddEdge the extra shadow volumn will gone:
		//-----------------------------------------------------------------------------		// Name: AddEdge()		// Desc: Adds an edge to a list of silohuette edges of a shadow volume.		//-----------------------------------------------------------------------------		void RSD3DShadowVolumn::AddEdge( WORD* pEdges, DWORD& dwNumEdges, WORD v0, WORD v1 )		{			// Remove interior edges (which appear in the list twice)			for( DWORD i=0; i < dwNumEdges; i++ )			{				if( ( pEdges[2*i+0] == v0 && pEdges[2*i+1] == v1 ) ||					( pEdges[2*i+0] == v1 && pEdges[2*i+1] == v0 ) )				{/*					if( dwNumEdges > 1 )					{						pEdges[2*i+0] = pEdges[2*(dwNumEdges-1)+0];						pEdges[2*i+1] = pEdges[2*(dwNumEdges-1)+1];					}					dwNumEdges--;					return;*/				}			}			pEdges[2*dwNumEdges+0] = v0;			pEdges[2*dwNumEdges+1] = v1;			dwNumEdges++;		}

so i guess this is what you said in your reply.
do you have any good algorithm to fix it up?

the thing is, i'm still a little bit confusing why it will display the shadow volumn~~~ why the stencil counter will add 1?

[Edited by - wangzhonnew on January 28, 2007 3:54:20 PM]
I've seen that issue referred to as "shadow bleed" before. As the above poster said, you need to fix your meshes.
NextWar: The Quest for Earth available now for Windows Phone 7.
Quote:Original post by Sc4Freak
I've seen that issue referred to as "shadow bleed" before. As the above poster said, you need to fix your meshes.


hmm.. i'm not a art guy... it's sooo hard for me to fix the mesh itself.
is there any other workaround in code to prevent it happens?
since it is static mesh, i can do the precomputation when loading the mesh... if there's any good algorithm to detect which edge caused the problem, maybe i can remove it from the list...
From the picture you have posted, it seems that all the meshes are bleeding shadows. That to me looks like a problem with silhouette calculations and/or incorrect edge table.

Try getting correct results with a simpler scene first. Try dumping the edge table/silhouette to a file and verify if calculations are correct. If your calculations are indeed correct and the mesh is at fault, then there is no choice but to fix the mesh, else you will have to go for another shadowing technique like shadow maps.

Hope that helps.
++ My::Game ++
Quote:Original post by _neutrin0_
From the picture you have posted, it seems that all the meshes are bleeding shadows. That to me looks like a problem with silhouette calculations and/or incorrect edge table.

Try getting correct results with a simpler scene first. Try dumping the edge table/silhouette to a file and verify if calculations are correct. If your calculations are indeed correct and the mesh is at fault, then there is no choice but to fix the mesh, else you will have to go for another shadowing technique like shadow maps.

Hope that helps.


uhhh... guess i have to try to find another tree model...
the shadow works perfectly fine for my car, building, and simple "tree model" that i created in Max...
only the tree model that i got online bleeding and now seems there's no way to fix it :(
thanks for you guys' suggestion...

This topic is closed to new replies.

Advertisement