Sign in to follow this  
webjeff

Stencil Shadow DirectX Weirdness (w/ screen)

Recommended Posts

OK, check out this screenshot: http://webjeff.netfirms.com/newshadow.jpg I've been on here before, I can't really test a simple shadow model, because I am using a third party animation library (cal3d) however, the vertices/indicies I'm getting from the library are used to render it, so rendering to the stencil buffer should be ok. I tried all different ways for culling, I checked the indicies I'm getting manually by breakpoints to the indicies in the model and they match up correctly. Here is my stencil shadow code:
#include "stdafx.h"
#include "ShadowVolume.h"

//-----------------------------------------------------------------------------
// Name: render
// Desc:
//-----------------------------------------------------------------------------
HRESULT ShadowVolume::render( LPDIRECT3DDEVICE9 pd3dDevice )
{

    pd3dDevice->SetFVF( D3DFVF_XYZ );

    return pd3dDevice->DrawPrimitiveUP( D3DPT_TRIANGLELIST, m_dwNumVertices/3,
                                        m_pVertices, sizeof(D3DXVECTOR3) );
}

//-----------------------------------------------------------------------------
// Name: buildShadowVolume()
// Desc: Takes a mesh as input, and uses it to build a shadow volume. The
//       technique used considers each triangle of the mesh, and adds it's
//       edges to a temporary list. The edge list is maintained, such that
//       only silohuette edges are kept. Finally, the silohuette edges are
//       extruded to make the shadow volume vertex list.
//-----------------------------------------------------------------------------
void ShadowVolume::CheckDual(LPDIRECT3DDEVICE9 pd3dDevice)
{
	D3DCAPS9 d3dCaps;
	pd3dDevice->GetDeviceCaps( &d3dCaps );// D3DADAPTER_DEFAULT, D3DDEVTYPE_HAL,
	if( ( d3dCaps.StencilCaps & D3DSTENCILCAPS_TWOSIDED ) != 0 )
		g_bTwoSidedStencilsAvailable = true;
}
HRESULT ShadowVolume::buildShadowVolume( void* pVert,void* pInd,DWORD numFaces,DWORD numVerts,D3DXVECTOR3 vLight,D3DXMATRIX mat )
{
	struct MeshVertex { 
		D3DXVECTOR3 p, n;
		FLOAT tu,tv;};

    MeshVertex *pVertices = (MeshVertex*)pVert;
    WORD       *pIndices  = (WORD*)pInd;

    DWORD dwNumFaces    = numFaces;
	DWORD dwNumVertices = numVerts;

    WORD* pEdges = new WORD[dwNumFaces*6];
    DWORD dwNumEdges = 0;

    // For each face
    for( DWORD i = 0; i < dwNumFaces; ++i )
    {
        WORD wFace0 = pIndices[3*i+0];
        WORD wFace1 = pIndices[3*i+1];
        WORD wFace2 = pIndices[3*i+2];
		D3DXVECTOR3 v0 = pVertices[wFace0].p;
		D3DXVECTOR3 v1 = pVertices[wFace1].p;
		D3DXVECTOR3 v2 = pVertices[wFace2].p;
		//D3DXVec3TransformCoord(&v0, &pVertices[wFace0].p,&mat);
		//D3DXVec3TransformCoord(&v1, &pVertices[wFace1].p,&mat);
		//D3DXVec3TransformCoord(&v2, &pVertices[wFace2].p,&mat);

		//D3DXVECTOR3 vLight(-489,-50,10);

        
        // Transform vertices or transform light?
        D3DXVECTOR3 vCross1(v2-v1);
        D3DXVECTOR3 vCross2(v1-v0);
        D3DXVECTOR3 vNormal;
        D3DXVec3Cross( &vNormal, &vCross1, &vCross2 );
		D3DXVec3Normalize(&vNormal,&vNormal);

        if( D3DXVec3Dot( &vNormal, &vLight ) >= 0.0f )
        {
            addEdge( pEdges, dwNumEdges, wFace0, wFace1 );
            addEdge( pEdges, dwNumEdges, wFace1, wFace2 );
            addEdge( pEdges, dwNumEdges, wFace2, wFace0 );
        }
    }

    for( i = 0; i < dwNumEdges; ++i )
    {
        D3DXVECTOR3 v1 = pVertices[pEdges[2*i+0]].p;
        D3DXVECTOR3 v2 = pVertices[pEdges[2*i+1]].p;
        D3DXVECTOR3 v3 = v1 - vLight*20;
        D3DXVECTOR3 v4 = v2 - vLight*20;
		
		if (m_dwNumVertices<100000-7){ 
        m_pVertices[m_dwNumVertices++] = v1;
        m_pVertices[m_dwNumVertices++] = v2;
        m_pVertices[m_dwNumVertices++] = v3;
        m_pVertices[m_dwNumVertices++] = v2;
        m_pVertices[m_dwNumVertices++] = v4;
        m_pVertices[m_dwNumVertices++] = v3;
		}
    }

    // Delete the temporary edge list
    delete[] pEdges;

    return S_OK;
}

//-----------------------------------------------------------------------------
// Name: addEdge()
// Desc: Adds an edge to a list of silohuette edges of a shadow volume.
//-----------------------------------------------------------------------------
void ShadowVolume::addEdge( WORD* pEdges, DWORD& dwNumEdges, WORD v0, WORD v1 )
{
    //// Remove interior edges (which appear in the list twice)
    //for( DWORD i = 0; i < dwNumEdges; ++i )
    //{
    //    if( ( pEdges[2*i+0] == v0 && pEdges[2*i+1] == v1 ) ||
    //        ( pEdges[2*i+0] == v1 && pEdges[2*i+1] == v0 ) )
    //    {
    //        if( dwNumEdges > 1 )
    //        {
    //            pEdges[2*i+0] = pEdges[2*(dwNumEdges-1)+0];
    //            pEdges[2*i+1] = pEdges[2*(dwNumEdges-1)+1];
    //        }

    //        --dwNumEdges;
    //        return;
    //    }
    //}

    pEdges[2*dwNumEdges+0] = v0;
    pEdges[2*dwNumEdges+1] = v1;
    dwNumEdges++;
}




void ShadowVolume::renderShadowToStencilBuffer( LPDIRECT3DDEVICE9 g_pd3dDevice, D3DXMATRIX m_mat )
{


    // Disable z-buffer writes (note: z-testing still occurs), and enable the
    // stencil-buffer
	DWORD shade,cull,zwrite,stencil,alpha;
	g_pd3dDevice->GetRenderState( D3DRS_SHADEMODE, &shade );
    g_pd3dDevice->GetRenderState( D3DRS_CULLMODE,  &cull );
    g_pd3dDevice->GetRenderState( D3DRS_ZWRITEENABLE,     &zwrite );
    g_pd3dDevice->GetRenderState( D3DRS_STENCILENABLE,    &stencil );
    g_pd3dDevice->GetRenderState( D3DRS_ALPHABLENDENABLE, &alpha );

    g_pd3dDevice->SetRenderState( D3DRS_ZWRITEENABLE,  FALSE );
    g_pd3dDevice->SetRenderState( D3DRS_STENCILENABLE, TRUE );

    // Dont bother with interpolating color
    g_pd3dDevice->SetRenderState( D3DRS_SHADEMODE, D3DSHADE_FLAT );

    // Set up stencil compare fuction, reference value, and masks.
    // Stencil test passes if ((ref & mask) cmpfn (stencil & mask)) is true.
    // Note: since we set up the stencil-test to always pass, the STENCILFAIL
    // renderstate is really not needed.
    g_pd3dDevice->SetRenderState( D3DRS_STENCILFUNC,  D3DCMP_ALWAYS );
    g_pd3dDevice->SetRenderState( D3DRS_STENCILZFAIL, D3DSTENCILOP_KEEP );
    g_pd3dDevice->SetRenderState( D3DRS_STENCILFAIL,  D3DSTENCILOP_KEEP );

    // If z-test passes, inc/decrement stencil buffer value
    g_pd3dDevice->SetRenderState( D3DRS_STENCILREF,       0x1 );
    g_pd3dDevice->SetRenderState( D3DRS_STENCILMASK,      0xffffffff );
    g_pd3dDevice->SetRenderState( D3DRS_STENCILWRITEMASK, 0xffffffff );
    g_pd3dDevice->SetRenderState( D3DRS_STENCILPASS,      D3DSTENCILOP_INCR );

    // Make sure that no pixels get drawn to the frame buffer
    g_pd3dDevice->SetRenderState( D3DRS_ALPHABLENDENABLE, TRUE );
    g_pd3dDevice->SetRenderState( D3DRS_SRCBLEND,  D3DBLEND_ZERO );
    g_pd3dDevice->SetRenderState( D3DRS_DESTBLEND, D3DBLEND_ONE );

	if( g_bTwoSidedStencilsAvailable == true )
    {
        // With 2-sided stencil, we can avoid rendering twice:
        g_pd3dDevice->SetRenderState( D3DRS_TWOSIDEDSTENCILMODE, TRUE );
        g_pd3dDevice->SetRenderState( D3DRS_CCW_STENCILFUNC,  D3DCMP_ALWAYS );
        g_pd3dDevice->SetRenderState( D3DRS_CCW_STENCILZFAIL, D3DSTENCILOP_KEEP );
        g_pd3dDevice->SetRenderState( D3DRS_CCW_STENCILFAIL,  D3DSTENCILOP_KEEP );
        g_pd3dDevice->SetRenderState( D3DRS_CCW_STENCILPASS, D3DSTENCILOP_DECR );

        g_pd3dDevice->SetRenderState( D3DRS_CULLMODE,  D3DCULL_NONE );

        // Draw both sides of shadow volume in stencil/z only
        g_pd3dDevice->SetTransform( D3DTS_WORLD, &m_mat );
        render( g_pd3dDevice );

        g_pd3dDevice->SetRenderState( D3DRS_TWOSIDEDSTENCILMODE, FALSE );
    }
    else
    {
        // Draw front-side of shadow volume in stencil/z only
        g_pd3dDevice->SetTransform( D3DTS_WORLD, &m_mat );
        render( g_pd3dDevice );

        // Now reverse cull order so back sides of shadow volume are written.
        g_pd3dDevice->SetRenderState( D3DRS_CULLMODE, D3DCULL_CW );

        // Decrement stencil buffer value
        g_pd3dDevice->SetRenderState( D3DRS_STENCILPASS, D3DSTENCILOP_DECR );

        // Draw back-side of shadow volume in stencil/z only
        g_pd3dDevice->SetTransform( D3DTS_WORLD, &m_mat );
        render( g_pd3dDevice );
    }

    // Restore render states
	g_pd3dDevice->SetRenderState( D3DRS_SHADEMODE, shade );
    g_pd3dDevice->SetRenderState( D3DRS_CULLMODE,  cull );
    g_pd3dDevice->SetRenderState( D3DRS_ZWRITEENABLE,     zwrite );
    g_pd3dDevice->SetRenderState( D3DRS_STENCILENABLE,    stencil );
    g_pd3dDevice->SetRenderState( D3DRS_ALPHABLENDENABLE, alpha );
}



void ShadowVolume::renderShadowToScene( LPDIRECT3DDEVICE9 g_pd3dDevice )
{
     // Set render states
    g_pd3dDevice->SetRenderState( D3DRS_ZENABLE,          FALSE );
    g_pd3dDevice->SetRenderState( D3DRS_STENCILENABLE,    TRUE );
    g_pd3dDevice->SetRenderState( D3DRS_ALPHABLENDENABLE, TRUE );
    g_pd3dDevice->SetRenderState( D3DRS_SRCBLEND,  D3DBLEND_SRCALPHA );
    g_pd3dDevice->SetRenderState( D3DRS_DESTBLEND, D3DBLEND_INVSRCALPHA );

    g_pd3dDevice->SetTextureStageState( 0, D3DTSS_COLORARG1, D3DTA_TEXTURE );
    g_pd3dDevice->SetTextureStageState( 0, D3DTSS_COLORARG2, D3DTA_DIFFUSE );
    g_pd3dDevice->SetTextureStageState( 0, D3DTSS_COLOROP,   D3DTOP_MODULATE );
    g_pd3dDevice->SetTextureStageState( 0, D3DTSS_ALPHAARG1, D3DTA_TEXTURE );
    g_pd3dDevice->SetTextureStageState( 0, D3DTSS_ALPHAARG2, D3DTA_DIFFUSE );
    g_pd3dDevice->SetTextureStageState( 0, D3DTSS_ALPHAOP,   D3DTOP_MODULATE );

    // Only write where stencil value >= 1 (count indicates # of shadows that
    // overlap that pixel)
    g_pd3dDevice->SetRenderState( D3DRS_STENCILREF,  0x1 );
    g_pd3dDevice->SetRenderState( D3DRS_STENCILFUNC, D3DCMP_LESSEQUAL );
    g_pd3dDevice->SetRenderState( D3DRS_STENCILPASS, D3DSTENCILOP_KEEP );

    // Draw a big, gray square
	g_pd3dDevice->SetFVF( ShadowVertex::FVF_Flags );
    g_pd3dDevice->SetStreamSource( 0, m_pBigSquareVB, 0, sizeof(ShadowVertex) );
    g_pd3dDevice->DrawPrimitive( D3DPT_TRIANGLESTRIP, 0, 2 );

    // Restore render states
    g_pd3dDevice->SetRenderState( D3DRS_ZENABLE,          TRUE );
    g_pd3dDevice->SetRenderState( D3DRS_STENCILENABLE,    FALSE );
    g_pd3dDevice->SetRenderState( D3DRS_ALPHABLENDENABLE, FALSE );
}

You may also note, I commented out the part where it deletes dupicates, if I leave that in and take out the duplicate edges, I get this picture: http://webjeff.netfirms.com/wrongshadows.jpg (just for the helmet) Any ideas guys, I'm really at a loss. Thanks in advance, Jeff King.

Share this post


Link to post
Share on other sites

Create an account or sign in to comment

You need to be a member in order to leave a comment

Create an account

Sign up for a new account in our community. It's easy!

Register a new account

Sign in

Already have an account? Sign in here.

Sign In Now

Sign in to follow this