Sign in to follow this  

Stupid Stencil Shadow Nightmare

This topic is 4851 days old which is more than the 365 day threshold we allow for new replies. Please post a new topic.

If you intended to correct an error in the post then please contact us.

Recommended Posts

Argg, OK, My stencil shadow looks like its taking the model and extruding the opposite side of it (sorda looks like tons of lines are going from the model to below the surface of the plane). I am NOT in wireframe mode. I have an example I am looking at and when I put it in wireframe mode it looks exactly like how mine looks, I just need the shadow, not the geometry going down to make the shadow.... any ideas why it would be showing up like that??? Thanks Jeff.

Share this post


Link to post
Share on other sites
Hmm,

Maybe I'm not being clear, its hard to explain, but the projected shadow volume shows up, not just the shadow. Maybe thats the term I'm looking for. I see tons of lines, I am clearing the stencil buffer each frame and I'm calculating it correctly, but for some reason its not right.

At this address (THIS IS NOT MY CODE):
http://www.codesampler.com/source/dx9_shadow_volume.zip

But when you click F1 for wireframe, thats exactly what my model looks like without displaying it in wireframe.

Any ideas?
Jeff King.

Share this post


Link to post
Share on other sites
Can you post some code? So it looks like you're rendering to the screen when you should be rendering to the stencil? (I don't mean that's what you're doing, but that's what it looks like right?)

Chris

Share this post


Link to post
Share on other sites
Here's my Shadow Volume Code:


#include "stdafx.h"
#include "ShadowVolume.h"

//-----------------------------------------------------------------------------
// Name: render
// Desc:
//-----------------------------------------------------------------------------
HRESULT ShadowVolume::render( LPDIRECT3DDEVICE9 pd3dDevice )
{
pd3dDevice->SetFVF( D3DFVF_XYZ );

return pd3dDevice->DrawPrimitiveUP( D3DPT_TRIANGLELIST, m_dwNumVertices/3,
m_pVertices, sizeof(D3DXVECTOR3) );
}

//-----------------------------------------------------------------------------
// Name: buildShadowVolume()
// Desc: Takes a mesh as input, and uses it to build a shadow volume. The
// technique used considers each triangle of the mesh, and adds it's
// edges to a temporary list. The edge list is maintained, such that
// only silohuette edges are kept. Finally, the silohuette edges are
// extruded to make the shadow volume vertex list.
//-----------------------------------------------------------------------------
void ShadowVolume::CheckDual(LPDIRECT3DDEVICE9 pd3dDevice)
{
D3DCAPS9 d3dCaps;
pd3dDevice->GetDeviceCaps( &d3dCaps );// D3DADAPTER_DEFAULT, D3DDEVTYPE_HAL,
if( ( d3dCaps.StencilCaps & D3DSTENCILCAPS_TWOSIDED ) != 0 )
g_bTwoSidedStencilsAvailable = true;
}
HRESULT ShadowVolume::buildShadowVolume( void* pVert,void* pInd,DWORD numFaces,DWORD numVerts,D3DXVECTOR3 vLight )
{
// Note: the MeshVertex format depends on the FVF of the mesh
struct MeshVertex {
D3DXVECTOR3 p, n;
FLOAT tu,tv;}; // Works fine for teapot, but won't do for all meshes!

MeshVertex *pVertices = (MeshVertex*)pVert;
WORD *pIndices = (WORD*)pInd;

DWORD dwNumFaces = numFaces;
DWORD dwNumVertices = numVerts;
// Allocate a temporary edge list
WORD* pEdges = new WORD[dwNumFaces*6];
DWORD dwNumEdges = 0;

// For each face
for( DWORD i = 0; i < dwNumFaces; ++i )
{
WORD wFace0 = pIndices[3*i+0];
WORD wFace1 = pIndices[3*i+1];
WORD wFace2 = pIndices[3*i+2];

D3DXVECTOR3 v0 = pVertices[wFace0].p;
D3DXVECTOR3 v1 = pVertices[wFace1].p;
D3DXVECTOR3 v2 = pVertices[wFace2].p;

// Transform vertices or transform light?
D3DXVECTOR3 vCross1(v2-v1);
D3DXVECTOR3 vCross2(v1-v0);
D3DXVECTOR3 vNormal;
D3DXVec3Cross( &vNormal, &vCross1, &vCross2 );

if( D3DXVec3Dot( &vNormal, &vLight ) >= 0.0f )
{
addEdge( pEdges, dwNumEdges, wFace0, wFace1 );
addEdge( pEdges, dwNumEdges, wFace1, wFace2 );
addEdge( pEdges, dwNumEdges, wFace2, wFace0 );
}
}

for( i = 0; i < dwNumEdges; ++i )
{
D3DXVECTOR3 v1 = pVertices[pEdges[2*i+0]].p;
D3DXVECTOR3 v2 = pVertices[pEdges[2*i+1]].p;
D3DXVECTOR3 v3 = v1 - vLight*10;
D3DXVECTOR3 v4 = v2 - vLight*10;

// Add a quad (two triangles) to the vertex list
m_pVertices[m_dwNumVertices++] = v1;
m_pVertices[m_dwNumVertices++] = v2;
m_pVertices[m_dwNumVertices++] = v3;

m_pVertices[m_dwNumVertices++] = v2;
m_pVertices[m_dwNumVertices++] = v4;
m_pVertices[m_dwNumVertices++] = v3;
}

// Delete the temporary edge list
delete[] pEdges;

return S_OK;
}

//-----------------------------------------------------------------------------
// Name: addEdge()
// Desc: Adds an edge to a list of silohuette edges of a shadow volume.
//-----------------------------------------------------------------------------
void ShadowVolume::addEdge( WORD* pEdges, DWORD& dwNumEdges, WORD v0, WORD v1 )
{
// Remove interior edges (which appear in the list twice)
for( DWORD i = 0; i < dwNumEdges; ++i )
{
if( ( pEdges[2*i+0] == v0 && pEdges[2*i+1] == v1 ) ||
( pEdges[2*i+0] == v1 && pEdges[2*i+1] == v0 ) )
{
if( dwNumEdges > 1 )
{
pEdges[2*i+0] = pEdges[2*(dwNumEdges-1)+0];
pEdges[2*i+1] = pEdges[2*(dwNumEdges-1)+1];
}

--dwNumEdges;
return;
}
}

pEdges[2*dwNumEdges+0] = v0;
pEdges[2*dwNumEdges+1] = v1;
dwNumEdges++;
}




void ShadowVolume::renderShadowToStencilBuffer( LPDIRECT3DDEVICE9 g_pd3dDevice, D3DXMATRIX m_mat )
{
// Disable z-buffer writes (note: z-testing still occurs), and enable the
// stencil-buffer
g_pd3dDevice->SetRenderState( D3DRS_ZWRITEENABLE, FALSE );
g_pd3dDevice->SetRenderState( D3DRS_STENCILENABLE, TRUE );

// Dont bother with interpolating color
g_pd3dDevice->SetRenderState( D3DRS_SHADEMODE, D3DSHADE_FLAT );

// Set up stencil compare fuction, reference value, and masks.
// Stencil test passes if ((ref & mask) cmpfn (stencil & mask)) is true.
// Note: since we set up the stencil-test to always pass, the STENCILFAIL
// renderstate is really not needed.
g_pd3dDevice->SetRenderState( D3DRS_STENCILFUNC, D3DCMP_ALWAYS );
g_pd3dDevice->SetRenderState( D3DRS_STENCILZFAIL, D3DSTENCILOP_KEEP );
g_pd3dDevice->SetRenderState( D3DRS_STENCILFAIL, D3DSTENCILOP_KEEP );

// If z-test passes, inc/decrement stencil buffer value
g_pd3dDevice->SetRenderState( D3DRS_STENCILREF, 0x1 );
g_pd3dDevice->SetRenderState( D3DRS_STENCILMASK, 0xffffffff );
g_pd3dDevice->SetRenderState( D3DRS_STENCILWRITEMASK, 0xffffffff );
g_pd3dDevice->SetRenderState( D3DRS_STENCILPASS, D3DSTENCILOP_INCR );

// Make sure that no pixels get drawn to the frame buffer
g_pd3dDevice->SetRenderState( D3DRS_ALPHABLENDENABLE, TRUE );
g_pd3dDevice->SetRenderState( D3DRS_SRCBLEND, D3DBLEND_ZERO );
g_pd3dDevice->SetRenderState( D3DRS_DESTBLEND, D3DBLEND_ONE );

if( g_bTwoSidedStencilsAvailable == true )
{
// With 2-sided stencil, we can avoid rendering twice:
g_pd3dDevice->SetRenderState( D3DRS_TWOSIDEDSTENCILMODE, TRUE );
g_pd3dDevice->SetRenderState( D3DRS_CCW_STENCILFUNC, D3DCMP_ALWAYS );
g_pd3dDevice->SetRenderState( D3DRS_CCW_STENCILZFAIL, D3DSTENCILOP_KEEP );
g_pd3dDevice->SetRenderState( D3DRS_CCW_STENCILFAIL, D3DSTENCILOP_KEEP );
g_pd3dDevice->SetRenderState( D3DRS_CCW_STENCILPASS, D3DSTENCILOP_DECR );

g_pd3dDevice->SetRenderState( D3DRS_CULLMODE, D3DCULL_NONE );

// Draw both sides of shadow volume in stencil/z only
g_pd3dDevice->SetTransform( D3DTS_WORLD, &m_mat );
render( g_pd3dDevice );

g_pd3dDevice->SetRenderState( D3DRS_TWOSIDEDSTENCILMODE, FALSE );
}
else
{
// Draw front-side of shadow volume in stencil/z only
g_pd3dDevice->SetTransform( D3DTS_WORLD, &m_mat );
render( g_pd3dDevice );

// Now reverse cull order so back sides of shadow volume are written.
g_pd3dDevice->SetRenderState( D3DRS_CULLMODE, D3DCULL_CW );

// Decrement stencil buffer value
g_pd3dDevice->SetRenderState( D3DRS_STENCILPASS, D3DSTENCILOP_DECR );

// Draw back-side of shadow volume in stencil/z only
g_pd3dDevice->SetTransform( D3DTS_WORLD, &m_mat );
render( g_pd3dDevice );
}

// Restore render states
g_pd3dDevice->SetRenderState( D3DRS_SHADEMODE, D3DSHADE_GOURAUD );
g_pd3dDevice->SetRenderState( D3DRS_CULLMODE, D3DCULL_CCW );
g_pd3dDevice->SetRenderState( D3DRS_ZWRITEENABLE, TRUE );
g_pd3dDevice->SetRenderState( D3DRS_STENCILENABLE, FALSE );
g_pd3dDevice->SetRenderState( D3DRS_ALPHABLENDENABLE, FALSE );
}



void ShadowVolume::renderShadowToScene( LPDIRECT3DDEVICE9 g_pd3dDevice )
{
// Set render states
g_pd3dDevice->SetRenderState( D3DRS_ZENABLE, FALSE );
g_pd3dDevice->SetRenderState( D3DRS_STENCILENABLE, TRUE );
g_pd3dDevice->SetRenderState( D3DRS_ALPHABLENDENABLE, TRUE );
g_pd3dDevice->SetRenderState( D3DRS_SRCBLEND, D3DBLEND_SRCALPHA );
g_pd3dDevice->SetRenderState( D3DRS_DESTBLEND, D3DBLEND_INVSRCALPHA );

g_pd3dDevice->SetTextureStageState( 0, D3DTSS_COLORARG1, D3DTA_TEXTURE );
g_pd3dDevice->SetTextureStageState( 0, D3DTSS_COLORARG2, D3DTA_DIFFUSE );
g_pd3dDevice->SetTextureStageState( 0, D3DTSS_COLOROP, D3DTOP_MODULATE );
g_pd3dDevice->SetTextureStageState( 0, D3DTSS_ALPHAARG1, D3DTA_TEXTURE );
g_pd3dDevice->SetTextureStageState( 0, D3DTSS_ALPHAARG2, D3DTA_DIFFUSE );
g_pd3dDevice->SetTextureStageState( 0, D3DTSS_ALPHAOP, D3DTOP_MODULATE );

// Only write where stencil value >= 1 (count indicates # of shadows that
// overlap that pixel)
g_pd3dDevice->SetRenderState( D3DRS_STENCILREF, 0x1 );
g_pd3dDevice->SetRenderState( D3DRS_STENCILFUNC, D3DCMP_LESSEQUAL );
g_pd3dDevice->SetRenderState( D3DRS_STENCILPASS, D3DSTENCILOP_KEEP );

// Draw a big, gray square
g_pd3dDevice->SetFVF( ShadowVertex::FVF_Flags );
g_pd3dDevice->SetStreamSource( 0, m_pBigSquareVB, 0, sizeof(ShadowVertex) );
g_pd3dDevice->DrawPrimitive( D3DPT_TRIANGLESTRIP, 0, 2 );

// Restore render states
g_pd3dDevice->SetRenderState( D3DRS_ZENABLE, TRUE );
g_pd3dDevice->SetRenderState( D3DRS_STENCILENABLE, FALSE );
g_pd3dDevice->SetRenderState( D3DRS_ALPHABLENDENABLE, FALSE );
}
[/code]


Here's where I actually call the shadow volume code in my render function:

[code]
void cCal3d::renderModel()
{



// get the renderer of the model
CalRenderer *pCalRenderer;
pCalRenderer = m_calModel.getRenderer();

// begin the rendering loop
if(!pCalRenderer->beginRendering()) return;

m_VBCursor=0;
m_IBCursor=0;

DWORD dwVBLockFlags=D3DLOCK_NOOVERWRITE;
DWORD dwIBLockFlags=D3DLOCK_NOOVERWRITE;



m_pD3DDevice->SetStreamSource( 0, m_pVB, 0, sizeof(MODELVERTEX) );
m_pD3DDevice->SetFVF(D3DFVF_VERTEX);
m_pD3DDevice->SetIndices(m_pIB);

shadow->reset();
shadow->CheckDual(m_pD3DDevice);


// get the number of meshes
int meshCount;
meshCount = pCalRenderer->getMeshCount();

// render all meshes of the model
int meshId;
for(meshId = 0; meshId < meshCount; meshId++)
{
// get the number of submeshes
int submeshCount;
submeshCount = pCalRenderer->getSubmeshCount(meshId);

// render all submeshes of the mesh
int submeshId;
for(submeshId = 0; submeshId < submeshCount; submeshId++)
{

// select mesh and submesh for further data access
if(pCalRenderer->selectMeshSubmesh(meshId, submeshId))
{

D3DMATERIAL9 mat;
unsigned char meshColor[4];
pCalRenderer->getAmbientColor(&meshColor[0]);
mat.Ambient.r=meshColor[0]/255.0f;mat.Ambient.g=meshColor[1]/255.0f;
mat.Ambient.b=meshColor[2]/255.0f;mat.Ambient.a=meshColor[3]/255.0f;

pCalRenderer->getDiffuseColor(&meshColor[0]);
mat.Diffuse.r=meshColor[0]/255.0f;mat.Diffuse.g=meshColor[1]/255.0f;
mat.Diffuse.b=meshColor[2]/255.0f;mat.Diffuse.a=meshColor[3]/255.0f;

pCalRenderer->getSpecularColor(&meshColor[0]);
mat.Specular.r=meshColor[0]/255.0f;mat.Specular.g=meshColor[1]/255.0f;
mat.Specular.b=meshColor[2]/255.0f;mat.Specular.a=meshColor[3]/255.0f;

mat.Power=0.0f;
mat.Emissive.r=0.0f;mat.Emissive.g=0.0f;mat.Emissive.b=0.0f;mat.Emissive.a=0.0f;


if(m_VBCursor+ pCalRenderer->getVertexCount()>=20000)
{
m_VBCursor=0;
dwVBLockFlags=D3DLOCK_DISCARD;
}

if(m_IBCursor + pCalRenderer->getFaceCount()>=40000)
{
m_IBCursor=0;
dwIBLockFlags=D3DLOCK_DISCARD;

}


// Get vertexbuffer from the model
MODELVERTEX *pVertices;
m_pVB->Lock(m_VBCursor*sizeof(MODELVERTEX), pCalRenderer->getVertexCount()*sizeof(MODELVERTEX), (void**)&pVertices, dwVBLockFlags);
int vertexCount = pCalRenderer->getVerticesNormalsAndTexCoords(&pVertices->pos.x);
m_pVB->Unlock();

CalIndex *meshFaces;
int faceCount = 0;
m_pIB->Lock(m_IBCursor* 3*sizeof(CalIndex), pCalRenderer->getFaceCount()*3* sizeof(CalIndex), (void**)&meshFaces,dwIBLockFlags);
faceCount = pCalRenderer->getFaces(meshFaces);
m_pIB->Unlock();


m_pD3DDevice->SetRenderState(D3DRS_CULLMODE,D3DCULL_NONE);

if((pCalRenderer->getMapCount() > 0))
{
LPDIRECT3DTEXTURE9 tex = (LPDIRECT3DTEXTURE9)pCalRenderer->getMapUserData(0);
m_pD3DDevice->SetTexture(0,tex);
}


if (genShadow)
{
D3DXVECTOR3 vec;
vec.x = 5;
vec.y = 5;
vec.z = 5;
D3DXMATRIX matz;
m_pD3DDevice->GetTransform(D3DTS_WORLD,&matz);

D3DXVECTOR3 vLightInWorldSpace( vec.x, vec.y, vec.z );
D3DXVECTOR3 vLightInObjectSpace;
D3DXMATRIXA16 matInverse;
D3DXMatrixInverse( &matInverse, NULL, &matz );
D3DXVec3TransformNormal( &vLightInObjectSpace, &vLightInWorldSpace, &matInverse );

//GENERATE SHADOW /////////////////////////////////////////////////////////////////////
shadow->buildShadowVolume(pVertices,meshFaces,faceCount,faceCount*3,vec);
//////////////////////////////////////////////////////////////////////////////////////

//D3DXMATRIX savMat = getMat();
//D3DXMATRIX matz;
//m_pD3DDevice->GetTransform(D3DTS_WORLD,&matz);

//D3DXMATRIX matFinal2,matShadow;D3DXMATRIX matz;
//D3DXMatrixIdentity(&matFinal2);
//D3DXMatrixIdentity(&matShadow);

//D3DXPLANE plane;
//D3DXVECTOR3 point = D3DXVECTOR3( 0.0f, -89.0f, 0.0f );
//D3DXVECTOR3 normal = D3DXVECTOR3( 0.0f, 1.0f, 0.0f );
//D3DXPlaneFromPointNormal( &plane, &point, &normal);
//D3DXVECTOR4 vlight = D3DXVECTOR4( 50.0f, 40.0f, 0.0f, 0.0f);
//D3DXMatrixShadow(&matShadow, &vlight ,&plane );
//D3DXMatrixMultiply(&matz, &matz, &matShadow);
//m_pD3DDevice->SetTransform( D3DTS_WORLD, &matz);

//D3DMATERIAL9 Materials;
//rgbaDiffuse = D3DXCOLOR(0.0, 0.0, 0.0, 0.05);
//rgbaAmbient = D3DXCOLOR(0.0, 0.0, 0.0, 0.05);
//rgbaSpecular = D3DXCOLOR(0.0, 0.0, 0.0, 0.05);
//rgbaEmissive = D3DXCOLOR(0.0, 0.0, 0.0, 0.05);

//Materials.Ambient = rgbaAmbient;
//Materials.Diffuse = rgbaDiffuse;
//Materials.Emissive = rgbaEmissive;
//Materials.Specular = rgbaSpecular;
//Materials.Power = 1.0f;

//m_pD3DDevice->SetMaterial(&Materials);
//m_pD3DDevice->SetRenderState(D3DRS_ZENABLE,TRUE);
//m_pD3DDevice->SetRenderState(D3DRS_ZWRITEENABLE, TRUE);

////DRAW
//m_pD3DDevice->DrawIndexedPrimitive(
// D3DPT_TRIANGLELIST,
// m_VBCursor,
// 0,
// vertexCount,
// m_IBCursor*3,
// faceCount
// );
//m_pD3DDevice->SetRenderState(D3DRS_ZFUNC,D3DCMP_LESSEQUAL);
//m_pD3DDevice->SetTransform( D3DTS_WORLD, &savMat);
//shadow->Render(m_pD3DDevice);

}








D3DMATERIAL9 Material;
//Set material default values (R, G, B, A)
rgbaDiffuse = D3DXCOLOR(1.0, 1.0, 1.0, 1.0);
rgbaAmbient = D3DXCOLOR(0.0, 0.0, 0.0, 1.0);
rgbaSpecular = D3DXCOLOR(0.0, 0.0, 0.0, 1.0);
rgbaEmissive = D3DXCOLOR(0.0, 0.0, 0.0, 1.0);
Material.Ambient = rgbaAmbient;
Material.Diffuse = rgbaDiffuse;
Material.Emissive = rgbaEmissive;
Material.Specular = rgbaSpecular;
Material.Power = 1.0f;

m_pD3DDevice->SetMaterial(&Material);

m_pD3DDevice->DrawIndexedPrimitive(
D3DPT_TRIANGLELIST,
m_VBCursor,
0,
vertexCount,
m_IBCursor*3,
faceCount
);


//char str[1000];
//sprintf(str,"VertCount: %i FaceCount: %i VB_CURSOR: %f MeshID: %i MeshCount: %i ---- ",vertexCount,faceCount,m_VBCursor,meshId,meshCount);
////sprintf(str," VBCURSOR: %f MeshID: %i ---- ",m_VBCursor,meshId,meshCount);
//log->LogInfo(str);

m_VBCursor+=vertexCount;
m_IBCursor+=faceCount;
}
}
}

if (genShadow)
{
D3DXVECTOR3 vec;
vec.x = 5;
vec.y = 5;
vec.z = 5;
D3DXMATRIX matz;
m_pD3DDevice->GetTransform(D3DTS_WORLD,&matz);

m_pD3DDevice->SetRenderState( D3DRS_FILLMODE, D3DFILL_WIREFRAME );
m_pD3DDevice->SetTransform( D3DTS_WORLD, &matz );
shadow->render( m_pD3DDevice );
m_pD3DDevice->SetRenderState( D3DRS_FILLMODE, D3DFILL_SOLID );



D3DXVECTOR3 vLightInWorldSpace( vec.x, vec.y, vec.z );
D3DXVECTOR3 vLightInObjectSpace;
D3DXMATRIXA16 matInverse;
D3DXMatrixInverse( &matInverse, NULL, &matz );
D3DXVec3TransformNormal( &vLightInObjectSpace, &vLightInWorldSpace, &matInverse );
shadow->renderShadowToStencilBuffer(m_pD3DDevice,matInverse);
shadow->renderShadowToScene(m_pD3DDevice);

}

// end the rendering
pCalRenderer->endRendering();





[Edited by - Coder on September 4, 2004 6:51:20 AM]

Share this post


Link to post
Share on other sites
Any other ideas? Am I the only one to have this problem :( I think it was something to do with the way I get the edges, the examples all use the DirectX Mesh, but I am using my own mesh, so I just pass the vertices\index buffer to the function to find the edges, I dunno, any ideas??

Thanks
Jeff

Share this post


Link to post
Share on other sites

This topic is 4851 days old which is more than the 365 day threshold we allow for new replies. Please post a new topic.

If you intended to correct an error in the post then please contact us.

Create an account or sign in to comment

You need to be a member in order to leave a comment

Create an account

Sign up for a new account in our community. It's easy!

Register a new account

Sign in

Already have an account? Sign in here.

Sign In Now

Sign in to follow this