Hello,

I am trying to use Frank Luna's great, great shader book to render terrain with billboarded trees that are alpha tested, but also use shadow maps to draw some nice, leafy shadows under the trees. It's an age old problem, and one I am unable to find a solution for after googling for a few days. At least now I've narrowed it down to alpha test and not alpha blend, but still not working. What I think should show as alpha blended shadows on the terrain instead turns up as solid black rectangles.

void ShadowMapDemo::drawShadowMap()
{
	mShadowMap->beginScene();
	HR(gd3dDevice->Clear(0, 0, D3DCLEAR_TARGET | D3DCLEAR_ZBUFFER, 0x00000000, 1.0f, 0));
 
	HR(mFX->SetTechnique(mhBuildShadowMapTech));

	
	UINT numPasses = 0;
	HR(mFX->Begin(&numPasses, 0));
	HR(mFX->BeginPass(0));

	// Draw scene mesh.
	HR(mFX->SetMatrix(mhLightWVP, &(mSceneWorld*mLightVP)));
	HR(mFX->CommitChanges());
	for(UINT j = 0; j < mSceneMtrls.size(); ++j)
	{
		for(UINT j = 0; j < mSceneMtrls.size(); ++j)
		{
			HR(mSceneMesh->DrawSubset(j));
		}
	}

	// Draw car mesh. (substituted w/ a tree with trunk and billboard leaves/branches)

	HR(mFX->SetMatrix(mhLightWVP, &(mCarWorld*mLightVP)));
	HR(mFX->CommitChanges());

	HR(gd3dDevice->SetRenderState(D3DRS_ALPHATESTENABLE, true));
	HR(gd3dDevice->SetRenderState(D3DRS_ALPHAFUNC, D3DCMP_GREATEREQUAL));
	HR(gd3dDevice->SetRenderState(D3DRS_ALPHAREF, 200));

	for(UINT j = 0; j < mCarMtrls.size(); ++j)
	{
		for(UINT j = 0; j < mCarMtrls.size(); ++j)
		{
			HR(mCarMesh->DrawSubset(j));
		}
	}

        HR(gd3dDevice->SetRenderState(D3DRS_ALPHATESTENABLE, false));

	HR(mFX->EndPass());
	HR(mFX->End());

	mShadowMap->endScene();
}

Here is the .fx file

//=============================================================================
// LightShadow.fx by Frank Luna (C) 2004 All Rights Reserved.
//
// Does lighting and shadowing with shadow maps.
//=============================================================================

uniform extern float4x4 gLightWVP;

static const float SHADOW_EPSILON = 0.00005f;
static const float SMAP_SIZE = 512.0f;

void BuildShadowMapVS(float3 posL : POSITION0,
                      out float4 posH : POSITION0,
                      out float2 depth : TEXCOORD0)
{
	// Render from light's perspective.
	posH = mul(float4(posL, 1.0f), gLightWVP);
	
	// Propagate z- and w-coordinates.
	depth = posH.zw;
}

float4 BuildShadowMapPS(float2 depth : TEXCOORD0) : COLOR
{
	// Each pixel in the shadow map stores the pixel depth from the 
	// light source in normalized device coordinates.
	return depth.x / depth.y; // z / w
}

technique BuildShadowMapTech
{
	pass P0
	{
		vertexShader = compile vs_2_0 BuildShadowMapVS();
        pixelShader  = compile ps_2_0 BuildShadowMapPS();
	}
}

struct Mtrl
{
	float4 ambient;
	float4 diffuse;
	float4 spec;
	float  specPower;
};

struct SpotLight
{
	float4 ambient;
	float4 diffuse;
	float4 spec;
	float3 posW;
	float3 dirW;  
	float  spotPower;
};

uniform extern float4x4  gWorld;
uniform extern float4x4  gWVP;
uniform extern Mtrl      gMtrl;
uniform extern SpotLight gLight;
uniform extern float3    gEyePosW;
uniform extern texture   gTex;
uniform extern texture   gShadowMap;
 
sampler TexS = sampler_state
{
	Texture = <gTex>;
	MinFilter = LINEAR;
	MagFilter = LINEAR;
	MipFilter = LINEAR;
	AddressU  = WRAP; 
    AddressV  = WRAP;
};

sampler ShadowMapS = sampler_state
{
	Texture = <gShadowMap>;
	MinFilter = POINT;
	MagFilter = POINT;
	MipFilter = POINT;
	AddressU  = CLAMP; 
    AddressV  = CLAMP;
};

void LightShadowVS(float3 posL         : POSITION0,
                   float3 normalL      : NORMAL0,
                   float2 tex0         : TEXCOORD0,
                   out float4 oPosH    : POSITION0,
                   out float3 oPosW    : TEXCOORD0,
                   out float3 oNormalW : TEXCOORD1,
                   out float3 oToEyeW  : TEXCOORD2,
                   out float2 oTex0    : TEXCOORD3,
                   out float4 oProjTex : TEXCOORD4)
{
	// Transform to homogeneous clip space.
	oPosH = mul(float4(posL, 1.0f), gWVP);
	
	// Transform vertex position to world space.
	oPosW = mul(float4(posL, 1.0f), gWorld).xyz;
	
	// Transform normal to world space (assume no non-uniform scaling).
	oNormalW = mul(float4(normalL, 0.0f), gWorld).xyz;
	
	// Compute the unit vector from the vertex to the eye.
	oToEyeW = gEyePosW - oPosW;
	
	// Pass on texture coords to PS
	oTex0 = tex0;
	
	// Generate projective texture coordinates.
	oProjTex = mul(float4(posL, 1.0f), gLightWVP);
}

float4 LightShadowPS(float3 posW    : TEXCOORD0,
                     float3 normalW : TEXCOORD1,
                     float3 toEyeW  : TEXCOORD2,
                     float2 tex0    : TEXCOORD3,
                     float4 projTex : TEXCOORD4) : COLOR
{
	// Interpolated normals can become unnormal--so normalize.
	normalW = normalize(normalW);
	toEyeW  = normalize(toEyeW);
	
	// Light vector is from pixel to spotlight position.
	float3 lightVecW = normalize(gLight.posW - posW);
	
	// Compute the reflection vector.
	float3 r = reflect(-lightVecW, normalW);
	
	// Determine how much (if any) specular light makes it into the eye.
	float t  = pow(max(dot(r, toEyeW), 0.0f), gMtrl.specPower);
	
	// Determine the diffuse light intensity that strikes the vertex.
	float s = max(dot(lightVecW, normalW), 0.0f);
	
	// Compute the ambient, diffuse and specular terms separately. 
	float3 spec = t*(gMtrl.spec*gLight.spec).rgb;
	float3 diffuse = s*(gMtrl.diffuse*gLight.diffuse.rgb);
	float3 ambient = gMtrl.ambient*gLight.ambient;
	
	// Compute spotlight coefficient.
	float spot = pow(max( dot(-lightVecW, gLight.dirW), 0.0f), gLight.spotPower);
	
	// Sample decal map.
	float4 texColor = tex2D(TexS, tex0); 
	
	// Project the texture coords and scale/offset to [0, 1].
	projTex.xy /= projTex.w;            
	projTex.x =  0.5f*projTex.x + 0.5f; 
	projTex.y = -0.5f*projTex.y + 0.5f;
	
	// Compute pixel depth for shadowing.
	float depth = projTex.z / projTex.w;
 
	// Transform to texel space
    float2 texelpos = SMAP_SIZE * projTex.xy;
        
    // Determine the lerp amounts.           
    float2 lerps = frac( texelpos );
    
    // 2x2 percentage closest filter.
    float dx = 1.0f / SMAP_SIZE;
	float s0 = (tex2D(ShadowMapS, projTex.xy).r + SHADOW_EPSILON < depth) ? 0.0f : 1.0f;
	float s1 = (tex2D(ShadowMapS, projTex.xy + float2(dx, 0.0f)).r + SHADOW_EPSILON < depth) ? 0.0f : 1.0f;
	float s2 = (tex2D(ShadowMapS, projTex.xy + float2(0.0f, dx)).r + SHADOW_EPSILON < depth) ? 0.0f : 1.0f;
	float s3 = (tex2D(ShadowMapS, projTex.xy + float2(dx, dx)).r   + SHADOW_EPSILON < depth) ? 0.0f : 1.0f;
	
	float shadowCoeff = lerp( lerp( s0, s1, lerps.x ),
                              lerp( s2, s3, lerps.x ),
                              lerps.y );
	
	// Light/Texture pixel.  Note that shadow coefficient only affects diffuse/spec.
	
	float3 litColor = spot*ambient*texColor.rgb + spot*shadowCoeff*(diffuse*texColor.rgb + spec);
	
	
	
	
	return float4(litColor, gMtrl.diffuse.a*texColor.a);
}

technique LightShadowTech
{
    pass P0
    {
        // Specify the vertex and pixel shader associated with this pass.
        vertexShader = compile vs_2_0 LightShadowVS();
        pixelShader  = compile ps_2_0 LightShadowPS();
       
        
    }
}

The tree draws fine with alpha blended leaves...it's the shadow that shows up as the quad based billboard. I know I have to do some type of test...maybe in the shader. Not too sure. Anyone know how to do this?

You have to sample the leaf texture and somehow output the texel alpha, either by multiplying it by your depth value (if you're using float textures) or just output it straightaway (if you're using a depth texture / PCF).

BTW, here are some useful values to use for alpha test reference value with alpha test func = greater:

0xBF - render when one of four pixels is color key value

0x7F - render when two of four pixels is color key value

0x3F - render when three of four pixels is color key value

I got something working...at least closer. I have clipped out the alpha tested shadow parts on the billboards themselves...not on the actual shadow area! I disabled the alpha testing in the .cpp code when rendering the tree to verify that I am getting an alpha channel SOMETHING on the billboards (in the image, the pink circle)...but I am trying to get the same thing to happen on the ground (the green circle). For the record, I do render the billboards alpha tested properly (the black is turned off and all you see are the leaves)...but I did that to show that I am getting the shadow to at least appear...just not where I want it to appear.

I added to the LightShadowPS function the following code:

        const float fAlphaTestValue = .01f;
	float4 textureSample = tex2D(TexS, projTex.xy);
	clip(textureSample.a - fAlphaTestValue);

So maybe I need to replace TexS with ShadowMapS...however, when I do that, I see zero change in anything.

[attachment=14534:shadowmapwrong.jpg]

Well, for now I have a placeholder of a 2k polycount mesh with a bunch of random poly's that I render in place of the TRUE shadow map (an alpha channel billboard). I don't know how much of a hit I am taking performance wise, because I could never get the billboard to cast a shadow with a bunch of holes in it. Using a 300 poly count placeholder mesh with which to render the shadows, I get about 200+ more FPS. So I know using a placeholder mesh will give me a big hit as opposed to 20 quads (billboards).

With that being said...I would really appreciate it if somebody knows ANYTHING about terrain engines, and shadow maps, and billboards, and HLSL, could look at the code and maybe take a stab at solving it. HELP!! Good grief, all over google I see this in terrain engines. I already asked Danny Green, and he has given me great help...but I am stuck. Shaders...ugh.

You need to turn on alpha testing when you render the billboard into the shadow map, and try changing your fAlphaTestValue to something like 0.5, that should reject all pixels that are half or more transparent.

Hi Hupsilardee,

I set the value to .5...but where do I turn on alpha testing? I'm not even sure where when I am rendering into the billboard. I am that new to shaders. Sorry.

I should also mention I am doing the clip test (w .5 alpatestvalue) in function LightShadowPS of the .fx file.

Ok. Can you post the actual image file used as texture?

Sure...it's a .dds (that might be the problem, too). I can't upload the .dds file, but here is the mipmipped image (and alpha channel) in two separate jpegs. I noticed this morning the technique I am using for drawing the shadow is mhBuildShadowMapTech, which corresponds to BuildShadowMapTech in the .fx file...the PS for that simply returns the depth.x/depth.y.

I notice that I am working in LightShadowPS, which is the technique set when drawing the tree, not the shadow. The tree does draw OK with alpha testing turned on. The black is definitely masked out. It's when I try to draw that shadow that I see the billboards. I apologize for not knowing which way is up when it comes to shaders. It could be that I need to add some code to the BuildShadowMapTech part, instead of returning just a depth. I am at a loss. Thanks for looking at it!!