I have created a shadow map

As I understand, I need to compare the current pixel depth with the shadow map, if the shadow map depth is LESS than the current pixel depth then I add shadow to the current pixel.

Now, I see half of the scene is darker (which is unexpected)

Shader

VS:

float4 oPosition = mul(float4(IN.Position, 1.0f), shadowWorldViewProj);
oPosition.x      = ((oPosition.x + oPosition.w) * ScreenParams.x + oPosition.w) * ScreenParams.z;
oPosition.y      = ((oPosition.w - oPosition.y) * ScreenParams.y + oPosition.w) * ScreenParams.w;
Out.screenPos    = oPosition;
Out.Z = mul(float4(IN.Position, 1.0f), shadowWorldView).z;

PS:

shadow = 0.0;
float scene_z = tex2Dproj(shadowMap, In.screenPos).x;
if (scene_z > 0)
{
    if (In.Z > scene_z)
        shadow = 1.0f;
}

In the PS you should recieve the (x,y,z) position of the pixel in screen space (WorldViewProj). This position needs to be transformed to shadow map space using Proj-1*View-1*SMView*SMProj*SMTexBias matrices which can be calculated outside the shader and be set as a constant. Then you can do the comparison, I think :)

@Tispe: I'm a little confused.

Can you show me any code example? I thought my above code is somehow correct.

BTW:

shadowWorldViewProj = The light worldViewProjection

shadowWorldView = The light worldView

There are several ways of doing this.

You can calculate the vertex position in both shadow map space and screen space and then pass both positions to the PS. The PS would then recieve the interpolated position both in screen space and shadow map space. With this method you calculate first the vertex pos in world space. Then you calculate two positions from that world pos using ViewProj and SMViewSMProj.

Another method is the one I described above, it has less performance but more flexibility. I think :)

@Tispe: I think I'm doing the same idea in my code

Comparing the depth in the shadow map with the current pixel depth to determine if the pixel should be darker or not

And it's still not working...

...
...
VS_OUTPUT VS( VERTEX IN )
{
     VS_OUTPUT OUT;
     OUT = (VS_OUTPUT)0;
     OUT.Position  = mul(float4(IN.Position, 1.0f), WorldViewProjection);
     OUT.UV        = IN.UV;
     OUT.Normal    = mul(float4(IN.Normal, 0.0f), World).xyz;
     float4 oPosition = mul(float4(IN.Position, 1.0f), lightWorldViewProj);
     oPosition.x      = ((oPosition.x + oPosition.w) * ScreenParams.x + oPosition.w) * ScreenParams.z;
     oPosition.y      = ((oPosition.w - oPosition.y) * ScreenParams.y + oPosition.w) * ScreenParams.w;
     OUT.screenPos    = oPosition;

     OUT.Z = mul(float4(IN.Position, 1.0f), lightWorldView).z;
     return OUT;
}

float4 PS(VS_OUTPUT IN) : COLOR
{
    float scene_z = tex2Dproj(shadowMap, IN.screenPos).x;
    if (IN.Z > scene_z)
        // Pixel in shadow
        shadow = float4(0.4f, 0.4f, 0.4f);
    else
        shadow = float4(1.0f, 1.0f, 1.0f)
    return tex2D( colorTex, IN.UV ) * shadow;
}

oPosition.x = ((oPosition.x + oPosition.w) * ScreenParams.x + oPosition.w) * ScreenParams.z;
oPosition.y = ((oPosition.w - oPosition.y) * ScreenParams.y + oPosition.w) * ScreenParams.w;

Where did you get this code? I have not seen it before.

Maybe you should remove it and apply SMTexBias to IN.screenPos in the PS.

float fTexOffs = 0.5 + (0.5 / (float)SHADOW_MAP_SIZE);
D3DXMATRIX SMTexBias( 0.5f,	 0.0f,	 0.0f, 0.0f,
					  0.0f,	 -0.5f,	0.0f, 0.0f,
					  0.0f,	 0.0f,	 1.0f, 0.0f,
					  fTexOffs, fTexOffs, 0.0f, 1.0f );

@Tispe: The code I posted is somehow close to the one that I use for soft particles, but of course the depth map is different as well as I'm comparing the pixel depth with the scene depth to determine if the pixel is in shadow

I tried using the code you mentioned, but it's still not working... what I see is that half of scene is darker instead of shadows.

Here is what I have done:

// Shadow map size: 1024 x 1024

VS_OUTPUT VS( VERTEX IN )
{
     VS_OUTPUT OUT;
     OUT = (VS_OUTPUT)0;
     OUT.Position  = mul(float4(IN.Position, 1.0f), WorldViewProjection);
     OUT.UV        = IN.UV;
     OUT.Normal    = mul(float4(IN.Normal, 0.0f), World).xyz;
     OUT.vTexCoord = mul( IN.Position, SMTexBias );
     OUT.ShadowZ = mul(float4(IN.Position, 1.0f), lightViewProj).z;
     return OUT;
}

float4 PS(VS_OUTPUT IN) : COLOR
{
    // Generate the 9 texture co-ordinates for a 3x3 PCF kernel
    float4 vTexCoords[9];
    // Texel size
    float fTexelSize = 1.0f / 1024.0f;

    // Generate the tecture co-ordinates for the specified depth-map size
    // 4 3 5
    // 1 0 2
    // 7 6 8
    vTexCoords[0] = IN.vTexCoord;
    vTexCoords[1] = IN.vTexCoord + float4( -fTexelSize, 0.0f, 0.0f, 0.0f );
    vTexCoords[2] = IN.vTexCoord + float4( fTexelSize, 0.0f, 0.0f, 0.0f );
    vTexCoords[3] = IN.vTexCoord + float4( 0.0f, -fTexelSize, 0.0f, 0.0f );
    vTexCoords[6] = IN.vTexCoord + float4( 0.0f, fTexelSize, 0.0f, 0.0f );
    vTexCoords[4] = IN.vTexCoord + float4( -fTexelSize, -fTexelSize, 0.0f, 0.0f );
    vTexCoords[5] = IN.vTexCoord + float4( fTexelSize, -fTexelSize, 0.0f, 0.0f );
    vTexCoords[7] = IN.vTexCoord + float4( -fTexelSize, fTexelSize, 0.0f, 0.0f );
    vTexCoords[8] = IN.vTexCoord + float4( fTexelSize, fTexelSize, 0.0f, 0.0f );
    // Sample each of them checking whether the pixel under test is shadowed or not
    float fShadowTerms[9];
    float fShadowTerm = 0.0f;
    for( int i = 0; i < 9; i++ )
    {
        float A = tex2Dproj( shadowMap, vTexCoords[i] ).r;
        float B = (IN.ShadowZ - 0.1f);
        // Texel is shadowed
        fShadowTerms[i] = A < B ? 0.0f : 1.0f;
        fShadowTerm += fShadowTerms[i];
    }
    // Get the average
    fShadowTerm /= 9.0f;
    return fShadowTerm;
}

I would not do any PCF before you atleast get shadows working.

@Tispe: Unfortunately, I'm still having troubles getting shadow to work.

Please check my code and let me know if there is something wrong:

Shader for rendering the final scene:

...
texture shadowMapTexture;
sampler shadowMap = sampler_state
{
    Texture     = <shadowMapTexture>;
    MinFilter = Point;
    MagFilter = Point;
    MipFilter = Point;
};

// Vertex shader
VS_OUTPUT VS( VERTEX IN )
{
     VS_OUTPUT OUT;
     OUT = (VS_OUTPUT)0;
     OUT.Position  = mul(float4(IN.Position, 1.0f), WorldViewProjection);
     OUT.UV        = IN.UV;
     OUT.Normal    = mul(float4(IN.Normal, 0.0f), World).xyz;
     OUT.vTexCoord = mul( IN.Position, g_matTexture[0] );
     OUT.Z = mul(float4(IN.Position, 1.0f), lightWorldView).z;
     return OUT;
}

// Pixel shader
float3 PS( VS_OUTPUT IN ) : COLOR
{
     float A = tex2Dproj( shadowMap, IN.vTexCoord ).r;
     float B = (IN.ShadowZ - 0.1f);
     float shadow = A < B ? 0.4f : 1.0f;
     return tex2D(colorMap, IN.UV) * shadow;
}

...
VS_OUTPUT VS( VS_INPUT IN )
{
    VS_OUTPUT OUT;
    OUT = (VS_OUTPUT)0;
    OUT.Pos = mul(float4(IN.Pos, 1.0f), WorldViewProj);
    OUT.UV = IN.UV;
    OUT.Normal    = mul(float4(IN.Normal, 0.0f), World).xyz;
    OUT.Z = mul(float4(IN.Pos, 1.0f), WorldView).z;
    return OUT;
}

PS_OUTPUT PS( VS_OUTPUT IN )
{
    OUT.Depth = IN.Z;
    OUT.Color = ...;
}

In the shadow map VS shader:

OUT.Z = mul(float4(IN.Pos, 1.0f), WorldView).z;

I think you need WorldViewProj here. But call it LightViewProj instead for proper naming convention for the second pass.

Also you need to pass two floats zw, not just z. Then in PS you need to divide z by w:

OUT.Depth = IN.z / IN.w;