What you see is directional light VSM. I think I somehow set variance value wrong. But I'm not sure. Here's the code
///////Shadow Map PS
{
OutputPSShadow outgoing = (OutputPSShadow)0;
//I did this so it stays in [0,1] range
float depth = length(incoming.view) / 1000;
float2 moment;
moment.x = depth;
float dx = ddx(depth), dy = ddy(depth);
moment.y = depth * depth + 0.25 * (dx*dx + dy*dy);
outgoing.color.r = moment.x;
outgoing.color.g = moment.y;
outgoing.color.a = 1;
return outgoing;
}
///////Shadow Casting function
{
float2 tex = incoming.mapSamplingPos.xy * float2(0.5, -0.5) + 0.5;
//if not in Shadow map Area then no shadow
if(saturate(tex.x) == tex.x && saturate(tex.y) == tex.y)
{
float disToLight = length(incoming.view) / 1000;
float lit = 0;
//Shadow Map
float2 moment = tex2D(ShadowSampler, tex).xy;
//if moment.x is zero then something's wrong so no shadow
if(moment.x == 0) return 1;
float e_x2 = moment.y;
float ex_2 = moment.x * moment.x;
float variance = e_x2 - ex_2;
float md = disToLight - moment.x;
float p = variance / (variance + md*md);
lit = max(p, disToLight <= moment.x);
return lit;
}
else return 1;
}
I just don't know where to look anymore, please help.