I wan't to render some primitive with a mask texture, one channel used to clip pixel, it works well when lighting.
this is the relevant code:

Texture2D g_MaskMap : MaskMap;
uniform float g_MaskSpecularColorMulti : MaskSpecMulti = 1.0f;
uniform float g_MaskTransmissionMaskMulti : TransmissionMaskMulti = 1.0f;
uniform float g_MaskClipValue : MaskClip = 0.3333f;
uniform float g_EnableMasked : EnableMasked = 0.0f;
float4 GetMaterialMaskMapData( FDeferredShadingPSInput Input )
{
return g_MaskMap.Sample( g_SamplerStateLinearFlit, Input.oTex.xy ).xyzw;
}
float GetMaskedValue( float4 MaskData )
{
return MaskData.r;
}
void TryClip( float2 vTex )
{
if( g_EnableMasked < 1.0 )
{
return;
}
float4 vMaskData = g_MaskMap.Sample( g_SamplerStateLinearFlit, vTex.xy ).xyzw;
float MaskValue = GetMaskedValue( vMaskData );
MaskValue -= g_MaskClipValue;
clip( MaskValue );
}
float4 StaticMeshSpotLighting( HStaticMeshMaterialVSOutput Input ) : SV_TARGET
{
TryClip( Input.oTex );
return SpotLightingShading( Input );
}

this is the final output:
[attachment=7426:2.png]

but it produce a problem in shadow map, the gap of leaf is not rendered to depth buffer, I wan't to make a depth buffer which is masked by that mask texture.
this is the depth buffer, copy from nsight.
[attachment=7427:1.png]

// this is draw shadow map code:


float4 DrawShadowMapVS( HStaticMeshPositionVertexFactory Input ):SV_Position
{
float4 vWorldPos = mul( float4(Input.vPos, 1.0f ), mWorld );
float4 oPos = mul( vWorldPos, mLightViewProj );
return oPos;
}
technique10 DrawShadowMap
{
pass p0
{
SetVertexShader ( CompileShader( vs_5_0, DrawShadowMapVS() ) );
SetGeometryShader( NULL );
SetPixelShader( NULL );
SetRasterizerState( DisableCulling );
SetDepthStencilState( ZTestLess_DS, 0x00000000 );
SetBlendState( NoBlending_BS, float4( 1.0f, 1.0f, 1.0f, 1.0f ), 0xffffffff );
}
}


I realize we can use texture sample in vertex shader in dx11, so i make a new vertex shader for make the depth value of masked vertex always 1.0, but it doesn't work.


struct FMaskedShadowMapVSOutput
{
float4 oPos:SV_Position;
float2 oTex:TEXCOORD0;
};
FMaskedShadowMapVSOutput DrawShadowMapMaskedVS( HStaticMeshPositionWithTexVertexFactory Input )
{
FMaskedShadowMapVSOutput Output = (FMaskedShadowMapVSOutput)0;
float4 vWorldPos = mul( float4(Input.vPos, 1.0f ), mWorld );
Output.oPos = mul( vWorldPos, mLightViewProj );
Output.oTex = Input.vTex;
float4 vMaskData = g_MaskMap.Load( float3(Input.vTex.xy,0) ).xyzw;
float MaskValue = GetMaskedValue( vMaskData );
MaskValue -= g_MaskClipValue;
// it always far!!!
if( MaskValue <= 0 )
{
Output.oPos.z = Output.oPos.w;
}
return Output;
}


so, i wan't to know, how to render a msked primitive into a depth buffer, when the masked value is less or equal to 0, discard that pixel, and did not write the depth value to depth buffer.
or other do you have a better way to do that.

If i have to make a render target to save depth value by my self, and then i can clip the pixel in the MakeShadowMap pixel shader?

the "clip" HLSL function ("discard" in GLSL) prevents the pixel from writing to any bound target.

this way, have a generic pixel shader for your shadow write that does the texture sample and clips the pixels as necissary. Basically, your shadow-pass shader will require a pixel shader, no way around that.

the "clip" HLSL function ("discard" in GLSL) prevents the pixel from writing to any bound target.

this way, have a generic pixel shader for your shadow write that does the texture sample and clips the pixels as necissary. Basically, your shadow-pass shader will require a pixel shader, no way around that.

thank you, i fixed it.
i have add a pixel shader, only output depth value.


struct FMaskedShadowMapPSOutput
{
float oDepth : SV_Depth;
};
FMaskedShadowMapPSOutput DrawShadowMapMaskedPixelShader( FMaskedShadowMapVSOutput Input )
{
FMaskedShadowMapPSOutput Output = (FMaskedShadowMapPSOutput)0;
TryClip( Input.oTex );

float z = Input.oPos.z;

Output.oDepth = z;
return Output;
}


technique10 DrawShadowMapMasked
{
pass p0
{
SetVertexShader ( CompileShader( vs_5_0, DrawShadowMapMaskedVS() ) );
SetGeometryShader( NULL );
SetPixelShader( CompileShader( ps_5_0, DrawShadowMapMaskedPixelShader() ) );

SetRasterizerState( DisableCulling );
SetDepthStencilState( ZTestLess_DS, 0x00000000 );
SetBlendState( NoBlending_BS, float4( 1.0f, 1.0f, 1.0f, 1.0f ), 0xffffffff );
}
}



You don't have to output SV_Depth. In fact, you really don't want to since it will just make it run slower. Just return SV_Target0, and return a value of 0 from your pixel shader. The only important part of your pixel shader is the part that calls clip or discard.

thank you, [color=#000000][font=arial]

You are really a warm-hearted man![/font]
http://www.gamedev.net/topic/620609-dose-all-primitive-in-the-viewport-will-execute-its-pixel-shader/


---------------------------
if i output depth to SV_Target0, does that mean, i will create a render target to do that,?
i think the depth buffer is necessary because it will help me to clip the shaded pixel, right?