Combine HLSL shaders

117

Author

June 01, 2009 12:09 PM

Hi! I'm trying to implement nice toon outlines to my model. I have found edge detection shader on NVIDIA: http://developer.download.nvidia.com/shaderlibrary/packages/scene_toonEdges.fx.zip Here are its contents:


/*********************************************************************NVMH3****
File:  $Id: //sw/devtools/ShaderLibrary/main/HLSL/scene_toonEdges.fx#1 $

Copyright NVIDIA Corporation 2004-2007
TO THE MAXIMUM EXTENT PERMITTED BY APPLICABLE LAW, THIS SOFTWARE IS PROVIDED
*AS IS* AND NVIDIA AND ITS SUPPLIERS DISCLAIM ALL WARRANTIES, EITHER EXPRESS
OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, IMPLIED WARRANTIES OF MERCHANTABILITY
AND FITNESS FOR A PARTICULAR PURPOSE.  IN NO EVENT SHALL NVIDIA OR ITS SUPPLIERS
BE LIABLE FOR ANY SPECIAL, INCIDENTAL, INDIRECT, OR CONSEQUENTIAL DAMAGES
WHATSOEVER (INCLUDING, WITHOUT LIMITATION, DAMAGES FOR LOSS OF BUSINESS PROFITS,
BUSINESS INTERRUPTION, LOSS OF BUSINESS INFORMATION, OR ANY OTHER PECUNIARY LOSS)
ARISING OUT OF THE USE OF OR INABILITY TO USE THIS SOFTWARE, EVEN IF NVIDIA HAS
BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.

% Toony stuff. Two kinds of edge detection combined: normals and
% depth edge detection, resulting in clean predictable lines.

keywords: image_processing animation bumpmap

******************************************************************************/

float Script : STANDARDSGLOBAL &lt;
	string UIWidget = "none";
	string ScriptClass = "scene";
	string ScriptOrder = "standard";
	string ScriptOutput = "color";
	string Script = "Technique=Main;";
&gt; = 0.8; // version #

float4x4 WorldITXf : WorldInverseTranspose &lt; string UIWidget="None"; &gt;;
float4x4 WorldViewProjectionXf : WorldViewProjection &lt; string UIWidget="None"; &gt;;
float4x4 WorldViewXf : WorldView &lt; string UIWidget="None"; &gt;;
float4x4 WorldXf : World &lt; string UIWidget="None"; &gt;;
float4x4 ViewIXf : ViewInverse &lt; string UIWidget="None"; &gt;;

float2 ScreenSize : VIEWPORTPIXELSIZE &lt; string UIWidget="None"; &gt;;

///////////////////////////////////////////////////////////
/////////////////////////////////////// Tweakables ////////
///////////////////////////////////////////////////////////

float4 ClearColor &lt;
	string UIWidget = "none";
	//string UIWidget = "color";
	string UIName = "Clear Color";
&gt; = {0,0,0,0.0};

float4 BlackColor &lt;
	string UIName = "BG";
	string UIWidget = "color";
	//string UIWidget = "none";
	//string UIName = "background 2";
&gt; = {0,0,0,0.0};

float ClearDepth &lt;string UIWidget = "none";&gt; = 1.0;

#include "include\\Quad.fxh"

float NPixels &lt;
    string UIName = "Edge Pixels Steps";
    string UIWidget = "slider";
    float UIMin = 1.0f;
    float UIMax = 5.0f;
    float UIStep = 0.5f;
&gt; = 1.5f;

float Threshhold &lt;
    string UIName = "Edge Threshhold";
    string UIWidget = "slider";
    float UIMin = 0.001f;
    float UIMax = 0.1f;
    float UIStep = 0.001f;
&gt; = 0.2;

float ThreshholdD &lt;
    string UIName = "Depth Threshhold";
    string UIWidget = "slider";
    float UIMin = 0.0001f;
    float UIMax = 0.1f;
    float UIStep = 0.0001f;
&gt; = 0.2;

float Far &lt;
    string UIName = "Far Depth";
    string UIWidget = "slider";
    float UIMin = 0.1f;
    float UIMax = 100.5f;
    float UIStep = 0.01f;
&gt; = 10.2;

float Near &lt;
    string UIName = "Near Depth";
    string UIWidget = "slider";
    float UIMin = 0.01f;
    float UIMax = 50.5f;
    float UIStep = 0.01f;
&gt; = 1.2;

float3 LightDir : Direction &lt;
    string Object = "DirectionalLight";
    string Space = "World";
&gt; = {-10.0f, 10.0f, -10.0f};

///////////////////////////////////////////////////////////
///////////////////////////// Render-to-Texture Data //////
///////////////////////////////////////////////////////////

DECLARE_QUAD_TEX(NormTexture,NormSampler,"X8R8G8B8")
DECLARE_QUAD_TEX(DeepTexture,DeepSampler,"X8R8G8B8")
DECLARE_QUAD_TEX(NEdgeTexture,NEdgeSampler,"X8R8G8B8")
DECLARE_QUAD_DEPTH_BUFFER(DepthBuffer, "D24S8")

//DECLARE_QUAD_TEX(UVTexture,UVSampler,"X8R8G8B8")

// QUAD_REAL Time : TIME &lt;string UIWidget="None";&gt;;

texture FileTexture : Diffuse
&lt;
	string ResourceName = "default_color.dds";
&gt;;

sampler FileSampler = sampler_state 
{
    texture = &lt;FileTexture&gt;;
    AddressU  = CLAMP;        
    AddressV  = CLAMP;
    AddressW  = CLAMP;
    MIPFILTER = LINEAR;
    MINFILTER = LINEAR;
    MAGFILTER = LINEAR;
};

//////////////////////////// geometric data ///////

/* data from application vertex buffer */
struct appdata {
    float3 Pos    : POSITION;
    float4 UV        : TEXCOORD0;
    float4 Normal    : NORMAL;
    float4 Tangent    : TANGENT0;
    float4 Binormal    : BINORMAL0;
};


/* data passed from vertex shader to pixel shader */
struct vertexOutput {
    float4 HPosition	: POSITION;
    float2 TexCoord		: TEXCOORD0;
    float3 LightVec		: TEXCOORD1;
    float3 WorldNormal	: TEXCOORD2;
    float3 WorldEyeVec	: TEXCOORD3;
    float3 WorldTangent	: TEXCOORD4;
    float3 WorldBinorm	: TEXCOORD5;
    float3 EyePos	: TEXCOORD6;
	// float4 Diff			: COLOR0;
};

vertexOutput simpleVS(appdata IN) {
    vertexOutput OUT = (vertexOutput)0;
    float4 Po = float4(IN.Pos,1.0);
    OUT.HPosition = mul(Po, WorldViewProjectionXf);
    // OUT.HPosition = mul(WorldViewProjectionXf, Po);
    OUT.WorldNormal = mul(WorldITXf, IN.Normal).xyz;
    OUT.WorldTangent = mul(WorldITXf, IN.Tangent).xyz;
    OUT.WorldBinorm = mul(WorldITXf, IN.Binormal).xyz;
    float4 Pw = mul(WorldXf, Po);
    OUT.LightVec = -LightDir.xyz;
    OUT.TexCoord = IN.UV.xy+float2(0,1); // hack for DUSK model - kb
    OUT.EyePos = mul(Po,WorldViewXf);
    OUT.WorldEyeVec = normalize(ViewIXf[3].xyz - Pw.xyz);
    return OUT;
}

/// geom pixel shaders ////

float4 vecColorN(float3 V) {
    float3 Nc = 0.5 * (normalize(V.xyz) + ((1.0).xxx));
    return float4(Nc,1);
}

float4 normPS(vertexOutput IN)   : COLOR { return vecColorN(IN.WorldNormal); }
float4 uvPS(vertexOutput IN)   : COLOR { return float4(IN.TexCoord.xy,0,1); }
float4 deepPS(vertexOutput IN)   : COLOR {
    float d = (IN.EyePos.z-Near)/(Far-Near);
	return float4(d.xxx,1);
	
}
float4 texPS(vertexOutput IN)   : COLOR {
	float3 Ln = IN.LightVec;
	float3 Nn = normalize(IN.WorldNormal.xyz);
	float3 Vn = normalize(IN.WorldEyeVec.xyz);
	Nn = faceforward(Nn,-Vn,Nn);
	float ldn = dot(Ln,Nn);
	ldn = max(ldn,0); // 0.5 + ldn*0.5;
	return float4(ldn.xxx,1)*tex2D(FileSampler,IN.TexCoord.xy);
}

/////// edge detection //////////////

struct EdgeVertexOutput
{
   	QUAD_REAL4 Position	: POSITION;
    QUAD_REAL2 UV00		: TEXCOORD0;
    QUAD_REAL2 UV01		: TEXCOORD1;
    QUAD_REAL2 UV02		: TEXCOORD2;
    QUAD_REAL2 UV10		: TEXCOORD3;
    QUAD_REAL2 UV12		: TEXCOORD4;
    QUAD_REAL2 UV20		: TEXCOORD5;
    QUAD_REAL2 UV21		: TEXCOORD6;
    QUAD_REAL2 UV22		: TEXCOORD7;
};

EdgeVertexOutput edgeVS(
		QUAD_REAL3 Position : POSITION, 
		QUAD_REAL3 TexCoord : TEXCOORD0
) {
    EdgeVertexOutput OUT;
    OUT.Position = QUAD_REAL4(Position, 1);
	QUAD_REAL2 off = QUAD_REAL2(QuadTexOffset/(QuadScreenSize.x),QuadTexOffset/(QuadScreenSize.y));
    QUAD_REAL2 ctr = QUAD_REAL2(TexCoord.xy+off); 
	QUAD_REAL2 ox = QUAD_REAL2(NPixels/QuadScreenSize.x,0.0);
	QUAD_REAL2 oy = QUAD_REAL2(0.0,NPixels/QuadScreenSize.y);
	OUT.UV00 = ctr - ox - oy;
	OUT.UV01 = ctr - oy;
	OUT.UV02 = ctr + ox - oy;
	OUT.UV10 = ctr - ox;
	OUT.UV12 = ctr + ox;
	OUT.UV20 = ctr - ox + oy;
	OUT.UV21 = ctr + oy;
	OUT.UV22 = ctr + ox + oy;
    return OUT;
}

//////////////////////////////////////////////////////
////////////////////////////////// pixel shader //////
//////////////////////////////////////////////////////

QUAD_REAL getGray(QUAD_REAL4 c)
{
    return(dot(c.rgb,((0.33333).xxx)));
}

QUAD_REAL4 edgeDetectPS(EdgeVertexOutput IN,
	uniform sampler2D ColorMap,
	uniform QUAD_REAL T2
) : COLOR {
	QUAD_REAL4 CC;
	CC = tex2D(ColorMap,IN.UV00); QUAD_REAL g00 = getGray(CC);
	CC = tex2D(ColorMap,IN.UV01); QUAD_REAL g01 = getGray(CC);
	CC = tex2D(ColorMap,IN.UV02); QUAD_REAL g02 = getGray(CC);
	CC = tex2D(ColorMap,IN.UV10); QUAD_REAL g10 = getGray(CC);
	CC = tex2D(ColorMap,IN.UV12); QUAD_REAL g12 = getGray(CC);
	CC = tex2D(ColorMap,IN.UV20); QUAD_REAL g20 = getGray(CC);
	CC = tex2D(ColorMap,IN.UV21); QUAD_REAL g21 = getGray(CC);
	CC = tex2D(ColorMap,IN.UV22); QUAD_REAL g22 = getGray(CC);
	QUAD_REAL sx = 0;
	sx -= g00;
	sx -= g01 * 2;
	sx -= g02;
	sx += g20;
	sx += g21 * 2;
	sx += g22;
	QUAD_REAL sy = 0;
	sy -= g00;
	sy += g02;
	sy -= g10 * 2;
	sy += g12 * 2;
	sy -= g20;
	sy += g22;
	QUAD_REAL dist = (sx*sx+sy*sy);
	QUAD_REAL result = 0;
	if (dist&gt;T2) { result = 1; }
	return result.xxxx;
}

QUAD_REAL4 edgeDetect2PS(EdgeVertexOutput IN,
	uniform sampler2D ColorMap,
	uniform sampler2D PrevPass,
	uniform QUAD_REAL T2
) : COLOR {
	QUAD_REAL4 CC;
	CC = tex2D(ColorMap,IN.UV00); QUAD_REAL g00 = getGray(CC);
	CC = tex2D(ColorMap,IN.UV01); QUAD_REAL g01 = getGray(CC);
	CC = tex2D(ColorMap,IN.UV02); QUAD_REAL g02 = getGray(CC);
	CC = tex2D(ColorMap,IN.UV10); QUAD_REAL g10 = getGray(CC);
	CC = tex2D(ColorMap,IN.UV12); QUAD_REAL g12 = getGray(CC);
	CC = tex2D(ColorMap,IN.UV20); QUAD_REAL g20 = getGray(CC);
	CC = tex2D(ColorMap,IN.UV21); QUAD_REAL g21 = getGray(CC);
	CC = tex2D(ColorMap,IN.UV22); QUAD_REAL g22 = getGray(CC);
	QUAD_REAL sx = 0;
	sx -= g00;
	sx -= g01 * 2;
	sx -= g02;
	sx += g20;
	sx += g21 * 2;
	sx += g22;
	QUAD_REAL sy = 0;
	sy -= g00;
	sy += g02;
	sy -= g10 * 2;
	sy += g12 * 2;
	sy -= g20;
	sy += g22;
	QUAD_REAL dist = (sx*sx+sy*sy);
	QUAD_REAL result = 0;
	if (dist&gt;T2) { result = 1; }
	QUAD_REAL prev = tex2D(PrevPass,IN.UV10).x; // should be UV11 but not in ps_2
	QUAD_REAL line = 1 - (result*prev);
	return line.xxxx;
}


QUAD_REAL4 overlayPS(QuadVertexOutput IN,
					uniform sampler2D LineSampler,
					uniform sampler2D ColorSampler) : COLOR
{   
	QUAD_REAL4 line = tex2D(LineSampler, IN.UV);
	QUAD_REAL4 col = tex2D(ColorSampler, IN.UV);
	return line*col;
}  

////////////////////////////////////////////////////////////
/////////////////////////////////////// techniques /////////
////////////////////////////////////////////////////////////

technique Main &lt;
	string Script =
			// these three could be a single MRT pass
        	"Pass=Norms;"
        	// "Pass=UVs;"
        	"Pass=Depth;"
        	"Pass=Nedge;"
        	"Pass=Tex;"
        	"Pass=ImageProc;"
        	"Pass=Overlay";
&gt; {
	pass Norms &lt;
    	string Script = "RenderColorTarget0=NormTexture;"
						"RenderDepthStencilTarget=DepthBuffer;"
						"ClearSetColor=BlackColor;"
						"ClearSetDepth=ClearDepth;"
						"Clear=Color;"
						"Clear=Depth;"
						"Draw=Geometry;";
	&gt; {
		VertexShader = compile vs_2_0 simpleVS();
		ZEnable = true;
		ZWriteEnable = true;
		CullMode = None;
		AlphaBlendEnable = false;
		PixelShader = compile ps_2_a normPS();
	}
	pass UVs &lt;
    	string Script = "RenderColorTarget0=UVTexture;"
						"RenderDepthStencilTarget=DepthBuffer;"
						"ClearSetColor=BlackColor;"
						"ClearSetDepth=ClearDepth;"
						"Clear=Color;"
						"Clear=Depth;"
						"Draw=Geometry;";
	&gt; {
		VertexShader = compile vs_2_0 simpleVS();
		ZEnable = true;
		ZWriteEnable = true;
		CullMode = None;
		AlphaBlendEnable = false;
		PixelShader = compile ps_2_a uvPS();
	}
	pass Depth &lt;
    	string Script = "RenderColorTarget0=DeepTexture;"
						"RenderDepthStencilTarget=DepthBuffer;"
						"ClearSetColor=BlackColor;"
						"ClearSetDepth=ClearDepth;"
						"Clear=Color;"
						"Clear=Depth;"
						"Draw=Geometry;";
	&gt; {
		VertexShader = compile vs_2_0 simpleVS();
		ZEnable = true;
		ZWriteEnable = true;
		CullMode = None;
		AlphaBlendEnable = false;
		PixelShader = compile ps_2_a deepPS();
	}
    pass Nedge &lt;
    	string Script = "RenderColorTarget0=NEdgeTexture;"
						"RenderDepthStencilTarget=DepthBuffer;"
						"Draw=Buffer;";
    &gt; {
		cullmode = none;
		ZEnable = false;
		ZWriteEnable = false;
		AlphaBlendEnable = false;
		VertexShader = compile vs_1_1 edgeVS();
		PixelShader = compile ps_2_0 edgeDetectPS(NormSampler,(Threshhold*Threshhold));
    }
	pass Tex &lt;
    	string Script = "RenderColorTarget0=NormTexture;"		// re-use Norm texture
						"RenderDepthStencilTarget=DepthBuffer;"
						"ClearSetColor=BlackColor;"
						"ClearSetDepth=ClearDepth;"
						"Clear=Color;"
						"Clear=Depth;"
						"Draw=Geometry;";
	&gt; {
		VertexShader = compile vs_2_0 simpleVS();
		ZEnable = true;
		ZWriteEnable = true;
		CullMode = None;
		AlphaBlendEnable = false;
		PixelShader = compile ps_2_a texPS();
	}
    pass ImageProc &lt;
    	string Script = "RenderColorTarget0=DeepTexture;"	// re-use
						"RenderDepthStencilTarget=DepthBuffer;"
								"Draw=Buffer;";
    &gt; {
		cullmode = none;
		ZEnable = false;
		ZWriteEnable = false;
		AlphaBlendEnable = false;
		VertexShader = compile vs_1_1 edgeVS();
		PixelShader = compile ps_2_0 edgeDetect2PS(DeepSampler,NEdgeSampler,(ThreshholdD*ThreshholdD));
    }
    pass Overlay &lt;
    	string Script = "RenderColorTarget0=;"
						"RenderDepthStencilTarget=;"
								"Draw=Buffer;";
    &gt; {
		cullmode = none;
		ZEnable = false;
		ZWriteEnable = false;
		AlphaBlendEnable = false;
		VertexShader = compile vs_1_1 ScreenQuadVS();
		PixelShader = compile ps_2_0 overlayPS(NormSampler,DeepSampler);
    }
}

////////////// eof ///

But I'm using animated model with bones, and in order to process animated model, I have to use this shader:


// These are required by the Animation library
    float4x4 MatrixPalette[56];
    float4x4 World;

    // These are not
    float4x4 View;
    float4x4 Projection;
    texture BasicTexture;

    // Stores sampler state info for the texture
    sampler TextureSampler = sampler_state
    {
       Texture = (BasicTexture);
    };
    // This is passed into our vertex shader from Xna
struct VS_INPUT
    {
        // This is the position of the vertex in the model file
	float4 position : POSITION;
        // The vertex normal
	float3 normal : NORMAL0;
	// This is the texture coordinate for the vertex in the model file
	float2 texcoord : TEXCOORD0;
	// These are the indices (4 of them) that index the bones that affect
// this vertex.  The indices refer to the MatrixPalette.
	half4 indices : BLENDINDICES0;
	// These are the weights (4 of them) that determine how much each bone
// affects this vertex.
	float4 weights : BLENDWEIGHT0;
    };
    // This is passed out from our vertex shader once we have processed the input
struct VS_OUTPUT
    {
        // The final position of the vertex in world space
	float4 position : POSITION;
	// The texture coordinate associated with the vertex
	float2 texcoord : TEXCOORD0;
    };
    // This is the output from our skinning method
struct SKIN_OUTPUT
    {
        float4 position;
        float4 normal;
    };
   // This method takes in a vertex and applies the bone transforms to it.
    SKIN_OUTPUT Skin4( const VS_INPUT input)
    {
        SKIN_OUTPUT output = (SKIN_OUTPUT)0;
        // Since the weights need to add up to one, store 1.0 - (sum of the weights)
float lastWeight = 1.0;
        float weight = 0;
        // Apply the transforms for the first 3 weights
for (int i = 0; i &lt; 3; ++i)
        {
            weight = input.weights;
            lastWeight -= weight;
            output.position     += mul( input.position, MatrixPalette[input.indices]) * weight;
            output.normal       += mul( input.normal, MatrixPalette[input.indices]) * weight;
        }
        // Apply the transform for the last weight
        output.position     += mul( input.position, MatrixPalette[input.indices[3]])*lastWeight;
        output.normal       += mul( input.normal, MatrixPalette[input.indices[3]])*lastWeight;
        return output;
    };
    void TransformVertex (in VS_INPUT input, out VS_OUTPUT output)
    {
        // Calculate the skinned position
        SKIN_OUTPUT skin = Skin4(input);
        // This is the final position of the vertex, and where it will be drawn on the screen
        float4x4 WorldViewProjection = mul(World,mul(View,Projection));
        output.position = mul(skin.position, WorldViewProjection);
        // This is not used by is included to demonstrate how to get the normal in world space
        float4 transformedNormal = mul(skin.normal, WorldViewProjection);
        output.texcoord = input.texcoord;
    }
    // This is passed into our pixel shader
struct PS_INPUT
    {
        float2 texcoord : TEXCOORD0;
    };
    // This is the final color rendered on the screen
struct PS_OUTPUT
    {
        float4 color : COLOR;
    };
   void TransformPixel (in PS_INPUT input, out PS_OUTPUT output)
    {
        output.color = tex2D(TextureSampler,input.texcoord);
    }
  technique TransformTechnique
    {
        pass P0
        {
            VertexShader = compile vs_2_0 TransformVertex();
            PixelShader  = compile ps_2_0 TransformPixel();
        }
    }

I have tried to combine them together, but thus stuff is so complex!! In what part of the first nvidia shader should I put all those bones and wights? Firstly I tried replacing all vertex shaders, but all this input-output stuff and strcuts do not match. Any help on this? Regards, Konstantin. [Edited by - jollyjeffers on June 1, 2009 12:26:28 PM]

swiftcoder

18,997

June 01, 2009 12:20 PM

Use [source] [/source] tags to post code - it will be syntax highlighted, and we won't have to scroll past a wall of text.

Now, when it comes to shaders, you need to start small. Get yourself a book, or failing that, there are plenty of tutorials on the internet. Start with simple shaders (texturing, diffuse lighting, etc.), and work your way up to the more complex shaders such as you have here: vertex skinning and toon shading are both fairly advanced topics.

Tristam MacDonald. Ex-BigTech Software Engineer. Future farmer. [https://trist.am]

jollyjeffers

1,571

June 01, 2009 12:33 PM

Quote:Original post by swiftcoder
Use [source] [/source] tags to post code - it will be syntax highlighted, and we won't have to scroll past a wall of text.

Yup, that is definitely preferred and I've edited accordingly. Although in RyuMaster's defence they did put [code] tags in, which is more than some people do [smile]

As for the original post... I would have to concur with swiftcoder - the Nvidia FX files tend to be quite hard to follow unless you know exactly what you're doing. I generally don't like them myself as they tend to obscure the very thing they're trying to demonstrate [headshake]

If you can't find any different/better sources for cel shading then you'll just have to keep crunching through the Nvidia sample. The main obfuscation they love is DXSAS so try and strip as much of that out as possible. Then go through all of the #include and other #define type stuff and try to clean the code up as much as possible. Eliminate all combinations of techniques/parameters/constants that you don't need.

In doing the above I have had some success at implementing code from Nvidia's SDK's without blindly assuming it just works [lol]

hth
Jack

<hr align="left" width="25%" />
Jack Hoxley <small>[</small><small> Forum FAQ | Revised FAQ | MVP Profile | Developer Journal ]</small>

RyuMaster

117

Author

June 01, 2009 01:59 PM

Thanks for the reply. Yes, looks like

 didn't work... I thought it is my browsers issue.

So I see now... It would not be easy. But isn't all this stuff is a matter of copy-past?

Like, if I take this part width bones and weights, and will past it into nvidia or some other vertex shader part - and it will work?

Or shaders can become so large and different, that... Well, probably so :)

I'll try to find less complex toon shader then and maybe start with it...

adt7

751

June 01, 2009 02:23 PM

Toon shading can be done as a post-processing effect with a sobel filter.

Although, as people have recommended above, I would recommend you learn some shader basics so that you at least have some idea what you're doing when you're working with them instead of copying and pasting.

Combine HLSL shaders

This topic is closed to new replies.

Popular Topics

Recommended Tutorials

Combine HLSL shaders

This topic is closed to new replies.

Popular Topics

Recommended Tutorials

Reticulating splines