Hi,

I have been converting a Toon-Shader from XNA 4.0 into DirectX 9 C++ (the XNA source code can be found here). I seem to have got most of the DirectX code in place but instead of seeing cel-shading, the mesh is completely rendered black. I suspect this might be due to the inverse transpose of the world matrix but I might be completely wrong since my DirectX programming isn't brilliant

Here's the code I have got so far:

Mesh.cpp



#include <Windows.h>
#include <mmsystem.h>
#include <d3dx9.h>
#pragma warning( disable : 4996 ) // disable deprecated warning
#include <strsafe.h>
#pragma warning( default : 4996 )




//-----------------------------------------------------------------------------
// Global variables
//-----------------------------------------------------------------------------
LPDIRECT3D9 g_pD3D = NULL; // Used to create the D3DDevice
LPDIRECT3DDEVICE9 g_pd3dDevice = NULL; // Our rendering device

LPD3DXMESH g_pMesh = NULL; // Our mesh object in sysmem
D3DMATERIAL9* g_pMeshMaterials = NULL; // Materials for our mesh
LPDIRECT3DTEXTURE9* g_pMeshTextures = NULL; // Textures for our mesh
DWORD g_dwNumMaterials = 0L; // Number of mesh materials
LPDIRECT3DTEXTURE9 g_pTexture = NULL;

D3DXMATRIXA16 matWorld;
D3DXMATRIXA16 matView;
D3DXMATRIXA16 matProj;

LPD3DXEFFECT anEffect;

D3DXHANDLE hTech;



//-----------------------------------------------------------------------------
// Name: InitD3D()
// Desc: Initializes Direct3D
//-----------------------------------------------------------------------------
HRESULT InitD3D( HWND hWnd )
{
// Create the D3D object.
if( NULL == ( g_pD3D = Direct3DCreate9( D3D_SDK_VERSION ) ) )
return E_FAIL;

// Set up the structure used to create the D3DDevice. Since we are now
// using more complex geometry, we will create a device with a zbuffer.
D3DPRESENT_PARAMETERS d3dpp;
ZeroMemory( &d3dpp, sizeof( d3dpp ) );
d3dpp.Windowed = TRUE;
d3dpp.SwapEffect = D3DSWAPEFFECT_DISCARD;
d3dpp.BackBufferFormat = D3DFMT_UNKNOWN;
d3dpp.EnableAutoDepthStencil = TRUE;
d3dpp.AutoDepthStencilFormat = D3DFMT_D16;

// Create the D3DDevice
if( FAILED( g_pD3D->CreateDevice( D3DADAPTER_DEFAULT, D3DDEVTYPE_HAL, hWnd,
D3DCREATE_SOFTWARE_VERTEXPROCESSING,
&d3dpp, &g_pd3dDevice ) ) )
{
return E_FAIL;
}

// Turn on the zbuffer
g_pd3dDevice->SetRenderState( D3DRS_ZENABLE, TRUE );

// Turn on ambient lighting
g_pd3dDevice->SetRenderState( D3DRS_AMBIENT, 0xffffffff );

return S_OK;
}




//-----------------------------------------------------------------------------
// Name: InitGeometry()
// Desc: Load the mesh and build the material and texture arrays
//-----------------------------------------------------------------------------
HRESULT InitGeometry()
{
LPD3DXBUFFER pD3DXMtrlBuffer;

// Load the mesh from the specified file
if( FAILED( D3DXLoadMeshFromX( L"Tiger.x", D3DXMESH_SYSTEMMEM,
g_pd3dDevice, NULL,
&pD3DXMtrlBuffer, NULL, &g_dwNumMaterials,
&g_pMesh ) ) )
{
// If model is not in current folder, try parent folder
if( FAILED( D3DXLoadMeshFromX( L"..\\Tiger.x", D3DXMESH_SYSTEMMEM,
g_pd3dDevice, NULL,
&pD3DXMtrlBuffer, NULL, &g_dwNumMaterials,
&g_pMesh ) ) )
{
MessageBox( NULL, L"Could not find tiger.x", L"Meshes.exe", MB_OK );
return E_FAIL;
}
}

// We need to extract the material properties and texture names from the
// pD3DXMtrlBuffer
D3DXMATERIAL* d3dxMaterials = ( D3DXMATERIAL* )pD3DXMtrlBuffer->GetBufferPointer();
g_pMeshMaterials = new D3DMATERIAL9[g_dwNumMaterials];
if( g_pMeshMaterials == NULL )
return E_OUTOFMEMORY;
g_pMeshTextures = new LPDIRECT3DTEXTURE9[g_dwNumMaterials];
if( g_pMeshTextures == NULL )
return E_OUTOFMEMORY;

for( DWORD i = 0; i < g_dwNumMaterials; i++ )
{
// Copy the material
g_pMeshMaterials = d3dxMaterials.MatD3D;

// Set the ambient color for the material (D3DX does not do this)
g_pMeshMaterials.Ambient = g_pMeshMaterials.Diffuse;

g_pMeshTextures = NULL;
if( d3dxMaterials.pTextureFilename != NULL &&
lstrlenA( d3dxMaterials.pTextureFilename ) > 0 )
{
// Create the texture
if( FAILED( D3DXCreateTextureFromFileA( g_pd3dDevice,
d3dxMaterials.pTextureFilename,
&g_pMeshTextures ) ) )
{
// If texture is not in current folder, try parent folder
const CHAR* strPrefix = "..\\";
CHAR strTexture[MAX_PATH];
strcpy_s( strTexture, MAX_PATH, strPrefix );
strcat_s( strTexture, MAX_PATH, d3dxMaterials.pTextureFilename );
// If texture is not in current folder, try parent folder
if( FAILED( D3DXCreateTextureFromFileA( g_pd3dDevice,
strTexture,
&g_pMeshTextures ) ) )
{
MessageBox( NULL, L"Could not find texture map", L"Meshes.exe", MB_OK );
}
}
}
}

//With this method you load your effect file
D3DXCreateEffectFromFile(g_pd3dDevice, L"Toon.fx", NULL, NULL, 0, NULL, &anEffect, NULL);

// Done with the material buffer
pD3DXMtrlBuffer->Release();

return S_OK;
}


//-----------------------------------------------------------------------------
// Name: Cleanup()
// Desc: Releases all previously initialized objects
//-----------------------------------------------------------------------------
VOID Cleanup()
{
if( g_pMeshMaterials != NULL )
delete[] g_pMeshMaterials;

if( g_pMeshTextures )
{
for( DWORD i = 0; i < g_dwNumMaterials; i++ )
{
if( g_pMeshTextures )
g_pMeshTextures->Release();
}
delete[] g_pMeshTextures;
}
if( g_pMesh != NULL )
g_pMesh->Release();

if( g_pd3dDevice != NULL )
g_pd3dDevice->Release();

if( g_pD3D != NULL )
g_pD3D->Release();

if( anEffect!= NULL )
anEffect->Release();
}



//-----------------------------------------------------------------------------
// Name: SetupMatrices()
// Desc: Sets up the world, view, and projection transform matrices.
//-----------------------------------------------------------------------------
VOID SetupMatrices()
{
// Set up world matrix
D3DXMatrixRotationY( &matWorld, timeGetTime() / 1000.0f );
g_pd3dDevice->SetTransform( D3DTS_WORLD, &matWorld );

// Set up our view matrix. A view matrix can be defined given an eye point,
// a point to lookat, and a direction for which way is up. Here, we set the
// eye five units back along the z-axis and up three units, look at the
// origin, and define "up" to be in the y-direction.
D3DXVECTOR3 vEyePt( 0.0f, 3.0f,-5.0f );
D3DXVECTOR3 vLookatPt( 0.0f, 0.0f, 0.0f );
D3DXVECTOR3 vUpVec( 0.0f, 1.0f, 0.0f );

D3DXMatrixLookAtLH( &matView, &vEyePt, &vLookatPt, &vUpVec );
g_pd3dDevice->SetTransform( D3DTS_VIEW, &matView );

// For the projection matrix, we set up a perspective transform (which
// transforms geometry from 3D view space to 2D viewport space, with
// a perspective divide making objects smaller in the distance). To build
// a perpsective transform, we need the field of view (1/4 pi is common),
// the aspect ratio, and the near and far clipping planes (which define at
// what distances geometry should be no longer be rendered).

D3DXMatrixPerspectiveFovLH( &matProj, D3DX_PI / 4, 1.0f, 1.0f, 100.0f );
g_pd3dDevice->SetTransform( D3DTS_PROJECTION, &matProj );
}

//-----------------------------------------------------------------------------
// Name: Render()
// Desc: Draws the scene
//-----------------------------------------------------------------------------
VOID Render()
{
UINT numberOfPasses;

// Clear the backbuffer and the zbuffer
g_pd3dDevice->Clear( 0, NULL, D3DCLEAR_TARGET | D3DCLEAR_ZBUFFER,
D3DCOLOR_XRGB( 0, 0, 255 ), 1.0f, 0 );

// Begin the scene
if( SUCCEEDED( g_pd3dDevice->BeginScene() ) )
{
// Setup the world, view, and projection matrices
SetupMatrices();

anEffect->SetMatrix("World", &matWorld);
anEffect->SetMatrix("View", &matView);
anEffect->SetMatrix("Projection", &matProj);

// Get Inverse Transpose of World Matrix
D3DXMATRIX iTWM;
D3DXMatrixInverse(&iTWM, 0, &matWorld);
D3DXMatrixTranspose(&iTWM, &iTWM);

anEffect->SetMatrix("WorldInverseTranspose", &iTWM);

anEffect->SetTechnique("Toon");

if( SUCCEEDED(anEffect->Begin(&numberOfPasses, NULL)))
{

for(UINT i = 0; i < numberOfPasses; i++)
{
anEffect->BeginPass(i);

for (DWORD j = 0; j < g_dwNumMaterials; j++)
{
anEffect->SetTexture("Texture", g_pMeshTextures[j]);
anEffect->CommitChanges();
g_pMesh->DrawSubset(j);
}

anEffect->EndPass();

anEffect->End();
}
// End the scene
g_pd3dDevice->EndScene();
}
}


// Present the backbuffer contents to the display
g_pd3dDevice->Present( NULL, NULL, NULL, NULL );
}




//-----------------------------------------------------------------------------
// Name: MsgProc()
// Desc: The window's message handler
//-----------------------------------------------------------------------------
LRESULT WINAPI MsgProc( HWND hWnd, UINT msg, WPARAM wParam, LPARAM lParam )
{
switch( msg )
{
case WM_DESTROY:
Cleanup();
PostQuitMessage( 0 );
return 0;
}

return DefWindowProc( hWnd, msg, wParam, lParam );
}




//-----------------------------------------------------------------------------
// Name: WinMain()
// Desc: The application's entry point
//-----------------------------------------------------------------------------
INT WINAPI wWinMain( HINSTANCE hInst, HINSTANCE, LPWSTR, INT )
{
UNREFERENCED_PARAMETER( hInst );

// Register the window class
WNDCLASSEX wc =
{
sizeof( WNDCLASSEX ), CS_CLASSDC, MsgProc, 0L, 0L,
GetModuleHandle( NULL ), NULL, NULL, NULL, NULL,
L"D3D Tutorial", NULL
};
RegisterClassEx( &wc );

// Create the application's window
HWND hWnd = CreateWindow( L"D3D Tutorial", L"D3D Tutorial 06: Meshes",
WS_OVERLAPPEDWINDOW, 100, 100, 300, 300,
NULL, NULL, wc.hInstance, NULL );

// Initialize Direct3D
if( SUCCEEDED( InitD3D( hWnd ) ) )
{
// Create the scene geometry
if( SUCCEEDED( InitGeometry() ) )
{
// Show the window
ShowWindow( hWnd, SW_SHOWDEFAULT );
UpdateWindow( hWnd );

// Enter the message loop
MSG msg;
ZeroMemory( &msg, sizeof( msg ) );
while( msg.message != WM_QUIT )
{
if( PeekMessage( &msg, NULL, 0U, 0U, PM_REMOVE ) )
{
TranslateMessage( &msg );
DispatchMessage( &msg );
}
else
{
Render();
}
}
}
}

UnregisterClass( L"D3D Tutorial", wc.hInstance );
return 0;
}





Toon.fx



//--------------------------- BASIC PROPERTIES ------------------------------
// The world transformation
float4x4 World;

// The view transformation
float4x4 View;

// The projection transformation
float4x4 Projection;

// The transpose of the inverse of the world transformation,
// used for transforming the vertex's normal
float4x4 WorldInverseTranspose;

//--------------------------- DIFFUSE LIGHT PROPERTIES ------------------------------
// The direction of the diffuse light
float3 DiffuseLightDirection = float3(1, 0, 0);

// The color of the diffuse light
float4 DiffuseColor = float4(1, 1, 1, 1);

// The intensity of the diffuse light
float DiffuseIntensity = 1.0;

//--------------------------- TOON SHADER PROPERTIES ------------------------------
// The color to draw the lines in. Black is a good default.
float4 LineColor = float4(0, 0, 0, 1);

// The thickness of the lines. This may need to change, depending on the scale of
// the objects you are drawing.
float LineThickness = .03;

//--------------------------- TEXTURE PROPERTIES ------------------------------
// The texture being used for the object
texture Texture;

// The texture sampler, which will get the texture color
sampler2D textureSampler = sampler_state
{
Texture = (Texture);
MinFilter = Linear;
MagFilter = Linear;
AddressU = Clamp;
AddressV = Clamp;
};

//--------------------------- DATA STRUCTURES ------------------------------
// The structure used to store information between the application and the
// vertex shader
struct AppToVertex
{
float4 Position : POSITION0; // The position of the vertex
float3 Normal : NORMAL0; // The vertex's normal
float2 TextureCoordinate : TEXCOORD0; // The texture coordinate of the vertex
};

// The structure used to store information between the vertex shader and the
// pixel shader
struct VertexToPixel
{
float4 Position : POSITION0;
float2 TextureCoordinate : TEXCOORD0;
float3 Normal : TEXCOORD1;
};

//--------------------------- SHADERS ------------------------------
// The vertex shader that does cel shading.
// It really only does the basic transformation of the vertex location,
// and normal, and copies the texture coordinate over.
VertexToPixel CelVertexShader(AppToVertex input)
{
VertexToPixel output;

// Transform the position
float4 worldPosition = mul(input.Position, World);
float4 viewPosition = mul(worldPosition, View);
output.Position = mul(viewPosition, Projection);

// Transform the normal
output.Normal = normalize(mul(input.Normal, WorldInverseTranspose));

// Copy over the texture coordinate
output.TextureCoordinate = input.TextureCoordinate;

return output;
}

// The pixel shader that does cel shading. Basically, it calculates
// the color like is should, and then it discretizes the color into
// one of four colors.
float4 CelPixelShader(VertexToPixel input) : COLOR0
{
// Calculate diffuse light amount
float intensity = dot(normalize(DiffuseLightDirection), input.Normal);
if(intensity < 0)
intensity = 0;

// Calculate what would normally be the final color, including texturing and diffuse lighting
float4 color = tex2D(textureSampler, input.TextureCoordinate) * DiffuseColor * DiffuseIntensity;
color.a = 1;

// Discretize the intensity, based on a few cutoff points
if (intensity > 0.95)
color = float4(1.0,1,1,1.0) * color;
else if (intensity > 0.5)
color = float4(0.7,0.7,0.7,1.0) * color;
else if (intensity > 0.05)
color = float4(0.35,0.35,0.35,1.0) * color;
else
color = float4(0.1,0.1,0.1,1.0) * color;

return color;
}

// The vertex shader that does the outlines
VertexToPixel OutlineVertexShader(AppToVertex input)
{
VertexToPixel output = (VertexToPixel)0;

// Calculate where the vertex ought to be. This line is equivalent
// to the transformations in the CelVertexShader.
float4 original = mul(mul(mul(input.Position, World), View), Projection);

// Calculates the normal of the vertex like it ought to be.
float4 normal = mul(mul(mul(input.Normal, World), View), Projection);

// Take the correct "original" location and translate the vertex a little
// bit in the direction of the normal to draw a slightly expanded object.
// Later, we will draw over most of this with the right color, except the expanded
// part, which will leave the outline that we want.
output.Position = original + (mul(LineThickness, normal));

return output;
}

// The pixel shader for the outline. It is pretty simple: draw everything with the
// correct line color.
float4 OutlinePixelShader(VertexToPixel input) : COLOR0
{
return LineColor;
}

// The entire technique for doing toon shading
technique Toon
{
// The first pass will go through and draw the back-facing triangles with the outline shader,
// which will draw a slightly larger version of the model with the outline color. Later, the
// model will get drawn normally, and draw over the top most of this, leaving only an outline.
pass Pass1
{
VertexShader = compile vs_1_1 OutlineVertexShader();
PixelShader = compile ps_2_0 OutlinePixelShader();
CullMode = CW;
}

// The second pass will draw the model like normal, but with the cel pixel shader, which will
// color the model with certain colors, giving us the cel/toon effect that we are looking for.
pass Pass2
{
VertexShader = compile vs_1_1 CelVertexShader();
PixelShader = compile ps_2_0 CelPixelShader();
CullMode = CCW;
}
}




Thanks in advance.

Two thoughts:

(1) It could be that the normals aren't the problem after all. Maybe it's just that the second pass never gets rendered? Change LineColor to a different color and see if that colorizes the model. If it does, your second pass isn't showing up and all you're seeing is the single-color outline.

(2) I don't understand why that cel shader multiplies the normals by the inverse transpose of the world matrix at all. I'm not an expert yet as far as matrix transformations are concerned but I would try bringing those normals from their original space (I call it model space) to world space (as your diffuse light direction is also in world space of course) by simply multiplying them by the world matrix:


// Transform the normal
output.Normal = normalize(mul(input.Normal, World));


Hope this helps!

EDIT: And why does the outline shader worry about normals at all? It simply colorizes the mesh with one single color (LineColor), you can greatly simplify that shader program, it only needs to return LineColor in the pixel shader (as it does already) but the vertex shader needs to only bring the vertices into screen space. Normals mean nothing to that shader!

Hi,

I noticed two things. First, you call EndPass a little too often.
It should be something like this:
if( SUCCEEDED(anEffect->Begin(&numberOfPasses, NULL)))
{
for(UINT i = 0; i < numberOfPasses; i++)
{
anEffect->BeginPass(i);
for (DWORD j = 0; j < g_dwNumMaterials; j++)
{
anEffect->SetTexture("Texture", g_pMeshTextures[j]);
anEffect->CommitChanges();
g_pMesh->DrawSubset(j);
}
anEffect->EndPass();
}
anEffect->End();
}

Second, the mesh you’re looking at doesn’t have normals.
Add this after you have loaded your mesh to generate them:
if (!(g_pMesh->GetFVF() & D3DFVF_NORMAL))
{
LPD3DXMESH pTempMesh;
g_pMesh->CloneMeshFVF(g_pMesh->GetOptions(), g_pMesh->GetFVF() | D3DFVF_NORMAL, g_pd3dDevice, &pTempMesh);
g_pMesh->Release();
D3DXComputeNormals(pTempMesh, NULL);
g_pMesh = pTempMesh;
}
That worked for me.

(2) I don't understand why that cel shader multiplies the normals by the inverse transpose of the world matrix at all.

As for the multiplication with the InverseTranspose of the world matrix: This is used to “remove” the scalation from the world matrix. That’s a very common trick to keep “well-behaved” normals. Imagine you have sphere and scale it by factor 3 along the x-axis. Now, your normals end up not perpendicular on the surface anymore. But when using the InverseTranspose, however, they are perpendicular. But, well, as long as you don’t have any non-uniform scalations you can skip that.

EDIT: And why does the outline shader worry about normals at all? It simply colorizes the mesh with one single color (LineColor), you can greatly simplify that shader program, it only needs to return LineColor in the pixel shader (as it does already) but the vertex shader needs to only bring the vertices into screen space. Normals mean nothing to that shader!

The normal is used to scale the object a little.

Cheers!

[quote name='d k h' timestamp='1324315432' post='4895363']
(2) I don't understand why that cel shader multiplies the normals by the inverse transpose of the world matrix at all.

As for the multiplication with the InverseTranspose of the world matrix: This is used to “remove” the scalation from the world matrix. That’s a very common trick to keep “well-behaved” normals. Imagine you have sphere and scale it by factor 3 along the x-axis. Now, your normals end up not perpendicular on the surface anymore. But when using the InverseTranspose, however, they are perpendicular. But, well, unless you don’t have any non-uniform scalations you can skip that.
[/quote]

Ah, so multiplying normals with the inverse transpose of the world matrix moves them from their original to world space just like multiplying the normals with the normal world matrix would, but it keeps non-uniformly scaled normals 'proper', didn't know that. Thanks!

[quote name='d k h' timestamp='1324315432' post='4895363']
EDIT: And why does the outline shader worry about normals at all? It simply colorizes the mesh with one single color (LineColor), you can greatly simplify that shader program, it only needs to return LineColor in the pixel shader (as it does already) but the vertex shader needs to only bring the vertices into screen space. Normals mean nothing to that shader!

The normal is used to scale the object a little.
[/quote]

Oops, yeah, I overlooked that part of the outline vertex shader, sorry OP!

It's working now . Thanks for your help guys