Jump to content

  • Log In with Google      Sign In   
  • Create Account

We're offering banner ads on our site from just $5!

1. Details HERE. 2. GDNet+ Subscriptions HERE. 3. Ad upload HERE.


Don't forget to read Tuesday's email newsletter for your chance to win a free copy of Construct 2!


HLSL, how do I turn on debug, disable optimizations?


Old topic!
Guest, the last post of this topic is over 60 days old and at this point you may not reply in this topic. If you wish to continue this conversation start a new topic.

  • You cannot reply to this topic
12 replies to this topic

#1 Tispe   Members   -  Reputation: 1039

Like
0Likes
Like

Posted 20 September 2012 - 04:34 PM

Hi

PIX is not showing me the HLSL code, how do I turn on debug, disable optimizations?

if(FAILED(D3DXCompileShaderFromResource(
  NULL,
  MAKEINTRESOURCE(IDR_VERTEXSHADER),	//filepath
  NULL,	//macro's
  NULL,	//includes
  "vs_main",   //main function
  "vs_3_0",   //shader profile
  NULL,	//flags
  &code,	//compiled operations
  &debugcode,   //errors
  &constantTable))) //constants
{
  MessageBox(NULL, L"The Compile Vertex shader function failed", L"Error", MB_OK);
		PostQuitMessage(0);
  return false;
}
else
{
  d3ddev->CreateVertexShader((DWORD*)code->GetBufferPointer(), &vertexShader);
  code->Release();
}


Sponsor:

#2 MJP   Moderators   -  Reputation: 11585

Like
0Likes
Like

Posted 20 September 2012 - 05:52 PM

Pass D3DXSHADER_DEBUG|D3DXSHADER_SKIPOPTIMIZATION as the "Flags" parameter of D3DXCompileShaderFromResource.

#3 Tispe   Members   -  Reputation: 1039

Like
0Likes
Like

Posted 21 September 2012 - 02:27 AM

It seems my vertex shader is still optimizing away my sampler:

// Vertex shader input structure
struct VS_INPUT
{
    float4 Position   : POSITION;
    float2 Texture    : TEXCOORD0;
};

// Vertex shader output structure
struct VS_OUTPUT
{
    float4 Position   : POSITION;
    float2 Texture    : TEXCOORD0;
};

// Global variables
float4x4 World;
float4x4 ViewProj;
sampler2D Tex1 : register(s1);

// Name: Simple Vertex Shader
// Type: Vertex shader
// Desc: Vertex transformation and texture coord pass-through
//
VS_OUTPUT vs_main( in VS_INPUT In )
{
    VS_OUTPUT Out;					  //create an output vertex
    float4x4 WorldViewProj = mul(World, ViewProj);
    Out.Position = mul(In.Position, WorldViewProj);  //apply vertex transformation
    Out.Texture  = In.Texture;		  //copy original texcoords
    return Out;						 //return output vertex
}

becomes

// Parameters:
//
//   float4x4 ViewProj;
//   float4x4 World;
//
//
// Registers:
//
//   Name		 Reg   Size
//   ------------ ----- ----
//   World	    c0	   4
//   ViewProj	 c4	   4
//
    vs_3_0
    dcl_position v0  // In<0,1,2,3>
    dcl_texcoord v1  // In<4,5>
    dcl_position o0
    dcl_texcoord o1.xy
    mov r0, c0  // ::World<0,4,8,12>
    mul r1, r0, c4.x
    mov r2, c1  // ::World<1,5,9,13>
    mul r3, r2, c4.y
    add r1, r1, r3
    mov r3, c2  // ::World<2,6,10,14>
    mul r4, r3, c4.z
    add r1, r1, r4
    mov r4, c3  // ::World<3,7,11,15>
    mul r5, r4, c4.w
    add r1, r1, r5  // ::WorldViewProj<0,4,8,12>
    mul r5, r0, c5.x
    mul r6, r2, c5.y
    add r5, r5, r6
    mul r6, r3, c5.z
    add r5, r5, r6
    mul r6, r4, c5.w
    add r5, r5, r6  // ::WorldViewProj<1,5,9,13>
    mul r6, r0, c6.x
    mul r7, r2, c6.y
    add r6, r6, r7
    mul r7, r3, c6.z
    add r6, r6, r7
    mul r7, r4, c6.w
    add r6, r6, r7  // ::WorldViewProj<2,6,10,14>
    mul r0, r0, c7.x
    mul r2, r2, c7.y
    add r0, r0, r2
    mul r2, r3, c7.z
    add r0, r0, r2
    mul r2, r4, c7.w
    add r0, r0, r2  // ::WorldViewProj<3,7,11,15>
    dp4 r1.x, v0, r1  // ::Out<0>
    dp4 r1.y, v0, r5  // ::Out<1>
    dp4 r1.z, v0, r6  // ::Out<2>
    dp4 r1.w, v0, r0  // ::Out<3>
    mov r0.xy, v1  // ::Out<4,5>
    mov o0, r1  // ::vs_main<0,1,2,3>
    mov o1.xy, r0  // ::vs_main<4,5>
// approximately 39 instruction slots used


#4 Hodgman   Moderators   -  Reputation: 31092

Like
0Likes
Like

Posted 21 September 2012 - 03:23 AM

That's because the sampler is never used so there's no need for the asm to declare it.

#5 Tispe   Members   -  Reputation: 1039

Like
0Likes
Like

Posted 21 September 2012 - 04:58 AM

Hmm, I want to validate that matrices are passed correctly but even this will be optimized away with the "no optimized flag"

pGraphicsDevice->constantTable->SetMatrixArray(pGraphicsDevice->GetDevice(), "FirstTenSkeleton", pPlayer->pSkeleton, 10);

// Vertex shader input structure
struct VS_INPUT
{
    float4 Position   : POSITION;
    float2 Texture    : TEXCOORD0;
};

// Vertex shader output structure
struct VS_OUTPUT
{
    float4 Position   : POSITION;
    float2 Texture    : TEXCOORD0;
};

// Global variables
float4x4 World;
float4x4 ViewProj;
sampler2D Tex1 : register(s1);
float4x4 FirstTenSkeleton[10];

// Name: Simple Vertex Shader
// Type: Vertex shader
// Desc: Vertex transformation and texture coord pass-through
//
VS_OUTPUT vs_main( in VS_INPUT In )
{
    VS_OUTPUT Out;					  //create an output vertex
    float4x4 WorldViewProj = mul(World, ViewProj);
float4x4 Bone0 = FirstTenSkeleton[0];
float4x4 Bone1 = FirstTenSkeleton[1];
float4x4 Bone2 = FirstTenSkeleton[2];
float4x4 Bone3 = FirstTenSkeleton[3];
float4x4 Bone4 = FirstTenSkeleton[4];
float4x4 Bone5 = FirstTenSkeleton[5];
float4x4 Bone6 = FirstTenSkeleton[6];
float4x4 Bone7 = FirstTenSkeleton[7];
float4x4 Bone8 = FirstTenSkeleton[8];
float4x4 Bone9 = FirstTenSkeleton[9];
    Out.Position = mul(In.Position, WorldViewProj);  //apply vertex transformation
    Out.Texture  = In.Texture;		  //copy original texcoords
    return Out;						 //return output vertex
}


#6 Hodgman   Moderators   -  Reputation: 31092

Like
0Likes
Like

Posted 21 September 2012 - 05:04 AM

If the compiler is stubbornly optimizing out variables like this, you can force it to include them by making sure they affect the output in some (very small) way, e.g.
float4x4 temp1 = FirstTenSkeleton[0]
               + FirstTenSkeleton[1]
               + FirstTenSkeleton[2]
               + FirstTenSkeleton[3];//etc
float4 temp2 = Bones[0]+Bones[1]+Bones[2]+Bones[3];
float temp3 = dot(temp2, (float4)1);
Out.Texture.x += temp3 * 0.0001;

Edited by Hodgman, 21 September 2012 - 05:05 AM.


#7 Tispe   Members   -  Reputation: 1039

Like
0Likes
Like

Posted 21 September 2012 - 08:36 AM

OK.... I got things showing up in the debugger now. But guess what, the matrices are incorrect.

pGraphicsDevice->constantTable->SetMatrixArray(pGraphicsDevice->GetDevice(), "FirstTenSkeleton", pPlayers[i]->pRaceSkeleton->pSkeleton, 10);
  D3DXMATRIX FirstBone = pPlayers[i]->pRaceSkeleton->pSkeleton[0];

In VS2010 FirstBone matrix has the following values:
  _11 5.9604645e-008 float
  _12 0.00000000 float
  _13 0.99999994 float
  _14 0.00000000 float
  _21 0.00000000 float
  _22 1.0000000 float
  _23 0.00000000 float
  _24 0.00000000 float
  _31 -0.99999994 float
  _32 0.00000000 float
  _33 5.9604645e-008 float
  _34 0.00000000 float
  _41 0.00000000 float
  _42 0.00000000 float
  _43 0.00000000 float
  _44 1.0000000 float


The Vertex Shader gives Bone0 this value:
Bone0 ( 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1065353216 ) float16


Row-major or Column major should only swap rows for columns right? So it is not about that here?

Here is the new VS:
// Vertex shader input structure
struct VS_INPUT
{
	float4 Position   : POSITION;
	float2 Texture	: TEXCOORD0;
};

// Vertex shader output structure
struct VS_OUTPUT
{
	float4 Position   : POSITION;
	float2 Texture	: TEXCOORD0;
};

// Global variables
float4x4 World;
float4x4 ViewProj;
sampler2D Tex1 : register(s1);
float4x4 FirstTenSkeleton[10];

// Name: Simple Vertex Shader
// Type: Vertex shader
// Desc: Vertex transformation and texture coord pass-through
//
VS_OUTPUT vs_main( in VS_INPUT In )
{
	VS_OUTPUT Out;					  //create an output vertex
	float4x4 WorldViewProj = mul(World, ViewProj);
float4x4 Bone0 = FirstTenSkeleton[0];
float4x4 Bone1 = FirstTenSkeleton[1];
float4x4 Bone2 = FirstTenSkeleton[2];
float4x4 Bone3 = FirstTenSkeleton[3];
float4x4 Bone4 = FirstTenSkeleton[4];
float4x4 Bone5 = FirstTenSkeleton[5];
float4x4 Bone6 = FirstTenSkeleton[6];
float4x4 Bone7 = FirstTenSkeleton[7];
float4x4 Bone8 = FirstTenSkeleton[8];
float4x4 Bone9 = FirstTenSkeleton[9];
	Out.Position = mul(In.Position, WorldViewProj);  //apply vertex transformation
	Out.Texture  = In.Texture;		  //copy original texcoords

//Remove, just for testing
float4x4 temp1 = Bone0 + Bone1 + Bone2 + Bone3 + Bone4;
float temp2 = temp1[0][0] + temp1[0][1] + temp1[0][2] + temp1[0][3] +
	 temp1[1][0] + temp1[1][1] + temp1[1][2] + temp1[1][3] +
	 temp1[2][0] + temp1[2][1] + temp1[2][2] + temp1[2][3] +
	 temp1[3][0] + temp1[3][1] + temp1[3][2] + temp1[3][3];
	
Out.Texture.x += temp2 * 0.0001;

	return Out;						 //return output vertex
}

Edited by Tispe, 21 September 2012 - 08:38 AM.


#8 Tispe   Members   -  Reputation: 1039

Like
0Likes
Like

Posted 21 September 2012 - 09:57 AM

I get the same with the World matrix. Visual Studio 2010 and PIX are giving me completly differnt values, why is that?

I get proper rendering but somehow PIX and VS2010 gives different values for a matrix......

#9 Hodgman   Moderators   -  Reputation: 31092

Like
0Likes
Like

Posted 21 September 2012 - 09:16 PM

This is D3D9, right?
Try running the "DirectX control panel" (installed as part of the SDK), and switch Direct3D 9 over to use the debug version, and tick the "shader debugging" tickbox.

#10 Tispe   Members   -  Reputation: 1039

Like
0Likes
Like

Posted 22 September 2012 - 04:34 AM

Yes, D3D9.

I did as you suggested but same result:

Posted Image

#11 Tispe   Members   -  Reputation: 1039

Like
0Likes
Like

Posted 22 September 2012 - 11:06 AM

Ok, so I went to both the x64 and x86 control panel and used "debug version of Direct3D 9". After setting the latter my mesh is no longer rendering with thousands of the following output logs:

Direct3D9: (ERROR) :Can specify D3DLOCK_DISCARD or D3DLOCK_NOOVERWRITE for only Vertex Buffers created with D3DUSAGE_DYNAMIC
Direct3D9: (WARN) :Ignoring redundant SetRenderState - 143
Direct3D9: (WARN) :Ignoring redundant SetSamplerState. Sampler: 0, State: 7
Direct3D9: :====> ENTER: DLLMAIN(646cd9a0): Process Detach 00000aec, tid=00001b38
Direct3D9: (INFO) :MemFini!
Direct3D9: (ERROR) :Memory still allocated! Alloc count = 363
Direct3D9: (ERROR) :Memory Address: 034d07fc lAllocID=1 dwSize=00004bc4, ReturnAddr=646cd5ea (pid=00000aec)
Direct3D9: (ERROR) :Memory Address: 034d53f4 lAllocID=2 dwSize=00000350, ReturnAddr=646d02fa (pid=00000aec)

Edited by Tispe, 22 September 2012 - 11:11 AM.


#12 Tispe   Members   -  Reputation: 1039

Like
0Likes
Like

Posted 22 September 2012 - 01:19 PM

Ok, changed LockVertexBuffer(D3DLOCK_DISCARD, (void**)&pVoid) to be LockVertexBuffer(0, (void**)&pVoid) and got rid of the Direct3D9: (ERROR) :Can specify D3DLOCK_DISCARD or D3DLOCK_NOOVERWRITE for only Vertex Buffers created with D3DUSAGE_DYNAMIC. Mesh now renders in debug mode from control panel.

But, same result. The matrix values in PIX still does not correspond to those in Visual Studio 2010.

#13 Tispe   Members   -  Reputation: 1039

Like
0Likes
Like

Posted 23 September 2012 - 11:12 AM

Hi, I have an update. I found the values in a different place. The Vertex State tab holds 256 constant float registers. I find my values rounded off here. I can also find them in the "registers" tab. But in the variables tab the same matrices as shown in the picture above has these wierd 1,1,1,1,1,1,1,1...... values.

Now, lets say I want to sample a texture for information in the vertex shader and place it in local variables. Which registers will those be? Since I can't rely on variable names in PIX?

Posted Image




Old topic!
Guest, the last post of this topic is over 60 days old and at this point you may not reply in this topic. If you wish to continue this conversation start a new topic.



PARTNERS