Hey there this here works fine for RTS types terrain and cammera views.
.
D3DXVECTOR3 RT = D3DXVECTOR3(0.0f, 0.0f, 0.0f);
// Compute the vector of the pick ray in world space
//D3DXVECTOR3 v;
// Get the inverse view matrix
D3DXMATRIX matView;//= *g_Camera.GetViewMatrix();
gpCamera->getViewMatrix(&matView);
D3D10_VIEWPORT vp;
vp.TopLeftX = 0;
vp.TopLeftY = 0;
vp.Width = width;
vp.Height = height;
vp.MinDepth = 0.0f;
vp.MaxDepth = 1.0f;
D3DXMATRIX Identityworld;
D3DXMatrixIdentity( &Identityworld );
// Put mouse coordinates in screen space
D3DXVECTOR3 mnear = D3DXVECTOR3(ptCursor.x, ptCursor.y, 0);
D3DXVECTOR3 mfar = D3DXVECTOR3(ptCursor.x, ptCursor.y, 1);
// Transform points to world space
D3DXVec3Unproject(&mnear,
&mnear,
&vp,
pmproj,
&matView,
&Identityworld);
D3DXVec3Unproject(&mfar,
&mfar,
&vp,
pmproj,
&matView,
&Identityworld);
// near = DX.Vector3.Unproject(near,m_device.Viewport, m_device.Transform.Projection, m_device.Transform.View, DX.Matrix.Identity);
// far = DX.Vector3.Unproject(far,m_device.Viewport, m_device.Transform.Projection, m_device.Transform.View, DX.Matrix.Identity);
// Find Y Intercept
D3DXVECTOR3 direction = mfar - mnear;
if (mfar.y < 0) // if your mouse ray ends below XZ Plane
{
float yFactor = -mnear.y / direction.y;
RT = mnear + direction * yFactor; // zeroWorldPoint.X and zeroWorldPoint.Z contain XZ Plane Intercept
}
else
return RT;//error
//make sure we are not out of bounds
int hw = mWidth * 0.5f;
int hd = mDepth * 0.5f;
if(RT.x < -hw )
RT.x = -hw;
else
if(RT.x > hw)////how big the map is
RT.x = hw;
if(RT.z < -hd )
RT.z = -hd;
else
if(RT.z > hd)////how big the map is
RT.z = hd;
//we will need to trace the rays direction back up to check height values on the height map
float heighmapheight = 0.0f;
heighmapheight = getHeight(RT.x, RT.z);
direction = mnear - mfar;//revers the direction so we move towards the ground
//RT.y = heighmapheight;
D3DXVec3Normalize(&direction, &direction);
while(RT.y - heighmapheight < 10)
{
RT += direction * 20;
//float yFactor = -mnear.y / direction.y;
//RT = mnear + direction * yFactor; // zeroWorldPoint.X and zeroWorldPoint.Z contain XZ Plane Intercept
heighmapheight = getHeight(RT.x, RT.z);
}
return RT;