Shouldn't this line...
float xScale = 1.0f / tanf(0.5f * fovy);
...be:
float xScale = 1.0f / tanf(0.5f * fovx);
?
No, because in that case image will be twisted.
To All
I think the real problem is in projection matrix. I used standard projection matrix by mistake and receive such distortions, when i switched to my custom projection matrix, I receive now more than twice less distortions (70px vs 30px now), but my question is: is it possible to improve projection matrix more to have less distortions leaving FOV 90 degrees?
For now I build my projection matrix like this:
void CameraDX_Free::perspective(float fovx, float aspect, float znear, float zfar)
{
// Construct a projection matrix based on the horizontal field of view
// 'fovx' rather than the more traditional vertical field of view 'fovy'.
float e = 1.0f / tanf(XMConvertToRadians(fovx) / 2.0f);
float aspectInv = 1.0f / aspect;
float fovy = 2.0f * atanf(aspectInv / e);
float xScale = 1.0f / tanf(0.5f * fovy);
float yScale = xScale / aspectInv;
XMFLOAT4X4 temp ;
XMStoreFloat4x4(&temp,m_projMatrix);
temp(0,0) = xScale;
temp(1,0) = 0.0f;
temp(2,0) = 0.0f;
temp(3,0) = 0.0f;
temp(0,1) = 0.0f;
temp(1,1) = yScale;
temp(2,1) = 0.0f;
temp(3,1) = 0.0f;
temp(0,2) = 0.0f;
temp(1,2) = 0.0f;
temp(2,2) = zfar / (zfar - znear);
temp(3,2) = -znear * zfar / (zfar - znear);
temp(0,3) = 0.0f;
temp(1,3) = 0.0f;
temp(2,3) = 1.0f;
temp(3,3) = 0.0f;
m_projMatrix = XMLoadFloat4x4(&temp);
m_fovx = fovx;
m_aspectRatio = aspect;
m_znear = znear;
m_zfar = zfar;
}