Hi,

Im having alot of trouble with determining the ray normal that goes into the scene through the pointer.

Heres an example:

I have.

A projection matrix that i generate with an implementation of gluPerspective and glFrustum

0.001953000      0             0        0
0                0.002604000   0        0
0                0             1        0
-1              -1,            0        1

A dead simple view matrix obtained from a camera that sits at [0,0,5] and doesn't have any orientation.

1  0  0  0
0  1  0  0
0  0  1  0
0  0 -5  1

mouse pointer dead center in the screen (so [0,0] in clip space)

I then do the following calculation. (that i found in a tutorial here: http://antongerdelan.net/opengl/raycasting.html)

float x = (2.0f * mouse_x) / width - 1.0f;               //0 in this case because the mouse is in the center of the screen
float y = 1.0f - (2.0f * mouse_y) / height;              //0 in this case because the mouse is in the center of the screen
float z = 1.0f;
vec3 ray_nds = vec3(x, y, z);
vec4 ray_clip = vec4(ray_nds.xy, -1.0, 1.0);
vec4 ray_eye = inverse(projection_matrix) * ray_clip;
ray_eye = vec4(ray_eye.xy, -1.0, 0.0);
vec3 ray_wor = (inverse(view_matrix) * ray_eye).xyz;
ray_wor = normalise(ray_wor);

I then expect a result of [0,0,-1] because the camera is looking straight down the z-axis and the mouse is in the center of the screen.

instead the result i get is [0.799999,0.59999925,-0.001562398]

Please take a look and point out what I'm missing here, Maybe the calculation is wrong, maybe I'm missing a step, maybe the projection or view matrixes are wrong or maybe my expectation is wrong and I'm just misunderstanding something.

Thnx in Advance!

This is my c# code, its my toolkit of functions i've collected over time, I had completely forgotten the implementation.

        public static void CalcRay(int x, int y, ref Vector3 p1, ref Vector3 p2, Matrix a_viewMatrix, Viewport a_viewPort)
        {
            float NEAR = 0.1f;
            float FAR = 1000.0f;
            float FOV = 0.9f;
            //float WIDTH = 1920.0f;
            //float HEIGHT = 1080.0f;
            float WIDTH_DIV_2 = (a_viewPort.Width * 0.5f);
            float HEIGHT_DIV_2 = (a_viewPort.Height * 0.5f);
            float ASPECT = a_viewPort.AspectRatio;
            //float ASPECT = 1.0f;
            float dx, dy;
            //D3DMATRIX invMatrix, viewMatrix;
            Matrix invMatrix = Matrix.Invert(a_viewMatrix);

            dx = (float)Math.Tan(FOV * 0.5f) * (x / WIDTH_DIV_2 - 1.0f) * ASPECT;
            dy = (float)Math.Tan(FOV * 0.5f) * (1.0f - y / HEIGHT_DIV_2);

            p1 = new Vector3(dx * NEAR, dy * NEAR, NEAR);
            p2 = new Vector3(dx * FAR, dy * FAR, -FAR);
            p1 = (Vector3)Vector3.Transform(p1, invMatrix);
            p2 = (Vector3)Vector3.Transform(p2, invMatrix);
        }

const double imopi          = 0.017453292519943295769236907684886;

works only with perspective projection
TRay RayFromScreen(int x, int y, int sw, int sh, float fov, float z_near, float z_far, float aspect)
{

x,y, sw,sh should be in pixels as far as i can see

mat4 mvm = CAM_MODEL * CAM_VIEW; //cam model is always identity
mvm.Transpose();
vec3 dirX, dirY;
	dirX.x = mvm.m[0];
	dirX.y = mvm.m[4];
	dirX.z = mvm.m[8];

	dirY.x =	mvm.m[1];
	dirY.y =	mvm.m[5];
	dirY.z =	mvm.m[9];
	TRay res;



	float a = fov / 2.0;
float cotangent = 1.0 / tan( a * imopi );

float ax = z_near / cotangent;

float screen_w = 2.0*ax;

float screen_h = screen_w;// * yratio;

screen_w = screen_w * aspect;

float scr_coord_x = float(x) / float(sw);
float scr_coord_y = float(sh - y) / float(sh);


vec3 dir = FPP_CAM->ReturnFrontVector();

//vec3 dirX = FPP_CAM->ReturnRightVector();
//vec3 dirY = FPP_CAM->ReturnUpVector();


//move to lower left corner
vec3 start_pos = (FPP_CAM->pos + dir * z_near) + (-dirX * (screen_w / 2.0)) + (-dirY * (screen_h/2.0));

res.start = start_pos + (dirX * (screen_w * scr_coord_x)) + (dirY * (screen_h * scr_coord_y));

res.end = res.start + Normalize( vectorAB(FPP_CAM->pos, res.start) ) * 10000.0;
return res;

float yratio = 1.0 / aspect;

//compute the world position on the other end since its a perspective projection
ax = z_far / cotangent;
screen_w = 2.0*ax;
screen_h = screen_w * yratio;

start_pos = (FPP_CAM->pos + dir * z_far) + (-dirX * (screen_w / 2.0)) + (-dirY * (screen_h/2.0));

res.end = start_pos + (dirX * (screen_w * scr_coord_x)) + (dirY * (screen_h * scr_coord_y));

ALOG("RAY START: "+POINT_TO_TEXT(res.start));

return res;

}

theres a tranpose thing to make it oglcompilant only because i was lazy to change mvm.m thing

Thank you, I will try your functions.