I want to pick 3D objects. I've done it in the past (with GameDev forum's help :P) but I used GL_SELECTION so this time I thought it'd be better if I did it the 'right' way. So I began reading tutorials and searching the gamedev forums, when I thought I was ready I started coding, and this is what I came up with.
For the time being I just want to know exactly which vertex my mouse was hovering on when I hit the left button, so I used gluUnProject() and glReadPixels() to get the 'exact' coordinates. But the modelview matrix wasn't the Identity Matrix what was left was to invert the matrix and multiply it with the vertex coordinates I got from gluUnProject(). I wrote 2 functions one that returns the determinant of a 4x4 matrix, and one that returns the inverted matrix. It should all work like a charm but ... it doesn't, I'm really confused here and I was wondering if any of you guys would mind helping me out a bit ...
What I think I'm doing right now is calculating the vertex that the mouse was on when the left mousebutton was clicked and then multiplying its coordinates with the inverted modelview matrix which should give me its coordinates as passed in the glBegin() ... glEnd() segment. Maybe what I'm trying to do is totally wrong please give me some pointers.
Thank you for your time.
float Determinant4f(const float *m) {
float det = 0.0f;
/*
I used this
|0 4 8 12| |5 9 13| |4 8 12| |4 8 12| |4 8 12|
|1 5 9 13| |6 10 14| |6 10 14| |5 9 13| |5 9 13|
|2 6 10 14| = m[0] * |7 11 15| - m[1] * |7 11 15| + m[2] * |7 11 15| - m[3] * |6 10 14|
|3 7 11 15|
|a b c|
|d e f| = aei + bfg + cdh - ceg - bdi - afh
|g h i|
*/
// parts of cofactors
float m_10_15 = m[10]*m[15];
float m_14_07 = m[14]*m[ 7];
float m_06_11 = m[ 6]*m[11];
float m_10_07 = m[10]*m[ 7];
float m_06_15 = m[ 6]*m[15];
float m_14_11 = m[14]*m[11];
float m_04_09 = m[ 4]*m[ 9];
float m_08_13 = m[ 8]*m[13];
float m_12_05 = m[12]*m[ 5];
float m_12_09 = m[12]*m[ 9];
float m_08_05 = m[ 8]*m[ 5];
float m_04_13 = m[ 4]*m[13];
det = m[ 0] * ( (m[ 5]*m_10_15 + m[ 9]*m_14_07 + m[13]*m_06_11) - (m[13]*m_10_07 + m[ 9]*m_06_15 + m[ 5]*m_14_11) );
det-= m[ 1] * ( (m[ 4]*m_10_15 + m[ 8]*m_14_07 + m[12]*m_06_11) - (m[12]*m_10_07 + m[ 8]*m_06_15 + m[ 4]*m_14_11) );
det+= m[ 2] * ( (m_04_09*m[15] + m_08_13*m[ 7] + m_12_05*m[11]) - (m_12_09*m[ 7] + m_08_05*m[15] + m_04_13*m[11]) );
det-= m[ 3] * ( (m_04_09*m[14] + m_08_13*m[ 6] + m_12_05*m[10]) - (m_12_09*m[ 6] + m_08_05*m[14] + m_04_13*m[10]) );
return det;
}
void InvertMatrix4f(const float *mat, float *dst) {
/*
A^-1 = 1/det(A) * adjoint(A)
*A^-1 = the inverse of A
|0 4 8 12|
|1 5 9 13|
|2 6 10 14|
|3 7 11 15|
*/
float _det = Determinant4f(mat);
float det = 1.0f/_det;
dst[ 0] = ( (mat[ 5]*mat[10]*mat[15] + mat[ 9]*mat[14]*mat[ 7] + mat[13]*mat[ 6]*mat[11]) -
(mat[13]*mat[10]*mat[ 7] + mat[ 9]*mat[ 6]*mat[15] + mat[ 5]*mat[14]*mat[11]) )* det;
dst[ 1] =-( (mat[ 1]*mat[10]*mat[15] + mat[ 9]*mat[14]*mat[ 3] + mat[13]*mat[ 2]*mat[11]) -
(mat[13]*mat[10]*mat[ 3] + mat[ 9]*mat[ 2]*mat[15] + mat[ 1]*mat[14]*mat[11]) )* det;
dst[ 2] = ( (mat[ 1]*mat[ 6]*mat[15] + mat[ 5]*mat[14]*mat[ 3] + mat[13]*mat[ 2]*mat[ 7]) -
(mat[13]*mat[ 6]*mat[ 3] + mat[ 5]*mat[ 2]*mat[15] + mat[ 1]*mat[14]*mat[ 7]) )* det;
dst[ 3] =-( (mat[ 1]*mat[ 6]*mat[11] + mat[ 5]*mat[10]*mat[ 3] + mat[ 9]*mat[ 2]*mat[ 7]) -
(mat[ 9]*mat[ 6]*mat[ 3] + mat[ 5]*mat[ 2]*mat[11] + mat[ 1]*mat[10]*mat[ 7]) )* det;
/*dst[ 4] =-( (mat[ 4]*mat[10]*mat[15] + mat[ 8]*mat[14]*mat[ 7] + mat[12]*mat[ 6]*mat[11]) -
(mat[12]*mat[10]*mat[ 7] + mat[ 8]*mat[ 6]*mat[13] + mat[ 4]*mat[14]*mat[11]) )* det;*/
dst[ 4] =-( (mat[ 4]*mat[10]*mat[15] + mat[ 8]*mat[14]*mat[ 7] + mat[12]*mat[ 6]*mat[11]) -
(mat[12]*mat[10]*mat[ 7] + mat[ 8]*mat[ 6]*mat[15] + mat[ 4]*mat[14]*mat[11]) )* det;
dst[ 5] = ( (mat[ 0]*mat[10]*mat[15] + mat[ 8]*mat[14]*mat[ 3] + mat[12]*mat[ 2]*mat[11]) -
(mat[12]*mat[10]*mat[ 3] + mat[ 8]*mat[ 2]*mat[15] + mat[ 0]*mat[14]*mat[11]) )* det;
dst[ 6] =-( (mat[ 0]*mat[ 6]*mat[15] + mat[ 4]*mat[14]*mat[ 3] + mat[12]*mat[ 2]*mat[ 7]) -
(mat[12]*mat[ 6]*mat[ 3] + mat[ 4]*mat[ 2]*mat[15] + mat[ 0]*mat[14]*mat[ 7]) )* det;
dst[ 7] = ( (mat[ 0]*mat[ 6]*mat[11] + mat[ 4]*mat[10]*mat[ 3] + mat[ 8]*mat[ 2]*mat[ 7]) -
(mat[ 8]*mat[ 6]*mat[ 3] + mat[ 4]*mat[ 2]*mat[11] + mat[ 0]*mat[10]*mat[ 7]) )* det;
dst[ 8] = ( (mat[ 4]*mat[ 9]*mat[15] + mat[ 8]*mat[13]*mat[ 7] + mat[12]*mat[ 5]*mat[11]) -
(mat[12]*mat[ 9]*mat[ 7] + mat[ 8]*mat[ 5]*mat[15] + mat[ 4]*mat[13]*mat[11]) )* det;
/*dst[ 9] =-( (mat[ 1]*mat[ 9]*mat[15] + mat[ 8]*mat[13]*mat[ 3] + mat[12]*mat[ 1]*mat[11]) -
(mat[12]*mat[ 9]*mat[ 3] + mat[ 8]*mat[ 1]*mat[15] + mat[ 0]*mat[13]*mat[11]) )* det;*/
dst[ 9] =-( (mat[ 0]*mat[ 9]*mat[15] + mat[ 8]*mat[13]*mat[ 3] + mat[12]*mat[ 1]*mat[11]) -
(mat[12]*mat[ 9]*mat[ 3] + mat[ 8]*mat[ 1]*mat[15] + mat[ 0]*mat[13]*mat[11]) )* det;
dst[10] = ( (mat[ 0]*mat[ 5]*mat[15] + mat[ 4]*mat[13]*mat[ 3] + mat[12]*mat[ 1]*mat[ 7]) -
(mat[12]*mat[ 5]*mat[ 3] + mat[ 4]*mat[ 1]*mat[15] + mat[ 0]*mat[13]*mat[ 7]) )* det;
dst[11] =-( (mat[ 0]*mat[ 5]*mat[11] + mat[ 4]*mat[ 9]*mat[ 3] + mat[ 8]*mat[ 1]*mat[ 7]) -
(mat[ 8]*mat[ 5]*mat[ 3] + mat[ 4]*mat[ 1]*mat[11] + mat[ 0]*mat[ 9]*mat[ 7]) )* det;
dst[12] =-( (mat[ 4]*mat[ 9]*mat[14] + mat[ 8]*mat[13]*mat[ 6] + mat[12]*mat[ 5]*mat[10]) -
(mat[12]*mat[ 9]*mat[ 6] + mat[ 8]*mat[ 5]*mat[14] + mat[ 4]*mat[13]*mat[10]) )* det;
dst[13] = ( (mat[ 0]*mat[ 9]*mat[14] + mat[ 8]*mat[13]*mat[ 2] + mat[12]*mat[ 1]*mat[10]) -
(mat[12]*mat[ 9]*mat[ 2] + mat[ 8]*mat[ 1]*mat[14] + mat[ 0]*mat[13]*mat[10]) )* det;
dst[14] =-( (mat[ 0]*mat[ 5]*mat[14] + mat[ 4]*mat[13]*mat[ 2] + mat[12]*mat[ 1]*mat[ 6]) -
(mat[12]*mat[ 5]*mat[ 2] + mat[ 4]*mat[ 1]*mat[14] + mat[ 0]*mat[13]*mat[ 6]) )* det;
dst[15] = ( (mat[ 0]*mat[ 5]*mat[10] + mat[ 4]*mat[ 9]*mat[ 2] + mat[ 8]*mat[ 1]*mat[ 6]) -
(mat[ 8]*mat[ 5]*mat[ 2] + mat[ 4]*mat[ 1]*mat[10] + mat[ 0]*mat[ 9]*mat[ 6]) )* det;
}
// here is my draw function
void draw() {
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
static float theta = 0.0f;
theta+= 0.025f;
if (theta>360.0f)
theta-=360.0f;
glMatrixMode(GL_MODELVIEW);
glLoadIdentity();
glTranslatef(1.0f,-1.0f,-5.0f);
glRotatef(theta,0.1f,1.0f,0.00005f);
glColor3f(r,g,b);
glBegin(GL_QUADS);
glVertex3f(0.0f,0.0f,0.0f);
glVertex3f(3.0f,0.0f,0.0f);
glVertex3f(3.0f,1.0f,0.0f);
glVertex3f(0.0f,1.0f,0.0f);
glEnd();
SDL_GL_SwapBuffers();
};
// and here is the point calculation function
void calculatePointFromMouse(int x, int y, Vector& location) {
GLdouble projMatrix[16], modelMatrix[16];
GLdouble x3d,y3d,z3d;
GLint viewport[4];
GLfloat depth;
glGetDoublev(GL_MODELVIEW_MATRIX,modelMatrix);
glGetDoublev(GL_PROJECTION_MATRIX,projMatrix);
glGetIntegerv(GL_VIEWPORT,viewport);
y = viewport[3] - y; // height - y
glReadPixels(x,y,1,1,GL_DEPTH_COMPONENT,GL_FLOAT,&depth);
gluUnProject(x,y, depth, modelMatrix,projMatrix,viewport, &x3d,&y3d,&z3d);
GLfloat fmodelMatrix[16],invertedMatrix[16];
glGetFloatv(GL_MODELVIEW_MATRIX,fmodelMatrix);
InvertMatrix4f(fmodelMatrix,invertedMatrix);
x3d = x3d*invertedMatrix[0] + y3d*invertedMatrix[4] + z3d*invertedMatrix[ 8];
y3d = x3d*invertedMatrix[1] + y3d*invertedMatrix[5] + z3d*invertedMatrix[ 9];
z3d = x3d*invertedMatrix[2] + y3d*invertedMatrix[6] + z3d*invertedMatrix[10];
location = Vector(x3d,y3d,z3d);
}
// that's the selection part
void mouseDown(int button,int x, int y) {
if (m_guiVisible)
m_gui->mouseDown(button, x, y);
if (button ==1 ) {
Vector origin;
calculatePointFromMouse(x,y,origin);
r = g = b = 0.5f;
if ((origin.x<=3.0f && origin.x >= 0.0f) && (origin.y >=0.0f && origin.y <= 1.0f))
r = g = b = 1.0f;
}
};
Edit: Changed the way I invert the matrix
