Hi ,
I am trying to do object picking.I calculate AABB and do ray - AABB intersaction.
When the object has position(0,0,0) everything works , but when I change the position the ray - AABB intersaction is alwayes true.

This is how I calculate the ray :

float pointX = (2.0f * localX / width - 1.0f) / shaderData.projection._11;
float pointY = (-2.0f * localY / height + 1.0f) / shaderData.projection._22;

XMVECTOR dir = XMVectorSet(pointX, pointY, 1.0f, 0.0f);

XMVECTOR det = XMMatrixDeterminant(view);
XMMATRIX viewInv = XMMatrixInverse(&det, view);

dir = XMVector3TransformNormal(dir, viewInv);

XMStoreFloat3((XMFLOAT3*)&pickRayDirection, dir);

And for every object I do this :

XMFLOAT4X4 mat = ent->GetWorldMatrix();
XMMATRIX world = XMLoadFloat4x4((XMFLOAT4X4*)&mat);
XMVECTOR det = XMMatrixDeterminant(world);

XMMATRIX invWorld = XMMatrixInverse(&det, world);
Vector3 org = cam->GetPosition();
Vector3 dir = cam->GetPickingRayDirection();
XMVECTOR orgInv = XMVector3TransformCoord(XMLoadFloat3((XMFLOAT3*)&org), invWorld);
XMVECTOR dirInv = XMVector3TransformNormal(XMLoadFloat3((XMFLOAT3*)&dir), invWorld);

dirInv = XMVector3Normalize(dirInv);
XMStoreFloat3((XMFLOAT3*)&org, orgInv);
XMStoreFloat3((XMFLOAT3*)&dir, dirInv);

Somewhere I saw that they get the translation matrix of the object , multiply it by the world matrix of the object and then inverse it.

Isn't the inverse of the translation done in the inverse world matrix ?!?!

I can apply any rotation and scale to the object and it will work , but if I change the position it does not work .

Tx

*EDIT :

I fixed it ! I just had to transpose the world matrix before I inverse it wow...

Hi Cat ,

It is interesting that you transpose the modelView matrix.Do you transpose the view matrix before you pass it to GetDirectionFromScreen ?

And dont you think that GetDirectionFromScreen is too big to be inlined ;)

Anyway I think that my AABB calculation is correct and my ray calculation is correct , I just can't put the ray and the AABB in the same space :)

Do not invert matrices unless you really need to during the game.

Make sure that you perform all the computations in the same space. Do all of it in world space just to make sure.

Not too different of your code, this is how I compute a ray in world space assuming DirectX (position = mouse position in client coordinates):

inline CVector3 ScreenToWorld(const CCamera& camera, const D3D11_VIEWPORT& screen, const CVector2& screenPos) {
	// Rewind (Screen->NDC->Projection->World)
	// To NDC.
	float scaleX = 0.5f * screen.Width;
	float scaleY = 0.5f * screen.Height;

	CVector3 ndcPos;
	ndcPos.x = (screenPos.x - scaleX) / scaleX;
	ndcPos.y = -(screenPos.y - scaleY) / scaleY;
	ndcPos.z = 1.0f;

	// To clip.
	CVector3 clipPos;
	clipPos.x = ndcPos.x / camera.ProjMatrix()._11;
	clipPos.y = ndcPos.y / camera.ProjMatrix()._22;
	clipPos.z = ndcPos.z;

	// To world.
	CMatrix4x4 invView = camera.Orientation().CreateTransformationMatrix();

	CVector3 worldPos = invView * clipPos;

	return worldPos;
}

Or, for readability if you will:

inline CMatrix4x4 GetInvMatrix(const D3D11_VIEWPORT& screen) {
	CMatrix4x4 out;
	out._11 = 1.0f / (0.5f * screen.Width);
	out._12 = 0.0f;
	out._13 = 0.0f;
	out._14 = 0.0f;

	out._21 = 0.0f;
	out._22 = 1.0f / (-0.5f * screen.Height);
	out._23 = 0.0f;
	out._24 = 0.0f;

	out._31 = 0.0f;
	out._32 = 0.0f;
	out._33 = 1.0f / (screen.MaxDepth - screen.MinDepth);
	out._34 = 0.0f;

	out._41 = -out._11 * (screen.TopLeftX + 0.5f * screen.Width);
	out._42 = -out._22 * (screen.TopLeftY + 0.5f * screen.Height);
	out._43 = -out._33 * screen.MinDepth;
	out._44 = 1.0f;
	return out;
}

inline CVector3 ScreenToWorld(const CCamera& camera, const D3D11_VIEWPORT& screen, const CVector2& screenPos) {
	// Screen->Projection->View->World
	CMatrix4x4 invScreen = GetInvMatrix(screen);
        CMatrix4x4 invView = camera.Orientation().CreateTransformationMatrix();

	CVector3 projPos = invScreen * CVector3(screenPos.x, screenPos.y, screen.MaxDepth);
	CVector3 viewPos(projPos.x / camera.ProjMatrix()._11, projPos.y / camera.ProjMatrix()._22, projPos.z);
	CVector3 worldPos = invView * viewPos;
	
	return worldPos;
}

And that's how I do a Ray-AABB test (p = ray origin, d = ray direction):

// Christer Ericson, Real-Time Collision Detection (2005), p. 180.
// Correction: http://realtimecollisiondetection.net/books/rtcd/errata/
bool RayCast(const b3Vec3& p, const b3Vec3& d, r32 tmax, r32& tmin) const {
		// Test all three slabs.
		for (u32 i = 0; i < 3; i++) {
			if (b3Abs(d[i]) < B3_EPSILON) { //epsilon = very small value
				// The segment is parallel to slab. No hit if origin not within slab.
				if (p[i] < min[i] || p[i] > max[i]) {
					return false; 
				}
			}
			else {
				// Compute intersection t value of segment with near and far plane of slab.
				r32 ood = B3_ONE / d[i];
				r32 t1 = ood * (min[i] - p[i]);
				r32 t2 = ood * (max[i] - p[i]);

				// Make t1 be intersection with near plane, t2 with far plane.
				if (t1 > t2) {
					b3Swap(t1, t2);
				}

				// Compute the intersection of slab intersection intervals
				tmin = b3Max(tmin, t1); // Rather than: if (t1 > tmin) tmin = t1;
				tmax = b3Min(tmax, t2); // Rather than: if (t2 < tmax) tmax = t2;

				// Exit with no collision as soon as slab intersection becomes empty.
				if (tmin > tmax) { 
					return false; 
				}
			}
		}

		return true;
	}

I calculate the AABB in object space , so I tried to put the ray in object space (and it is not working if I change the position of the object) and I tried to put the AABB in world space and it is not working at all.

ray in object space I do - rayDir * invWorldMatrix(of the object) and rayOrg * invWorldMatrix(of the object)

AABB in world space I do - AABBmax * worldMatrix and AABBmin * worldMatrix ( I am not sure if the math here is correct)

Also when I calculate the ray when I do :

vec3 p = (normalizedMouseY,normalizedMouseY,1.0f);

p = inverseViewMatrix * p

Is p now a point or a direction ?