I guess you use c#. Here is c# class representing triangle mesh. You can use picking functionality from this code:
/// <summary:
/// 3D Geometry that consist from indices, vertices, normals, texture coordinates.
/// </summary>
public sealed class TriangleMesh
{
private static uint s_internalIdCounter;
private BoundingSphere m_boundingSphere;
private BoundingBox m_boundingBox;
/// <summary>
/// Gets an internal ID that value is unique for each instance.
/// </summary>
public uint InternalID { get; private set; }
/// <summary>
/// Gets an array of the indices of the geometry.
/// </summary>
public ushort[] Indices { get; private set; }
/// <summary>
/// Gets an array of the vertices of the geometry.
/// </summary>
public Float3[] Vertices { get; private set; }
/// <summary>
/// Gets an array of the normals of the geometry. Value can be <c>null</c>.
/// </summary>
public Float3[] Normals { get; private set; }
/// <summary>
/// Gets an array of the normals of the geometry. Value can be <c>null</c>.
/// </summary>
public Float2[] TexCoords { get; private set; }
public TriangleMesh(ushort[] indices, Float3[] vertices, Float3[] normals, Float2[] texCoords)
{
if (indices.Length < 3)
throw new ArgumentException("The length of the indices array shouldn't be less than 3 elements.", "indices");
if (vertices.Length < 3)
throw new ArgumentException("The length of the vertices array shouldn't be less than 3 elements.", "vertices");
Indices = indices;
Vertices = vertices;
Normals = normals;
TexCoords = texCoords;
unchecked
{
s_internalIdCounter++;
}
InternalID = s_internalIdCounter;
CalculateBounds();
}
/// <summary>
/// Calculates a bounding box and a bonding sphere of the geometry.
/// </summary>
private void CalculateBounds()
{
m_boundingBox = BoundingBox.FromPoints(Vertices);
m_boundingSphere = BoundingSphere.FromBox(m_boundingBox);
}
/// <summary>
/// Gets transformed bounding sphere of the geometry.
/// </summary>
/// <param name="transform">Transformation matrix.</param>
/// <returns>Transformed bounding sphere.</returns>
public BoundingSphere CalculateBoundingSphere(Float4x4 transform)
{
Float3 center = Float3.TransformCoordinate(m_boundingSphere.Center, transform);
return new BoundingSphere(center, m_boundingSphere.Radius);
}
/// <summary>
/// Gets transformed bounding box of the geometry.
/// </summary>
/// <param name="transform">Transformation matrix.</param>
/// <returns>Transformed bounding box.</returns>
public BoundingBox CalculateBoundingBox(Float4x4 transform)
{
Float3 min = Float3.TransformCoordinate(m_boundingBox.Minimum, transform);
Float3 max = Float3.TransformCoordinate(m_boundingBox.Maximum, transform);
return new BoundingBox(min, max);
}
/// <summary>
/// Determines whether a ray intersects the geometry.
/// </summary>
/// <param name="transform">Transformation matrix of the geometry</param>
/// <param name="ray">The ray which will be tested for intersection.</param>
/// <param name="distance">When the method completes, contains the distance at which the ray intersected the plane.</param>
/// <param name="faceIndex">When the method completes, contains the index of face which the ray intersects.</param>
/// <returns><c>true</c> if the ray intersects the plane; otherwise, <c>false</c>.</returns>
public bool Intersects(Float4x4 transform, Ray ray, out float distance, out int faceIndex)
{
float u, v;
return Intersects(transform , ray, out distance, out faceIndex, out u, out v);
}
/// <summary>
/// Determines whether a ray intersects the geometry.
/// </summary>
/// <param name="transform">Transformation matrix of the geometry</param>
/// <param name="ray">The ray which will be tested for intersection.</param>
/// <param name="distance">When the method completes, contains the distance at which the ray intersected the plane.</param>
/// <param name="faceIndex">When the method completes, contains the index of face which the ray intersects.</param>
/// <param name="u">Barycentric U of face which the ray intersects.</param>
/// <param name="v">Barycentric V of face which the ray intersects.</param>
/// <returns><c>true</c> if the ray intersects the plane; otherwise, <c>false</c>.</returns>
public bool Intersects(Float4x4 transform, Ray ray, out float distance, out int faceIndex, out float u, out float v)
{
// Convert ray to model space
Float3 near = ray.Position;
Float3 dir = ray.Direction;
transform.Invert();
Float3 tmp = near;
Float3.TransformCoordinate(ref tmp, ref transform, out near);
tmp = dir;
Float3.TransformNormal(ref tmp, ref transform, out dir);
Ray modelSpaceRay = new Ray(near, dir);
// Test bounding sphere first
BoundingSphere bs = CalculateBoundingSphere(transform);
if (Ray.Intersects(ray, bs, out distance))
{
if (Indices != null && Indices.Length > 0)
{
// Intersect indexed geometry
for (faceIndex = 0; faceIndex < Indices.Length; faceIndex += 3)
{
Float3 vertex1 = Vertices[Indices[faceIndex]];
Float3 vertex2 = Vertices[Indices[faceIndex + 1]];
Float3 vertex3 = Vertices[Indices[faceIndex + 2]];
if (Ray.Intersects(modelSpaceRay, vertex1, vertex2, vertex3, out distance, out u, out v))
{
return true;
}
}
}
else
{
// Intersect non-indexed geometry
for (faceIndex = 0; faceIndex < Vertices.Length; faceIndex += 3)
{
Float3 vertex1 = Vertices[faceIndex];
Float3 vertex2 = Vertices[faceIndex + 1];
Float3 vertex3 = Vertices[faceIndex + 2];
if (Ray.Intersects(modelSpaceRay, vertex1, vertex2, vertex3, out distance, out u, out v))
{
return true;
}
}
}
}
faceIndex = -1;
distance = u = v = -1f;
return false;
}
/// <summary>
/// Determines whether a ray intersects the geometry.
/// </summary>
/// <param name="transform">Transformation matrix of the geometry</param>
/// <param name="ray">The ray which will be tested for intersection.</param>
/// <param name="distance">When the method completes, contains the distance at which the ray intersected the plane.</param>
/// <param name="faceIndex">When the method completes, contains the index of face which the ray intersects.</param>
/// <param name="hits">All intersection hits.</param>
/// <returns><c>true</c> if the ray intersects the plane; otherwise, <c>false</c>.</returns>
public bool Intersects(Float4x4 transform, Ray ray, out float distance, out int faceIndex, out IntersectInformation[] hits)
{
var hitsList = new List<IntersectInformation>();
float curDistance;
int curIndex;
distance = float.MaxValue;
faceIndex = -1;
// Create bounding sphere before inverting transform matrix
BoundingSphere bs = CalculateBoundingSphere(transform);
// Convert ray to model space
Float3 near = ray.Position;
Float3 dir = ray.Direction;
transform.Invert();
Float3 tmp = near;
Float3.TransformCoordinate(ref tmp, ref transform, out near);
tmp = dir;
Float3.TransformNormal(ref tmp, ref transform, out dir);
Ray modelSpaceRay = new Ray(near, dir);
// Test bounding sphere first
if (Ray.Intersects(ray, bs, out curDistance))
{
if (Indices != null && Indices.Length > 0)
{
// Intersect indexed geometry
for (curIndex = 0; curIndex < Indices.Length; curIndex += 3)
{
Float3 vertex1 = Vertices[Indices[curIndex]];
Float3 vertex2 = Vertices[Indices[curIndex + 1]];
Float3 vertex3 = Vertices[Indices[curIndex + 2]];
float u, v;
if (Ray.Intersects(modelSpaceRay, vertex1, vertex2, vertex3, out curDistance, out u, out v))
{
if (curDistance < distance)
{
distance = curDistance;
faceIndex = curIndex / 3;
}
var hit = new IntersectInformation
{
Distance = curDistance,
FaceIndex = faceIndex,
U = u,
V = v
};
hitsList.Add(hit);
}
}
}
else
{
// Intersect non-indexed geometry
for (curIndex = 0; curIndex < Vertices.Length; curIndex += 3)
{
Float3 vertex1 = Vertices[curIndex];
Float3 vertex2 = Vertices[curIndex + 1];
Float3 vertex3 = Vertices[curIndex + 2];
float u, v;
if (Ray.Intersects(modelSpaceRay, vertex1, vertex2, vertex3, out curDistance, out u, out v))
{
if (curDistance < distance)
{
distance = curDistance;
faceIndex = curIndex / 3;
}
var hit = new IntersectInformation
{
Distance = curDistance,
FaceIndex = faceIndex,
U = u,
V = v
};
hitsList.Add(hit);
}
}
}
}
hits = hitsList.ToArray();
return hits.Length > 0;
}
}
Also ray you have to get from picked pixel on screen rather than from near/far. Have a look to this code:
void CalculatePickRay(int x, int y, int width, int height, float near, Float4x4 view, Float4x4 projection, out Float3 pickRayDir, out Float3 pickRayOrig)
{
pickRayDir.X = (((2.0f * x) / width) - 1);
pickRayDir.Y = -(((2.0f * y) / height) - 1);
pickRayDir.Z = 1.0f;
projection.M41 = 0;
projection.M42 = 0;
projection.M43 = 0;
projection.M44 = 1;
projection.Invert();
Float3 tmp = pickRayDir;
Float3.TransformNormal(ref tmp, ref projection, out pickRayDir);
// Get the inverse view matrix
view.Invert();
tmp = pickRayDir;
Float3.TransformNormal(ref tmp, ref view, out pickRayDir);
pickRayDir.Normalize();
pickRayOrig.X = view.M41;
pickRayOrig.Y = view.M42;
pickRayOrig.Z = view.M43;
// calc origin as intersection with near frustum
pickRayOrig += pickRayDir * near;
}
