I have been trying to generate a view matrix from a matrix that contains a rotation and position. When it is generate and I use it for anything then all of the calculations for everything else seem to be off in the y direction. I seem to have narrowed it down to the generation of my view matrix and was hoping if anyone could point out the flaw in this.

/// generate the right, up and look at from the camera matrix
right.x = cameraMatrix->_11;
right.y = cameraMatrix->_12;
right.z = cameraMatrix->_13;

up.x = cameraMatrix->_21;
up.y = cameraMatrix->_22;
up.z = cameraMatrix->_23;

lookAt.x = cameraMatrix->_31;
lookAt.y = cameraMatrix->_32;
lookAt.z = cameraMatrix->_33;


/// normalize the vectors to keep them unit length
NormalizeVector(&lookAt, &lookAt);

Cross(&up, &lookAt, &right);
NormalizeVector(&up, &up);

Cross(&right, &up, &lookAt);
NormalizeVector(&right, &right);

/// build the view matrix
viewMatrix._11 = right.x;
viewMatrix._21 = right.y;
viewMatrix._31 = right.z;

viewMatrix._12 = up.x;
viewMatrix._22 = up.y;
viewMatrix._32 = up.z;

viewMatrix._13 = lookAt.x;
viewMatrix._23 = lookAt.y;
viewMatrix._33 = lookAt.z;

viewMatrix._41 = -Dot(&cameraPosition, &right);
viewMatrix._42 = -Dot(&cameraPosition, &up);
viewMatrix._43 = -Dot(&cameraPosition, &lookAt);
viewMatrix._44 = 1.0f;

/// make sure these are empty
viewMatrix._14 = 0.0f;
viewMatrix._24 = 0.0f;
viewMatrix._34 = 0.0f;

I thought it was the issue after being able to cross check all of the math in my calculations to go from screen coordinates to ray to line segment\plane intersection test. I checked the math against the SlimDX library and then the collision checks against the orange collision detection book

    Vector3 start = Unproject(Vector3(mouseX, mouseY, 0.0f), viewport.x, viewport.y, viewport.width,
        viewport.height, viewport.minZ, viewport.maxZ, worldViewProjection);

    Vector3	end = Unproject(Vector3(mouseX, mouseY, 1.0f), viewport.x, viewport.y, viewport.width,
        viewport.height, viewport.minZ, viewport.maxZ, worldViewProjection);

    float t = 0.0f;
    return interestSegmentPlane(start, end, fPlaneD, vPlaneNormal, t, vCollisionPoint);


bool interestSegmentPlane(const Vector3 &a, const Vector3 &b, const float &pd, const Vector3 &pn, float &t, Vector3 &q)
{
    Vector3 ab = b-a;
    t = (pd - Vec3Dot(&pn, &a)) / Vec3Dot(&pn, &ab);

    if(t >= 0.0f && t <= 1.0f)
    {
        q = a + t * ab;
        return true;
    }

    return false;
}

Vector3 Unproject(const Vector3 &vVector, float viewportx, float viewportY, float viewportWidth, 
                          float viewportHeight, float viewportMinZ, float viewportMaxZ,
                          const Matrix& worldViewProj)
{
    Vector3 vResult;
    Vector3 vWorking;

    Matrix invViewProjMatrix;
    MatrixInverse(&invViewProjMatrix, worldViewProj);

    vWorking.x = (((vVector.x - viewportX) / viewportWidth) * 2.0f) - 1.0f;
    vWorking.y = -((((vVector.y - viewportY) / viewportHeight) * 2.0f) - 1.0f);
    vWorking.z = (vVector.z - viewportMinZ) / (viewportMaxZ - viewportMinZ);

    Vec3TransformCoord(&vResult, &vWorking, &invViewProjMatrix);

    return vResult;
}

Vector3* Vec3TransformCoord(Vector3* result, const Vector3* coordinate, const Matrix* transform)
{
    Vector4 vWorking;

    vWorking.x = (coordinate->x * transform->_11) + (coordinate->y * transform->_21) + (coordinate->z * transform->_31) + transform->_41;
    vWorking.y = (coordinate->x * transform->_12) + (coordinate->y * transform->_22) + (coordinate->z * transform->_32) + transform->_42;
    vWorking.z = (coordinate->x * transform->_13) + (coordinate->y * transform->_23) + (coordinate->z * transform->_33) + transform->_43;
    vWorking.w = 1.0f / ((coordinate->x * transform->_14) + (coordinate->y * transform->_24) + (coordinate->z * transform->_34) + transform->_44);

    result->x = vWorking.x * vWorking.w;
    result->y = vWorking.y * vWorking.w;
    result->z = vWorking.z * vWorking.w;

    return result;
}