How to calculate rotation matrix based on cone direction? I have a spotlight defined like this:

I'm creating vertices array for a cone to draw spot light shape and then I'm creating a buffer (line loop mode):

private static float[] createConeVerticesArray(float radius,
                                               float height,
                                               int segments)
{
    Array<Float> verticesArray = new Array<>();

    float angle = 2 * MathUtils.PI / segments;
    float cos = MathUtils.cos(angle);
    float sin = MathUtils.sin(angle);
    float cx = radius, cy = 0;

    for(int i = 0; i < segments; i++)
    {
        verticesArray.add(cx);
        verticesArray.add(cy);
        verticesArray.add(height);

        verticesArray.add(0f);
        verticesArray.add(0f);
        verticesArray.add(0f);

        verticesArray.add(cx);
        verticesArray.add(cy);
        verticesArray.add(height);

        float temp = cx;
        cx = cos * cx - sin * cy;
        cy = sin * temp + cos * cy;

        verticesArray.add(cx);
        verticesArray.add(cy);
        verticesArray.add(height);
    }
    verticesArray.add(cx);
    verticesArray.add(cy);
    verticesArray.add(height);

    cx = radius;
    cy = 0;
    verticesArray.add(cx);
    verticesArray.add(cy);
    verticesArray.add(height);

    float[] result = new float[verticesArray.size];
    for(int i = 0; i < verticesArray.size; i++)
    {
        result[i] = verticesArray.get(i);
    }

    return result;
}

Next step I'm creating three spot lights like this which represents 3 rotations around each axis:

	this.spotLights.add(new CSpotLight(
            new Color(1f, 0f, 0f, 1f), //Color
            new Vector3(0f, 0f, 5000f), //Position
            1f, //Intensity
            new Vector3(0f, 0f, -1f), //Direction
            15f, //Inner angle
            30f, //Outer angle
            4000f, //Radius
            1f)); //Attenuation
    this.spotLights.add(new CSpotLight(
            new Color(1f, 0f, 0f, 1f),
            new Vector3(0f, 5000f, 0f),
            1f,
            new Vector3(0f, -1f, 0f),
            15f,
            30f,
            4000f,
            1f));
    this.spotLights.add(new CSpotLight(
            new Color(1f, 0f, 0f, 1f),
            new Vector3(5000f, 0f, 0f),
            1f,
            new Vector3(-1f, 0f, 0f),
            15f,
            30f,
            4000f,
            1f));

And then I'm calculating quaternion for each spot light to create rotation matrix for model matrix (just for vertex shader purpose):

private static Quaternion quaternionLookRotation(Vector3 direction, Vector3 up)
{
    Vector3 vector = Pools.obtain(Vector3.class).set(direction).nor();
    Vector3 vector2 = Pools.obtain(Vector3.class).set(up).crs(vector).nor();
    Vector3 vector3 = Pools.obtain(Vector3.class).set(vector).crs(vector2);

    float m00 = vector2.x;
    float m01 = vector2.y;
    float m02 = vector2.z;
    float m10 = vector3.x;
    float m11 = vector3.y;
    float m12 = vector3.z;
    float m20 = vector.x;
    float m21 = vector.y;
    float m22 = vector.z;

    Pools.free(vector);
    Pools.free(vector2);
    Pools.free(vector3);

    float num8 = (m00 + m11) + m22;
    Quaternion quaternion = Pools.obtain(Quaternion.class);
    if (num8 > 0f)
    {
        float num = (float) Math.sqrt(num8 + 1f);
        quaternion.w = num * 0.5f;
        num = 0.5f / num;
        quaternion.x = (m12 - m21) * num;
        quaternion.y = (m20 - m02) * num;
        quaternion.z = (m01 - m10) * num;
        return quaternion;
    }
    if ((m00 >= m11) && (m00 >= m22))
    {
        float num7 = (float) Math.sqrt(((1f + m00) - m11) - m22);
        float num4 = 0.5f / num7;
        quaternion.x = 0.5f * num7;
        quaternion.y = (m01 + m10) * num4;
        quaternion.z = (m02 + m20) * num4;
        quaternion.w = (m12 - m21) * num4;
        return quaternion;
    }
    if (m11 > m22)
    {
        float num6 = (float) Math.sqrt(((1f + m11) - m00) - m22);
        float num3 = 0.5f / num6;
        quaternion.x = (m10+ m01) * num3;
        quaternion.y = 0.5f * num6;
        quaternion.z = (m21 + m12) * num3;
        quaternion.w = (m20 - m02) * num3;
        return quaternion;
    }
    float num5 = (float) Math.sqrt(((1f + m22) - m00) - m11);
    float num2 = 0.5f / num5;
    quaternion.x = (m20 + m02) * num2;
    quaternion.y = (m21 + m12) * num2;
    quaternion.z = 0.5f * num5;
    quaternion.w = (m01 - m10) * num2;
    return quaternion;
}

this.rotMatrix.set(quaternionLookRotation(
                spotLight.getDirection(),
                new Vector3(0, 1, 0)));

Two rotations from three works perfectly. The problem appears only in the third rotation axis which should faced point (0, 0, 0):

Not sure if I'm understanding you correctly and it's hard for me to analyze your code, but I can show you what works for me.

So as far as I know a quaternion for these purposes should represent an axis and a rotation. So I'm guessing what you want to do is take the cross product of your light cone vector (in your case looks like (0,0,1)), and a normalized vector formed by the source of each spotlight pointing to the point you are illuminating. Once you normalize this, you should get the axis for your quaternion. To get the rotation angle you can do a dot product of the same two vectors and take the arccos. Might have to flip the sign.

So then your quaternion gets built something like this (I guess you can optimize it a bit by also passing in the cos of the angle directly instead of doing the arccos and cos):

   TDLQuaternion(const TDLVector3D<T> &clAxis, T fTheta, EDLQuatAxisRotate clDummy) 
   { 
      fTheta /= 2.0;
      // Set vector part to clAxis * sin(fTheta) 
      this->TDLVector3D<T>::MultiplyScalar(clAxis,sin(fTheta));
      m_fW = cos(fTheta);
   }

So this is from my old code and it basically just sets the X, Y and Z to the axis * sin(angle) and W to the cos(angle). Then to use it to rotate your cone points you can do:

   TDLVector3D<T> Rotate(const TDLVector3D<T> &clVec)
   {
      return  TDLVector3D<T>( (T) 2.0 *      this->TDLVector3D<T>::DotProduct(clVec)     * (*this) ,
                            (  m_fW * m_fW - this->TDLVector3D<T>::DotProduct(*this)   ) * clVec   ,
                            (T) 2.0 * m_fW * this->TDLVector3D<T>::CrossProduct(clVec)             ,
                            EDLVectorSum::Yes); 
   }

In this code *this is basically the (X,Y,Z) part of the quaternion since my quaternion inherits from a 3D vector class and then I add on the W to it to get a quaternion . Also this particularTDLVector3D<T> constructor adds of the all the Xs, Ys and Zs of the 3 input vector to construct the final vector. After this you will also have to translate all the points by your spotlight positions.

You can also simply convert your quaternion to a rotation matrix as you did before and just rotate stuff that way and add in the translation at the same time. I can post the code if you want to see it, but it doesn't look like you need it since you are using quaternionLookRotation.

Hopefully you can glean something out of this. good luck.

@dragomirus18 (respectfully) your quaternion calculation is incomplete or wrong because of this:

this.rotMatrix.set(quaternionLookRotation(
                spotLight.getDirection(),
 // <--- when this is 0,-1,0
                new Vector3(0, 1, 0)));
                ...
	float m00 = 0
    float m01 = 0
    float m02 = 0
    float m10 = 0
    float m11 = 0
    float m12 = 0
    float m20 = 0
    float m21 = -1
    float m22 = 0
    ...
    	quaternion.x = 0.5f 
    	quaternion.y = 0
        quaternion.z = 0
        quaternion.w = 0.5f

this quaternion is no other than the Axis and Angle:

{ [ 1, 0, 0 ], 90degrees }

This is wrong. Allow me to explain.

When you look for a quaternion between 2 vectors va to vb (for example), you are looking for a quaternion OR an angle-axis notation that is going to rotate vector va to vector vb.

As an example:
Let's say my vector va is (1,0,0) and i want it to get to vb (0,1,0). This means I want to rotate va by +90 degrees around the Z axis (0,0,1) (Assuming that my rotation are counter-clockwise when positive).
So i could write this pseudo code:
Quaternion get_Quaternion(vector3 from /* this is va vector */, 
						  vector3 to /* this is vb vector */)
{
	Quaternion q = quaternionLookRotation(from, to)
	return q
}

This is good so far, but it doesn't cater for special cases when va == vb OR va == -vb

So now if we look for a quaternion to rotate your input vector from(0,-1,0) to input vector to(0,1,0), your quaternionLookRotation returns q = { [ 1, 0, 0 ], 90degrees }

which is incorrect because it is impossible to rotate vector (0,-1,0) about 90degrees around the X axis (1,0,0) in order to get to vector (0,1,0). It will never get there.

This is all because your calculation do not cater for when 2 vectors are equally directed or facing in opposite directions.

Now have a look at this solution:

// copied from: http://lolengine.net/blog/2014/02/24/quaternion-from-two-vectors-final​

/* Build a unit quaternion representing the rotation
 * from u to v. The input vectors need not be normalised. */
quat quat::fromtwovectors(vec3 u, 
							vec3 v)
{
    float norm_u_norm_v = sqrt(dot(u, u) * dot(v, v));
    float real_part = norm_u_norm_v + dot(u, v);
    vec3 w;

    if (real_part < 1.e-6f * norm_u_norm_v)
    {
        /* If u and v are exactly opposite, rotate 180 degrees
         * around an arbitrary orthogonal axis. Axis normalisation
         * can happen later, when we normalise the quaternion. */
        real_part = 0.0f;
        w = abs(u.x) > abs(u.z) ? vec3(-u.y, u.x, 0.f)
                                : vec3(0.f, -u.z, u.y);
    }
    else
    {
        /* Otherwise, build quaternion the standard way. */
        w = cross(u, v);
    }

    return normalize(quat(real_part, w.x, w.y, w.z));
}

It may not seem like so, but this solution caters for those special cases.

Try and change your code like this and see if it solves it for you.

Here are some more solutions:

// see line 54
https://github.com/toji/gl-matrix/blob/f0583ef53e94bc7e78b78c8a24f09ed5e2f7a20c/src/gl-matrix/quat.js#L54​

I hope this helps ;

have fun ?

EDIT: If you use it, the Eigen Math library also works fine: quat = Quaternion::FromTwoVectors(from, to)

Ok, thank you guys for help me to understand how quaternion exactly works :D
Here is my final code which works fine in any cases:

private static Quaternion rotationBetweenTwoVectors(Vector3 vec1, Vector3 vec2)
{
Quaternion result = Pools.obtain(Quaternion.class);
Vector3 u = Pools.obtain(Vector3.class).set(vec1).nor();
Vector3 v = Pools.obtain(Vector3.class).set(vec2).nor();
float dot = u.dot(v);

if(dot < -0.999999)
{
Vector3 tmp1 = Pools.obtain(Vector3.class).set(Vector3.X).crs(u);
if(tmp1.len() < 0.000001)
{
tmp1.set(Vector3.Y).crs(u);
}
tmp1.nor();
result.setFromAxisRad(tmp1, MathUtils.PI).nor();
Pools.free(tmp1);
}
else if(dot > 0.999999)
{
result.idt();
}
else
{
result.setFromCross(u, v);
}
Pools.free(u);
Pools.free(v);

return result;
}

Where setFromCross calculates axis from cross product of two vectors and angle like this:

final float dot = MathUtils.clamp(Vector3.dot(x1, y1, z1, x2, y2, z2), -1f, 1f);
final float angle = (float)Math.acos(dot);

Hope this will helps somebody ?

well done, that looks better ?