Don't compare the keyframes, compare the differences between the frames (and the key can be removed if the two differences are almost the same). Comparing dot products is usually better for quats.

bool canRemoveMiddleFrame(vec3 v0, vec3 v1, vec3 v2, float diff)
{
   vec3 va = v1 - v0;
   vec3 vb = v2 - v1;
   vec3 vc = va - vb;
   return (abs(vc.x) < diff) && (abs(vc.y) < diff) && (abs(vc.z) < diff);
}
bool canRemoveMiddleFrame(quat v0, quat v1, quat v2, float diff)
{
   quat va = v1 * invert(v0);
   quat vb = v2 * invert(v1);
   float difference = dot(va, vb);
   return abs( difference - 1 )  < diff;
}

Don't compare the keyframes, compare the differences between the frames (and the key can be removed if the two differences are almost the same). Comparing dot products is usually better for quats.

bool canRemoveMiddleFrame(vec3 v0, vec3 v1, vec3 v2, float diff)
{
   vec3 va = v1 - v0;
   vec3 vb = v2 - v1;
   vec3 vc = va - vb;
   return (abs(vc.x) < diff) && (abs(vc.y) < diff) && (abs(vc.z) < diff);
}
bool canRemoveMiddleFrame(quat v0, quat v1, quat v2, float diff)
{
   quat va = v1 * invert(v0);
   quat vb = v2 * invert(v1);
   float difference = dot(va, vb);
   return abs( difference - 1 )  < diff;
}

Don't compare the keyframes, compare the differences between the frames (and the key can be removed if the two differences are almost the same). Comparing dot products is usually better for quats.

bool canRemoveMiddleFrame(vec3 v0, vec3 v1, vec3 v2, float diff)
{
  vec3 va = v1 - v0;
  vec3 vb = v2 - v1;
  vec3 vc = va - vb;
  return (abs(vc.x) < diff) && (abs(vc.y) < diff) && (abs(vc.z) < diff);
}

bool canRemoveMiddleFrame(quat v0, quat v1, quat v2, float diff)
{
  vec3 va = v1 * invert(v0);
  vec3 vb = v2 * invert(v1);
  float difference = dot(va, vb);
  return abs( difference - 1 )  < diff;
}