Solved it. See http://gamedev.stackexchange.com/questions/73588/how-do-i-fix-my-planet-facing-camera for a pretty good explanation. I'll post the code here.

glm::mat4 trans; float factor = 1.0f; float real_vertical = vertical; m_horizontal += horizontal; m_vertical += vertical; while (m_horizontal > TWO_PI) { m_horizontal -= TWO_PI; } while (m_horizontal < -TWO_PI) { m_horizontal += TWO_PI; } if (m_vertical > MAX_VERTICAL) { m_vertical = MAX_VERTICAL; } else if (m_vertical < -MAX_VERTICAL) { m_vertical = -MAX_VERTICAL; } glm::quat world_axes_rotation = glm::angleAxis(m_horizontal * ONEEIGHTY_PI, glm::vec3(0.0f, 1.0f, 0.0f)); world_axes_rotation = glm::normalize(world_axes_rotation); world_axes_rotation = glm::rotate(world_axes_rotation, m_vertical * ONEEIGHTY_PI, glm::vec3(1.0f, 0.0f, 0.0f)); m_pole = glm::normalize(m_pole - glm::dot(m_orientation, m_pole) * m_orientation); glm::mat4 local_transform; local_transform[0] = glm::vec4(m_pole.x, m_pole.y, m_pole.z, 0.0f); local_transform[1] = glm::vec4(m_orientation.x, m_orientation.y, m_orientation.z, 0.0f); glm::vec3 tmp = glm::cross(m_pole, m_orientation); local_transform[2] = glm::vec4(tmp.x, tmp.y, tmp.z, 0.0f); local_transform[3] = glm::vec4(m_position.x, m_position.y, m_position.z, 1.0f); world_axes_rotation = glm::normalize(world_axes_rotation); m_view = local_transform * glm::mat4_cast(world_axes_rotation); m_direction = -1.0f * glm::vec3(m_view[2]); m_up = glm::vec3(m_view[1]); m_right = glm::vec3(m_view[0]); m_view = glm::inverse(m_view);