Hi,

so normally I feel pretty competent about anything math related and can figure it out, but this issue is driving me nuts. I want a landscape to blend into the background at a given distance. Now simply using the linearized z values results in larger view distance in the corners than in the center (since the z value is the distance from the projection plane an not the actual camera position). So I figured I can apply that distance falloff in my deferred step. But it seems I'm incapable of calculating the correct "spherical distance". First I figured I'd do the brute force approach and reconstruct the world position of my fragment and calculate the distance that way. So this is how I calculate the world space coordinates in the shader:

[source]
vec3 worldspace(vec3 x) {
vec4 pos = invprojection*vec4(x, 1);
return pos.xyz / pos.w;
}
[/source]
where x is vec3(screen.xy, depth) (depth as read from the depth buffer)

It almost seems to work but it's always slightly off and the coordinates "creep" around when moving the camera. I did check that invprojection is actually the inverse (proj*invproj = identity). Also it seems to work when I transform a vector back and forth on the cpu side. As a result I now got the reverse effect that the view distance in the corners was lower...

After spending hours on trying to figure out what was wrong I started over and tried to do it without the explicit reverse transform:

[source]
float spherical(float z, float f) {
return f*(near_far.x + (near_far.y-near_far.x)*2.*near_far.x/(near_far.x + near_far.y - z * (near_far.y-near_far.x)))/near_far.y;
}
[/source]

where f is a correction factor that is calculated as sqrt(1+dot(screen.xy, screen.xy)), which should be the cosine between view and fragment direction.
Interestingly this gives the exact same result as my former approach... ARGH

Anyway:
1. is there something Obviously wrong with the way i calculate the world space coordinates?
2. Am I missing something else that prevents this from working that is not related to my actual distance calculation?
3. Is there a better (canonical) way to do this?

I simply cannot do it ARRRRGH. This is driving me nuts.

I actually want worldspace coordinates. But since I don't make the distinction between projection and modelview matrix that shouldn't matter anyway. Before I get back to my original problem I simply want my world space thing to work.

I already do the scaling of gl_FragCoords...

I tried to reduce my code to show the issue but it is far from being a "minimal" example sadly (see zip for a version with all the headers, should only depend on sfml).

#define GL_GLEXT_PROTOTYPES
#include <SFML/System.hpp>
#include <SFML/Window.hpp>

#include <GL/gl.h>
#include <GL/glu.h>

#include <iostream>

#include "include/MathMatrix.h"
#include "include/MathVector.h"
#include "include/GraphicsMatrices.h"
#include "include/GLSLShader.h"
#include "include/GLVBO2.h"

#include <boost/fusion/include/vector.hpp>

namespace fusion = boost::fusion;


int main(/*int argc, char *argv[]*/)
{
const int width = 1600;
const int height = 900;
const float ratio = float(width)/height;

sf::Window app(sf::VideoMode(width, height, 32), "Test"/*, sf::Style::None*/);
app.UseVerticalSync(true);
app.ShowMouseCursor(false);
sf::Clock clock;

//define shader
GLSLShader shader;
shader.setVertexShaderSource(
"#version 150\n"
"uniform mat4 projection;\n"
"in vec4 vertex;\n"
"in vec4 color;\n"
"out vec4 vertexcolor;\n"
"out vec3 vertexpos;\n"
"void main() {\n"
" vertexcolor = color;\n"
" vertexpos = vec3(vertex);\n"
" gl_Position = projection*vertex;\n"
"}\n"
);

shader.setFragmentShaderSource(
"#version 150\n"

"uniform mat4 invprojection;\n"
"in vec4 vertexcolor;\n"
"in vec3 vertexpos;\n"
"out vec4 FragColor;\n"

"vec3 worldspace(vec3 x) {\n"
" vec4 pos = invprojection*vec4(x, 1);\n"
" return pos.xyz / pos.w;\n"
"}\n"

"void main() {\n"
" vec3 pos = worldspace(vec3(2*gl_FragCoord.xy/vec2(1600,900)-1,gl_FragCoord.z));\n"
//~ " vec3 pos = vertexpos;\n" //uncomment this to see how it should look like
" float blend = float((int(pos.x) + int(pos.y) + int(pos.z))&1);\n"
" FragColor = vertexcolor*blend;\n"
"}\n"
);

shader.compileProgram();
shader.bindAttributeLocation(0, "vertex");
shader.bindAttributeLocation(1, "color");
shader.bindFragDataLocation(0, "FragColor");
shader.linkProgram();

//vbo for our cube

GlVBO<fusion::vector< Vector<float, 3>, Vector<float, 3> > > cube_vbo;
cube_vbo.addVertex(Vector<float,3>( 5.5, 5.5, 5.5), Vector<float,3>(1,0,0));
cube_vbo.addVertex(Vector<float,3>(-5.5, 5.5, 5.5), Vector<float,3>(1,0,0));
cube_vbo.addVertex(Vector<float,3>( 5.5,-5.5, 5.5), Vector<float,3>(1,0,0));
cube_vbo.addVertex(Vector<float,3>( 5.5,-5.5, 5.5), Vector<float,3>(1,0,0));
cube_vbo.addVertex(Vector<float,3>(-5.5, 5.5, 5.5), Vector<float,3>(1,0,0));
cube_vbo.addVertex(Vector<float,3>(-5.5,-5.5, 5.5), Vector<float,3>(1,0,0));

cube_vbo.addVertex(Vector<float,3>( 5.5, 5.5,-5.5), Vector<float,3>(1,0,0));
cube_vbo.addVertex(Vector<float,3>( 5.5,-5.5,-5.5), Vector<float,3>(1,0,0));
cube_vbo.addVertex(Vector<float,3>(-5.5, 5.5,-5.5), Vector<float,3>(1,0,0));
cube_vbo.addVertex(Vector<float,3>(-5.5, 5.5,-5.5), Vector<float,3>(1,0,0));
cube_vbo.addVertex(Vector<float,3>( 5.5,-5.5,-5.5), Vector<float,3>(1,0,0));
cube_vbo.addVertex(Vector<float,3>(-5.5,-5.5,-5.5), Vector<float,3>(1,0,0));

cube_vbo.addVertex(Vector<float,3>( 5.5, 5.5, 5.5), Vector<float,3>(0,1,0));
cube_vbo.addVertex(Vector<float,3>( 5.5, 5.5,-5.5), Vector<float,3>(0,1,0));
cube_vbo.addVertex(Vector<float,3>(-5.5, 5.5, 5.5), Vector<float,3>(0,1,0));
cube_vbo.addVertex(Vector<float,3>(-5.5, 5.5, 5.5), Vector<float,3>(0,1,0));
cube_vbo.addVertex(Vector<float,3>( 5.5, 5.5,-5.5), Vector<float,3>(0,1,0));
cube_vbo.addVertex(Vector<float,3>(-5.5, 5.5,-5.5), Vector<float,3>(0,1,0));

cube_vbo.addVertex(Vector<float,3>( 5.5,-5.5, 5.5), Vector<float,3>(0,1,0));
cube_vbo.addVertex(Vector<float,3>(-5.5,-5.5, 5.5), Vector<float,3>(0,1,0));
cube_vbo.addVertex(Vector<float,3>( 5.5,-5.5,-5.5), Vector<float,3>(0,1,0));
cube_vbo.addVertex(Vector<float,3>( 5.5,-5.5,-5.5), Vector<float,3>(0,1,0));
cube_vbo.addVertex(Vector<float,3>(-5.5,-5.5, 5.5), Vector<float,3>(0,1,0));
cube_vbo.addVertex(Vector<float,3>(-5.5,-5.5,-5.5), Vector<float,3>(0,1,0));

cube_vbo.addVertex(Vector<float,3>( 5.5, 5.5, 5.5), Vector<float,3>(0,0,1));
cube_vbo.addVertex(Vector<float,3>( 5.5,-5.5, 5.5), Vector<float,3>(0,0,1));
cube_vbo.addVertex(Vector<float,3>( 5.5, 5.5,-5.5), Vector<float,3>(0,0,1));
cube_vbo.addVertex(Vector<float,3>( 5.5, 5.5,-5.5), Vector<float,3>(0,0,1));
cube_vbo.addVertex(Vector<float,3>( 5.5,-5.5, 5.5), Vector<float,3>(0,0,1));
cube_vbo.addVertex(Vector<float,3>( 5.5,-5.5,-5.5), Vector<float,3>(0,0,1));

cube_vbo.addVertex(Vector<float,3>(-5.5, 5.5, 5.5), Vector<float,3>(0,0,1));
cube_vbo.addVertex(Vector<float,3>(-5.5, 5.5,-5.5), Vector<float,3>(0,0,1));
cube_vbo.addVertex(Vector<float,3>(-5.5,-5.5, 5.5), Vector<float,3>(0,0,1));
cube_vbo.addVertex(Vector<float,3>(-5.5,-5.5, 5.5), Vector<float,3>(0,0,1));
cube_vbo.addVertex(Vector<float,3>(-5.5, 5.5,-5.5), Vector<float,3>(0,0,1));
cube_vbo.addVertex(Vector<float,3>(-5.5,-5.5,-5.5), Vector<float,3>(0,0,1));
cube_vbo.build();


float alpha = 0.0f, beta = 0.0f;

const sf::Input &input = app.GetInput();
double t = clock.GetElapsedTime();
bool has_focus = true;
Vector<float, 3> position(0,50,0);

glEnable(GL_CULL_FACE);
glDisable(GL_BLEND);

while(app.IsOpened())
{
double dt = clock.GetElapsedTime() - t;
t += dt;

Matrix<float, 4, 4> rotation = RotationMatrix(beta, 1.f, 0.f, 0.f)*RotationMatrix(alpha, 0.f, 1.f, 0.f);

//input processing
if(input.IsKeyDown(sf::Key::W))
{
position -= 20.f*float(dt)*Transpose(SubMatrix<3,3>(rotation, 0, 0))*Vector<float, 3>(0,0,1);
}
if(input.IsKeyDown(sf::Key::S))
{
position += 20.f*float(dt)*Transpose(SubMatrix<3,3>(rotation, 0, 0))*Vector<float, 3>(0,0,1);
}
if(has_focus)
{
alpha += 0.004*(input.GetMouseX()-width/2);
beta += 0.004*(input.GetMouseY()-height/2);
app.SetCursorPosition(width/2, height/2);
}
sf::Event event;
while(app.GetEvent(event))
{
if(event.Type == sf::Event::LostFocus)
{
has_focus = false;
}
if(event.Type == sf::Event::GainedFocus)
{
has_focus = true;
}
if(event.Type == sf::Event::Closed)
{
app.Close();
}
if(event.Type == sf::Event::KeyPressed)
{
if(event.Key.Code == sf::Key::Escape)
{
app.Close();
}
}
}


//setup projection and its inverse

float view_dist = 128;

float s = 0.1;
float near = 5.f*s;
float far = view_dist;

Matrix<float, 4, 4> projection = FrustumMatrix(-3.f*s, 3.f*s, -3.f*s/ratio, 3.f*s/ratio, near, far);
rotation = RotationMatrix(beta, 1.f, 0.f, 0.f)*RotationMatrix(alpha, 0.f, 1.f, 0.f);
projection *= rotation;
projection *= TranslationMatrix(-position[0], -position[1], -position[2]);

Matrix<float, 4, 4> invprojection = TranslationMatrix(position[0], position[1], position[2]);
invprojection *= Transpose(rotation);
invprojection *= InverseFrustumMatrix(-3.f*s, 3.f*s, -3.f*s/ratio, 3.f*s/ratio, near, far);

std::cout << projection*invprojection << std::endl; // show that it is the actual inverse


//draw

glEnable(GL_DEPTH_TEST);
glClearColor(0, 0, 0, 0);
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);

shader.bindProgram();
shader.setUniformMatrix("projection", 4, 4, projection.raw());
shader.setUniformMatrix("invprojection", 4, 4, invprojection.raw());

cube_vbo.draw(GL_TRIANGLES);

app.Display();
}

return 0;
}




Well it turns out i also have to use 2*glFragCoord.z-1...

You should keep the calculation in view space, it is the same as in world space without error-prone performance killer
When you have reconstructed your view space vector, just take the length of it, that's it, the rest is a simple fog calculation, maybe combined with a linear fading at the far end to ensure that it fades away when no fog is available. Something like this:



vec3 view_vector = ...

// calculate normlized dist
float dist = min(1.0, length(view_vector) / far_z);

// calculate fog value 0=no fog, 1=dense fog
float fog_value = fog_calculation(dist);

// ensure to blend at the far clip plane, 10%
fog_value = 1.0- (1.0-fog_value) * (1.0-smoothstep(0.9,1.0,dist));