I am lost I can't figure out what to fix to get this lighting correct? I am using GLSL 1.2 and Irrlicht to load and calculate the meshes tangents and bitangents ect...

http://imageshack.com/a/img849/4338/255f.png

http://imageshack.com/a/img827/25/0klu.png

GLSL code

//VS
 
#version 120
uniform mat4 mWorldViewProj;
uniform mat4 mInvWorld;
uniform mat4 mTransWorld;
uniform vec3 mLightPos;
uniform vec4 mLightColor;
uniform sampler2D displacementMap;
 
//gl_MultiTexCoord1 is for Tangent Vectors with Irrlicht
//gl_MultiTexCoord2 is for BiTangent Vectors with Irrlicht
 
varying vec3 normal, tangent, bitangent;
varying vec3 lightPos;
 
void main()
{
    normal    = normalize(gl_NormalMatrix * gl_Normal);
    tangent   = normalize(gl_NormalMatrix * gl_MultiTexCoord1.xyz);
    bitangent = normalize(gl_NormalMatrix * gl_MultiTexCoord2.xyz);
    float scale = 10.0;
    vec4 dv = texture2DLod(displacementMap, vec2(gl_MultiTexCoord0), 0.0);
    float df = 0.30*dv.x + 0.59*dv.y + 0.11*dv.z;
    vec4 newVertexPos = vec4(gl_Normal * df * scale, 0.0) + gl_Vertex;
 
    lightPos = normalize(mLightPos - (vec3(gl_Vertex) * 1.0));
 
    gl_TexCoord[0] = gl_MultiTexCoord0;   
    gl_Position = mWorldViewProj * newVertexPos;
}
 
//FS

#version 120
uniform sampler2D decalMap, aoMap, normalMap;
 
varying vec3 lightPos;
varying vec3 normal, tangent, bitangent;
 
void main()
{
    vec3 N = normalize(normal);
    vec3 T = normalize(tangent);
    vec3 B = normalize(bitangent);
 
    vec3 nmap = (2.0 * texture2D(normalMap, gl_TexCoord[0].st).rgb - 1.0);
    nmap.z = sqrt(1.0 - dot(nmap.xy, nmap.xy));
    N = normalize((T * nmap.x) + (B * nmap.y) + (N * nmap.z));
 
   mat3 TBN = mat3(tangent, bitangent, normal);
   
   vec3 light_pos = normalize(lightPos);
 
   light_pos = light_pos * TBN;
 
   float diffuse = max(dot(N, light_pos), 0.0);
 
 
   vec3 ambient = texture2D(aoMap, gl_TexCoord[0].st).rgb;
   vec3 color = ambient * diffuse * texture2D(decalMap, gl_TexCoord[0].st).rgb; 
 
   color  +=(gl_LightSource[1].diffuse.rgb * diffuse);
 
   gl_FragColor = vec4(color, 1.0);
}

Your pixel shader looks suspicious.

You create your TBN matrix from the interpolated non-normalized vectors. I think that this isn't your intention, although the effect on the screen is minor.

You should transform your normal with the TBN matrix

...

mat3 TBN = mat3(T, B, N);

N = normalize((T * nmap.x) + (B * nmap.y) + (N * nmap.z)); <- this line looks strange. 
could be like : vec3 Normal = nmap * TBN; assuming that you want to transform the normal from tangent space to another space
 
light_pos = light_pos * TBN; //either transform the normal to some common space or transform the light to tangent space. Not the both.

What is the normal matrix you use in the vertex shader?

Cheers!

Fixed it! Here is the updated code...

If you want please look over the code see if anything looks wrong? The lights stay on now and looks good... but not sure the math is correct still.

THanks Kauna!!!

//VS
#version 120
uniform mat4 mWorldViewProj;
uniform mat4 mInvWorld;
uniform mat4 mTransWorld;
uniform vec3 mLightPos;
uniform vec4 mLightColor;
uniform sampler2D displacementMap;
 
varying vec3 normal, tangent, bitangent;
varying vec3 lightPos;
varying vec4 ambient;
 
void main()
{
normal    = normalize(gl_NormalMatrix * gl_Normal);
    tangent   = normalize(gl_NormalMatrix * gl_MultiTexCoord1.xyz);
    bitangent = normalize(gl_NormalMatrix * gl_MultiTexCoord2.xyz);
    
float scale = 10.0;
    vec4 dv = texture2DLod(displacementMap, vec2(gl_MultiTexCoord0), 0.0);
    float df = 0.30*dv.x + 0.59*dv.y + 0.11*dv.z;
    vec4 newVertexPos = vec4(gl_Normal * df * scale, 0.0) + gl_Vertex;
 
lightPos = mLightPos - vec3(gl_Vertex);
 
ambient = mLightColor;
 
    gl_TexCoord[0] = gl_MultiTexCoord0;   
    gl_Position = mWorldViewProj * newVertexPos;
}
 
//FS

#version 120
uniform sampler2D decalMap, aoMap, normalMap;
 
varying vec3 lightPos;
varying vec3 normal, tangent, bitangent;
varying vec4 ambient;
 
void main()
{
    vec3 N = normalize(normal);
    vec3 T = normalize(tangent);
    vec3 B = normalize(bitangent);
mat3 TBN = mat3(T, B, N);
vec3 nmap;
    
nmap = (2.0 * texture2D(normalMap, gl_TexCoord[0].st).rgb - 1.0);
    nmap.z = sqrt(1.0 - dot(nmap.xy, nmap.xy));
 
    N = nmap * TBN;
 
vec3 light_pos = normalize(lightPos);
light_pos = light_pos * TBN;
float diffuse = max(dot(N, light_pos), 0.0);
 
vec3 color = texture2D(decalMap, gl_TexCoord[0].st).rgb;
color *= (ambient.rgb * texture2D(aoMap, gl_TexCoord[0].st).rgb);
color *= diffuse;
 
gl_FragColor = vec4(color, 1.0);
}

 

According to your vertex shader light_pos is actually the light direction? So transforming it by the TBN matrix doesn't make sense, since the light isn't in the tangent space originally.

What kind of lighting you are looking for? directional or omni light?

What is the gl_NormalMatrix matrix?

Cheers!

Hmmm when I delete the call to

light_pos = light_pos * TBN;

the mesh goes black again so no that is needed...