detail tessellation "jumping" surface

[Yours3!f 1533]

Hi,

I'm trying to implement detail tessellation as in the DirectX SDK's DirectX 11 section: Detail tessellation.
My problem is that when I'm displacing the tessellated vertices in the tessellation evaluation shader, the resulting surface keeps "jumping" up and down.

Video:

here's the source code:

[vertex shader]

#version 410

uniform mat4 modelview;
uniform mat3 normal_mat;
uniform float tex_repeat;

in vec4 in_vertex;
in vec3 in_normal;
in vec3 in_tangent;
in vec2 in_texture;

out cross_shader_data
{
vec2 tex_coord;
vec4 vs_position;
vec3 normal;
vec3 tangent;
vec3 bitangent;
} o;

void main()
{
o.normal = normal_mat * in_normal; //transform normals, tangents, and bitangents into view space
o.tangent = normal_mat * in_tangent;
o.bitangent = cross(o.normal, o.tangent);

o.tex_coord = in_texture * tex_repeat;

o.vs_position = modelview * in_vertex; //transform vertices into view space
}

[tessellation control shader]

#version 410

layout(vertices = 3) out; //3 vertices define a patch (that is a triangle)

uniform vec4 tess_level; //=vec4(16)
uniform mat4 proj;
uniform vec2 screen_size; //screen width, height

in cross_shader_data
{
vec2 tex_coord;
vec4 vs_position;
vec3 normal;
vec3 tangent;
vec3 bitangent;
} i[];

out cross_shader_data
{
vec2 tex_coord;
vec4 vs_position;
vec3 normal;
vec3 tangent;
vec3 bitangent;
} o[];

vec2 get_screen_space_pos(vec4 pos)
{
vec4 ps_position = proj * pos; //transform view space position to projection space
vec2 ndc_pos = ps_position.xy / ps_position.w; //tranfrom into normalized device coordinates
return (ndc_pos + 1.0) * 0.5 * screen_size; //convert ndc coordinates to [0...1], and scale them to [0...screen size]
}

void main()
{
vec2 ss_position_0 = get_screen_space_pos(i[0].vs_position); //get screen space positions for the vertices
vec2 ss_position_1 = get_screen_space_pos(i[1].vs_position);
vec2 ss_position_2 = get_screen_space_pos(i[2].vs_position);

vec4 tess_factor = tess_level.xxxy; //get initial tessellation factor

//calculate tessellation factors, depending on screen space position
//(screen space LOD)
tess_factor.x = tess_level.w * distance(ss_position_2, ss_position_1);
tess_factor.y = tess_level.w * distance(ss_position_2, ss_position_0);
tess_factor.z = tess_level.w * distance(ss_position_0, ss_position_1);
tess_factor.w = 0.33 * (tess_factor.x + tess_factor.y + tess_factor.z);

gl_TessLevelOuter[0] = tess_factor.x;
gl_TessLevelOuter[1] = tess_factor.y;
gl_TessLevelOuter[2] = tess_factor.z;

gl_TessLevelOuter[0] = tess_factor.w;

//copy data
o[gl_InvocationID].tex_coord = i[gl_InvocationID].tex_coord;
o[gl_InvocationID].vs_position = i[gl_InvocationID].vs_position;
o[gl_InvocationID].normal = i[gl_InvocationID].normal;
o[gl_InvocationID].tangent = i[gl_InvocationID].tangent;
o[gl_InvocationID].bitangent = i[gl_InvocationID].bitangent;
}

[tessellation evaluation shader]

#version 410

//make triangles using fractional odd spacing, vertices are in counter clockwise order
layout(triangles, fractional_odd_spacing, ccw) in;

uniform sampler2D texture1; //XYZ normals, W height
uniform mat4 proj;
uniform float tess_height_scale; //=0.04

in cross_shader_data
{
vec2 tex_coord;
vec4 vs_position;
vec3 normal;
vec3 tangent;
vec3 bitangent;
} i[];

out cross_shader_data
{
vec2 tex_coord;
vec4 vs_position;
vec3 normal;
vec3 tangent;
vec3 bitangent;
} o;

void main()
{
//compute the new vertex parameters using barycentric coordinates
o.tex_coord = gl_TessCoord.x * i[0].tex_coord + gl_TessCoord.y * i[1].tex_coord + gl_TessCoord.z * i[2].tex_coord;
o.vs_position = gl_TessCoord.x * i[0].vs_position + gl_TessCoord.y * i[1].vs_position + gl_TessCoord.z * i[2].vs_position;
o.normal = gl_TessCoord.x * i[0].normal + gl_TessCoord.y * i[1].normal + gl_TessCoord.z * i[2].normal;
o.tangent = gl_TessCoord.x * i[0].tangent + gl_TessCoord.y * i[1].tangent + gl_TessCoord.z * i[2].tangent;
o.bitangent = gl_TessCoord.x * i[0].bitangent + gl_TessCoord.y * i[1].bitangent + gl_TessCoord.z * i[2].bitangent;

//calculate heightmap lod using viewspace position
float heightmap_miplevel = clamp( (length(o.vs_position) - 100.0 ) / 100.0, 0.0, 3.0);

//sample height
float height = textureLod( texture1, o.tex_coord, heightmap_miplevel ).w;

//displace vertex along normal
o.vs_position.xyz += normalize(o.normal) * ( tess_height_scale * (height - 1.0) );

gl_Position = proj * o.vs_position;
}

[pixel shader]

#version 410

uniform sampler2D texture0;
uniform sampler2D texture1;
uniform sampler2D texture2;
uniform float inv_neg_far;

in cross_shader_data
{
vec2 tex_coord;
vec4 vs_position;
vec3 normal;
vec3 tangent;
vec3 bitangent;
} i;

out vec4 albedo;
out vec4 normal;
out vec4 depth;
out vec4 extra;

void main()
{
//simple bumpmapping
mat3 ts_to_vs = mat3( normalize(i.tangent),
normalize(i.bitangent),
normalize(i.normal) );

albedo = vec4(texture(texture0, i.tex_coord).xyz, texture(texture2, i.tex_coord).x);
normal = vec4(ts_to_vs * (texture(texture1, i.tex_coord).xyz * 2.0 - 1.0) * 0.5 + 0.5, 1.0);
depth.x = i.vs_position.z * inv_neg_far;
extra = vec4(0.0);
}


Best regards,
Yours3!f

[Yours3!f 1533]

ok, so I replaced the tessellated mesh with a plane as it was in the sample, and I saw that the plane there was subdivided a little bit, it wasn't just 2 triangles. After that the flickering (jumping) is gone, but not all of the triangles get tessellated. See screenshot.
Any ideas what am I doing wrong?

[Tsus 1186]

Hi!

Looks like here is some typo:
gl_TessLevelOuter[0] = tess_factor.w;

I guess it should rather be
gl_TessLevelInner[0] = tess_factor.w;
Does it?

Best regards

[Yours3!f 1533]

Hi!

Looks like here is some typo:
gl_TessLevelOuter[0] = tess_factor.w;

I guess it should rather be
gl_TessLevelInner[0] = tess_factor.w;
Does it?

Best regards

OMG lol this was the mistake I cannot believe this... I've been struggling with this for like a week xD
BIG THANK YOU Tsus!!!
here's a pic about it: