Hey there,

So I have been having some problems with SSAO shader in Deffered Rendering for very long long time. And mostly due to this problem stopped doing anything for several months.

So this time decided to ask for help as would like to solve this nastiness and keep on learning different shader techniques.

I mostly tried to follow all SSAO tutorials and checked shader examples (https://github.com/SaschaWillems/Vulkan/blob/master/data/shaders/ssao)

I also profilled in RenderDoc and data seems very similar. But for some reason SSAO does not work.

Here is the screen with range check (the white framebuffer image show ssao outpout):

[attachment=34881:SSAO1.png]

Here is the image without range check (the dark framebuffer image show ssao output):

As you can see (its dark pitch black... No idea really why)

[attachment=34880:ssao2.png]

In first shader pass I generate textures for deffered shader and SSAO.

This code generates normals in vertex shader for SSAO:

mat3 normalMatrix = transpose(inverse(mat3(mvMatrix)));
vs_out.normal = normalMatrix * inNormal;

and I convert vertices into model space as my deffered calculation is in modelspace

vs_out.ws_coords =  vec3(ubo.modelMatrix * tmpPos);

In fragment shader I pass it like this:

outNormal = vec4(normalize(fs_in.normal) * 0.5 + 0.5, LinearizeDepth(gl_FragCoord.z));

also I compress the texture for better perfomance:

//Position
outvec0.x = packHalf2x16(fs_in.ws_coords.xy);

//Pos And Specular
outvec0.y = packHalf2x16(vec2(fs_in.ws_coords.z, specularTexture.x));

When I come to my SSAO pass (in fragment shader):

float SSAOAlgo0()
{
	ivec2 P1 = ivec2(inUV * textureSize(PosSpecularPacked, 0));

	uvec2 uvec2_PosSpecularPacked = texelFetch(PosSpecularPacked, P1, 0).rg;
	vec4 normalDepthTexture = texture(NormalDepth, inUV, 0);

    //Get position and depth texture
	vec2 tempPosition0 = unpackHalf2x16(uvec2_PosSpecularPacked.x);
	vec2 tempPosAndSpec = unpackHalf2x16(uvec2_PosSpecularPacked.y);


        vec3 fragPos = vec3(tempPosition0, tempPosAndSpec.x);

	//Convert frag pos to view space as the fragPos is in modelSpace at the moment
	fragPos = vec3(ubo.view * vec4(fragPos, 1.0f));
	//fragPos.y = -fragPos.y;

	//Get normal
	vec3 normal = normalize(normalDepthTexture.xyz * 2.0 - 1.0);

	//Random vec using noise lookup
	ivec2 texDim = textureSize(NormalDepth, 0); 
	ivec2 noiseDim = textureSize(texNoise, 0);
	const vec2 noiseUV = vec2(float(texDim.x)/float(noiseDim.x), float(texDim.y)/(noiseDim.y)) * inUV;  
	vec3 randomVec = texture(texNoise, noiseUV).xyz * 2.0 - 1.0;

    // Create TBN change-of-basis matrix: from tangent-space to view-space
    vec3 tangent = normalize(randomVec - normal * dot(randomVec, normal));
    vec3 bitangent = cross(normal, tangent);
    mat3 TBN = mat3(tangent, bitangent, normal);

    // Iterate over the sample kernel and calculate occlusion factor
    float f_occlusion = 0.0f;
    for(int i = 0; i < SSAO_KERNEL_SIZE; ++i)
    {
        // get sample position
        vec3 Sample = TBN * ubossaokernel.samples[i].xyz; // From tangent to view-space
        Sample = fragPos + Sample * SSAO_RADIUS; 
        
		
        // project sample position (to sample texture) (to get position on screen/texture)
        vec4 offset = vec4(Sample, 1.0f);
        offset = ubo.projection * offset; // from view to clip-space
        offset.xyz /= offset.w; // perspective divide
        offset.xyz = offset.xyz * 0.5f + 0.5f; // transform to range 0.0 - 1.0
        
		// get sample depth
        float sampleDepth = -texture(NormalDepth, offset.xy, 0).w; // Get depth value of kernel sample
			
        // range check & accumulate
//#define RANGE_CHECK
#ifdef  RANGE_CHECK
			float rangeCheck = smoothstep(0.0f, 1.0f, SSAO_RADIUS / abs(fragPos.z - sampleDepth ));
			f_occlusion += (sampleDepth >= Sample.z ? 1.0f : 0.0f) * rangeCheck;
#else
			f_occlusion += (sampleDepth >= Sample.z ? 1.0f : 0.0f);  
#endif
    }
    f_occlusion = 1.0f - (f_occlusion / float(SSAO_KERNEL_SIZE));
	return f_occlusion;
}

void main()
{
    FragColor = SSAOAlgo0();
}

SSAO kernel and Noise generation looks like this:

void Renderer::InitializeSSAOData()
{
	std::uniform_real_distribution<float> randomFloats(0.0f, 1.0f); // random floats between 0.0 - 1.0
	std::default_random_engine generator;

	for (uint32_t i = 0; i < 64; ++i)
	{
		glm::vec3 sample(
			randomFloats(generator) * 2.0f - 1.0f,
			randomFloats(generator) * 2.0f - 1.0f,
			randomFloats(generator)
		);
		sample = glm::normalize(sample);
		sample *= randomFloats(generator);

		float scale = scale = static_cast<float>(i) / 64.0;
		scale = lerp(0.1f, 1.0f, scale*scale);

		sample *= scale;
		uboSSAOKernel.ssaoKernel[i] = glm::vec4(sample, 0.0f);
	}

	std::vector<glm::vec4> ssaoNoise;
	ssaoNoise.reserve(16);
	for (uint32_t i = 0; i < 16; i++)
	{
		glm::vec4 noise(
			randomFloats(generator) * 2.0 - 1.0,
			randomFloats(generator) * 2.0 - 1.0,
			0.0f, 0.0f);
		ssaoNoise.push_back(noise);
	}

	//Generates texture. But cant see anything in debugger in it. Better generate on gpu... at the moment
	GenerateTexture(ssaoNoise, 4, 4, 1, VK_FORMAT_R32G32B32A32_SFLOAT, &m_NoiseGeneratedTexture, VK_IMAGE_USAGE_SAMPLED_BIT, VK_FILTER_NEAREST);

	//Send data
	void * pData;
	VK_CHECK_RESULT(vkMapMemory(m_pWRenderer->m_SwapChain.device, uniformData.ssaokernel.memory, 0, sizeof(uboSSAOKernel), 0, (void **)&pData));
	memcpy(pData, &uboSSAOKernel, sizeof(uboSSAOKernel));
	vkUnmapMemory(m_pWRenderer->m_SwapChain.device, uniformData.ssaokernel.memory);
}

The full code can be found in here: (https://github.com/TywyllSoftware/TywRenderer/tree/master/Projects/SSAO)

Thanks.