Hi,
I have been wrestling with FXAA for a few days now, and after getting updated to the latest version (3.11) I have finally gotten my FXAA shader to return usable values. I am using deferred rendering for the engine, and am running the FXAA shader immediately after combining my lightMap and Diffuse GBuffer, and outputting to a different renderTarget. The problem is, I am getting the below results; worse (and different) aliasing than before using it:




Do note, that if I comment out the "return FxaaPixelShader(...)" line for a standard texture sampler, the render looks like its normal aliased self (aka much better than the above). My code below comprises the area where I think there would have to be an issue. Any idea where it might be?

Note: I am using the Fxaa3_11.h file as is from Timothy Lottes, so the issue is definitely on my end.

[font=arial,helvetica,sans-serif]Renderer Call:[/font]
[source lang="csharp"]

void MakeFinal(GraphicsDevice GraphicsDevice, RenderTarget2D Output)
{
//Set Composition Target
GraphicsDevice.SetRenderTarget(finalRender);

//Clear
GraphicsDevice.Clear(Color.Transparent);

//Set Textures
GraphicsDevice.Textures[0] = GBufferTargets[0].RenderTarget;
GraphicsDevice.SamplerStates[0] = SamplerState.PointClamp;

GraphicsDevice.Textures[1] = lightRT;
GraphicsDevice.SamplerStates[1] = SamplerState.PointClamp;

//Set Effect Parameters
finalCombinedEffect.Parameters["GBufferTextureSize"].SetValue(GBufferTextureSize);

//Apply
finalCombinedEffect.CurrentTechnique.Passes[0].Apply();
//Draw
//game.GraphicsDevice.DepthStencilState = DepthStencilState.None;

quadRenderer.Draw(game.GraphicsDevice);

//Set Composition Target
GraphicsDevice.SetRenderTarget(Output);

//Clear
GraphicsDevice.Clear(Color.Transparent);

FXAA.Parameters["SCREEN_WIDTH"].SetValue(finalRender.Width);
FXAA.Parameters["SCREEN_HEIGHT"].SetValue(finalRender.Height);
FXAA.Parameters["gScreenTexture"].SetValue(finalRender as Texture2D);
//FXAA.Parameters["_rcpFrame"].SetValue(new Vector2(.4f, .4f));
FXAA.CurrentTechnique.Passes[0].Apply();
quadRenderer.Draw(game.GraphicsDevice);
}
[/source]

[font=arial, sans-serif]

"finalCombinedEffect" Pixel Shader (where Luma is encoded):[/font]
[source lang="HLSL"]float4 PS(VSO input) : COLOR0
{
//Sample Albedo
float3 Color = tex2D(Albedo, input.UV).xyz;

//Sample Light Map
float4 Lighting = tex2D(LightMap, input.UV);

//Accumulate to Final Color
float4 output = float4(Color.xyz * Lighting.xyz + Lighting.w, 1);

output.a = sqrt(dot(output.rgb, float3(0.299, 0.587, 0.114)));

//Return
return output;
//return FxaaPixelShader(input.pos, posPos, tex, rcpFrame, rcpFrameOpt);
}[/source]

[font=arial, sans-serif]

FXAAShader:[/font]
[source lang="HLSL"]uniform extern float SCREEN_WIDTH;
uniform extern float SCREEN_HEIGHT;
uniform extern texture gScreenTexture;

sampler screenSampler = sampler_state
{
Texture = <gScreenTexture>;
MinFilter = POINT;
MagFilter = POINT;
MipFilter = POINT;
AddressU = Clamp;
AddressV = Clamp;
};

#define FXAA_PC 1
//#define FXAA_GREEN_AS_LUMA 1

#if SHADER_MODEL >= 0x400
#if SHADER_MODEL >= 0x500
#define FXAA_HLSL_5 1
#else
#define FXAA_HLSL_4 1
#endif
#elif SHADER_MODEL <= 0x300
#define FXAA_HLSL_3 1
#endif

#include "Fxaa3_11.h"

float4 PS(float2 tc : TEXCOORD0) : COLOR0
{
float pixelWidth = (1 / SCREEN_WIDTH);
float pixelHeight = (1 / SCREEN_HEIGHT);

float2 pixelCenter = float2(tc.x, tc.y);
float4 fxaaConsolePosPos = float4(tc.x - .5f, tc.y - .5f,
tc.x + .5f, tc.y + .5f);

//return tex2D(screenSampler, tc);

return FxaaPixelShader(
tc,
fxaaConsolePosPos,
screenSampler,
screenSampler,
screenSampler,
float2(1.0f/SCREEN_WIDTH, 1.0f/SCREEN_HEIGHT),
float4(2.0 / SCREEN_WIDTH, 2.0 / SCREEN_HEIGHT, .5 / SCREEN_WIDTH, .5 / SCREEN_HEIGHT),
float4(0,0,0,0),
float4(0,0,0,0),
.75f,
.166f,
.08f,
8.0f,
.125f,
.05f,
float4(0,0,0,0));
}

//Technique
technique Default
{
pass p0
{
PixelShader = compile ps_3_0 PS();
}
}[/source]

In case it's not clear, the finalCombine shader above is run first, and drawn to my render target. I then use that render target as the source for the Pixel Shader in my Fxaa PixelShader (separate shader). Are you saying Luma needs to be encoded in the pixel shader above that calls FxaaPixelShader? It might be easiest if you could illustrate where it should be encoded, because there are 3 pixel shaders at work here: finalCombine, my FXAA pixel shader, and then the FxaaPixelShader, which is called by my FXAA pixel shader.

I will try Linear IP tonight, although I did try that and didn't do much, BUT I have changed stuff since then. Also, it seems that my code block for my FXAA shader got cut off (and is breaking the forums maybe ). Considering this is a pretty critical peice, I have copied it below without the code box. Might help you explain exactly where a (Luma) should be calculated.

My FXAA Shader (which calls FxaaPixelShader):[color=rgb(34,34,34)][background=rgb(255,255,255)]
uniform extern float SCREEN_WIDTH;[/background][color=rgb(34,34,34)][background=rgb(255,255,255)]
uniform extern float SCREEN_HEIGHT;[/background][color=rgb(34,34,34)][background=rgb(255,255,255)]
uniform extern texture gScreenTexture;[/background]
[color=rgb(34,34,34)][background=rgb(255,255,255)]
sampler screenSampler = sampler_state[/background][color=rgb(34,34,34)][background=rgb(255,255,255)]
{[/background][color=rgb(34,34,34)][background=rgb(255,255,255)]
Texture = <gScreenTexture>;[/background][color=rgb(34,34,34)][background=rgb(255,255,255)]
MinFilter = POINT;[/background][color=rgb(34,34,34)][background=rgb(255,255,255)]
MagFilter = POINT;[/background][color=rgb(34,34,34)][background=rgb(255,255,255)]
MipFilter = POINT;[/background][color=rgb(34,34,34)][background=rgb(255,255,255)]
AddressU = Clamp;[/background][color=rgb(34,34,34)][background=rgb(255,255,255)]
AddressV = Clamp;[/background][color=rgb(34,34,34)][background=rgb(255,255,255)]
};[/background]
[color=rgb(34,34,34)][background=rgb(255,255,255)]
#define FXAA_PC 1[/background][color=rgb(34,34,34)][background=rgb(255,255,255)]
//#define FXAA_GREEN_AS_LUMA 1[/background]
[color=rgb(34,34,34)][background=rgb(255,255,255)]
#if SHADER_MODEL >= 0x400[/background][color=rgb(34,34,34)][background=rgb(255,255,255)]
#if SHADER_MODEL >= 0x500[/background][color=rgb(34,34,34)][background=rgb(255,255,255)]
#define FXAA_HLSL_5 1[/background][color=rgb(34,34,34)][background=rgb(255,255,255)]
#else [/background][color=rgb(34,34,34)][background=rgb(255,255,255)]
#define FXAA_HLSL_4 1[/background][color=rgb(34,34,34)][background=rgb(255,255,255)]
#endif[/background][color=rgb(34,34,34)][background=rgb(255,255,255)]
#elif SHADER_MODEL <= 0x300[/background][color=rgb(34,34,34)][background=rgb(255,255,255)]
#define FXAA_HLSL_3 1[/background][color=rgb(34,34,34)][background=rgb(255,255,255)]
#endif[/background]
[color=rgb(34,34,34)][background=rgb(255,255,255)]
#include "Fxaa3_11.h"[/background]
[color=rgb(34,34,34)][background=rgb(255,255,255)]
float4 PS(float2 tc : TEXCOORD0) : COLOR0[/background][color=rgb(34,34,34)][background=rgb(255,255,255)]
{[/background][color=rgb(34,34,34)][background=rgb(255,255,255)]
float pixelWidth = (1 / SCREEN_WIDTH);[/background][color=rgb(34,34,34)][background=rgb(255,255,255)]
float pixelHeight = (1 / SCREEN_HEIGHT);[/background]
[color=rgb(34,34,34)][background=rgb(255,255,255)]
float2 pixelCenter = float2(tc.x, tc.y);[/background][color=rgb(34,34,34)][background=rgb(255,255,255)]
float4 fxaaConsolePosPos = float4(tc.x - .5f, tc.y - .5f, [/background][color=rgb(34,34,34)][background=rgb(255,255,255)]
tc.x + .5f, tc.y + .5f);[/background]
[color=rgb(34,34,34)][background=rgb(255,255,255)]
//return tex2D(screenSampler, tc);[/background]
[color=rgb(34,34,34)][background=rgb(255,255,255)]
return FxaaPixelShader([/background][color=rgb(34,34,34)][background=rgb(255,255,255)]
tc,[/background][color=rgb(34,34,34)][background=rgb(255,255,255)]
fxaaConsolePosPos,[/background][color=rgb(34,34,34)][background=rgb(255,255,255)]
screenSampler,[/background][color=rgb(34,34,34)][background=rgb(255,255,255)]
screenSampler,[/background][color=rgb(34,34,34)][background=rgb(255,255,255)]
screenSampler,[/background][color=rgb(34,34,34)][background=rgb(255,255,255)]
float2(1.0f/SCREEN_WIDTH, 1.0f/SCREEN_HEIGHT),[/background][color=rgb(34,34,34)][background=rgb(255,255,255)]
float4(2.0 / SCREEN_WIDTH, 2.0 / SCREEN_HEIGHT, .5 / SCREEN_WIDTH, .5 / SCREEN_HEIGHT),[/background][color=rgb(34,34,34)][background=rgb(255,255,255)]
float4(0,0,0,0),[/background][color=rgb(34,34,34)][background=rgb(255,255,255)]
float4(0,0,0,0),[/background][color=rgb(34,34,34)][background=rgb(255,255,255)]
.75f,[/background][color=rgb(34,34,34)][background=rgb(255,255,255)]
.166f,[/background][color=rgb(34,34,34)][background=rgb(255,255,255)]
.08f,[/background][color=rgb(34,34,34)][background=rgb(255,255,255)]
8.0f,[/background][color=rgb(34,34,34)][background=rgb(255,255,255)]
.125f,[/background][color=rgb(34,34,34)][background=rgb(255,255,255)]
.05f,[/background][color=rgb(34,34,34)][background=rgb(255,255,255)]
float4(0,0,0,0));[/background][color=rgb(34,34,34)][background=rgb(255,255,255)]
}[/background]
[color=rgb(34,34,34)][background=rgb(255,255,255)]
//Technique[/background][color=rgb(34,34,34)][background=rgb(255,255,255)]
technique Default[/background][color=rgb(34,34,34)][background=rgb(255,255,255)]
{[/background][color=rgb(34,34,34)][background=rgb(255,255,255)]
pass p0[/background][color=rgb(34,34,34)][background=rgb(255,255,255)]
{[/background][color=rgb(34,34,34)][background=rgb(255,255,255)]
PixelShader = compile ps_3_0 PS();[/background][color=rgb(34,34,34)][background=rgb(255,255,255)]
}[/background][color=rgb(34,34,34)][background=rgb(255,255,255)]
}[/background]

I know all of that is probably pretty hard to read, but I can't even go back and edit my posts, so I apologize. I guess something in my initial topic broke the forums. I'll let the admins know.

So I updated the code to use linear interpolation and am using the green channel as Luma (to be safe in case I am setting Luma incorrectly), but I'm not certain things are working correctly. Below is an image of what it looks like, as well as my new shader code. Note that I am using above average quality settings and still seeing obvious aliasing:




uniform extern float SCREEN_WIDTH;
uniform extern float SCREEN_HEIGHT;
uniform extern texture gScreenTexture;
sampler screenSampler = sampler_state
{
Texture = <gScreenTexture>;
MinFilter = LINEAR;
MagFilter = LINEAR;
MipFilter = LINEAR;
AddressU = Clamp;
AddressV = Clamp;
};
#define FXAA_PC 1
#define FXAA_QUALITY__PRESET 23
#define FXAA_GREEN_AS_LUMA 1
#if SHADER_MODEL >= 0x400
#if SHADER_MODEL >= 0x500
#define FXAA_HLSL_5 1
#else
#define FXAA_HLSL_4 1
#endif
#elif SHADER_MODEL <= 0x300
#define FXAA_HLSL_3 1
#endif
#include "Fxaa3_11.h"
//Texture2D Texture;
SamplerState TextureSampler;
cbuffer cb0
{
float4 _rcpFrame;
float4 _rcpFrameOpt;
};
//Vertex Input Structure
struct VSI
{
float3 Position : POSITION0;
float2 UV : TEXCOORD0;
};
struct FxaaVS_Output {
float4 Pos : SV_POSITION;
float2 Tex : TEXCOORD0; };
FxaaVS_Output FxaaVS(uint id : SV_VertexID)
{
FxaaVS_Output Output;
Output.Tex = float2((id << 1) & 2, id & 2);
Output.Pos = float4(Output.Tex * float2(2.0f, -2.0f) +
float2(-1.0f, 1.0f), 0.0f, 1.0f);
return Output;
}
sampler Texture : register(s0);
struct PS_OUTPUT
{
float4 c : COLOR;
};
float4 PS(FxaaVS_Output input) : COLOR0
{
float pixelWidth = (1 / SCREEN_WIDTH);
float pixelHeight = (1 / SCREEN_HEIGHT);
float2 pixelCenter = float2(input.Tex.x, input.Tex.y);
float4 fxaaConsolePosPos = float4(input.Tex.x - .5f, input.Tex.y - .5f,
input.Tex.x + .5f, input.Tex.y + .5f);
//FxaaTex =
//return float4(0,1,0,1);
return FxaaPixelShader(
pixelCenter,
input.Pos,
screenSampler,
screenSampler,
screenSampler,
float2(1.0f/SCREEN_WIDTH, 1.0f/SCREEN_HEIGHT),
float4(2.0 / SCREEN_WIDTH, 2.0 / SCREEN_HEIGHT, .5 / SCREEN_WIDTH, .5 / SCREEN_HEIGHT),
float4(0,0,0,0),
float4(0,0,0,0),
1.0f,
.063f,
.0833f,
8.0f,
.125f,
.05f,
float4(0,0,0,0));
}
//Technique
technique Default
{
pass p0
{
PixelShader = compile ps_3_0 PS();
//VertexShader = compile vs_3_0 VS();
}
}