OK, I'm bumping this thread because I'm revisiting the Fresnel equation, this time using complex IOR values. I'm having a hard time converting this to complex numbers.
float Fresnel(float CosThetaI, float n)
{
float CosThetaT = sqrt(max(0, 1 - (1 - CosThetaI * CosThetaI) / (n * n)));
float NCosThetaT = n * CosThetaT;
float NCosThetaI = n * CosThetaI;
float Rs = pow(abs((CosThetaI - NCosThetaT) / (CosThetaI + NCosThetaT)), 2);
float Rp = pow(abs((CosThetaT - NCosThetaI) / (CosThetaT + NCosThetaI)), 2);
return (Rs + Rp) / 2;
}
This is the basic formula, but I need to re-write it so that it looks like:
float Fresnel(float CosThetaI, vec3 n, vec3 k)
{
...
}
Where n and k make up the complex IOR (n + ki). I've taken a few stabs at it, but it's gotten me nowhere. Here is my train wreck of an attempt:
vec3 Fresnel(float CosThetaI, vec3 n, vec3 k)
{
float temp = 1 - CosThetaI * CosThetaI;
vec3 NKSqr_real = n * n - k * k;
vec3 NKSqr_imag = n * k * 2;
vec3 temp2_real = (temp * NKSqr_real) / (NKSqr_real * NKSqr_real + NKSqr_imag * NKSqr_imag);
vec3 temp2_imag = -(temp * NKSqr_imag) / (NKSqr_real * NKSqr_real + NKSqr_imag * NKSqr_imag);
temp2_real = 1 - temp2_real;
temp2_imag = -temp2_imag;
vec3 CosThetaT_real = sqrt((temp2_real + sqrt(temp2_real * temp2_real + temp2_imag * temp2_imag)) / 2);
vec3 CosThetaT_imag = sign(temp2_imag) * sqrt((-temp2_real + sqrt(temp2_real * temp2_real + temp2_imag * temp2_imag)) / 2);
vec3 NCosThetaT_real = n * CosThetaT_real - k * CosThetaT_imag;
vec3 NCosThetaT_imag = k * CosThetaT_real + n * CosThetaT_imag;
vec3 NCosThetaI_real = n * CosThetaI;
vec3 NCosThetaI_imag = k * CosThetaI;
vec3 CosThetaI_minus_NCosThetaT_real = CosThetaI - NCosThetaT_real;
vec3 CosThetaI_minus_NCosThetaT_imag = -NCosThetaT_imag;
vec3 CosThetaI_plus_NCosThetaT_real = CosThetaI + NCosThetaT_real;
vec3 CosThetaI_plus_NCosThetaT_imag = NCosThetaT_imag;
vec3 a, b, c, d;
a = CosThetaI_minus_NCosThetaT_real;
b = CosThetaI_minus_NCosThetaT_imag;
c = CosThetaI_plus_NCosThetaT_real;
d = CosThetaI_plus_NCosThetaT_imag;
vec3 Rs_real = (a * c + b * d) / (c * c + d * d);
vec3 Rs_imag = (b * c + a * d) / (c * c + d * d);
vec3 Rs = sqrt(Rs_real * Rs_real + Rs_imag * Rs_imag);
Rs = Rs * Rs;
vec3 CosThetaT_minus_NCosThetaI_real = CosThetaT_real - NCosThetaI_real;
vec3 CosThetaT_minus_NCosThetaI_imag = CosThetaT_imag - NCosThetaI_imag;
vec3 CosThetaT_plus_NCosThetaI_real = CosThetaT_real + NCosThetaI_real;
vec3 CosThetaT_plus_NCosThetaI_imag = CosThetaT_imag + NCosThetaI_imag;
a = CosThetaT_minus_NCosThetaI_real;
b = CosThetaT_minus_NCosThetaI_imag;
c = CosThetaT_plus_NCosThetaI_real;
d = CosThetaT_plus_NCosThetaI_imag;
vec3 Rp_real = (a * c + b * d) / (c * c + d * d);
vec3 Rp_imag = (b * c + a * d) / (c * c + d * d);
vec3 Rp = sqrt(Rp_real * Rp_real + Rp_imag * Rp_imag);
Rp = Rp * Rp;
return (Rs + Rp) / 2;
}
It would be so much easier if HLSL/GLSL had first class support for complex values. 
EDIT:
Never mind. I managed to come up with this.
vec2 CADD(vec2 a, vec2 b) { return a + b; }
vec2 CSUB(vec2 a, vec2 b) { return a - b; }
vec2 CMUL(vec2 a, vec2 b) { return vec2(a.x * b.x - a.y * b.y, a.y * b.x + a.x * b.y); }
vec2 CDIV(vec2 a, vec2 b) { return vec2((a.x * b.x + a.y * b.y) / (b.x * b.x + b.y * b.y), (a.y * b.x - a.x * b.y) / (b.x * b.x + b.y * b.y)); }
float CABS(vec2 a) { return sqrt(a.x * a.x + a.y * a.y); }
vec2 CSQRT(vec2 a) { return vec2(sqrt((a.x + sqrt(a.x * a.x + a.y * a.y)) / 2), sign(a.y) * sqrt((-a.x + sqrt(a.x * a.x + a.y * a.y)) / 2)); }
float _Fresnel(float _CosThetaI, vec2 n)
{
vec2 CosThetaI = vec2(_CosThetaI, 0);
vec2 CosThetaT = CSQRT(CSUB(vec2(1.0, 0), CDIV(CSUB(vec2(1.0, 0), CMUL(CosThetaI, CosThetaI)), CMUL(n, n))));
vec2 NCosThetaI = CMUL(n, CosThetaI);
vec2 NCosThetaT = CMUL(n, CosThetaT);
float Rs = pow(CABS(CDIV(CSUB(CosThetaI, NCosThetaT), CADD(CosThetaI, NCosThetaT))), 2);
float Rp = pow(CABS(CDIV(CSUB(CosThetaT, NCosThetaI), CADD(CosThetaT, NCosThetaI))), 2);
return (Rs + Rp) / 2;
}
vec3 Fresnel(float CosThetaI, vec3 n, vec3 k)
{
return vec3(
_Fresnel(CosThetaI, vec2(n.r, k.r)),
_Fresnel(CosThetaI, vec2(n.g, k.g)),
_Fresnel(CosThetaI, vec2(n.b, k.b))
);
}
