its not clear to me from your description whether this problem must be solved in 3d or can be solved in 2d.
if you just want to test if the lit area from a light is in view, you may be able to cull in 2d, perhaps with something as simple as bounding box distance.
if you're culling a large light sphere in 3d (hanging from a post, etc, FPS shooter view), you might try "cull if outside all 6 planes" vs "don't cull if within any of the 6 planes".
here's the code i use to cull large chunks to the frustum. the chunks are 300x300. the far plane is 1000. the frustum is NOT tilted up/down. chuncks are only clipped to the near, far, left, and right planes:
// calculate the planes
void Zcalc_frustum_planes2() // rotation only
{
D3DXMATRIX projectionMatrix, // ROCKLAND: copy of current projection matrix
viewMatrix, // ROCKLAND: copy of current view matrix
matrix,m; // ROCKLAND: frustum matrix ( view * modified proj )
projectionMatrix = Zprojection_matrix;
D3DXMatrixTranslation(&viewMatrix,-Zcamx,-Zcamy,-Zcamz);
D3DXMatrixRotationY(&m,-Zcamyr);
D3DXMatrixMultiply(&viewMatrix,&viewMatrix,&m);
D3DXMatrixMultiply(&matrix, &viewMatrix, &projectionMatrix);
// Calculate left plane of frustum.
m_planes2[2].a = matrix._14 + matrix._11;
m_planes2[2].b = matrix._24 + matrix._21;
m_planes2[2].c = matrix._34 + matrix._31;
m_planes2[2].d = matrix._44 + matrix._41;
D3DXPlaneNormalize(&m_planes2[2], &m_planes2[2]);
// Calculate right plane of frustum.
m_planes2[3].a = matrix._14 - matrix._11;
m_planes2[3].b = matrix._24 - matrix._21;
m_planes2[3].c = matrix._34 - matrix._31;
m_planes2[3].d = matrix._44 - matrix._41;
D3DXPlaneNormalize(&m_planes2[3], &m_planes2[3]);
// Calculate top plane of frustum.
m_planes2[4].a = matrix._14 - matrix._12;
m_planes2[4].b = matrix._24 - matrix._22;
m_planes2[4].c = matrix._34 - matrix._32;
m_planes2[4].d = matrix._44 - matrix._42;
D3DXPlaneNormalize(&m_planes2[4], &m_planes2[4]);
// Calculate bottom plane of frustum.
m_planes2[5].a = matrix._14 + matrix._12;
m_planes2[5].b = matrix._24 + matrix._22;
m_planes2[5].c = matrix._34 + matrix._32;
m_planes2[5].d = matrix._44 + matrix._42;
D3DXPlaneNormalize(&m_planes2[5], &m_planes2[5]);
// Calculate near plane of frustum.
m_planes2[0].a = matrix._13;
m_planes2[0].b = matrix._23;
m_planes2[0].c = matrix._33;
m_planes2[0].d = matrix._43;
D3DXPlaneNormalize(&m_planes2[0], &m_planes2[0]);
// Calculate far plane of frustum.
m_planes2[1].a = matrix._14 - matrix._13;
m_planes2[1].b = matrix._24 - matrix._23;
m_planes2[1].c = matrix._34 - matrix._33;
m_planes2[1].d = matrix._44 - matrix._43;
D3DXPlaneNormalize(&m_planes2[1], &m_planes2[1]);
}
// test corner vs plane.
int Zpoint_inside_frustum_plane(int x,int y,int z,int plane_index)
{
D3DXVECTOR3 v;
v.x=(float)x;
v.y=(float)y;
v.z=(float)z;
if ( D3DXPlaneDotCoord(&m_planes2[plane_index],&v) < 0.0f )
{
return(0);
}
return(1);
}
// test area vs plane.
int Zarea_inside_frustum_plane(int x1,int z1,int x2,int z2,int plane_index)
{
if ( Zpoint_inside_frustum_plane(x1,0,z1,plane_index) ||
Zpoint_inside_frustum_plane(x1,0,z2,plane_index) ||
Zpoint_inside_frustum_plane(x2,0,z1,plane_index) ||
Zpoint_inside_frustum_plane(x2,0,z2,plane_index) ) return(1);
return(0);
}
// test area vs near,far,left,and right planes...
int Zarea_in_frustum(int x1,int z1,int x2,int z2)
{
if (! Zarea_inside_frustum_plane(x1,z1,x2,z2,2)) return(0); // left
if (! Zarea_inside_frustum_plane(x1,z1,x2,z2,3)) return(0); // right
if (! Zarea_inside_frustum_plane(x1,z1,x2,z2,0)) return(0); // near
if (! Zarea_inside_frustum_plane(x1,z1,x2,z2,1)) return(0); // far
return(1);
}