im using first suggestion to test if all points are outside of one frustum plane.

liek that (didint check it though)

[spoiler]

#ifndef frustumH
#define frustumH
#include "DxcMath.h"
#include "DxcMatrix.h"


const int F_LEFT 	= 0;
const int F_RIGHT 	= 1;
const int F_TOP 	= 2;
const int F_BOTTOM 	= 3;
const int F_NEAR 	= 4;
const int F_FAR 	= 5;

struct TFrustumPlane
{
	vec3  n;
	float d;
};

struct TFrustum
{
TFrustumPlane F[6];

void CalcFrustum(TSpecCamera * FPP_CAM, mat4 CAM_VIEW, int sw, int sh, float fovy, float z_near, float z_far, float aspect)
{
mat4 CAM_MODEL;
mat4 mvm =  CAM_MODEL * CAM_VIEW;
mvm.Transpose();
vec3 dirX, dirY;
	dirX.x = mvm.m[0];
	dirX.y = mvm.m[4];
	dirX.z = mvm.m[8];

	dirY.x =	mvm.m[1];
	dirY.y =	mvm.m[5];
	dirY.z =	mvm.m[9];


	float a = fovy / 2.0;
float cotangent = 1.0 / tan( a * imopi );

float ax = z_near / cotangent;

float screen_w = 2.0*ax;

float screen_h = screen_w;

screen_w = screen_w * aspect;


vec3 dir = FPP_CAM->ReturnFrontVector();

//move to lower left corner
vec3 A = FPP_CAM->pos;


vec3 NEAR_BOTTOM_LEFT  = (FPP_CAM->pos + dir * z_near) + (-dirX * (screen_w / 2.0)) + (-dirY * (screen_h/2.0));
vec3 NEAR_BOTTOM_RIGHT = (FPP_CAM->pos + dir * z_near) + (dirX * (screen_w / 2.0)) + (-dirY * (screen_h/2.0));
vec3 NEAR_TOP_LEFT  = (FPP_CAM->pos + dir * z_near) + (-dirX * (screen_w / 2.0)) + (dirY * (screen_h/2.0));
vec3 NEAR_TOP_RIGHT = (FPP_CAM->pos + dir * z_near) + (dirX * (screen_w / 2.0)) + (dirY * (screen_h/2.0));



ax = z_far / cotangent;
screen_w = 2.0*ax;
screen_h = screen_w;
screen_w = screen_w * aspect;


vec3 FAR_BOTTOM_LEFT  = (FPP_CAM->pos + dir * z_far) + (-dirX * (screen_w / 2.0)) + (-dirY * (screen_h/2.0));
vec3 FAR_BOTTOM_RIGHT = (FPP_CAM->pos + dir * z_far) + (dirX * (screen_w / 2.0)) + (-dirY * (screen_h/2.0));
vec3 FAR_TOP_LEFT  = (FPP_CAM->pos + dir * z_far) + (-dirX * (screen_w / 2.0)) + (dirY * (screen_h/2.0));
vec3 FAR_TOP_RIGHT = (FPP_CAM->pos + dir * z_far) + (dirX * (screen_w / 2.0)) + (dirY * (screen_h/2.0));

F[F_LEFT].n 	= Normal(A, FAR_BOTTOM_LEFT, FAR_TOP_LEFT, true);
F[F_RIGHT].n 	= Normal(A, FAR_BOTTOM_RIGHT, FAR_TOP_RIGHT, true);

F[F_TOP].n 		= Normal(A, FAR_TOP_LEFT, FAR_TOP_RIGHT, true);
F[F_BOTTOM].n 	= Normal(A, FAR_BOTTOM_LEFT, FAR_BOTTOM_RIGHT, true);

F[F_NEAR].n 	= Normal(NEAR_BOTTOM_LEFT, NEAR_TOP_LEFT, NEAR_TOP_RIGHT, true);
F[F_FAR].n 		= Normal(FAR_BOTTOM_LEFT, FAR_TOP_LEFT, FAR_TOP_RIGHT, true);

F[F_LEFT].d 	= getplaneD(F[F_LEFT].n, A);
F[F_RIGHT].n 	= getplaneD(F[F_RIGHT].n, A);

F[F_TOP].n 		= getplaneD(F[F_TOP].n, A);
F[F_BOTTOM].n 	= getplaneD(F[F_BOTTOM].n, A);

F[F_NEAR].n 	= getplaneD(F[F_NEAR].n, NEAR_BOTTOM_LEFT);
F[F_FAR].n 		= getplaneD(F[F_FAR].n, FAR_BOTTOM_LEFT);

vec3 cp = A + dir * (z_far / 2.0);

for (int i=0; i < 4; i++)
if (classifyapointagainstaplane(cp, F[i].n, F[i].d) != isBack)
{
F[i].n = -F[i].n;
F[i].d = getplaneD(F[i].n, A);
}


if (classifyapointagainstaplane(cp, F[F_NEAR].n, F[F_NEAR].d) != isBack)
{
F[F_NEAR].n = -F[F_NEAR].n;
F[F_NEAR].d = getplaneD(F[F_NEAR].n, NEAR_BOTTOM_LEFT);
}

if (classifyapointagainstaplane(cp, F[F_FAR].n, F[F_FAR].d) != isBack)
{
F[F_FAR].n = -F[F_FAR].n;
F[F_FAR].d = getplaneD(F[F_FAR].n, FAR_BOTTOM_LEFT);
}


}



bool BoxIsVisible(vec3 * verts, int len)
{
for (int i=0; i < 6; i++) //through all frustum planes, check if all points are outside that plane if yes then its invisible
{
	bool all_one_side = true;

for (int j=0; j < len; j++)
if ( classifyapointagainstaplane(verts[j], F[i].n, F[i].d) != isFront ) {all_one_side = false; break; }

if (all_one_side) return false;
}

return true;
}

};

#endif
 

[/spoiler]