I am implement a ambient occlusion, which throw several rays to check how many rays intersect with geometry. in my pixel shader I throw rays, if a ray intersects with any triangles, one variable will be increased by 1 and will be used afterward for shading. I initialize the variable ( iNumberOfIntersectedRays in my case)as 0, and increase it by one in the condition mensioned above. the problem is that when I debug the shader, sometime the variable is -1 or -2, and it just comes randomly. very strange. what possible mistake I have made. following is part of my code, anyone can help my out, thanks. float4 AO_PS( in float3 inColour : COLOR0, in float3 inPosition:COLOR1, in float3 inNormal : COLOR2, in float2 vPosition: VPOS ) : COLOR0 { float3 indices; float3 direction = {0,0,1}; float iNumberOfRays = 5; int1 iNumberOfIntersectedRays; int iIntersected = 0; float shading = 0; float3 colorReturn; /*{*/ iNumberOfIntersectedRays = 0.0f; for( float i = 0.5; i < iNumberOfRays; i++) { direction = tex2Dlod(g_sampleRandom, float4((i) / 100.0,1.0, 0, 0)); //direction = inNormal + direction; //direction = direction / length(direction); //iNumberOfIntersectedRays = 0; if(dot(direction, inNormal)<0) { direction = direction * (-1); } iIntersected = 0; if(iNumberOfIntersectedRays<0) { iNumberOfIntersectedRays=0; } if (vPosition.x == 10 && vPosition.y == 3) // for debug { for (float j = 0.5; j < 12; j++) { colorReturn = float3 (0.0,0.0,1.0); indices.xyz = tex2Dlod(g_sampleIndex, float4((j) / g_indexTexSize, (j+1) / g_indexTexSize, 0, 0)).xyz; structVertex v1 = getVertexFromVertexMaps( indices, 0 ); structVertex v2 = getVertexFromVertexMaps( indices, 1 ); structVertex v3 = getVertexFromVertexMaps( indices, 2 ); if(calculateIntersection(inPosition, direction, v1.position,v2.position,v3.position) == 1 && iIntersected == 0) { iIntersected = 1; iNumberOfIntersectedRays += 1; } } } } /*if (vPosition.x == 10 && vPosition.y == 3) { colorReturn = float3 (1.0,0.0,0.0); } else {*/ shading = 1 - iNumberOfIntersectedRays / iNumberOfRays; colorReturn = float3(shading, shading, shading); //} return float4(colorReturn, 1.0); }

sorry, I made a mistake somewhere else.