3D Ugly HLSL output when converting pos->UV in VS, not in PS?

Recommended Posts

I'm an amateur, trying to learn HLSL techniques.  I'm currently trying to implement texture projection (making a movie projector) in a DX9 environment.

I'm running my vertices through an alternate view and projection and using that as UV coordinates on a texture.  However, I find that the coordinates are very different depending on whether I convert them from screen coordinates to texture coordinates in the vertex shader or the pixel shader and I don't know why.  I suspect it may have something to do with some kind of automatic conversions going on between the vertex shader and the pixel shader?

I don't care much about performance, but I really want to use the vertex shader for this calculation so that I can shadow the projection, shadow-buffer style.  But there are artifacts and clones that I can't live with.

I'm attaching two pics, one showing the artifacts when calculating UV coordinates in the vertex shader, one when calculating the UV coordinates in the pixel shader (which, other than shadowing, I'm happy with.)

Here is the almost-complete code (I'm leaving out the wide variety of technique calls that all look the same).  I'm never sure whether to whittle this down to what's relevant in order to save you some effort in understanding, or to leave it complete in case I turn out unqualified to be the one-that-whittles.  Here, there is a single line in the pixel shader that I'm uncommenting in order to replace the UV coordinates with those computed in the vertex shader.

I'm certain that there are a lot of other things that I'm doing poorly as well, and appreciate any extra recommendations.  I don't have access to the main executable, just the HLSL.

I greatly appreciate any help anyone is willing to offer.  Thanks for looking.

#define MOVIETEX "b.png"
//#define MOVIETEX "test.gif"
//#define MOVIETEX "NT.gif"

#define VSVRS vs_2_0
#define PSVRS ps_2_0    //animated textures don't work in v3.0
#define PI 3.14159265f
#define IDENTITYMATRIX {{1,0,0,0},{0,1,0,0},{0,0,1,0},{0, 0, 0, 1}}

#define BLACK float4(0,0,0,1)
#define CONT_MODEL_INSTANCE "Projector.pmx"

float4x4 cProjector    : CONTROLOBJECT < string name = CONT_MODEL_INSTANCE; string item = "Projector"; >;
float4 cFOV : CONTROLOBJECT < string name = CONT_MODEL_INSTANCE; string item = "FOV"; >;
float4 cBrightness : CONTROLOBJECT < string name = CONT_MODEL_INSTANCE; string item = "Brightness"; >;
float4 cCol : CONTROLOBJECT < string name = CONT_MODEL_INSTANCE; string item = "Color"; >;
float4 cNearFar : CONTROLOBJECT < string name = CONT_MODEL_INSTANCE; string item = "NearFar"; >;
float3 cZVec : CONTROLOBJECT < string name = CONT_MODEL_INSTANCE; string item = "NearFar"; >;
static float3 projWPos = float3(cProjector._41, cProjector._42, cProjector._43);

float4x4 WorldMatrix              : WORLD;
float4x4 ViewMatrix               : VIEW;
float4x4 ViewProjMatrix           : VIEWPROJECTION;
float4x4 WorldViewProjMatrix            : WORLDVIEWPROJECTION;
float4x4 ProjMatrix                        : PROJECTION;
float4 MaterialDiffuse   : DIFFUSE  < string Object = "Geometry"; >;
float3 MaterialAmbient   : AMBIENT  < string Object = "Geometry"; >;
float4 TextureAddValue  : ADDINGTEXTURE;
float4 TextureMulValue  : MULTIPLYINGTEXTURE;

texture MovieTex : ANIMATEDTEXTURE <
    string ResourceName = MOVIETEX;

sampler MovieSamp = sampler_state {
    texture = <MovieTex>;

texture ObjectTexture: MATERIALTEXTURE;
sampler ObjTexSampler = sampler_state {
    texture = <ObjectTexture>;

technique EdgeTec < string MMDPass = "edge"; > {        //disable

technique ShadowTec < string MMDPass = "shadow"; > {    //disable

technique ZplotTec <string MMDPass = "zplot";> {        //disable

float4x4 mat3tomat4 (float3x3 inpM) {
    float4x4 outp = IDENTITYMATRIX;
    outp._11 = inpM._11; outp._12 = inpM._12; outp._13 = inpM._13;
    outp._21 = inpM._21; outp._22 = inpM._22; outp._23 = inpM._23;
    outp._31 = inpM._31; outp._32 = inpM._32; outp._33 = inpM._33;
    outp._41 = 0.0f; outp._42 = 0.0f; outp._43 = 0.0f;
    outp._14 = 0.0f; outp._24 = 0.0f; outp._34 = 0.0f;
    return outp;

float4x4 invertTR4x4 (float4x4 inpM) {
//inverts a typical 4x4 matrix composed of only translations and rotations
    float4x4 invTr = IDENTITYMATRIX;
    invTr._41 = -inpM._41; invTr._42 = -inpM._42; invTr._43 = -inpM._43;
    float3x3 invRot3x3 = transpose((float3x3)inpM);
    float4x4 invRot4x4 = mat3tomat4(invRot3x3);
    float4x4 outpM = mul(invTr, invRot4x4);
    return outpM;

float4x4 getPerspProj (float2 Fov, float near, float far) {
//receives FOV in degrees
    Fov *= PI / 180.0f;
    Fov = 1.0f/Fov;
    float4x4 outp = IDENTITYMATRIX;
    outp._11 = atan(Fov.x/2.0f);
    outp._22 = atan(Fov.y/2.0f);
    outp._33 = -(far+near)/(far-near);
    outp._43 = (-2.0f*near*far)/(far-near);
    outp._34 = -1.0f;
    outp._44 = 0.0f;
    return outp;

struct BufferShadow_OUTPUT {
    float4 Pos      : POSITION;
    float4 PTex        : TEXCOORD0;        //texture coordinates in alternate projection
    float4 UV        : TEXCOORD1;
    float3 Normal   : TEXCOORD2;
    float3 PEye        : TEXCOORD3;
    float2 Tex        : TEXCOORD4;
    float4 wPos        : TEXCOORD5;
    float4 Color    : COLOR0;

BufferShadow_OUTPUT BufferShadow_VS(float4 Pos : POSITION, float3 Normal : NORMAL, float2 Tex : TEXCOORD0, float2 Tex2 : TEXCOORD1, uniform bool useTexture, uniform bool useSphereMap, uniform bool useToon)
    BufferShadow_OUTPUT Out = (BufferShadow_OUTPUT)0;
    Pos = mul( Pos, WorldMatrix );
    Out.PEye = cZVec - projWPos.xyz;  //easier than transforming Zvec
    Out.wPos = Pos;
    Out.Pos = mul(Pos, ViewProjMatrix);

    float4x4 invTR = invertTR4x4(cProjector);
    Out.PTex = mul(Pos, invTR);
    float4x4 altProj = getPerspProj((cFOV.xy)*cFOV.z, cNearFar.x, cNearFar.y);
    Out.PTex = mul(Out.PTex, altProj);
    Out.UV = Out.PTex;
    Out.UV.xyz /= Out.UV.w;
    Out.UV.x = (Out.UV.x + 0.5f)*2.0f;
    Out.UV.y = (-Out.UV.y + 0.5f)*2.0f;
    Out.UV.xy -= 0.5f;    //texture is centered on 0,0
    Out.Normal = normalize( mul( Normal, (float3x3)WorldMatrix ) );
    Out.Tex = Tex;
    Out.Color.rgb = MaterialAmbient;
    Out.Color.a = MaterialDiffuse.a;
    return Out;

float4 BufferShadow_PS(BufferShadow_OUTPUT IN, uniform bool useTexture, uniform bool useSphereMap, uniform bool useToon) : COLOR
    float4 Color = IN.Color;
    float3 PEn = normalize(IN.PEye); float3 Nn = normalize(IN.Normal);
    if ( useTexture ) {
        float4 TexColor = tex2D( ObjTexSampler, IN.Tex );
        TexColor.rgb = lerp(1, TexColor * TextureMulValue + TextureAddValue, TextureMulValue.a + TextureAddValue.a).rgb;
        Color *= TexColor;
    float4 UV = IN.PTex;
    UV.xyz /= UV.w;
    UV.x = (UV.x + 0.5f) *2.0f;
    UV.y = (-UV.y+0.5f) * 2.0f;
    UV.xy -= 0.5f;
    //uncommenting seems like it should provide same output yet doesn't
    //UV = IN.UV;
    float4 projTex = tex2D(MovieSamp, UV.xy);
    Color *= projTex;
    Color = projTex;
    Color.rgb *= pow(dot(Nn, PEn), 0.6f);
    Color.rgb *= cCol.rgb;
    Color.rgb *= cBrightness.x;
    if ((UV.z < 0.0f) || (UV.z > 1.0f) || (UV.x < 0.0f) || (UV.x > 1.0f) || (UV.y < 0.0f) || (UV.y > 1.0f)){
        return BLACK;
    //outside range; using border mode giving me artifacts i don't understand
    else {return Color;}

technique MainTecBS0 < string MMDPass = "object_ss"; bool UseTexture = false; bool UseSphereMap = false; bool UseToon = false; > {
    pass DrawObject {
        VertexShader = compile vs_3_0 BufferShadow_VS(false, false, false);
        PixelShader  = compile ps_3_0 BufferShadow_PS(false, false, false);




Share this post

Link to post
Share on other sites

The results of your UV calculation are not going to interpolate linearly across a triangle. In other words, Lerp(CalcUV(Vtx0), CalcUV(Vtx1)) != CalcUV(Lerp(Vtx0, Vtx1)). You can either do the whole calculation in the pixel shader, or you can do everything before the homogoneous divide-by-w in the vertex shader and then perform the divide and the rest in the pixel shader (you can actually bake the scale/offset stuff that goes from [-1, 1] -> [0, 1] range into your projection matrix, in which case you only need to do the divide-by-w in your pixel shader).

Share this post

Link to post
Share on other sites

Thanks, I think I see what you're saying.  If I understand correctly, the UV coordinates won't quite be the same if I apply them before the w dvide, but that's probably an error in my current version.  (This is the first time I've ever made a projection matrix, or even an inverse matrix, and I wanted to keep them clean.  Because I guess I'm scared I'll never be able to get close again :) But I'll make a new matrix and multiply it in before the w divide, which should let me make a shadow buffer.)


I don't know if you have any comments on anything else?  With that 3x3-matrix-to-4x4 function, it feels like there should be a better way, one that I just don't know about.  (The fov is also not what I'm treating it as, but that may be related to my scale+shift after divide.)

Share this post

Link to post
Share on other sites

Right: the results aren't the same under interpolation. You can imagine linear interpolation as taking two points on a 2D graph, and drawing a straight line through them (hence the "linear" name). For a linear function like y = 4x + 5 you could take two points, draw a line between them, and anywhere on that line will match the original function. However for a non-linear function like y = x^2, your straight line won't match the function and will therefore give you incorrect results. In your case you're dealing with a non-linear function due to using a perspective projection (linear transformations like rotation and translation are still ok).

You can do the 3x3->4x4 matrix a bit more concisely by using an alternate matrix constructor and some swizzles:

float4x4 mat3tomat4 (float3x3 inpM) {
    return float4x4(float4(inpM._11_12_13, 0.0f),
                    float4(inpM._21_22_23, 0.0f),
                    float4(inpM._31_32_33, 0.0f),
                    float4(0.0f, 0.0f, 0.0f, 0.0f));

The scale + shift is normal when you're calculating UV coordinates after applying a projection, since a typical projection matrix will give you coordinates in the range [-1, 1] while UV's are in [0, 1] space (since [-1, 1] matches the normalized device coordinates expected by the rasterizer). If you want to create a projection matrix that produces [0, 1] coordinates after division-by-w, you can transform your projection by a scale + translation matrix:

// Transforms from [-1,1] post-projection space to [0,1] UV space, and also
// flips the Y coordinate so that it puts [0, 0] in the top-left corner
float4x4 projScaleOffset = float4x4(float4(0.5f,  0.0f, 0.0f, 0.0f),
                                    float4(0.0f, -0.5f, 0.0f, 0.0f),
                                    float4(0.0f,  0.0f, 1.0f, 0.0f),
                                    float4(0.5f,  0.5f, 0.0f, 1.0f));
float4x4 proj = mul(proj, projScaleOffset);


Share this post

Link to post
Share on other sites
Posted (edited)

Thanks.  I'm doing everything but the divide in the VS now and it looks good.  I'm having some issues implementing my shadow buffer, but it's a situation where I'll probably need to play with it for a few days (sleeping on it always seems to help).  Appreciate the help regarding mat3tomat4: seeing that example will help me simplify other things that I do as well.


Edit: Oh, I misunderstood something, but I see now.  Rather than trying to fit my screen into to the (0-1, 0-1) range, I should try to fit my texture into -1 to 1 range.  Doing this after the w-divide is appropriate.


Edit2: I believe everything is working, but I need to do more testing to be sure, and make sure I'm handling things like alpha.  There is something extremely magical about making my own shadow buffer for the first time.  Thank you again for all of your help.

Edited by bandages

Share this post

Link to post
Share on other sites

Create an account or sign in to comment

You need to be a member in order to leave a comment

Create an account

Sign up for a new account in our community. It's easy!

Register a new account

Sign in

Already have an account? Sign in here.

Sign In Now

  • Forum Statistics

    • Total Topics
    • Total Posts
  • Similar Content

    • By DavidMT
      A team has a position open for a 3D artist and/or modeler.
      Excellent opportunity to gain experience working with a team remotely. Great chance to extend your portfolio Candidates, if successful have the chance to join the team permanently with entry-level financial benefits. Revenue Share What they expect
      Excellent communication skills Portfolio of previous work If you are interested please attach a CV and/or relevant works to jobs@iamdavidmt.com
    • By ForgedInteractive
      Who We Are
      We are Forged Interactive, a small team of like-minded game developers with the sole purpose of making games we love! Currently, we're progressing very quickly with our first project and there are plenty of opportunities and work for new interested programmers. With this project, our development platform is Unity 5.5.2 and C# as our behavioral language. Since this project is our first release, the game itself is a smaller project though progress is moving quickly. We are looking to finalize the current project and get started on future projects in the near future and are expanding our team to do so.
      Who We Are Looking For:
      Programmer Level Designer  
      About the Game
      Ours is the tale of two siblings, thrown into a world of chaos. Living in the shadow of their parents' heroic deeds and their Uncle's colorful military career, Finn and Atia are about to become the next force to shape our world. How will you rise through the ranks of Hereilla and what will be your legacy? Once defeated your enemies turn coat and join you in your adventures. Players can enjoy a range of troops and abilities based on their gameplay style which become more important as maps introduce more challenging terrain, enemies and bosses. Strong orc knights, dangerous shamans, and even a dragon are out on the prowl. Knowing when to fight and when to run, and how to manage your army is essential. Your actions alone decide the fate of this world.
      Previous Work by Team
      Although we are working towards our first game as Forged Interactive, our team members themselves have worked on titles including and not limited to:
      Final Fantasy Kingsglaive FIFA 2017 Xcom 2 Civilization  
      What do we expect?
      Reference work or portfolio. Examples what have you already done and what projects you have worked on academic or otherwise. The ability to commit to the project on a regular basis. If you are going on a two-week trip, we don't mind, but it would be good if you could commit 10+ hours to the project each week. Willingness to work with a royalty based compensation model, you will be paid when the game launches. Openness to learning new tools and techniques
      What can we offer?
      Continuous support and availability from our side. You have the ability to give design input, and creative say in the development of the game. Shown in credits on websites, in-game and more. Insight and contacts from within the Industry.
      If you are interested in knowing more or joining, please email or PM us on Skype. A member of our management team will reply to you within 48 hours.
      E-mail: Recruitment@ForgedInteractive.com
      Skype: ForgedInteractive
      David, Colin and Joseph
      Follow us on:
      Facebook: https://www.facebook.com/ForgedInteractive/
      Twitter: @ForgedInteract
      Youtube: https://www.youtube.com/channel/UCpK3zhq5ToOeDpdI0Eik-Ug?view_as=subscriber
      Reddit: www.reddit.com/user/Forged_Interactive

    • By Michael Pearson
      Hello all!   I'm currently in my third year on my 3D Animation & Games Development course, and I am in the process of doing some basic primary research for my dissertation project, which is to create a high quality 3D Environment for use in video games and potentially VR.   I have a questionnaire (targeting other artists in the field), and I would really appreciate it if you took some time to have a look and fill it out:   https://docs.google.com/forms/d/e/1FAIpQLScW12nFI8-fMlAUMNSQvOInxhnIfXpG91iRCm25TVlZufrvbQ/viewform?usp=sf_link   Thank you in advance!   Mike.
    • By i3DTutorials

      Game School Online™ Announces Winter Enrollment Open Through December
      Los Angeles, CA - December 2017
      Free Winter Term Enrollment!
      Free Winter Enrollment:
      Game School Online™, the first and only free online game development school of its kind, has opened enrollment for the winter term, which begins in January of 2018. GSO™ currently offers curricula in environment art and lighting for games, focusing specifically on Unreal Engine 4. GSO™ will be adding two new courses for the winter term, “Advanced Lighting Concepts with Unreal Engine 4” and “Advanced Hard Surface Modeling.”

      Free Workshop!

      December Free Workshop:
      In addition to winter enrollment, GSO™ will have a free online workshop, “Animating For Games and Movies”, with Veara Suon, Senior Animator at Double Negative. Veara has worked in the AAA games industry, working on franchises like Destiny and Bioshock. Veara also works in the film industry, having worked for famed studios like Weta, ILM, and Sony, contributing to films like Pacific Rim 2, Spiderman, and Avengers. In this month’s free GSO™ workshop, Veara will be discussing topics such as:
      Sharing techniques on how he animates for games and film
      Advice on how students can get a job in games or film
      Advice on how working professionals can switch from games to film, and vice versa
      Job relocation
      The workshop will last for an hour, with allocated time for Q & A from the audience. The workshop will take place on December 14 at 8pm PST. As usual, our monthly workshops are absolutely free and anyone is welcomed to join us. To RSVP, please visit our Facebook page here: https://www.facebook.com/events/139461393377301/?active_tab=about

      Scholars Available For Mentoring!
      Scholar Lineup:
      Scholars are our mentors, working industry professionals currently working on your favorite games and franchises- here to help you learn to be a professional game dev, with one on one live private sessions. Our current lineup of scholars includes:
      Brian Yam - Director of Visual Development @ Section Studios
      Olaf Piesche - Senior Engineer @ Epic Games
      Kevin DeBolt - World Artist @ 343 Industries
      Rosie Katz - Senior Game Designer @ EA, Visceral, Sledgehammer Games
      Leo Gonzalez - Senior Artist @ Certain Affinity
      Brandon Pham - Lead Environment Artist

      We believe that education should be free for everyone! Come join us at Game School Online™ and see what the future of game development education looks like today! For more information about Game School Online™, available courses, scholars, and events, please visit http://gameschoolonline.com/

      Facebook: https://www.facebook.com/gameschoolonline/
      Twitter: @GSOscholar
      Discord: https://discordapp.com/invite/BzU5Fq2
      Patreon: https://www.patreon.com/gameschoolonline

      About Game School Online™ (GSO™)
      Game School Online™ is the first ever, completely free game development training program. GSO™ was founded by veteran developers working in the AAA games and entertainment industries. Featuring courses authored by working industry professionals, students are able to learn production techniques and workflows used to ship some of the biggest IPs and AAA franchises. Industry pros known as “scholars”, work with students providing private live sessions to help you become “industry” ready. All courses are available for free, no trials or demos- it’s completely free for anyone that wants to learn how to become a skilled game developer. For more information, please visit our website: http://gameschoolonline.com/
      Game School Online™ and GSO™ are registered trademarks Game School Online, LLC. All other brand names, product names or trademarks belong to their respective holders.
      Game School Online, LLC.

      View full story
  • Popular Now