Strange behaviour in vertex shader with divisions

Question

This is the vertex shader code:

#version 330 core

layout (location = 0) in vec3 position;
layout (location = 1) in vec3 myColor;

out vec3 MyColor;

void main()
{
    MyColor = myColor;
    vec3 v = position;
    v.x = v.x/v.z;
    v.y = v.y/v.z;
    gl_Position = vec4(v, 1.0f);
}

This shader is fairly simple, but when I draw a square at the bottom instead of leaving the screen empty draws this shapes and I was wondering why does it happen:

The following is an example code showing this behaviour so you only have to uncomment certain code snippets for checking it:

#include <iostream>

#define GLEW_STATIC
#include <GL/glew.h>

#include <GLFW/glfw3.h>

int main()
{
    //INIT LIBRARIES
    glfwInit();
    glfwWindowHint(GLFW_CONTEXT_VERSION_MAJOR, 3);
    glfwWindowHint(GLFW_CONTEXT_VERSION_MINOR, 3);
    glfwWindowHint(GLFW_OPENGL_PROFILE, GLFW_OPENGL_CORE_PROFILE);
    glfwWindowHint(GLFW_RESIZABLE, GL_FALSE);
    GLFWwindow* window = glfwCreateWindow(200 ,200 , "", nullptr, nullptr);
    glfwMakeContextCurrent(window);
    glewExperimental = GL_TRUE;
    glewInit();
    glViewport(0, 0, 200, 200);



    //SHADERS
    const GLchar* VERTEX_SHADER_SOURCE =
        "#version 330 core                      \n"
        "                                       \n"
        "layout (location = 0) in vec3 position;\n"
        "layout (location = 1) in vec3 myColor; \n"
        "                                       \n"
        "out vec3 MyColor;                      \n"
        "                                       \n"
        "void main()                            \n"
        "{                                      \n"
        "   MyColor = myColor;                  \n"
        "   vec3 v = position;                  \n"
        "   v.x = v.x/v.z;                      \n"
        "   v.y = v.y/v.z;                      \n"
        "   gl_Position = vec4(v, 1.0f);        \n"
        "}                                      \n";

    const GLchar* FRAGMENT_SHADER_SOURCE =
        "#version 330 core              \n"
        "                               \n"
        "in vec3 MyColor;               \n"
        "                               \n"
        "out vec4 color;                \n"
        "                               \n"
        "void main()                    \n"
        "{                              \n"
        "   color = vec4(MyColor,1.0f); \n"
        "}                              \n";

    GLuint program;
    GLuint vertex, fragment;

    vertex = glCreateShader(GL_VERTEX_SHADER);
    glShaderSource(vertex, 1, &VERTEX_SHADER_SOURCE, NULL);
    glCompileShader(vertex);

    fragment = glCreateShader(GL_FRAGMENT_SHADER);
    glShaderSource(fragment, 1, &FRAGMENT_SHADER_SOURCE, NULL);
    glCompileShader(fragment);

    program = glCreateProgram();
    glAttachShader(program, vertex);
    glAttachShader(program, fragment);
    glLinkProgram(program);

    glDeleteShader(vertex);
    glDeleteShader(fragment);
    glUseProgram(program);

    //VAOs & VBOs
    GLfloat vertices[] =
//Uncomment one of the following set of vertices to see the program in action
//The last one produces the strange behaviour
    /*{
            //This set will show you a square occuping the full window
            -0.5f, -0.5f,  0.5f,        1.0f, 0.0f, 0.0f,
            -0.5f,  0.5f,  0.5f,        0.0f, 1.0f, 0.0f,
             0.5f, -0.5f,  0.5f,        0.0f, 0.0f, 1.0f,
             0.5f,  0.5f,  0.5f,        1.0f, 1.0f, 0.0f
    };/**/
    /*{
            //This set draw a square at the bottom
             0.5f, -0.5f,  0.5f,        1.0f, 0.0f, 0.0f,
            -0.5f, -0.5f,  0.5f,        0.0f, 1.0f, 0.0f,
             0.5f, -0.5f,  1.0f,        0.0f, 0.0f, 1.0f,
            -0.5f, -0.5f,  1.0f,        1.0f, 1.0f, 0.0f
    };/**/
    /*{
            //This set draw the square to the left (Z is negative therefore z/x is also negative and the x value is inversed)
            -0.5f, -0.5f, -0.5f,        1.0f, 0.0f, 0.0f,
            -0.5f,  0.5f, -0.5f,        0.0f, 1.0f, 0.0f,
            -0.5f, -0.5f, -1.0f,        0.0f, 0.0f, 1.0f,
            -0.5f,  0.5f, -1.0f,        1.0f, 1.0f, 0.0f
    };/**/
    {
            //This set should not draw anything because y = y/z -> y = -0.5/[-0.25,+0.25] -> y = (-inf,-4] U [4,inf)
            //[-0.25,+0.25] means any value between -0.25 and 0.25
            //(-inf,-4] U [4,inf) means any value that is not between -4 and 4
            //Whathever value 'y' gets is out of the range [-1,1] so it should not be visible, but instead a figure is drawn causing the strange behaviour
             0.5f, -0.5f, -0.25f,       1.0f, 0.0f, 0.0f,
            -0.5f, -0.5f, -0.25f,       0.0f, 1.0f, 0.0f,
             0.5f, -0.5f,  0.25f,       0.0f, 0.0f, 1.0f,
            -0.5f, -0.5f,  0.25f,       1.0f, 1.0f, 0.0f
    };/**/

    GLuint VAO, VBO;
    glGenVertexArrays(1, &VAO);
    glBindVertexArray(VAO);
    glGenBuffers(1, &VBO);
    glBindBuffer(GL_ARRAY_BUFFER, VBO);
    glBufferData(GL_ARRAY_BUFFER, sizeof(vertices), vertices, GL_STATIC_DRAW);
    glVertexAttribPointer(0, 3, GL_FLOAT, GL_FALSE, 6 * sizeof(GLfloat), (GLvoid*)0);
    glEnableVertexAttribArray(0);
    glVertexAttribPointer(1, 3, GL_FLOAT, GL_FALSE, 6 * sizeof(GLfloat), (GLvoid*)(3*sizeof(GLfloat)));
    glEnableVertexAttribArray(1);


    //DRAW
    glClear(GL_COLOR_BUFFER_BIT);
    glDrawArrays(GL_TRIANGLE_STRIP, 0,4);
    glfwSwapBuffers(window);
    while (!glfwWindowShouldClose(window))
        glfwPollEvents();



    //END
    glDeleteVertexArrays(1, &VAO);
    glDeleteBuffers(1, &VBO);
    glfwTerminate();

    return 0;
}

EDIT The following link is a youtube video that will show you how the square gets transformed with different Z values:

https://youtu.be/S4TsfZjCIbM

And this is the code that I used for the video:

#include <iostream>

#define GLEW_STATIC
#include <GL/glew.h>

#include <GLFW/glfw3.h>

int main()
{
    //INIT LIBRARIES
    glfwInit();
    glfwWindowHint(GLFW_CONTEXT_VERSION_MAJOR, 3);
    glfwWindowHint(GLFW_CONTEXT_VERSION_MINOR, 3);
    glfwWindowHint(GLFW_OPENGL_PROFILE, GLFW_OPENGL_CORE_PROFILE);
    glfwWindowHint(GLFW_RESIZABLE, GL_FALSE);
    GLFWwindow* window = glfwCreateWindow(400 ,400 , "", nullptr, nullptr);
    glfwMakeContextCurrent(window);
    glewExperimental = GL_TRUE;
    glewInit();
    glViewport(0, 0, 400, 400);



    //SHADERS
    const GLchar* VERTEX_SHADER_SOURCE =
        "#version 330 core                      \n"
        "                                       \n"
        "layout (location = 0) in vec3 position;\n"
        "layout (location = 1) in vec3 myColor; \n"
        ""
        "uniform float movement;"
        "                                       \n"
        "out vec3 MyColor;                      \n"
        "                                       \n"
        "void main()                            \n"
        "{                                      \n"
        "   MyColor = myColor;                  \n"
        "   vec3 v = position;                  \n"
        "   v.z = v.z - movement;               \n"
        "   v.x = v.x/v.z;                      \n"
        "   v.y = v.y/v.z;                      \n"
        "   gl_Position = vec4(v, 1.0f);        \n"
        "}                                      \n";

    const GLchar* FRAGMENT_SHADER_SOURCE =
        "#version 330 core              \n"
        "                               \n"
        "in vec3 MyColor;               \n"
        "                               \n"
        "out vec4 color;                \n"
        "                               \n"
        "void main()                    \n"
        "{                              \n"
        "   color = vec4(MyColor,1.0f); \n"
        "}                              \n";

    GLuint program;
    GLuint vertex, fragment;

    vertex = glCreateShader(GL_VERTEX_SHADER);
    glShaderSource(vertex, 1, &VERTEX_SHADER_SOURCE, NULL);
    glCompileShader(vertex);

    fragment = glCreateShader(GL_FRAGMENT_SHADER);
    glShaderSource(fragment, 1, &FRAGMENT_SHADER_SOURCE, NULL);
    glCompileShader(fragment);

    program = glCreateProgram();
    glAttachShader(program, vertex);
    glAttachShader(program, fragment);
    glLinkProgram(program);

    glDeleteShader(vertex);
    glDeleteShader(fragment);

    GLfloat uniform = glGetUniformLocation(program, "movement");
    glUseProgram(program);



    //VAOs & VBOs
    GLfloat vertices[] =
    {
             0.25f, -0.25f,  1.0f,      1.0f, 0.0f, 0.0f,
            -0.25f, -0.25f,  1.0f,      0.0f, 1.0f, 0.0f,
             0.25f, -0.25f,  1.5f,      0.0f, 0.0f, 1.0f,
            -0.25f, -0.25f,  1.5f,      1.0f, 1.0f, 0.0f
    };

    GLuint VAO, VBO;
    glGenVertexArrays(1, &VAO);
    glBindVertexArray(VAO);
    glGenBuffers(1, &VBO);
    glBindBuffer(GL_ARRAY_BUFFER, VBO);
    glBufferData(GL_ARRAY_BUFFER, sizeof(vertices), vertices, GL_DYNAMIC_DRAW);
    glVertexAttribPointer(0, 3, GL_FLOAT, GL_FALSE, 6 * sizeof(GLfloat), (GLvoid*)0);
    glEnableVertexAttribArray(0);
    glVertexAttribPointer(1, 3, GL_FLOAT, GL_FALSE, 6 * sizeof(GLfloat), (GLvoid*)(3*sizeof(GLfloat)));
    glEnableVertexAttribArray(1);



    //DRAW
    GLfloat movement = 0.0f;
    glfwSetTime(0.0);

    while (!glfwWindowShouldClose(window))
    {
        if(movement >= 2.5f)
            glfwSetTime(0.0);
        glClear(GL_COLOR_BUFFER_BIT);
        movement = glfwGetTime()/4;
        glUniform1f(uniform, movement);
        glDrawArrays(GL_TRIANGLE_STRIP, 0,4);
        glfwSwapBuffers(window);
        glfwPollEvents();
    }



    //END
    glDeleteVertexArrays(1, &VAO);
    glDeleteBuffers(1, &VBO);
    glfwTerminate();

    return 0;
}

EDIT 2 The problem was that I believed the vertex shader runs for every point in the square, but instead it only run for the 4 vertex specified. Now the problem is... How can I fix it?

Answer 1

The vertex shader only runs per vertex, not for every point in the square. So the four vertices are mapped to (±2, ±2), and then the GPU draws a polygon between those vertices, which does cross the viewport.

The points interior to the polygon do not go through the v.y = v.y/v.z transform, so the resulting point set is not $(-\infty,-2] \cup [2,\infty)$ after all.

For this reason, one can't generally do non-linear geometry transformations in the vertex shader alone—except for perspective projection, which has special hardware support. (You run into this problem as well if you try to do any non-linear camera projection, such as a fisheye projection.) There are two ways to solve it:

1. Subdivide the geometry into small pieces so that the vertices are closely spaced. Then, applying the nonlinear transform per vertex may be a good-enough approximation.

   Also, since in your case the transform has a discontinuity at $z = 0$, you would need to split the original polygon into two pieces (each with its own set of vertices) where it crosses the $z = 0$ plane, and push the split vertices away from that plane, so you'd have $z = \pm \epsilon$, where $\epsilon$ is some small number. [It might even work to use IEEE floating point "negative zero" values here; not sure.] Otherwise the pieces that cross $z = 0$ will cause the same problem all over again.

2. Do the transform as an image-space process, i.e. in the pixel shader rather than the vertex shader. This is used for example for lens distortion correction in VR: the image is rendered first using an ordinary perspective projection, then resampled to the distorted projection. This requires being able to invert the transform, i.e. map from the destination pixel position back to the corresponding source position, then sampling the source pixel from a texture.

Answer 2

Going by the code in your vertex shader, the last set of vertices produces the following:

( 0.5, -0.5, -0.25) → (-2,  2)
(-0.5, -0.5, -0.25) → ( 2,  2)
( 0.5, -0.5,  0.25) → ( 2, -2)
(-0.5, -0.5,  0.25) → (-2, -2)

…which is a 4×4 square, centered on (0,0), with the points specified in clockwise order (which won’t draw correctly with a triangle strip, as you’re seeing in the above screenshot).

It looks like you’re expecting y values outside of the [-1, 1] range to not be drawn at all, but all that’s doing is drawing triangles that are larger than the viewport.