How to use Multisampling with OpenGL FBOs

Question

I'm trying to enable mutlisampling and alpha-to-coverage for an FBO. Using the default framebuffer, all I have to do is call glEnable(GL_MULTISAMPLE) and glEnable(GL_SAMPLE_ALPHA_TO_COVERAGE). However, I am unable to achieve the same effect using my own FBO.

My goal: Draw the scene to an FBO the same way it would be drawn to the default framebuffer with the above properties. From there I want to be able to use the image as a texture for future passes through a shader.

This works: Code for making an FBO without multisampling/alpha-to-coverage, 1 color attachment, 1 depth attachment:

// Generate the color attachment
glGenTextures(1,&defaultColorAttachment0);
glActiveTexture(GL_TEXTURE0);
glBindTexture(GL_TEXTURE_2D,defaultColorAttachment0);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
glTexImage2D(GL_TEXTURE_2D,0,GL_RGBA,screenWidth,screenHeight,0,GL_RGBA,GL_UNSIGNED_BYTE,NULL);

// Bind the texture to the FBO
glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, defaultColorAttachment0,0);

// Generate the depth attachment
glGenRenderbuffers(1,&defaultDepthBuffer);
glBindRenderbuffer(GL_RENDERBUFFER, defaultDepthBuffer);
glRenderbufferStorage(GL_RENDERBUFFER, GL_DEPTH_COMPONENT, screenWidth, screenHeight);
glFramebufferRenderbuffer(GL_FRAMEBUFFER, GL_DEPTH_ATTACHMENT, GL_RENDERBUFFER, defaultDepthBuffer);

This doesn't work. Code trying to make a multisampled FBO:

// Generate the color attachment
glGenTextures(1,&defaultColorAttachment0);
glBindTexture(GL_TEXTURE_2D_MULTISAMPLE, defaultColorAttachment0);
glTexImage2DMultisample(GL_TEXTURE_2D_MULTISAMPLE, 4, GL_RGBA, screenWidth, screenHeight, GL_FALSE);
glFramebufferTexture(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, defaultColorAttachment0,0);

// Generate the depth attachment
glGenRenderbuffers(1,&defaultDepthBuffer);
glBindRenderbuffer(GL_RENDERBUFFER, defaultDepthBuffer);
glRenderbufferStorage(GL_RENDERBUFFER, GL_DEPTH_COMPONENT, screenWidth, screenHeight);
glFramebufferRenderbuffer(GL_FRAMEBUFFER, GL_DEPTH_ATTACHMENT, GL_RENDERBUFFER, defaultDepthBuffer);

I have tried looking through the OpenGL wiki on this, although the it's incomplete (various unfinished headings make it look unprofessional). glGetError never complains. I've tried messing around with this, but I either get a black screen or a screen full of garbage pixels.

Main Question: What things do I need to consider/change and where (FBO creation, textures, shaders) in order to get multisampling and alpha-to-coverage to work with an FBO?

Answer 1

You need to allocate a multisampled depth buffer for this to work correctly and give it the same number of samples as your color buffer. In other words, you should be calling glRenderbufferStorageMultisample (...) instead of glRenderbufferStorage (...).

Your FBO should be failing a completeness check the way it is allocated right now. A call to glCheckFramebufferStatus (...) ought to return GL_FRAMEBUFFER_INCOMPLETE_MULTISAMPLE because your depth buffer has exactly 1 sample and your color buffer attachment has 4.

---

Since you are also using a multisampled texture attachment in this FBO, you should be aware of differences between sampling a single-sampled texture vs. multisampled in GLSL shaders.

Multisampled textures have a special sampler uniform type (e.g. sampler2DMS) and you have to explicitly fetch each sample in the texture by its integer (non-normalized) texel coordinate and sample index using texelFetch (...). This also means that they cannot be filtered or mip-mapped.

You probably do not want a multisampled texture in this case, you probably want to use glBlitFramebuffer (...) to do the MSAA resolve into a single-sampled FBO. If you do this instead you can read the anti-aliased results in your shaders rather than having to fetch each sample and implement the anti-aliasing yourself.

Answer 2

Here is a working example to go along with the accepted answer. It is a modified example of the triangle example from the LearnopenGL tutorials to draw a MSAA custom framebuffer to a quad which is then draw to the default framebuffer (the screen):

#include <iostream>
#include <glad/glad.h>
#include <GLFW/glfw3.h>

const char *vertexShaderSource = "#version 330 core\n"
    "layout (location = 0) in vec3 aPos;\n"
    "void main()\n"
    "{\n"
    "   gl_Position = vec4(aPos.x, aPos.y, aPos.z, 1.0);\n"
    "}\0";
const char *fragmentShaderSource = "#version 330 core\n"
    "out vec4 FragColor;\n"
    "void main()\n"
    "{\n"
    "   FragColor = vec4(1.0f, 0.5f, 0.2f, 1.0f);\n"
    "}\n\0";

const char *postProcessvertexShaderSource = "#version 330 core\n"
"layout (location = 0) in vec2 position;\n"             
"layout (location = 1) in vec2 inTexCoord;\n"

"out vec2 texCoord;\n"
"void main(){\n"
"    texCoord = inTexCoord;\n"
"    gl_Position = vec4(position.x, position.y, 0.0f, 1.0f);\n"
"}\n\0";

const char *postProcessFragmentShaderSource = "#version 330 core\n"
"out vec4 fragmentColor;\n"
"in vec2 texCoord;\n"
"//notice the sampler\n"
"uniform sampler2DMS screencapture;\n"
"uniform int viewport_width;\n"
"uniform int viewport_height;\n"

"void main(){\n"
"   //texelFetch requires a vec of ints for indexing (since we're indexing pixel locations)\n"
"   //texture coords is range [0, 1], we need range [0, viewport_dim].\n"
"   //texture coords are essentially a percentage, so we can multiply text coords by total size \n"
"   ivec2 vpCoords = ivec2(viewport_width, viewport_height);\n"
"   vpCoords.x = int(vpCoords.x * texCoord.x); \n"
"   vpCoords.y = int(vpCoords.y * texCoord.y);\n"
"   //do a simple average since this is just a demo\n"
"   vec4 sample1 = texelFetch(screencapture, vpCoords, 0);\n"
"   vec4 sample2 = texelFetch(screencapture, vpCoords, 1);\n"
"   vec4 sample3 = texelFetch(screencapture, vpCoords, 2);\n"
"   vec4 sample4 = texelFetch(screencapture, vpCoords, 3);\n"
"   fragmentColor = vec4(sample1 + sample2 + sample3 + sample4) / 4.0f;\n"
"}\n\0";

int main()
{
    int width = 800;
    int height = 600;
    
    glfwInit();
    glfwWindowHint(GLFW_CONTEXT_VERSION_MINOR, 3);
    glfwWindowHint(GLFW_CONTEXT_VERSION_MAJOR, 3);
    glfwWindowHint(GLFW_OPENGL_PROFILE, GLFW_OPENGL_CORE_PROFILE);
    glfwWindowHint(GLFW_OPENGL_FORWARD_COMPAT, GL_TRUE);

    GLFWwindow* window = glfwCreateWindow(width, height, "OpenglContext", nullptr, nullptr);
    if (!window)
    {
        std::cerr << "failed to create window" << std::endl;
        exit(-1);
    }
    glfwMakeContextCurrent(window);

    if (!gladLoadGLLoader((GLADloadproc)glfwGetProcAddress))
    {
        std::cerr << "failed to initialize glad with processes " << std::endl;
        exit(-1);
    }

    glfwSetInputMode(window, GLFW_CURSOR, GLFW_CURSOR_DISABLED);

    int samples = 4;
    float quadVerts[] = {
        -1.0, -1.0,     0.0, 0.0,
        -1.0, 1.0,      0.0, 1.0,
        1.0, -1.0,      1.0, 0.0,

        1.0, -1.0,      1.0, 0.0,
        -1.0, 1.0,      0.0, 1.0,
        1.0, 1.0,       1.0, 1.0
    };

    GLuint postVAO;
    glGenVertexArrays(1, &postVAO);
    glBindVertexArray(postVAO);

    GLuint postVBO;
    glGenBuffers(1, &postVBO);
    glBindBuffer(GL_ARRAY_BUFFER, postVBO);
    glBufferData(GL_ARRAY_BUFFER, sizeof(quadVerts), quadVerts, GL_STATIC_DRAW);

    glVertexAttribPointer(0, 2, GL_FLOAT, GL_FALSE, 4 * sizeof(float), reinterpret_cast<void*>(0));
    glEnableVertexAttribArray(0);

    glVertexAttribPointer(1, 2, GL_FLOAT, GL_FALSE, 4 * sizeof(float), reinterpret_cast<void*>(2 * sizeof(float)));
    glEnableVertexAttribArray(1);

    glBindVertexArray(0);


    GLuint msaaFB;
    glGenFramebuffers(1, &msaaFB);
    glBindFramebuffer(GL_FRAMEBUFFER, msaaFB); //bind both read/write to the target framebuffer

    GLuint texMutiSampleColor;
    glGenTextures(1, &texMutiSampleColor);
    glBindTexture(GL_TEXTURE_2D_MULTISAMPLE, texMutiSampleColor);
    glTexImage2DMultisample(GL_TEXTURE_2D_MULTISAMPLE, samples, GL_RGB, width, height, GL_TRUE);
    glBindTexture(GL_TEXTURE_2D_MULTISAMPLE, 0);
    glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D_MULTISAMPLE, texMutiSampleColor, 0);

    glBindFramebuffer(GL_FRAMEBUFFER, 0);


    // vertex shader
    unsigned int vertexShader = glCreateShader(GL_VERTEX_SHADER);
    glShaderSource(vertexShader, 1, &vertexShaderSource, NULL);
    glCompileShader(vertexShader);
    // check for shader compile errors

    // fragment shader
    unsigned int fragmentShader = glCreateShader(GL_FRAGMENT_SHADER);
    glShaderSource(fragmentShader, 1, &fragmentShaderSource, NULL);
    glCompileShader(fragmentShader);
    // check for shader compile errors

    // link shaders
    unsigned int shaderProgram = glCreateProgram();
    glAttachShader(shaderProgram, vertexShader);
    glAttachShader(shaderProgram, fragmentShader);
    glLinkProgram(shaderProgram);
    // check for linking errors

    glDeleteShader(vertexShader);
    glDeleteShader(fragmentShader);


    //postprocess vertex shader
    unsigned int postProcessVertexShader = glCreateShader(GL_VERTEX_SHADER);
    glShaderSource(postProcessVertexShader, 1, &postProcessvertexShaderSource, NULL);
    glCompileShader(postProcessVertexShader);

    // postprocess fragment shader
    unsigned int postProcessFragmentShader = glCreateShader(GL_FRAGMENT_SHADER);
    glShaderSource(postProcessFragmentShader, 1, &postProcessFragmentShaderSource, NULL);
    glCompileShader(postProcessFragmentShader);
    // check for shader compile errors

    // link shaders
    unsigned int postProcessShaderProgram = glCreateProgram();
    glAttachShader(postProcessShaderProgram, postProcessVertexShader);
    glAttachShader(postProcessShaderProgram, postProcessFragmentShader);
    glLinkProgram(postProcessShaderProgram);
    // check for linking errors

    glDeleteShader(postProcessVertexShader);
    glDeleteShader(postProcessFragmentShader);

    glUseProgram(postProcessShaderProgram);
    glUniform1i(glGetUniformLocation(postProcessShaderProgram, "screencapture"), 0); 
    glUniform1i(glGetUniformLocation(postProcessShaderProgram, "viewport_width"), width); 
    glUniform1i(glGetUniformLocation(postProcessShaderProgram, "viewport_height"), height); 

    float vertices[] = {
        -0.5f, -0.5f, 0.0f,
         0.5f, -0.5f, 0.0f,
         0.0f,  0.5f, 0.0f 
    }; 

    unsigned int VBO, VAO;
    glGenVertexArrays(1, &VAO);
    glGenBuffers(1, &VBO);
    glBindVertexArray(VAO);
    glBindBuffer(GL_ARRAY_BUFFER, VBO);
    glBufferData(GL_ARRAY_BUFFER, sizeof(vertices), vertices, GL_STATIC_DRAW);
    glVertexAttribPointer(0, 3, GL_FLOAT, GL_FALSE, 3 * sizeof(float), (void*)0);
    glEnableVertexAttribArray(0);
    glBindBuffer(GL_ARRAY_BUFFER, 0); 
    glBindVertexArray(0); 

    bool use_msaa = true;

    while (!glfwWindowShouldClose(window))
    {

        if (glfwGetKey(window, GLFW_KEY_ESCAPE) == GLFW_PRESS)
        {
            glfwSetWindowShouldClose(window, true);
        }

        if (glfwGetKey(window, GLFW_KEY_R) == GLFW_PRESS)
            use_msaa = true;
        if (glfwGetKey(window, GLFW_KEY_T) == GLFW_PRESS)
            use_msaa = false;     

        glClearColor(0.0f, 0.0f, 0.0f, 1.0f);
        glClear(GL_COLOR_BUFFER_BIT);

        if (use_msaa) {
            glBindFramebuffer(GL_FRAMEBUFFER, msaaFB);
        }

        glClearColor(0.0f, 0.0f, 0.0f, 1.0f);
        glClear(GL_COLOR_BUFFER_BIT);

        // draw our first triangle
        glUseProgram(shaderProgram);
        glBindVertexArray(VAO);
        glDrawArrays(GL_TRIANGLES, 0, 3);

        if (use_msaa) {
            glBindFramebuffer(GL_FRAMEBUFFER, 0);
            glUseProgram(postProcessShaderProgram);
            glActiveTexture(GL_TEXTURE0);
            glBindTexture(GL_TEXTURE_2D_MULTISAMPLE, texMutiSampleColor);
            glBindVertexArray(postVAO);
            glDrawArrays(GL_TRIANGLES, 0, 6);
        }

        glfwSwapBuffers(window);
        glfwPollEvents();

    }
    glfwTerminate();
    // cleanup
}

Thankyou Matt Stone from the comment section of LearnOpenGL for the working code.