Silhouette-Outlined shader

Question

I'm trying to implement GLSL shader which would highlight the outer edges of rendered 3D mesh. The problem is that I do not have access to the OpenGL client side code so this must be done only in GLSL shaders.

My first attempt was to use/adopt this shader from Unity and do it in OpenGL GLSL. Here how it should look:

And here is what I got:

I'm not sure If I compute the stuff correctly but as you can see the output is nowhere near my expectations.

Here is the ogre material

material Chassis 
    {
    technique
        {    
        pass standard
            {
            cull_software back         
            scene_blend zero one
            }
        pass psssm
            {         
            cull_software front 
            scene_blend src_alpha one_minus_src_alpha         
            vertex_program_ref reflection_cube_specularmap_normalmap_vs100 
                {
                param_named_auto modelViewProjectionMatrix worldviewproj_matrix
                param_named_auto normalMatrix inverse_transpose_world_matrix
                param_named_auto modelView worldview_matrix
                param_named_auto camera_world_position camera_position
                param_named_auto inverse_projection_matrix inverse_projection_matrix
                param_named_auto  projection_matrix projection_matrix
                param_named_auto  p_InverseModelView inverse_worldview_matrix
                }
            fragment_program_ref reflection_cube_specularmap_normalmap_fs100
                {                
                }    
            }
        }
    }

Here is the vertex shader

#version 140
#define lowp
#define mediump
#define highp

in vec4 vertex;
in vec3 normal;   

uniform mat4 normalMatrix;
uniform mat4 modelViewProjectionMatrix;
uniform mat4 modelView;
uniform vec3 camera_world_position;
uniform mat4 projection_matrix;
uniform mat4 inverse_projection_matrix;
void main()
   {        
   vec4 pos = modelViewProjectionMatrix * vertex;
   mat4 modelView = inverse_projection_matrix * modelViewProjectionMatrix;

   vec4 norm   =   inverse(transpose(modelView)) * vec4(normal, 0.0);
   vec2 offset =   vec2( norm.x * projection_matrix[0][0], norm.y * projection_matrix[1][1] );

   pos.xy += offset * pos.z * 0.18;
   gl_Position = pos;
   } 

EDIT: I have added the material script which ogre uses and I have added the vertex shader code.

Answer 1

I assume single complex 3D mesh. I would do this with 2 pass rendering:

1. clear screen

   let use (0,0,0) as clear color.

2. render mesh

   disable depth output,test (or clear it afterwards). Do not use shading fill just with some predefined color for example (1,1,1) Lets do this for simple cube:

   

3. read the frame buffer and use it as a texture

   So either use FBO and render to texture for #1,#2 or use glReadPixels instead and load it as some texture back to GPU (I know it slower but works also on Intel). For more info see both answers in here:

   • OpenGL Scale Single Pixel Line

4. clear screen with background color

5. render

   so either render GL_QUAD covering whole screen or render your mesh with shading and what ever you want. You need to pass also the texture from previous step into GLSL.

   In fragment render as usual ... but at the end also add this:

   Scan all texels around current fragment screen position up to distance equal to outline thickness in the texture from previous step. If any black pixel found in it override outputted color with your outline color. You can even modulate it with the smallest distance to black color.

   This is very similar to this:

   • How to implement 2D raycasting light effect in GLSL

   but much simpler. Here result:

   

I took this example Analysis of a shader in VR of mine and converted it to this:

Fragment:

// Fragment
#version 400 core
#extension GL_ARB_explicit_uniform_location : enable
layout(location =64) uniform vec3 lt_pnt_pos;// point light source position [GCS]
layout(location =67) uniform vec3 lt_pnt_col;// point light source color&strength
layout(location =70) uniform vec3 lt_amb_col;// ambient light source color&strength
in vec3 LCS_pos;        // fragment position [LCS]
in vec3 pixel_pos;      // fragment position [GCS]
in vec3 pixel_col;      // fragment surface color
in vec3 pixel_nor;      // fragment surface normal [GCS]
out vec4 col;

// outline
uniform sampler2D txr;  // texture from previous pass
uniform int thickness;  // [pixels] outline thickness
uniform float xs,ys;    // [pixels] texture/screen resolution
void main()
    {
    // standard rendering
    float li;
    vec3 c,lt_dir;
    lt_dir=normalize(lt_pnt_pos-pixel_pos); // vector from fragment to point light source in [GCS]
    li=dot(pixel_nor,lt_dir);
    if (li<0.0) li=0.0;
    c=pixel_col*(lt_amb_col+(lt_pnt_col*li));

    // outline effect
    if (thickness>0)            // thickness effect in second pass
        {
        int i,j,r=thickness;
        float xx,yy,rr,x,y,dx,dy;
        dx=1.0/xs;              // texel size
        dy=1.0/ys;
        x=gl_FragCoord.x*dx;
        y=gl_FragCoord.y*dy;
        rr=thickness*thickness;
        for (yy=y-(float(thickness)*dy),i=-r;i<=r;i++,yy+=dy)
         for (xx=x-(float(thickness)*dx),j=-r;j<=r;j++,xx+=dx)
          if ((i*i)+(j*j)<=rr)
           if ((texture(txr,vec2(xx,yy)).r)<0.01)
            {
            c=vec3(1.0,0.0,0.0);    // outline color
            i=r+r+1;
            j=r+r+1;
            break;
            }
        }
    else c=vec3(1.0,1.0,1.0);   // render with white in first pass

    // output color
    col=vec4(c,1.0);
    }

The Vertex shader is without change:

// Vertex
#version 400 core
#extension GL_ARB_explicit_uniform_location : enable
layout(location = 0) in vec3 pos;
layout(location = 2) in vec3 nor;
layout(location = 3) in vec3 col;
layout(location = 0) uniform mat4 m_model;  // model matrix
layout(location =16) uniform mat4 m_normal; // model matrix with origin=(0,0,0)
layout(location =32) uniform mat4 m_view;   // inverse of camera matrix
layout(location =48) uniform mat4 m_proj;   // projection matrix
out vec3 LCS_pos;       // fragment position [LCS]
out vec3 pixel_pos;     // fragment position [GCS]
out vec3 pixel_col;     // fragment surface color
out vec3 pixel_nor;     // fragment surface normal [GCS]

void main()
    {
    LCS_pos=pos;
    pixel_col=col;
    pixel_pos=(m_model*vec4(pos,1)).xyz;
    pixel_nor=(m_normal*vec4(nor,1)).xyz;
    gl_Position=m_proj*m_view*m_model*vec4(pos,1);
    }

And CPU side code looks like this:

//---------------------------------------------------------------------------
#include <vcl.h>
#pragma hdrstop
#include "Unit1.h"
#include "gl_simple.h"
//---------------------------------------------------------------------------
#pragma package(smart_init)
#pragma resource "*.dfm"
TForm1 *Form1;
//---------------------------------------------------------------------------
GLfloat lt_pnt_pos[3]={+2.5,+2.5,+2.5};
GLfloat lt_pnt_col[3]={0.8,0.8,0.8};
GLfloat lt_amb_col[3]={0.2,0.2,0.2};
GLuint txrid=0;
GLfloat animt=0.0;
//---------------------------------------------------------------------------
// https://stackoverflow.com/q/46603878/2521214
//---------------------------------------------------------------------------
void gl_draw()
    {
    // load values into shader
    GLint i,id;
    GLfloat m[16];
    glUseProgram(prog_id);

    GLfloat x,y,z,d=0.25;

    id=glGetUniformLocation(prog_id,"txr"); glUniform1i(id,0);
    id=glGetUniformLocation(prog_id,"xs"); glUniform1f(id,xs);
    id=glGetUniformLocation(prog_id,"ys"); glUniform1f(id,ys);

    id=64; glUniform3fv(id,1,lt_pnt_pos);
    id=67; glUniform3fv(id,1,lt_pnt_col);
    id=70; glUniform3fv(id,1,lt_amb_col);
    glGetFloatv(GL_MODELVIEW_MATRIX,m);
    id=0; glUniformMatrix4fv(id,1,GL_FALSE,m);
    m[12]=0.0; m[13]=0.0; m[14]=0.0;
    id=16; glUniformMatrix4fv(id,1,GL_FALSE,m);
    for (i=0;i<16;i++) m[i]=0.0; m[0]=1.0; m[5]=1.0; m[10]=1.0; m[15]=1.0;
    id=32; glUniformMatrix4fv(id,1,GL_FALSE,m);
    glGetFloatv(GL_PROJECTION_MATRIX,m);
    id=48; glUniformMatrix4fv(id,1,GL_FALSE,m);


    // draw VAO cube (no outline)
    id=glGetUniformLocation(prog_id,"thickness"); glUniform1i(id,0);
    glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
    vao_draw(); // render cube

    // copy frame buffer to CPU memory and than back to GPU as Texture
    BYTE *map=new BYTE[xs*ys*4];
    glReadPixels(0,0,xs,ys,GL_RGB,GL_UNSIGNED_BYTE,map);    // framebuffer -> map[]
    glEnable(GL_TEXTURE_2D);
    glBindTexture(GL_TEXTURE_2D,txrid);
    glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, xs, ys, 0, GL_RGB, GL_UNSIGNED_BYTE, map); // map[] -> texture txrid
    delete[] map;

    // draw VAO cube (outline)
    id=glGetUniformLocation(prog_id,"thickness"); glUniform1i(id,5);
    glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
    vao_draw(); // render cube
    glDisable(GL_TEXTURE_2D);

    // turn of shader
    glUseProgram(0);

    // rotate the cube to see animation
    glMatrixMode(GL_MODELVIEW);
//  glRotatef(1.0,0.0,1.0,0.0);
//  glRotatef(1.0,1.0,0.0,0.0);

    glFlush();
    SwapBuffers(hdc);
    }
//---------------------------------------------------------------------------
__fastcall TForm1::TForm1(TComponent* Owner):TForm(Owner)
    {
    gl_init(Handle);

    glGenTextures(1,&txrid);
    glEnable(GL_TEXTURE_2D);
    glBindTexture(GL_TEXTURE_2D,txrid);
    glPixelStorei(GL_UNPACK_ALIGNMENT, 4);
    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S,GL_CLAMP_TO_EDGE);
    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T,GL_CLAMP_TO_EDGE);
    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER,GL_NEAREST);
    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER,GL_NEAREST);
    glTexEnvf(GL_TEXTURE_ENV, GL_TEXTURE_ENV_MODE,GL_COPY);
    glDisable(GL_TEXTURE_2D);


    int hnd,siz; char vertex[4096],fragment[4096];
    hnd=FileOpen("normal_shading.glsl_vert",fmOpenRead); siz=FileSeek(hnd,0,2); FileSeek(hnd,0,0); FileRead(hnd,vertex  ,siz); vertex  [siz]=0; FileClose(hnd);
    hnd=FileOpen("normal_shading.glsl_frag",fmOpenRead); siz=FileSeek(hnd,0,2); FileSeek(hnd,0,0); FileRead(hnd,fragment,siz); fragment[siz]=0; FileClose(hnd);
    glsl_init(vertex,fragment);
//  hnd=FileCreate("GLSL.txt"); FileWrite(hnd,glsl_log,glsl_logs); FileClose(hnd);

    int i0,i;
    mm_log->Lines->Clear();
    for (i=i0=0;i<glsl_logs;i++)
     if ((glsl_log[i]==13)||(glsl_log[i]==10))
        {
        glsl_log[i]=0;
        mm_log->Lines->Add(glsl_log+i0);
        glsl_log[i]=13;
        for (;((glsl_log[i]==13)||(glsl_log[i]==10))&&(i<glsl_logs);i++);
        i0=i;
        }
    if (i0<glsl_logs) mm_log->Lines->Add(glsl_log+i0);

    vao_init();
    }
//---------------------------------------------------------------------------
void __fastcall TForm1::FormDestroy(TObject *Sender)
    {
    glDeleteTextures(1,&txrid);
    gl_exit();
    glsl_exit();
    vao_exit();
    }
//---------------------------------------------------------------------------
void __fastcall TForm1::FormResize(TObject *Sender)
    {
    gl_resize(ClientWidth,ClientHeight-mm_log->Height);
    glMatrixMode(GL_PROJECTION);
    glTranslatef(0,0,-15.0);

    glMatrixMode(GL_MODELVIEW);
    glRotatef(-15.0,0.0,1.0,0.0);
    glRotatef(-125.0,1.0,0.0,0.0);
    }
//---------------------------------------------------------------------------
void __fastcall TForm1::FormPaint(TObject *Sender)
    {
    gl_draw();
    }
//---------------------------------------------------------------------------
void __fastcall TForm1::Timer1Timer(TObject *Sender)
    {
    gl_draw();
    animt+=0.02; if (animt>1.5) animt=-0.5;
    Caption=animt;
    }
//---------------------------------------------------------------------------
void __fastcall TForm1::FormMouseWheel(TObject *Sender, TShiftState Shift, int WheelDelta, TPoint &MousePos, bool &Handled)
    {
    GLfloat dz=2.0;
    if (WheelDelta<0) dz=-dz;
    glMatrixMode(GL_PROJECTION);
    glTranslatef(0,0,dz);
    gl_draw();
    }
//---------------------------------------------------------------------------

As usual the code is using/based on this:

• complete GL+GLSL+VAO/VBO C++ example

[Notes]

In case you got multiple objects then use for each object different color in #2. Then in #5 scan for any different color then the one that is in the texel at current position instead of scanning for black.

Also this can be done on 2D image instead of using mesh. You just need to know the background color. So you can use pre-renderd/grabed/screenshoted images for this too.

You can add discard and or change the final if logic to change behaviour (like you want just outline and no mesh inside etc ...). Or you can add the outline color to render color instead of assigning it directly to get the impression of highlight ... instead of coloring

see a),b),c) options in modified fragment:

// Fragment
#version 400 core
#extension GL_ARB_explicit_uniform_location : enable
layout(location =64) uniform vec3 lt_pnt_pos;// point light source position [GCS]
layout(location =67) uniform vec3 lt_pnt_col;// point light source color&strength
layout(location =70) uniform vec3 lt_amb_col;// ambient light source color&strength
in vec3 LCS_pos;        // fragment position [LCS]
in vec3 pixel_pos;      // fragment position [GCS]
in vec3 pixel_col;      // fragment surface color
in vec3 pixel_nor;      // fragment surface normal [GCS]
out vec4 col;

// outline
uniform sampler2D txr;  // texture from previous pass
uniform int thickness;  // [pixels] outline thickness
uniform float xs,ys;    // [pixels] texture/screen resolution
void main()
    {
    // standard rendering
    float li;
    vec3 c,lt_dir;
    lt_dir=normalize(lt_pnt_pos-pixel_pos); // vector from fragment to point light source in [GCS]
    li=dot(pixel_nor,lt_dir);
    if (li<0.0) li=0.0;
    c=pixel_col*(lt_amb_col+(lt_pnt_col*li));

    // outline effect
    if (thickness>0)            // thickness effect in second pass
        {
        int i,j,r=thickness;
        float xx,yy,rr,x,y,dx,dy;
        dx=1.0/xs;              // texel size
        dy=1.0/ys;
        x=gl_FragCoord.x*dx;
        y=gl_FragCoord.y*dy;
        rr=thickness*thickness;
        for (yy=y-(float(thickness)*dy),i=-r;i<=r;i++,yy+=dy)
         for (xx=x-(float(thickness)*dx),j=-r;j<=r;j++,xx+=dx)
          if ((i*i)+(j*j)<=rr)
           if ((texture(txr,vec2(xx,yy)).r)<0.01)
            {
            c =vec3(1.0,0.0,0.0);   // a) assign outline color
//          c+=vec3(1.0,0.0,0.0);   // b) add outline color
            i=r+r+1;
            j=r+r+1;
            r=0;
            break;
            }
//      if (r!=0) discard; // c) do not render inside
        }
    else c=vec3(1.0,1.0,1.0);   // render with white in first pass

    // output color
    col=vec4(c,1.0);
    }

[Edit1] single pass approach for smooth edges

As you can not access client side code this approach will work in shader only. For smooth (curved) edged shapes the surface normal is near perpendicular to camera view axis (z). So dot between them is near zero. This can be exploited directly ... Here update of the shaders:

Vertex

// Vertex
#version 400 core
#extension GL_ARB_explicit_uniform_location : enable
layout(location = 0) in vec3 pos;
layout(location = 2) in vec3 nor;
layout(location = 3) in vec3 col;
layout(location = 0) uniform mat4 m_model;  // model matrix
layout(location =16) uniform mat4 m_normal; // model matrix with origin=(0,0,0)
layout(location =32) uniform mat4 m_view;   // inverse of camera matrix
layout(location =48) uniform mat4 m_proj;   // projection matrix
out vec3 pixel_pos;     // fragment position [GCS]
out vec3 pixel_col;     // fragment surface color
out vec3 pixel_nor;     // fragment surface normal [GCS]
out vec3 view_nor;     // surface normal in camera [LCS]

void main()
    {
    pixel_col=col;
    pixel_pos=(m_model*vec4(pos,1)).xyz;
    pixel_nor=(m_normal*vec4(nor,1)).xyz;

    mat4 m;
    m=m_model*m_view;                   // model view matrix
    m[3].xyz=vec3(0.0,0.0,0.0);         // with origin set to (0,0,0)
    view_nor=(m*vec4(nor,1.0)).xyz;     // object local normal to camera local normal

    gl_Position=m_proj*m_view*m_model*vec4(pos,1);
    }

Fragment

// Fragment
#version 400 core
#extension GL_ARB_explicit_uniform_location : enable
layout(location =64) uniform vec3 lt_pnt_pos;// point light source position [GCS]
layout(location =67) uniform vec3 lt_pnt_col;// point light source color&strength
layout(location =70) uniform vec3 lt_amb_col;// ambient light source color&strength
in vec3 pixel_pos;      // fragment position [GCS]
in vec3 pixel_col;      // fragment surface color
in vec3 pixel_nor;      // fragment surface normal [GCS]
out vec4 col;

// outline
in vec3 view_nor;     // surface normal in camera [LCS]

void main()
    {
    // standard rendering
    float li;
    vec3 c,lt_dir;
    lt_dir=normalize(lt_pnt_pos-pixel_pos); // vector from fragment to point light source in [GCS]
    li=dot(pixel_nor,lt_dir);
    if (li<0.0) li=0.0;
    c=pixel_col*(lt_amb_col+(lt_pnt_col*li));

    // outline effect
    if (abs(dot(view_nor,vec3(0.0,0.0,1.0)))<=0.5) c=vec3(1.0,0.0,0.0);

    // output color
    col=vec4(c,1.0);
    }

Here preview:

As you can see it works properly for smooth objects but for sharp edges like on cube is this not working at all... You can use the same combinations (a,b,c) as in previous approach.

The m holds modelview matrix with origin set to (0,0,0). That enables it for vector conversion (no translation). For more info see Understanding 4x4 homogenous transform matrices.

The 0.5 in the dot product result if is the thickness of outline. 0.0 means no outline and 1.0 means whole object is outline.