How to make an OpenGL rendering context with transparent background?

Question

Rendering contexts usually have a solid color on the background (black or whatever, see the image below):

I'm wondering if it's possible to setup a window, with no decorations AND with the transparent background, while allowing me to render OpenGL stuff on it.

This would give the illusion that the triangle is floating on the screen. The transparent background should allow you to see the desktop or other applications that might be behind it.

Could you please exemplify with source code?

Platform: Windows (win32 only)

Answer 1

After spending some reputation on a unsuccessful bounty to get some help on this issue, I finally realized how complex was the problem I was interested in.

The few individuals that have accomplished this task don't share much. During my research I found different ways to achieve what I was looking for. One of the most interesting ones is AeroGL, and it shows snippets of code using a technique that was not mentioned so far, which is rendering the graphics to a device-independent bitmap (DIB).

To close this thread permanently, the source code below implements that technique. The code itself is a slight modification of an application presented here (big thanks to Andrei Sapronov Y.).

The end result can be seen in the image below:

The code has been tested on Windows XP (32-bits) and Windows 8.1 (32-bits). Enjoy!

#define _WIN32_WINNT 0x0500

#include <windows.h>
#include <windowsx.h>
#include <GL/gl.h>
#include <GL/glu.h>

#pragma comment (lib, "opengl32.lib")
#pragma comment (lib, "glu32.lib")

#include <assert.h>
#include <tchar.h>

#ifdef  assert
#define verify(expr) if(!expr) assert(0)
#else verify(expr) expr
#endif

const TCHAR szAppName[]=_T("TransparentGL");
const TCHAR wcWndName[]=_T("WS_EX_LAYERED OpenGL");

HDC hDC;            
HGLRC m_hrc;        
int w(240);
int h(240); 

HDC pdcDIB;                 
HBITMAP hbmpDIB;            
void *bmp_cnt(NULL);        
int cxDIB(0); 
int cyDIB(0);   
BITMAPINFOHEADER BIH;       


BOOL initSC()
{
    glEnable(GL_ALPHA_TEST);        
    glEnable(GL_DEPTH_TEST);        
    glEnable(GL_COLOR_MATERIAL);

    glEnable(GL_LIGHTING);          
    glEnable(GL_LIGHT0);            

    glEnable(GL_BLEND);             
    glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA);
    glClearColor(0, 0, 0, 0);

    return 0;
}

void resizeSC(int width,int height)
{
    glViewport(0,0,width,height);
    glMatrixMode(GL_PROJECTION);
    glLoadIdentity();

    glMatrixMode(GL_MODELVIEW );
    glLoadIdentity();
}

BOOL renderSC()
{   
    glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT );

    glPushMatrix();

    glColor3f(0, 1, 1);
    glBegin(GL_TRIANGLES);                              // Drawing Using Triangles
        glColor3f(1.0f,0.0f,0.0f);                      // Set The Color To Red
        glVertex3f( 0.0f, 1.0f, 0.0f);                  // Top
        glColor3f(0.0f,1.0f,0.0f);                      // Set The Color To Green
        glVertex3f(-1.0f,-1.0f, 0.0f);                  // Bottom Left
        glColor3f(0.0f,0.0f,1.0f);                      // Set The Color To Blue
        glVertex3f( 1.0f,-1.0f, 0.0f);                  // Bottom Right
    glEnd();

    glPopMatrix();
    glFlush();

    return 0;
}

// DIB -> hDC
void draw(HDC pdcDest)
{
    assert(pdcDIB);

    verify(BitBlt(pdcDest, 0, 0, w, h, pdcDIB, 0, 0, SRCCOPY));
}

void CreateDIB(int cx, int cy)
{
    assert(cx > 0); 
    assert(cy > 0);

    cxDIB = cx ;
    cyDIB = cy ;

    int iSize = sizeof(BITMAPINFOHEADER);   
    memset(&BIH, 0, iSize);

    BIH.biSize = iSize;
    BIH.biWidth = cx;   
    BIH.biHeight = cy;  
    BIH.biPlanes = 1;   
    BIH.biBitCount = 24;    
    BIH.biCompression = BI_RGB;

    if(pdcDIB) 
        verify(DeleteDC(pdcDIB));

    pdcDIB = CreateCompatibleDC(NULL);
    assert(pdcDIB);

    if(hbmpDIB) 
        verify(DeleteObject(hbmpDIB));

    hbmpDIB = CreateDIBSection(
        pdcDIB,         
        (BITMAPINFO*)&BIH,  
        DIB_RGB_COLORS,     
        &bmp_cnt,       
        NULL,
        0);

    assert(hbmpDIB);
    assert(bmp_cnt);

    if(hbmpDIB)
        SelectObject(pdcDIB, hbmpDIB);
}

BOOL CreateHGLRC()
{
    DWORD dwFlags = PFD_SUPPORT_OPENGL | PFD_DRAW_TO_BITMAP;

    PIXELFORMATDESCRIPTOR pfd ;
    memset(&pfd,0, sizeof(PIXELFORMATDESCRIPTOR)) ;
    pfd.nSize = sizeof(PIXELFORMATDESCRIPTOR); 
    pfd.nVersion = 1;                       
    pfd.dwFlags =  dwFlags ;                
    pfd.iPixelType = PFD_TYPE_RGBA ;        
    pfd.cColorBits = 24 ;                   
    pfd.cDepthBits = 32 ;                   
    pfd.iLayerType = PFD_MAIN_PLANE ;       

   int PixelFormat = ChoosePixelFormat(pdcDIB, &pfd);
   if (PixelFormat == 0){
      assert(0);
      return FALSE ;
   }

   BOOL bResult = SetPixelFormat(pdcDIB, PixelFormat, &pfd);
   if (bResult==FALSE){
      assert(0);
      return FALSE ;
   }

   m_hrc = wglCreateContext(pdcDIB);
   if (!m_hrc){
      assert(0);
      return FALSE;
   }

   return TRUE;
}

LRESULT CALLBACK WindowFunc(HWND hWnd,UINT msg, WPARAM wParam, LPARAM lParam)
{
    PAINTSTRUCT ps;

    switch(msg) 
    {
        case WM_ERASEBKGND:
            return 0;
        break;

        case WM_CREATE:
        break;

        case WM_DESTROY:
            if(m_hrc)
            {
                wglMakeCurrent(NULL, NULL);
                wglDeleteContext(m_hrc) ;
            }
            PostQuitMessage(0) ;
        break;

        case WM_PAINT:
            hDC = BeginPaint(hWnd, &ps);
            renderSC(); // OpenGL -> DIB
            draw(hDC);  // DIB -> hDC
            EndPaint(hWnd, &ps);
        break;

        case WM_SIZE:
            w = LOWORD(lParam); h = HIWORD(lParam);         
            wglMakeCurrent(NULL, NULL);
            wglDeleteContext(m_hrc);

            CreateDIB(w, h);
            CreateHGLRC();
            verify(wglMakeCurrent(pdcDIB, m_hrc));

            initSC();
            resizeSC(w, h);
            renderSC();
        break;

        default: 
            return DefWindowProc(hWnd,msg,wParam,lParam);
    }

    return 0;
}

int WINAPI _tWinMain(HINSTANCE hThisInst, HINSTANCE hPrevInst, LPSTR str,int nWinMode)
{   
    WNDCLASSEX wc;
    memset(&wc, 0, sizeof(wc));
    wc.cbSize = sizeof(WNDCLASSEX);
    wc.hIconSm = LoadIcon(NULL, IDI_APPLICATION);
    wc.style = CS_HREDRAW | CS_VREDRAW;
    wc.lpfnWndProc = (WNDPROC)WindowFunc;
    wc.cbClsExtra  = 0;
    wc.cbWndExtra  = 0;
    wc.hInstance = hThisInst;
    wc.hIcon = LoadIcon(NULL, IDI_APPLICATION);
    wc.hCursor = LoadCursor(NULL, IDC_ARROW);
    wc.hbrBackground = (HBRUSH) (COLOR_WINDOW);
    wc.lpszClassName = szAppName;

    if(!RegisterClassEx(&wc))
    {
        MessageBox(NULL, _T("RegisterClassEx - failed"), _T("Error"), MB_OK | MB_ICONERROR);
        return FALSE;
    }

    HWND hWnd = CreateWindowEx(WS_EX_LAYERED, szAppName, wcWndName,
                    WS_VISIBLE | WS_POPUP, 200, 150, w, h,
                    NULL, NULL, hThisInst, NULL);
    if(!hWnd){
        MessageBox(NULL, _T("CreateWindowEx - failed"), _T("Error"), MB_OK | MB_ICONERROR);
        return FALSE;
    }

    verify(SetLayeredWindowAttributes(hWnd, 0x0, 0, LWA_COLORKEY));

    MSG msg;
    while(1) 
    {
        while (PeekMessage(&msg,NULL,0,0,PM_NOREMOVE)){
            if (GetMessage(&msg, NULL, 0, 0))
            {
                TranslateMessage(&msg);
                DispatchMessage(&msg);
            }
            else return 0;
        }
    } 

    return (FALSE); 
}

Answer 2

Since all the answers given so far target Windows only, but there's surely also a demand doing this on X11 with a composited window manager, for reference I post my example code here (also to be found at https://github.com/datenwolf/codesamples/blob/master/samples/OpenGL/x11argb_opengl/x11argb_opengl.c

/*------------------------------------------------------------------------
 * A demonstration of OpenGL in a  ARGB window 
 *    => support for composited window transparency
 *
 * (c) 2011 by Wolfgang 'datenwolf' Draxinger
 *     See me at comp.graphics.api.opengl and StackOverflow.com
  
 * License agreement: This source code is provided "as is". You
 * can use this source code however you want for your own personal
 * use. If you give this source code to anybody else then you must
 * leave this message in it.
 *
 * This program is based on the simplest possible 
 * Linux OpenGL program by FTB (see info below)
 
  The simplest possible Linux OpenGL program? Maybe...

  (c) 2002 by FTB. See me in comp.graphics.api.opengl

  --
  <\___/>
  / O O \
  \_____/  FTB.

------------------------------------------------------------------------*/

#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <math.h>

#include <GL/gl.h>
#include <GL/glx.h>
#include <GL/glxext.h>
#include <X11/Xatom.h>
#include <X11/extensions/Xrender.h>
#include <X11/Xutil.h>

#define USE_CHOOSE_FBCONFIG

static void fatalError(const char *why)
{
    fprintf(stderr, "%s", why);
    exit(0x666);
}

static int Xscreen;
static Atom del_atom;
static Colormap cmap;
static Display *Xdisplay;
static XVisualInfo *visual;
static XRenderPictFormat *pict_format;
static GLXFBConfig *fbconfigs, fbconfig;
static int numfbconfigs;
static GLXContext render_context;
static Window Xroot, window_handle;
static GLXWindow glX_window_handle;
static int width, height;

static int VisData[] = {
GLX_RENDER_TYPE, GLX_RGBA_BIT,
GLX_DRAWABLE_TYPE, GLX_WINDOW_BIT,
GLX_DOUBLEBUFFER, True,
GLX_RED_SIZE, 8,
GLX_GREEN_SIZE, 8,
GLX_BLUE_SIZE, 8,
GLX_ALPHA_SIZE, 8,
GLX_DEPTH_SIZE, 16,
None
};

static int isExtensionSupported(const char *extList, const char *extension)
{
 
  const char *start;
  const char *where, *terminator;
 
  /* Extension names should not have spaces. */
  where = strchr(extension, ' ');
  if ( where || *extension == '\0' )
    return 0;
 
  /* It takes a bit of care to be fool-proof about parsing the
     OpenGL extensions string. Don't be fooled by sub-strings,
     etc. */
  for ( start = extList; ; ) {
    where = strstr( start, extension );
 
    if ( !where )
      break;
 
    terminator = where + strlen( extension );
 
    if ( where == start || *(where - 1) == ' ' )
      if ( *terminator == ' ' || *terminator == '\0' )
        return 1;
 
    start = terminator;
  }
  return 0;
}

static Bool WaitForMapNotify(Display *d, XEvent *e, char *arg)
{    
    return d && e && arg && (e->type == MapNotify) && (e->xmap.window == *(Window*)arg);
}

static void describe_fbconfig(GLXFBConfig fbconfig)
{
    int doublebuffer;
    int red_bits, green_bits, blue_bits, alpha_bits, depth_bits;

    glXGetFBConfigAttrib(Xdisplay, fbconfig, GLX_DOUBLEBUFFER, &doublebuffer);
    glXGetFBConfigAttrib(Xdisplay, fbconfig, GLX_RED_SIZE, &red_bits);
    glXGetFBConfigAttrib(Xdisplay, fbconfig, GLX_GREEN_SIZE, &green_bits);
    glXGetFBConfigAttrib(Xdisplay, fbconfig, GLX_BLUE_SIZE, &blue_bits);
    glXGetFBConfigAttrib(Xdisplay, fbconfig, GLX_ALPHA_SIZE, &alpha_bits);
    glXGetFBConfigAttrib(Xdisplay, fbconfig, GLX_DEPTH_SIZE, &depth_bits);

    fprintf(stderr, "FBConfig selected:\n"
        "Doublebuffer: %s\n"
        "Red Bits: %d, Green Bits: %d, Blue Bits: %d, Alpha Bits: %d, Depth Bits: %d\n",
        doublebuffer == True ? "Yes" : "No", 
        red_bits, green_bits, blue_bits, alpha_bits, depth_bits);
}

static void createTheWindow()
{
    XEvent event;
    int x,y, attr_mask;
    XSizeHints hints;
    XWMHints *startup_state;
    XTextProperty textprop;
    XSetWindowAttributes attr = {0,};
    static char *title = "FTB's little OpenGL example - ARGB extension by WXD";

    Xdisplay = XOpenDisplay(NULL);
    if (!Xdisplay) {
        fatalError("Couldn't connect to X server\n");
    }
    Xscreen = DefaultScreen(Xdisplay);
    Xroot = RootWindow(Xdisplay, Xscreen);

    fbconfigs = glXChooseFBConfig(Xdisplay, Xscreen, VisData, &numfbconfigs);
    fbconfig = 0;
    for(int i = 0; i<numfbconfigs; i++) {
        visual = (XVisualInfo*) glXGetVisualFromFBConfig(Xdisplay, fbconfigs[i]);
        if(!visual)
            continue;

        pict_format = XRenderFindVisualFormat(Xdisplay, visual->visual);
        XFree(visual);
        if(!pict_format)
            continue;

        fbconfig = fbconfigs[i];
        if(pict_format->direct.alphaMask > 0) {
            break;
        }
    }

    if(!fbconfig) {
        fatalError("No matching FB config found");
    }

    describe_fbconfig(fbconfig);

    /* Create a colormap - only needed on some X clients, eg. IRIX */
    cmap = XCreateColormap(Xdisplay, Xroot, visual->visual, AllocNone);

    attr.colormap = cmap;
    attr.background_pixmap = None;
    attr.border_pixmap = None;
    attr.border_pixel = 0;
    attr.event_mask =
        StructureNotifyMask |
        EnterWindowMask |
        LeaveWindowMask |
        ExposureMask |
        ButtonPressMask |
        ButtonReleaseMask |
        OwnerGrabButtonMask |
        KeyPressMask |
        KeyReleaseMask;

    attr_mask = 
        CWBackPixmap|
        CWColormap|
        CWBorderPixel|
        CWEventMask;

    width = DisplayWidth(Xdisplay, DefaultScreen(Xdisplay))/2;
    height = DisplayHeight(Xdisplay, DefaultScreen(Xdisplay))/2;
    x=width/2, y=height/2;

    window_handle = XCreateWindow(  Xdisplay,
                    Xroot,
                    x, y, width, height,
                    0,
                    visual->depth,
                    InputOutput,
                    visual->visual,
                    attr_mask, &attr);

    if( !window_handle ) {
        fatalError("Couldn't create the window\n");
    }

#if USE_GLX_CREATE_WINDOW
    int glXattr[] = { None };
    glX_window_handle = glXCreateWindow(Xdisplay, fbconfig, window_handle, glXattr);
    if( !glX_window_handle ) {
        fatalError("Couldn't create the GLX window\n");
    }
#else
    glX_window_handle = window_handle;
#endif

    textprop.value = (unsigned char*)title;
    textprop.encoding = XA_STRING;
    textprop.format = 8;
    textprop.nitems = strlen(title);

    hints.x = x;
    hints.y = y;
    hints.width = width;
    hints.height = height;
    hints.flags = USPosition|USSize;

    startup_state = XAllocWMHints();
    startup_state->initial_state = NormalState;
    startup_state->flags = StateHint;

    XSetWMProperties(Xdisplay, window_handle,&textprop, &textprop,
            NULL, 0,
            &hints,
            startup_state,
            NULL);

    XFree(startup_state);

    XMapWindow(Xdisplay, window_handle);
    XIfEvent(Xdisplay, &event, WaitForMapNotify, (char*)&window_handle);

    if ((del_atom = XInternAtom(Xdisplay, "WM_DELETE_WINDOW", 0)) != None) {
        XSetWMProtocols(Xdisplay, window_handle, &del_atom, 1);
    }
}

static int ctxErrorHandler( Display *dpy, XErrorEvent *ev )
{
    fputs("Error at context creation", stderr);
    return 0;
}

static void createTheRenderContext()
{
    int dummy;
    if (!glXQueryExtension(Xdisplay, &dummy, &dummy)) {
        fatalError("OpenGL not supported by X server\n");
    }

#if USE_GLX_CREATE_CONTEXT_ATTRIB
    #define GLX_CONTEXT_MAJOR_VERSION_ARB       0x2091
    #define GLX_CONTEXT_MINOR_VERSION_ARB       0x2092
    render_context = NULL;
    if( isExtensionSupported( glXQueryExtensionsString(Xdisplay, DefaultScreen(Xdisplay)), "GLX_ARB_create_context" ) ) {
        typedef GLXContext (*glXCreateContextAttribsARBProc)(Display*, GLXFBConfig, GLXContext, Bool, const int*);
        glXCreateContextAttribsARBProc glXCreateContextAttribsARB = (glXCreateContextAttribsARBProc)glXGetProcAddressARB( (const GLubyte *) "glXCreateContextAttribsARB" );
        if( glXCreateContextAttribsARB ) {
            int context_attribs[] =
            {
                GLX_CONTEXT_MAJOR_VERSION_ARB, 3,
                GLX_CONTEXT_MINOR_VERSION_ARB, 0,
                //GLX_CONTEXT_FLAGS_ARB        , GLX_CONTEXT_FORWARD_COMPATIBLE_BIT_ARB,
                None
            };

            int (*oldHandler)(Display*, XErrorEvent*) = XSetErrorHandler(&ctxErrorHandler);
            
            render_context = glXCreateContextAttribsARB( Xdisplay, fbconfig, 0, True, context_attribs );

            XSync( Xdisplay, False );
            XSetErrorHandler( oldHandler );

            fputs("glXCreateContextAttribsARB failed", stderr);
        } else {
            fputs("glXCreateContextAttribsARB could not be retrieved", stderr);
        }
    } else {
            fputs("glXCreateContextAttribsARB not supported", stderr);
    }

    if(!render_context)
    {
#else
    {
#endif
        render_context = glXCreateNewContext(Xdisplay, fbconfig, GLX_RGBA_TYPE, 0, True);
        if (!render_context) {
            fatalError("Failed to create a GL context\n");
        }
    }

    if (!glXMakeContextCurrent(Xdisplay, glX_window_handle, glX_window_handle, render_context)) {
        fatalError("glXMakeCurrent failed for window\n");
    }
}

static int updateTheMessageQueue()
{
    XEvent event;
    XConfigureEvent *xc;

    while (XPending(Xdisplay))
    {
        XNextEvent(Xdisplay, &event);
        switch (event.type)
        {
        case ClientMessage:
            if (event.xclient.data.l[0] == del_atom)
            {
                return 0;
            }
        break;

        case ConfigureNotify:
            xc = &(event.xconfigure);
            width = xc->width;
            height = xc->height;
            break;
        }
    }
    return 1;
}

/*  6----7
   /|   /|
  3----2 |
  | 5--|-4
  |/   |/
  0----1

*/

GLfloat cube_vertices[][8] =  {
    /*  X     Y     Z   Nx   Ny   Nz    S    T */
    {-1.0, -1.0,  1.0, 0.0, 0.0, 1.0, 0.0, 0.0}, // 0
    { 1.0, -1.0,  1.0, 0.0, 0.0, 1.0, 1.0, 0.0}, // 1
    { 1.0,  1.0,  1.0, 0.0, 0.0, 1.0, 1.0, 1.0}, // 2
    {-1.0,  1.0,  1.0, 0.0, 0.0, 1.0, 0.0, 1.0}, // 3

    { 1.0, -1.0, -1.0, 0.0, 0.0, -1.0, 0.0, 0.0}, // 4
    {-1.0, -1.0, -1.0, 0.0, 0.0, -1.0, 1.0, 0.0}, // 5
    {-1.0,  1.0, -1.0, 0.0, 0.0, -1.0, 1.0, 1.0}, // 6
    { 1.0,  1.0, -1.0, 0.0, 0.0, -1.0, 0.0, 1.0}, // 7

    {-1.0, -1.0, -1.0, -1.0, 0.0, 0.0, 0.0, 0.0}, // 5
    {-1.0, -1.0,  1.0, -1.0, 0.0, 0.0, 1.0, 0.0}, // 0
    {-1.0,  1.0,  1.0, -1.0, 0.0, 0.0, 1.0, 1.0}, // 3
    {-1.0,  1.0, -1.0, -1.0, 0.0, 0.0, 0.0, 1.0}, // 6
    
    { 1.0, -1.0,  1.0,  1.0, 0.0, 0.0, 0.0, 0.0}, // 1
    { 1.0, -1.0, -1.0,  1.0, 0.0, 0.0, 1.0, 0.0}, // 4
    { 1.0,  1.0, -1.0,  1.0, 0.0, 0.0, 1.0, 1.0}, // 7
    { 1.0,  1.0,  1.0,  1.0, 0.0, 0.0, 0.0, 1.0}, // 2
    
    {-1.0, -1.0, -1.0,  0.0, -1.0, 0.0, 0.0, 0.0}, // 5
    { 1.0, -1.0, -1.0,  0.0, -1.0, 0.0, 1.0, 0.0}, // 4
    { 1.0, -1.0,  1.0,  0.0, -1.0, 0.0, 1.0, 1.0}, // 1
    {-1.0, -1.0,  1.0,  0.0, -1.0, 0.0, 0.0, 1.0}, // 0

    {-1.0, 1.0,  1.0,  0.0,  1.0, 0.0, 0.0, 0.0}, // 3
    { 1.0, 1.0,  1.0,  0.0,  1.0, 0.0, 1.0, 0.0}, // 2
    { 1.0, 1.0, -1.0,  0.0,  1.0, 0.0, 1.0, 1.0}, // 7
    {-1.0, 1.0, -1.0,  0.0,  1.0, 0.0, 0.0, 1.0}, // 6
};

static void draw_cube(void)
{
    glEnableClientState(GL_VERTEX_ARRAY);
    glEnableClientState(GL_NORMAL_ARRAY);
    glEnableClientState(GL_TEXTURE_COORD_ARRAY);

    glVertexPointer(3, GL_FLOAT, sizeof(GLfloat) * 8, &cube_vertices[0][0]);
    glNormalPointer(GL_FLOAT, sizeof(GLfloat) * 8, &cube_vertices[0][3]);
    glTexCoordPointer(2, GL_FLOAT, sizeof(GLfloat) * 8, &cube_vertices[0][6]);

    glDrawArrays(GL_QUADS, 0, 24);
}

float const light0_dir[]={0,1,0,0};
float const light0_color[]={78./255., 80./255., 184./255.,1};

float const light1_dir[]={-1,1,1,0};
float const light1_color[]={255./255., 220./255., 97./255.,1};

float const light2_dir[]={0,-1,0,0};
float const light2_color[]={31./255., 75./255., 16./255.,1};

static void redrawTheWindow()
{
    float const aspect = (float)width / (float)height;

    static float a=0;
    static float b=0;
    static float c=0;

    glDrawBuffer(GL_BACK);

    glViewport(0, 0, width, height);

    // Clear with alpha = 0.0, i.e. full transparency
        glClearColor(0.0, 0.0, 0.0, 0.0);
    glClear(GL_COLOR_BUFFER_BIT|GL_DEPTH_BUFFER_BIT);
    
    glMatrixMode(GL_PROJECTION);
    glLoadIdentity();
    glFrustum(-aspect, aspect, -1, 1, 2.5, 10);

    glMatrixMode(GL_MODELVIEW);
    glLoadIdentity();

    glEnable(GL_DEPTH_TEST);
    glEnable(GL_CULL_FACE);

    glEnable(GL_BLEND);
    glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA);

    glLightfv(GL_LIGHT0, GL_POSITION, light0_dir);
    glLightfv(GL_LIGHT0, GL_DIFFUSE, light0_color);

    glLightfv(GL_LIGHT1, GL_POSITION, light1_dir);
    glLightfv(GL_LIGHT1, GL_DIFFUSE, light1_color);

    glLightfv(GL_LIGHT2, GL_POSITION, light2_dir);
    glLightfv(GL_LIGHT2, GL_DIFFUSE, light2_color);

    glTranslatef(0., 0., -5.);

    glRotatef(a, 1, 0, 0);
    glRotatef(b, 0, 1, 0);
    glRotatef(c, 0, 0, 1);

    glEnable(GL_LIGHT0);
    glEnable(GL_LIGHT1);
    glEnable(GL_LIGHTING);

    glEnable(GL_COLOR_MATERIAL);
    glColorMaterial(GL_FRONT_AND_BACK, GL_AMBIENT_AND_DIFFUSE);

    glColor4f(1., 1., 1., 0.5);

    glCullFace(GL_FRONT);
    draw_cube();
    glCullFace(GL_BACK);
    draw_cube();

    a = fmod(a+0.1, 360.);
    b = fmod(b+0.5, 360.);
    c = fmod(c+0.25, 360.);

    glXSwapBuffers(Xdisplay, glX_window_handle);
}

int main(int argc, char *argv[])
{
    createTheWindow();
    createTheRenderContext();

    while (updateTheMessageQueue()) {
        redrawTheWindow();
    }

    return 0;
}

The main trick is getting the right FBConfig. You need to ask for a alpha channel and test the associated XRenderPictFormat for the presence of an alpha mask.

Answer 3

I know this is possible with Windows 7, not sure about earlier versions.

To get rid of the window border you need to remove the WS_OVERLAPPEDWINDOW style from the window and add the WS_POPUP style:

DWORD style = ::GetWindowLong(hWnd, GWL_STYLE);
style &= ~WS_OVERLAPPEDWINDOW;
style |= WS_POPUP;
::SetWindowLong(hWnd, GWL_STYLE, style);

To make the background of the OpenGL window transparent, you will need to use the DwmEnableBlurBehindWindow function:

DWM_BLURBEHIND bb = {0};
bb.dwFlags = DWM_BB_ENABLE;
bb.fEnable = true;
bb.hRgnBlur = NULL;
DwmEnableBlurBehindWindow(hWnd, &bb);

You will also need to specify 0 for the alpha value when calling glClearColor.

glClearColor(0.0f,0.0f,0.0f,0.0f);

Also, when creating your OpenGL context, make sure you allocate an alpha channel.

Now your background should be fully transparent. If you keep the window decorations, then the background will use the Aero blur look and you can adjust the level of transparency using the alpha value in glClearColor.

Answer 4

This is an old question, but since newer versions of Windows have compositing and support, as datenwolf hints, for opengl, we can use some of that special sauce for accomplishing this. Although it is also trivial with DirectX (go figure...) Microsoft did add compositing hints to opengl contexts. Yay anti-trust fears!

So instead of an inefficient copy-to-physical-memory action, we can have the compositing engine just understand how to make use of the opengl context.

So, you have to create an opengl context with a pixelformat that specifies an alpha channel and that it should use composition (line 82). Then, you use the DwmApi.h routines to enable a blurred window (line 179) with a completely invalid region specified, which will blur nothing and leave the window transparent. (You need to specify a black+transparent brush on the window class! Oddly!) Then, you actually just use opengl as you are used to using it. In the event loop, every chance you get, you can just draw and swap buffers (line 201) and remember to enable GL_BLEND! :)

Please review/fork https://gist.github.com/3644466 or just view the following code snippet based off of the OP's own answer with this technique instead (you can just plop this in an empty project):

#define _WIN32_WINNT 0x0500

#include <windows.h>
#include <windowsx.h>
#include <GL/gl.h>
#include <GL/glu.h>

#include <dwmapi.h>

#pragma comment (lib, "opengl32.lib")
#pragma comment (lib, "glu32.lib")

#pragma comment (lib, "dwmapi.lib")

#include <assert.h>
#include <tchar.h>

#ifdef  assert
#define verify(expr) if(!expr) assert(0)
#else verify(expr) expr
#endif

const TCHAR szAppName[]=_T("TransparentGL");
const TCHAR wcWndName[]=_T("TransparentGL");

HDC hDC;            
HGLRC m_hrc;        
int w = 240;
int h = 240;

BOOL initSC() {
    glEnable(GL_ALPHA_TEST);        
    glEnable(GL_DEPTH_TEST);        
    glEnable(GL_COLOR_MATERIAL);

    glEnable(GL_LIGHTING);          
    glEnable(GL_LIGHT0);            

    glEnable(GL_BLEND);             
    glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA);
    glClearColor(0, 0, 0, 0);

    return 0;
}

void resizeSC(int width,int height) {
    glViewport(0,0,width,height);
    glMatrixMode(GL_PROJECTION);
    glLoadIdentity();

    glMatrixMode(GL_MODELVIEW );
    glLoadIdentity();
}

BOOL renderSC() {
    glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT );

    glPushMatrix();

    glColor3f(0, 1, 1);
    glBegin(GL_TRIANGLES);                              // Drawing Using Triangles
        glColor3f(1.0f,0.0f,0.0f);                      // Set The Color To Red
        glVertex3f( 0.0f, 1.0f, 0.0f);                  // Top
        glColor3f(0.0f,1.0f,0.0f);                      // Set The Color To Green
        glVertex3f(-1.0f,-1.0f, 0.0f);                  // Bottom Left
        glColor3f(0.0f,0.0f,1.0f);                      // Set The Color To Blue
        glVertex3f( 1.0f,-1.0f, 0.0f);                  // Bottom Right
    glEnd();

    glPopMatrix();
    glFlush();

    return 0;
}

BOOL CreateHGLRC(HWND hWnd) {
    PIXELFORMATDESCRIPTOR pfd = {
      sizeof(PIXELFORMATDESCRIPTOR),
      1,                                // Version Number
      PFD_DRAW_TO_WINDOW      |         // Format Must Support Window
      PFD_SUPPORT_OPENGL      |         // Format Must Support OpenGL
      PFD_SUPPORT_COMPOSITION |         // Format Must Support Composition
      PFD_DOUBLEBUFFER,                 // Must Support Double Buffering
      PFD_TYPE_RGBA,                    // Request An RGBA Format
      32,                               // Select Our Color Depth
      0, 0, 0, 0, 0, 0,                 // Color Bits Ignored
      8,                                // An Alpha Buffer
      0,                                // Shift Bit Ignored
      0,                                // No Accumulation Buffer
      0, 0, 0, 0,                       // Accumulation Bits Ignored
      24,                               // 16Bit Z-Buffer (Depth Buffer)
      8,                                // Some Stencil Buffer
      0,                                // No Auxiliary Buffer
      PFD_MAIN_PLANE,                   // Main Drawing Layer
      0,                                // Reserved
      0, 0, 0                           // Layer Masks Ignored
   };     

   HDC hdc = GetDC(hWnd);
   int PixelFormat = ChoosePixelFormat(hdc, &pfd);
   if (PixelFormat == 0) {
      assert(0);
      return FALSE ;
   }

   BOOL bResult = SetPixelFormat(hdc, PixelFormat, &pfd);
   if (bResult==FALSE) {
      assert(0);
      return FALSE ;
   }

   m_hrc = wglCreateContext(hdc);
   if (!m_hrc){
      assert(0);
      return FALSE;
   }

   ReleaseDC(hWnd, hdc);

   return TRUE;
}

LRESULT CALLBACK WindowFunc(HWND hWnd,UINT msg, WPARAM wParam, LPARAM lParam) {
    PAINTSTRUCT ps;

    switch(msg) {
        case WM_CREATE:
        break;

        case WM_DESTROY:
            if(m_hrc) {
                wglMakeCurrent(NULL, NULL);
                wglDeleteContext(m_hrc) ;
            }
            PostQuitMessage(0) ;
        break;

        default: 
            return DefWindowProc(hWnd,msg,wParam,lParam);
    }

    return 0;
}

int WINAPI _tWinMain(HINSTANCE hThisInst, HINSTANCE hPrevInst, LPSTR str,int nWinMode) {
    WNDCLASSEX wc;
    memset(&wc, 0, sizeof(wc));
    wc.cbSize = sizeof(WNDCLASSEX);
    wc.hIconSm = LoadIcon(NULL, IDI_APPLICATION);
    wc.style = CS_HREDRAW | CS_VREDRAW;
    wc.lpfnWndProc = (WNDPROC)WindowFunc;
    wc.cbClsExtra  = 0;
    wc.cbWndExtra  = 0;
    wc.hInstance = hThisInst;
    wc.hIcon = LoadIcon(NULL, IDI_APPLICATION);
    wc.hCursor = LoadCursor(NULL, IDC_ARROW);
    wc.hbrBackground = (HBRUSH)CreateSolidBrush(0x00000000);
    wc.lpszClassName = szAppName;

    if(!RegisterClassEx(&wc)) {
        MessageBox(NULL, _T("RegisterClassEx - failed"), _T("Error"), MB_OK | MB_ICONERROR);
        return FALSE;
    }

    HWND hWnd = CreateWindowEx(WS_EX_APPWINDOW, szAppName, wcWndName,
                    WS_VISIBLE | WS_POPUP, 200, 150, w, h,
                    NULL, NULL, hThisInst, NULL);

    if(!hWnd) {
        MessageBox(NULL, _T("CreateWindowEx - failed"), _T("Error"), MB_OK | MB_ICONERROR);
        return FALSE;
    }

    DWM_BLURBEHIND bb = {0};
    HRGN hRgn = CreateRectRgn(0, 0, -1, -1);
    bb.dwFlags = DWM_BB_ENABLE | DWM_BB_BLURREGION;
    bb.hRgnBlur = hRgn;
    bb.fEnable = TRUE;
    DwmEnableBlurBehindWindow(hWnd, &bb);

    CreateHGLRC(hWnd);

    HDC hdc = GetDC(hWnd);
    wglMakeCurrent(hdc, m_hrc);
    initSC();
    resizeSC(w, h);
    ReleaseDC(hWnd, hdc);

    MSG msg;  
    while(1) {
        if (PeekMessage(&msg, NULL, 0, 0, PM_REMOVE)) {
            TranslateMessage(&msg);
            DispatchMessage(&msg);
        }
        else {
            HDC hdc = GetDC(hWnd);
            wglMakeCurrent(hdc, m_hrc);

            renderSC();

            SwapBuffers(hdc);
            ReleaseDC(hWnd, hdc);
        }
    } 

    return (FALSE); 
}

Answer 5

This would be very easy if OpenGL windows were allowed to be layered. But they are not, so you'll have to go for something else.

What you could do is to create a layered window (WS_EX_LAYERED + SetLayeredWindowAttributes() - Google 'em if you don't know them) to handle the transparency, and a hidden OpenGL window for the rendering. Render the OpenGL scene to an off-screen buffer, read it back and share it with the layered window, then bitblt (GDI function) it to the layered window.

This might be too slow for very complex stuff, but will give you the effect you are asking for, and work on Windows 2000 and above.

EDIT: When it comes to creating the actual off-screen buffer, framebuffer objects (FBOs) are probably your best bet. You could just draw on the hidden OpenGL window, though I think I recall someone posting about running into troubles with that, because of pixel ownership - FBOs are recommended. You could also use pixel buffers (pbuffers), but these are kind of outdated (stamped "legacy"), and FBOs are considered the modern way to do this. FBOs should give you hardware acceleration (if supported), and won't itself limit you to a specific OpenGL version. You'll need an OpenGL context to use it, so you'll have to create that hidden OpenGL window and set up the FBO from there.

Here are some resources on FBOs:
Wikipedia
FBO
Gamedev article
Guide (for mac, but might be helpful)

Answer 6

great set of demos with source taking you through step by step:

http://www.dhpoware.com/demos/index.html

Answer 7

I know this is old, but I was trying to port the Xlib solution to Gtk+. After a lot of study I finally made it so I really want to share it here for anyone in need.

#include <gtk/gtk.h>
#include <gdk/gdkscreen.h>
#include <gdk/gdkkeysyms.h>
#include <gtk/gtkgl.h>
#include <GL/gl.h>
#include <GL/glu.h>

static gboolean supports_alpha = FALSE;

/***
 *** Configure the OpenGL framebuffer.
***/
static GdkGLConfig* configure_gl(void)
{
    GdkGLConfig* glconfig;

    /* Try double-buffered visual */
    glconfig = gdk_gl_config_new_by_mode(GDK_GL_MODE_RGBA |
        GDK_GL_MODE_ALPHA |
        GDK_GL_MODE_DEPTH |
        GDK_GL_MODE_DOUBLE);
    if (glconfig == NULL)
    {
        printf("Cannot find the double-buffered visual.\n");
        printf("No appropriate OpenGL-capable visual found.\n");
        exit(1);
    }
    printf("Find GLConfig with alpha channel.\n");
    return glconfig;
}

static void screen_changed(GtkWidget* widget, GdkScreen* old_screen, gpointer userdata)
{
    /* To check if the display supports alpha channels, get the colormap */
    GdkScreen* screen = gtk_widget_get_screen(widget);
    GdkColormap* colormap = gdk_screen_get_rgba_colormap(screen);

    if (!colormap)
    {
        printf("Your screen does not support alpha channels!\n");
        colormap = gdk_screen_get_rgb_colormap(screen);
        supports_alpha = FALSE;
    }
    else
    {
        printf("Your screen supports alpha channels!\n");
        supports_alpha = TRUE;
    }

    gtk_widget_set_colormap(widget, colormap);
}

static gboolean expose(GtkWidget* widget, GdkEventExpose* event, gpointer userdata)
{
    GdkGLContext* glcontext = gtk_widget_get_gl_context(widget);
    GdkGLDrawable* gldrawable = gtk_widget_get_gl_drawable(widget);

    if (!gdk_gl_drawable_gl_begin(gldrawable, glcontext))
    {
        return FALSE;
    }

    glDrawBuffer(GL_BACK);

    glClearColor(0.0, 0.0, 0.0, 0.0);
    glClear(GL_COLOR_BUFFER_BIT);

    glBegin(GL_QUADS);
    glColor4f(1.0f, 0.0f, 0.0f, 0.3f);
    glVertex3f(-0.5f, -0.5f, 0);
    glVertex3f(+0.5f, -0.5f, 0);
    glVertex3f(+0.5f, +0.5f, 0);
    glVertex3f(-0.5f, +0.5f, 0);
    glEnd();

    gdk_gl_drawable_swap_buffers(gldrawable);

    gdk_gl_drawable_gl_end(gldrawable);
    return TRUE;
}

int main(int argc, char** argv)
{
    gtk_init(&argc, &argv);

    GtkWidget* window = gtk_window_new(GTK_WINDOW_TOPLEVEL);

    /* Added to config GLConfig */
    GdkGLConfig* glconfig = configure_gl();
    gtk_widget_set_gl_capability(window,
        glconfig,
        NULL,
        TRUE,
        GDK_GL_RGBA_TYPE);

    gtk_window_set_position(GTK_WINDOW(window), GTK_WIN_POS_CENTER);
    gtk_window_set_default_size(GTK_WINDOW(window), 400, 400);
    gtk_window_set_title(GTK_WINDOW(window), "Alpha Demo");
    g_signal_connect(G_OBJECT(window), "delete-event", gtk_main_quit, NULL);

    gtk_widget_set_app_paintable(window, TRUE);

    g_signal_connect(G_OBJECT(window), "expose-event", G_CALLBACK(expose), NULL);
    g_signal_connect(G_OBJECT(window), "screen-changed", G_CALLBACK(screen_changed), NULL);

    screen_changed(window, NULL, NULL);

    gtk_widget_show_all(window);
    gtk_main();

    return 0;
}

Compiled with gcc main.c -o main `pkg-config --libs --cflags gtk+-2.0 gtkglext-1.0`. Tested on Ubuntu 18.04 (in addition to gtk, you'll need to install libgtkglext1-dev).

EDIT

I changed the rendering code from simply a glClear to a rectangle.

The code is a modified version from this question and also this question.

Answer 8

You can render the 3d scene to a pbuffer and blit it to screen using a color key.

Answer 9

The ClanLib game SDK does this.

If you only require a static transparent border use the following technique:

Creates 5 windows

AAAAA

B###C

B###C

DDDDD

A,B,C,D are layered windows

"#" is the main window.

See the images at the bottom of - http://clanlib.org/wiki/ClanLib_2.2.9_Release_Notes