Capture screen of any Windows app?

Question

I'm trying to write a Windows C++ program that will try to pick out a color of interest from whatever is currently being displayed on the screen.

I've tried following examples for GDI, Direct3D9, and Direct3D11 DXGI, and they all seem to work only for capturing the Windows desktop and/or my own application's own output. When I launch a full-screen Direct3D game, I seem to to end up with some flavor of blank pixel data.

It must be possible to accomplish this, or else OBS Studio, FRAPS, etc. would not work as transparently as they do.

I know I could try to reverse engineer OBS Studio, but does anybody have a more succinct C++ solution for capturing an arbitrary Windows application's video output as some kind of pixel buffer?

Edit: I should also mention that capture of regular desktop windows seems to work. It's fullscreen games that are giving me trouble.

Edit: A commenter requested my GDI code. Here is my GDI and D3D9 code. As you can see, I tried a few variations based on conflicting examples that I found:

std::wstring GetScreenColor(COLORREF& colorRef)
{
    std::wstring retVal;

    //const int desktopWidth(GetDeviceCaps(desktopHdc, HORZRES));
    //const int desktopHeight(GetDeviceCaps(desktopHdc, VERTRES));
//        const int desktopWidth(GetSystemMetrics(SM_CXVIRTUALSCREEN));
//        const int desktopHeight(GetSystemMetrics(SM_CYVIRTUALSCREEN));
    const int desktopWidth(GetSystemMetrics(SM_CXSCREEN));
    const int desktopHeight(GetSystemMetrics(SM_CYSCREEN));
    HWND desktopHwnd(GetDesktopWindow());
//        HDC desktopHdc(GetDC(NULL));
    HDC desktopHdc(GetDC(desktopHwnd));
    HDC myHdc(CreateCompatibleDC(desktopHdc));
    const HBITMAP desktopBitmap(CreateCompatibleBitmap(desktopHdc, desktopWidth, desktopHeight));
    SelectObject(myHdc, desktopBitmap);
//        BitBlt(myHdc, 0, 0, desktopWidth, desktopHeight, desktopHdc, 0, 0, SRCCOPY);
    BitBlt(myHdc, 0, 0, desktopWidth, desktopHeight, desktopHdc, 0, 0, SRCCOPY | CAPTUREBLT);
    //SelectObject(myHdc, hOld);
    ReleaseDC(NULL, desktopHdc);

    BITMAPINFO bitmapInfo = { 0 };
    bitmapInfo.bmiHeader.biSize = sizeof(bitmapInfo.bmiHeader);
    bitmapInfo.bmiHeader.biWidth = desktopWidth;
    bitmapInfo.bmiHeader.biHeight = -desktopHeight;
    bitmapInfo.bmiHeader.biPlanes = 1;
    bitmapInfo.bmiHeader.biBitCount = 24;
    bitmapInfo.bmiHeader.biCompression = BI_RGB;
    bitmapInfo.bmiHeader.biSizeImage = 0;

    // TODO: use a persistent buffer?
    const unsigned long numPixels(desktopHeight * desktopWidth);
    ColorBGRS* rawPixels(new ColorBGRS[numPixels]);
    if (!GetDIBits(myHdc, desktopBitmap, 0, desktopHeight, rawPixels, &bitmapInfo, DIB_RGB_COLORS))
    {
        delete[] rawPixels;
        ReleaseDC(desktopHwnd, desktopHdc);
        DeleteDC(myHdc);
        DeleteObject(desktopBitmap);
        return L"GetDIBits() failed";
    }

    unsigned long redSum(0);
    unsigned long greenSum(0);
    unsigned long blueSum(0);

    for (unsigned long index(0); index < numPixels; ++index)
    {
        blueSum  += rawPixels[index].blue;
        greenSum += rawPixels[index].green;
        redSum   += rawPixels[index].red;
    }

    const unsigned long redAverage(redSum / numPixels);
    const unsigned long blueAverage(blueSum / numPixels);
    const unsigned long greenAverage(greenSum / numPixels);

    colorRef = RGB(redAverage, greenAverage, blueAverage);

    delete[] rawPixels;
    ReleaseDC(desktopHwnd, desktopHdc);
    DeleteDC(myHdc);
    DeleteObject(desktopBitmap);

    return std::wstring();
}

std::wstring GetScreenColor2(COLORREF& colorRef)
{
    IDirect3D9* d3d9(Direct3DCreate9(D3D_SDK_VERSION));
    if (!d3d9)
    {
        d3d9->Release();
        return L"Direct3DCreate9() failed";
    }

    D3DDISPLAYMODE d3dDisplayMode;
    if (FAILED(d3d9->GetAdapterDisplayMode(D3DADAPTER_DEFAULT, &d3dDisplayMode)))
    {
        return L"GetAdapterDisplayMode() failed";
    }

    D3DPRESENT_PARAMETERS d3dPresentParams;
    ZeroMemory(&d3dPresentParams, sizeof(D3DPRESENT_PARAMETERS));        
    d3dPresentParams.Windowed = TRUE;
    d3dPresentParams.Flags = D3DPRESENTFLAG_LOCKABLE_BACKBUFFER;
    d3dPresentParams.BackBufferFormat = d3dDisplayMode.Format;
    d3dPresentParams.BackBufferCount = 1;
    d3dPresentParams.BackBufferHeight = d3dDisplayMode.Height;
    d3dPresentParams.BackBufferWidth = d3dDisplayMode.Width;
    d3dPresentParams.MultiSampleType = D3DMULTISAMPLE_NONE;
    d3dPresentParams.SwapEffect = D3DSWAPEFFECT_DISCARD;
    //d3dPresentParams.SwapEffect = D3DSWAPEFFECT_COPY;
    d3dPresentParams.hDeviceWindow = NULL; //hWnd;
    d3dPresentParams.PresentationInterval = D3DPRESENT_INTERVAL_DEFAULT;
    d3dPresentParams.FullScreen_RefreshRateInHz = D3DPRESENT_RATE_DEFAULT;

    IDirect3DDevice9* d3d9Device(0);
    if (FAILED(d3d9->CreateDevice(D3DADAPTER_DEFAULT, D3DDEVTYPE_HAL, d3dPresentParams.hDeviceWindow, D3DCREATE_SOFTWARE_VERTEXPROCESSING, &d3dPresentParams, &d3d9Device)))
    {
        d3d9->Release();
        return L"CreateDevice() failed";
    }

    IDirect3DSurface9* d3d9Surface(0);
//        if (FAILED(d3d9Device->GetBackBuffer(0, 0, D3DBACKBUFFER_TYPE_MONO, &d3d9Surface))) return L"GetBackBuffer() failed";
    if (FAILED(d3d9Device->CreateOffscreenPlainSurface(d3dDisplayMode.Width, d3dDisplayMode.Height, D3DFMT_A8R8G8B8, D3DPOOL_SCRATCH, &d3d9Surface, NULL)))
//        if (FAILED(d3d9Device->CreateOffscreenPlainSurface(d3dDisplayMode.Width, d3dDisplayMode.Height, D3DFMT_A8R8G8B8, D3DPOOL_SYSTEMMEM, &d3d9Surface, NULL)))
    {
        d3d9Device->Release();
        d3d9->Release();
        return L"CreateOffscreenPlainSurface() failed";
    }

    if (FAILED(d3d9Device->GetFrontBufferData(0, d3d9Surface)))
    {
        d3d9Surface->Release();
        d3d9Device->Release();
        d3d9->Release();
        return L"GetFrontBufferData() failed";
    }

    D3DLOCKED_RECT d3dLockedRect;
    if (FAILED(d3d9Surface->LockRect(&d3dLockedRect, 0, D3DLOCK_NO_DIRTY_UPDATE |
                                                        D3DLOCK_NOSYSLOCK |
                                                        D3DLOCK_READONLY)))
    {
        d3d9Surface->UnlockRect();
        d3d9Surface->Release();
        d3d9Device->Release();
        d3d9->Release();
        return L"LockRect() failed";
    }

    const unsigned long numPixels(d3dDisplayMode.Height * d3dDisplayMode.Width);
    BYTE* rawPixels((BYTE*)(d3dLockedRect.pBits));
    colorRef = RGB(*(rawPixels + 2), *(rawPixels + 1), *(rawPixels));

    d3d9Surface->UnlockRect();
    d3d9Surface->Release();
    d3d9Device->Release();
    d3d9->Release();

    return std::wstring();
}

Answer 1

There is the Desktop Duplication API since Windows 8 that is able to record fullscreen applications like games. I have recently made this library for one of my projects that you maybe can use for reference. Only in your case you need to get the raw pixel data from the textures instead of writing them to video or images.

edit: added a small example of reinitialization on lost access.

{
    CComPtr<ID3D11Device> pDevice;
    CComPtr<IDXGIOutputDuplication> pDeskDupl;
    //(...)create devices and duplication interface etc here..
    InitializeDesktopDupl(pDevice, &pDeskDupl, &OutputDuplDesc);
    while(true) //capture loop gist.
    {
    IDXGIResource *pDesktopResource = nullptr;
    DXGI_OUTDUPL_FRAME_INFO FrameInfo;
    RtlZeroMemory(&FrameInfo, sizeof(FrameInfo));
    // Get new frame
    HRESULT hr = pDeskDupl->AcquireNextFrame(
                99,//timeout in ms
                &FrameInfo,
                &pDesktopResource);

            if (hr == DXGI_ERROR_ACCESS_LOST) {
                pDeskDupl->ReleaseFrame();
                pDeskDupl.Release();
                pDesktopResource->Release();

                hr = InitializeDesktopDupl(pDevice, &pDeskDupl, &OutputDuplDesc);
                if(FAILED(hr)){
                 //Check if everything is OK before continuing
                }
            }
     }
}
HRESULT InitializeDesktopDupl(ID3D11Device *pDevice, IDXGIOutputDuplication **ppDesktopDupl, DXGI_OUTDUPL_DESC *pOutputDuplDesc)
{
    *ppDesktopDupl = NULL;
    *pOutputDuplDesc;

    // Get DXGI device
    CComPtr<IDXGIDevice> pDxgiDevice;
    CComPtr<IDXGIOutputDuplication> pDeskDupl = NULL;
    DXGI_OUTDUPL_DESC OutputDuplDesc;

    HRESULT hr = pDevice->QueryInterface(IID_PPV_ARGS(&pDxgiDevice));

    if (FAILED(hr)) { return hr; }
    // Get DXGI adapter
    CComPtr<IDXGIAdapter> pDxgiAdapter;
    hr = pDxgiDevice->GetParent(
        __uuidof(IDXGIAdapter),
        reinterpret_cast<void**>(&pDxgiAdapter));
    pDxgiDevice.Release();
    if (FAILED(hr)) { return hr; }
    // Get output
    CComPtr<IDXGIOutput> pDxgiOutput;
    hr = pDxgiAdapter->EnumOutputs(
        m_DisplayOutput,
        &pDxgiOutput);
    if (FAILED(hr)) { return hr; }
    pDxgiAdapter.Release();

    CComPtr<IDXGIOutput1> pDxgiOutput1;

    hr = pDxgiOutput->QueryInterface(IID_PPV_ARGS(&pDxgiOutput1));
    if (FAILED(hr)) { return hr; }
    pDxgiOutput.Release();

    // Create desktop duplication
    hr = pDxgiOutput1->DuplicateOutput(
        pDevice,
        &pDeskDupl);
    if (FAILED(hr)) { return hr; }
    pDxgiOutput1.Release();

    // Create GUI drawing texture
    pDeskDupl->GetDesc(&OutputDuplDesc);

    pDxgiOutput1.Release();

    *ppDesktopDupl = pDeskDupl;
    (*ppDesktopDupl)->AddRef();
    *pOutputDuplDesc = OutputDuplDesc;
    return hr;
}