SetWindowshookEx sometimes not working after dll injection

Question

I am using this lib http://code.google.com/p/injex/ to inject a dll that contains a keyboard hook function. The dll is simple, just create a thread and use the 'setWindowsHookEx' function.

The injection was always successful, but the keyboard hook didn't work for all the cases.

1. I wrote a simple 'hello world' win32 app, and use injex.exe to inject the keyboard hook dll into it. All works as expected.

2. If I choose a complex program to inject (A game written in delphi), the dll was successfully injected(I can see the injected main form), but it failed to hook the keyboard.

So my question is:

1. Is the failure of keyboard hook because of the injection method, or the keyboard hook function?

2. Is it possible that a dll can be injected, but its functionalities are not guaranteed?

So any advices are welcome.

Here is the dll code:

// dllmain.cpp : Defines the entry point for the DLL application.
#include "stdafx.h"

#include <Windows.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <tchar.h>

//GUI==============================================================================
#define MYMENU_EXIT         (WM_APP + 101)
#define MYMENU_MESSAGEBOX   (WM_APP + 102) 

HINSTANCE  inj_hModule;          //Injected Modules Handle
HWND       prnt_hWnd;            //Parent Window Handle

//WndProc for the new window
LRESULT CALLBACK DLLWindowProc (HWND, UINT, WPARAM, LPARAM);

//Register our windows Class
BOOL RegisterDLLWindowClass(wchar_t szClassName[])
{
    WNDCLASSEX wc;
    wc.hInstance =  inj_hModule;
    wc.lpszClassName = (LPCWSTR)L"InjectedDLLWindowClass";
    wc.lpszClassName = (LPCWSTR)szClassName;
    wc.lpfnWndProc = DLLWindowProc;
    wc.style = CS_DBLCLKS;
    wc.cbSize = sizeof (WNDCLASSEX);
    wc.hIcon = LoadIcon (NULL, IDI_APPLICATION);
    wc.hIconSm = LoadIcon (NULL, IDI_APPLICATION);
    wc.hCursor = LoadCursor (NULL, IDC_ARROW);
    wc.lpszMenuName = NULL;
    wc.cbClsExtra = 0;
    wc.cbWndExtra = 0;
    wc.hbrBackground = (HBRUSH) COLOR_BACKGROUND;
    if (!RegisterClassEx (&wc))
        return 0;
}
//Creating our windows Menu
HMENU CreateDLLWindowMenu()
{
    HMENU hMenu;
    hMenu = CreateMenu();
    HMENU hMenuPopup;
    if(hMenu==NULL)
        return FALSE;
    hMenuPopup = CreatePopupMenu();
    AppendMenu (hMenuPopup, MF_STRING, MYMENU_EXIT, TEXT("Exit"));
    AppendMenu (hMenu, MF_POPUP, (UINT_PTR) hMenuPopup, TEXT("File")); 

    hMenuPopup = CreatePopupMenu();
    AppendMenu (hMenuPopup, MF_STRING,MYMENU_MESSAGEBOX, TEXT("MessageBox")); 
    AppendMenu (hMenu, MF_POPUP, (UINT_PTR) hMenuPopup, TEXT("Test")); 
    return hMenu;
}


//Our new windows proc
LRESULT CALLBACK DLLWindowProc (HWND hwnd, UINT message, WPARAM wParam, LPARAM lParam)
{
    switch (message)
    {
        case WM_KEYDOWN:
            if(wParam == VK_HOME) {
            //TextOut(hdc,15, 15, keydown, _tcslen(keydown));
            // etc.
                MessageBox(NULL, _T("Call to RegisterClassEx failed!"), _T("Win32 Guided Tour"), NULL);
            }
        break;
        case WM_COMMAND:
               switch(wParam)
               {
                    case MYMENU_EXIT:
                        SendMessage(hwnd, WM_CLOSE, 0, 0);
                        break;
                    case MYMENU_MESSAGEBOX:
                        MessageBox(hwnd, L"Test", L"MessageBox",MB_OK);
                        break;
               }
               break;
        case WM_DESTROY:
            PostQuitMessage (0);
            break;
        default:
            return DefWindowProc (hwnd, message, wParam, lParam);
    }
    return 0;
}

//HOOK LIB=====================================================================================================
// This will be called when keys are pressed.

HHOOK g_kbHook=NULL;

LRESULT CALLBACK KeyboardHook(int nCode,WPARAM wParam,LPARAM lParam)
{
    KBDLLHOOKSTRUCT* key;
    if(wParam == WM_KEYDOWN || wParam == WM_SYSKEYDOWN){
        key = (KBDLLHOOKSTRUCT*)lParam;
        if(key->vkCode == VkKeyScan('a')){
            //odprintf("You pressed 'a'");
            MessageBox(NULL, _T("Pressed a"), _T("Win32 Guided Tour"), NULL);
        }

        if(key->vkCode == VK_F1){
            MessageBox(NULL, _T("Pressed F1"), _T("Win32 Guided Tour"), NULL);
            //odprintf("You pressed F1");
        }
    }

    return 0;
}

LRESULT CALLBACK _KeyboardHookProc(int nCode,WPARAM wParam,LPARAM lParam){
    //g_kbHookFunction(nCode,wParam,lParam);
    KeyboardHook(nCode,wParam,lParam);
    return CallNextHookEx(g_kbHook,nCode,wParam,lParam);
};

//ENTRY POINT================================================================================================
//our start function, start everything here

DWORD WINAPI start(LPVOID lpParam){
    // START YOUR CODE HERE...

    // Lay some hooks, season libraly with pwnsauce.
    // Do whatever you want here, this is your thread to play with.
    // NOTE: This is used as the starting function in most examples.

    // STOP HERE.

    MSG messages;
    wchar_t *pString = reinterpret_cast<wchar_t * > (lpParam);
    HMENU hMenu = CreateDLLWindowMenu();
    RegisterDLLWindowClass(L"InjectedDLLWindowClass");
    prnt_hWnd = FindWindow(L"Window Injected Into ClassName", L"Window Injected Into Caption");
    HWND hwnd = CreateWindowEx (0, L"InjectedDLLWindowClass", L"mywindow", WS_EX_PALETTEWINDOW, CW_USEDEFAULT, CW_USEDEFAULT, 400, 300, prnt_hWnd, hMenu,inj_hModule, NULL );
    ShowWindow (hwnd, SW_SHOWNORMAL);

    g_kbHook = SetWindowsHookEx(WH_KEYBOARD_LL,_KeyboardHookProc,GetModuleHandle(NULL),0);
    if (!g_kbHook) {

    }

    while (GetMessage (&messages, NULL, 0, 0))
    {
        TranslateMessage(&messages);
        DispatchMessage(&messages);
    }
    return 1;
    g_kbHook = NULL;
}


BOOL APIENTRY DllMain( HMODULE hModule, DWORD  ul_reason_for_call, LPVOID lpReserved) {
    switch (ul_reason_for_call) {
        case DLL_PROCESS_ATTACH:
            // We create a new thread to do our bidding in, this way we don't hold the loader lock.
            // We close the handle to the thread for various reasons.
            inj_hModule = hModule;
            CloseHandle(CreateThread(NULL, 0, start, (LPVOID)hModule, 0, NULL));

        break;
        case DLL_THREAD_ATTACH:
        case DLL_THREAD_DETACH:
        case DLL_PROCESS_DETACH:
        break;
    }

    return TRUE;
}

Answer 1

If the process you're trying to hook already installed its own keyboard hook, that hook may not be calling CallNextHookEx() to pass the message along to your hook. See the description at MSDN (note the explanation in the Return Value section). If that is indeed the cause of your problem, your only option may be to install your hook before the process installs its own hook.