When Hooking API with MinHook get "The value of ESP was not properly saved across a function call"

  api-hook, c++, minhook

In Visual Studio 2019 on Windows 10 x64 20H2 I’m trying to hook the exported function of a DLL for which I don’t have a lib or include file. While I have used MinHook hundreds of times successfully in the past it was always with functions I had the lib and include file for. I am wondering what is needed to stop the hook from triggering crash.

When my hook function executer I get this error:

Microsoft Visual C++ Runtime Library Debug Error!

Program: HookingModule.dll Module: HookingModule.dll File:

Run-Time Check Failure #0 – The value of ESP was not properly saved
across a function call. This is usually a result of calling a
function declared with one calling convention with a function pointer
declared with a different calling convention.

Now I am aware usually this is an indication of hooking with incorrect calling convention. As far as I can tell the function I am hooking is a WndProc handler and uses __stdcall.
The disassembly of function being hooked is:

The beginning of function:

.text:009A14E0 55                                      push    ebp
.text:009A14E1 8B EC                                   mov     ebp, esp
.text:009A14E3 81 EC BC 02 00 00                       sub     esp, 2BCh
.text:009A14E9 8B 45 0C                                mov     eax, [ebp+Msg]
.text:009A14EC 89 85 64 FD FF FF                       mov     [ebp+var_29C], eax
.text:009A14F2 83 BD 64 FD FF FF 20                    cmp     [ebp+var_29C], 20h ; ' '
.text:009A14F9 77 3D                                   ja      short loc_9A1538
.text:009A14FB 83 BD 64 FD FF FF 20                    cmp     [ebp+var_29C], 20h ; ' '
.text:009A1502 0F 84 7D 12 00 00                       jz      loc_9A2785
.text:009A1508 8B 8D 64 FD FF FF                       mov     ecx, [ebp+var_29C]

The end of function:

.text:009A2C35                         loc_9A2C35:                             ; CODE XREF: FN_WindowWnd+1751↑j
.text:009A2C35 8B 4D 08                                mov     ecx, [ebp+hWndParent]
.text:009A2C38 51                                      push    ecx             ; hWnd
.text:009A2C39 E8 02 DC 05 00                          call    sub_A00840
.text:009A2C3E 83 C4 04                                add     esp, 4
.text:009A2C41
.text:009A2C41                         loc_9A2C41:                             ; CODE XREF: FN_WindowWnd+1753↑j
.text:009A2C41 8B 45 A0                                mov     eax, [ebp+var_60]
.text:009A2C44
.text:009A2C44                         loc_9A2C44:                             ; CODE XREF: FN_WindowWnd+351↑j
.text:009A2C44                                                                 ; FN_WindowWnd+36D↑j ...
.text:009A2C44 8B E5                                   mov     esp, ebp
.text:009A2C46 5D                                      pop     ebp
.text:009A2C47 C2 10 00                                retn    10h

The way I am hooking function :

typedef LRESULT (*CALLBACK LPFN_WindowWnd)(_In_ HWND hwnd, _In_ UINT uMsg, _In_ WPARAM wParam, _In_ LPARAM lParam);
LPFN_WindowWnd original_FN_WindowWnd;
LRESULT CALLBACK hooked_FN_WindowWnd(_In_ HWND hwnd, _In_ UINT uMsg, _In_ WPARAM wParam, _In_ LPARAM lParam)
{
    if (uMsg == WM_GETOBJECT) return 0;
    return original_FN_WindowWnd(hwnd, uMsg, wParam, lParam);
}

if (MH_Initialize() != MH_OK)
    {
        TRACE(L"HOOK INITIALIZATION FAILED!");
        return;
    }

HMODULE hModule = GetModuleHandleW(L"hookedmodule.dll");
FN_WindowWnd = (LPFN_WindowWnd)GetProcAddress(hModule, "FN_WindowWnd");

if (hModule != NULL)
    {
        
        FN_WindowWnd = (LPFN_WindowWnd)GetProcAddress(hModule, "FN_WindowWnd");
TRACE(L"HOOK: Create Hook for pbvm100!FN_WindowWnd");
        if (MH_CreateHook(FN_WindowWnd, &hooked_FN_WindowWnd,
            reinterpret_cast<LPVOID*>(&original_FN_WindowWnd)) != MH_OK)
        {
            TRACE(L"CREATE HOOK FAILED!");
        }
MH_EnableHook(MH_ALL_HOOKS);
}

This results in the following disassembly, which as far as I can tell is saving and restoring ESP.

text:100A4410 [email protected]@[email protected]@[email protected] proc near
.text:100A4410                                         ; CODE XREF: hooked_FN_WindowWnd(HWND__ *,uint,uint,long)↑j
.text:100A4410
.text:100A4410 var_C0          = byte ptr -0C0h
.text:100A4410 arg_0           = dword ptr  8
.text:100A4410 arg_4           = dword ptr  0Ch
.text:100A4410 arg_8           = dword ptr  10h
.text:100A4410 arg_C           = dword ptr  14h
.text:100A4410
.text:100A4410                 push    ebp
.text:100A4411                 mov     ebp, esp
.text:100A4413                 sub     esp, 0C0h
.text:100A4419                 push    ebx
.text:100A441A                 push    esi
.text:100A441B                 push    edi
.text:100A441C                 lea     edi, [ebp+var_C0]
.text:100A4422                 mov     ecx, 30h
.text:100A4427                 mov     eax, 0CCCCCCCCh
.text:100A442C                 rep stosd
.text:100A442E                 mov     ecx, offset unk_1021F073
.text:100A4433                 call    [email protected][email protected] ; __CheckForDebuggerJustMyCode(x)
.text:100A4438                 cmp     [ebp+arg_4], 3Dh ; '='
.text:100A443C                 jnz     short loc_100A4442
.text:100A443E                 xor     eax, eax
.text:100A4440                 jmp     short loc_100A4464
.text:100A4442 ; ---------------------------------------------------------------------------
.text:100A4442
.text:100A4442 loc_100A4442:                           ; CODE XREF: hooked_FN_WindowWnd(HWND__ *,uint,uint,long)+2C↑j
.text:100A4442                 mov     esi, esp
.text:100A4444                 mov     eax, [ebp+arg_C]
.text:100A4447                 push    eax
.text:100A4448                 mov     ecx, [ebp+arg_8]
.text:100A444B                 push    ecx
.text:100A444C                 mov     edx, [ebp+arg_4]
.text:100A444F                 push    edx
.text:100A4450                 mov     eax, [ebp+arg_0]
.text:100A4453                 push    eax
.text:100A4454                 call    [email protected]@[email protected]@[email protected] ; int (*original_FN_WindowWnd)(HWND__ *,uint,uint,long)
.text:100A445A                 add     esp, 10h
.text:100A445D                 cmp     esi, esp
.text:100A445F                 call    j___RTC_CheckEsp
.text:100A4464
.text:100A4464 loc_100A4464:                           ; CODE XREF: hooked_FN_WindowWnd(HWND__ *,uint,uint,long)+30↑j
.text:100A4464                 pop     edi
.text:100A4465                 pop     esi
.text:100A4466                 pop     ebx
.text:100A4467                 add     esp, 0C0h
.text:100A446D                 cmp     ebp, esp
.text:100A446F                 call    j___RTC_CheckEsp
.text:100A4474                 mov     esp, ebp
.text:100A4476                 pop     ebp
.text:100A4477                 retn    10h
.text:100A4477 [email protected]@[email protected]@[email protected] endp

In WinDbg this what we can see:

Hooked function is hit and continues to original function:

 1:002> .step_filter "ntdll!*;kernelbase!*;user32!*"
Filter out code symbols matching:
  ntdll!*
  kernelbase!*
  user32!*
    1:002> t
    eax=7b7ef073 ebx=10b514e0 ecx=7b7ef073 edx=00000001 esi=000f1df6 edi=0019ebf8
    eip=7b674438 esp=0019eb2c ebp=0019ebf8 iopl=0         nv up ei pl zr na pe nc
    cs=0023  ss=002b  ds=002b  es=002b  fs=0053  gs=002b             efl=00000246
    HookInit32!hooked_FN_WindowWnd+0x28:
    7b674438 837d0c3d        cmp     dword ptr [ebp+0Ch],3Dh ss:002b:0019ec04=00000081
    1:002> t
    eax=7b7ef073 ebx=10b514e0 ecx=7b7ef073 edx=00000001 esi=000f1df6 edi=0019ebf8
    eip=7b674442 esp=0019eb2c ebp=0019ebf8 iopl=0         nv up ei pl nz ac pe nc
    cs=0023  ss=002b  ds=002b  es=002b  fs=0053  gs=002b             efl=00000216
    HookInit32!hooked_FN_WindowWnd+0x32:
    7b674442 8bf4            mov     esi,esp
    1:002> t
    eax=000f1df6 ebx=10b514e0 ecx=00000000 edx=00000081 esi=0019eb2c edi=0019ebf8
    eip=024a0fe0 esp=0019eb18 ebp=0019ebf8 iopl=0         nv up ei pl nz ac pe nc
    cs=0023  ss=002b  ds=002b  es=002b  fs=0053  gs=002b             efl=00000216
    024a0fe0 55              push    ebp
    1:002> t
    eax=000f1df6 ebx=10b514e0 ecx=00000000 edx=00000081 esi=0019eb2c edi=0019ebf8
    eip=024a0fe1 esp=0019eb14 ebp=0019ebf8 iopl=0         nv up ei pl nz ac pe nc
    cs=0023  ss=002b  ds=002b  es=002b  fs=0053  gs=002b             efl=00000216
    024a0fe1 8bec            mov     ebp,esp
    1:002> t
    eax=000f1df6 ebx=10b514e0 ecx=00000000 edx=00000081 esi=0019eb2c edi=0019ebf8
    eip=024a0fe3 esp=0019eb14 ebp=0019eb14 iopl=0         nv up ei pl nz ac pe nc
    cs=0023  ss=002b  ds=002b  es=002b  fs=0053  gs=002b             efl=00000216
    024a0fe3 81ecbc020000    sub     esp,2BCh
    1:002> t
    eax=000f1df6 ebx=10b514e0 ecx=00000000 edx=00000081 esi=0019eb2c edi=0019ebf8
    eip=024a0fe9 esp=0019e858 ebp=0019eb14 iopl=0         nv up ei pl nz ac po nc
    cs=0023  ss=002b  ds=002b  es=002b  fs=0053  gs=002b             efl=00000212
    024a0fe9 e9fb046b0e      jmp     hookedmodule!FN_WindowWnd+0x9 (10b514e9)
    1:002> t
    eax=000f1df6 ebx=10b514e0 ecx=00000000 edx=00000081 esi=0019eb2c edi=0019ebf8
    eip=10b514e9 esp=0019e858 ebp=0019eb14 iopl=0         nv up ei pl nz ac po nc
    cs=0023  ss=002b  ds=002b  es=002b  fs=0053  gs=002b             efl=00000212
    hookedmodule!FN_WindowWnd+0x9:
    10b514e9 8b450c          mov     eax,dword ptr [ebp+0Ch] ss:002b:0019eb20=00000081
    1:002> t
    eax=00000081 ebx=10b514e0 ecx=00000000 edx=00000081 esi=0019eb2c edi=0019ebf8
    eip=10b514ec esp=0019e858 ebp=0019eb14 iopl=0         nv up ei pl nz ac po nc
    cs=0023  ss=002b  ds=002b  es=002b  fs=0053  gs=002b             efl=00000212
    

etc…

    hookedmodule!fn_txnservice_create_instance+0x300f:
    10bec77f 52              push    edx
    1:002> t
    eax=00000001 ebx=10b514e0 ecx=0000c000 edx=000f1df6 esi=0019eb2c edi=0019ebf8
    eip=10bec780 esp=0019e7b0 ebp=0019e7b8 iopl=0         nv up ei pl nz na po nc
    cs=0023  ss=002b  ds=002b  es=002b  fs=0053  gs=002b             efl=00000202
    hookedmodule!fn_txnservice_create_instance+0x3010:
    10bec780 ff15289ed310    call    dword ptr [hookedmodule!getVtableInfo_plugincontextkeyword+0x229b8 (10d39e28)] ds:002b:10d39e28={USER32!GetPropW (756ea160)}
    1:002> t
    eax=0000c000 ebx=00000000 ecx=ffffe000 edx=000f1df6 esi=000f1df6 edi=00fae220
    eip=763810e0 esp=0019e788 ebp=0019e7a8 iopl=0         nv up ei pl zr na pe nc
    cs=0023  ss=002b  ds=002b  es=002b  fs=0053  gs=002b             efl=00000246
    win32u!NtUserGetProp:
    763810e0 b80e100000      mov     eax,100Eh
    1:002> t
    eax=0000100e ebx=00000000 ecx=ffffe000 edx=000f1df6 esi=000f1df6 edi=00fae220
    eip=763810e5 esp=0019e788 ebp=0019e7a8 iopl=0         nv up ei pl zr na pe nc
    cs=0023  ss=002b  ds=002b  es=002b  fs=0053  gs=002b             efl=00000246
    win32u!NtUserGetProp+0x5:
    763810e5 ba10633876      mov     edx,offset win32u!Wow64SystemServiceCall (76386310)
    1:002> t
    eax=0000100e ebx=00000000 ecx=ffffe000 edx=76386310 esi=000f1df6 edi=00fae220
    eip=763810ea esp=0019e788 ebp=0019e7a8 iopl=0         nv up ei pl zr na pe nc
    cs=0023  ss=002b  ds=002b  es=002b  fs=0053  gs=002b             efl=00000246
    win32u!NtUserGetProp+0xa:
    763810ea ffd2            call    edx {win32u!Wow64SystemServiceCall (76386310)}
    1:002> t
    eax=0000100e ebx=00000000 ecx=ffffe000 edx=76386310 esi=000f1df6 edi=00fae220
    eip=76386310 esp=0019e784 ebp=0019e7a8 iopl=0         nv up ei pl zr na pe nc
    cs=0023  ss=002b  ds=002b  es=002b  fs=0053  gs=002b             efl=00000246
    win32u!Wow64SystemServiceCall:
    76386310 ff25cc703876    jmp     dword ptr [win32u!Wow64Transition (763870cc)] ds:002b:763870cc=776a7000
    1:002> t
    eax=0000100e ebx=00000000 ecx=ffffe000 edx=76386310 esi=000f1df6 edi=00fae220
    eip=776a7000 esp=0019e784 ebp=0019e7a8 iopl=0         nv up ei pl zr na pe nc
    cs=0023  ss=002b  ds=002b  es=002b  fs=0053  gs=002b             efl=00000246
    776a7000 ea09706a773300  jmp     0033:776A7009
    1:002> t
    *** The C++ standard library and CRT step filter can be enabled to skip this function. Run .settings set Sources.SkipCrtCode = true to enable it. ***
    eax=00000000 ebx=0019dccc ecx=db0544f1 edx=00000000 esi=7b674464 edi=7b657329
    eip=7b67728c esp=0019dc54 ebp=0019dc7c iopl=0         nv up ei pl zr na pe nc
    cs=0023  ss=002b  ds=002b  es=002b  fs=0053  gs=002b             efl=00000246
    HookInit32!notify_debugger+0x4c:
    7b67728c eb16            jmp     HookInit32!notify_debugger+0x64 (7b6772a4)
    1:002> t
    eax=00000000 ebx=0019dccc ecx=db0544f1 edx=00000000 esi=7b674464 edi=7b657329
    eip=7b6772a4 esp=0019dc54 ebp=0019dc7c iopl=0         nv up ei pl zr na pe nc
    cs=0023  ss=002b  ds=002b  es=002b  fs=0053  gs=002b             efl=00000246
    HookInit32!notify_debugger+0x64:
    7b6772a4 c745fcfeffffff  mov     dword ptr [ebp-4],0FFFFFFFEh ss:002b:0019dc78=00000000
    1:002> t
    eax=00000000 ebx=0019dccc ecx=05a8dc1e edx=00000000 esi=7b674464 edi=7b657329
    eip=7b676c16 esp=0019dc84 ebp=0019dca4 iopl=0         nv up ei pl zr na pe nc
    cs=0023  ss=002b  ds=002b  es=002b  fs=0053  gs=002b             efl=00000246
    HookInit32!DebuggerProbe+0x26:
    7b676c16 83c404          add     esp,4
    1:002> t
    eax=00000000 ebx=0019dccc ecx=05a8dc1e edx=00000000 esi=7b674464 edi=7b657329
    eip=7b676c19 esp=0019dc88 ebp=0019dca4 iopl=0         nv up ei pl nz na pe nc
    cs=0023  ss=002b  ds=002b  es=002b  fs=0053  gs=002b             efl=00000206
    HookInit32!DebuggerProbe+0x29:
    7b676c19 807dff00        cmp     byte ptr [ebp-1],0         ss:002b:0019dca3=00
    1:002> t
    eax=00000000 ebx=0019dccc ecx=05a8dc1e edx=00000000 esi=7b674464 edi=7b657329
    eip=7b676c20 esp=0019dc88 ebp=0019dca4 iopl=0         nv up ei pl zr na pe nc
    cs=0023  ss=002b  ds=002b  es=002b  fs=0053  gs=002b             efl=00000246
    HookInit32!DebuggerProbe+0x30:
    7b676c20 8be5            mov     esp,ebp
    1:002> t
    eax=00000000 ebx=0019dccc ecx=05a8dc1e edx=00000000 esi=7b674464 edi=7b657329
    eip=7b67705f esp=0019dcac ebp=0019eaf8 iopl=0         nv up ei pl zr na pe nc
    cs=0023  ss=002b  ds=002b  es=002b  fs=0053  gs=002b             efl=00000246
    HookInit32!failwithmessage+0x9f:
    7b67705f 83c404          add     esp,4
    1:002> t
    eax=00000000 ebx=0019dccc ecx=05a8dc1e edx=00000000 esi=7b674464 edi=7b657329
    eip=7b677088 esp=0019dcb0 ebp=0019eaf8 iopl=0         nv up ei pl zr na pe nc
    cs=0023  ss=002b  ds=002b  es=002b  fs=0053  gs=002b             efl=00000246
    HookInit32!failwithmessage+0xc8:
    7b677088 b001            mov     al,1
    1:002> t
    eax=00000001 ebx=0019dccc ecx=05a8dc1e edx=00000000 esi=7b674464 edi=7b657329
    eip=7b67708a esp=0019dcb0 ebp=0019eaf8 iopl=0         nv up ei pl zr na pe nc
    cs=0023  ss=002b  ds=002b  es=002b  fs=0053  gs=002b             efl=00000246
    HookInit32!failwithmessage+0xca:
    7b67708a 83bdccf1ffff00  cmp     dword ptr [ebp-0E34h],0 ss:002b:0019dcc4=00000000
    1:002> t
    eax=00000001 ebx=0019dccc ecx=05a8dc1e edx=00000000 esi=7b674464 edi=7b657329
    eip=775d20d0 esp=0019dcac ebp=0019eaf8 iopl=0         nv up ei pl nz na po nc
    cs=0023  ss=002b  ds=002b  es=002b  fs=0053  gs=002b             efl=00000202
    KERNEL32!IsDebuggerPresentStub:
    775d20d0 ff25e40e6377    jmp     dword ptr [KERNEL32!_imp__IsDebuggerPresent (77630ee4)] ds:002b:77630ee4={KERNELBASE!IsDebuggerPresent (76fa92c0)}
    1:002> t
    eax=00000001 ebx=0019dccc ecx=05a8dc1e edx=00000000 esi=7b674464 edi=7b657329
    eip=7b6770a5 esp=0019dcb0 ebp=0019eaf8 iopl=0         nv up ei pl nz na po nc
    cs=0023  ss=002b  ds=002b  es=002b  fs=0053  gs=002b             efl=00000202
    HookInit32!failwithmessage+0xe5:
    7b6770a5 85c0            test    eax,eax
    1:002> t
    eax=00000001 ebx=0019dccc ecx=05a8dc1e edx=00000000 esi=7b674464 edi=7b657329
    eip=7b6771ac esp=0019dcb0 ebp=0019eaf8 iopl=0         nv up ei pl nz na po nc
    cs=0023  ss=002b  ds=002b  es=002b  fs=0053  gs=002b             efl=00000202
    HookInit32!failwithmessage+0x1ec:
    7b6771ac cc              int     3
    1:002> t
    eax=00000001 ebx=10b514e0 ecx=05b10062 edx=00000000 esi=0019eb2c edi=0019ebf8
    eip=7b6580f3 esp=0019dcb8 ebp=0019eaf8 iopl=0         nv up ei pl nz na po nc
    cs=0023  ss=002b  ds=002b  es=002b  fs=0053  gs=002b             efl=00000202
    HookInit32!ILT+8430(__security_check_cookie:
    7b6580f3 e918050200      jmp     HookInit32!__security_check_cookie (7b678610)
    1:002> t
    eax=00000001 ebx=10b514e0 ecx=05b10062 edx=00000000 esi=0019eb2c edi=0019ebf8
    eip=7b678610 esp=0019dcb8 ebp=0019eaf8 iopl=0         nv up ei pl nz na po nc
    cs=0023  ss=002b  ds=002b  es=002b  fs=0053  gs=002b             efl=00000202
    HookInit32!__security_check_cookie:
    7b678610 3b0d98127b7b    cmp     ecx,dword ptr [HookInit32!__security_cookie (7b7b1298)]

Based On WinDbg tracing instructions it seems a call to GetPropW API from within a function called by the hooked function triggers the crash, or at least is the last code from hooked module to execute.

.text:10BEC760 55                                      push    ebp
.text:10BEC761 8B EC                                   mov     ebp, esp
.text:10BEC763 8B 45 08                                mov     eax, [ebp+hWnd]
.text:10BEC766 50                                      push    eax             ; hWnd
.text:10BEC767 FF 15 4C 9E D3 10                       call    ds:IsWindow
.text:10BEC76D 85 C0                                   test    eax, eax
.text:10BEC76F 75 04                                   jnz     short loc_10BEC775
.text:10BEC771 33 C0                                   xor     eax, eax
.text:10BEC773 EB 11                                   jmp     short loc_10BEC786
.text:10BEC775                         ; ---------------------------------------------------------------------------
.text:10BEC775
.text:10BEC775                         loc_10BEC775:                           ; CODE XREF: sub_10BEC760+F↑j
.text:10BEC775 8B 0D D8 B4 DA 10                       mov     ecx, lpString
.text:10BEC77B 51                                      push    ecx             ; lpString
.text:10BEC77C 8B 55 08                                mov     edx, [ebp+hWnd]
.text:10BEC77F 52                                      push    edx             ; hWnd
.text:10BEC780 FF 15 28 9E D3 10                       call    ds:GetPropW     ; <-- This seems to trigger crashs It's the last code in hooked DLL to run
.text:10BEC786
.text:10BEC786                         loc_10BEC786:                           ; CODE XREF: sub_10BEC760+13↑j
.text:10BEC786 5D                                      pop     ebp
.text:10BEC787 C3                                      retn

Source: Windows Questions C++

LEAVE A COMMENT