[Bug 36013] New: bleem! (PlayStation emulator) fails on startup, complaining with 'Not enough memory to load program!'

https://bugs.winehq.org/show_bug.cgi?id=36013

Bug ID: 36013
           Summary: bleem! (PlayStation emulator) fails on startup,
                    complaining with 'Not enough memory to load program!'
           Product: Wine
           Version: 1.7.16
          Hardware: x86
                OS: Linux
            Status: NEW
          Severity: normal
          Priority: P2
         Component: -unknown
          Assignee: [e-mail]
          Reporter: [e-mail]

Hello folks,

continuation of bug 34618

You need my 1-byte binary patch from bug 34618 to bypass the Win9X debugger
check:

--- snip ---
$ printf '\x30' | dd of=bleem!.exe bs=1 seek=349588 count=1 conv=notrunc
--- snip ---

WINEPREFIX Windows version must be set to 'Win9X' variant.

Relevant part of trace log:

--- snip ---
$ pwd
/home/focht/.wine/drive_c/Program Files/bleem/bleem!

$ WINEDEBUG=+tid,+seh,+relay,+vxd wine ./bleem!.exe >>log.txt 2>&1
...

0023:Call KERNEL32.CreateFileA(004011e0 "C:\Program
Files\bleem\bleem!\bleem!.exe",80000000,00000001,00000000,00000003,00000080,00000000)
ret=0040107c
0023:Ret  KERNEL32.CreateFileA() retval=0000005c ret=0040107c
0023:Call
KERNEL32.CreateFileMappingA(0000005c,00000000,00000002,00000000,00000000,00000000)
ret=00401096
0023:Ret  KERNEL32.CreateFileMappingA() retval=00000060 ret=00401096
0023:Call KERNEL32.MapViewOfFile(00000060,00000004,00000000,00000000,00000000)
ret=004010ad
0023:Ret  KERNEL32.MapViewOfFile() retval=00340000 ret=004010ad
0023:Call krnl386.exe16.VxDCall0(002a0000) ret=003958ad
0023:  eax=004022a4 ebx=000000b5 ecx=00000000 edx=00000005 esi=0033feb0
edi=0033fbac ebp=0033fe38 esp=0033fb08 ds=002b es=002b fs=0063 gs=006b
flags=00000202
0023:Call KERNEL32.LoadLibraryW(7e6dd0e4 L"vwin32.vxd") ret=7e6acb8c
...
0023:Ret  KERNEL32.LoadLibraryW() retval=7e400000 ret=7e6acb8c
0023:Call KERNEL32.GetProcAddress(7e400000,7e6c08de "VxDCall") ret=7e6acbf8
0023:Ret  KERNEL32.GetProcAddress() retval=7e4066ac ret=7e6acbf8
0023:Call vwin32.vxd.VxDCall(002a0000,0033f794) ret=7e6acc68
0023:Call KERNEL32.GetVersion() ret=7e406cc7
0023:Ret  KERNEL32.GetVersion() retval=c0000a04 ret=7e406cc7
0023:Ret  vwin32.vxd.VxDCall() retval=0000040a ret=7e6acc68
0023:Ret  krnl386.exe16.VxDCall0() retval=0000040a ret=003958ad
0023:  eax=0000040a ebx=000000b5 ecx=00000000 edx=00000005 esi=0033feb0
edi=0033fbac ebp=0033fe38 esp=0033fb08 ds=002b es=002b fs=0063 gs=006b
flags=00000202
0023:Call krnl386.exe16.VxDCall0(002a002a) ret=003958c0
0023:  eax=004022a4 ebx=002a002a ecx=00000000 edx=00000005 esi=0033feb0
edi=0033fbac ebp=0033fe38 esp=0033fb0c ds=002b es=002b fs=0063 gs=006b
flags=00000206
0023:Call vwin32.vxd.VxDCall(002a002a,0033f798) ret=7e6acc68
0023:Ret  vwin32.vxd.VxDCall() retval=0000004f ret=7e6acc68
0023:Ret  krnl386.exe16.VxDCall0() retval=0000004f ret=003958c0
0023:  eax=0000004f ebx=002a002a ecx=00000000 edx=00000005 esi=0033feb0
edi=0033fbac ebp=0033fe38 esp=0033fb10 ds=002b es=002b fs=0063 gs=006b
flags=00000206
0023:Call krnl386.exe16.VxDCall0(00010000) ret=00395737
0023:  eax=004022a4 ebx=00000010 ecx=0005b944 edx=00000000 esi=0033fb60
edi=0034fa0c ebp=8005a244 esp=0033fafc ds=002b es=002b fs=0063 gs=006b
flags=00000202
0023:Call KERNEL32.LoadLibraryW(7e6dd0c0 L"vmm.vxd") ret=7e6acb8c
...
0023:Ret  KERNEL32.LoadLibraryW() retval=7e1e0000 ret=7e6acb8c
0023:Call KERNEL32.GetProcAddress(7e1e0000,7e6c08de "VxDCall") ret=7e6acbf8
0023:Ret  KERNEL32.GetProcAddress() retval=7e1ee794 ret=7e6acbf8
0023:Call vmm.vxd.VxDCall(00010000,0033f788) ret=7e6acc68
0023:trace:vxd:VMM_VxDCall PageReserve: page: 80000400, npages: 0000005c,
flags: 00000008 partial stub!
0023:Call KERNEL32.VirtualAlloc(00000000,0005c000,00002000,00000000)
ret=7e1eea61
0023:Ret  KERNEL32.VirtualAlloc() retval=00000000 ret=7e1eea61
0023:trace:vxd:VMM_VxDCall PageReserve: returning: (nil)
0023:Ret  vmm.vxd.VxDCall() retval=ffffffff ret=7e6acc68
0023:Ret  krnl386.exe16.VxDCall0() retval=ffffffff ret=00395737
0023:  eax=ffffffff ebx=00000010 ecx=0005b944 edx=00000000 esi=0033fb60
edi=0034fa0c ebp=8005a244 esp=0033fb08 ds=002b es=002b fs=0063 gs=006b
flags=00000202
0023:Call KERNEL32.UnmapViewOfFile(00340000) ret=004010d6
0023:Ret  KERNEL32.UnmapViewOfFile() retval=00000001 ret=004010d6
0023:Call KERNEL32.CloseHandle(00000060) ret=004010e1
0023:Ret  KERNEL32.CloseHandle() retval=00000001 ret=004010e1
0023:Call KERNEL32.CloseHandle(0000005c) ret=004010ec
0023:Ret  KERNEL32.CloseHandle() retval=00000001 ret=004010ec
0023:Call user32.MessageBeep(00000010) ret=004010ff
0023:Call winex11.drv.SystemParametersInfo(00000001,00000000,0033fdcc,00000000)
ret=7ec19bd2
0023:Ret  winex11.drv.SystemParametersInfo() retval=00000000 ret=7ec19bd2
0023:Call winex11.drv.Beep() ret=7ebffc49
0023:Ret  winex11.drv.Beep() retval=00000001 ret=7ebffc49
0023:Ret  user32.MessageBeep() retval=00000001 ret=004010ff
0023:Call user32.MessageBoxA(00000000,00401336 "Not enough memory to load
program!\n\nClose other applications and try again.",004012e5 "bleem,
llc.",00011010) ret=0040111c 
--- snip ---

The emulator uses VMM.VXD service 'PageReserve' with 'PR_FIXED' attributes to
request a chunk of memory.

VirtualAlloc() with '0' page protection attributes (line 162) doesn't make
sense here, probably a typo.

Source:
http://source.winehq.org/git/wine.git/blob/ae9b07fdc0f8b179d87db07c8ecd2c07f...

--- snip ---
136 DWORD WINAPI VMM_VxDCall( DWORD service, CONTEXT *context )
137 {
138     static int warned;
139
140     switch ( LOWORD(service) )
141     {
142     case 0x0000: /* PageReserve */
143     {
144         LPVOID address;
145         LPVOID ret;
146         ULONG page = stack32_pop( context );
147         ULONG npages = stack32_pop( context );
148         ULONG flags = stack32_pop( context );
149
150         TRACE("PageReserve: page: %08x, npages: %08x, flags: %08x partial
stub!\n",
151             page, npages, flags );
152
153         if ( page == PR_SYSTEM ) {
154             ERR("Can't reserve ring 1 memory\n");
155             return -1;
156         }
157         /* FIXME: This has to be handled separately for the separate
158                 address-spaces we now have */
159         if ( page == PR_PRIVATE || page == PR_SHARED ) page = 0;
160         /* FIXME: Handle flags in some way */
161         address = (LPVOID )(page * page_size);
162         ret = VirtualAlloc ( address, npages * page_size, MEM_RESERVE, 0 );
163         TRACE("PageReserve: returning: %p\n", ret );
164         if ( ret == NULL )
165             return -1;
166         else
167             return (DWORD )ret;
168     }
...
--- snip ---

$ sha1sum demo-v1_4.exe 
b05c7c4023046fabec5a47aac0754ad0ac6b2325  demo-v1_4.exe

$ du -sh demo-v1_4.exe 
444K    demo-v1_4.exe

$ wine --version
wine-1.7.16-178-g7e874ae

Regards