From: Rémi Bernon rbernon@codeweavers.com
--- dlls/kernel32/tests/heap.c | 29 +---- dlls/ntdll/heap.c | 237 ++++++++++++++++++++++++++++++++++++- 2 files changed, 240 insertions(+), 26 deletions(-)
diff --git a/dlls/kernel32/tests/heap.c b/dlls/kernel32/tests/heap.c index ed84441efe0..0c95b08780e 100644 --- a/dlls/kernel32/tests/heap.c +++ b/dlls/kernel32/tests/heap.c @@ -452,9 +452,7 @@ static void test_HeapCreate(void)
SetLastError( 0xdeadbeef ); ptr1 = HeapAlloc( heap, 0, alloc_size - (0x200 + 0x80 * sizeof(void *)) ); - todo_wine ok( !ptr1, "HeapAlloc succeeded\n" ); - todo_wine ok( GetLastError() == ERROR_NOT_ENOUGH_MEMORY, "got error %lu\n", GetLastError() ); ret = HeapFree( heap, 0, ptr1 ); ok( ret, "HeapFree failed, error %lu\n", GetLastError() ); @@ -953,7 +951,6 @@ static void test_HeapCreate(void) SetLastError( 0xdeadbeef ); while ((ret = HeapWalk( heap, &entry ))) entries[count++] = entry; ok( GetLastError() == ERROR_NO_MORE_ITEMS, "got error %lu\n", GetLastError() ); - todo_wine ok( count > 24, "got count %lu\n", count ); if (count < 2) count = 2;
@@ -966,7 +963,6 @@ static void test_HeapCreate(void)
for (i = 0; i < 0x12; i++) { - todo_wine ok( entries[4 + i].wFlags == 0, "got wFlags %#x\n", entries[4 + i].wFlags ); todo_wine ok( entries[4 + i].cbData == 0x20, "got cbData %#lx\n", entries[4 + i].cbData ); @@ -985,7 +981,6 @@ static void test_HeapCreate(void) rtl_entry.lpData = NULL; SetLastError( 0xdeadbeef ); while (!RtlWalkHeap( heap, &rtl_entry )) rtl_entries[count++] = rtl_entry; - todo_wine ok( count > 24, "got count %lu\n", count ); if (count < 2) count = 2;
@@ -1028,7 +1023,6 @@ static void test_HeapCreate(void) SetLastError( 0xdeadbeef ); while ((ret = HeapWalk( heap, &entry ))) entries[count++] = entry; ok( GetLastError() == ERROR_NO_MORE_ITEMS, "got error %lu\n", GetLastError() ); - todo_wine ok( count > 24, "got count %lu\n", count ); if (count < 2) count = 2;
@@ -1057,7 +1051,6 @@ static void test_HeapCreate(void) rtl_entry.lpData = NULL; SetLastError( 0xdeadbeef ); while (!RtlWalkHeap( heap, &rtl_entry )) rtl_entries[count++] = rtl_entry; - todo_wine ok( count > 24, "got count %lu\n", count ); if (count < 2) count = 2;
@@ -1071,11 +1064,8 @@ static void test_HeapCreate(void) if (!entries[i].wFlags) ok( rtl_entries[i].wFlags == 0 || rtl_entries[i].wFlags == RTL_HEAP_ENTRY_LFH, "got wFlags %#x\n", rtl_entries[i].wFlags ); else if (entries[i].wFlags & PROCESS_HEAP_ENTRY_BUSY) - { - todo_wine ok( rtl_entries[i].wFlags == (RTL_HEAP_ENTRY_LFH|RTL_HEAP_ENTRY_BUSY) || broken(rtl_entries[i].wFlags == 1) /* win7 */, "got wFlags %#x\n", rtl_entries[i].wFlags ); - } else if (entries[i].wFlags & PROCESS_HEAP_UNCOMMITTED_RANGE) ok( rtl_entries[i].wFlags == RTL_HEAP_ENTRY_UNCOMMITTED || broken(rtl_entries[i].wFlags == 0x100) /* win7 */, "got wFlags %#x\n", rtl_entries[i].wFlags ); @@ -1220,7 +1210,6 @@ static void test_HeapCreate(void) thread_params.flags = 0; SetEvent( thread_params.start_event ); res = WaitForSingleObject( thread_params.ready_event, 100 ); - todo_wine ok( !res, "WaitForSingleObject returned %#lx, error %lu\n", res, GetLastError() ); ret = HeapUnlock( heap ); ok( ret, "HeapUnlock failed, error %lu\n", GetLastError() ); @@ -1667,9 +1656,7 @@ static void test_GlobalAlloc(void) ok( size == small_size, "GlobalSize returned %Iu\n", size ); SetLastError( 0xdeadbeef ); tmp_mem = GlobalReAlloc( mem, small_size, 0 ); - todo_wine ok( !tmp_mem, "GlobalReAlloc succeeded\n" ); - todo_wine ok( GetLastError() == ERROR_NOT_ENOUGH_MEMORY, "got error %lu\n", GetLastError() ); if (tmp_mem) mem = tmp_mem; tmp_mem = GlobalReAlloc( mem, 1024 * 1024, GMEM_MODIFY ); @@ -1685,7 +1672,6 @@ static void test_GlobalAlloc(void) ok( !!mem, "GlobalAlloc failed, error %lu\n", GetLastError() ); tmp_mem = GlobalReAlloc( mem, small_size + 1, GMEM_MOVEABLE ); ok( !!tmp_mem, "GlobalReAlloc failed, error %lu\n", GetLastError() ); - todo_wine ok( tmp_mem != mem, "GlobalReAlloc didn't relocate memory\n" ); ptr = GlobalLock( tmp_mem ); ok( !!ptr, "GlobalLock failed, error %lu\n", GetLastError() ); @@ -1832,8 +1818,8 @@ static void test_GlobalAlloc(void) { ok( !is_mem_entry( tmp_mem ), "unexpected moveable %p\n", tmp_mem ); if (flags == GMEM_MODIFY) ok( tmp_mem == mem, "GlobalReAlloc returned %p\n", tmp_mem ); - else if (flags != GMEM_MOVEABLE) todo_wine_if(!flags) ok( !tmp_mem, "GlobalReAlloc succeeded\n" ); - else todo_wine ok( tmp_mem != mem, "GlobalReAlloc returned %p\n", tmp_mem ); + else if (flags != GMEM_MOVEABLE) ok( !tmp_mem, "GlobalReAlloc succeeded\n" ); + else ok( tmp_mem != mem, "GlobalReAlloc returned %p\n", tmp_mem ); } else { @@ -1847,7 +1833,7 @@ static void test_GlobalAlloc(void)
size = GlobalSize( mem ); if (flags == GMEM_MOVEABLE) ok( size == 0 || broken( size == 1 ) /* w7 */, "GlobalSize returned %Iu\n", size ); - else todo_wine_if(!flags) ok( size == small_size, "GlobalSize returned %Iu\n", size ); + else ok( size == small_size, "GlobalSize returned %Iu\n", size );
mem = GlobalFree( mem ); ok( !mem, "GlobalFree failed, error %lu\n", GetLastError() ); @@ -2405,9 +2391,7 @@ static void test_LocalAlloc(void) ok( size == small_size, "LocalSize returned %Iu\n", size ); SetLastError( 0xdeadbeef ); tmp_mem = LocalReAlloc( mem, small_size, 0 ); - todo_wine ok( !tmp_mem, "LocalReAlloc succeeded\n" ); - todo_wine ok( GetLastError() == ERROR_NOT_ENOUGH_MEMORY, "got error %lu\n", GetLastError() ); if (tmp_mem) mem = tmp_mem; tmp_mem = LocalReAlloc( mem, 1024 * 1024, LMEM_MODIFY ); @@ -2423,7 +2407,6 @@ static void test_LocalAlloc(void) ok( !!mem, "LocalAlloc failed, error %lu\n", GetLastError() ); tmp_mem = LocalReAlloc( mem, small_size + 1, LMEM_MOVEABLE ); ok( !!tmp_mem, "LocalReAlloc failed, error %lu\n", GetLastError() ); - todo_wine ok( tmp_mem != mem, "LocalReAlloc didn't relocate memory\n" ); ptr = LocalLock( tmp_mem ); ok( !!ptr, "LocalLock failed, error %lu\n", GetLastError() ); @@ -2516,13 +2499,13 @@ static void test_LocalAlloc(void) tmp_mem = LocalReAlloc( mem, 0, flags ); ok( !is_mem_entry( tmp_mem ), "unexpected moveable %p\n", tmp_mem ); if (flags & LMEM_MODIFY) ok( tmp_mem == mem, "LocalReAlloc returned %p\n", tmp_mem ); - else if (flags != LMEM_MOVEABLE) todo_wine_if(!flags) ok( !tmp_mem, "LocalReAlloc succeeded\n" ); - else todo_wine ok( tmp_mem != mem, "LocalReAlloc returned %p\n", tmp_mem ); + else if (flags != LMEM_MOVEABLE) ok( !tmp_mem, "LocalReAlloc succeeded\n" ); + else ok( tmp_mem != mem, "LocalReAlloc returned %p\n", tmp_mem ); if (tmp_mem) mem = tmp_mem;
size = LocalSize( mem ); if (flags == LMEM_MOVEABLE) ok( size == 0 || broken( size == 1 ) /* w7 */, "LocalSize returned %Iu\n", size ); - else todo_wine_if(!flags) ok( size == small_size, "LocalSize returned %Iu\n", size ); + else ok( size == small_size, "LocalSize returned %Iu\n", size );
mem = LocalFree( mem ); ok( !mem, "LocalFree failed, error %lu\n", GetLastError() ); diff --git a/dlls/ntdll/heap.c b/dlls/ntdll/heap.c index e5e7044affe..dbda193f185 100644 --- a/dlls/ntdll/heap.c +++ b/dlls/ntdll/heap.c @@ -200,7 +200,8 @@ static const SIZE_T free_list_sizes[] = typedef struct DECLSPEC_ALIGN(BLOCK_ALIGN) tagSUBHEAP { SIZE_T __pad[sizeof(SIZE_T) / sizeof(DWORD) - 1]; - DWORD affinity; + WORD affinity; + WORD user_count; SIZE_T block_size; SIZE_T data_size; struct list entry; @@ -1081,10 +1082,61 @@ static DWORD next_thread_data_index(void) return index; }
+/* release thread blocks to the main heap free lists, clearing the split flags */ +static void heap_release_blocks_lfh( struct heap *heap, SUBHEAP *subheap ) +{ + struct block *block, *next; + + for (block = first_block( subheap ); block; block = next) + { + /* remove the flag one block at a time, so heap_free_block only merges with the previous */ + block_set_flags( block, BLOCK_FLAG_SPLIT, 0 ); + if ((next = next_block( subheap, block ))) block_set_flags( next, 0, BLOCK_FLAG_SPLIT ); + + if (block_get_flags( block ) & BLOCK_FLAG_FREE) + { + struct entry *entry = CONTAINING_RECORD( block, struct entry, block ); + list_remove( &entry->entry ); + heap_free_block( heap, 0, block ); + } + } +} + +/* release subheaps owned by current thread, clearing the subheap affinity */ +static void heap_release_subheaps_lfh( struct heap *heap ) +{ + SUBHEAP *subheap, *next; + + LIST_FOR_EACH_ENTRY_SAFE( subheap, next, &heap->subheap_list, SUBHEAP, entry ) + { + if (subheap_get_affinity( subheap ) == NtCurrentTeb()->HeapVirtualAffinity) + { + subheap_set_affinity( subheap, 0 ); + heap_release_blocks_lfh( heap, subheap ); + } + } +} + void heap_thread_detach(void) { + struct heap *heap; + DWORD index = NtCurrentTeb()->HeapVirtualAffinity - 1; if (index >= ARRAY_SIZE(thread_data_tid)) return; + + heap_lock( process_heap, 0 ); + + LIST_FOR_EACH_ENTRY( heap, &process_heap->entry, struct heap, entry ) + { + heap_lock( heap, 0 ); + heap_release_subheaps_lfh( heap ); + heap_unlock( heap, 0 ); + } + + heap_release_subheaps_lfh( process_heap ); + + heap_unlock( process_heap, 0 ); + InterlockedExchange( thread_data_tid + index, 0 ); }
@@ -1124,7 +1176,9 @@ static SUBHEAP *create_subheap( struct heap *heap, DWORD flags, SIZE_T total_siz block_size = (SIZE_T)ROUND_ADDR( subheap_size( subheap ) - subheap_overhead( subheap ), BLOCK_ALIGN - 1 ); block_init_free( first_block( subheap ), flags, subheap, block_size );
- list_add_head( &heap->subheap_list, &subheap->entry ); + heap_lock( heap, 0 ); + list_add_tail( &heap->subheap_list, &subheap->entry ); + heap_unlock( heap, 0 );
return subheap; } @@ -1722,11 +1776,127 @@ static SIZE_T heap_get_block_size( const struct heap *heap, ULONG flags, SIZE_T return block_size; }
+static void insert_free_block_lfh( struct heap *heap, struct entry *entry, struct heap_thread_data *thread_data ) +{ + SIZE_T block_size = block_get_size( &entry->block ), category = BLOCK_SIZE_CATEGORY( block_size ); + struct entry *free_list = thread_data->free_lists + category; + SUBHEAP *subheap = block_get_subheap( heap, &entry->block ); + + list_add_head( &free_list->entry, &entry->entry ); + if (!--subheap->user_count) + { + void *addr = subheap_base( subheap ); + struct block *block; + SIZE_T size = 0; + + for (block = first_block( subheap ); block; block = next_block( subheap, block )) + { + entry = (struct entry *)block; + list_remove( &entry->entry ); + } + + heap_lock( heap, 0 ); + list_remove( &subheap->entry ); + heap_unlock( heap, 0 ); + + NtFreeVirtualMemory( NtCurrentProcess(), &addr, &size, MEM_RELEASE ); + } +} + +static void flush_defer_list( struct heap *heap, ULONG flags, struct heap_thread_data *thread_data ) +{ + struct list pending = LIST_INIT( pending ); + struct entry *entry, *next; + + if (!thread_data->has_defer) return; + + heap_lock( heap, flags ); + list_move_tail( &pending, &thread_data->defer_list.entry ); + thread_data->has_defer = FALSE; + heap_unlock( heap, flags ); + + LIST_FOR_EACH_ENTRY_SAFE( entry, next, &pending, struct entry, entry ) + { + list_remove( &entry->entry ); + insert_free_block_lfh( heap, entry, thread_data ); + } +} + +static void split_block_lfh( struct heap *heap, ULONG flags, struct block *block, SIZE_T block_size, struct entry *free_list ) +{ + SIZE_T i, old_block_size = block_get_size( block ), count = min( 16, old_block_size / block_size ); + SUBHEAP *subheap = block_get_subheap( heap, block ); + struct entry *entry; + + if (!subheap_commit( heap, subheap, block, count * block_size )) return; + + for (i = 0; i < count; ++i) + { + block_init_free( block, flags, subheap, block_size ); + block_set_flags( block, 0, BLOCK_FLAG_SPLIT ); + old_block_size -= block_size; + + entry = (struct entry *)block; + if (i) list_add_head( &free_list->entry, &entry->entry ); + + block = next_block( subheap, block ); + } + + if ((entry = (struct entry *)block)) + { + block_init_free( block, flags, subheap, old_block_size ); + /* only if large enough to carve more blocks from it */ + if (old_block_size < block_size) list_init( &entry->entry ); + else list_add_tail( &free_list->entry, &entry->entry ); + } + + /* we've split the region entirely, we can consider releasing it */ + if (old_block_size < block_size || !block) subheap->user_count--; +} + +static struct block *find_free_block_lfh( struct heap *heap, ULONG flags, SIZE_T category, + struct heap_thread_data *thread_data ) +{ + SIZE_T block_size = BLOCK_CATEGORY_SIZE(category), commit_size, total_size; + WORD affinity = NtCurrentTeb()->HeapVirtualAffinity; + struct entry *entry, *free_list; + struct list *ptr; + SUBHEAP *subheap; + + free_list = thread_data->free_lists + category; + commit_size = sizeof(*subheap) + 16 * block_size + sizeof(*entry); + total_size = ROUND_SIZE( sizeof(*subheap) + 512 * block_size + sizeof(*entry), REGION_ALIGN ); + + if ((ptr = list_head( &free_list->entry ))) + { + entry = LIST_ENTRY( ptr, struct entry, entry ); + list_remove( &entry->entry ); + subheap = block_get_subheap( heap, &entry->block ); + } + else if (!(flags & HEAP_GROWABLE)) + return NULL; + else if (!(subheap = create_subheap( heap, flags, commit_size, total_size, affinity ))) + return NULL; + else + { + TRACE( "created new sub-heap %p of %#Ix bytes for heap %p\n", subheap, subheap_size( subheap ), heap ); + subheap->user_count = 1; /* avoid releasing the region until we've split it entirely */ + entry = first_block( subheap ); + } + + if (!(block_get_flags( &entry->block ) & BLOCK_FLAG_SPLIT)) + split_block_lfh( heap, flags, &entry->block, block_size, free_list ); + + subheap->user_count++; + return &entry->block; +} + static NTSTATUS heap_allocate_block_lfh( struct heap *heap, ULONG flags, SIZE_T block_size, SIZE_T size, void **ret ) { SIZE_T category = BLOCK_SIZE_CATEGORY( block_size ); struct heap_thread_data *thread_data; + struct block *block;
if (category >= ARRAY_SIZE(heap->stats)) return STATUS_UNSUCCESSFUL; if (heap->stats[category].live <= 0x10 && heap->stats[category].total <= 0x800) @@ -1743,7 +1913,12 @@ static NTSTATUS heap_allocate_block_lfh( struct heap *heap, ULONG flags, SIZE_T } if (!(thread_data = heap_get_thread_data( heap ))) return STATUS_UNSUCCESSFUL;
- return STATUS_NOT_CAPABLE; + if (!(block = find_free_block_lfh( heap, flags, category, thread_data ))) return STATUS_NO_MEMORY; + block_init_used( block, flags, size ); + flush_defer_list( heap, flags, thread_data ); + + *ret = block + 1; + return STATUS_SUCCESS; }
static NTSTATUS heap_allocate_block( struct heap *heap, ULONG flags, SIZE_T block_size, SIZE_T size, void **ret ) @@ -1806,6 +1981,40 @@ void *WINAPI DECLSPEC_HOTPATCH RtlAllocateHeap( HANDLE handle, ULONG flags, SIZE return ptr; }
+static NTSTATUS heap_free_block_lfh( struct heap *heap, ULONG flags, struct block *block, WORD block_affinity ) +{ + SIZE_T block_size = block_get_size( block ), category = BLOCK_SIZE_CATEGORY( block_size ); + ULONG affinity = NtCurrentTeb()->HeapVirtualAffinity; + SUBHEAP *subheap = block_get_subheap( heap, block ); + struct entry *entry = (struct entry *)block; + + valgrind_make_writable( &block, sizeof(*entry) ); + block_set_type( block, BLOCK_TYPE_FREE ); + block_set_flags( block, ~BLOCK_FLAG_SPLIT, BLOCK_FLAG_FREE ); + block_set_base( block, subheap_base( subheap ) ); + block_set_size( block, BLOCK_CATEGORY_SIZE(category) ); + mark_block_free( block + 1, BLOCK_CATEGORY_SIZE(category) - sizeof(*block), flags ); + + if (block_affinity == affinity) + { + struct heap_thread_data *thread_data = heap_get_thread_data( heap ); + insert_free_block_lfh( heap, entry, thread_data ); + } + else + { + heap_lock( heap, flags ); + /* the subheap affinity may be reset to zero when the owner thread is detached */ + if ((block_affinity = block_get_affinity( heap, block ))) + { + list_add_tail( &heap->tls[block_affinity - 1].defer_list.entry, &entry->entry ); + heap->tls[block_affinity - 1].has_defer = TRUE; + } + heap_unlock( heap, flags ); + return block_affinity ? STATUS_SUCCESS : STATUS_NOT_CAPABLE; + } + + return STATUS_SUCCESS; +}
/*********************************************************************** * RtlFreeHeap (NTDLL.@) @@ -1816,6 +2025,7 @@ BOOLEAN WINAPI DECLSPEC_HOTPATCH RtlFreeHeap( HANDLE handle, ULONG flags, void * struct heap *heap; ULONG heap_flags; NTSTATUS status; + WORD affinity;
if (!ptr) return TRUE;
@@ -1829,6 +2039,8 @@ BOOLEAN WINAPI DECLSPEC_HOTPATCH RtlFreeHeap( HANDLE handle, ULONG flags, void * status = heap_free_large( heap, heap_flags, block ); else if (!(block = heap_delay_free( heap, heap_flags, block ))) status = STATUS_SUCCESS; + else if ((affinity = block_get_affinity( heap, block ))) + status = heap_free_block_lfh( heap, heap_flags, block, affinity ); else { heap_lock( heap, heap_flags ); @@ -1848,6 +2060,7 @@ static NTSTATUS heap_resize_block( struct heap *heap, ULONG flags, struct block SIZE_T old_block_size; struct entry *entry; struct block *next; + WORD affinity;
if (block_get_flags( block ) & BLOCK_FLAG_LARGE) { @@ -1875,6 +2088,24 @@ static NTSTATUS heap_resize_block( struct heap *heap, ULONG flags, struct block
if (block_size >= HEAP_MIN_LARGE_BLOCK_SIZE) return STATUS_NO_MEMORY; /* growing small block to large block */
+ if ((affinity = block_get_affinity( heap, block ))) + { + if (affinity != NtCurrentTeb()->HeapVirtualAffinity) return STATUS_NO_MEMORY; /* resizing block owned by another thread */ + + /* as native LFH does it, different block size even though we could resize */ + if (ROUND_SIZE( *old_size, BLOCK_ALIGN - 1) != ROUND_SIZE( size, BLOCK_ALIGN - 1)) return STATUS_NO_MEMORY; + if (size >= *old_size) return STATUS_NO_MEMORY; + + valgrind_notify_resize( block + 1, *old_size, size ); + block_set_flags( block, BLOCK_FLAG_USER_MASK, BLOCK_USER_FLAGS( flags ) ); + block->tail_size = block_get_size( block ) - sizeof(*block) - size; + initialize_block( block, *old_size, size, flags ); + mark_block_tail( block, flags ); + + *ret = block + 1; + return STATUS_SUCCESS; + } + heap_lock( heap, flags );
if (block_size > old_block_size)