From: Rémi Bernon rbernon@codeweavers.com
--- dlls/kernel32/tests/heap.c | 2 - dlls/ntdll/heap.c | 116 ++++++++++++++++++++++++++++++++++++- 2 files changed, 115 insertions(+), 3 deletions(-)
diff --git a/dlls/kernel32/tests/heap.c b/dlls/kernel32/tests/heap.c index 29faf8c902e..ed84441efe0 100644 --- a/dlls/kernel32/tests/heap.c +++ b/dlls/kernel32/tests/heap.c @@ -904,7 +904,6 @@ static void test_HeapCreate(void)
ret = pHeapQueryInformation( heap, HeapCompatibilityInformation, &compat_info, sizeof(compat_info), &size ); ok( ret, "HeapQueryInformation failed, error %lu\n", GetLastError() ); - todo_wine ok( compat_info == 2, "got HeapCompatibilityInformation %lu\n", compat_info );
ret = HeapDestroy( heap ); @@ -1195,7 +1194,6 @@ static void test_HeapCreate(void)
ret = pHeapQueryInformation( heap, HeapCompatibilityInformation, &compat_info, sizeof(compat_info), &size ); ok( ret, "HeapQueryInformation failed, error %lu\n", GetLastError() ); - todo_wine ok( compat_info == 2, "got HeapCompatibilityInformation %lu\n", compat_info );
/* locking is serialized */ diff --git a/dlls/ntdll/heap.c b/dlls/ntdll/heap.c index b300c7700af..4b1d9dfea20 100644 --- a/dlls/ntdll/heap.c +++ b/dlls/ntdll/heap.c @@ -189,6 +189,53 @@ typedef struct DECLSPEC_ALIGN(BLOCK_ALIGN) tagSUBHEAP C_ASSERT( sizeof(SUBHEAP) == offsetof(SUBHEAP, block) + sizeof(struct block) ); C_ASSERT( sizeof(SUBHEAP) == 4 * BLOCK_ALIGN );
+ +/* LFH block size categories */ +#define BLOCK_SIZE_SMALL_STEP (16) +#define BLOCK_SIZE_SMALL_MAX (1024) +#define BLOCK_SIZE_MEDIUM_STEP (192) +#define BLOCK_SIZE_MEDIUM_MAX (32768) + +#define BLOCK_SIZE_CATEGORY_SMALL( size ) ((size) > 0 ? ((size) - 1) / BLOCK_SIZE_SMALL_STEP : 0) +#define BLOCK_SIZE_CATEGORY_MEDIUM( size ) (BLOCK_SIZE_CATEGORY_SMALL( BLOCK_SIZE_SMALL_MAX ) + 1 + \ + (((size) > BLOCK_SIZE_SMALL_MAX) ? ((size) - BLOCK_SIZE_SMALL_MAX - 1) / BLOCK_SIZE_MEDIUM_STEP : 0)) + +#define BLOCK_SIZE_CATEGORY_COUNT (BLOCK_SIZE_CATEGORY_MEDIUM( BLOCK_SIZE_MEDIUM_MAX ) + 1) +#define BLOCK_SIZE_CATEGORY( size ) (((size) >= BLOCK_SIZE_MEDIUM_MAX) \ + ? (BLOCK_SIZE_CATEGORY_COUNT - 1) \ + : ((size) > BLOCK_SIZE_SMALL_MAX) \ + ? BLOCK_SIZE_CATEGORY_MEDIUM( size ) \ + : BLOCK_SIZE_CATEGORY_SMALL( size )) +#define BLOCK_CATEGORY_SIZE( category ) (category > BLOCK_SIZE_CATEGORY_SMALL( BLOCK_SIZE_SMALL_MAX ) \ + ? (BLOCK_SIZE_SMALL_MAX + (category - BLOCK_SIZE_CATEGORY_SMALL( BLOCK_SIZE_SMALL_MAX )) * BLOCK_SIZE_MEDIUM_STEP) \ + : ((SIZE_T)16 + category * BLOCK_SIZE_SMALL_STEP)) + +/* macro sanity checks */ +C_ASSERT( BLOCK_SIZE_CATEGORY( 0 ) == 0 ); +C_ASSERT( BLOCK_CATEGORY_SIZE( 0 ) == 16 ); +C_ASSERT( BLOCK_SIZE_CATEGORY( 16 ) == 0 ); +C_ASSERT( BLOCK_SIZE_CATEGORY( BLOCK_SIZE_SMALL_MAX ) == 63 ); +C_ASSERT( BLOCK_CATEGORY_SIZE( 63 ) == BLOCK_SIZE_SMALL_MAX ); +C_ASSERT( BLOCK_SIZE_CATEGORY( BLOCK_SIZE_SMALL_MAX + 1 ) == 64 ); +C_ASSERT( BLOCK_CATEGORY_SIZE( 64 ) == BLOCK_SIZE_SMALL_MAX + BLOCK_SIZE_MEDIUM_STEP ); +C_ASSERT( BLOCK_SIZE_CATEGORY( BLOCK_SIZE_SMALL_MAX + BLOCK_SIZE_MEDIUM_STEP ) == 64 ); + +/* keep the block size category count reasonable */ +C_ASSERT( BLOCK_SIZE_CATEGORY_COUNT <= 256 ); + +/* difference between block classes and all possible validation overhead must fit into block tail_size */ +C_ASSERT( BLOCK_SIZE_MEDIUM_STEP + 3 * BLOCK_ALIGN <= FIELD_MAX( struct block, tail_size ) ); + +/* a category of heap blocks of a certain size */ + +struct category +{ + /* counters for LFH activation */ + volatile LONG blocks_alive; + volatile LONG blocks_total; + volatile BOOL enabled; +}; + struct heap { /* win32/win64 */ DWORD_PTR unknown1[2]; /* 0000/0000 */ @@ -212,6 +259,7 @@ struct heap struct block **pending_free; /* Ring buffer for pending free requests */ RTL_CRITICAL_SECTION cs; struct entry free_lists[HEAP_NB_FREE_LISTS]; + struct category *categories; SUBHEAP subheap; };
@@ -544,6 +592,15 @@ static void heap_dump( const struct heap *heap ) TRACE( "heap: %p\n", heap ); TRACE( " next %p\n", LIST_ENTRY( heap->entry.next, struct heap, entry ) );
+ TRACE( " categories:\n" ); + for (i = 0; heap->categories && i < BLOCK_SIZE_CATEGORY_COUNT; i++) + { + struct category *category = heap->categories + i; + if (!category->blocks_alive && !category->blocks_total) continue; + TRACE( " %3u: size %#4Ix, blocks alive %ld, total %ld\n", i, BLOCK_CATEGORY_SIZE(i), + category->blocks_alive, category->blocks_total ); + } + TRACE( " free_lists: %p\n", heap->free_lists ); for (i = 0; i < HEAP_NB_FREE_LISTS; i++) TRACE( " %p: size %#8Ix, prev %p, next %p\n", heap->free_lists + i, get_free_list_block_size( i ), @@ -1430,6 +1487,14 @@ HANDLE WINAPI RtlCreateHeap( ULONG flags, void *addr, SIZE_T total_size, SIZE_T
heap_set_debug_flags( heap );
+ if (heap->flags & HEAP_GROWABLE) + { + /* allocate a large block to keep LFH block categories */ + SIZE_T size = sizeof(struct category) * BLOCK_SIZE_CATEGORY_COUNT; + NtAllocateVirtualMemory( NtCurrentProcess(), (void *)&heap->categories, + 0, &size, MEM_COMMIT, PAGE_READWRITE ); + } + /* link it into the per-process heap list */ if (process_heap) { @@ -1515,6 +1580,11 @@ HANDLE WINAPI RtlDestroyHeap( HANDLE handle ) NtFreeVirtualMemory( NtCurrentProcess(), &addr, &size, MEM_RELEASE ); } valgrind_notify_free_all( &heap->subheap, heap ); + if ((addr = heap->categories)) + { + size = 0; + NtFreeVirtualMemory( NtCurrentProcess(), &addr, &size, MEM_RELEASE ); + } size = 0; addr = heap; NtFreeVirtualMemory( NtCurrentProcess(), &addr, &size, MEM_RELEASE ); @@ -1572,6 +1642,28 @@ static NTSTATUS heap_allocate_block( struct heap *heap, ULONG flags, SIZE_T bloc return STATUS_SUCCESS; }
+static NTSTATUS heap_allocate_block_lfh( struct heap *heap, ULONG flags, SIZE_T block_size, + SIZE_T size, void **ret ) +{ + struct category *category, *last = heap->categories + BLOCK_SIZE_CATEGORY_COUNT - 1; + + category = heap->categories + BLOCK_SIZE_CATEGORY( block_size ); + if (!heap->categories || category == last) return STATUS_UNSUCCESSFUL; + + if (!category->enabled && category->blocks_alive <= 0x10 && category->blocks_total <= 0x800) + return STATUS_UNSUCCESSFUL; + category->enabled = TRUE; + + if (heap->compat_info != HEAP_LFH) + { + ULONG info = HEAP_LFH; + RtlSetHeapInformation( heap, HeapCompatibilityInformation, &info, sizeof(info) ); + return STATUS_UNSUCCESSFUL; + } + + return STATUS_NOT_IMPLEMENTED; +} + /*********************************************************************** * RtlAllocateHeap (NTDLL.@) */ @@ -1589,11 +1681,20 @@ void *WINAPI DECLSPEC_HOTPATCH RtlAllocateHeap( HANDLE handle, ULONG flags, SIZE status = STATUS_NO_MEMORY; else if (block_size >= HEAP_MIN_LARGE_BLOCK_SIZE) status = heap_allocate_large( heap, heap_flags, block_size, size, &ptr ); + else if (!heap_allocate_block_lfh( heap, heap_flags, block_size, size, &ptr )) + status = STATUS_SUCCESS; else { heap_lock( heap, heap_flags ); status = heap_allocate_block( heap, heap_flags, block_size, size, &ptr ); heap_unlock( heap, heap_flags ); + + if (!status && heap->categories) + { + block_size = block_get_size( (struct block *)ptr - 1 ); + heap->categories[BLOCK_SIZE_CATEGORY( block_size )].blocks_alive++; + heap->categories[BLOCK_SIZE_CATEGORY( block_size )].blocks_total++; + } }
if (!status) valgrind_notify_alloc( ptr, size, flags & HEAP_ZERO_MEMORY ); @@ -1628,9 +1729,14 @@ BOOLEAN WINAPI DECLSPEC_HOTPATCH RtlFreeHeap( HANDLE handle, ULONG flags, void * status = STATUS_SUCCESS; else { + SIZE_T block_size = block_get_size( block ); + heap_lock( heap, heap_flags ); status = heap_free_block( heap, heap_flags, block ); heap_unlock( heap, heap_flags ); + + if (!status && heap->categories) + heap->categories[BLOCK_SIZE_CATEGORY( block_size )].blocks_alive--; }
TRACE( "handle %p, flags %#lx, ptr %p, return %u, status %#lx.\n", handle, flags, ptr, !status, status ); @@ -1642,7 +1748,7 @@ static NTSTATUS heap_resize_block( struct heap *heap, ULONG flags, struct block SIZE_T size, SIZE_T *old_size, void **ret ) { SUBHEAP *subheap = block_get_subheap( heap, block ); - SIZE_T old_block_size; + SIZE_T old_category, old_block_size; struct entry *entry; struct block *next;
@@ -1669,6 +1775,7 @@ static NTSTATUS heap_resize_block( struct heap *heap, ULONG flags, struct block
old_block_size = block_get_size( block ); *old_size = old_block_size - block_get_overhead( block ); + old_category = BLOCK_SIZE_CATEGORY( old_block_size );
if (block_size >= HEAP_MIN_LARGE_BLOCK_SIZE) return STATUS_NO_MEMORY; /* growing small block to large block */
@@ -1708,6 +1815,13 @@ static NTSTATUS heap_resize_block( struct heap *heap, ULONG flags, struct block
heap_unlock( heap, flags );
+ if (heap->categories) + { + heap->categories[old_category].blocks_alive--; + heap->categories[BLOCK_SIZE_CATEGORY( block_size )].blocks_alive++; + heap->categories[BLOCK_SIZE_CATEGORY( block_size )].blocks_total++; + } + *ret = block + 1; return STATUS_SUCCESS; }