Not completely sure if it's worth having for 8.0, but opening this to share the target I'm trying to reach.
-- v4: ntdll: Implement Low Fragmentation Heap. ntdll: Introduce per-thread free lists for heap blocks. ntdll: Introduce a new BLOCK_FLAG_SPLIT heap block flag. ntdll: Introduce a new subheap thread affinity field. ntdll: Introduce a new heap block_init_used helper. ntdll: Introduce a new heap free_list_init helper. ntdll: Count allocations and automatically enable LFH. ntdll: Implement HeapCompatibilityInformation. ntdll: Fix HeapWalk with empty uncommitted consecutive subheaps.
From: Rémi Bernon rbernon@codeweavers.com
--- dlls/ntdll/heap.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/dlls/ntdll/heap.c b/dlls/ntdll/heap.c index 9403c83e92c..a2410957a44 100644 --- a/dlls/ntdll/heap.c +++ b/dlls/ntdll/heap.c @@ -653,7 +653,7 @@ static SUBHEAP *find_subheap( const struct heap *heap, const struct block *block if (!check_subheap( subheap, heap )) return NULL; if (contains( first_block( subheap ), blocks_size, block, sizeof(*block) )) return subheap; /* outside of blocks region, possible corruption or heap_walk */ - if (contains( subheap_base( subheap ), subheap_size( subheap ), block, 0 )) return heap_walk ? subheap : NULL; + if (contains( subheap_base( subheap ), subheap_size( subheap ), block, 1 )) return heap_walk ? subheap : NULL; }
return NULL;
From: Rémi Bernon rbernon@codeweavers.com
--- dlls/kernel32/tests/heap.c | 10 ------- dlls/ntdll/heap.c | 56 ++++++++++++++++++++++++++++++++------ 2 files changed, 47 insertions(+), 19 deletions(-)
diff --git a/dlls/kernel32/tests/heap.c b/dlls/kernel32/tests/heap.c index a81cb1b7a15..29faf8c902e 100644 --- a/dlls/kernel32/tests/heap.c +++ b/dlls/kernel32/tests/heap.c @@ -827,11 +827,8 @@ static void test_HeapCreate(void) size = 0; SetLastError( 0xdeadbeef ); ret = pHeapQueryInformation( 0, HeapCompatibilityInformation, &compat_info, sizeof(compat_info), &size ); - todo_wine ok( !ret, "HeapQueryInformation succeeded\n" ); - todo_wine ok( GetLastError() == ERROR_NOACCESS, "got error %lu\n", GetLastError() ); - todo_wine ok( size == 0, "got size %Iu\n", size );
size = 0; @@ -871,7 +868,6 @@ static void test_HeapCreate(void) ok( ret, "HeapSetInformation failed, error %lu\n", GetLastError() ); ret = pHeapQueryInformation( heap, HeapCompatibilityInformation, &compat_info, sizeof(compat_info), &size ); ok( ret, "HeapQueryInformation failed, error %lu\n", GetLastError() ); - todo_wine ok( compat_info == 2, "got HeapCompatibilityInformation %lu\n", compat_info );
/* cannot be undone */ @@ -879,20 +875,15 @@ static void test_HeapCreate(void) compat_info = 0; SetLastError( 0xdeadbeef ); ret = pHeapSetInformation( heap, HeapCompatibilityInformation, &compat_info, sizeof(compat_info) ); - todo_wine ok( !ret, "HeapSetInformation succeeded\n" ); - todo_wine ok( GetLastError() == ERROR_GEN_FAILURE, "got error %lu\n", GetLastError() ); compat_info = 1; SetLastError( 0xdeadbeef ); ret = pHeapSetInformation( heap, HeapCompatibilityInformation, &compat_info, sizeof(compat_info) ); - todo_wine ok( !ret, "HeapSetInformation succeeded\n" ); - todo_wine ok( GetLastError() == ERROR_GEN_FAILURE, "got error %lu\n", GetLastError() ); ret = pHeapQueryInformation( heap, HeapCompatibilityInformation, &compat_info, sizeof(compat_info), &size ); ok( ret, "HeapQueryInformation failed, error %lu\n", GetLastError() ); - todo_wine ok( compat_info == 2, "got HeapCompatibilityInformation %lu\n", compat_info );
ret = HeapDestroy( heap ); @@ -931,7 +922,6 @@ static void test_HeapCreate(void) ok( ret, "HeapSetInformation failed, error %lu\n", GetLastError() ); ret = pHeapQueryInformation( heap, HeapCompatibilityInformation, &compat_info, sizeof(compat_info), &size ); ok( ret, "HeapQueryInformation failed, error %lu\n", GetLastError() ); - todo_wine ok( compat_info == 2, "got HeapCompatibilityInformation %lu\n", compat_info );
for (i = 0; i < 0x11; i++) ptrs[i] = pHeapAlloc( heap, 0, 24 + 2 * sizeof(void *) ); diff --git a/dlls/ntdll/heap.c b/dlls/ntdll/heap.c index a2410957a44..e72f65da7ed 100644 --- a/dlls/ntdll/heap.c +++ b/dlls/ntdll/heap.c @@ -39,6 +39,13 @@
WINE_DEFAULT_DEBUG_CHANNEL(heap);
+/* HeapCompatibilityInformation values */ + +#define HEAP_STD 0 +#define HEAP_LAL 1 +#define HEAP_LFH 2 + + /* undocumented RtlWalkHeap structure */
struct rtl_heap_entry @@ -194,6 +201,7 @@ struct heap DWORD force_flags; /* 0044/0074 */ /* end of the Windows 10 compatible struct layout */
+ LONG compat_info; /* HeapCompatibilityInformation / heap frontend type */ struct list entry; /* Entry in process heap list */ struct list subheap_list; /* Sub-heap list */ struct list large_list; /* Large blocks list */ @@ -1378,6 +1386,7 @@ HANDLE WINAPI RtlCreateHeap( ULONG flags, void *addr, SIZE_T total_size, SIZE_T heap->ffeeffee = 0xffeeffee; heap->auto_flags = (flags & HEAP_GROWABLE); heap->flags = (flags & ~HEAP_SHARED); + heap->compat_info = HEAP_STD; heap->magic = HEAP_MAGIC; heap->grow_size = max( HEAP_DEF_SIZE, total_size ); heap->min_size = commit_size; @@ -2039,21 +2048,24 @@ ULONG WINAPI RtlGetProcessHeaps( ULONG count, HANDLE *heaps ) * RtlQueryHeapInformation (NTDLL.@) */ NTSTATUS WINAPI RtlQueryHeapInformation( HANDLE handle, HEAP_INFORMATION_CLASS info_class, - void *info, SIZE_T size_in, PSIZE_T size_out ) + void *info, SIZE_T size_in, SIZE_T *size_out ) { + struct heap *heap; + ULONG flags; + + TRACE( "handle %p, info_class %u, info %p, size_in %Iu, size_out %p.\n", handle, info_class, info, size_in, size_out ); + switch (info_class) { case HeapCompatibilityInformation: + if (!(heap = unsafe_heap_from_handle( handle, 0, &flags ))) return STATUS_ACCESS_VIOLATION; if (size_out) *size_out = sizeof(ULONG); - - if (size_in < sizeof(ULONG)) - return STATUS_BUFFER_TOO_SMALL; - - *(ULONG *)info = 0; /* standard heap */ + if (size_in < sizeof(ULONG)) return STATUS_BUFFER_TOO_SMALL; + *(ULONG *)info = heap->compat_info; return STATUS_SUCCESS;
default: - FIXME("Unknown heap information class %u\n", info_class); + FIXME( "HEAP_INFORMATION_CLASS %u not implemented!\n", info_class ); return STATUS_INVALID_INFO_CLASS; } } @@ -2063,8 +2075,34 @@ NTSTATUS WINAPI RtlQueryHeapInformation( HANDLE handle, HEAP_INFORMATION_CLASS i */ NTSTATUS WINAPI RtlSetHeapInformation( HANDLE handle, HEAP_INFORMATION_CLASS info_class, void *info, SIZE_T size ) { - FIXME( "handle %p, info_class %d, info %p, size %Id stub!\n", handle, info_class, info, size ); - return STATUS_SUCCESS; + struct heap *heap; + ULONG flags; + + TRACE( "handle %p, info_class %u, info %p, size %Iu.\n", handle, info_class, info, size ); + + switch (info_class) + { + case HeapCompatibilityInformation: + { + ULONG compat_info; + + if (size < sizeof(ULONG)) return STATUS_BUFFER_TOO_SMALL; + if (!(heap = unsafe_heap_from_handle( handle, 0, &flags ))) return STATUS_INVALID_HANDLE; + + compat_info = *(ULONG *)info; + if (compat_info != HEAP_STD && compat_info != HEAP_LFH) + { + FIXME( "HeapCompatibilityInformation %lu not implemented!\n", compat_info ); + return STATUS_UNSUCCESSFUL; + } + if (InterlockedCompareExchange( &heap->compat_info, compat_info, HEAP_STD )) return STATUS_UNSUCCESSFUL; + return STATUS_SUCCESS; + } + + default: + FIXME( "HEAP_INFORMATION_CLASS %u not implemented!\n", info_class ); + return STATUS_SUCCESS; + } }
/***********************************************************************
From: Rémi Bernon rbernon@codeweavers.com
--- dlls/kernel32/tests/heap.c | 2 -- dlls/ntdll/heap.c | 56 +++++++++++++++++++++++++++++++++++++- 2 files changed, 55 insertions(+), 3 deletions(-)
diff --git a/dlls/kernel32/tests/heap.c b/dlls/kernel32/tests/heap.c index 29faf8c902e..ed84441efe0 100644 --- a/dlls/kernel32/tests/heap.c +++ b/dlls/kernel32/tests/heap.c @@ -904,7 +904,6 @@ static void test_HeapCreate(void)
ret = pHeapQueryInformation( heap, HeapCompatibilityInformation, &compat_info, sizeof(compat_info), &size ); ok( ret, "HeapQueryInformation failed, error %lu\n", GetLastError() ); - todo_wine ok( compat_info == 2, "got HeapCompatibilityInformation %lu\n", compat_info );
ret = HeapDestroy( heap ); @@ -1195,7 +1194,6 @@ static void test_HeapCreate(void)
ret = pHeapQueryInformation( heap, HeapCompatibilityInformation, &compat_info, sizeof(compat_info), &size ); ok( ret, "HeapQueryInformation failed, error %lu\n", GetLastError() ); - todo_wine ok( compat_info == 2, "got HeapCompatibilityInformation %lu\n", compat_info );
/* locking is serialized */ diff --git a/dlls/ntdll/heap.c b/dlls/ntdll/heap.c index e72f65da7ed..9bf845412d4 100644 --- a/dlls/ntdll/heap.c +++ b/dlls/ntdll/heap.c @@ -149,6 +149,27 @@ C_ASSERT( sizeof(ARENA_LARGE) == 4 * BLOCK_ALIGN ); C_ASSERT( sizeof(struct block) <= HEAP_MIN_BLOCK_SIZE ); C_ASSERT( sizeof(struct entry) <= HEAP_MIN_BLOCK_SIZE );
+/* block size categories for LFH */ + +#define BLOCK_SIZE_SMALL_MAX (16 * HEAP_MIN_BLOCK_SIZE) +#define BLOCK_SIZE_MEDIUM_STEP (12 * BLOCK_ALIGN) +#define BLOCK_SIZE_CATEGORY_SMALL(size) (((size) - HEAP_MIN_BLOCK_SIZE) / BLOCK_ALIGN) +#define BLOCK_SIZE_CATEGORY_MEDIUM(size) (BLOCK_SIZE_CATEGORY_SMALL(BLOCK_SIZE_SMALL_MAX) + ((size) - (BLOCK_SIZE_SMALL_MAX)) / BLOCK_SIZE_MEDIUM_STEP) +#define BLOCK_SIZE_CATEGORY(size) (((size) >= BLOCK_SIZE_SMALL_MAX) ? BLOCK_SIZE_CATEGORY_MEDIUM(size) : BLOCK_SIZE_CATEGORY_SMALL(size)) +#define BLOCK_SIZE_CATEGORY_COUNT BLOCK_SIZE_CATEGORY(0x400 * BLOCK_ALIGN) +#define BLOCK_SIZE_MEDIUM_MIN (BLOCK_SIZE_SMALL_MAX + BLOCK_SIZE_MEDIUM_STEP - BLOCK_ALIGN) + +#define BLOCK_CATEGORY_SIZE(category) (category >= BLOCK_SIZE_CATEGORY_MEDIUM(BLOCK_SIZE_SMALL_MAX) \ + ? (BLOCK_SIZE_MEDIUM_MIN + (category - BLOCK_SIZE_CATEGORY_MEDIUM(BLOCK_SIZE_SMALL_MAX)) * BLOCK_SIZE_MEDIUM_STEP) \ + : (HEAP_MIN_BLOCK_SIZE + category * BLOCK_ALIGN)) + +C_ASSERT( BLOCK_SIZE_CATEGORY(HEAP_MIN_BLOCK_SIZE) == 0 ); +C_ASSERT( BLOCK_SIZE_CATEGORY_SMALL(BLOCK_SIZE_SMALL_MAX - 1) == 44 ); +C_ASSERT( BLOCK_SIZE_CATEGORY_MEDIUM(BLOCK_SIZE_SMALL_MAX) == 45 ); +C_ASSERT( BLOCK_SIZE_CATEGORY_COUNT == 126 ); + +/* difference between block classes must fit into block tail_size */ +C_ASSERT( BLOCK_CATEGORY_SIZE(126) - BLOCK_CATEGORY_SIZE(125) <= FIELD_MAX( struct block, tail_size ) ); /* used block size is coded into block_size */ #define HEAP_MAX_USED_BLOCK_SIZE (FIELD_MAX( struct block, block_size ) * BLOCK_ALIGN) /* free block size is coded into block_size + tail_size */ @@ -189,6 +210,12 @@ typedef struct DECLSPEC_ALIGN(BLOCK_ALIGN) tagSUBHEAP C_ASSERT( sizeof(SUBHEAP) == offsetof(SUBHEAP, block) + sizeof(struct block) ); C_ASSERT( sizeof(SUBHEAP) == 4 * BLOCK_ALIGN );
+struct heap_counters +{ + volatile BYTE live; + volatile WORD total; +}; + struct heap { /* win32/win64 */ DWORD_PTR unknown1[2]; /* 0000/0000 */ @@ -201,6 +228,7 @@ struct heap DWORD force_flags; /* 0044/0074 */ /* end of the Windows 10 compatible struct layout */
+ struct heap_counters stats[BLOCK_SIZE_CATEGORY_COUNT]; /* Stats counter for LFH activation */ LONG compat_info; /* HeapCompatibilityInformation / heap frontend type */ struct list entry; /* Entry in process heap list */ struct list subheap_list; /* Sub-heap list */ @@ -776,11 +804,13 @@ static struct block *heap_delay_free( struct heap *heap, ULONG flags, struct blo
static NTSTATUS heap_free_block( struct heap *heap, ULONG flags, struct block *block ) { + SIZE_T block_size = block_get_size( block ), category = BLOCK_SIZE_CATEGORY(block_size); SUBHEAP *subheap = block_get_subheap( heap, block ); - SIZE_T block_size = block_get_size( block ); struct entry *entry; struct block *next;
+ if (category < ARRAY_SIZE(heap->stats) && heap->stats[category].live <= 0x10) heap->stats[category].live--; + if ((next = next_block( subheap, block )) && (block_get_flags( next ) & BLOCK_FLAG_FREE)) { /* merge with next block if it is free */ @@ -1541,6 +1571,28 @@ static SIZE_T heap_get_block_size( const struct heap *heap, ULONG flags, SIZE_T return block_size; }
+static NTSTATUS heap_allocate_block_lfh( struct heap *heap, ULONG flags, SIZE_T block_size, + SIZE_T size, void **ret ) +{ + SIZE_T category = BLOCK_SIZE_CATEGORY( block_size ); + + if (category >= ARRAY_SIZE(heap->stats)) return STATUS_UNSUCCESSFUL; + if (heap->stats[category].live <= 0x10 && heap->stats[category].total <= 0x800) + { + heap->stats[category].live++; + heap->stats[category].total++; + return STATUS_UNSUCCESSFUL; + } + if (heap->compat_info != HEAP_LFH) + { + ULONG info = HEAP_LFH; + RtlSetHeapInformation( heap, HeapCompatibilityInformation, &info, sizeof(info) ); + return STATUS_UNSUCCESSFUL; + } + + return STATUS_NOT_CAPABLE; +} + static NTSTATUS heap_allocate_block( struct heap *heap, ULONG flags, SIZE_T block_size, SIZE_T size, void **ret ) { struct block *block, *next; @@ -1589,6 +1641,8 @@ void *WINAPI DECLSPEC_HOTPATCH RtlAllocateHeap( HANDLE handle, ULONG flags, SIZE status = STATUS_NO_MEMORY; else if (block_size >= HEAP_MIN_LARGE_BLOCK_SIZE) status = heap_allocate_large( heap, heap_flags, block_size, size, &ptr ); + else if (!heap_allocate_block_lfh( heap, heap_flags, block_size, size, &ptr )) + status = STATUS_SUCCESS; else { heap_lock( heap, heap_flags );
From: Rémi Bernon rbernon@codeweavers.com
--- dlls/ntdll/heap.c | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-)
diff --git a/dlls/ntdll/heap.c b/dlls/ntdll/heap.c index 9bf845412d4..528b13c9cf6 100644 --- a/dlls/ntdll/heap.c +++ b/dlls/ntdll/heap.c @@ -510,7 +510,7 @@ static inline struct entry *find_free_list( struct heap *heap, SIZE_T block_size
if (block_size <= HEAP_MAX_SMALL_FREE_LIST) i = (block_size - HEAP_MIN_BLOCK_SIZE) / BLOCK_ALIGN; - else for (i = HEAP_NB_SMALL_FREE_LISTS; i < HEAP_NB_FREE_LISTS - 1; i++) + else for (i = HEAP_NB_SMALL_FREE_LISTS; i < ARRAY_SIZE(heap->free_lists) - 1; i++) if (block_size <= free_list_sizes[i - HEAP_NB_SMALL_FREE_LISTS]) break;
list = heap->free_lists + i; @@ -518,6 +518,15 @@ static inline struct entry *find_free_list( struct heap *heap, SIZE_T block_size return list; }
+static void free_list_init( struct entry *entry ) +{ + list_init( &entry->entry ); + block_set_flags( &entry->block, ~0, BLOCK_FLAG_FREE_LINK ); + block_set_size( &entry->block, 0 ); + block_set_base( &entry->block, &entry ); + block_set_type( &entry->block, BLOCK_TYPE_FREE ); +} + /* get the memory protection type to use for a given heap */ static inline ULONG get_protection_type( DWORD flags ) { @@ -573,10 +582,13 @@ static void heap_dump( const struct heap *heap ) TRACE( " next %p\n", LIST_ENTRY( heap->entry.next, struct heap, entry ) );
TRACE( " free_lists: %p\n", heap->free_lists ); - for (i = 0; i < HEAP_NB_FREE_LISTS; i++) + for (i = 0; i < ARRAY_SIZE(heap->free_lists); i++) + { TRACE( " %p: size %#8Ix, prev %p, next %p\n", heap->free_lists + i, get_free_list_block_size( i ), LIST_ENTRY( heap->free_lists[i].entry.prev, struct entry, entry ), LIST_ENTRY( heap->free_lists[i].entry.next, struct entry, entry ) ); + } +
TRACE( " subheaps: %p\n", &heap->subheap_list ); LIST_FOR_EACH_ENTRY( subheap, &heap->subheap_list, SUBHEAP, entry ) @@ -1050,7 +1062,9 @@ static BOOL is_valid_free_block( const struct heap *heap, const struct block *bl unsigned int i;
if ((subheap = find_subheap( heap, block, FALSE ))) return TRUE; - for (i = 0; i < HEAP_NB_FREE_LISTS; i++) if (block == &heap->free_lists[i].block) return TRUE; + for (i = 0; i < ARRAY_SIZE(heap->free_lists); i++) + if (block == &heap->free_lists[i].block) return TRUE; + return FALSE; }
@@ -1423,13 +1437,9 @@ HANDLE WINAPI RtlCreateHeap( ULONG flags, void *addr, SIZE_T total_size, SIZE_T list_init( &heap->subheap_list ); list_init( &heap->large_list );
- list_init( &heap->free_lists[0].entry ); for (i = 0, entry = heap->free_lists; i < HEAP_NB_FREE_LISTS; i++, entry++) { - block_set_flags( &entry->block, ~0, BLOCK_FLAG_FREE_LINK ); - block_set_size( &entry->block, 0 ); - block_set_type( &entry->block, BLOCK_TYPE_FREE ); - block_set_base( &entry->block, heap ); + free_list_init( entry ); if (i) list_add_after( &entry[-1].entry, &entry->entry ); }
From: Rémi Bernon rbernon@codeweavers.com
--- dlls/ntdll/heap.c | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-)
diff --git a/dlls/ntdll/heap.c b/dlls/ntdll/heap.c index 528b13c9cf6..287b62a884b 100644 --- a/dlls/ntdll/heap.c +++ b/dlls/ntdll/heap.c @@ -757,6 +757,15 @@ static inline BOOL subheap_decommit( const struct heap *heap, SUBHEAP *subheap, return TRUE; }
+static void block_init_used( struct block *block, ULONG flags, SIZE_T size ) +{ + block_set_type( block, BLOCK_TYPE_USED ); + block_set_flags( block, 0, BLOCK_USER_FLAGS( flags ) ); + block->tail_size = block_get_size( block ) - sizeof(*block) - size; + initialize_block( block, 0, size, flags ); + mark_block_tail( block, flags ); +} + static void block_init_free( struct block *block, ULONG flags, SUBHEAP *subheap, SIZE_T block_size ) { const char *end = (char *)block + block_size, *commit_end = subheap_commit_end( subheap ); @@ -1622,11 +1631,7 @@ static NTSTATUS heap_allocate_block( struct heap *heap, ULONG flags, SIZE_T bloc insert_free_block( heap, flags, subheap, next ); }
- block_set_type( block, BLOCK_TYPE_USED ); - block_set_flags( block, ~0, BLOCK_USER_FLAGS( flags ) ); - block->tail_size = block_get_size( block ) - sizeof(*block) - size; - initialize_block( block, 0, size, flags ); - mark_block_tail( block, flags ); + block_init_used( block, flags, size );
if ((next = next_block( subheap, block ))) block_set_flags( next, BLOCK_FLAG_PREV_FREE, 0 );
From: Rémi Bernon rbernon@codeweavers.com
To associate a subheap with a specific thread that will be its only user. Other threads may only defer the block free of subheap with a different affinity, through locking and defer list. --- dlls/ntdll/heap.c | 46 +++++++++++++++++++++++++++++++++++++++------- 1 file changed, 39 insertions(+), 7 deletions(-)
diff --git a/dlls/ntdll/heap.c b/dlls/ntdll/heap.c index 287b62a884b..4904dd87544 100644 --- a/dlls/ntdll/heap.c +++ b/dlls/ntdll/heap.c @@ -198,7 +198,10 @@ static const SIZE_T free_list_sizes[] =
typedef struct DECLSPEC_ALIGN(BLOCK_ALIGN) tagSUBHEAP { - SIZE_T __pad[sizeof(SIZE_T) / sizeof(DWORD)]; +#ifdef _WIN64 + SIZE_T __padding; +#endif + DWORD affinity; SIZE_T block_size; SIZE_T data_size; struct list entry; @@ -356,6 +359,21 @@ static void subheap_set_bounds( SUBHEAP *subheap, char *commit_end, char *end ) subheap->data_size = commit_end - (char *)(subheap + 1); }
+static void subheap_set_affinity( SUBHEAP *subheap, WORD affinity ) +{ + subheap->affinity = affinity; +} + +static inline WORD subheap_get_affinity( const SUBHEAP *subheap ) +{ + return subheap->affinity; +} + +static inline WORD block_get_affinity( const struct heap *heap, const struct block *block ) +{ + return subheap_get_affinity( block_get_subheap( heap, block ) ); +} + static inline void *first_block( const SUBHEAP *subheap ) { return (void *)&subheap->block; @@ -593,11 +611,12 @@ static void heap_dump( const struct heap *heap ) TRACE( " subheaps: %p\n", &heap->subheap_list ); LIST_FOR_EACH_ENTRY( subheap, &heap->subheap_list, SUBHEAP, entry ) { - SIZE_T free_size = 0, used_size = 0, overhead = 0; + SIZE_T free_size = 0, used_size = 0, overhead = 0, affinity = subheap_get_affinity( subheap ); const char *base = subheap_base( subheap );
TRACE( " %p: base %p first %p last %p end %p\n", subheap, base, first_block( subheap ), last_block( subheap ), base + subheap_size( subheap ) ); + if (affinity) TRACE( " affinity %#Ix\n", affinity );
if (!check_subheap( subheap, heap )) return;
@@ -1000,7 +1019,7 @@ static BOOL validate_large_block( const struct heap *heap, const struct block *b }
-static SUBHEAP *create_subheap( struct heap *heap, DWORD flags, SIZE_T total_size, SIZE_T commit_size ) +static SUBHEAP *create_subheap( struct heap *heap, DWORD flags, SIZE_T total_size, SIZE_T commit_size, WORD affinity ) { SIZE_T block_size; SUBHEAP *subheap; @@ -1011,6 +1030,7 @@ static SUBHEAP *create_subheap( struct heap *heap, DWORD flags, SIZE_T total_siz if (!(subheap = allocate_region( heap, flags, &total_size, &commit_size ))) return NULL;
subheap->user_value = heap; + subheap_set_affinity( subheap, affinity ); subheap_set_bounds( subheap, (char *)subheap + commit_size, (char *)subheap + total_size ); block_size = (SIZE_T)ROUND_ADDR( subheap_size( subheap ) - subheap_overhead( subheap ), BLOCK_ALIGN - 1 ); block_init_free( first_block( subheap ), flags, subheap, block_size ); @@ -1048,7 +1068,7 @@ static struct block *find_free_block( struct heap *heap, ULONG flags, SIZE_T blo total_size = sizeof(SUBHEAP) + block_size + sizeof(struct entry); if (total_size < block_size) return NULL; /* overflow */
- if ((subheap = create_subheap( heap, flags, max( heap->grow_size, total_size ), total_size ))) + if ((subheap = create_subheap( heap, flags, max( heap->grow_size, total_size ), total_size, 0 ))) { if (heap->grow_size <= HEAP_MAX_FREE_BLOCK_SIZE / 2) heap->grow_size *= 2; } @@ -1056,7 +1076,7 @@ static struct block *find_free_block( struct heap *heap, ULONG flags, SIZE_T blo { if (heap->grow_size <= total_size || heap->grow_size <= 4 * 1024 * 1024) return NULL; heap->grow_size /= 2; - subheap = create_subheap( heap, flags, max( heap->grow_size, total_size ), total_size ); + subheap = create_subheap( heap, flags, max( heap->grow_size, total_size ), total_size, 0 ); }
TRACE( "created new sub-heap %p of %#Ix bytes for heap %p\n", subheap, subheap_size( subheap ), heap ); @@ -1209,12 +1229,19 @@ static BOOL heap_validate_ptr( const struct heap *heap, const void *ptr )
static BOOL heap_validate( const struct heap *heap ) { + DWORD affinity = NtCurrentTeb()->HeapVirtualAffinity, block_affinity; const ARENA_LARGE *large_arena; const struct block *block; const SUBHEAP *subheap;
LIST_FOR_EACH_ENTRY( subheap, &heap->subheap_list, SUBHEAP, entry ) { + if ((block_affinity = subheap_get_affinity( subheap )) && block_affinity != affinity) + { + TRACE( "Skipping subheap %p with affinity %#lx\n", subheap, block_affinity ); + continue; + } + if (!check_subheap( subheap, heap )) { ERR( "heap %p, subheap %p corrupted sizes or user_value\n", heap, subheap ); @@ -1470,6 +1497,7 @@ HANDLE WINAPI RtlCreateHeap( ULONG flags, void *addr, SIZE_T total_size, SIZE_T
subheap = &heap->subheap; subheap->user_value = heap; + subheap_set_affinity( subheap, 0 ); subheap_set_bounds( subheap, (char *)heap + commit_size, (char *)heap + total_size ); block_size = (SIZE_T)ROUND_ADDR( subheap_size( subheap ) - subheap_overhead( subheap ), BLOCK_ALIGN - 1 ); block_init_free( first_block( subheap ), flags, subheap, block_size ); @@ -1980,7 +2008,8 @@ static NTSTATUS heap_walk_blocks( const struct heap *heap, const SUBHEAP *subhea entry->cbData = commit_end - (char *)entry->lpData - 4 * BLOCK_ALIGN; entry->cbOverhead = 2 * BLOCK_ALIGN; entry->iRegionIndex = 0; - entry->wFlags = 0; + if (subheap_get_affinity( subheap )) entry->wFlags = RTL_HEAP_ENTRY_LFH; + else entry->wFlags = 0; } else { @@ -1988,7 +2017,9 @@ static NTSTATUS heap_walk_blocks( const struct heap *heap, const SUBHEAP *subhea entry->cbData = block_get_size( block ) - block_get_overhead( block ); entry->cbOverhead = block_get_overhead( block ); entry->iRegionIndex = 0; - entry->wFlags = RTL_HEAP_ENTRY_COMMITTED|RTL_HEAP_ENTRY_BLOCK|RTL_HEAP_ENTRY_BUSY; + entry->wFlags = RTL_HEAP_ENTRY_BUSY; + if (subheap_get_affinity( subheap )) entry->wFlags |= RTL_HEAP_ENTRY_LFH; + else entry->wFlags |= RTL_HEAP_ENTRY_COMMITTED|RTL_HEAP_ENTRY_BLOCK; }
return STATUS_SUCCESS; @@ -2034,6 +2065,7 @@ static NTSTATUS heap_walk( const struct heap *heap, struct rtl_heap_entry *entry entry->cbOverhead = 0; entry->iRegionIndex = 0; entry->wFlags = RTL_HEAP_ENTRY_REGION; + if (subheap_get_affinity( subheap )) entry->wFlags |= RTL_HEAP_ENTRY_LFH; entry->Region.dwCommittedSize = (char *)subheap_commit_end( subheap ) - base; entry->Region.dwUnCommittedSize = subheap_size( subheap ) - entry->Region.dwCommittedSize; entry->Region.lpFirstBlock = base + entry->cbData;
From: Rémi Bernon rbernon@codeweavers.com
To avoid coalescing free LFH blocks until their owning thread is detached. --- dlls/ntdll/heap.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-)
diff --git a/dlls/ntdll/heap.c b/dlls/ntdll/heap.c index 4904dd87544..b2f71316492 100644 --- a/dlls/ntdll/heap.c +++ b/dlls/ntdll/heap.c @@ -101,6 +101,7 @@ C_ASSERT( sizeof(struct block) == 8 ); #define BLOCK_FLAG_PREV_FREE 0x02 #define BLOCK_FLAG_FREE_LINK 0x03 #define BLOCK_FLAG_LARGE 0x04 +#define BLOCK_FLAG_SPLIT 0x08 /* block is split and must not be coalesced */ #define BLOCK_FLAG_USER_INFO 0x10 /* user flags up to 0xf0 */ #define BLOCK_FLAG_USER_MASK 0xf0
@@ -779,7 +780,7 @@ static inline BOOL subheap_decommit( const struct heap *heap, SUBHEAP *subheap, static void block_init_used( struct block *block, ULONG flags, SIZE_T size ) { block_set_type( block, BLOCK_TYPE_USED ); - block_set_flags( block, 0, BLOCK_USER_FLAGS( flags ) ); + block_set_flags( block, ~BLOCK_FLAG_SPLIT, BLOCK_USER_FLAGS( flags ) ); block->tail_size = block_get_size( block ) - sizeof(*block) - size; initialize_block( block, 0, size, flags ); mark_block_tail( block, flags ); @@ -851,7 +852,8 @@ static NTSTATUS heap_free_block( struct heap *heap, ULONG flags, struct block *b
if (category < ARRAY_SIZE(heap->stats) && heap->stats[category].live <= 0x10) heap->stats[category].live--;
- if ((next = next_block( subheap, block )) && (block_get_flags( next ) & BLOCK_FLAG_FREE)) + if ((next = next_block( subheap, block )) && (block_get_flags( next ) & BLOCK_FLAG_FREE) && + !(block_get_flags( next ) & BLOCK_FLAG_SPLIT)) { /* merge with next block if it is free */ entry = (struct entry *)next; @@ -1120,7 +1122,7 @@ static BOOL validate_free_block( const struct heap *heap, const SUBHEAP *subheap err = "invalid previous free block pointer"; else if (!(block_get_flags( prev ) & BLOCK_FLAG_FREE) || block_get_type( prev ) != BLOCK_TYPE_FREE) err = "invalid previous free block header"; - else if ((next = next_block( subheap, block ))) + else if ((next = next_block( subheap, block )) && !(block_get_flags( block ) & BLOCK_FLAG_SPLIT)) { if (!(block_get_flags( next ) & BLOCK_FLAG_PREV_FREE)) err = "invalid next block flags";
From: Rémi Bernon rbernon@codeweavers.com
--- dlls/ntdll/heap.c | 112 ++++++++++++++++++++++++++++++++++++++-- dlls/ntdll/loader.c | 2 + dlls/ntdll/ntdll_misc.h | 1 + 3 files changed, 112 insertions(+), 3 deletions(-)
diff --git a/dlls/ntdll/heap.c b/dlls/ntdll/heap.c index b2f71316492..73df7dc3526 100644 --- a/dlls/ntdll/heap.c +++ b/dlls/ntdll/heap.c @@ -220,6 +220,17 @@ struct heap_counters volatile WORD total; };
+struct heap_thread_data +{ + struct entry free_lists[BLOCK_SIZE_CATEGORY_COUNT]; + /* requires heap lock to access */ + struct entry defer_list; + volatile ULONG defer_count; +}; + +/* affinity to tid mapping array, defines the amount of supported concurrent threads */ +static LONG thread_data_tid[64]; + struct heap { /* win32/win64 */ DWORD_PTR unknown1[2]; /* 0000/0000 */ @@ -232,6 +243,7 @@ struct heap DWORD force_flags; /* 0044/0074 */ /* end of the Windows 10 compatible struct layout */
+ struct heap_thread_data *tls; /* Thread local storage data */ struct heap_counters stats[BLOCK_SIZE_CATEGORY_COUNT]; /* Stats counter for LFH activation */ LONG compat_info; /* HeapCompatibilityInformation / heap frontend type */ struct list entry; /* Entry in process heap list */ @@ -595,7 +607,7 @@ static void heap_dump( const struct heap *heap ) const struct block *block; const ARENA_LARGE *large; const SUBHEAP *subheap; - unsigned int i; + unsigned int i, j;
TRACE( "heap: %p\n", heap ); TRACE( " next %p\n", LIST_ENTRY( heap->entry.next, struct heap, entry ) ); @@ -608,6 +620,24 @@ static void heap_dump( const struct heap *heap ) LIST_ENTRY( heap->free_lists[i].entry.next, struct entry, entry ) ); }
+ if (heap->tls) for (i = 0; i < ARRAY_SIZE(thread_data_tid); ++i) + { + struct heap_thread_data *thread_data = heap->tls + i; + if (!thread_data_tid[i]) continue; + TRACE( " index %#x thread %04lx:\n", i, thread_data_tid[i] ); + + for (j = 0; j < ARRAY_SIZE(thread_data->free_lists); j++) + { + if (list_empty( &thread_data->free_lists[j].entry )) continue; + TRACE( " %p: size %8Ix, prev %p, next %p\n", thread_data->free_lists + j, HEAP_MIN_BLOCK_SIZE + j * BLOCK_ALIGN, + LIST_ENTRY( thread_data->free_lists[j].entry.prev, struct entry, entry ), + LIST_ENTRY( thread_data->free_lists[j].entry.next, struct entry, entry ) ); + } + + TRACE( " %p: pending, prev %p, next %p\n", &thread_data->defer_list, + LIST_ENTRY( thread_data->defer_list.entry.prev, struct entry, entry ), + LIST_ENTRY( thread_data->defer_list.entry.next, struct entry, entry ) ); + }
TRACE( " subheaps: %p\n", &heap->subheap_list ); LIST_FOR_EACH_ENTRY( subheap, &heap->subheap_list, SUBHEAP, entry ) @@ -617,7 +647,7 @@ static void heap_dump( const struct heap *heap )
TRACE( " %p: base %p first %p last %p end %p\n", subheap, base, first_block( subheap ), last_block( subheap ), base + subheap_size( subheap ) ); - if (affinity) TRACE( " affinity %#Ix\n", affinity ); + if (affinity) TRACE( " affinity %#Ix, thread %04lx\n", affinity, thread_data_tid[affinity - 1] );
if (!check_subheap( subheap, heap )) return;
@@ -1021,6 +1051,65 @@ static BOOL validate_large_block( const struct heap *heap, const struct block *b }
+static struct heap_thread_data *alloc_thread_data_block( struct heap *heap ) +{ + struct heap_thread_data *thread_data_block = NULL; + SIZE_T size, i, j; + + size = sizeof(*thread_data_block) * ARRAY_SIZE(thread_data_tid); + if (NtAllocateVirtualMemory( NtCurrentProcess(), (void *)&thread_data_block, 0, &size, + MEM_COMMIT, PAGE_READWRITE )) return NULL; + + for (i = 0; i < ARRAY_SIZE(thread_data_tid); ++i) + { + struct heap_thread_data *thread_data = thread_data_block + i; + for (j = 0; j < ARRAY_SIZE(thread_data->free_lists); ++j) + free_list_init( &thread_data->free_lists[j] ); + free_list_init( &thread_data->defer_list ); + } + + return thread_data_block; +} + +static DWORD next_thread_data_index(void) +{ + LONG index, tid = GetCurrentThreadId(); + + for (index = ARRAY_SIZE(thread_data_tid) - 1; index >= 0; index--) + if (!InterlockedCompareExchange( thread_data_tid + index, tid, 0 )) + break; + + NtCurrentTeb()->HeapVirtualAffinity = index + 1; + return index; +} + +void heap_thread_detach(void) +{ + DWORD index = NtCurrentTeb()->HeapVirtualAffinity - 1; + if (index >= ARRAY_SIZE(thread_data_tid)) return; + InterlockedExchange( thread_data_tid + index, 0 ); +} + +static struct heap_thread_data *heap_get_thread_data( struct heap *heap ) +{ + DWORD index = NtCurrentTeb()->HeapVirtualAffinity - 1; + + if (!process_heap || heap->compat_info != HEAP_LFH) return NULL; + if (index >= ARRAY_SIZE(thread_data_tid)) index = next_thread_data_index(); + if (index >= ARRAY_SIZE(thread_data_tid)) return NULL; + + if (!heap->tls) + { + RtlLockHeap( heap ); + if (!heap->tls) heap->tls = alloc_thread_data_block( heap ); + RtlUnlockHeap( heap ); + } + + if (!heap->tls) return NULL; + return heap->tls + index; +} + + static SUBHEAP *create_subheap( struct heap *heap, DWORD flags, SIZE_T total_size, SIZE_T commit_size, WORD affinity ) { SIZE_T block_size; @@ -1090,11 +1179,20 @@ static struct block *find_free_block( struct heap *heap, ULONG flags, SIZE_T blo static BOOL is_valid_free_block( const struct heap *heap, const struct block *block ) { const SUBHEAP *subheap; - unsigned int i; + unsigned int i, j;
if ((subheap = find_subheap( heap, block, FALSE ))) return TRUE; for (i = 0; i < ARRAY_SIZE(heap->free_lists); i++) if (block == &heap->free_lists[i].block) return TRUE; + if (!heap->tls) return FALSE; + + for (i = 0; i < ARRAY_SIZE(thread_data_tid); ++i) + { + struct heap_thread_data *thread_data = heap->tls + i; + if (block == &thread_data->defer_list.block) return TRUE; + for (j = 0; j < ARRAY_SIZE(thread_data->free_lists); ++j) + if (block == &thread_data->free_lists[j].block) return TRUE; + }
return FALSE; } @@ -1594,6 +1692,12 @@ HANDLE WINAPI RtlDestroyHeap( HANDLE handle ) NtFreeVirtualMemory( NtCurrentProcess(), &addr, &size, MEM_RELEASE ); } valgrind_notify_free_all( &heap->subheap, heap ); + if (heap->tls) + { + size = 0; + addr = heap->tls; + NtFreeVirtualMemory( NtCurrentProcess(), &addr, &size, MEM_RELEASE ); + } size = 0; addr = heap; NtFreeVirtualMemory( NtCurrentProcess(), &addr, &size, MEM_RELEASE ); @@ -1624,6 +1728,7 @@ static NTSTATUS heap_allocate_block_lfh( struct heap *heap, ULONG flags, SIZE_T SIZE_T size, void **ret ) { SIZE_T category = BLOCK_SIZE_CATEGORY( block_size ); + struct heap_thread_data *thread_data;
if (category >= ARRAY_SIZE(heap->stats)) return STATUS_UNSUCCESSFUL; if (heap->stats[category].live <= 0x10 && heap->stats[category].total <= 0x800) @@ -1638,6 +1743,7 @@ static NTSTATUS heap_allocate_block_lfh( struct heap *heap, ULONG flags, SIZE_T RtlSetHeapInformation( heap, HeapCompatibilityInformation, &info, sizeof(info) ); return STATUS_UNSUCCESSFUL; } + if (!(thread_data = heap_get_thread_data( heap ))) return STATUS_UNSUCCESSFUL;
return STATUS_NOT_CAPABLE; } diff --git a/dlls/ntdll/loader.c b/dlls/ntdll/loader.c index 01a30742678..6c643575f8d 100644 --- a/dlls/ntdll/loader.c +++ b/dlls/ntdll/loader.c @@ -3737,6 +3737,8 @@ void WINAPI LdrShutdownThread(void) /* don't call DbgUiGetThreadDebugObject as some apps hook it and terminate if called */ if (NtCurrentTeb()->DbgSsReserved[1]) NtClose( NtCurrentTeb()->DbgSsReserved[1] ); RtlFreeThreadActivationContextStack(); + + heap_thread_detach(); }
diff --git a/dlls/ntdll/ntdll_misc.h b/dlls/ntdll/ntdll_misc.h index d1a7790991b..f6b77b79cde 100644 --- a/dlls/ntdll/ntdll_misc.h +++ b/dlls/ntdll/ntdll_misc.h @@ -127,5 +127,6 @@ static inline void ascii_to_unicode( WCHAR *dst, const char *src, size_t len )
/* FLS data */ extern TEB_FLS_DATA *fls_alloc_data(void) DECLSPEC_HIDDEN; +extern void heap_thread_detach(void) DECLSPEC_HIDDEN;
#endif
From: Rémi Bernon rbernon@codeweavers.com
--- dlls/kernel32/tests/heap.c | 29 +---- dlls/ntdll/heap.c | 261 ++++++++++++++++++++++++++++++++++++- 2 files changed, 263 insertions(+), 27 deletions(-)
diff --git a/dlls/kernel32/tests/heap.c b/dlls/kernel32/tests/heap.c index ed84441efe0..0c95b08780e 100644 --- a/dlls/kernel32/tests/heap.c +++ b/dlls/kernel32/tests/heap.c @@ -452,9 +452,7 @@ static void test_HeapCreate(void)
SetLastError( 0xdeadbeef ); ptr1 = HeapAlloc( heap, 0, alloc_size - (0x200 + 0x80 * sizeof(void *)) ); - todo_wine ok( !ptr1, "HeapAlloc succeeded\n" ); - todo_wine ok( GetLastError() == ERROR_NOT_ENOUGH_MEMORY, "got error %lu\n", GetLastError() ); ret = HeapFree( heap, 0, ptr1 ); ok( ret, "HeapFree failed, error %lu\n", GetLastError() ); @@ -953,7 +951,6 @@ static void test_HeapCreate(void) SetLastError( 0xdeadbeef ); while ((ret = HeapWalk( heap, &entry ))) entries[count++] = entry; ok( GetLastError() == ERROR_NO_MORE_ITEMS, "got error %lu\n", GetLastError() ); - todo_wine ok( count > 24, "got count %lu\n", count ); if (count < 2) count = 2;
@@ -966,7 +963,6 @@ static void test_HeapCreate(void)
for (i = 0; i < 0x12; i++) { - todo_wine ok( entries[4 + i].wFlags == 0, "got wFlags %#x\n", entries[4 + i].wFlags ); todo_wine ok( entries[4 + i].cbData == 0x20, "got cbData %#lx\n", entries[4 + i].cbData ); @@ -985,7 +981,6 @@ static void test_HeapCreate(void) rtl_entry.lpData = NULL; SetLastError( 0xdeadbeef ); while (!RtlWalkHeap( heap, &rtl_entry )) rtl_entries[count++] = rtl_entry; - todo_wine ok( count > 24, "got count %lu\n", count ); if (count < 2) count = 2;
@@ -1028,7 +1023,6 @@ static void test_HeapCreate(void) SetLastError( 0xdeadbeef ); while ((ret = HeapWalk( heap, &entry ))) entries[count++] = entry; ok( GetLastError() == ERROR_NO_MORE_ITEMS, "got error %lu\n", GetLastError() ); - todo_wine ok( count > 24, "got count %lu\n", count ); if (count < 2) count = 2;
@@ -1057,7 +1051,6 @@ static void test_HeapCreate(void) rtl_entry.lpData = NULL; SetLastError( 0xdeadbeef ); while (!RtlWalkHeap( heap, &rtl_entry )) rtl_entries[count++] = rtl_entry; - todo_wine ok( count > 24, "got count %lu\n", count ); if (count < 2) count = 2;
@@ -1071,11 +1064,8 @@ static void test_HeapCreate(void) if (!entries[i].wFlags) ok( rtl_entries[i].wFlags == 0 || rtl_entries[i].wFlags == RTL_HEAP_ENTRY_LFH, "got wFlags %#x\n", rtl_entries[i].wFlags ); else if (entries[i].wFlags & PROCESS_HEAP_ENTRY_BUSY) - { - todo_wine ok( rtl_entries[i].wFlags == (RTL_HEAP_ENTRY_LFH|RTL_HEAP_ENTRY_BUSY) || broken(rtl_entries[i].wFlags == 1) /* win7 */, "got wFlags %#x\n", rtl_entries[i].wFlags ); - } else if (entries[i].wFlags & PROCESS_HEAP_UNCOMMITTED_RANGE) ok( rtl_entries[i].wFlags == RTL_HEAP_ENTRY_UNCOMMITTED || broken(rtl_entries[i].wFlags == 0x100) /* win7 */, "got wFlags %#x\n", rtl_entries[i].wFlags ); @@ -1220,7 +1210,6 @@ static void test_HeapCreate(void) thread_params.flags = 0; SetEvent( thread_params.start_event ); res = WaitForSingleObject( thread_params.ready_event, 100 ); - todo_wine ok( !res, "WaitForSingleObject returned %#lx, error %lu\n", res, GetLastError() ); ret = HeapUnlock( heap ); ok( ret, "HeapUnlock failed, error %lu\n", GetLastError() ); @@ -1667,9 +1656,7 @@ static void test_GlobalAlloc(void) ok( size == small_size, "GlobalSize returned %Iu\n", size ); SetLastError( 0xdeadbeef ); tmp_mem = GlobalReAlloc( mem, small_size, 0 ); - todo_wine ok( !tmp_mem, "GlobalReAlloc succeeded\n" ); - todo_wine ok( GetLastError() == ERROR_NOT_ENOUGH_MEMORY, "got error %lu\n", GetLastError() ); if (tmp_mem) mem = tmp_mem; tmp_mem = GlobalReAlloc( mem, 1024 * 1024, GMEM_MODIFY ); @@ -1685,7 +1672,6 @@ static void test_GlobalAlloc(void) ok( !!mem, "GlobalAlloc failed, error %lu\n", GetLastError() ); tmp_mem = GlobalReAlloc( mem, small_size + 1, GMEM_MOVEABLE ); ok( !!tmp_mem, "GlobalReAlloc failed, error %lu\n", GetLastError() ); - todo_wine ok( tmp_mem != mem, "GlobalReAlloc didn't relocate memory\n" ); ptr = GlobalLock( tmp_mem ); ok( !!ptr, "GlobalLock failed, error %lu\n", GetLastError() ); @@ -1832,8 +1818,8 @@ static void test_GlobalAlloc(void) { ok( !is_mem_entry( tmp_mem ), "unexpected moveable %p\n", tmp_mem ); if (flags == GMEM_MODIFY) ok( tmp_mem == mem, "GlobalReAlloc returned %p\n", tmp_mem ); - else if (flags != GMEM_MOVEABLE) todo_wine_if(!flags) ok( !tmp_mem, "GlobalReAlloc succeeded\n" ); - else todo_wine ok( tmp_mem != mem, "GlobalReAlloc returned %p\n", tmp_mem ); + else if (flags != GMEM_MOVEABLE) ok( !tmp_mem, "GlobalReAlloc succeeded\n" ); + else ok( tmp_mem != mem, "GlobalReAlloc returned %p\n", tmp_mem ); } else { @@ -1847,7 +1833,7 @@ static void test_GlobalAlloc(void)
size = GlobalSize( mem ); if (flags == GMEM_MOVEABLE) ok( size == 0 || broken( size == 1 ) /* w7 */, "GlobalSize returned %Iu\n", size ); - else todo_wine_if(!flags) ok( size == small_size, "GlobalSize returned %Iu\n", size ); + else ok( size == small_size, "GlobalSize returned %Iu\n", size );
mem = GlobalFree( mem ); ok( !mem, "GlobalFree failed, error %lu\n", GetLastError() ); @@ -2405,9 +2391,7 @@ static void test_LocalAlloc(void) ok( size == small_size, "LocalSize returned %Iu\n", size ); SetLastError( 0xdeadbeef ); tmp_mem = LocalReAlloc( mem, small_size, 0 ); - todo_wine ok( !tmp_mem, "LocalReAlloc succeeded\n" ); - todo_wine ok( GetLastError() == ERROR_NOT_ENOUGH_MEMORY, "got error %lu\n", GetLastError() ); if (tmp_mem) mem = tmp_mem; tmp_mem = LocalReAlloc( mem, 1024 * 1024, LMEM_MODIFY ); @@ -2423,7 +2407,6 @@ static void test_LocalAlloc(void) ok( !!mem, "LocalAlloc failed, error %lu\n", GetLastError() ); tmp_mem = LocalReAlloc( mem, small_size + 1, LMEM_MOVEABLE ); ok( !!tmp_mem, "LocalReAlloc failed, error %lu\n", GetLastError() ); - todo_wine ok( tmp_mem != mem, "LocalReAlloc didn't relocate memory\n" ); ptr = LocalLock( tmp_mem ); ok( !!ptr, "LocalLock failed, error %lu\n", GetLastError() ); @@ -2516,13 +2499,13 @@ static void test_LocalAlloc(void) tmp_mem = LocalReAlloc( mem, 0, flags ); ok( !is_mem_entry( tmp_mem ), "unexpected moveable %p\n", tmp_mem ); if (flags & LMEM_MODIFY) ok( tmp_mem == mem, "LocalReAlloc returned %p\n", tmp_mem ); - else if (flags != LMEM_MOVEABLE) todo_wine_if(!flags) ok( !tmp_mem, "LocalReAlloc succeeded\n" ); - else todo_wine ok( tmp_mem != mem, "LocalReAlloc returned %p\n", tmp_mem ); + else if (flags != LMEM_MOVEABLE) ok( !tmp_mem, "LocalReAlloc succeeded\n" ); + else ok( tmp_mem != mem, "LocalReAlloc returned %p\n", tmp_mem ); if (tmp_mem) mem = tmp_mem;
size = LocalSize( mem ); if (flags == LMEM_MOVEABLE) ok( size == 0 || broken( size == 1 ) /* w7 */, "LocalSize returned %Iu\n", size ); - else todo_wine_if(!flags) ok( size == small_size, "LocalSize returned %Iu\n", size ); + else ok( size == small_size, "LocalSize returned %Iu\n", size );
mem = LocalFree( mem ); ok( !mem, "LocalFree failed, error %lu\n", GetLastError() ); diff --git a/dlls/ntdll/heap.c b/dlls/ntdll/heap.c index 73df7dc3526..0916dc82a76 100644 --- a/dlls/ntdll/heap.c +++ b/dlls/ntdll/heap.c @@ -202,7 +202,8 @@ typedef struct DECLSPEC_ALIGN(BLOCK_ALIGN) tagSUBHEAP #ifdef _WIN64 SIZE_T __padding; #endif - DWORD affinity; + WORD affinity; + WORD user_count; SIZE_T block_size; SIZE_T data_size; struct list entry; @@ -228,7 +229,9 @@ struct heap_thread_data volatile ULONG defer_count; };
-/* affinity to tid mapping array, defines the amount of supported concurrent threads */ +/* affinity to tid mapping array, defines the amount of + * supported concurrent threads for threaded heaps. + */ static LONG thread_data_tid[64];
struct heap @@ -1083,10 +1086,76 @@ static DWORD next_thread_data_index(void) return index; }
+/* release thread blocks to the main heap free lists, clearing the split flags */ +static void heap_release_blocks_lfh( struct heap *heap, SUBHEAP *subheap ) +{ + struct block *block, *next; + + for (block = first_block( subheap ); block; block = next) + { + /* remove the flag one block at a time, so heap_free_block only merges with the previous */ + block_set_flags( block, BLOCK_FLAG_SPLIT, 0 ); + if ((next = next_block( subheap, block ))) + { + if (block_get_size( next ) >= HEAP_MIN_BLOCK_SIZE) + block_set_flags( next, 0, BLOCK_FLAG_SPLIT ); + else + { + block_set_flags( next, BLOCK_FLAG_SPLIT, 0 ); + next = NULL; + } + } + + if (block_get_flags( block ) & BLOCK_FLAG_FREE) + { + struct entry *entry = CONTAINING_RECORD( block, struct entry, block ); + list_remove( &entry->entry ); + heap_free_block( heap, heap->flags, block ); + } + } +} + +static void flush_defer_list( struct heap *heap, ULONG flags, struct heap_thread_data *thread_data ); +static struct heap_thread_data *heap_get_thread_data( struct heap *heap ); + +/* release subheaps owned by current thread, clearing the subheap affinity */ +static void heap_release_subheaps_lfh( struct heap *heap ) +{ + struct heap_thread_data *thread_data; + SUBHEAP *subheap, *next; + + if ((thread_data = heap_get_thread_data( heap ))) flush_defer_list( heap, heap->flags, thread_data ); + + LIST_FOR_EACH_ENTRY_SAFE( subheap, next, &heap->subheap_list, SUBHEAP, entry ) + { + if (subheap_get_affinity( subheap ) == NtCurrentTeb()->HeapVirtualAffinity) + { + subheap_set_affinity( subheap, 0 ); + heap_release_blocks_lfh( heap, subheap ); + } + } +} + void heap_thread_detach(void) { + struct heap *heap; + DWORD index = NtCurrentTeb()->HeapVirtualAffinity - 1; if (index >= ARRAY_SIZE(thread_data_tid)) return; + + heap_lock( process_heap, 0 ); + + LIST_FOR_EACH_ENTRY( heap, &process_heap->entry, struct heap, entry ) + { + heap_lock( heap, 0 ); + heap_release_subheaps_lfh( heap ); + heap_unlock( heap, 0 ); + } + + heap_release_subheaps_lfh( process_heap ); + + heap_unlock( process_heap, 0 ); + InterlockedExchange( thread_data_tid + index, 0 ); }
@@ -1126,7 +1195,9 @@ static SUBHEAP *create_subheap( struct heap *heap, DWORD flags, SIZE_T total_siz block_size = (SIZE_T)ROUND_ADDR( subheap_size( subheap ) - subheap_overhead( subheap ), BLOCK_ALIGN - 1 ); block_init_free( first_block( subheap ), flags, subheap, block_size );
- list_add_head( &heap->subheap_list, &subheap->entry ); + heap_lock( heap, 0 ); + list_add_tail( &heap->subheap_list, &subheap->entry ); + heap_unlock( heap, 0 );
return subheap; } @@ -1724,11 +1795,129 @@ static SIZE_T heap_get_block_size( const struct heap *heap, ULONG flags, SIZE_T return block_size; }
+static void insert_free_block_lfh( struct heap *heap, SUBHEAP *subheap, struct entry *free_list, struct entry *entry ) +{ + list_add_head( &free_list->entry, &entry->entry ); + + if (!--subheap->user_count) + { + void *addr = subheap_base( subheap ); + struct block *block; + SIZE_T size = 0; + + for (block = first_block( subheap ); block; block = next_block( subheap, block )) + { + entry = (struct entry *)block; + if (block_get_size( block ) >= sizeof(*entry)) list_remove( &entry->entry ); + } + + heap_lock( heap, 0 ); + list_remove( &subheap->entry ); + heap_unlock( heap, 0 ); + + NtFreeVirtualMemory( NtCurrentProcess(), &addr, &size, MEM_RELEASE ); + } +} + +static void flush_defer_list( struct heap *heap, ULONG flags, struct heap_thread_data *thread_data ) +{ + struct list pending = LIST_INIT( pending ); + struct entry *entry, *next; + + heap_lock( heap, flags ); + list_move_tail( &pending, &thread_data->defer_list.entry ); + thread_data->defer_count = 0; + heap_unlock( heap, flags ); + + LIST_FOR_EACH_ENTRY_SAFE( entry, next, &pending, struct entry, entry ) + { + SIZE_T block_size = block_get_size( &entry->block ), category = BLOCK_SIZE_CATEGORY( block_size ); + SUBHEAP *subheap = block_get_subheap( heap, &entry->block ); + + /* remove the block from the pending list */ + list_remove( &entry->entry ); + + block_init_free( &entry->block, flags, subheap, BLOCK_CATEGORY_SIZE(category) ); + block_set_flags( &entry->block, 0, BLOCK_FLAG_SPLIT ); + insert_free_block_lfh( heap, subheap, thread_data->free_lists + category, entry ); + } +} + +static void split_block_lfh( struct heap *heap, ULONG flags, struct block *block, SIZE_T block_size, struct entry *free_list ) +{ + SIZE_T i, old_block_size = block_get_size( block ), count = min( 16, old_block_size / block_size ); + SUBHEAP *subheap = block_get_subheap( heap, block ); + struct entry *entry; + + if (!subheap_commit( heap, subheap, block, count * block_size )) return; + + for (i = 0; i < count; ++i) + { + block_init_free( block, flags, subheap, block_size ); + block_set_flags( block, 0, BLOCK_FLAG_SPLIT ); + old_block_size -= block_size; + + entry = (struct entry *)block; + if (i) list_add_head( &free_list->entry, &entry->entry ); + + block = next_block( subheap, block ); + } + + if (block) block_init_free( block, flags, subheap, old_block_size ); + if ((entry = (struct entry *)block) && old_block_size >= sizeof(*entry)) + { + /* only if large enough to carve more blocks from it */ + if (old_block_size < block_size) list_init( &entry->entry ); + else list_add_tail( &free_list->entry, &entry->entry ); + } + + /* we've split the region entirely, we can consider releasing it */ + if (old_block_size < block_size || !block) subheap->user_count--; +} + +static struct block *find_free_block_lfh( struct heap *heap, ULONG flags, SIZE_T category, + struct heap_thread_data *thread_data ) +{ + SIZE_T block_size = BLOCK_CATEGORY_SIZE(category), commit_size, total_size; + WORD affinity = NtCurrentTeb()->HeapVirtualAffinity; + struct entry *entry, *free_list; + struct list *ptr; + SUBHEAP *subheap; + + free_list = thread_data->free_lists + category; + commit_size = ROUND_SIZE( sizeof(*subheap) + 32 * block_size + sizeof(*entry), REGION_ALIGN - 1 ); + total_size = ROUND_SIZE( sizeof(*subheap) + 64 * block_size + sizeof(*entry), REGION_ALIGN - 1 ); + + if ((ptr = list_head( &free_list->entry ))) + { + entry = LIST_ENTRY( ptr, struct entry, entry ); + list_remove( &entry->entry ); + subheap = block_get_subheap( heap, &entry->block ); + } + else if (!(flags & HEAP_GROWABLE)) + return NULL; + else if (!(subheap = create_subheap( heap, flags, commit_size, total_size, affinity ))) + return NULL; + else + { + TRACE( "created new sub-heap %p of %#Ix bytes for heap %p\n", subheap, subheap_size( subheap ), heap ); + subheap->user_count = 1; /* avoid releasing the region until we've split it entirely */ + entry = first_block( subheap ); + } + + if (!(block_get_flags( &entry->block ) & BLOCK_FLAG_SPLIT)) + split_block_lfh( heap, flags, &entry->block, block_size, free_list ); + + subheap->user_count++; + return &entry->block; +} + static NTSTATUS heap_allocate_block_lfh( struct heap *heap, ULONG flags, SIZE_T block_size, SIZE_T size, void **ret ) { SIZE_T category = BLOCK_SIZE_CATEGORY( block_size ); struct heap_thread_data *thread_data; + struct block *block;
if (category >= ARRAY_SIZE(heap->stats)) return STATUS_UNSUCCESSFUL; if (heap->stats[category].live <= 0x10 && heap->stats[category].total <= 0x800) @@ -1745,7 +1934,12 @@ static NTSTATUS heap_allocate_block_lfh( struct heap *heap, ULONG flags, SIZE_T } if (!(thread_data = heap_get_thread_data( heap ))) return STATUS_UNSUCCESSFUL;
- return STATUS_NOT_CAPABLE; + if (thread_data->defer_count) flush_defer_list( heap, flags, thread_data ); + if (!(block = find_free_block_lfh( heap, flags, category, thread_data ))) return STATUS_NO_MEMORY; + block_init_used( block, flags, size ); + + *ret = block + 1; + return STATUS_SUCCESS; }
static NTSTATUS heap_allocate_block( struct heap *heap, ULONG flags, SIZE_T block_size, SIZE_T size, void **ret ) @@ -1808,6 +2002,43 @@ void *WINAPI DECLSPEC_HOTPATCH RtlAllocateHeap( HANDLE handle, ULONG flags, SIZE return ptr; }
+static NTSTATUS heap_free_block_lfh( struct heap *heap, ULONG flags, struct block *block, WORD block_affinity ) +{ + SIZE_T block_size = block_get_size( block ), category = BLOCK_SIZE_CATEGORY( block_size ); + ULONG affinity = NtCurrentTeb()->HeapVirtualAffinity; + SUBHEAP *subheap = block_get_subheap( heap, block ); + struct entry *entry = (struct entry *)block; + + if (block_affinity == affinity) + { + /* the block is owned by the current thread, clear add it to the corresponding freelist */ + struct heap_thread_data *thread_data = heap_get_thread_data( heap ); + block_init_free( block, flags, subheap, BLOCK_CATEGORY_SIZE(category) ); + block_set_flags( block, 0, BLOCK_FLAG_SPLIT ); + insert_free_block_lfh( heap, subheap, thread_data->free_lists + category, entry ); + } + else + { + struct heap_thread_data *thread_data = &heap->tls[block_affinity - 1]; + heap_lock( heap, flags ); + + /* the subheap affinity may be reset to zero when the owner thread is detached */ + if ((block_affinity = block_get_affinity( heap, block ))) + { + /* the block is owned by another thread but we can clear it while it's unlinked, + * then add it to the defer list. The owner thread will move the block out of the + * defer list at some later time. It cannot also detach while we hold the heap CS. + */ + list_add_tail( &thread_data->defer_list.entry, &entry->entry ); + thread_data->defer_count++; + } + + heap_unlock( heap, flags ); + return block_affinity ? STATUS_SUCCESS : STATUS_NOT_CAPABLE; + } + + return STATUS_SUCCESS; +}
/*********************************************************************** * RtlFreeHeap (NTDLL.@) @@ -1818,6 +2049,7 @@ BOOLEAN WINAPI DECLSPEC_HOTPATCH RtlFreeHeap( HANDLE handle, ULONG flags, void * struct heap *heap; ULONG heap_flags; NTSTATUS status; + WORD affinity;
if (!ptr) return TRUE;
@@ -1831,6 +2063,8 @@ BOOLEAN WINAPI DECLSPEC_HOTPATCH RtlFreeHeap( HANDLE handle, ULONG flags, void * status = heap_free_large( heap, heap_flags, block ); else if (!(block = heap_delay_free( heap, heap_flags, block ))) status = STATUS_SUCCESS; + else if ((affinity = block_get_affinity( heap, block ))) + status = heap_free_block_lfh( heap, heap_flags, block, affinity ); else { heap_lock( heap, heap_flags ); @@ -1850,6 +2084,7 @@ static NTSTATUS heap_resize_block( struct heap *heap, ULONG flags, struct block SIZE_T old_block_size; struct entry *entry; struct block *next; + WORD affinity;
if (block_get_flags( block ) & BLOCK_FLAG_LARGE) { @@ -1877,6 +2112,24 @@ static NTSTATUS heap_resize_block( struct heap *heap, ULONG flags, struct block
if (block_size >= HEAP_MIN_LARGE_BLOCK_SIZE) return STATUS_NO_MEMORY; /* growing small block to large block */
+ if ((affinity = block_get_affinity( heap, block ))) + { + if (affinity != NtCurrentTeb()->HeapVirtualAffinity) return STATUS_NO_MEMORY; /* resizing block owned by another thread */ + + /* as native LFH does it with different block size: refuse to resize even though we could */ + if (ROUND_SIZE( *old_size, BLOCK_ALIGN - 1) != ROUND_SIZE( size, BLOCK_ALIGN - 1)) return STATUS_NO_MEMORY; + if (size >= *old_size) return STATUS_NO_MEMORY; + + valgrind_notify_resize( block + 1, *old_size, size ); + block_set_flags( block, BLOCK_FLAG_USER_MASK, BLOCK_USER_FLAGS( flags ) ); + block->tail_size = block_get_size( block ) - sizeof(*block) - size; + initialize_block( block, *old_size, size, flags ); + mark_block_tail( block, flags ); + + *ret = block + 1; + return STATUS_SUCCESS; + } + heap_lock( heap, flags );
if (block_size > old_block_size)
Hi,
It looks like your patch introduced the new failures shown below. Please investigate and fix them before resubmitting your patch. If they are not new, fixing them anyway would help a lot. Otherwise please ask for the known failures list to be updated.
The tests also ran into some preexisting test failures. If you know how to fix them that would be helpful. See the TestBot job for the details:
The full results can be found at: https://testbot.winehq.org/JobDetails.pl?Key=128864
Your paranoid android.
=== debian11b (64 bit WoW report) ===
kernel32: heap.c:455: Test failed: HeapAlloc succeeded heap.c:456: Test failed: got error 3735928559
This should pass the tests, but I'm still not completely sure about some of the implementation choices.
-
Marvin -
Rémi Bernon -
Rémi Bernon (＠rbernon)