Not completely sure if it's worth having for 8.0, but opening this to share the target I'm trying to reach.
-- v3: ntdll: Implement Low Fragmentation Heap. ntdll: Introduce per-thread free lists for heap blocks. ntdll: Introduce a new BLOCK_FLAG_SPLIT heap block flag. ntdll: Introduce a new subheap thread affinity field. ntdll: Introduce a new heap block_init_used helper. ntdll: Introduce a new heap free_list_init helper. ntdll: Count allocations and automatically enable LFH. ntdll: Implement HeapCompatibilityInformation.
From: Rémi Bernon rbernon@codeweavers.com
--- dlls/kernel32/tests/heap.c | 10 ------- dlls/ntdll/heap.c | 56 ++++++++++++++++++++++++++++++++------ 2 files changed, 47 insertions(+), 19 deletions(-)
diff --git a/dlls/kernel32/tests/heap.c b/dlls/kernel32/tests/heap.c index a81cb1b7a15..29faf8c902e 100644 --- a/dlls/kernel32/tests/heap.c +++ b/dlls/kernel32/tests/heap.c @@ -827,11 +827,8 @@ static void test_HeapCreate(void) size = 0; SetLastError( 0xdeadbeef ); ret = pHeapQueryInformation( 0, HeapCompatibilityInformation, &compat_info, sizeof(compat_info), &size ); - todo_wine ok( !ret, "HeapQueryInformation succeeded\n" ); - todo_wine ok( GetLastError() == ERROR_NOACCESS, "got error %lu\n", GetLastError() ); - todo_wine ok( size == 0, "got size %Iu\n", size );
size = 0; @@ -871,7 +868,6 @@ static void test_HeapCreate(void) ok( ret, "HeapSetInformation failed, error %lu\n", GetLastError() ); ret = pHeapQueryInformation( heap, HeapCompatibilityInformation, &compat_info, sizeof(compat_info), &size ); ok( ret, "HeapQueryInformation failed, error %lu\n", GetLastError() ); - todo_wine ok( compat_info == 2, "got HeapCompatibilityInformation %lu\n", compat_info );
/* cannot be undone */ @@ -879,20 +875,15 @@ static void test_HeapCreate(void) compat_info = 0; SetLastError( 0xdeadbeef ); ret = pHeapSetInformation( heap, HeapCompatibilityInformation, &compat_info, sizeof(compat_info) ); - todo_wine ok( !ret, "HeapSetInformation succeeded\n" ); - todo_wine ok( GetLastError() == ERROR_GEN_FAILURE, "got error %lu\n", GetLastError() ); compat_info = 1; SetLastError( 0xdeadbeef ); ret = pHeapSetInformation( heap, HeapCompatibilityInformation, &compat_info, sizeof(compat_info) ); - todo_wine ok( !ret, "HeapSetInformation succeeded\n" ); - todo_wine ok( GetLastError() == ERROR_GEN_FAILURE, "got error %lu\n", GetLastError() ); ret = pHeapQueryInformation( heap, HeapCompatibilityInformation, &compat_info, sizeof(compat_info), &size ); ok( ret, "HeapQueryInformation failed, error %lu\n", GetLastError() ); - todo_wine ok( compat_info == 2, "got HeapCompatibilityInformation %lu\n", compat_info );
ret = HeapDestroy( heap ); @@ -931,7 +922,6 @@ static void test_HeapCreate(void) ok( ret, "HeapSetInformation failed, error %lu\n", GetLastError() ); ret = pHeapQueryInformation( heap, HeapCompatibilityInformation, &compat_info, sizeof(compat_info), &size ); ok( ret, "HeapQueryInformation failed, error %lu\n", GetLastError() ); - todo_wine ok( compat_info == 2, "got HeapCompatibilityInformation %lu\n", compat_info );
for (i = 0; i < 0x11; i++) ptrs[i] = pHeapAlloc( heap, 0, 24 + 2 * sizeof(void *) ); diff --git a/dlls/ntdll/heap.c b/dlls/ntdll/heap.c index 9403c83e92c..2a763bc825f 100644 --- a/dlls/ntdll/heap.c +++ b/dlls/ntdll/heap.c @@ -39,6 +39,13 @@
WINE_DEFAULT_DEBUG_CHANNEL(heap);
+/* HeapCompatibilityInformation values */ + +#define HEAP_STD 0 +#define HEAP_LAL 1 +#define HEAP_LFH 2 + + /* undocumented RtlWalkHeap structure */
struct rtl_heap_entry @@ -194,6 +201,7 @@ struct heap DWORD force_flags; /* 0044/0074 */ /* end of the Windows 10 compatible struct layout */
+ LONG compat_info; /* HeapCompatibilityInformation / heap frontend type */ struct list entry; /* Entry in process heap list */ struct list subheap_list; /* Sub-heap list */ struct list large_list; /* Large blocks list */ @@ -1378,6 +1386,7 @@ HANDLE WINAPI RtlCreateHeap( ULONG flags, void *addr, SIZE_T total_size, SIZE_T heap->ffeeffee = 0xffeeffee; heap->auto_flags = (flags & HEAP_GROWABLE); heap->flags = (flags & ~HEAP_SHARED); + heap->compat_info = HEAP_STD; heap->magic = HEAP_MAGIC; heap->grow_size = max( HEAP_DEF_SIZE, total_size ); heap->min_size = commit_size; @@ -2039,21 +2048,24 @@ ULONG WINAPI RtlGetProcessHeaps( ULONG count, HANDLE *heaps ) * RtlQueryHeapInformation (NTDLL.@) */ NTSTATUS WINAPI RtlQueryHeapInformation( HANDLE handle, HEAP_INFORMATION_CLASS info_class, - void *info, SIZE_T size_in, PSIZE_T size_out ) + void *info, SIZE_T size_in, SIZE_T *size_out ) { + struct heap *heap; + ULONG flags; + + TRACE( "handle %p, info_class %u, info %p, size_in %Iu, size_out %p.\n", handle, info_class, info, size_in, size_out ); + switch (info_class) { case HeapCompatibilityInformation: + if (!(heap = unsafe_heap_from_handle( handle, 0, &flags ))) return STATUS_ACCESS_VIOLATION; if (size_out) *size_out = sizeof(ULONG); - - if (size_in < sizeof(ULONG)) - return STATUS_BUFFER_TOO_SMALL; - - *(ULONG *)info = 0; /* standard heap */ + if (size_in < sizeof(ULONG)) return STATUS_BUFFER_TOO_SMALL; + *(ULONG *)info = heap->compat_info; return STATUS_SUCCESS;
default: - FIXME("Unknown heap information class %u\n", info_class); + FIXME( "HEAP_INFORMATION_CLASS %u not implemented!\n", info_class ); return STATUS_INVALID_INFO_CLASS; } } @@ -2063,8 +2075,34 @@ NTSTATUS WINAPI RtlQueryHeapInformation( HANDLE handle, HEAP_INFORMATION_CLASS i */ NTSTATUS WINAPI RtlSetHeapInformation( HANDLE handle, HEAP_INFORMATION_CLASS info_class, void *info, SIZE_T size ) { - FIXME( "handle %p, info_class %d, info %p, size %Id stub!\n", handle, info_class, info, size ); - return STATUS_SUCCESS; + struct heap *heap; + ULONG flags; + + TRACE( "handle %p, info_class %u, info %p, size %Iu.\n", handle, info_class, info, size ); + + switch (info_class) + { + case HeapCompatibilityInformation: + { + ULONG compat_info; + + if (size < sizeof(ULONG)) return STATUS_BUFFER_TOO_SMALL; + if (!(heap = unsafe_heap_from_handle( handle, 0, &flags ))) return STATUS_INVALID_HANDLE; + + compat_info = *(ULONG *)info; + if (compat_info != HEAP_STD && compat_info != HEAP_LFH) + { + FIXME( "HeapCompatibilityInformation %lu not implemented!\n", compat_info ); + return STATUS_UNSUCCESSFUL; + } + if (InterlockedCompareExchange( &heap->compat_info, compat_info, HEAP_STD )) return STATUS_UNSUCCESSFUL; + return STATUS_SUCCESS; + } + + default: + FIXME( "HEAP_INFORMATION_CLASS %u not implemented!\n", info_class ); + return STATUS_SUCCESS; + } }
/***********************************************************************
From: Rémi Bernon rbernon@codeweavers.com
--- dlls/kernel32/tests/heap.c | 2 -- dlls/ntdll/heap.c | 56 +++++++++++++++++++++++++++++++++++++- 2 files changed, 55 insertions(+), 3 deletions(-)
diff --git a/dlls/kernel32/tests/heap.c b/dlls/kernel32/tests/heap.c index 29faf8c902e..ed84441efe0 100644 --- a/dlls/kernel32/tests/heap.c +++ b/dlls/kernel32/tests/heap.c @@ -904,7 +904,6 @@ static void test_HeapCreate(void)
ret = pHeapQueryInformation( heap, HeapCompatibilityInformation, &compat_info, sizeof(compat_info), &size ); ok( ret, "HeapQueryInformation failed, error %lu\n", GetLastError() ); - todo_wine ok( compat_info == 2, "got HeapCompatibilityInformation %lu\n", compat_info );
ret = HeapDestroy( heap ); @@ -1195,7 +1194,6 @@ static void test_HeapCreate(void)
ret = pHeapQueryInformation( heap, HeapCompatibilityInformation, &compat_info, sizeof(compat_info), &size ); ok( ret, "HeapQueryInformation failed, error %lu\n", GetLastError() ); - todo_wine ok( compat_info == 2, "got HeapCompatibilityInformation %lu\n", compat_info );
/* locking is serialized */ diff --git a/dlls/ntdll/heap.c b/dlls/ntdll/heap.c index 2a763bc825f..4bee3769ef3 100644 --- a/dlls/ntdll/heap.c +++ b/dlls/ntdll/heap.c @@ -149,6 +149,27 @@ C_ASSERT( sizeof(ARENA_LARGE) == 4 * BLOCK_ALIGN ); C_ASSERT( sizeof(struct block) <= HEAP_MIN_BLOCK_SIZE ); C_ASSERT( sizeof(struct entry) <= HEAP_MIN_BLOCK_SIZE );
+/* block size categories for LFH */ + +#define BLOCK_SIZE_SMALL_MAX (16 * HEAP_MIN_BLOCK_SIZE) +#define BLOCK_SIZE_MEDIUM_STEP (12 * BLOCK_ALIGN) +#define BLOCK_SIZE_CATEGORY_SMALL(size) (((size) - HEAP_MIN_BLOCK_SIZE) / BLOCK_ALIGN) +#define BLOCK_SIZE_CATEGORY_MEDIUM(size) (BLOCK_SIZE_CATEGORY_SMALL(BLOCK_SIZE_SMALL_MAX) + ((size) - (BLOCK_SIZE_SMALL_MAX)) / BLOCK_SIZE_MEDIUM_STEP) +#define BLOCK_SIZE_CATEGORY(size) (((size) >= BLOCK_SIZE_SMALL_MAX) ? BLOCK_SIZE_CATEGORY_MEDIUM(size) : BLOCK_SIZE_CATEGORY_SMALL(size)) +#define BLOCK_SIZE_CATEGORY_COUNT BLOCK_SIZE_CATEGORY(0x400 * BLOCK_ALIGN) +#define BLOCK_SIZE_MEDIUM_MIN (BLOCK_SIZE_SMALL_MAX + BLOCK_SIZE_MEDIUM_STEP - BLOCK_ALIGN) + +#define BLOCK_CATEGORY_SIZE(category) (category >= BLOCK_SIZE_CATEGORY_MEDIUM(BLOCK_SIZE_SMALL_MAX) \ + ? (BLOCK_SIZE_MEDIUM_MIN + (category - BLOCK_SIZE_CATEGORY_MEDIUM(BLOCK_SIZE_SMALL_MAX)) * BLOCK_SIZE_MEDIUM_STEP) \ + : (HEAP_MIN_BLOCK_SIZE + category * BLOCK_ALIGN)) + +C_ASSERT( BLOCK_SIZE_CATEGORY(HEAP_MIN_BLOCK_SIZE) == 0 ); +C_ASSERT( BLOCK_SIZE_CATEGORY_SMALL(BLOCK_SIZE_SMALL_MAX - 1) == 44 ); +C_ASSERT( BLOCK_SIZE_CATEGORY_MEDIUM(BLOCK_SIZE_SMALL_MAX) == 45 ); +C_ASSERT( BLOCK_SIZE_CATEGORY_COUNT == 126 ); + +/* difference between block classes must fit into block tail_size */ +C_ASSERT( BLOCK_CATEGORY_SIZE(126) - BLOCK_CATEGORY_SIZE(125) <= FIELD_MAX( struct block, tail_size ) ); /* used block size is coded into block_size */ #define HEAP_MAX_USED_BLOCK_SIZE (FIELD_MAX( struct block, block_size ) * BLOCK_ALIGN) /* free block size is coded into block_size + tail_size */ @@ -189,6 +210,12 @@ typedef struct DECLSPEC_ALIGN(BLOCK_ALIGN) tagSUBHEAP C_ASSERT( sizeof(SUBHEAP) == offsetof(SUBHEAP, block) + sizeof(struct block) ); C_ASSERT( sizeof(SUBHEAP) == 4 * BLOCK_ALIGN );
+struct heap_counters +{ + volatile BYTE live; + volatile WORD total; +}; + struct heap { /* win32/win64 */ DWORD_PTR unknown1[2]; /* 0000/0000 */ @@ -201,6 +228,7 @@ struct heap DWORD force_flags; /* 0044/0074 */ /* end of the Windows 10 compatible struct layout */
+ struct heap_counters stats[BLOCK_SIZE_CATEGORY_COUNT]; /* Stats counter for LFH activation */ LONG compat_info; /* HeapCompatibilityInformation / heap frontend type */ struct list entry; /* Entry in process heap list */ struct list subheap_list; /* Sub-heap list */ @@ -776,11 +804,13 @@ static struct block *heap_delay_free( struct heap *heap, ULONG flags, struct blo
static NTSTATUS heap_free_block( struct heap *heap, ULONG flags, struct block *block ) { + SIZE_T block_size = block_get_size( block ), category = BLOCK_SIZE_CATEGORY(block_size); SUBHEAP *subheap = block_get_subheap( heap, block ); - SIZE_T block_size = block_get_size( block ); struct entry *entry; struct block *next;
+ if (category < ARRAY_SIZE(heap->stats) && heap->stats[category].live <= 0x10) heap->stats[category].live--; + if ((next = next_block( subheap, block )) && (block_get_flags( next ) & BLOCK_FLAG_FREE)) { /* merge with next block if it is free */ @@ -1541,6 +1571,28 @@ static SIZE_T heap_get_block_size( const struct heap *heap, ULONG flags, SIZE_T return block_size; }
+static NTSTATUS heap_allocate_block_lfh( struct heap *heap, ULONG flags, SIZE_T block_size, + SIZE_T size, void **ret ) +{ + SIZE_T category = BLOCK_SIZE_CATEGORY( block_size ); + + if (category >= ARRAY_SIZE(heap->stats)) return STATUS_UNSUCCESSFUL; + if (heap->stats[category].live <= 0x10 && heap->stats[category].total <= 0x800) + { + heap->stats[category].live++; + heap->stats[category].total++; + return STATUS_UNSUCCESSFUL; + } + if (heap->compat_info != HEAP_LFH) + { + ULONG info = HEAP_LFH; + RtlSetHeapInformation( heap, HeapCompatibilityInformation, &info, sizeof(info) ); + return STATUS_UNSUCCESSFUL; + } + + return STATUS_NOT_CAPABLE; +} + static NTSTATUS heap_allocate_block( struct heap *heap, ULONG flags, SIZE_T block_size, SIZE_T size, void **ret ) { struct block *block, *next; @@ -1589,6 +1641,8 @@ void *WINAPI DECLSPEC_HOTPATCH RtlAllocateHeap( HANDLE handle, ULONG flags, SIZE status = STATUS_NO_MEMORY; else if (block_size >= HEAP_MIN_LARGE_BLOCK_SIZE) status = heap_allocate_large( heap, heap_flags, block_size, size, &ptr ); + else if (!heap_allocate_block_lfh( heap, heap_flags, block_size, size, &ptr )) + status = STATUS_SUCCESS; else { heap_lock( heap, heap_flags );
From: Rémi Bernon rbernon@codeweavers.com
--- dlls/ntdll/heap.c | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-)
diff --git a/dlls/ntdll/heap.c b/dlls/ntdll/heap.c index 4bee3769ef3..61526dd5d50 100644 --- a/dlls/ntdll/heap.c +++ b/dlls/ntdll/heap.c @@ -510,7 +510,7 @@ static inline struct entry *find_free_list( struct heap *heap, SIZE_T block_size
if (block_size <= HEAP_MAX_SMALL_FREE_LIST) i = (block_size - HEAP_MIN_BLOCK_SIZE) / BLOCK_ALIGN; - else for (i = HEAP_NB_SMALL_FREE_LISTS; i < HEAP_NB_FREE_LISTS - 1; i++) + else for (i = HEAP_NB_SMALL_FREE_LISTS; i < ARRAY_SIZE(heap->free_lists) - 1; i++) if (block_size <= free_list_sizes[i - HEAP_NB_SMALL_FREE_LISTS]) break;
list = heap->free_lists + i; @@ -518,6 +518,15 @@ static inline struct entry *find_free_list( struct heap *heap, SIZE_T block_size return list; }
+static void free_list_init( struct entry *entry ) +{ + list_init( &entry->entry ); + block_set_flags( &entry->block, ~0, BLOCK_FLAG_FREE_LINK ); + block_set_size( &entry->block, 0 ); + block_set_base( &entry->block, &entry ); + block_set_type( &entry->block, BLOCK_TYPE_FREE ); +} + /* get the memory protection type to use for a given heap */ static inline ULONG get_protection_type( DWORD flags ) { @@ -573,10 +582,13 @@ static void heap_dump( const struct heap *heap ) TRACE( " next %p\n", LIST_ENTRY( heap->entry.next, struct heap, entry ) );
TRACE( " free_lists: %p\n", heap->free_lists ); - for (i = 0; i < HEAP_NB_FREE_LISTS; i++) + for (i = 0; i < ARRAY_SIZE(heap->free_lists); i++) + { TRACE( " %p: size %#8Ix, prev %p, next %p\n", heap->free_lists + i, get_free_list_block_size( i ), LIST_ENTRY( heap->free_lists[i].entry.prev, struct entry, entry ), LIST_ENTRY( heap->free_lists[i].entry.next, struct entry, entry ) ); + } +
TRACE( " subheaps: %p\n", &heap->subheap_list ); LIST_FOR_EACH_ENTRY( subheap, &heap->subheap_list, SUBHEAP, entry ) @@ -1050,7 +1062,9 @@ static BOOL is_valid_free_block( const struct heap *heap, const struct block *bl unsigned int i;
if ((subheap = find_subheap( heap, block, FALSE ))) return TRUE; - for (i = 0; i < HEAP_NB_FREE_LISTS; i++) if (block == &heap->free_lists[i].block) return TRUE; + for (i = 0; i < ARRAY_SIZE(heap->free_lists); i++) + if (block == &heap->free_lists[i].block) return TRUE; + return FALSE; }
@@ -1423,13 +1437,9 @@ HANDLE WINAPI RtlCreateHeap( ULONG flags, void *addr, SIZE_T total_size, SIZE_T list_init( &heap->subheap_list ); list_init( &heap->large_list );
- list_init( &heap->free_lists[0].entry ); for (i = 0, entry = heap->free_lists; i < HEAP_NB_FREE_LISTS; i++, entry++) { - block_set_flags( &entry->block, ~0, BLOCK_FLAG_FREE_LINK ); - block_set_size( &entry->block, 0 ); - block_set_type( &entry->block, BLOCK_TYPE_FREE ); - block_set_base( &entry->block, heap ); + free_list_init( entry ); if (i) list_add_after( &entry[-1].entry, &entry->entry ); }
From: Rémi Bernon rbernon@codeweavers.com
--- dlls/ntdll/heap.c | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-)
diff --git a/dlls/ntdll/heap.c b/dlls/ntdll/heap.c index 61526dd5d50..7d85205e17d 100644 --- a/dlls/ntdll/heap.c +++ b/dlls/ntdll/heap.c @@ -757,6 +757,15 @@ static inline BOOL subheap_decommit( const struct heap *heap, SUBHEAP *subheap, return TRUE; }
+static void block_init_used( struct block *block, ULONG flags, SIZE_T size ) +{ + block_set_type( block, BLOCK_TYPE_USED ); + block_set_flags( block, 0, BLOCK_USER_FLAGS( flags ) ); + block->tail_size = block_get_size( block ) - sizeof(*block) - size; + initialize_block( block, 0, size, flags ); + mark_block_tail( block, flags ); +} + static void block_init_free( struct block *block, ULONG flags, SUBHEAP *subheap, SIZE_T block_size ) { const char *end = (char *)block + block_size, *commit_end = subheap_commit_end( subheap ); @@ -1622,11 +1631,7 @@ static NTSTATUS heap_allocate_block( struct heap *heap, ULONG flags, SIZE_T bloc insert_free_block( heap, flags, subheap, next ); }
- block_set_type( block, BLOCK_TYPE_USED ); - block_set_flags( block, ~0, BLOCK_USER_FLAGS( flags ) ); - block->tail_size = block_get_size( block ) - sizeof(*block) - size; - initialize_block( block, 0, size, flags ); - mark_block_tail( block, flags ); + block_init_used( block, flags, size );
if ((next = next_block( subheap, block ))) block_set_flags( next, BLOCK_FLAG_PREV_FREE, 0 );
From: Rémi Bernon rbernon@codeweavers.com
To associate a subheap with a specific thread that will be its only user. Other threads may only defer the block free of subheap with a different affinity, through locking and defer list. --- dlls/ntdll/heap.c | 46 ++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 38 insertions(+), 8 deletions(-)
diff --git a/dlls/ntdll/heap.c b/dlls/ntdll/heap.c index 7d85205e17d..40bda92c0e5 100644 --- a/dlls/ntdll/heap.c +++ b/dlls/ntdll/heap.c @@ -198,7 +198,8 @@ static const SIZE_T free_list_sizes[] =
typedef struct DECLSPEC_ALIGN(BLOCK_ALIGN) tagSUBHEAP { - SIZE_T __pad[sizeof(SIZE_T) / sizeof(DWORD)]; + SIZE_T __pad[sizeof(SIZE_T) / sizeof(DWORD) - 1]; + DWORD affinity; SIZE_T block_size; SIZE_T data_size; struct list entry; @@ -356,6 +357,21 @@ static void subheap_set_bounds( SUBHEAP *subheap, char *commit_end, char *end ) subheap->data_size = commit_end - (char *)(subheap + 1); }
+static void subheap_set_affinity( SUBHEAP *subheap, WORD affinity ) +{ + subheap->affinity = affinity; +} + +static inline WORD subheap_get_affinity( const SUBHEAP *subheap ) +{ + return subheap->affinity; +} + +static inline WORD block_get_affinity( const struct heap *heap, const struct block *block ) +{ + return subheap_get_affinity( block_get_subheap( heap, block ) ); +} + static inline void *first_block( const SUBHEAP *subheap ) { return (void *)&subheap->block; @@ -593,11 +609,12 @@ static void heap_dump( const struct heap *heap ) TRACE( " subheaps: %p\n", &heap->subheap_list ); LIST_FOR_EACH_ENTRY( subheap, &heap->subheap_list, SUBHEAP, entry ) { - SIZE_T free_size = 0, used_size = 0, overhead = 0; + SIZE_T free_size = 0, used_size = 0, overhead = 0, affinity = subheap_get_affinity( subheap ); const char *base = subheap_base( subheap );
TRACE( " %p: base %p first %p last %p end %p\n", subheap, base, first_block( subheap ), last_block( subheap ), base + subheap_size( subheap ) ); + if (affinity) TRACE( " affinity %#Ix\n", affinity );
if (!check_subheap( subheap, heap )) return;
@@ -1000,7 +1017,7 @@ static BOOL validate_large_block( const struct heap *heap, const struct block *b }
-static SUBHEAP *create_subheap( struct heap *heap, DWORD flags, SIZE_T total_size, SIZE_T commit_size ) +static SUBHEAP *create_subheap( struct heap *heap, DWORD flags, SIZE_T total_size, SIZE_T commit_size, WORD affinity ) { SIZE_T block_size; SUBHEAP *subheap; @@ -1011,6 +1028,7 @@ static SUBHEAP *create_subheap( struct heap *heap, DWORD flags, SIZE_T total_siz if (!(subheap = allocate_region( heap, flags, &total_size, &commit_size ))) return NULL;
subheap->user_value = heap; + subheap_set_affinity( subheap, affinity ); subheap_set_bounds( subheap, (char *)subheap + commit_size, (char *)subheap + total_size ); block_size = (SIZE_T)ROUND_ADDR( subheap_size( subheap ) - subheap_overhead( subheap ), BLOCK_ALIGN - 1 ); block_init_free( first_block( subheap ), flags, subheap, block_size ); @@ -1048,7 +1066,7 @@ static struct block *find_free_block( struct heap *heap, ULONG flags, SIZE_T blo total_size = sizeof(SUBHEAP) + block_size + sizeof(struct entry); if (total_size < block_size) return NULL; /* overflow */
- if ((subheap = create_subheap( heap, flags, max( heap->grow_size, total_size ), total_size ))) + if ((subheap = create_subheap( heap, flags, max( heap->grow_size, total_size ), total_size, 0 ))) { if (heap->grow_size <= HEAP_MAX_FREE_BLOCK_SIZE / 2) heap->grow_size *= 2; } @@ -1056,7 +1074,7 @@ static struct block *find_free_block( struct heap *heap, ULONG flags, SIZE_T blo { if (heap->grow_size <= total_size || heap->grow_size <= 4 * 1024 * 1024) return NULL; heap->grow_size /= 2; - subheap = create_subheap( heap, flags, max( heap->grow_size, total_size ), total_size ); + subheap = create_subheap( heap, flags, max( heap->grow_size, total_size ), total_size, 0 ); }
TRACE( "created new sub-heap %p of %#Ix bytes for heap %p\n", subheap, subheap_size( subheap ), heap ); @@ -1100,7 +1118,7 @@ static BOOL validate_free_block( const struct heap *heap, const SUBHEAP *subheap err = "invalid previous free block pointer"; else if (!(block_get_flags( prev ) & BLOCK_FLAG_FREE) || block_get_type( prev ) != BLOCK_TYPE_FREE) err = "invalid previous free block header"; - else if ((next = next_block( subheap, block ))) + else if (!(block_get_affinity( heap, block )) && (next = next_block( subheap, block ))) { if (!(block_get_flags( next ) & BLOCK_FLAG_PREV_FREE)) err = "invalid next block flags"; @@ -1209,12 +1227,19 @@ static BOOL heap_validate_ptr( const struct heap *heap, const void *ptr )
static BOOL heap_validate( const struct heap *heap ) { + DWORD affinity = NtCurrentTeb()->HeapVirtualAffinity, block_affinity; const ARENA_LARGE *large_arena; const struct block *block; const SUBHEAP *subheap;
LIST_FOR_EACH_ENTRY( subheap, &heap->subheap_list, SUBHEAP, entry ) { + if ((block_affinity = subheap_get_affinity( subheap )) && block_affinity != affinity) + { + TRACE( "Skipping subheap %p with affinity %#lx\n", subheap, block_affinity ); + continue; + } + if (!check_subheap( subheap, heap )) { ERR( "heap %p, subheap %p corrupted sizes or user_value\n", heap, subheap ); @@ -1470,6 +1495,7 @@ HANDLE WINAPI RtlCreateHeap( ULONG flags, void *addr, SIZE_T total_size, SIZE_T
subheap = &heap->subheap; subheap->user_value = heap; + subheap_set_affinity( subheap, 0 ); subheap_set_bounds( subheap, (char *)heap + commit_size, (char *)heap + total_size ); block_size = (SIZE_T)ROUND_ADDR( subheap_size( subheap ) - subheap_overhead( subheap ), BLOCK_ALIGN - 1 ); block_init_free( first_block( subheap ), flags, subheap, block_size ); @@ -1980,7 +2006,8 @@ static NTSTATUS heap_walk_blocks( const struct heap *heap, const SUBHEAP *subhea entry->cbData = commit_end - (char *)entry->lpData - 4 * BLOCK_ALIGN; entry->cbOverhead = 2 * BLOCK_ALIGN; entry->iRegionIndex = 0; - entry->wFlags = 0; + if (subheap_get_affinity( subheap )) entry->wFlags = RTL_HEAP_ENTRY_LFH; + else entry->wFlags = 0; } else { @@ -1988,7 +2015,9 @@ static NTSTATUS heap_walk_blocks( const struct heap *heap, const SUBHEAP *subhea entry->cbData = block_get_size( block ) - block_get_overhead( block ); entry->cbOverhead = block_get_overhead( block ); entry->iRegionIndex = 0; - entry->wFlags = RTL_HEAP_ENTRY_COMMITTED|RTL_HEAP_ENTRY_BLOCK|RTL_HEAP_ENTRY_BUSY; + entry->wFlags = RTL_HEAP_ENTRY_BUSY; + if (subheap_get_affinity( subheap )) entry->wFlags |= RTL_HEAP_ENTRY_LFH; + else entry->wFlags |= RTL_HEAP_ENTRY_COMMITTED|RTL_HEAP_ENTRY_BLOCK; }
return STATUS_SUCCESS; @@ -2034,6 +2063,7 @@ static NTSTATUS heap_walk( const struct heap *heap, struct rtl_heap_entry *entry entry->cbOverhead = 0; entry->iRegionIndex = 0; entry->wFlags = RTL_HEAP_ENTRY_REGION; + if (subheap_get_affinity( subheap )) entry->wFlags |= RTL_HEAP_ENTRY_LFH; entry->Region.dwCommittedSize = (char *)subheap_commit_end( subheap ) - base; entry->Region.dwUnCommittedSize = subheap_size( subheap ) - entry->Region.dwCommittedSize; entry->Region.lpFirstBlock = base + entry->cbData;
From: Rémi Bernon rbernon@codeweavers.com
To avoid coalescing free LFH blocks until their owning thread is detached. --- dlls/ntdll/heap.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/dlls/ntdll/heap.c b/dlls/ntdll/heap.c index 40bda92c0e5..c5b473559cf 100644 --- a/dlls/ntdll/heap.c +++ b/dlls/ntdll/heap.c @@ -101,6 +101,7 @@ C_ASSERT( sizeof(struct block) == 8 ); #define BLOCK_FLAG_PREV_FREE 0x02 #define BLOCK_FLAG_FREE_LINK 0x03 #define BLOCK_FLAG_LARGE 0x04 +#define BLOCK_FLAG_SPLIT 0x08 /* block is split and must not be coalesced */ #define BLOCK_FLAG_USER_INFO 0x10 /* user flags up to 0xf0 */ #define BLOCK_FLAG_USER_MASK 0xf0
@@ -777,7 +778,7 @@ static inline BOOL subheap_decommit( const struct heap *heap, SUBHEAP *subheap, static void block_init_used( struct block *block, ULONG flags, SIZE_T size ) { block_set_type( block, BLOCK_TYPE_USED ); - block_set_flags( block, 0, BLOCK_USER_FLAGS( flags ) ); + block_set_flags( block, ~BLOCK_FLAG_SPLIT, BLOCK_USER_FLAGS( flags ) ); block->tail_size = block_get_size( block ) - sizeof(*block) - size; initialize_block( block, 0, size, flags ); mark_block_tail( block, flags ); @@ -849,7 +850,8 @@ static NTSTATUS heap_free_block( struct heap *heap, ULONG flags, struct block *b
if (category < ARRAY_SIZE(heap->stats) && heap->stats[category].live <= 0x10) heap->stats[category].live--;
- if ((next = next_block( subheap, block )) && (block_get_flags( next ) & BLOCK_FLAG_FREE)) + if ((next = next_block( subheap, block )) && (block_get_flags( next ) & BLOCK_FLAG_FREE) && + !(block_get_flags( next ) & BLOCK_FLAG_SPLIT)) { /* merge with next block if it is free */ entry = (struct entry *)next;
From: Rémi Bernon rbernon@codeweavers.com
--- dlls/ntdll/heap.c | 112 ++++++++++++++++++++++++++++++++++++++-- dlls/ntdll/loader.c | 2 + dlls/ntdll/ntdll_misc.h | 1 + 3 files changed, 112 insertions(+), 3 deletions(-)
diff --git a/dlls/ntdll/heap.c b/dlls/ntdll/heap.c index c5b473559cf..e5e7044affe 100644 --- a/dlls/ntdll/heap.c +++ b/dlls/ntdll/heap.c @@ -218,6 +218,17 @@ struct heap_counters volatile WORD total; };
+struct heap_thread_data +{ + struct entry free_lists[BLOCK_SIZE_CATEGORY_COUNT]; + /* requires heap lock to access */ + struct entry defer_list; + volatile BOOL has_defer; +}; + +/* affinity to tid mapping array, defines the amount of supported concurrent threads */ +static LONG thread_data_tid[64]; + struct heap { /* win32/win64 */ DWORD_PTR unknown1[2]; /* 0000/0000 */ @@ -230,6 +241,7 @@ struct heap DWORD force_flags; /* 0044/0074 */ /* end of the Windows 10 compatible struct layout */
+ struct heap_thread_data *tls; /* Thread local storage data */ struct heap_counters stats[BLOCK_SIZE_CATEGORY_COUNT]; /* Stats counter for LFH activation */ LONG compat_info; /* HeapCompatibilityInformation / heap frontend type */ struct list entry; /* Entry in process heap list */ @@ -593,7 +605,7 @@ static void heap_dump( const struct heap *heap ) const struct block *block; const ARENA_LARGE *large; const SUBHEAP *subheap; - unsigned int i; + unsigned int i, j;
TRACE( "heap: %p\n", heap ); TRACE( " next %p\n", LIST_ENTRY( heap->entry.next, struct heap, entry ) ); @@ -606,6 +618,24 @@ static void heap_dump( const struct heap *heap ) LIST_ENTRY( heap->free_lists[i].entry.next, struct entry, entry ) ); }
+ if (heap->tls) for (i = 0; i < ARRAY_SIZE(thread_data_tid); ++i) + { + struct heap_thread_data *thread_data = heap->tls + i; + if (!thread_data_tid[i]) continue; + TRACE( " index %#x thread %04lx:\n", i, thread_data_tid[i] ); + + for (j = 0; j < ARRAY_SIZE(thread_data->free_lists); j++) + { + if (list_empty( &thread_data->free_lists[j].entry )) continue; + TRACE( " %p: size %8Ix, prev %p, next %p\n", thread_data->free_lists + j, HEAP_MIN_BLOCK_SIZE + j * BLOCK_ALIGN, + LIST_ENTRY( thread_data->free_lists[j].entry.prev, struct entry, entry ), + LIST_ENTRY( thread_data->free_lists[j].entry.next, struct entry, entry ) ); + } + + TRACE( " %p: pending, prev %p, next %p\n", &thread_data->defer_list, + LIST_ENTRY( thread_data->defer_list.entry.prev, struct entry, entry ), + LIST_ENTRY( thread_data->defer_list.entry.next, struct entry, entry ) ); + }
TRACE( " subheaps: %p\n", &heap->subheap_list ); LIST_FOR_EACH_ENTRY( subheap, &heap->subheap_list, SUBHEAP, entry ) @@ -615,7 +645,7 @@ static void heap_dump( const struct heap *heap )
TRACE( " %p: base %p first %p last %p end %p\n", subheap, base, first_block( subheap ), last_block( subheap ), base + subheap_size( subheap ) ); - if (affinity) TRACE( " affinity %#Ix\n", affinity ); + if (affinity) TRACE( " affinity %#Ix, thread %04lx\n", affinity, thread_data_tid[affinity - 1] );
if (!check_subheap( subheap, heap )) return;
@@ -1019,6 +1049,65 @@ static BOOL validate_large_block( const struct heap *heap, const struct block *b }
+static struct heap_thread_data *alloc_thread_data_block( struct heap *heap ) +{ + struct heap_thread_data *thread_data_block = NULL; + SIZE_T size, i, j; + + size = sizeof(*thread_data_block) * ARRAY_SIZE(thread_data_tid); + if (NtAllocateVirtualMemory( NtCurrentProcess(), (void *)&thread_data_block, 0, &size, + MEM_COMMIT, PAGE_READWRITE )) return NULL; + + for (i = 0; i < ARRAY_SIZE(thread_data_tid); ++i) + { + struct heap_thread_data *thread_data = thread_data_block + i; + for (j = 0; j < ARRAY_SIZE(thread_data->free_lists); ++j) + free_list_init( &thread_data->free_lists[j] ); + free_list_init( &thread_data->defer_list ); + } + + return thread_data_block; +} + +static DWORD next_thread_data_index(void) +{ + LONG index, tid = GetCurrentThreadId(); + + for (index = ARRAY_SIZE(thread_data_tid) - 1; index >= 0; index--) + if (!InterlockedCompareExchange( thread_data_tid + index, tid, 0 )) + break; + + NtCurrentTeb()->HeapVirtualAffinity = index + 1; + return index; +} + +void heap_thread_detach(void) +{ + DWORD index = NtCurrentTeb()->HeapVirtualAffinity - 1; + if (index >= ARRAY_SIZE(thread_data_tid)) return; + InterlockedExchange( thread_data_tid + index, 0 ); +} + +static struct heap_thread_data *heap_get_thread_data( struct heap *heap ) +{ + DWORD index = NtCurrentTeb()->HeapVirtualAffinity - 1; + + if (!process_heap || heap->compat_info != HEAP_LFH) return NULL; + if (index >= ARRAY_SIZE(thread_data_tid)) index = next_thread_data_index(); + if (index >= ARRAY_SIZE(thread_data_tid)) return NULL; + + if (!heap->tls) + { + RtlLockHeap( heap ); + if (!heap->tls) heap->tls = alloc_thread_data_block( heap ); + RtlUnlockHeap( heap ); + } + + if (!heap->tls) return NULL; + return heap->tls + index; +} + + static SUBHEAP *create_subheap( struct heap *heap, DWORD flags, SIZE_T total_size, SIZE_T commit_size, WORD affinity ) { SIZE_T block_size; @@ -1088,11 +1177,20 @@ static struct block *find_free_block( struct heap *heap, ULONG flags, SIZE_T blo static BOOL is_valid_free_block( const struct heap *heap, const struct block *block ) { const SUBHEAP *subheap; - unsigned int i; + unsigned int i, j;
if ((subheap = find_subheap( heap, block, FALSE ))) return TRUE; for (i = 0; i < ARRAY_SIZE(heap->free_lists); i++) if (block == &heap->free_lists[i].block) return TRUE; + if (!heap->tls) return FALSE; + + for (i = 0; i < ARRAY_SIZE(thread_data_tid); ++i) + { + struct heap_thread_data *thread_data = heap->tls + i; + if (block == &thread_data->defer_list.block) return TRUE; + for (j = 0; j < ARRAY_SIZE(thread_data->free_lists); ++j) + if (block == &thread_data->free_lists[j].block) return TRUE; + }
return FALSE; } @@ -1592,6 +1690,12 @@ HANDLE WINAPI RtlDestroyHeap( HANDLE handle ) NtFreeVirtualMemory( NtCurrentProcess(), &addr, &size, MEM_RELEASE ); } valgrind_notify_free_all( &heap->subheap, heap ); + if (heap->tls) + { + size = 0; + addr = heap->tls; + NtFreeVirtualMemory( NtCurrentProcess(), &addr, &size, MEM_RELEASE ); + } size = 0; addr = heap; NtFreeVirtualMemory( NtCurrentProcess(), &addr, &size, MEM_RELEASE ); @@ -1622,6 +1726,7 @@ static NTSTATUS heap_allocate_block_lfh( struct heap *heap, ULONG flags, SIZE_T SIZE_T size, void **ret ) { SIZE_T category = BLOCK_SIZE_CATEGORY( block_size ); + struct heap_thread_data *thread_data;
if (category >= ARRAY_SIZE(heap->stats)) return STATUS_UNSUCCESSFUL; if (heap->stats[category].live <= 0x10 && heap->stats[category].total <= 0x800) @@ -1636,6 +1741,7 @@ static NTSTATUS heap_allocate_block_lfh( struct heap *heap, ULONG flags, SIZE_T RtlSetHeapInformation( heap, HeapCompatibilityInformation, &info, sizeof(info) ); return STATUS_UNSUCCESSFUL; } + if (!(thread_data = heap_get_thread_data( heap ))) return STATUS_UNSUCCESSFUL;
return STATUS_NOT_CAPABLE; } diff --git a/dlls/ntdll/loader.c b/dlls/ntdll/loader.c index 01a30742678..6c643575f8d 100644 --- a/dlls/ntdll/loader.c +++ b/dlls/ntdll/loader.c @@ -3737,6 +3737,8 @@ void WINAPI LdrShutdownThread(void) /* don't call DbgUiGetThreadDebugObject as some apps hook it and terminate if called */ if (NtCurrentTeb()->DbgSsReserved[1]) NtClose( NtCurrentTeb()->DbgSsReserved[1] ); RtlFreeThreadActivationContextStack(); + + heap_thread_detach(); }
diff --git a/dlls/ntdll/ntdll_misc.h b/dlls/ntdll/ntdll_misc.h index d1a7790991b..f6b77b79cde 100644 --- a/dlls/ntdll/ntdll_misc.h +++ b/dlls/ntdll/ntdll_misc.h @@ -127,5 +127,6 @@ static inline void ascii_to_unicode( WCHAR *dst, const char *src, size_t len )
/* FLS data */ extern TEB_FLS_DATA *fls_alloc_data(void) DECLSPEC_HIDDEN; +extern void heap_thread_detach(void) DECLSPEC_HIDDEN;
#endif
From: Rémi Bernon rbernon@codeweavers.com
--- dlls/kernel32/tests/heap.c | 29 +---- dlls/ntdll/heap.c | 237 ++++++++++++++++++++++++++++++++++++- 2 files changed, 240 insertions(+), 26 deletions(-)
diff --git a/dlls/kernel32/tests/heap.c b/dlls/kernel32/tests/heap.c index ed84441efe0..0c95b08780e 100644 --- a/dlls/kernel32/tests/heap.c +++ b/dlls/kernel32/tests/heap.c @@ -452,9 +452,7 @@ static void test_HeapCreate(void)
SetLastError( 0xdeadbeef ); ptr1 = HeapAlloc( heap, 0, alloc_size - (0x200 + 0x80 * sizeof(void *)) ); - todo_wine ok( !ptr1, "HeapAlloc succeeded\n" ); - todo_wine ok( GetLastError() == ERROR_NOT_ENOUGH_MEMORY, "got error %lu\n", GetLastError() ); ret = HeapFree( heap, 0, ptr1 ); ok( ret, "HeapFree failed, error %lu\n", GetLastError() ); @@ -953,7 +951,6 @@ static void test_HeapCreate(void) SetLastError( 0xdeadbeef ); while ((ret = HeapWalk( heap, &entry ))) entries[count++] = entry; ok( GetLastError() == ERROR_NO_MORE_ITEMS, "got error %lu\n", GetLastError() ); - todo_wine ok( count > 24, "got count %lu\n", count ); if (count < 2) count = 2;
@@ -966,7 +963,6 @@ static void test_HeapCreate(void)
for (i = 0; i < 0x12; i++) { - todo_wine ok( entries[4 + i].wFlags == 0, "got wFlags %#x\n", entries[4 + i].wFlags ); todo_wine ok( entries[4 + i].cbData == 0x20, "got cbData %#lx\n", entries[4 + i].cbData ); @@ -985,7 +981,6 @@ static void test_HeapCreate(void) rtl_entry.lpData = NULL; SetLastError( 0xdeadbeef ); while (!RtlWalkHeap( heap, &rtl_entry )) rtl_entries[count++] = rtl_entry; - todo_wine ok( count > 24, "got count %lu\n", count ); if (count < 2) count = 2;
@@ -1028,7 +1023,6 @@ static void test_HeapCreate(void) SetLastError( 0xdeadbeef ); while ((ret = HeapWalk( heap, &entry ))) entries[count++] = entry; ok( GetLastError() == ERROR_NO_MORE_ITEMS, "got error %lu\n", GetLastError() ); - todo_wine ok( count > 24, "got count %lu\n", count ); if (count < 2) count = 2;
@@ -1057,7 +1051,6 @@ static void test_HeapCreate(void) rtl_entry.lpData = NULL; SetLastError( 0xdeadbeef ); while (!RtlWalkHeap( heap, &rtl_entry )) rtl_entries[count++] = rtl_entry; - todo_wine ok( count > 24, "got count %lu\n", count ); if (count < 2) count = 2;
@@ -1071,11 +1064,8 @@ static void test_HeapCreate(void) if (!entries[i].wFlags) ok( rtl_entries[i].wFlags == 0 || rtl_entries[i].wFlags == RTL_HEAP_ENTRY_LFH, "got wFlags %#x\n", rtl_entries[i].wFlags ); else if (entries[i].wFlags & PROCESS_HEAP_ENTRY_BUSY) - { - todo_wine ok( rtl_entries[i].wFlags == (RTL_HEAP_ENTRY_LFH|RTL_HEAP_ENTRY_BUSY) || broken(rtl_entries[i].wFlags == 1) /* win7 */, "got wFlags %#x\n", rtl_entries[i].wFlags ); - } else if (entries[i].wFlags & PROCESS_HEAP_UNCOMMITTED_RANGE) ok( rtl_entries[i].wFlags == RTL_HEAP_ENTRY_UNCOMMITTED || broken(rtl_entries[i].wFlags == 0x100) /* win7 */, "got wFlags %#x\n", rtl_entries[i].wFlags ); @@ -1220,7 +1210,6 @@ static void test_HeapCreate(void) thread_params.flags = 0; SetEvent( thread_params.start_event ); res = WaitForSingleObject( thread_params.ready_event, 100 ); - todo_wine ok( !res, "WaitForSingleObject returned %#lx, error %lu\n", res, GetLastError() ); ret = HeapUnlock( heap ); ok( ret, "HeapUnlock failed, error %lu\n", GetLastError() ); @@ -1667,9 +1656,7 @@ static void test_GlobalAlloc(void) ok( size == small_size, "GlobalSize returned %Iu\n", size ); SetLastError( 0xdeadbeef ); tmp_mem = GlobalReAlloc( mem, small_size, 0 ); - todo_wine ok( !tmp_mem, "GlobalReAlloc succeeded\n" ); - todo_wine ok( GetLastError() == ERROR_NOT_ENOUGH_MEMORY, "got error %lu\n", GetLastError() ); if (tmp_mem) mem = tmp_mem; tmp_mem = GlobalReAlloc( mem, 1024 * 1024, GMEM_MODIFY ); @@ -1685,7 +1672,6 @@ static void test_GlobalAlloc(void) ok( !!mem, "GlobalAlloc failed, error %lu\n", GetLastError() ); tmp_mem = GlobalReAlloc( mem, small_size + 1, GMEM_MOVEABLE ); ok( !!tmp_mem, "GlobalReAlloc failed, error %lu\n", GetLastError() ); - todo_wine ok( tmp_mem != mem, "GlobalReAlloc didn't relocate memory\n" ); ptr = GlobalLock( tmp_mem ); ok( !!ptr, "GlobalLock failed, error %lu\n", GetLastError() ); @@ -1832,8 +1818,8 @@ static void test_GlobalAlloc(void) { ok( !is_mem_entry( tmp_mem ), "unexpected moveable %p\n", tmp_mem ); if (flags == GMEM_MODIFY) ok( tmp_mem == mem, "GlobalReAlloc returned %p\n", tmp_mem ); - else if (flags != GMEM_MOVEABLE) todo_wine_if(!flags) ok( !tmp_mem, "GlobalReAlloc succeeded\n" ); - else todo_wine ok( tmp_mem != mem, "GlobalReAlloc returned %p\n", tmp_mem ); + else if (flags != GMEM_MOVEABLE) ok( !tmp_mem, "GlobalReAlloc succeeded\n" ); + else ok( tmp_mem != mem, "GlobalReAlloc returned %p\n", tmp_mem ); } else { @@ -1847,7 +1833,7 @@ static void test_GlobalAlloc(void)
size = GlobalSize( mem ); if (flags == GMEM_MOVEABLE) ok( size == 0 || broken( size == 1 ) /* w7 */, "GlobalSize returned %Iu\n", size ); - else todo_wine_if(!flags) ok( size == small_size, "GlobalSize returned %Iu\n", size ); + else ok( size == small_size, "GlobalSize returned %Iu\n", size );
mem = GlobalFree( mem ); ok( !mem, "GlobalFree failed, error %lu\n", GetLastError() ); @@ -2405,9 +2391,7 @@ static void test_LocalAlloc(void) ok( size == small_size, "LocalSize returned %Iu\n", size ); SetLastError( 0xdeadbeef ); tmp_mem = LocalReAlloc( mem, small_size, 0 ); - todo_wine ok( !tmp_mem, "LocalReAlloc succeeded\n" ); - todo_wine ok( GetLastError() == ERROR_NOT_ENOUGH_MEMORY, "got error %lu\n", GetLastError() ); if (tmp_mem) mem = tmp_mem; tmp_mem = LocalReAlloc( mem, 1024 * 1024, LMEM_MODIFY ); @@ -2423,7 +2407,6 @@ static void test_LocalAlloc(void) ok( !!mem, "LocalAlloc failed, error %lu\n", GetLastError() ); tmp_mem = LocalReAlloc( mem, small_size + 1, LMEM_MOVEABLE ); ok( !!tmp_mem, "LocalReAlloc failed, error %lu\n", GetLastError() ); - todo_wine ok( tmp_mem != mem, "LocalReAlloc didn't relocate memory\n" ); ptr = LocalLock( tmp_mem ); ok( !!ptr, "LocalLock failed, error %lu\n", GetLastError() ); @@ -2516,13 +2499,13 @@ static void test_LocalAlloc(void) tmp_mem = LocalReAlloc( mem, 0, flags ); ok( !is_mem_entry( tmp_mem ), "unexpected moveable %p\n", tmp_mem ); if (flags & LMEM_MODIFY) ok( tmp_mem == mem, "LocalReAlloc returned %p\n", tmp_mem ); - else if (flags != LMEM_MOVEABLE) todo_wine_if(!flags) ok( !tmp_mem, "LocalReAlloc succeeded\n" ); - else todo_wine ok( tmp_mem != mem, "LocalReAlloc returned %p\n", tmp_mem ); + else if (flags != LMEM_MOVEABLE) ok( !tmp_mem, "LocalReAlloc succeeded\n" ); + else ok( tmp_mem != mem, "LocalReAlloc returned %p\n", tmp_mem ); if (tmp_mem) mem = tmp_mem;
size = LocalSize( mem ); if (flags == LMEM_MOVEABLE) ok( size == 0 || broken( size == 1 ) /* w7 */, "LocalSize returned %Iu\n", size ); - else todo_wine_if(!flags) ok( size == small_size, "LocalSize returned %Iu\n", size ); + else ok( size == small_size, "LocalSize returned %Iu\n", size );
mem = LocalFree( mem ); ok( !mem, "LocalFree failed, error %lu\n", GetLastError() ); diff --git a/dlls/ntdll/heap.c b/dlls/ntdll/heap.c index e5e7044affe..dbda193f185 100644 --- a/dlls/ntdll/heap.c +++ b/dlls/ntdll/heap.c @@ -200,7 +200,8 @@ static const SIZE_T free_list_sizes[] = typedef struct DECLSPEC_ALIGN(BLOCK_ALIGN) tagSUBHEAP { SIZE_T __pad[sizeof(SIZE_T) / sizeof(DWORD) - 1]; - DWORD affinity; + WORD affinity; + WORD user_count; SIZE_T block_size; SIZE_T data_size; struct list entry; @@ -1081,10 +1082,61 @@ static DWORD next_thread_data_index(void) return index; }
+/* release thread blocks to the main heap free lists, clearing the split flags */ +static void heap_release_blocks_lfh( struct heap *heap, SUBHEAP *subheap ) +{ + struct block *block, *next; + + for (block = first_block( subheap ); block; block = next) + { + /* remove the flag one block at a time, so heap_free_block only merges with the previous */ + block_set_flags( block, BLOCK_FLAG_SPLIT, 0 ); + if ((next = next_block( subheap, block ))) block_set_flags( next, 0, BLOCK_FLAG_SPLIT ); + + if (block_get_flags( block ) & BLOCK_FLAG_FREE) + { + struct entry *entry = CONTAINING_RECORD( block, struct entry, block ); + list_remove( &entry->entry ); + heap_free_block( heap, 0, block ); + } + } +} + +/* release subheaps owned by current thread, clearing the subheap affinity */ +static void heap_release_subheaps_lfh( struct heap *heap ) +{ + SUBHEAP *subheap, *next; + + LIST_FOR_EACH_ENTRY_SAFE( subheap, next, &heap->subheap_list, SUBHEAP, entry ) + { + if (subheap_get_affinity( subheap ) == NtCurrentTeb()->HeapVirtualAffinity) + { + subheap_set_affinity( subheap, 0 ); + heap_release_blocks_lfh( heap, subheap ); + } + } +} + void heap_thread_detach(void) { + struct heap *heap; + DWORD index = NtCurrentTeb()->HeapVirtualAffinity - 1; if (index >= ARRAY_SIZE(thread_data_tid)) return; + + heap_lock( process_heap, 0 ); + + LIST_FOR_EACH_ENTRY( heap, &process_heap->entry, struct heap, entry ) + { + heap_lock( heap, 0 ); + heap_release_subheaps_lfh( heap ); + heap_unlock( heap, 0 ); + } + + heap_release_subheaps_lfh( process_heap ); + + heap_unlock( process_heap, 0 ); + InterlockedExchange( thread_data_tid + index, 0 ); }
@@ -1124,7 +1176,9 @@ static SUBHEAP *create_subheap( struct heap *heap, DWORD flags, SIZE_T total_siz block_size = (SIZE_T)ROUND_ADDR( subheap_size( subheap ) - subheap_overhead( subheap ), BLOCK_ALIGN - 1 ); block_init_free( first_block( subheap ), flags, subheap, block_size );
- list_add_head( &heap->subheap_list, &subheap->entry ); + heap_lock( heap, 0 ); + list_add_tail( &heap->subheap_list, &subheap->entry ); + heap_unlock( heap, 0 );
return subheap; } @@ -1722,11 +1776,127 @@ static SIZE_T heap_get_block_size( const struct heap *heap, ULONG flags, SIZE_T return block_size; }
+static void insert_free_block_lfh( struct heap *heap, struct entry *entry, struct heap_thread_data *thread_data ) +{ + SIZE_T block_size = block_get_size( &entry->block ), category = BLOCK_SIZE_CATEGORY( block_size ); + struct entry *free_list = thread_data->free_lists + category; + SUBHEAP *subheap = block_get_subheap( heap, &entry->block ); + + list_add_head( &free_list->entry, &entry->entry ); + if (!--subheap->user_count) + { + void *addr = subheap_base( subheap ); + struct block *block; + SIZE_T size = 0; + + for (block = first_block( subheap ); block; block = next_block( subheap, block )) + { + entry = (struct entry *)block; + list_remove( &entry->entry ); + } + + heap_lock( heap, 0 ); + list_remove( &subheap->entry ); + heap_unlock( heap, 0 ); + + NtFreeVirtualMemory( NtCurrentProcess(), &addr, &size, MEM_RELEASE ); + } +} + +static void flush_defer_list( struct heap *heap, ULONG flags, struct heap_thread_data *thread_data ) +{ + struct list pending = LIST_INIT( pending ); + struct entry *entry, *next; + + if (!thread_data->has_defer) return; + + heap_lock( heap, flags ); + list_move_tail( &pending, &thread_data->defer_list.entry ); + thread_data->has_defer = FALSE; + heap_unlock( heap, flags ); + + LIST_FOR_EACH_ENTRY_SAFE( entry, next, &pending, struct entry, entry ) + { + list_remove( &entry->entry ); + insert_free_block_lfh( heap, entry, thread_data ); + } +} + +static void split_block_lfh( struct heap *heap, ULONG flags, struct block *block, SIZE_T block_size, struct entry *free_list ) +{ + SIZE_T i, old_block_size = block_get_size( block ), count = min( 16, old_block_size / block_size ); + SUBHEAP *subheap = block_get_subheap( heap, block ); + struct entry *entry; + + if (!subheap_commit( heap, subheap, block, count * block_size )) return; + + for (i = 0; i < count; ++i) + { + block_init_free( block, flags, subheap, block_size ); + block_set_flags( block, 0, BLOCK_FLAG_SPLIT ); + old_block_size -= block_size; + + entry = (struct entry *)block; + if (i) list_add_head( &free_list->entry, &entry->entry ); + + block = next_block( subheap, block ); + } + + if ((entry = (struct entry *)block)) + { + block_init_free( block, flags, subheap, old_block_size ); + /* only if large enough to carve more blocks from it */ + if (old_block_size < block_size) list_init( &entry->entry ); + else list_add_tail( &free_list->entry, &entry->entry ); + } + + /* we've split the region entirely, we can consider releasing it */ + if (old_block_size < block_size || !block) subheap->user_count--; +} + +static struct block *find_free_block_lfh( struct heap *heap, ULONG flags, SIZE_T category, + struct heap_thread_data *thread_data ) +{ + SIZE_T block_size = BLOCK_CATEGORY_SIZE(category), commit_size, total_size; + WORD affinity = NtCurrentTeb()->HeapVirtualAffinity; + struct entry *entry, *free_list; + struct list *ptr; + SUBHEAP *subheap; + + free_list = thread_data->free_lists + category; + commit_size = sizeof(*subheap) + 16 * block_size + sizeof(*entry); + total_size = ROUND_SIZE( sizeof(*subheap) + 512 * block_size + sizeof(*entry), REGION_ALIGN ); + + if ((ptr = list_head( &free_list->entry ))) + { + entry = LIST_ENTRY( ptr, struct entry, entry ); + list_remove( &entry->entry ); + subheap = block_get_subheap( heap, &entry->block ); + } + else if (!(flags & HEAP_GROWABLE)) + return NULL; + else if (!(subheap = create_subheap( heap, flags, commit_size, total_size, affinity ))) + return NULL; + else + { + TRACE( "created new sub-heap %p of %#Ix bytes for heap %p\n", subheap, subheap_size( subheap ), heap ); + subheap->user_count = 1; /* avoid releasing the region until we've split it entirely */ + entry = first_block( subheap ); + } + + if (!(block_get_flags( &entry->block ) & BLOCK_FLAG_SPLIT)) + split_block_lfh( heap, flags, &entry->block, block_size, free_list ); + + subheap->user_count++; + return &entry->block; +} + static NTSTATUS heap_allocate_block_lfh( struct heap *heap, ULONG flags, SIZE_T block_size, SIZE_T size, void **ret ) { SIZE_T category = BLOCK_SIZE_CATEGORY( block_size ); struct heap_thread_data *thread_data; + struct block *block;
if (category >= ARRAY_SIZE(heap->stats)) return STATUS_UNSUCCESSFUL; if (heap->stats[category].live <= 0x10 && heap->stats[category].total <= 0x800) @@ -1743,7 +1913,12 @@ static NTSTATUS heap_allocate_block_lfh( struct heap *heap, ULONG flags, SIZE_T } if (!(thread_data = heap_get_thread_data( heap ))) return STATUS_UNSUCCESSFUL;
- return STATUS_NOT_CAPABLE; + if (!(block = find_free_block_lfh( heap, flags, category, thread_data ))) return STATUS_NO_MEMORY; + block_init_used( block, flags, size ); + flush_defer_list( heap, flags, thread_data ); + + *ret = block + 1; + return STATUS_SUCCESS; }
static NTSTATUS heap_allocate_block( struct heap *heap, ULONG flags, SIZE_T block_size, SIZE_T size, void **ret ) @@ -1806,6 +1981,40 @@ void *WINAPI DECLSPEC_HOTPATCH RtlAllocateHeap( HANDLE handle, ULONG flags, SIZE return ptr; }
+static NTSTATUS heap_free_block_lfh( struct heap *heap, ULONG flags, struct block *block, WORD block_affinity ) +{ + SIZE_T block_size = block_get_size( block ), category = BLOCK_SIZE_CATEGORY( block_size ); + ULONG affinity = NtCurrentTeb()->HeapVirtualAffinity; + SUBHEAP *subheap = block_get_subheap( heap, block ); + struct entry *entry = (struct entry *)block; + + valgrind_make_writable( &block, sizeof(*entry) ); + block_set_type( block, BLOCK_TYPE_FREE ); + block_set_flags( block, ~BLOCK_FLAG_SPLIT, BLOCK_FLAG_FREE ); + block_set_base( block, subheap_base( subheap ) ); + block_set_size( block, BLOCK_CATEGORY_SIZE(category) ); + mark_block_free( block + 1, BLOCK_CATEGORY_SIZE(category) - sizeof(*block), flags ); + + if (block_affinity == affinity) + { + struct heap_thread_data *thread_data = heap_get_thread_data( heap ); + insert_free_block_lfh( heap, entry, thread_data ); + } + else + { + heap_lock( heap, flags ); + /* the subheap affinity may be reset to zero when the owner thread is detached */ + if ((block_affinity = block_get_affinity( heap, block ))) + { + list_add_tail( &heap->tls[block_affinity - 1].defer_list.entry, &entry->entry ); + heap->tls[block_affinity - 1].has_defer = TRUE; + } + heap_unlock( heap, flags ); + return block_affinity ? STATUS_SUCCESS : STATUS_NOT_CAPABLE; + } + + return STATUS_SUCCESS; +}
/*********************************************************************** * RtlFreeHeap (NTDLL.@) @@ -1816,6 +2025,7 @@ BOOLEAN WINAPI DECLSPEC_HOTPATCH RtlFreeHeap( HANDLE handle, ULONG flags, void * struct heap *heap; ULONG heap_flags; NTSTATUS status; + WORD affinity;
if (!ptr) return TRUE;
@@ -1829,6 +2039,8 @@ BOOLEAN WINAPI DECLSPEC_HOTPATCH RtlFreeHeap( HANDLE handle, ULONG flags, void * status = heap_free_large( heap, heap_flags, block ); else if (!(block = heap_delay_free( heap, heap_flags, block ))) status = STATUS_SUCCESS; + else if ((affinity = block_get_affinity( heap, block ))) + status = heap_free_block_lfh( heap, heap_flags, block, affinity ); else { heap_lock( heap, heap_flags ); @@ -1848,6 +2060,7 @@ static NTSTATUS heap_resize_block( struct heap *heap, ULONG flags, struct block SIZE_T old_block_size; struct entry *entry; struct block *next; + WORD affinity;
if (block_get_flags( block ) & BLOCK_FLAG_LARGE) { @@ -1875,6 +2088,24 @@ static NTSTATUS heap_resize_block( struct heap *heap, ULONG flags, struct block
if (block_size >= HEAP_MIN_LARGE_BLOCK_SIZE) return STATUS_NO_MEMORY; /* growing small block to large block */
+ if ((affinity = block_get_affinity( heap, block ))) + { + if (affinity != NtCurrentTeb()->HeapVirtualAffinity) return STATUS_NO_MEMORY; /* resizing block owned by another thread */ + + /* as native LFH does it, different block size even though we could resize */ + if (ROUND_SIZE( *old_size, BLOCK_ALIGN - 1) != ROUND_SIZE( size, BLOCK_ALIGN - 1)) return STATUS_NO_MEMORY; + if (size >= *old_size) return STATUS_NO_MEMORY; + + valgrind_notify_resize( block + 1, *old_size, size ); + block_set_flags( block, BLOCK_FLAG_USER_MASK, BLOCK_USER_FLAGS( flags ) ); + block->tail_size = block_get_size( block ) - sizeof(*block) - size; + initialize_block( block, *old_size, size, flags ); + mark_block_tail( block, flags ); + + *ret = block + 1; + return STATUS_SUCCESS; + } + heap_lock( heap, flags );
if (block_size > old_block_size)
Hi,
It looks like your patch introduced the new failures shown below. Please investigate and fix them before resubmitting your patch. If they are not new, fixing them anyway would help a lot. Otherwise please ask for the known failures list to be updated.
The tests also ran into some preexisting test failures. If you know how to fix them that would be helpful. See the TestBot job for the details:
The full results can be found at: https://testbot.winehq.org/JobDetails.pl?Key=128834
Your paranoid android.
=== debian11 (32 bit report) ===
d2d1: d2d1: Timeout
d3d10core: d3d10core: Timeout
d3d10: effect: Timeout
d3d11: d3d11: Timeout
d3d8: visual: Timeout
d3d9: d3d9ex: Timeout device: Timeout
d3dcompiler_43: hlsl_d3d11: Timeout
d3dx10_34: d3dx10: Timeout
d3dx10_35: d3dx10: Timeout
d3dx10_36: d3dx10: Timeout
d3dx10_37: d3dx10: Timeout
d3dx10_38: d3dx10: Timeout
d3dx10_39: d3dx10: Timeout
d3dx10_40: d3dx10: Timeout
d3dx10_41: d3dx10: Timeout
d3dx10_42: d3dx10: Timeout
d3dx10_43: d3dx10: Timeout
ddraw: ddraw1: Timeout ddraw2: Timeout ddraw4: Timeout ddraw7: Timeout visual: Timeout
kernel32: heap: Timeout
ntoskrnl.exe: ntoskrnl: Timeout
qmgr: enum_files: Timeout enum_jobs: Timeout
Report validation errors: d3d9:visual crashed (c0000005) qmgr:file timeout
=== debian11 (build log) ===
WineRunWineTest.pl:error: The task timed out
=== debian11b (32 bit WoW report) ===
kernel32: heap: Timeout
=== debian11b (64 bit WoW report) ===
kernel32: heap.c:455: Test failed: HeapAlloc succeeded heap.c:456: Test failed: got error 3735928559 heap: Timeout
Jinoh Kang (@iamahuman) commented about dlls/ntdll/heap.c:
typedef struct DECLSPEC_ALIGN(BLOCK_ALIGN) tagSUBHEAP {
- SIZE_T __pad[sizeof(SIZE_T) / sizeof(DWORD)];
- SIZE_T __pad[sizeof(SIZE_T) / sizeof(DWORD) - 1];
The size expression evaluates to zero if `sizeof(SIZE_T) = sizeof(DWORD)`. Zero-sized arrays are a GCC/MSVC extension.
Jinoh Kang (@iamahuman) commented about dlls/ntdll/heap.c:
+struct heap_counters +{
- volatile BYTE live;
- volatile WORD total;
+};
+struct heap_thread_data +{
- struct entry free_lists[BLOCK_SIZE_CATEGORY_COUNT];
- /* requires heap lock to access */
- struct entry defer_list;
- volatile BOOL has_defer;
+};
+/* affinity to tid mapping array, defines the amount of supported concurrent threads */
```suggestion:-0+0 /* affinity to tid mapping array, defines the amount of * supported concurrent threads for threaded heaps. */ ```
-
Jinoh Kang (＠iamahuman) -
Marvin -
Rémi Bernon
-
Rémi Bernon (＠rbernon)