This serie is the first of a long serie for rewriting the PDB debug format support in dbnghelp.
For historical reasons, builtin dbghelp always fully loads every debug information (type, symbol...) for a given module, and transforms it into an internal representation (common to all debug formats: stabs, dwarf*, PDB...).
This turns out to be slow and memory hungry. For example, using a chromium based DLL, where PDB file breaks the 4G size barrier, load time of debug info is ~80s, and requires ~20GB of virtual memory.
Target of rewrite: - break monolothic approach in dbghelp to allow finer granularity in what's loaded (moving to a per compilation unit approach instead of full module) - reduce memory usage (for above example, current rewrite state is 20MB of virtual memory, and <4s of load time) - (potential) degradation of performance for some requests + may need to load/peruse additional information (not loaded at once) + request code hasn't been optimized for performance (on purpose), so that we can balance performance vs in-process memory (caching some more bits)
We start with some series about cleanup, fixes, optimisation in dbghelp (that will favor all debug formats, not only PDB).
This serie: - fixes a crash for .DBG file - fixes >4G PDB loading (so that we can measure improvements ;-) - introduces a new approach for internal vector representation (reduces memory usage), - some cleanup,
From: Eric Pouech epouech@codeweavers.com
Signed-off-by: Eric Pouech epouech@codeweavers.com --- dlls/dbghelp/msc.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/dlls/dbghelp/msc.c b/dlls/dbghelp/msc.c index 22ec5230f7a..a383afbe90c 100644 --- a/dlls/dbghelp/msc.c +++ b/dlls/dbghelp/msc.c @@ -4440,8 +4440,9 @@ static BOOL codeview_process_info(const struct process *pcs, if (ret) { msc_dbg->module->module.CVSig = *signature; - memcpy(msc_dbg->module->module.CVData, msc_dbg->root, - sizeof(msc_dbg->module->module.CVData)); + if (*signature == CODEVIEW_RSDS_SIG) + memcpy(msc_dbg->module->module.CVData, msc_dbg->root, + sizeof(msc_dbg->module->module.CVData)); } return ret; }
From: Eric Pouech epouech@codeweavers.com
Signed-off-by: Eric Pouech epouech@codeweavers.com --- dlls/dbghelp/msc.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/dlls/dbghelp/msc.c b/dlls/dbghelp/msc.c index a383afbe90c..fd773dbf9a6 100644 --- a/dlls/dbghelp/msc.c +++ b/dlls/dbghelp/msc.c @@ -3071,12 +3071,12 @@ static void* pdb_ds_read(const struct PDB_DS_HEADER* pdb, const UINT *block_list if (!size) return NULL;
num_blocks = (size + pdb->block_size - 1) / pdb->block_size; - buffer = HeapAlloc(GetProcessHeap(), 0, num_blocks * pdb->block_size); + buffer = HeapAlloc(GetProcessHeap(), 0, (SIZE_T)num_blocks * pdb->block_size); if (!buffer) return NULL;
for (i = 0; i < num_blocks; i++) memcpy(buffer + i * pdb->block_size, - (const char*)pdb + block_list[i] * pdb->block_size, pdb->block_size); + (const char*)pdb + (DWORD_PTR)block_list[i] * pdb->block_size, pdb->block_size);
return buffer; } @@ -3521,7 +3521,7 @@ static BOOL pdb_init(struct pdb_file_info* pdb_file, const char* image) struct PDB_DS_ROOT* root; struct PDB_DS_TOC* ds_toc;
- ds_toc = pdb_ds_read(pdb, (const UINT*)((const char*)pdb + pdb->toc_block * pdb->block_size), + ds_toc = pdb_ds_read(pdb, (const UINT*)((const char*)pdb + (DWORD_PTR)pdb->toc_block * pdb->block_size), pdb->toc_size); if (!ds_toc) { @@ -3635,7 +3635,7 @@ DWORD pdb_get_file_indexinfo(void* image, DWORD size, SYMSRV_INDEX_INFOW* info) struct PDB_DS_ROOT* root; DWORD ec = ERROR_SUCCESS;
- ds_toc = pdb_ds_read(pdb, (const UINT*)((const char*)pdb + pdb->toc_block * pdb->block_size), + ds_toc = pdb_ds_read(pdb, (const UINT*)((const char*)pdb + (DWORD_PTR)pdb->toc_block * pdb->block_size), pdb->toc_size); root = pdb_read_ds_stream(pdb, ds_toc, 1); if (!root)
From: Eric Pouech epouech@codeweavers.com
Signed-off-by: Eric Pouech epouech@codeweavers.com --- dlls/dbghelp/dbghelp_private.h | 1 + dlls/dbghelp/dwarf.c | 20 ++++++++++++-------- dlls/dbghelp/storage.c | 9 +++++++++ 3 files changed, 22 insertions(+), 8 deletions(-)
diff --git a/dlls/dbghelp/dbghelp_private.h b/dlls/dbghelp/dbghelp_private.h index 42458306571..63b2b8862de 100644 --- a/dlls/dbghelp/dbghelp_private.h +++ b/dlls/dbghelp/dbghelp_private.h @@ -45,6 +45,7 @@ void* pool_alloc(struct pool* a, size_t len) __WINE_ALLOC_SIZE(2) __WINE_MALL void* pool_realloc(struct pool* a, void* ptr, size_t len) __WINE_ALLOC_SIZE(3); char* pool_strdup(struct pool* a, const char* str) __WINE_MALLOC; WCHAR* pool_wcsdup(struct pool* a, const WCHAR* str) __WINE_MALLOC; +void pool_free(struct pool* a, void* ptr);
struct vector { diff --git a/dlls/dbghelp/dwarf.c b/dlls/dbghelp/dwarf.c index 84e861890d5..ebf71a4d148 100644 --- a/dlls/dbghelp/dwarf.c +++ b/dlls/dbghelp/dwarf.c @@ -2858,7 +2858,7 @@ static dwarf2_parse_context_t* dwarf2_locate_cu(dwarf2_parse_module_context_t* m const BYTE* where; for (i = 0; i < module_ctx->unit_contexts.num_elts; ++i) { - ctx = vector_at(&module_ctx->unit_contexts, i); + ctx = *(dwarf2_parse_context_t**)vector_at(&module_ctx->unit_contexts, i); where = module_ctx->sections[ctx->section].address + ref; if (where >= ctx->traverse_DIE.data && where < ctx->traverse_DIE.end_data) return ctx; @@ -4114,7 +4114,7 @@ static BOOL dwarf2_load_CU_module(dwarf2_parse_module_context_t* module_ctx, str module_ctx->module = module; module_ctx->thunks = thunks; module_ctx->load_offset = load_offset; - vector_init(&module_ctx->unit_contexts, sizeof(dwarf2_parse_context_t), 16); + vector_init(&module_ctx->unit_contexts, sizeof(dwarf2_parse_context_t*), 16); module_ctx->cu_versions = 0;
/* phase I: parse all CU heads */ @@ -4122,10 +4122,13 @@ static BOOL dwarf2_load_CU_module(dwarf2_parse_module_context_t* module_ctx, str mod_ctx.end_data = mod_ctx.data + sections[section_debug].size; while (mod_ctx.data < mod_ctx.end_data) { - dwarf2_parse_context_t* unit_ctx = vector_add(&module_ctx->unit_contexts, &module_ctx->module->pool); + dwarf2_parse_context_t **punit_ctx = vector_add(&module_ctx->unit_contexts, &module_ctx->module->pool);
- unit_ctx->module_ctx = module_ctx; - dwarf2_parse_compilation_unit_head(unit_ctx, &mod_ctx); + if (!(*punit_ctx = pool_alloc(&module_ctx->module->pool, sizeof(dwarf2_parse_context_t)))) + return FALSE; + + (*punit_ctx)->module_ctx = module_ctx; + dwarf2_parse_compilation_unit_head(*punit_ctx, &mod_ctx); }
/* phase2: load content of all CU @@ -4135,7 +4138,7 @@ static BOOL dwarf2_load_CU_module(dwarf2_parse_module_context_t* module_ctx, str */ if (!is_dwz) for (i = 0; i < module_ctx->unit_contexts.num_elts; ++i) - dwarf2_parse_compilation_unit((dwarf2_parse_context_t*)vector_at(&module_ctx->unit_contexts, i)); + dwarf2_parse_compilation_unit(*(dwarf2_parse_context_t**)vector_at(&module_ctx->unit_contexts, i));
return TRUE; } @@ -4189,9 +4192,10 @@ static BOOL dwarf2_unload_CU_module(dwarf2_parse_module_context_t* module_ctx) unsigned i; for (i = 0; i < module_ctx->unit_contexts.num_elts; ++i) { - dwarf2_parse_context_t* unit = vector_at(&module_ctx->unit_contexts, i); - if (unit->status != UNIT_ERROR) + dwarf2_parse_context_t* unit = *(dwarf2_parse_context_t**)vector_at(&module_ctx->unit_contexts, i); + if (unit && unit->status != UNIT_ERROR) pool_destroy(&unit->pool); + pool_free(&module_ctx->module->pool, unit); } dwarf2_unload_dwz(module_ctx->dwz); return TRUE; diff --git a/dlls/dbghelp/storage.c b/dlls/dbghelp/storage.c index f7c08adab8a..2344f0b848c 100644 --- a/dlls/dbghelp/storage.c +++ b/dlls/dbghelp/storage.c @@ -48,6 +48,15 @@ void* pool_realloc(struct pool* pool, void* ptr, size_t len) return ptr ? HeapReAlloc(pool->heap, 0, ptr, len) : pool_alloc(pool, len); }
+void pool_free(struct pool* pool, void* ptr) +{ +#ifdef USE_STATS + if (ptr) + mem_stats_down(&pool->stats, HeapSize(pool->heap, 0, ptr)); +#endif + HeapFree(pool->heap, 0, ptr); +} + char* pool_strdup(struct pool* pool, const char* str) { char* ret;
From: Eric Pouech epouech@codeweavers.com
Introduce a new model for vector allocation: - No longer requiring initial size (using a pure quadratic growth), - No longer providing stability of element address across add operations.
This allows some reduction in memory usage.
Signed-off-by: Eric Pouech epouech@codeweavers.com --- dlls/dbghelp/dwarf.c | 8 ++++---- dlls/dbghelp/module.c | 6 +++--- dlls/dbghelp/msc.c | 2 +- dlls/dbghelp/storage.c | 42 ++++++++++++++++++++++++++++++++++++------ dlls/dbghelp/symbol.c | 10 +++++----- dlls/dbghelp/type.c | 6 +++--- 6 files changed, 52 insertions(+), 22 deletions(-)
diff --git a/dlls/dbghelp/dwarf.c b/dlls/dbghelp/dwarf.c index ebf71a4d148..156f72343fe 100644 --- a/dlls/dbghelp/dwarf.c +++ b/dlls/dbghelp/dwarf.c @@ -1369,7 +1369,7 @@ static BOOL dwarf2_read_one_debug_info(dwarf2_parse_context_t* ctx, else di->data = NULL; if (abbrev->have_child) { - vector_init(&di->children, sizeof(dwarf2_debug_info_t*), 16); + vector_init(&di->children, sizeof(dwarf2_debug_info_t*), 0); while (traverse->data < traverse->end_data) { if (!dwarf2_read_one_debug_info(ctx, traverse, di, &child)) return FALSE; @@ -2685,7 +2685,7 @@ static BOOL dwarf2_parse_line_numbers(dwarf2_parse_context_t* ctx, opcode_len = traverse.data; traverse.data += opcode_base - 1;
- vector_init(&dirs, sizeof(const char*), 4); + vector_init(&dirs, sizeof(const char*), 0); p = vector_add(&dirs, &ctx->pool); *p = compile_dir ? compile_dir : "."; while (traverse.data < traverse.end_data && *traverse.data) @@ -2712,7 +2712,7 @@ static BOOL dwarf2_parse_line_numbers(dwarf2_parse_context_t* ctx, } traverse.data++;
- vector_init(&files, sizeof(unsigned), 16); + vector_init(&files, sizeof(unsigned), 0); while (traverse.data < traverse.end_data && *traverse.data) { unsigned int dir_index, mod_time; @@ -4114,7 +4114,7 @@ static BOOL dwarf2_load_CU_module(dwarf2_parse_module_context_t* module_ctx, str module_ctx->module = module; module_ctx->thunks = thunks; module_ctx->load_offset = load_offset; - vector_init(&module_ctx->unit_contexts, sizeof(dwarf2_parse_context_t*), 16); + vector_init(&module_ctx->unit_contexts, sizeof(dwarf2_parse_context_t*), 0); module_ctx->cu_versions = 0;
/* phase I: parse all CU heads */ diff --git a/dlls/dbghelp/module.c b/dlls/dbghelp/module.c index 3465d9ac396..80236d39ee3 100644 --- a/dlls/dbghelp/module.c +++ b/dlls/dbghelp/module.c @@ -197,14 +197,14 @@ struct module* module_new(struct process* pcs, const WCHAR* name, module->cpu = dbghelp_current_cpu; module->debug_format_bitmask = 0;
- vector_init(&module->vsymt, sizeof(struct symt*), 128); - vector_init(&module->vcustom_symt, sizeof(struct symt*), 16); + vector_init(&module->vsymt, sizeof(struct symt*), 0); + vector_init(&module->vcustom_symt, sizeof(struct symt*), 0); /* FIXME: this seems a bit too high (on a per module basis) * need some statistics about this */ hash_table_init(&module->pool, &module->ht_symbols, 4096); hash_table_init(&module->pool, &module->ht_types, 4096); - vector_init(&module->vtypes, sizeof(struct symt*), 32); + vector_init(&module->vtypes, sizeof(struct symt*), 0);
module->sources_used = 0; module->sources_alloc = 0; diff --git a/dlls/dbghelp/msc.c b/dlls/dbghelp/msc.c index fd773dbf9a6..2eadde29485 100644 --- a/dlls/dbghelp/msc.c +++ b/dlls/dbghelp/msc.c @@ -4187,7 +4187,7 @@ static void pev_init(struct pevaluator* pev, struct cpu_stack_walk* csw, { pev->csw = csw; pool_init(&pev->pool, 512); - vector_init(&pev->stack, sizeof(char*), 8); + vector_init(&pev->stack, sizeof(char*), 0); pev->stk_index = 0; hash_table_init(&pev->pool, &pev->values, 8); pev->error[0] = '\0'; diff --git a/dlls/dbghelp/storage.c b/dlls/dbghelp/storage.c index 2344f0b848c..0589ed699ee 100644 --- a/dlls/dbghelp/storage.c +++ b/dlls/dbghelp/storage.c @@ -74,10 +74,11 @@ WCHAR* pool_wcsdup(struct pool* pool, const WCHAR* str) void vector_init(struct vector* v, unsigned esz, unsigned bucket_sz) { v->buckets = NULL; - /* align size on DWORD boundaries */ - v->elt_size = (esz + 3) & ~3; + /* align size */ + v->elt_size = (esz + sizeof(void*) - 1) & ~(sizeof(void*) - 1); switch (bucket_sz) { + case 0: v->shift = 0; break; /* special case see below */ case 2: v->shift = 1; break; case 4: v->shift = 2; break; case 8: v->shift = 3; break; @@ -102,15 +103,44 @@ unsigned vector_length(const struct vector* v)
void* vector_at(const struct vector* v, unsigned pos) { - unsigned o; - if (pos >= v->num_elts) return NULL; - o = pos & ((1 << v->shift) - 1); - return (char*)v->buckets[pos >> v->shift] + o * v->elt_size; + if (v->shift) + { + unsigned o = pos & ((1 << v->shift) - 1); + return (char*)v->buckets[pos >> v->shift] + o * v->elt_size; + } + else + { + return (char*)v->buckets + pos * v->elt_size; + } }
void* vector_add(struct vector* v, struct pool* pool) { + if (!v->shift) + { + if (v->num_elts == 1024) + { + /* we'll need a second bucket, so go directly for it */ + void **new = pool_alloc(pool, 2 * sizeof(void*)); + if (!new) return NULL; + *new = v->buckets; + v->buckets = new; + v->num_buckets = 1; + v->buckets_allocated = 2; + v->shift = 10; + } + else + { + if (!v->num_elts || !(v->num_elts & (v->num_elts - 1))) + { + void *new = pool_realloc(pool, v->buckets, (v->num_elts ? v->num_elts * 2 : 1) * v->elt_size); + if (!new) return NULL; + v->buckets = new; + } + return vector_at(v, v->num_elts++); + } + } if (v->num_elts == (v->num_buckets << v->shift)) { if (v->num_buckets == v->buckets_allocated) diff --git a/dlls/dbghelp/symbol.c b/dlls/dbghelp/symbol.c index 38382b006d7..7c8acf8e68c 100644 --- a/dlls/dbghelp/symbol.c +++ b/dlls/dbghelp/symbol.c @@ -226,7 +226,7 @@ struct symt_module* symt_new_module(struct module* module) { sym->symt.tag = SymTagExe; sym->module = module; - vector_init(&sym->vchildren, sizeof(struct symt*), 8); + vector_init(&sym->vchildren, sizeof(struct symt*), 0); } return sym; } @@ -244,7 +244,7 @@ struct symt_compiland* symt_new_compiland(struct module* module, unsigned src_id sym->container = module->top; sym->address = 0; sym->source = src_idx; - vector_init(&sym->vchildren, sizeof(struct symt*), 32); + vector_init(&sym->vchildren, sizeof(struct symt*), 0); sym->user = NULL; p = vector_add(&module->top->vchildren, &module->pool); *p = sym; @@ -333,8 +333,8 @@ static struct symt_function* init_function_or_inlinesite(struct module* module, sym->hash_elt.name = pool_strdup(&module->pool, name); sym->container = container; sym->type = sig_type; - vector_init(&sym->vlines, sizeof(struct line_info), tag == SymTagFunction ? 8 : 4); - vector_init(&sym->vchildren, sizeof(struct symt*), 8); + vector_init(&sym->vlines, sizeof(struct line_info), 0); + vector_init(&sym->vchildren, sizeof(struct symt*), 0); sym->num_ranges = num_ranges; } return sym; @@ -548,7 +548,7 @@ struct symt_block* symt_open_func_block(struct module* module, block->symt.tag = SymTagBlock; block->num_ranges = num_ranges; block->container = parent_block ? &parent_block->symt : &func->symt; - vector_init(&block->vchildren, sizeof(struct symt*), 4); + vector_init(&block->vchildren, sizeof(struct symt*), 0); if (parent_block) p = vector_add(&parent_block->vchildren, &module->pool); else diff --git a/dlls/dbghelp/type.c b/dlls/dbghelp/type.c index b89c63c5b3d..9a1482e73ab 100644 --- a/dlls/dbghelp/type.c +++ b/dlls/dbghelp/type.c @@ -260,7 +260,7 @@ struct symt_udt* symt_new_udt(struct module* module, const char* typename, sym->hash_elt.name = pool_strdup(&module->pool, typename); hash_table_add(&module->ht_types, &sym->hash_elt); } else sym->hash_elt.name = NULL; - vector_init(&sym->vchildren, sizeof(struct symt*), 8); + vector_init(&sym->vchildren, sizeof(struct symt*), 0); symt_add_type(module, &sym->symt); } return sym; @@ -345,7 +345,7 @@ struct symt_enum* symt_new_enum(struct module* module, const char* typename, hash_table_add(&module->ht_types, &sym->hash_elt); } else sym->hash_elt.name = NULL; sym->base_type = basetype; - vector_init(&sym->vchildren, sizeof(struct symt*), 8); + vector_init(&sym->vchildren, sizeof(struct symt*), 0); symt_add_type(module, &sym->symt); } return sym; @@ -404,7 +404,7 @@ struct symt_function_signature* symt_new_function_signature(struct module* modul { sym->symt.tag = SymTagFunctionType; sym->rettype = ret_type; - vector_init(&sym->vchildren, sizeof(struct symt*), 4); + vector_init(&sym->vchildren, sizeof(struct symt*), 0); sym->call_conv = call_conv; symt_add_type(module, &sym->symt); }
From: Eric Pouech epouech@codeweavers.com
We can't (no longer) get inlined functions here.
Signed-off-by: Eric Pouech epouech@codeweavers.com --- dlls/dbghelp/symbol.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/dlls/dbghelp/symbol.c b/dlls/dbghelp/symbol.c index 7c8acf8e68c..961f77094b5 100644 --- a/dlls/dbghelp/symbol.c +++ b/dlls/dbghelp/symbol.c @@ -1910,9 +1910,9 @@ static BOOL get_line_from_addr(HANDLE hProcess, DWORD64 addr, struct symt_ht* symt;
if (!module_init_pair(&pair, hProcess, addr)) return FALSE; - if ((symt = symt_find_symbol_at(pair.effective, addr)) == NULL) return FALSE; + symt = symt_find_symbol_at(pair.effective, addr);
- if (symt->symt.tag != SymTagFunction && symt->symt.tag != SymTagInlineSite) return FALSE; + if (!symt_check_tag(&symt->symt, SymTagFunction)) return FALSE; return get_line_from_function(&pair, (struct symt_function*)symt, addr, pdwDisplacement, intl); }
-
Eric Pouech -
eric pouech (＠epo)