This is the mail archive of the
binutils@sourceware.org
mailing list for the binutils project.
Re: PATCH: PR ld/3111: LD very slow linking object files containing dwarf2 symbols
- From: "H. J. Lu" <hjl at lucon dot org>
- To: binutils at sources dot redhat dot com
- Date: Fri, 1 Sep 2006 10:47:31 -0700
- Subject: Re: PATCH: PR ld/3111: LD very slow linking object files containing dwarf2 symbols
- References: <20060829230012.GA19841@lucon.org>
On Tue, Aug 29, 2006 at 04:00:12PM -0700, H. J. Lu wrote:
> When we are comparing symbols in a section against another section in
> a different file, we swap in 2 symbol tables, sort them and free the
> symbol tables for each section. It is a O^2 problem. When there are
> many sections and symbols in a file, it can be very slow. This patch
> caches the symbol table unless --reduce-memory-overheads is used. The
> results for the testcase are
>
> /usr/bin/time ./ld -shared -o libfoo.so b2dynamic_nonlinear_solver.o b2dynamic_nonlinear_utile.o b2static_nonlinear_utile.o b2static_nonlinear_solver.o b2free_vibration_solver.o b2frequency_dependent_free_vibration_solver.o b2linearized_prebuckling_solver.o
> 1.06user 0.13system 0:01.20elapsed 99%CPU
> /usr/bin/time ./ld --reduce-memory-overheads -shared -o libbar.so b2dynamic_nonlinear_solver.o b2dynamic_nonlinear_utile.o b2static_nonlinear_utile.o b2static_nonlinear_solver.o b2free_vibration_solver.o b2frequency_dependent_free_vibration_solver.o b2linearized_prebuckling_solver.o
> 139.24user 8.93system 2:28.19elapsed 99%CPU
>
> It is about 130X speed up. The only concern I have is the memory
> overhead. It seems small for the testcase.
>
>
Here is a small update. I reset the buffer pointer after freeing it.
H.J.
---
bfd/
2006-08-29 H.J. Lu <hongjiu.lu@intel.com>
PR ld/3111
* elf-bfd.h (elf_obj_tdata): Add symbuf.
(_bfd_elf_section_already_linked): Add struct bfd_link_info *.
(_bfd_elf_check_kept_section): Likewise.
(bfd_elf_match_symbols_in_sections): Likewise.
* elf.c (assign_section_numbers): Updated to add
struct bfd_link_info *.
(bfd_elf_match_symbols_in_sections): Updated. Cache symbol
buffer if info->reduce_memory_overheads is false.
* elflink.c (match_group_member): Updated to add
struct bfd_link_info *.
(_bfd_elf_check_kept_section): Likewise.
(elf_link_input_bfd): Likewise.
(_bfd_elf_section_already_linked): Likewise.
(bfd_elf_final_link): Free symbol buffer if
info->reduce_memory_overheads is false.
* libbfd-in.h (_bfd_nolink_section_already_linked): Add
struct bfd_link_info *.
(_bfd_generic_section_already_linked): Likewise.
* libbfd.h: Regenerated.
* linker.c (bfd_section_already_linked): Add
struct bfd_link_info *.
(_bfd_generic_section_already_linked): Likewise.
* targets.c (bfd_target): Add struct bfd_link_info * to
_section_already_linked.
* bfd-in2.h: Regenerated.
include/
2006-08-29 H.J. Lu <hongjiu.lu@intel.com>
PR ld/3111
* bfdlink.h (bfd_link_info): Add reduce_memory_overheads.
ld/
2006-08-29 H.J. Lu <hongjiu.lu@intel.com>
PR ld/3111
* ld.h (args_type): Remove reduce_memory_overheads.
* ldlang.c (lang_map): Updated.
(section_already_linked): Likewise.
(print_input_section): Likewise.
* ldmain.c (main): Likewise.
* lexsup.c (parse_args): Likewise.
--- binutils/bfd/elf-bfd.h.symtab 2006-08-29 15:12:56.000000000 -0700
+++ binutils/bfd/elf-bfd.h 2006-08-29 15:12:56.000000000 -0700
@@ -1397,6 +1397,9 @@ struct elf_obj_tdata
/* Used to determine if the e_flags field has been initialized */
bfd_boolean flags_init;
+
+ /* Symbol buffer. */
+ Elf_Internal_Sym *symbuf;
};
#define elf_tdata(bfd) ((bfd) -> tdata.elf_obj_data)
@@ -1531,11 +1534,11 @@ extern bfd_boolean _bfd_elf_match_sectio
extern bfd_boolean bfd_elf_is_group_section
(bfd *, const struct bfd_section *);
extern void _bfd_elf_section_already_linked
- (bfd *, struct bfd_section *);
+ (bfd *, struct bfd_section *, struct bfd_link_info *);
extern void bfd_elf_set_group_contents
(bfd *, asection *, void *);
extern asection *_bfd_elf_check_kept_section
- (asection *);
+ (asection *, struct bfd_link_info *);
extern void _bfd_elf_link_just_syms
(asection *, struct bfd_link_info *);
extern bfd_boolean _bfd_elf_copy_private_header_data
@@ -1733,7 +1736,7 @@ extern bfd_boolean _bfd_elf_symbol_refs_
(struct elf_link_hash_entry *, struct bfd_link_info *, bfd_boolean);
extern bfd_boolean bfd_elf_match_symbols_in_sections
- (asection *sec1, asection *sec2);
+ (asection *, asection *, struct bfd_link_info *);
extern bfd_boolean _bfd_elf_setup_sections
(bfd *);
--- binutils/bfd/elf.c.symtab 2006-08-29 15:12:56.000000000 -0700
+++ binutils/bfd/elf.c 2006-08-29 15:12:56.000000000 -0700
@@ -3165,7 +3165,7 @@ assign_section_numbers (bfd *abfd, struc
s, s->owner);
/* Point to the kept section if it has the same
size as the discarded one. */
- kept = _bfd_elf_check_kept_section (s);
+ kept = _bfd_elf_check_kept_section (s, link_info);
if (kept == NULL)
{
bfd_set_error (bfd_error_bad_value);
@@ -8632,7 +8632,8 @@ elf_sym_name_compare (const void *arg1,
symbols. */
bfd_boolean
-bfd_elf_match_symbols_in_sections (asection *sec1, asection *sec2)
+bfd_elf_match_symbols_in_sections (asection *sec1, asection *sec2,
+ struct bfd_link_info *info)
{
bfd *bfd1, *bfd2;
const struct elf_backend_data *bed1, *bed2;
@@ -8690,21 +8691,37 @@ bfd_elf_match_symbols_in_sections (asect
if (symcount1 == 0 || symcount2 == 0)
return FALSE;
- isymbuf1 = bfd_elf_get_elf_syms (bfd1, hdr1, symcount1, 0,
- NULL, NULL, NULL);
- isymbuf2 = bfd_elf_get_elf_syms (bfd2, hdr2, symcount2, 0,
- NULL, NULL, NULL);
-
result = FALSE;
- if (isymbuf1 == NULL || isymbuf2 == NULL)
- goto done;
+ isymbuf1 = elf_tdata (bfd1)->symbuf;
+ isymbuf2 = elf_tdata (bfd2)->symbuf;
- /* Sort symbols by binding and section. Global definitions are at
- the beginning. */
- qsort (isymbuf1, symcount1, sizeof (Elf_Internal_Sym),
- elf_sort_elf_symbol);
- qsort (isymbuf2, symcount2, sizeof (Elf_Internal_Sym),
- elf_sort_elf_symbol);
+ if (isymbuf1 == NULL)
+ {
+ isymbuf1 = bfd_elf_get_elf_syms (bfd1, hdr1, symcount1, 0,
+ NULL, NULL, NULL);
+ if (isymbuf1 == NULL)
+ goto done;
+ /* Sort symbols by binding and section. Global definitions are at
+ the beginning. */
+ qsort (isymbuf1, symcount1, sizeof (Elf_Internal_Sym),
+ elf_sort_elf_symbol);
+ if (!info->reduce_memory_overheads)
+ elf_tdata (bfd1)->symbuf = isymbuf1;
+ }
+
+ if (isymbuf2 == NULL)
+ {
+ isymbuf2 = bfd_elf_get_elf_syms (bfd2, hdr2, symcount2, 0,
+ NULL, NULL, NULL);
+ if (isymbuf2 == NULL)
+ goto done;
+ /* Sort symbols by binding and section. Global definitions are at
+ the beginning. */
+ qsort (isymbuf2, symcount2, sizeof (Elf_Internal_Sym),
+ elf_sort_elf_symbol);
+ if (!info->reduce_memory_overheads)
+ elf_tdata (bfd2)->symbuf = isymbuf2;
+ }
/* Count definitions in the section. */
count1 = 0;
@@ -8788,10 +8805,13 @@ done:
free (symtable1);
if (symtable2)
free (symtable2);
- if (isymbuf1)
- free (isymbuf1);
- if (isymbuf2)
- free (isymbuf2);
+ if (info->reduce_memory_overheads)
+ {
+ if (isymbuf1)
+ free (isymbuf1);
+ if (isymbuf2)
+ free (isymbuf2);
+ }
return result;
}
--- binutils/bfd/elflink.c.symtab 2006-08-29 15:12:56.000000000 -0700
+++ binutils/bfd/elflink.c 2006-09-01 10:43:48.000000000 -0700
@@ -7193,14 +7193,15 @@ _bfd_elf_default_action_discarded (asect
/* Find a match between a section and a member of a section group. */
static asection *
-match_group_member (asection *sec, asection *group)
+match_group_member (asection *sec, asection *group,
+ struct bfd_link_info *info)
{
asection *first = elf_next_in_group (group);
asection *s = first;
while (s != NULL)
{
- if (bfd_elf_match_symbols_in_sections (s, sec))
+ if (bfd_elf_match_symbols_in_sections (s, sec, info))
return s;
s = elf_next_in_group (s);
@@ -7216,7 +7217,7 @@ match_group_member (asection *sec, asect
NULL. */
asection *
-_bfd_elf_check_kept_section (asection *sec)
+_bfd_elf_check_kept_section (asection *sec, struct bfd_link_info *info)
{
asection *kept;
@@ -7231,7 +7232,7 @@ _bfd_elf_check_kept_section (asection *s
if (kept != NULL)
{
if (elf_sec_group (sec) != NULL)
- kept = match_group_member (sec, kept);
+ kept = match_group_member (sec, kept, info);
if (kept != NULL && sec->size != kept->size)
kept = NULL;
}
@@ -7590,7 +7591,8 @@ elf_link_input_bfd (struct elf_final_lin
{
asection *kept;
- kept = _bfd_elf_check_kept_section (sec);
+ kept = _bfd_elf_check_kept_section (sec,
+ finfo->info);
if (kept != NULL)
{
*ps = kept;
@@ -8721,6 +8723,17 @@ bfd_elf_final_link (bfd *abfd, struct bf
}
}
+ /* Free symbol buffer if needed. */
+ if (!info->reduce_memory_overheads)
+ {
+ for (sub = info->input_bfds; sub != NULL; sub = sub->link_next)
+ if (elf_tdata (sub)->symbuf)
+ {
+ free (elf_tdata (sub)->symbuf);
+ elf_tdata (sub)->symbuf = NULL;
+ }
+ }
+
/* Output any global symbols that got converted to local in a
version script or due to symbol visibility. We do this in a
separate step since ELF requires all local symbols to appear
@@ -10184,7 +10197,8 @@ bfd_elf_discard_info (bfd *output_bfd, s
}
void
-_bfd_elf_section_already_linked (bfd *abfd, struct bfd_section * sec)
+_bfd_elf_section_already_linked (bfd *abfd, struct bfd_section *sec,
+ struct bfd_link_info *info)
{
flagword flags;
const char *name, *p;
@@ -10350,7 +10364,8 @@ _bfd_elf_section_already_linked (bfd *ab
if ((l->sec->flags & SEC_GROUP) == 0
&& bfd_coff_get_comdat_section (l->sec->owner, l->sec) == NULL
&& bfd_elf_match_symbols_in_sections (l->sec,
- elf_next_in_group (sec)))
+ elf_next_in_group (sec),
+ info))
{
elf_next_in_group (sec)->output_section = bfd_abs_section_ptr;
elf_next_in_group (sec)->kept_section = l->sec;
@@ -10371,7 +10386,7 @@ _bfd_elf_section_already_linked (bfd *ab
if (first != NULL
&& elf_next_in_group (first) == first
- && bfd_elf_match_symbols_in_sections (first, sec))
+ && bfd_elf_match_symbols_in_sections (first, sec, info))
{
sec->output_section = bfd_abs_section_ptr;
sec->kept_section = l->sec;
--- binutils/bfd/libbfd-in.h.symtab 2006-06-20 11:59:14.000000000 -0700
+++ binutils/bfd/libbfd-in.h 2006-08-29 15:12:56.000000000 -0700
@@ -410,7 +410,7 @@ extern bfd_boolean _bfd_generic_set_sect
#define _bfd_nolink_bfd_link_split_section \
((bfd_boolean (*) (bfd *, struct bfd_section *)) bfd_false)
#define _bfd_nolink_section_already_linked \
- ((void (*) (bfd *, struct bfd_section *)) bfd_void)
+ ((void (*) (bfd *, struct bfd_section *, struct bfd_link_info *)) bfd_void)
/* Routines to use for BFD_JUMP_TABLE_DYNAMIC for targets which do not
have dynamic symbols or relocs. Use BFD_JUMP_TABLE_DYNAMIC
@@ -524,7 +524,7 @@ extern bfd_boolean _bfd_generic_link_spl
(bfd *, struct bfd_section *);
extern void _bfd_generic_section_already_linked
- (bfd *, struct bfd_section *);
+ (bfd *, struct bfd_section *, struct bfd_link_info *);
/* Generic reloc_link_order processing routine. */
extern bfd_boolean _bfd_generic_reloc_link_order
--- binutils/bfd/linker.c.symtab 2006-03-16 12:37:43.000000000 -0800
+++ binutils/bfd/linker.c 2006-08-29 15:12:56.000000000 -0700
@@ -2877,14 +2877,15 @@ FUNCTION
bfd_section_already_linked
SYNOPSIS
- void bfd_section_already_linked (bfd *abfd, asection *sec);
+ void bfd_section_already_linked (bfd *abfd, asection *sec,
+ struct bfd_link_info *info);
DESCRIPTION
Check if @var{sec} has been already linked during a reloceatable
or final link.
-.#define bfd_section_already_linked(abfd, sec) \
-. BFD_SEND (abfd, _section_already_linked, (abfd, sec))
+.#define bfd_section_already_linked(abfd, sec, info) \
+. BFD_SEND (abfd, _section_already_linked, (abfd, sec, info))
.
*/
@@ -2970,7 +2971,8 @@ bfd_section_already_linked_table_free (v
/* This is used on non-ELF inputs. */
void
-_bfd_generic_section_already_linked (bfd *abfd, asection *sec)
+_bfd_generic_section_already_linked (bfd *abfd, asection *sec,
+ struct bfd_link_info *info ATTRIBUTE_UNUSED)
{
flagword flags;
const char *name;
--- binutils/bfd/targets.c.symtab 2006-08-21 15:20:21.000000000 -0700
+++ binutils/bfd/targets.c 2006-08-29 15:12:56.000000000 -0700
@@ -481,7 +481,8 @@ BFD_JUMP_TABLE macros.
.
. {* Check if SEC has been already linked during a reloceatable or
. final link. *}
-. void (*_section_already_linked) (bfd *, struct bfd_section *);
+. void (*_section_already_linked) (bfd *, struct bfd_section *,
+. struct bfd_link_info *);
.
. {* Routines to handle dynamic symbols and relocs. *}
.#define BFD_JUMP_TABLE_DYNAMIC(NAME) \
--- binutils/include/bfdlink.h.symtab 2006-08-29 15:12:56.000000000 -0700
+++ binutils/include/bfdlink.h 2006-08-29 15:12:56.000000000 -0700
@@ -335,6 +335,11 @@ struct bfd_link_info
/* TRUE if .gnu.hash section should be created. */
unsigned int emit_gnu_hash: 1;
+ /* If TRUE reduce memory overheads, at the expense of speed. This will
+ cause map file generation to use an O(N^2) algorithm and disable
+ caching ELF symbol buffer. */
+ unsigned int reduce_memory_overheads: 1;
+
/* What to do with unresolved symbols in an object file.
When producing executables the default is GENERATE_ERROR.
When producing shared libraries the default is IGNORE. The
--- binutils/ld/ld.h.symtab 2006-07-28 13:51:47.000000000 -0700
+++ binutils/ld/ld.h 2006-08-29 15:12:56.000000000 -0700
@@ -200,11 +200,6 @@ typedef struct {
behaviour of the linker. The new default behaviour is to reject such
input files. */
bfd_boolean accept_unknown_input_arch;
-
- /* If TRUE reduce memory overheads, at the expense of speed.
- This will cause map file generation to use an O(N^2) algorithm. */
- bfd_boolean reduce_memory_overheads;
-
} args_type;
extern args_type command_line;
--- binutils/ld/ldlang.c.symtab 2006-08-29 15:12:56.000000000 -0700
+++ binutils/ld/ldlang.c 2006-08-29 15:12:56.000000000 -0700
@@ -1795,7 +1795,7 @@ lang_map (void)
fprintf (config.map_file, _("\nLinker script and memory map\n\n"));
- if (! command_line.reduce_memory_overheads)
+ if (! link_info.reduce_memory_overheads)
{
obstack_begin (&map_obstack, 1000);
for (p = link_info.input_bfds; p != (bfd *) NULL; p = p->link_next)
@@ -1874,7 +1874,7 @@ init_os (lang_output_section_statement_t
s->bfd_section->output_section = s->bfd_section;
s->bfd_section->output_offset = 0;
- if (!command_line.reduce_memory_overheads)
+ if (!link_info.reduce_memory_overheads)
{
fat_section_userdata_type *new
= stat_alloc (sizeof (fat_section_userdata_type));
@@ -1967,7 +1967,7 @@ section_already_linked (bfd *abfd, asect
}
if (!(abfd->flags & DYNAMIC))
- bfd_section_already_linked (abfd, sec);
+ bfd_section_already_linked (abfd, sec, &link_info);
}
/* The wild routines.
@@ -3660,7 +3660,7 @@ print_input_section (asection *i)
if (i->output_section != NULL && i->output_section->owner == output_bfd)
{
- if (command_line.reduce_memory_overheads)
+ if (link_info.reduce_memory_overheads)
bfd_link_hash_traverse (link_info.hash, print_one_symbol, i);
else
print_all_symbols (i);
--- binutils/ld/ldmain.c.symtab 2006-08-29 15:12:56.000000000 -0700
+++ binutils/ld/ldmain.c 2006-08-29 15:12:56.000000000 -0700
@@ -256,7 +256,6 @@ main (int argc, char **argv)
command_line.warn_mismatch = TRUE;
command_line.check_section_addresses = TRUE;
command_line.accept_unknown_input_arch = FALSE;
- command_line.reduce_memory_overheads = FALSE;
sort_section = none;
@@ -320,6 +319,7 @@ main (int argc, char **argv)
link_info.gc_sections = FALSE;
link_info.print_gc_sections = FALSE;
link_info.dynamic = NULL;
+ link_info.reduce_memory_overheads = FALSE;
config.maxpagesize = 0;
config.commonpagesize = 0;
--- binutils/ld/lexsup.c.symtab 2006-08-29 15:12:56.000000000 -0700
+++ binutils/ld/lexsup.c 2006-08-29 15:12:56.000000000 -0700
@@ -1368,7 +1368,7 @@ parse_args (unsigned argc, char **argv)
break;
case OPTION_REDUCE_MEMORY_OVERHEADS:
- command_line.reduce_memory_overheads = TRUE;
+ link_info.reduce_memory_overheads = TRUE;
if (config.hash_table_size == 0)
config.hash_table_size = 1021;
break;