@@ -1874,6 +1874,9 @@ public:
if we did not do any inter-procedural code movement. */
unsigned used_by_single_function : 1;
+ /* Set if -fsection-anchors is set. */
+ unsigned section_anchor : 1;
+
private:
/* Assemble thunks and aliases associated to varpool node. */
void assemble_aliases (void);
@@ -800,6 +800,9 @@ varpool_node::finalize_decl (tree decl)
it is available to notice_global_symbol. */
node->definition = true;
notice_global_symbol (decl);
+
+ node->section_anchor = flag_section_anchors;
+
if (TREE_THIS_VOLATILE (decl) || DECL_PRESERVE_P (decl)
/* Traditionally we do not eliminate static variables when not
optimizing and when not doing toplevel reoder. */
@@ -1590,6 +1590,10 @@ fira-algorithm=
Common Joined RejectNegative Enum(ira_algorithm) Var(flag_ira_algorithm) Init(IRA_ALGORITHM_CB) Optimization
-fira-algorithm=[CB|priority] Set the used IRA algorithm.
+fipa-increase_alignment
+Common Report Var(flag_ipa_increase_alignment) Init(0) Optimization
+Option to gate increase_alignment ipa pass.
+
Enum
Name(ira_algorithm) Type(enum ira_algorithm) UnknownError(unknown IRA algorithm %qs)
@@ -2133,7 +2137,7 @@ Common Report Var(flag_sched_dep_count_heuristic) Init(1) Optimization
Enable the dependent count heuristic in the scheduler.
fsection-anchors
-Common Report Var(flag_section_anchors) Optimization
+Common Report Var(flag_section_anchors)
Access data in the same section from shared anchor points.
fsee
@@ -8252,6 +8252,8 @@ aarch64_override_options (void)
aarch64_register_fma_steering ();
+ /* Enable increase_alignment pass. */
+ flag_ipa_increase_alignment = 1;
}
/* Implement targetm.override_options_after_change. */
@@ -3458,6 +3458,9 @@ arm_option_override (void)
/* Init initial mode for testing. */
thumb_flipper = TARGET_THUMB;
+
+ /* Enable increase_alignment pass. */
+ flag_ipa_increase_alignment = 1;
}
static void
@@ -5011,6 +5011,9 @@ rs6000_option_override (void)
= { pass_analyze_swaps, "cse1", 1, PASS_POS_INSERT_BEFORE };
register_pass (&analyze_swaps_info);
+
+ /* Enable increase_alignment pass. */
+ flag_ipa_increase_alignment = 1;
}
@@ -627,6 +627,7 @@ lto_output_varpool_node (struct lto_simple_output_block *ob, varpool_node *node,
bp_pack_value (&bp, node->tls_model, 3);
bp_pack_value (&bp, node->used_by_single_function, 1);
bp_pack_value (&bp, node->need_bounds_init, 1);
+ bp_pack_value (&bp, node->section_anchor, 1);
streamer_write_bitpack (&bp);
group = node->get_comdat_group ();
@@ -1401,6 +1402,7 @@ input_varpool_node (struct lto_file_decl_data *file_data,
node->tls_model = (enum tls_model)bp_unpack_value (&bp, 3);
node->used_by_single_function = (enum tls_model)bp_unpack_value (&bp, 1);
node->need_bounds_init = bp_unpack_value (&bp, 1);
+ node->section_anchor = bp_unpack_value (&bp, 1);
group = read_identifier (ib);
if (group)
{
@@ -52,7 +52,8 @@ const char *lto_section_name[LTO_N_SECTION_TYPES] =
"icf",
"offload_table",
"mode_table",
- "hsa"
+ "hsa",
+ "increase_alignment"
};
@@ -245,6 +245,7 @@ enum lto_section_type
LTO_section_offload_table,
LTO_section_mode_table,
LTO_section_ipa_hsa,
+ LTO_section_increase_alignment,
LTO_N_SECTION_TYPES /* Must be last. */
};
@@ -138,12 +138,12 @@ along with GCC; see the file COPYING3. If not see
PUSH_INSERT_PASSES_WITHIN (pass_ipa_tree_profile)
NEXT_PASS (pass_feedback_split_functions);
POP_INSERT_PASSES ()
- NEXT_PASS (pass_ipa_increase_alignment);
NEXT_PASS (pass_ipa_tm);
NEXT_PASS (pass_ipa_lower_emutls);
TERMINATE_PASS_LIST (all_small_ipa_passes)
INSERT_PASSES_AFTER (all_regular_ipa_passes)
+ NEXT_PASS (pass_ipa_increase_alignment);
NEXT_PASS (pass_ipa_whole_program_visibility);
NEXT_PASS (pass_ipa_profile);
NEXT_PASS (pass_ipa_icf);
new file mode 100644
@@ -0,0 +1,25 @@
+/* { dg-do compile } */
+/* { dg-require-effective-target section_anchors } */
+/* { dg-require-effective-target vect_int } */
+
+#define N 32
+
+/* Clone of section-anchors-vect-70.c with foo() having -fno-tree-loop-vectorize. */
+
+static struct A {
+ int p1, p2;
+ int e[N];
+} a, b, c;
+
+__attribute__((optimize("-fno-tree-loop-vectorize")))
+int foo(void)
+{
+ for (int i = 0; i < N; i++)
+ a.e[i] = b.e[i] + c.e[i];
+
+ return a.e[0];
+}
+
+/* { dg-final { scan-ipa-dump-times "Increasing alignment of decl" 0 "increase_alignment" { target aarch64*-*-* } } } */
+/* { dg-final { scan-ipa-dump-times "Increasing alignment of decl" 0 "increase_alignment" { target powerpc64*-*-* } } } */
+/* { dg-final { scan-ipa-dump-times "Increasing alignment of decl" 0 "increase_alignment" { target arm*-*-* } } } */
@@ -483,7 +483,7 @@ extern simple_ipa_opt_pass *make_pass_local_optimization_passes (gcc::context *c
extern ipa_opt_pass_d *make_pass_ipa_whole_program_visibility (gcc::context
*ctxt);
-extern simple_ipa_opt_pass *make_pass_ipa_increase_alignment (gcc::context
+extern ipa_opt_pass_d *make_pass_ipa_increase_alignment (gcc::context
*ctxt);
extern ipa_opt_pass_d *make_pass_ipa_inline (gcc::context *ctxt);
extern simple_ipa_opt_pass *make_pass_ipa_free_lang_data (gcc::context *ctxt);
@@ -75,7 +75,10 @@ along with GCC; see the file COPYING3. If not see
#include "tree-ssa-propagate.h"
#include "dbgcnt.h"
#include "tree-scalar-evolution.h"
-
+#include "tree-ssa-loop.h"
+#include "tree-streamer.h"
+#include "stringpool.h"
+#include "gimple-pretty-print.h"
/* Loop or bb location. */
source_location vect_location;
@@ -796,6 +799,7 @@ make_pass_slp_vectorize (gcc::context *ctxt)
static unsigned get_vec_alignment_for_type (tree);
static hash_map<tree, unsigned> *type_align_map;
+static hash_set<varpool_node *> *vars = NULL;
/* Return alignment of array's vector type corresponding to scalar type.
0 if no vector type exists. */
@@ -899,46 +903,186 @@ get_vec_alignment_for_type (tree type)
return (alignment > TYPE_ALIGN (type)) ? alignment : 0;
}
+/* Return true if alignment should be increased for this vnode.
+ This is done if every function that referring to vnode
+ has flag_tree_loop_vectorize set. */
+
+static bool
+increase_alignment_p (varpool_node *vnode)
+{
+ ipa_ref *ref;
+ cgraph_node *cnode;
+
+ /* FIXME: Is this really necessary, since we check if cnode has
+ flag_tree_loop_vectorize_set during analysis phase in
+ increase_alignment_analyze_function ? */
+
+ for (int i = 0; vnode->iterate_referring (i, ref); i++)
+ /* Walk those functions that read/write/take address of vnode. */
+ if ((cnode = dyn_cast<cgraph_node *> (ref->referring)))
+ if (!opt_for_fn (cnode->decl, flag_tree_loop_vectorize))
+ return false;
+
+ return true;
+}
+
+static bool
+increase_alignment_callback (varpool_node *vnode, void *data ATTRIBUTE_UNUSED)
+{
+ tree decl = vnode->decl;
+
+ if (vnode->section_anchor
+ && decl_in_symtab_p (decl)
+ && symtab_node::get (decl)->can_increase_alignment_p ()
+ && !DECL_USER_ALIGN (decl)
+ && increase_alignment_p (vnode))
+ {
+ unsigned alignment = get_vec_alignment_for_type (TREE_TYPE (decl));
+ if (alignment && vect_can_force_dr_alignment_p (decl, alignment))
+ {
+ vnode->increase_alignment (alignment);
+ dump_printf (MSG_NOTE, "Increasing alignment of decl: ");
+ dump_generic_expr (MSG_NOTE, TDF_SLIM, decl);
+ dump_printf (MSG_NOTE, "\n");
+ }
+ }
+
+ return true;
+}
+
/* Entry point to increase_alignment pass. */
static unsigned int
increase_alignment (void)
{
- varpool_node *vnode;
-
vect_location = UNKNOWN_LOCATION;
type_align_map = new hash_map<tree, unsigned>;
/* Increase the alignment of all global arrays for vectorization. */
- FOR_EACH_DEFINED_VARIABLE (vnode)
+ for (hash_set<varpool_node *>::iterator it = vars->begin (); it != vars->end (); ++it)
{
- tree decl = vnode->decl;
- unsigned int alignment;
-
- if ((decl_in_symtab_p (decl)
- && !symtab_node::get (decl)->can_increase_alignment_p ())
- || DECL_USER_ALIGN (decl) || DECL_ARTIFICIAL (decl))
- continue;
-
- alignment = get_vec_alignment_for_type (TREE_TYPE (decl));
- if (alignment && vect_can_force_dr_alignment_p (decl, alignment))
- {
- vnode->increase_alignment (alignment);
- dump_printf (MSG_NOTE, "Increasing alignment of decl: ");
- dump_generic_expr (MSG_NOTE, TDF_SLIM, decl);
- dump_printf (MSG_NOTE, "\n");
- }
+ varpool_node *vnode = *it;
+ vnode->call_for_symbol_and_aliases (increase_alignment_callback, NULL, true);
}
delete type_align_map;
+ delete vars;
return 0;
}
+static void
+increase_alignment_analyze_function (cgraph_node *cnode)
+{
+ ipa_ref *ref;
+
+ for (int i = 0; cnode->iterate_reference (i, ref); i++)
+ if (varpool_node *vnode = dyn_cast<varpool_node *> (ref->referred))
+ {
+ if (vars->contains (vnode))
+ return;
+
+ tree decl = vnode->decl;
+ if (! (TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE
+ || TREE_CODE (TREE_TYPE (decl)) == RECORD_TYPE)
+ && !DECL_ARTIFICIAL (decl))
+ continue;
+
+ if (ref->use == IPA_REF_ADDR)
+ vars->add (vnode);
+ else if (loop_containing_stmt (ref->stmt))
+ vars->add (vnode);
+ }
+}
+
+static void
+increase_alignment_generate_summary (void)
+{
+ vars = new hash_set<varpool_node *> ();
+ cgraph_node *cnode;
+
+ FOR_EACH_DEFINED_FUNCTION (cnode)
+ if (opt_for_fn (cnode->decl, flag_tree_loop_optimize))
+ increase_alignment_analyze_function (cnode);
+}
+
+static void
+increase_alignment_write_summary (void)
+{
+ /* Following is streamed to the section:
+ a) count of type uhwi, representing number of decls.
+ b) the decls. */
+
+ struct output_block *ob = create_output_block (LTO_section_increase_alignment);
+ streamer_write_uhwi (ob, vars->elements ());
+
+ for (hash_set<varpool_node *>::iterator it = vars->begin (); it != vars->end (); ++it)
+ {
+ varpool_node *vnode = *it;
+ stream_write_tree (ob, vnode->decl, true);
+ }
+
+ produce_asm (ob, NULL);
+ destroy_output_block (ob);
+ delete vars;
+}
+
+static void
+increase_alignment_read_section (struct lto_file_decl_data *file_data,
+ const char *data, size_t len)
+{
+ const struct lto_function_header *header =
+ (const struct lto_function_header *) data;
+
+ const int cfg_offset = sizeof (struct lto_function_header);
+ const int main_offset = cfg_offset + header->cfg_size;
+ const int string_offset = main_offset + header->main_size;
+
+ lto_input_block ib (data + main_offset, header->main_size,
+ file_data->mode_table);
+
+ struct data_in *data_in =
+ lto_data_in_create (file_data, data + string_offset,
+ header->string_size, vNULL);
+
+ unsigned HOST_WIDE_INT count = streamer_read_uhwi (&ib);
+
+ for (unsigned HOST_WIDE_INT i = 0; i < count; i++)
+ {
+ tree decl = stream_read_tree (&ib, data_in);
+ varpool_node *vnode = varpool_node::get_create (decl);
+ if (vnode)
+ vars->add (vnode);
+ }
+
+ lto_free_section_data (file_data, LTO_section_increase_alignment, NULL,
+ data, len);
+
+ lto_data_in_delete (data_in);
+}
+
+static void
+increase_alignment_read_summary (void)
+{
+ struct lto_file_decl_data **file_data_vec = lto_get_file_decl_data ();
+ struct lto_file_decl_data *file_data;
+ unsigned i = 0;
+
+ vars = new hash_set<varpool_node *> ();
+
+ while ((file_data = file_data_vec[i++]))
+ {
+ size_t len;
+ const char *data = lto_get_section_data (file_data, LTO_section_increase_alignment, NULL, &len);
+
+ if (data)
+ increase_alignment_read_section (file_data, data, len);
+ }
+}
namespace {
const pass_data pass_data_ipa_increase_alignment =
{
- SIMPLE_IPA_PASS, /* type */
+ IPA_PASS, /* type */
"increase_alignment", /* name */
OPTGROUP_LOOP | OPTGROUP_VEC, /* optinfo_flags */
TV_IPA_OPT, /* tv_id */
@@ -949,17 +1093,26 @@ const pass_data pass_data_ipa_increase_alignment =
0, /* todo_flags_finish */
};
-class pass_ipa_increase_alignment : public simple_ipa_opt_pass
+class pass_ipa_increase_alignment : public ipa_opt_pass_d
{
public:
pass_ipa_increase_alignment (gcc::context *ctxt)
- : simple_ipa_opt_pass (pass_data_ipa_increase_alignment, ctxt)
+ : ipa_opt_pass_d (pass_data_ipa_increase_alignment, ctxt,
+ increase_alignment_generate_summary,
+ increase_alignment_write_summary,
+ increase_alignment_read_summary,
+ NULL, /* write optimization summary */
+ NULL, /* read optimization summary */
+ NULL, /* stmt fixup */
+ 0, /* function_transform_todo_flags_start */
+ NULL, /* transform function */
+ NULL )/* variable transform */
{}
/* opt_pass methods: */
virtual bool gate (function *)
{
- return flag_section_anchors && flag_tree_loop_vectorize;
+ return flag_ipa_increase_alignment != 0;
}
virtual unsigned int execute (function *) { return increase_alignment (); }
@@ -968,7 +1120,7 @@ public:
} // anon namespace
-simple_ipa_opt_pass *
+ipa_opt_pass_d *
make_pass_ipa_increase_alignment (gcc::context *ctxt)
{
return new pass_ipa_increase_alignment (ctxt);