From 2bc57c17faa1dd494ed3898298e9fbe91f8a8675 Mon Sep 17 00:00:00 2001
From: Joel Hutton <Joel.Hutton@arm.com>
Date: Wed, 2 Oct 2019 17:38:53 +0100
Subject: [PATCH] SLP Vectorization: Vectorize Vector Constructors

---
 gcc/testsuite/gcc.dg/vect/bb-slp-40.c |  33 +++++++
 gcc/tree-vect-slp.c                   | 127 ++++++++++++++++++++++++++
 gcc/tree-vectorizer.h                 |   5 +
 3 files changed, 165 insertions(+)
 create mode 100644 gcc/testsuite/gcc.dg/vect/bb-slp-40.c

diff --git a/gcc/testsuite/gcc.dg/vect/bb-slp-40.c b/gcc/testsuite/gcc.dg/vect/bb-slp-40.c
new file mode 100644
index 0000000000000000000000000000000000000000..51566b716bcda2fe82f50c50e9e9685cb3eb10ae
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/bb-slp-40.c
@@ -0,0 +1,33 @@
+/* { dg-do compile } */
+/* { dg-options "-O3 -fdump-tree-slp-all" } */
+/* { dg-require-effective-target vect_int } */
+
+char g_d[1024], g_s1[1024], g_s2[1024];
+void foo(void)
+{
+    char *d = g_d, *s1 = g_s1, *s2 = g_s2;
+
+    for ( int y = 0; y < 128; y++ )
+    {
+      d[0 ] = s1[0 ] + s2[0 ];
+      d[1 ] = s1[1 ] + s2[1 ];
+      d[2 ] = s1[2 ] + s2[2 ];
+      d[3 ] = s1[3 ] + s2[3 ];
+      d[4 ] = s1[4 ] + s2[4 ];
+      d[5 ] = s1[5 ] + s2[5 ];
+      d[6 ] = s1[6 ] + s2[6 ];
+      d[7 ] = s1[7 ] + s2[7 ];
+      d[8 ] = s1[8 ] + s2[8 ];
+      d[9 ] = s1[9 ] + s2[9 ];
+      d[10] = s1[10] + s2[10];
+      d[11] = s1[11] + s2[11];
+      d[12] = s1[12] + s2[12];
+      d[13] = s1[13] + s2[13];
+      d[14] = s1[14] + s2[14];
+      d[15] = s1[15] + s2[15];
+      d += 16;
+    }
+}
+
+/* See that we vectorize an SLP instance.  */
+/* { dg-final { scan-tree-dump-times "vectorizing stmts using SLP" 1 "slp1" } } */
diff --git a/gcc/tree-vect-slp.c b/gcc/tree-vect-slp.c
index 9b86b67734ad3e3506e9cee6a532b68decf24ae6..c4d452e3dfd46acdaa94dc047e4c6114d8295458 100644
--- a/gcc/tree-vect-slp.c
+++ b/gcc/tree-vect-slp.c
@@ -1922,6 +1922,7 @@ vect_analyze_slp_instance (vec_info *vinfo,
   unsigned int i;
   struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info);
   vec<stmt_vec_info> scalar_stmts;
+  bool constructor = false;
 
   if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
     {
@@ -1935,6 +1936,13 @@ vect_analyze_slp_instance (vec_info *vinfo,
       vectype = STMT_VINFO_VECTYPE (stmt_info);
       group_size = REDUC_GROUP_SIZE (stmt_info);
     }
+  else if (is_gimple_assign (stmt_info->stmt)
+	    && gimple_assign_rhs_code (stmt_info->stmt) == CONSTRUCTOR)
+    {
+      vectype = TREE_TYPE (gimple_assign_rhs1 (stmt_info->stmt));
+      group_size = CONSTRUCTOR_NELTS (gimple_assign_rhs1 (stmt_info->stmt));
+      constructor = true;
+    }
   else
     {
       gcc_assert (is_a <loop_vec_info> (vinfo));
@@ -1981,6 +1989,25 @@ vect_analyze_slp_instance (vec_info *vinfo,
       STMT_VINFO_REDUC_DEF (vect_orig_stmt (stmt_info))
 	= STMT_VINFO_REDUC_DEF (vect_orig_stmt (scalar_stmts.last ()));
     }
+  else if (constructor)
+    {
+      tree rhs = gimple_assign_rhs1 (stmt_info->stmt);
+      tree val;
+      FOR_EACH_CONSTRUCTOR_VALUE (CONSTRUCTOR_ELTS (rhs), i, val)
+	{
+	  if (TREE_CODE (val) == SSA_NAME)
+	    {
+	      gimple* def = SSA_NAME_DEF_STMT (val);
+	      stmt_vec_info def_info = vinfo->lookup_stmt (def);
+	      /* Value is defined in another basic block.  */
+	      if (!def_info)
+		return false;
+	      scalar_stmts.safe_push (def_info);
+	    }
+	  else
+	    return false;
+	}
+    }
   else
     {
       /* Collect reduction statements.  */
@@ -2038,6 +2065,14 @@ vect_analyze_slp_instance (vec_info *vinfo,
 	  SLP_INSTANCE_GROUP_SIZE (new_instance) = group_size;
 	  SLP_INSTANCE_UNROLLING_FACTOR (new_instance) = unrolling_factor;
 	  SLP_INSTANCE_LOADS (new_instance) = vNULL;
+
+	  if (constructor)
+	    {
+	      SLP_INSTANCE_ROOT_STMT (new_instance) = stmt_info->stmt;
+	    }
+	  else
+	    SLP_INSTANCE_ROOT_STMT (new_instance) = NULL;
+
 	  vect_gather_slp_loads (new_instance, node);
 	  if (dump_enabled_p ())
 	    dump_printf_loc (MSG_NOTE, vect_location,
@@ -2725,6 +2760,10 @@ vect_bb_slp_scalar_cost (basic_block bb,
 		stmt_vec_info use_stmt_info = vinfo->lookup_stmt (use_stmt);
 		if (!use_stmt_info || !PURE_SLP_STMT (use_stmt_info))
 		  {
+		    /* Check this is not a constructor that will be vectorized
+		       away.  */
+		    if (BB_VINFO_GROUPED_STORES (vinfo).contains (use_stmt_info))
+			continue;
 		    (*life)[i] = true;
 		    BREAK_FROM_IMM_USE_STMT (use_iter);
 		  }
@@ -2836,6 +2875,72 @@ vect_bb_vectorization_profitable_p (bb_vec_info bb_vinfo)
   return true;
 }
 
+static bool
+vect_ssa_use_outside_bb (tree ssa)
+{
+  imm_use_iterator use_iter;
+  gimple *use_stmt;
+  bool use_outside_of_block = false;
+  gcc_checking_assert (TREE_CODE (ssa) == SSA_NAME);
+  gimple* def = SSA_NAME_DEF_STMT (ssa);
+
+  FOR_EACH_IMM_USE_STMT (use_stmt, use_iter, ssa)
+  {
+    if (use_stmt->bb != def->bb)
+    {
+      use_outside_of_block = true;
+      BREAK_FROM_IMM_USE_STMT (use_iter);
+    }
+    /* In the following pattern, we consider _1 and vect_1
+       equivalent.
+       _1 = {a,b,c}
+       vect_1 = _1  */
+    else if (is_gimple_assign (use_stmt)
+	     && gimple_assign_rhs_code (use_stmt) == SSA_NAME
+	     && TREE_CODE (gimple_assign_lhs (use_stmt)) == SSA_NAME)
+    {
+      use_outside_of_block = vect_ssa_use_outside_bb (gimple_assign_lhs (use_stmt));
+      BREAK_FROM_IMM_USE_STMT (use_iter);
+    }
+    else
+      BREAK_FROM_IMM_USE_STMT (use_iter);
+  }
+  return use_outside_of_block;
+}
+
+static void
+vect_slp_check_for_constructors (bb_vec_info bb_vinfo)
+{
+  gimple_stmt_iterator gsi;
+
+  for (gsi = bb_vinfo->region_begin;
+      gsi_stmt (gsi) != gsi_stmt (bb_vinfo->region_end); gsi_next (&gsi))
+    {
+      gimple *stmt = gsi_stmt (gsi);
+      bool vectorizable = true;
+
+      if (is_gimple_assign (stmt)
+	  && gimple_assign_rhs_code (stmt) == CONSTRUCTOR
+	  && TREE_CODE (gimple_assign_lhs (stmt)) == SSA_NAME
+	  && TREE_CODE (TREE_TYPE (gimple_assign_lhs (stmt))) == VECTOR_TYPE)
+	{
+	  tree rhs = gimple_assign_rhs1 (stmt);
+
+	  if (CONSTRUCTOR_NELTS (rhs) == 0)
+	    vectorizable = false;
+
+	  if (!vect_ssa_use_outside_bb (gimple_assign_lhs (stmt)))
+	    vectorizable = false;
+
+	  if (vectorizable)
+	    {
+	      stmt_vec_info stmt_info = bb_vinfo->lookup_stmt (stmt);
+	      BB_VINFO_GROUPED_STORES (bb_vinfo).safe_push (stmt_info);
+	    }
+	}
+    }
+}
+
 /* Check if the basic block can be vectorized.  Returns a bb_vec_info
    if so and sets fatal to true if failure is independent of
    current_vector_size.  */
@@ -2908,6 +3013,8 @@ vect_slp_analyze_bb_1 (gimple_stmt_iterator region_begin,
       return NULL;
     }
 
+  vect_slp_check_for_constructors (bb_vinfo);
+
   /* If there are no grouped stores in the region there is no need
      to continue with pattern recog as vect_analyze_slp will fail
      anyway.  */
@@ -4053,6 +4160,26 @@ vect_schedule_slp_instance (slp_tree node, slp_instance instance,
 	FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (child), j, child_stmt_info)
 	  STMT_VINFO_DEF_TYPE (child_stmt_info) = vect_internal_def;
       }
+
+  /* For vector constructors, the same SSA name must be used to maintain data
+     flow into other basic blocks.  */
+  if (instance->root == node && SLP_INSTANCE_ROOT_STMT (instance)
+      && SLP_TREE_NUMBER_OF_VEC_STMTS (node) == 1
+      && SLP_TREE_VEC_STMTS (node).exists ())
+    {
+	  stmt_vec_info child_stmt_info;
+	  int j;
+	  FOR_EACH_VEC_ELT (SLP_TREE_VEC_STMTS (node), j, child_stmt_info)
+	    {
+		gassign *rstmt
+		  = gimple_build_assign (gimple_get_lhs (instance->root_stmt),
+					 gimple_get_lhs (child_stmt_info->stmt));
+		gimple_stmt_iterator rgsi = gsi_for_stmt (instance->root_stmt);
+		gsi_replace (&rgsi, rstmt, true);
+		break;
+	    }
+    }
+
 }
 
 /* Replace scalar calls from SLP node NODE with setting of their lhs to zero.
diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h
index 837fb5ab52537cdf95a413557335b30704f9dc26..906579f0bc0efce955a1cde177fe1404ea8ce843 100644
--- a/gcc/tree-vectorizer.h
+++ b/gcc/tree-vectorizer.h
@@ -149,6 +149,10 @@ public:
   /* The root of SLP tree.  */
   slp_tree root;
 
+  /* For vector constructors, the constructor stmt that the SLP tree is built
+     from, NULL otherwise.  */
+  gimple *root_stmt;
+
   /* Size of groups of scalar stmts that will be replaced by SIMD stmt/s.  */
   unsigned int group_size;
 
@@ -168,6 +172,7 @@ public:
 #define SLP_INSTANCE_GROUP_SIZE(S)               (S)->group_size
 #define SLP_INSTANCE_UNROLLING_FACTOR(S)         (S)->unrolling_factor
 #define SLP_INSTANCE_LOADS(S)                    (S)->loads
+#define SLP_INSTANCE_ROOT_STMT(S)                (S)->root_stmt
 
 #define SLP_TREE_CHILDREN(S)                     (S)->children
 #define SLP_TREE_SCALAR_STMTS(S)                 (S)->stmts
-- 
2.17.1

