Skip to content

Commit 7b96b9c

Browse files
tesujitesuji
committed
gvn: promote propagatable const local arrays
1 parent 5203642 commit 7b96b9c

8 files changed

+85
-36
lines changed

‎compiler/rustc_mir_transform/src/gvn.rs

+17-14
Original file line numberDiff line numberDiff line change
@@ -138,7 +138,7 @@ fn propagate_ssa<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
138138
let value = state.simplify_rvalue(rvalue, location);
139139
// FIXME(#112651) `rvalue` may have a subtype to `local`. We can only mark `local` as
140140
// reusable if we have an exact type match.
141-
if state.local_decls[local].ty != rvalue.ty(state.local_decls, tcx) {
141+
if state.local_decls[local].ty != rvalue.ty(state.local_decls, state.tcx) {
142142
return;
143143
}
144144
value
@@ -382,7 +382,8 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
382382
let ty = match kind {
383383
AggregateTy::Array => {
384384
assert!(fields.len() > 0);
385-
Ty::new_array(self.tcx, fields[0].layout.ty, fields.len() as u64)
385+
let field_ty = fields[0].layout.ty;
386+
Ty::new_array(self.tcx, field_ty, fields.len() as u64)
386387
}
387388
AggregateTy::Tuple => {
388389
Ty::new_tup_from_iter(self.tcx, fields.iter().map(|f| f.layout.ty))
@@ -406,7 +407,9 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
406407
};
407408
let ptr_imm = Immediate::new_pointer_with_meta(data, meta, &self.ecx);
408409
ImmTy::from_immediate(ptr_imm, ty).into()
409-
} else if matches!(ty.abi, Abi::Scalar(..) | Abi::ScalarPair(..)) {
410+
} else if matches!(kind, AggregateTy::Array)
411+
|| matches!(ty.abi, Abi::Scalar(..) | Abi::ScalarPair(..))
412+
{
410413
let dest = self.ecx.allocate(ty, MemoryKind::Stack).ok()?;
411414
let variant_dest = if let Some(variant) = variant {
412415
self.ecx.project_downcast(&dest, variant).ok()?
@@ -418,9 +421,7 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
418421
self.ecx.copy_op(op, &field_dest).ok()?;
419422
}
420423
self.ecx.write_discriminant(variant.unwrap_or(FIRST_VARIANT), &dest).ok()?;
421-
self.ecx
422-
.alloc_mark_immutable(dest.ptr().provenance.unwrap().alloc_id())
423-
.ok()?;
424+
let dest = dest.map_provenance(|prov| prov.as_immutable());
424425
dest.into()
425426
} else {
426427
return None;
@@ -704,7 +705,7 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
704705
place.projection = self.tcx.mk_place_elems(&projection);
705706
}
706707

707-
trace!(?place);
708+
trace!(after_place = ?place);
708709
}
709710

710711
/// Represent the *value* which would be read from `place`, and point `place` to a preexisting
@@ -884,7 +885,7 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
884885
}
885886

886887
let (mut ty, variant_index) = match *kind {
887-
AggregateKind::Array(..) => {
888+
AggregateKind::Array(_) => {
888889
assert!(!field_ops.is_empty());
889890
(AggregateTy::Array, FIRST_VARIANT)
890891
}
@@ -1347,6 +1348,7 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
13471348
}
13481349
}
13491350

1351+
#[instrument(level = "trace", skip(ecx), ret)]
13501352
fn op_to_prop_const<'tcx>(
13511353
ecx: &mut InterpCx<'tcx, DummyMachine>,
13521354
op: &OpTy<'tcx>,
@@ -1361,8 +1363,11 @@ fn op_to_prop_const<'tcx>(
13611363
return Some(ConstValue::ZeroSized);
13621364
}
13631365

1364-
// Do not synthetize too large constants. Codegen will just memcpy them, which we'd like to avoid.
1365-
if !matches!(op.layout.abi, Abi::Scalar(..) | Abi::ScalarPair(..)) {
1366+
// Do not synthesize too large constants, except constant arrays.
1367+
// For arrays, codegen will just memcpy them, but LLVM will optimize out those unneeded memcpy.
1368+
// For others, we'd prefer in-place initialization over memcpy them.
1369+
if !(op.layout.ty.is_array() || matches!(op.layout.abi, Abi::Scalar(..) | Abi::ScalarPair(..)))
1370+
{
13661371
return None;
13671372
}
13681373

@@ -1433,6 +1438,7 @@ impl<'tcx> VnState<'_, 'tcx> {
14331438
}
14341439

14351440
/// If `index` is a `Value::Constant`, return the `Constant` to be put in the MIR.
1441+
#[instrument(level = "trace", skip(self, index), ret)]
14361442
fn try_as_constant(&mut self, index: VnIndex) -> Option<ConstOperand<'tcx>> {
14371443
// This was already constant in MIR, do not change it.
14381444
if let Value::Constant { value, disambiguator: _ } = *self.get(index)
@@ -1444,10 +1450,6 @@ impl<'tcx> VnState<'_, 'tcx> {
14441450
}
14451451

14461452
let op = self.evaluated[index].as_ref()?;
1447-
if op.layout.is_unsized() {
1448-
// Do not attempt to propagate unsized locals.
1449-
return None;
1450-
}
14511453

14521454
let value = op_to_prop_const(&mut self.ecx, op)?;
14531455

@@ -1484,6 +1486,7 @@ impl<'tcx> MutVisitor<'tcx> for VnState<'_, 'tcx> {
14841486
self.simplify_operand(operand, location);
14851487
}
14861488

1489+
#[instrument(level = "trace", skip(self, stmt))]
14871490
fn visit_statement(&mut self, stmt: &mut Statement<'tcx>, location: Location) {
14881491
if let StatementKind::Assign(box (ref mut lhs, ref mut rvalue)) = stmt.kind {
14891492
self.simplify_place_projection(lhs, location);

‎compiler/rustc_mir_transform/src/ssa.rs

+1
Original file line numberDiff line numberDiff line change
@@ -147,6 +147,7 @@ impl SsaLocals {
147147
})
148148
}
149149

150+
#[instrument(level = "trace", skip_all)]
150151
pub fn for_each_assignment_mut<'tcx>(
151152
&self,
152153
basic_blocks: &mut IndexSlice<BasicBlock, BasicBlockData<'tcx>>,

‎tests/mir-opt/const_array_locals.main.GVN.diff

+50-18
Original file line numberDiff line numberDiff line change
@@ -37,49 +37,60 @@
3737
}
3838

3939
bb0: {
40-
- StorageLive(_1);
41-
+ nop;
42-
_1 = [const 255_i32, const 105_i32, const 15_i32, const 39_i32, const 62_i32];
40+
StorageLive(_1);
41+
- _1 = [const 255_i32, const 105_i32, const 15_i32, const 39_i32, const 62_i32];
42+
+ _1 = const [255_i32, 105_i32, 15_i32, 39_i32, 62_i32];
4343
StorageLive(_2);
4444
- _2 = [const 255_i32, const 105_i32, const 15_i32, const 39_i32, const 62_i32];
45-
+ _2 = _1;
45+
+ _2 = const [255_i32, 105_i32, 15_i32, 39_i32, 62_i32];
4646
StorageLive(_3);
4747
StorageLive(_4);
48-
_4 = [const 178_i32, const 9_i32, const 4_i32, const 56_i32, const 221_i32];
48+
- _4 = [const 178_i32, const 9_i32, const 4_i32, const 56_i32, const 221_i32];
49+
+ _4 = const [178_i32, 9_i32, 4_i32, 56_i32, 221_i32];
4950
StorageLive(_5);
50-
_5 = [const 193_i32, const 164_i32, const 194_i32, const 197_i32, const 6_i32];
51-
_3 = [move _4, move _5];
51+
- _5 = [const 193_i32, const 164_i32, const 194_i32, const 197_i32, const 6_i32];
52+
- _3 = [move _4, move _5];
53+
+ _5 = const [193_i32, 164_i32, 194_i32, 197_i32, 6_i32];
54+
+ _3 = const [[178_i32, 9_i32, 4_i32, 56_i32, 221_i32], [193_i32, 164_i32, 194_i32, 197_i32, 6_i32]];
5255
StorageDead(_5);
5356
StorageDead(_4);
5457
StorageLive(_6);
5558
StorageLive(_7);
5659
_17 = const main::promoted[0];
5760
_7 = &(*_17);
5861
- _6 = (*_7);
59-
+ _6 = (*_17);
62+
+ _6 = const [254_i32, 42_i32, 15_i32, 39_i32, 62_i32];
6063
StorageDead(_7);
6164
StorageLive(_9);
6265
StorageLive(_10);
63-
_10 = [const 31_u32, const 96_u32, const 173_u32, const 50_u32, const 1_u32];
64-
_9 = consume(move _10) -> [return: bb1, unwind continue];
66+
- _10 = [const 31_u32, const 96_u32, const 173_u32, const 50_u32, const 1_u32];
67+
- _9 = consume(move _10) -> [return: bb1, unwind continue];
68+
+ _10 = const [31_u32, 96_u32, 173_u32, 50_u32, 1_u32];
69+
+ _9 = consume(const [31_u32, 96_u32, 173_u32, 50_u32, 1_u32]) -> [return: bb1, unwind continue];
6570
}
6671

6772
bb1: {
6873
StorageDead(_10);
6974
StorageDead(_9);
7075
StorageLive(_11);
7176
StorageLive(_12);
72-
_12 = [const 1f32, const 2f32, const 3f32, const 1f32, const 1f32, const 1f32, const 1f32, const 42f32];
73-
_11 = F32x8(move _12);
77+
- _12 = [const 1f32, const 2f32, const 3f32, const 1f32, const 1f32, const 1f32, const 1f32, const 42f32];
78+
- _11 = F32x8(move _12);
79+
+ _12 = const [1f32, 2f32, 3f32, 1f32, 1f32, 1f32, 1f32, 42f32];
80+
+ _11 = F32x8(const [1f32, 2f32, 3f32, 1f32, 1f32, 1f32, 1f32, 42f32]);
7481
StorageDead(_12);
7582
StorageLive(_13);
7683
StorageLive(_14);
77-
_14 = [const 1_i32, const 0_i32, const 0_i32];
84+
- _14 = [const 1_i32, const 0_i32, const 0_i32];
85+
+ _14 = const [1_i32, 0_i32, 0_i32];
7886
StorageLive(_15);
79-
_15 = [const 0_i32, const 1_i32, const 0_i32];
87+
- _15 = [const 0_i32, const 1_i32, const 0_i32];
88+
+ _15 = const [0_i32, 1_i32, 0_i32];
8089
StorageLive(_16);
81-
_16 = [const 0_i32, const 0_i32, const 1_i32];
82-
_13 = [move _14, move _15, move _16];
90+
- _16 = [const 0_i32, const 0_i32, const 1_i32];
91+
- _13 = [move _14, move _15, move _16];
92+
+ _16 = const [0_i32, 0_i32, 1_i32];
93+
+ _13 = const [[1_i32, 0_i32, 0_i32], [0_i32, 1_i32, 0_i32], [0_i32, 0_i32, 1_i32]];
8394
StorageDead(_16);
8495
StorageDead(_15);
8596
StorageDead(_14);
@@ -89,9 +100,30 @@
89100
StorageDead(_6);
90101
StorageDead(_3);
91102
StorageDead(_2);
92-
- StorageDead(_1);
93-
+ nop;
103+
StorageDead(_1);
94104
return;
95105
}
96106
}
107+
+
108+
+ ALLOC0 (size: 36, align: 4) { .. }
109+
+
110+
+ ALLOC1 (size: 12, align: 4) { .. }
111+
+
112+
+ ALLOC2 (size: 12, align: 4) { .. }
113+
+
114+
+ ALLOC3 (size: 12, align: 4) { .. }
115+
+
116+
+ ALLOC4 (size: 32, align: 4) { .. }
117+
+
118+
+ ALLOC5 (size: 20, align: 4) { .. }
119+
+
120+
+ ALLOC6 (size: 40, align: 4) { .. }
121+
+
122+
+ ALLOC7 (size: 20, align: 4) { .. }
123+
+
124+
+ ALLOC8 (size: 20, align: 4) { .. }
125+
+
126+
+ ALLOC9 (size: 20, align: 4) { .. }
127+
+
128+
+ ALLOC10 (size: 20, align: 4) { .. }
97129

‎tests/mir-opt/const_prop/array_index.main.GVN.32bit.panic-unwind.diff

+4-1
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,8 @@
1515
bb0: {
1616
StorageLive(_1);
1717
StorageLive(_2);
18-
_2 = [const 0_u32, const 1_u32, const 2_u32, const 3_u32];
18+
- _2 = [const 0_u32, const 1_u32, const 2_u32, const 3_u32];
19+
+ _2 = const [0_u32, 1_u32, 2_u32, 3_u32];
1920
StorageLive(_3);
2021
_3 = const 2_usize;
2122
- _4 = Len(_2);
@@ -36,4 +37,6 @@
3637
return;
3738
}
3839
}
40+
+
41+
+ ALLOC0 (size: 16, align: 4) { .. }
3942

‎tests/mir-opt/const_prop/array_index.main.GVN.64bit.panic-unwind.diff

+4-1
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,8 @@
1515
bb0: {
1616
StorageLive(_1);
1717
StorageLive(_2);
18-
_2 = [const 0_u32, const 1_u32, const 2_u32, const 3_u32];
18+
- _2 = [const 0_u32, const 1_u32, const 2_u32, const 3_u32];
19+
+ _2 = const [0_u32, 1_u32, 2_u32, 3_u32];
1920
StorageLive(_3);
2021
_3 = const 2_usize;
2122
- _4 = Len(_2);
@@ -36,4 +37,6 @@
3637
return;
3738
}
3839
}
40+
+
41+
+ ALLOC0 (size: 16, align: 4) { .. }
3942

‎tests/mir-opt/const_prop/array_index.rs

+1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
//@ test-mir-pass: GVN
2+
//@ compile-flags: -Zdump-mir-exclude-alloc-bytes
23
// EMIT_MIR_FOR_EACH_PANIC_STRATEGY
34
// EMIT_MIR_FOR_EACH_BIT_WIDTH
45

‎tests/mir-opt/pre-codegen/optimizes_into_variable.main.GVN.32bit.panic-unwind.diff

+4-1
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,8 @@
3535
+ _1 = const 4_i32;
3636
StorageLive(_3);
3737
StorageLive(_4);
38-
_4 = [const 0_i32, const 1_i32, const 2_i32, const 3_i32, const 4_i32, const 5_i32];
38+
- _4 = [const 0_i32, const 1_i32, const 2_i32, const 3_i32, const 4_i32, const 5_i32];
39+
+ _4 = const [0_i32, 1_i32, 2_i32, 3_i32, 4_i32, 5_i32];
3940
StorageLive(_5);
4041
_5 = const 3_usize;
4142
_6 = const 6_usize;
@@ -64,4 +65,6 @@
6465
}
6566
+
6667
+ ALLOC0 (size: 8, align: 4) { .. }
68+
+
69+
+ ALLOC1 (size: 24, align: 4) { .. }
6770

‎tests/mir-opt/pre-codegen/optimizes_into_variable.main.GVN.64bit.panic-unwind.diff

+4-1
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,8 @@
3535
+ _1 = const 4_i32;
3636
StorageLive(_3);
3737
StorageLive(_4);
38-
_4 = [const 0_i32, const 1_i32, const 2_i32, const 3_i32, const 4_i32, const 5_i32];
38+
- _4 = [const 0_i32, const 1_i32, const 2_i32, const 3_i32, const 4_i32, const 5_i32];
39+
+ _4 = const [0_i32, 1_i32, 2_i32, 3_i32, 4_i32, 5_i32];
3940
StorageLive(_5);
4041
_5 = const 3_usize;
4142
_6 = const 6_usize;
@@ -64,4 +65,6 @@
6465
}
6566
+
6667
+ ALLOC0 (size: 8, align: 4) { .. }
68+
+
69+
+ ALLOC1 (size: 24, align: 4) { .. }
6770

0 commit comments

Comments
 (0)