From 25ccdaec23ba8d8fafbd5d6995a4023d9386e639 Mon Sep 17 00:00:00 2001 From: Mara Bos Date: Thu, 20 Feb 2025 00:36:59 +0100 Subject: [PATCH] Refactor fmt::Arguments to just two pointers in size. --- compiler/rustc_ast/src/format.rs | 7 + compiler/rustc_ast_lowering/messages.ftl | 3 + compiler/rustc_ast_lowering/src/errors.rs | 7 + compiler/rustc_ast_lowering/src/expr.rs | 26 +- compiler/rustc_ast_lowering/src/format.rs | 517 ++++++++++++---------- compiler/rustc_hir/src/lang_items.rs | 5 +- compiler/rustc_span/src/symbol.rs | 8 +- library/core/src/fmt/mod.rs | 374 +++++++++++----- library/core/src/fmt/rt.rs | 118 ++++- library/core/src/panicking.rs | 24 +- 10 files changed, 697 insertions(+), 392 deletions(-) diff --git a/compiler/rustc_ast/src/format.rs b/compiler/rustc_ast/src/format.rs index b611ddea1d9f1..b227961f9aa7d 100644 --- a/compiler/rustc_ast/src/format.rs +++ b/compiler/rustc_ast/src/format.rs @@ -160,6 +160,13 @@ impl FormatArgumentKind { &Self::Captured(id) => Some(id), } } + + pub fn is_captured(&self) -> bool { + match self { + Self::Captured(_) => true, + _ => false, + } + } } #[derive(Clone, Encodable, Decodable, Debug, PartialEq, Eq)] diff --git a/compiler/rustc_ast_lowering/messages.ftl b/compiler/rustc_ast_lowering/messages.ftl index 1b91c33742d8b..bab94eb4d9c1f 100644 --- a/compiler/rustc_ast_lowering/messages.ftl +++ b/compiler/rustc_ast_lowering/messages.ftl @@ -175,6 +175,9 @@ ast_lowering_template_modifier = template modifier ast_lowering_this_not_async = this is not `async` +ast_lowering_too_many_format_arguments = + too many arguments used in format string + ast_lowering_underscore_array_length_unstable = using `_` for array lengths is unstable diff --git a/compiler/rustc_ast_lowering/src/errors.rs b/compiler/rustc_ast_lowering/src/errors.rs index f31e2db051d81..f21b331471242 100644 --- a/compiler/rustc_ast_lowering/src/errors.rs +++ b/compiler/rustc_ast_lowering/src/errors.rs @@ -467,3 +467,10 @@ pub(crate) struct UseConstGenericArg { #[suggestion_part(code = "{other_args}")] pub call_args: Span, } + +#[derive(Diagnostic)] +#[diag(ast_lowering_too_many_format_arguments)] +pub(crate) struct TooManyFormatArguments { + #[primary_span] + pub span: Span, +} diff --git a/compiler/rustc_ast_lowering/src/expr.rs b/compiler/rustc_ast_lowering/src/expr.rs index d967772c93bfd..1840153556ca3 100644 --- a/compiler/rustc_ast_lowering/src/expr.rs +++ b/compiler/rustc_ast_lowering/src/expr.rs @@ -2129,12 +2129,12 @@ impl<'hir> LoweringContext<'_, 'hir> { self.arena.alloc(self.expr(sp, hir::ExprKind::Tup(&[]))) } - pub(super) fn expr_usize(&mut self, sp: Span, value: usize) -> hir::Expr<'hir> { + pub(super) fn expr_u64(&mut self, sp: Span, value: u64) -> hir::Expr<'hir> { let lit = self.arena.alloc(hir::Lit { span: sp, node: ast::LitKind::Int( - (value as u128).into(), - ast::LitIntType::Unsigned(ast::UintTy::Usize), + u128::from(value).into(), + ast::LitIntType::Unsigned(ast::UintTy::U64), ), }); self.expr(sp, hir::ExprKind::Lit(lit)) @@ -2151,12 +2151,12 @@ impl<'hir> LoweringContext<'_, 'hir> { self.expr(sp, hir::ExprKind::Lit(lit)) } - pub(super) fn expr_u16(&mut self, sp: Span, value: u16) -> hir::Expr<'hir> { + pub(super) fn expr_usize(&mut self, sp: Span, value: u16) -> hir::Expr<'hir> { let lit = self.arena.alloc(hir::Lit { span: sp, node: ast::LitKind::Int( u128::from(value).into(), - ast::LitIntType::Unsigned(ast::UintTy::U16), + ast::LitIntType::Unsigned(ast::UintTy::Usize), ), }); self.expr(sp, hir::ExprKind::Lit(lit)) @@ -2287,6 +2287,22 @@ impl<'hir> LoweringContext<'_, 'hir> { self.expr(b.span, hir::ExprKind::Block(b, None)) } + pub(super) fn expr_unsafe_block( + &mut self, + span: Span, + expr: &'hir hir::Expr<'hir>, + ) -> hir::Expr<'hir> { + let hir_id = self.next_id(); + self.expr_block(self.arena.alloc(hir::Block { + stmts: &[], + expr: Some(expr), + hir_id, + rules: hir::BlockCheckMode::UnsafeBlock(hir::UnsafeSource::CompilerGenerated), + span, + targeted_by_break: false, + })) + } + pub(super) fn expr_array_ref( &mut self, span: Span, diff --git a/compiler/rustc_ast_lowering/src/format.rs b/compiler/rustc_ast_lowering/src/format.rs index 343895984ca42..2adc56b216636 100644 --- a/compiler/rustc_ast_lowering/src/format.rs +++ b/compiler/rustc_ast_lowering/src/format.rs @@ -1,14 +1,15 @@ -use core::ops::ControlFlow; use std::borrow::Cow; +use std::ops::ControlFlow; use rustc_ast::visit::Visitor; use rustc_ast::*; use rustc_data_structures::fx::FxIndexMap; use rustc_hir as hir; use rustc_session::config::FmtDebug; -use rustc_span::{Ident, Span, Symbol, kw, sym}; +use rustc_span::{Ident, Span, Symbol, sym}; use super::LoweringContext; +use super::errors::TooManyFormatArguments; impl<'hir> LoweringContext<'_, 'hir> { pub(crate) fn lower_format_args(&mut self, sp: Span, fmt: &FormatArgs) -> hir::ExprKind<'hir> { @@ -84,28 +85,31 @@ impl<'hir> LoweringContext<'_, 'hir> { let mut was_inlined = vec![false; fmt.arguments.all_args().len()]; let mut inlined_anything = false; - for i in 0..fmt.template.len() { - let FormatArgsPiece::Placeholder(placeholder) = &fmt.template[i] else { continue }; - let Ok(arg_index) = placeholder.argument.index else { continue }; + let mut i = 0; - let mut literal = None; - - if let FormatTrait::Display = placeholder.format_trait + while i < fmt.template.len() { + if let FormatArgsPiece::Placeholder(placeholder) = &fmt.template[i] + && let Ok(arg_index) = placeholder.argument.index + && let FormatTrait::Display = placeholder.format_trait && placeholder.format_options == Default::default() && let arg = fmt.arguments.all_args()[arg_index].expr.peel_parens_and_refs() && let ExprKind::Lit(lit) = arg.kind + && let Some(literal) = self.try_inline_lit(lit) { - literal = self.try_inline_lit(lit); - } - - if let Some(literal) = literal { // Now we need to mutate the outer FormatArgs. // If this is the first time, this clones the outer FormatArgs. let fmt = fmt.to_mut(); // Replace the placeholder with the literal. - fmt.template[i] = FormatArgsPiece::Literal(literal); + if literal.is_empty() { + fmt.template.remove(i); + } else { + fmt.template[i] = FormatArgsPiece::Literal(literal); + i += 1; + } was_inlined[arg_index] = true; inlined_anything = true; + } else { + i += 1; } } @@ -221,6 +225,7 @@ fn flatten_format_args(mut fmt: Cow<'_, FormatArgs>) -> Cow<'_, FormatArgs> { enum ArgumentType { Format(FormatTrait), Usize, + Constant(u16), } /// Generate a hir expression representing an argument to a format_args invocation. @@ -254,103 +259,49 @@ fn make_argument<'hir>( Format(Binary) => sym::new_binary, Format(LowerHex) => sym::new_lower_hex, Format(UpperHex) => sym::new_upper_hex, - Usize => sym::from_usize, + Usize | Constant(_) => sym::from_usize, }, )); ctx.expr_call_mut(sp, new_fn, std::slice::from_ref(arg)) } -/// Generate a hir expression for a format_args Count. +/// Generate a hir expression for a format_piece. /// /// Generates: /// /// ```text -/// ::Is(…) -/// ``` -/// -/// or -/// -/// ```text -/// ::Param(…) -/// ``` -/// -/// or -/// -/// ```text -/// ::Implied +/// ::…(…) /// ``` -fn make_count<'hir>( +fn make_piece<'hir>( ctx: &mut LoweringContext<'_, 'hir>, + constructor: Symbol, + expr: hir::Expr<'hir>, sp: Span, - count: &Option, - argmap: &mut FxIndexMap<(usize, ArgumentType), Option>, ) -> hir::Expr<'hir> { - match count { - Some(FormatCount::Literal(n)) => { - let count_is = ctx.arena.alloc(ctx.expr_lang_item_type_relative( - sp, - hir::LangItem::FormatCount, - sym::Is, - )); - let value = ctx.arena.alloc_from_iter([ctx.expr_u16(sp, *n)]); - ctx.expr_call_mut(sp, count_is, value) - } - Some(FormatCount::Argument(arg)) => { - if let Ok(arg_index) = arg.index { - let (i, _) = argmap.insert_full((arg_index, ArgumentType::Usize), arg.span); - let count_param = ctx.arena.alloc(ctx.expr_lang_item_type_relative( - sp, - hir::LangItem::FormatCount, - sym::Param, - )); - let value = ctx.arena.alloc_from_iter([ctx.expr_usize(sp, i)]); - ctx.expr_call_mut(sp, count_param, value) - } else { - ctx.expr( - sp, - hir::ExprKind::Err( - ctx.dcx().span_delayed_bug(sp, "lowered bad format_args count"), - ), - ) - } - } - None => ctx.expr_lang_item_type_relative(sp, hir::LangItem::FormatCount, sym::Implied), - } + let new_fn = ctx.arena.alloc(ctx.expr_lang_item_type_relative( + sp, + hir::LangItem::FormatPiece, + constructor, + )); + let new_args = ctx.arena.alloc_from_iter([expr]); + ctx.expr(sp, hir::ExprKind::Call(new_fn, new_args)) } -/// Generate a hir expression for a format_args placeholder specification. -/// -/// Generates -/// -/// ```text -/// ::…, // alignment -/// …u32, // flags -/// , // width -/// , // precision -/// ) -/// ``` -fn make_format_spec<'hir>( - ctx: &mut LoweringContext<'_, 'hir>, - sp: Span, +/// Generate the 64 bit descriptor for a format_args placeholder specification. +// This needs to match the constants in library/core/src/fmt/rt.rs! +fn make_format_spec( placeholder: &FormatPlaceholder, argmap: &mut FxIndexMap<(usize, ArgumentType), Option>, -) -> hir::Expr<'hir> { - let position = match placeholder.argument.index { - Ok(arg_index) => { - let (i, _) = argmap.insert_full( - (arg_index, ArgumentType::Format(placeholder.format_trait)), - placeholder.span, - ); - ctx.expr_usize(sp, i) - } - Err(_) => ctx.expr( - sp, - hir::ExprKind::Err(ctx.dcx().span_delayed_bug(sp, "lowered bad format_args count")), - ), - }; +) -> u64 { + let position = argmap + .insert_full( + ( + placeholder.argument.index.unwrap_or(usize::MAX), + ArgumentType::Format(placeholder.format_trait), + ), + placeholder.span, + ) + .0 as u64; let &FormatOptions { ref width, ref precision, @@ -362,14 +313,12 @@ fn make_format_spec<'hir>( debug_hex, } = &placeholder.format_options; let fill = fill.unwrap_or(' '); - // These need to match the constants in library/core/src/fmt/rt.rs. let align = match alignment { Some(FormatAlignment::Left) => 0, Some(FormatAlignment::Right) => 1, Some(FormatAlignment::Center) => 2, None => 3, }; - // This needs to match the constants in library/core/src/fmt/rt.rs. let flags: u32 = fill as u32 | ((sign == Some(FormatSign::Plus)) as u32) << 21 | ((sign == Some(FormatSign::Minus)) as u32) << 22 @@ -380,17 +329,39 @@ fn make_format_spec<'hir>( | (width.is_some() as u32) << 27 | (precision.is_some() as u32) << 28 | align << 29 - | 1 << 31; // Highest bit always set. - let flags = ctx.expr_u32(sp, flags); - let precision = make_count(ctx, sp, precision, argmap); - let width = make_count(ctx, sp, width, argmap); - let format_placeholder_new = ctx.arena.alloc(ctx.expr_lang_item_type_relative( - sp, - hir::LangItem::FormatPlaceholder, - sym::new, - )); - let args = ctx.arena.alloc_from_iter([position, flags, precision, width]); - ctx.expr_call_mut(sp, format_placeholder_new, args) + | 1 << 31; + let (width_indirect, width) = make_count(width, argmap); + let (precision_indirect, precision) = make_count(precision, argmap); + (flags as u64) << 32 + | (precision_indirect as u64) << 31 + | (width_indirect as u64) << 30 + | precision << 20 + | width << 10 + | position +} + +fn make_count( + count: &Option, + argmap: &mut FxIndexMap<(usize, ArgumentType), Option>, +) -> (bool, u64) { + match count { + None => (false, 0), + &Some(FormatCount::Literal(n)) => { + if n < 1 << 10 { + (false, n as u64) + } else { + // Too big. Upgrade to an argument. + let index = + argmap.insert_full((usize::MAX, ArgumentType::Constant(n)), None).0 as u64; + (true, index) + } + } + Some(FormatCount::Argument(arg)) => ( + true, + argmap.insert_full((arg.index.unwrap_or(usize::MAX), ArgumentType::Usize), arg.span).0 + as u64, + ), + } } fn expand_format_args<'hir>( @@ -399,82 +370,143 @@ fn expand_format_args<'hir>( fmt: &FormatArgs, allow_const: bool, ) -> hir::ExprKind<'hir> { + // // Create a list of all _unique_ (argument, format trait) combinations. + // E.g. "{0} {0:x} {0} {1}" -> [(0, Display), (0, LowerHex), (1, Display)] + // + // We use usize::MAX for arguments that don't exist, because that can never be a valid index + // into the arguments array. + let mut argmap = FxIndexMap::default(); + + // Generate: + // + // ``` + // &[ + // Piece { i: 4 }, + // Piece { p: "meow" as *const str as *const u8 }, + // Piece { i: 0xE000_0020_0000_0002 }, + // …, + // Piece { i: 0 }, + // ] + // ``` + let mut pieces = Vec::new(); + let mut incomplete_lit = String::new(); - let lit_pieces = - ctx.arena.alloc_from_iter(fmt.template.iter().enumerate().filter_map(|(i, piece)| { - match piece { - &FormatArgsPiece::Literal(s) => { - // Coalesce adjacent literal pieces. - if let Some(FormatArgsPiece::Literal(_)) = fmt.template.get(i + 1) { - incomplete_lit.push_str(s.as_str()); - None - } else if !incomplete_lit.is_empty() { - incomplete_lit.push_str(s.as_str()); - let s = Symbol::intern(&incomplete_lit); - incomplete_lit.clear(); - Some(ctx.expr_str(fmt.span, s)) - } else { - Some(ctx.expr_str(fmt.span, s)) - } + + let default_options = 0xE000_0020_0000_0000; + let mut implicit_arg_index = 0; + + for (i, piece) in fmt.template.iter().enumerate() { + match piece { + &FormatArgsPiece::Literal(sym) => { + assert!(!sym.is_empty()); + // Coalesce adjacent literal pieces. + if let Some(FormatArgsPiece::Literal(_)) = fmt.template.get(i + 1) { + incomplete_lit.push_str(sym.as_str()); + continue; } - &FormatArgsPiece::Placeholder(_) => { - // Inject empty string before placeholders when not already preceded by a literal piece. - if i == 0 || matches!(fmt.template[i - 1], FormatArgsPiece::Placeholder(_)) { - Some(ctx.expr_str(fmt.span, kw::Empty)) - } else { - None + let (sym, len) = if incomplete_lit.is_empty() { + (sym, sym.as_str().len()) + } else { + incomplete_lit.push_str(sym.as_str()); + let sym = Symbol::intern(&incomplete_lit); + let len = incomplete_lit.len(); + incomplete_lit.clear(); + (sym, len) + }; + + // ``` + // Piece::num(4), + // ``` + let i = if ctx.tcx.sess.target.pointer_width >= 64 { + ctx.expr_u64(fmt.span, len as u64) + } else { + ctx.expr_u32(fmt.span, len as u32) + }; + pieces.push(make_piece(ctx, sym::num, i, macsp)); + + // ``` + // Piece::str("meow"), + // ``` + let s = ctx.expr_str(fmt.span, sym); + pieces.push(make_piece(ctx, sym::str, s, macsp)); + } + FormatArgsPiece::Placeholder(ref p) => { + // ``` + // Piece::num(0xE000_0020_0000_0000), + // ``` + // Or, on <64 bit platforms: + // ``` + // Piece::num(0xE000_0020), + // Piece::num(0x0000_0000), + // ``` + + let bits = make_format_spec(p, &mut argmap); + + // If this placeholder uses the next argument index, is surrounded by literal string + // pieces, and uses default formatting options, then we can skip it, as this kind of + // placeholder is implied by two consequtive string pieces. + if bits == default_options + implicit_arg_index { + if let (Some(FormatArgsPiece::Literal(_)), Some(FormatArgsPiece::Literal(_))) = + (fmt.template.get(i.wrapping_sub(1)), fmt.template.get(i + 1)) + { + implicit_arg_index += 1; + continue; } } - } - })); - let lit_pieces = ctx.expr_array_ref(fmt.span, lit_pieces); - // Whether we'll use the `Arguments::new_v1_formatted` form (true), - // or the `Arguments::new_v1` form (false). - let mut use_format_options = false; + if ctx.tcx.sess.target.pointer_width >= 64 { + let bits = ctx.expr_u64(fmt.span, bits); + pieces.push(make_piece(ctx, sym::num, bits, macsp)); + } else { + let high = ctx.expr_u32(fmt.span, (bits >> 32) as u32); + let low = ctx.expr_u32(fmt.span, bits as u32); + pieces.push(make_piece(ctx, sym::num, high, macsp)); + pieces.push(make_piece(ctx, sym::num, low, macsp)); + } - // Create a list of all _unique_ (argument, format trait) combinations. - // E.g. "{0} {0:x} {0} {1}" -> [(0, Display), (0, LowerHex), (1, Display)] - let mut argmap = FxIndexMap::default(); - for piece in &fmt.template { - let FormatArgsPiece::Placeholder(placeholder) = piece else { continue }; - if placeholder.format_options != Default::default() { - // Can't use basic form if there's any formatting options. - use_format_options = true; - } - if let Ok(index) = placeholder.argument.index { - if argmap - .insert((index, ArgumentType::Format(placeholder.format_trait)), placeholder.span) - .is_some() - { - // Duplicate (argument, format trait) combination, - // which we'll only put once in the args array. - use_format_options = true; + implicit_arg_index = (bits & 0x3FF) + 1; } } } - let format_options = use_format_options.then(|| { - // Generate: - // &[format_spec_0, format_spec_1, format_spec_2] - let elements = ctx.arena.alloc_from_iter(fmt.template.iter().filter_map(|piece| { - let FormatArgsPiece::Placeholder(placeholder) = piece else { return None }; - Some(make_format_spec(ctx, macsp, placeholder, &mut argmap)) - })); - ctx.expr_array_ref(macsp, elements) - }); + assert!(incomplete_lit.is_empty()); + + // Zero terminator. + // + // ``` + // Piece::num(0), + // ``` + let zero = ctx.expr_u64(fmt.span, 0); + pieces.push(make_piece(ctx, sym::num, zero, macsp)); + + // ``` + // unsafe { ::new(&[pieces…]) } + // ``` + let template_new = + ctx.expr_lang_item_type_relative(macsp, hir::LangItem::FormatTemplate, sym::new); + let pieces = ctx.expr_array_ref(fmt.span, ctx.arena.alloc_from_iter(pieces)); + let template = ctx.expr( + macsp, + hir::ExprKind::Call(ctx.arena.alloc(template_new), ctx.arena.alloc_from_iter([pieces])), + ); + let template = ctx.expr_unsafe_block(macsp, ctx.arena.alloc(template)); + + // Ensure all argument indexes actually fit in 10 bits, as we truncated them to 10 bits before. + if argmap.len() >= 1 << 10 { + ctx.dcx().emit_err(TooManyFormatArguments { span: fmt.span }); + } let arguments = fmt.arguments.all_args(); if allow_const && arguments.is_empty() && argmap.is_empty() { // Generate: - // ::new_const(lit_pieces) + // ::new_const(template) let new = ctx.arena.alloc(ctx.expr_lang_item_type_relative( macsp, hir::LangItem::FormatArguments, sym::new_const, )); - let new_args = ctx.arena.alloc_from_iter([lit_pieces]); + let new_args = ctx.arena.alloc_from_iter([template]); return hir::ExprKind::Call(new, new_args); } @@ -485,11 +517,19 @@ fn expand_format_args<'hir>( // // This is an optimization, speeding up compilation about 1-2% in some cases. // See https://github.com/rust-lang/rust/pull/106770#issuecomment-1380790609 - let use_simple_array = argmap.len() == arguments.len() - && argmap.iter().enumerate().all(|(i, (&(j, _), _))| i == j) + let use_simple_array = argmap + .iter() + // Don't care about non-arguments. + .filter(|(&(arg, _), _)| arg != usize::MAX) + .enumerate() + // Don't care about captured arguments, as their expression has no side effects. + .filter(|(_, (&(arg, _), _))| !arguments[arg].kind.is_captured()) + // Check that the argument indexes are used one by one in order. + .all(|(i, (&(arg, _), _))| i == arg) + // And check that none except possibly the first argument have a yield point. && arguments.iter().skip(1).all(|arg| !may_contain_yield_point(&arg.expr)); - let args = if arguments.is_empty() { + let args = if argmap.is_empty() && arguments.is_empty() { // Generate: // &::none() // @@ -520,22 +560,37 @@ fn expand_format_args<'hir>( // ::new_debug(&arg2), // … // ] - let elements = ctx.arena.alloc_from_iter(arguments.iter().zip(argmap).map( - |(arg, ((_, ty), placeholder_span))| { - let placeholder_span = - placeholder_span.unwrap_or(arg.expr.span).with_ctxt(macsp.ctxt()); - let arg_span = match arg.kind { - FormatArgumentKind::Captured(_) => placeholder_span, - _ => arg.expr.span.with_ctxt(macsp.ctxt()), - }; - let arg = ctx.lower_expr(&arg.expr); - let ref_arg = ctx.arena.alloc(ctx.expr( - arg_span, - hir::ExprKind::AddrOf(hir::BorrowKind::Ref, hir::Mutability::Not, arg), - )); - make_argument(ctx, placeholder_span, ref_arg, ty) - }, - )); + let elements = + ctx.arena.alloc_from_iter(argmap.iter().map(|(&(arg_index, ty), placeholder_span)| { + if let ArgumentType::Constant(c) = ty { + let arg = ctx.arena.alloc(ctx.expr_usize(macsp, c)); + let arg = ctx.arena.alloc(ctx.expr( + macsp, + hir::ExprKind::AddrOf(hir::BorrowKind::Ref, hir::Mutability::Not, arg), + )); + make_argument(ctx, macsp, arg, ty) + } else if let Some(arg) = arguments.get(arg_index) { + let placeholder_span = + placeholder_span.unwrap_or(arg.expr.span).with_ctxt(macsp.ctxt()); + let arg_span = match arg.kind { + FormatArgumentKind::Captured(_) => placeholder_span, + _ => arg.expr.span.with_ctxt(macsp.ctxt()), + }; + let arg = ctx.lower_expr(&arg.expr); + let ref_arg = ctx.arena.alloc(ctx.expr( + arg_span, + hir::ExprKind::AddrOf(hir::BorrowKind::Ref, hir::Mutability::Not, arg), + )); + make_argument(ctx, placeholder_span, ref_arg, ty) + } else { + ctx.expr( + macsp, + hir::ExprKind::Err( + ctx.dcx().span_delayed_bug(macsp, "missing format_args argument"), + ), + ) + } + })); ctx.expr_array_ref(macsp, elements) } else { // Generate: @@ -551,22 +606,37 @@ fn expand_format_args<'hir>( let (args_pat, args_hir_id) = ctx.pat_ident(macsp, args_ident); let args = ctx.arena.alloc_from_iter(argmap.iter().map( |(&(arg_index, ty), &placeholder_span)| { - let arg = &arguments[arg_index]; - let placeholder_span = - placeholder_span.unwrap_or(arg.expr.span).with_ctxt(macsp.ctxt()); - let arg_span = match arg.kind { - FormatArgumentKind::Captured(_) => placeholder_span, - _ => arg.expr.span.with_ctxt(macsp.ctxt()), - }; - let args_ident_expr = ctx.expr_ident(macsp, args_ident, args_hir_id); - let arg = ctx.arena.alloc(ctx.expr( - arg_span, - hir::ExprKind::Field( - args_ident_expr, - Ident::new(sym::integer(arg_index), macsp), - ), - )); - make_argument(ctx, placeholder_span, arg, ty) + if let ArgumentType::Constant(c) = ty { + let arg = ctx.arena.alloc(ctx.expr_usize(macsp, c)); + let arg = ctx.arena.alloc(ctx.expr( + macsp, + hir::ExprKind::AddrOf(hir::BorrowKind::Ref, hir::Mutability::Not, arg), + )); + make_argument(ctx, macsp, arg, ty) + } else if let Some(arg) = arguments.get(arg_index) { + let placeholder_span = + placeholder_span.unwrap_or(arg.expr.span).with_ctxt(macsp.ctxt()); + let arg_span = match arg.kind { + FormatArgumentKind::Captured(_) => placeholder_span, + _ => arg.expr.span.with_ctxt(macsp.ctxt()), + }; + let args_ident_expr = ctx.expr_ident(macsp, args_ident, args_hir_id); + let arg = ctx.arena.alloc(ctx.expr( + arg_span, + hir::ExprKind::Field( + args_ident_expr, + Ident::new(sym::integer(arg_index), macsp), + ), + )); + make_argument(ctx, placeholder_span, arg, ty) + } else { + ctx.expr( + macsp, + hir::ExprKind::Err( + ctx.dcx().span_delayed_bug(macsp, "missing format_args argument"), + ), + ) + } }, )); let elements = ctx.arena.alloc_from_iter(arguments.iter().map(|arg| { @@ -591,50 +661,15 @@ fn expand_format_args<'hir>( ) }; - if let Some(format_options) = format_options { - // Generate: - // ::new_v1_formatted( - // lit_pieces, - // args, - // format_options, - // unsafe { ::core::fmt::UnsafeArg::new() } - // ) - let new_v1_formatted = ctx.arena.alloc(ctx.expr_lang_item_type_relative( - macsp, - hir::LangItem::FormatArguments, - sym::new_v1_formatted, - )); - let unsafe_arg_new = ctx.arena.alloc(ctx.expr_lang_item_type_relative( - macsp, - hir::LangItem::FormatUnsafeArg, - sym::new, - )); - let unsafe_arg_new_call = ctx.expr_call(macsp, unsafe_arg_new, &[]); - let hir_id = ctx.next_id(); - let unsafe_arg = ctx.expr_block(ctx.arena.alloc(hir::Block { - stmts: &[], - expr: Some(unsafe_arg_new_call), - hir_id, - rules: hir::BlockCheckMode::UnsafeBlock(hir::UnsafeSource::CompilerGenerated), - span: macsp, - targeted_by_break: false, - })); - let args = ctx.arena.alloc_from_iter([lit_pieces, args, format_options, unsafe_arg]); - hir::ExprKind::Call(new_v1_formatted, args) - } else { - // Generate: - // ::new_v1( - // lit_pieces, - // args, - // ) - let new_v1 = ctx.arena.alloc(ctx.expr_lang_item_type_relative( - macsp, - hir::LangItem::FormatArguments, - sym::new_v1, - )); - let new_args = ctx.arena.alloc_from_iter([lit_pieces, args]); - hir::ExprKind::Call(new_v1, new_args) - } + // Generate: + // ::new(template, args) + let new = ctx.arena.alloc(ctx.expr_lang_item_type_relative( + macsp, + hir::LangItem::FormatArguments, + sym::new, + )); + let new_args = ctx.arena.alloc_from_iter([template, args]); + hir::ExprKind::Call(new, new_args) } fn may_contain_yield_point(e: &ast::Expr) -> bool { diff --git a/compiler/rustc_hir/src/lang_items.rs b/compiler/rustc_hir/src/lang_items.rs index 35b3f871f5cd9..68cc3b5f2c368 100644 --- a/compiler/rustc_hir/src/lang_items.rs +++ b/compiler/rustc_hir/src/lang_items.rs @@ -323,9 +323,8 @@ language_item_table! { // Lang items needed for `format_args!()`. FormatArgument, sym::format_argument, format_argument, Target::Struct, GenericRequirement::None; FormatArguments, sym::format_arguments, format_arguments, Target::Struct, GenericRequirement::None; - FormatCount, sym::format_count, format_count, Target::Enum, GenericRequirement::None; - FormatPlaceholder, sym::format_placeholder, format_placeholder, Target::Struct, GenericRequirement::None; - FormatUnsafeArg, sym::format_unsafe_arg, format_unsafe_arg, Target::Struct, GenericRequirement::None; + FormatPiece, sym::format_piece, format_piece, Target::Union, GenericRequirement::None; + FormatTemplate, sym::format_template, format_template, Target::Struct, GenericRequirement::None; ExchangeMalloc, sym::exchange_malloc, exchange_malloc_fn, Target::Fn, GenericRequirement::None; DropInPlace, sym::drop_in_place, drop_in_place_fn, Target::Fn, GenericRequirement::Minimum(1); diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs index be48296ef5348..76af8958c4769 100644 --- a/compiler/rustc_span/src/symbol.rs +++ b/compiler/rustc_span/src/symbol.rs @@ -242,7 +242,7 @@ symbols! { HashMapEntry, HashSet, Hasher, - Implied, + IMPLIED, InCleanup, IndexOutput, Input, @@ -981,10 +981,9 @@ symbols! { format_args_nl, format_argument, format_arguments, - format_count, format_macro, - format_placeholder, - format_unsafe_arg, + format_piece, + format_template, freeze, freeze_impls, freg, @@ -1422,6 +1421,7 @@ symbols! { not, notable_trait, note, + num, object_safe_for_dispatch, of, off, diff --git a/library/core/src/fmt/mod.rs b/library/core/src/fmt/mod.rs index 6a8fe258f5ec0..437dff6b56a40 100644 --- a/library/core/src/fmt/mod.rs +++ b/library/core/src/fmt/mod.rs @@ -7,6 +7,8 @@ use crate::char::EscapeDebugExtArgs; use crate::marker::PhantomData; use crate::num::fmt as numfmt; use crate::ops::Deref; +#[cfg(not(bootstrap))] +use crate::ptr::NonNull; use crate::{iter, mem, result, str}; mod builders; @@ -297,7 +299,7 @@ pub struct FormattingOptions { /// └─ Always set. /// This makes it possible to distinguish formatting flags from /// a &str size when stored in (the upper bits of) the same field. - /// (fmt::Arguments will make use of this property in the future.) + /// (fmt::Arguments makes use of this property.) /// ``` flags: u32, /// Width if width flag (bit 27) above is set. Otherwise, always 0. @@ -583,6 +585,71 @@ impl<'a> Formatter<'a> { } } +/// Bootstrap only. +#[cfg(bootstrap)] +#[lang = "format_arguments"] +#[stable(feature = "rust1", since = "1.0.0")] +#[derive(Copy, Clone)] +pub struct Arguments<'a> { + pieces: &'a [&'static str], + fmt: Option<&'a [rt::Placeholder]>, + args: &'a [rt::Argument<'a>], +} + +#[cfg(bootstrap)] +#[doc(hidden)] +#[unstable(feature = "fmt_internals", issue = "none")] +impl<'a> Arguments<'a> { + #[inline] + pub const fn new_const(pieces: &'a [&'static str; N]) -> Self { + const { assert!(N <= 1) }; + Arguments { pieces, fmt: None, args: &[] } + } + + #[inline] + pub const fn new_v1( + pieces: &'a [&'static str; P], + args: &'a [rt::Argument<'a>; A], + ) -> Arguments<'a> { + const { assert!(P >= A && P <= A + 1, "invalid args") } + Arguments { pieces, fmt: None, args } + } + + #[inline] + pub const fn new_v1_formatted( + pieces: &'a [&'static str], + args: &'a [rt::Argument<'a>], + fmt: &'a [rt::Placeholder], + _unsafe_arg: rt::UnsafeArg, + ) -> Arguments<'a> { + Arguments { pieces, fmt: Some(fmt), args } + } + + #[inline] + pub fn estimated_capacity(&self) -> usize { + 0 + } + + #[stable(feature = "fmt_as_str", since = "1.52.0")] + #[rustc_const_stable(feature = "const_arguments_as_str", since = "1.84.0")] + #[must_use] + #[inline] + pub const fn as_str(&self) -> Option<&'static str> { + None + } + + // These two methods are used in library/core/src/panicking.rs to create a + // `fmt::Arguments` for a `&'static str`. + #[inline] + pub(crate) const fn pieces_for_str(s: &'static str) -> [&'static str; 1] { + [s] + } + #[inline] + pub(crate) const unsafe fn from_pieces(p: &'a [&'static str; 1]) -> Self { + Self::new_const(p) + } +} + /// This structure represents a safely precompiled version of a format string /// and its arguments. This cannot be generated at runtime because it cannot /// safely be done, so no constructors are given and the fields are private @@ -605,57 +672,32 @@ impl<'a> Formatter<'a> { /// ``` /// /// [`format()`]: ../../std/fmt/fn.format.html +#[cfg(not(bootstrap))] +#[cfg(target_pointer_width = "64")] #[lang = "format_arguments"] #[stable(feature = "rust1", since = "1.0.0")] #[derive(Copy, Clone)] pub struct Arguments<'a> { - // Format string pieces to print. - pieces: &'a [&'static str], - - // Placeholder specs, or `None` if all specs are default (as in "{}{}"). - fmt: Option<&'a [rt::Placeholder]>, - - // Dynamic arguments for interpolation, to be interleaved with string - // pieces. (Every argument is preceded by a string piece.) - args: &'a [rt::Argument<'a>], + template: rt::Template<'a>, + args: NonNull>, } /// Used by the format_args!() macro to create a fmt::Arguments object. +#[cfg(not(bootstrap))] #[doc(hidden)] #[unstable(feature = "fmt_internals", issue = "none")] impl<'a> Arguments<'a> { #[inline] - pub const fn new_const(pieces: &'a [&'static str; N]) -> Self { - const { assert!(N <= 1) }; - Arguments { pieces, fmt: None, args: &[] } + pub const fn new_const(template: rt::Template<'a>) -> Arguments<'a> { + Arguments { template, args: NonNull::dangling() } } - /// When using the format_args!() macro, this function is used to generate the - /// Arguments structure. #[inline] - pub const fn new_v1( - pieces: &'a [&'static str; P], - args: &'a [rt::Argument<'a>; A], - ) -> Arguments<'a> { - const { assert!(P >= A && P <= A + 1, "invalid args") } - Arguments { pieces, fmt: None, args } - } - - /// Specifies nonstandard formatting parameters. - /// - /// An `rt::UnsafeArg` is required because the following invariants must be held - /// in order for this function to be safe: - /// 1. The `pieces` slice must be at least as long as `fmt`. - /// 2. Every `rt::Placeholder::position` value within `fmt` must be a valid index of `args`. - /// 3. Every `rt::Count::Param` within `fmt` must contain a valid index of `args`. - #[inline] - pub const fn new_v1_formatted( - pieces: &'a [&'static str], - args: &'a [rt::Argument<'a>], - fmt: &'a [rt::Placeholder], - _unsafe_arg: rt::UnsafeArg, + pub const fn new( + template: rt::Template<'a>, + args: &'a [rt::Argument<'a>; N], ) -> Arguments<'a> { - Arguments { pieces, fmt: Some(fmt), args } + Arguments { template, args: NonNull::from_ref(args).cast() } } /// Estimates the length of the formatted text. @@ -664,22 +706,66 @@ impl<'a> Arguments<'a> { /// when using `format!`. Note: this is neither the lower nor upper bound. #[inline] pub fn estimated_capacity(&self) -> usize { - let pieces_length: usize = self.pieces.iter().map(|x| x.len()).sum(); + // Iterate over the template, counting the length of literal pieces. + let mut length = 0usize; + let mut starts_with_placeholder = false; + let mut template = self.template; + let mut has_placeholders = false; + loop { + // SAFETY: We can assume the template is valid. + unsafe { + let n = template.next().i; + if n == 0 { + // End of template. + break; + } else if n <= isize::MAX as _ { + // Literal string piece. + if length != 0 { + // More than one literal string piece means we have placeholders. + has_placeholders = true; + } + length += n as usize; + let _ptr = template.next(); // Skip the string pointer. + } else { + // Placeholder piece. + if length == 0 { + starts_with_placeholder = true; + } + has_placeholders = true; + #[cfg(not(target_pointer_width = "64"))] + let _ = template.next(); // Skip second half of placeholder. + } + } + } - if self.args.is_empty() { - pieces_length - } else if !self.pieces.is_empty() && self.pieces[0].is_empty() && pieces_length < 16 { - // If the format string starts with an argument, + if !has_placeholders { + // If the template has no placeholders, we know the length exactly. + length + } else if starts_with_placeholder && length < 16 { + // If the format string starts with a placeholder, // don't preallocate anything, unless length - // of pieces is significant. + // of literal pieces is significant. 0 } else { - // There are some arguments, so any additional push + // There are some placeholders, so any additional push // will reallocate the string. To avoid that, // we're "pre-doubling" the capacity here. - pieces_length.checked_mul(2).unwrap_or(0) + length.wrapping_mul(2) } } + + // These two methods are used in library/core/src/panicking.rs to create a + // `fmt::Arguments` for a `&'static str`. + #[inline] + pub(crate) const fn pieces_for_str(s: &'static str) -> [rt::Piece; 3] { + [rt::Piece { i: s.len() as _ }, rt::Piece::str(s), rt::Piece { i: 0 }] + } + /// Safety: Only call this with the result of `pieces_for_str`. + #[inline] + pub(crate) const unsafe fn from_pieces(p: &'a [rt::Piece; 3]) -> Self { + // SAFETY: Guaranteed by caller. + Self::new_const(unsafe { rt::Template::new(p) }) + } } impl<'a> Arguments<'a> { @@ -725,16 +811,33 @@ impl<'a> Arguments<'a> { /// assert_eq!(format_args!("").as_str(), Some("")); /// assert_eq!(format_args!("{:?}", std::env::current_dir()).as_str(), None); /// ``` + #[cfg(not(bootstrap))] #[stable(feature = "fmt_as_str", since = "1.52.0")] #[rustc_const_stable(feature = "const_arguments_as_str", since = "1.84.0")] #[must_use] #[inline] pub const fn as_str(&self) -> Option<&'static str> { - match (self.pieces, self.args) { - ([], []) => Some(""), - ([s], []) => Some(s), - _ => None, + let mut template = self.template; + // SAFETY: We can assume the template is valid. + let n = unsafe { template.next().i }; + if n == 0 { + // The template is empty. + return Some(""); } + if n <= isize::MAX as _ { + // Template starts with a string piece. + // SAFETY: We can assume the template is valid. + unsafe { + let ptr = template.next().p; + if template.next().i == 0 { + // The template has only one piece. + return Some(str::from_utf8_unchecked(crate::slice::from_raw_parts( + ptr, n as usize, + ))); + } + } + } + None } /// Same as [`Arguments::as_str`], but will only return `Some(s)` if it can be determined at compile time. @@ -1424,37 +1527,8 @@ pub trait UpperExp { fn fmt(&self, f: &mut Formatter<'_>) -> Result; } -/// Takes an output stream and an `Arguments` struct that can be precompiled with -/// the `format_args!` macro. -/// -/// The arguments will be formatted according to the specified format string -/// into the output stream provided. -/// -/// # Examples -/// -/// Basic usage: -/// -/// ``` -/// use std::fmt; -/// -/// let mut output = String::new(); -/// fmt::write(&mut output, format_args!("Hello {}!", "world")) -/// .expect("Error occurred while trying to write in String"); -/// assert_eq!(output, "Hello world!"); -/// ``` -/// -/// Please note that using [`write!`] might be preferable. Example: -/// -/// ``` -/// use std::fmt::Write; -/// -/// let mut output = String::new(); -/// write!(&mut output, "Hello {}!", "world") -/// .expect("Error occurred while trying to write in String"); -/// assert_eq!(output, "Hello world!"); -/// ``` -/// -/// [`write!`]: crate::write! +/// Bootstrap only +#[cfg(bootstrap)] #[stable(feature = "rust1", since = "1.0.0")] pub fn write(output: &mut dyn Write, args: Arguments<'_>) -> Result { let mut formatter = Formatter::new(output, FormattingOptions::new()); @@ -1505,14 +1579,14 @@ pub fn write(output: &mut dyn Write, args: Arguments<'_>) -> Result { Ok(()) } +#[cfg(bootstrap)] unsafe fn run(fmt: &mut Formatter<'_>, arg: &rt::Placeholder, args: &[rt::Argument<'_>]) -> Result { let (width, precision) = // SAFETY: arg and args come from the same Arguments, // which guarantees the indexes are always within bounds. unsafe { (getcount(args, &arg.width), getcount(args, &arg.precision)) }; - #[cfg(bootstrap)] - let options = + fmt.options = *FormattingOptions { flags: flags::ALWAYS_SET | arg.flags << 21, width: 0, precision: 0 } .align(match arg.align { rt::Alignment::Left => Some(Alignment::Left), @@ -1523,17 +1597,13 @@ unsafe fn run(fmt: &mut Formatter<'_>, arg: &rt::Placeholder, args: &[rt::Argume .fill(arg.fill) .width(width) .precision(precision); - #[cfg(not(bootstrap))] - let options = FormattingOptions { flags: arg.flags, width, precision }; + let position = arg.position as usize; // Extract the correct argument - debug_assert!(arg.position < args.len()); + debug_assert!(position < args.len()); // SAFETY: arg and args come from the same Arguments, // which guarantees its index is always within bounds. - let value = unsafe { args.get_unchecked(arg.position) }; - - // Set all the formatting options. - fmt.options = options; + let value = unsafe { args.get_unchecked(position) }; // Then actually do some printing // SAFETY: this is a placeholder argument. @@ -1554,16 +1624,99 @@ unsafe fn getcount(args: &[rt::Argument<'_>], cnt: &rt::Count) -> Option { } } +/// Takes an output stream and an `Arguments` struct that can be precompiled with +/// the `format_args!` macro. +/// +/// The arguments will be formatted according to the specified format string +/// into the output stream provided. +/// +/// # Examples +/// +/// Basic usage: +/// +/// ``` +/// use std::fmt; +/// +/// let mut output = String::new(); +/// fmt::write(&mut output, format_args!("Hello {}!", "world")) +/// .expect("Error occurred while trying to write in String"); +/// assert_eq!(output, "Hello world!"); +/// ``` +/// +/// Please note that using [`write!`] might be preferable. Example: +/// +/// ``` +/// use std::fmt::Write; +/// +/// let mut output = String::new(); +/// write!(&mut output, "Hello {}!", "world") +/// .expect("Error occurred while trying to write in String"); +/// assert_eq!(output, "Hello world!"); +/// ``` +/// +/// [`write!`]: crate::write! #[cfg(not(bootstrap))] -unsafe fn getcount(args: &[rt::Argument<'_>], cnt: &rt::Count) -> u16 { - match *cnt { - rt::Count::Is(n) => n, - rt::Count::Implied => 0, - rt::Count::Param(i) => { - debug_assert!(i < args.len()); - // SAFETY: cnt and args come from the same Arguments, - // which guarantees this index is always within bounds. - unsafe { args.get_unchecked(i).as_u16().unwrap_unchecked() } +#[stable(feature = "rust1", since = "1.0.0")] +pub fn write(output: &mut dyn Write, fmt: Arguments<'_>) -> Result { + let mut template = fmt.template; + let args = fmt.args; + + let mut last_piece_was_str = false; + let mut implicit_arg_index = 0; + + loop { + // SAFETY: We can assume the template is valid. + let n = unsafe { template.next().i }; + if n == 0 { + // End of template. + return Ok(()); + } else if n <= isize::MAX as _ { + // Literal string piece. + if last_piece_was_str { + // Two consecutive string pieces means we need to insert + // an implicit argument with default options. + let options = FormattingOptions::new(); + // SAFETY: We can assume the template only refers to arguments that exist. + let arg = unsafe { *args.add(implicit_arg_index).as_ref() }; + implicit_arg_index += 1; + // SAFETY: We can assume the placeholders match the arguments. + unsafe { arg.fmt(&mut Formatter::new(output, options)) }?; + } + // SAFETY: We can assume the strings in the template are valid. + let s = unsafe { crate::str::from_raw_parts(template.next().p, n as usize) }; + output.write_str(s)?; + last_piece_was_str = true; + } else { + // Placeholder piece. + #[cfg(target_pointer_width = "64")] + let (high, low) = ((n >> 32) as u32, n as u32); + #[cfg(not(target_pointer_width = "64"))] + // SAFETY: We can assume the template is valid. + let (high, low) = (n as u32, unsafe { template.next().i } as u32); + let arg_index = (low & 0x3FF) as usize; + let mut width = (low >> 10 & 0x3FF) as u16; + let mut precision = (low >> 20 & 0x3FF) as u16; + if low & 1 << 30 != 0 { + // Dynamic width from a usize argument. + // SAFETY: We can assume the template only refers to arguments that exist. + unsafe { + width = args.add(width as usize).as_ref().as_u16().unwrap_unchecked(); + } + } + if low & 1 << 31 != 0 { + // Dynamic precision from a usize argument. + // SAFETY: We can assume the template only refers to arguments that exist. + unsafe { + precision = args.add(precision as usize).as_ref().as_u16().unwrap_unchecked(); + } + } + let options = FormattingOptions { flags: high, width, precision }; + // SAFETY: We can assume the template only refers to arguments that exist. + let arg = unsafe { *args.add(arg_index).as_ref() }; + // SAFETY: We can assume the placeholders match the arguments. + unsafe { arg.fmt(&mut Formatter::new(output, options)) }?; + last_piece_was_str = false; + implicit_arg_index = arg_index + 1; } } } @@ -1738,7 +1891,7 @@ impl<'a> Formatter<'a> { #[stable(feature = "rust1", since = "1.0.0")] pub fn pad(&mut self, s: &str) -> Result { // Make sure there's a fast path up front - if self.options.flags & (flags::WIDTH_FLAG | flags::PRECISION_FLAG) == 0 { + if self.options.flags & (rt::WIDTH_FLAG | rt::PRECISION_FLAG) == 0 { return self.buf.write_str(s); } // The `precision` field can be interpreted as a `max-width` for the @@ -1973,9 +2126,8 @@ impl<'a> Formatter<'a> { or `sign_aware_zero_pad` methods instead" )] pub fn flags(&self) -> u32 { - // Extract the debug upper/lower hex, zero pad, alternate, and plus/minus flags - // to stay compatible with older versions of Rust. - self.options.flags >> 21 & 0x3F + // Shift out the fill character from the flags field. + self.options.flags >> 21 } /// Returns the character used as 'fill' whenever there is alignment. @@ -2073,7 +2225,7 @@ impl<'a> Formatter<'a> { #[must_use] #[stable(feature = "fmt_flags", since = "1.5.0")] pub fn width(&self) -> Option { - if self.options.flags & flags::WIDTH_FLAG == 0 { + if self.options.flags & rt::WIDTH_FLAG == 0 { None } else { Some(self.options.width as usize) @@ -2108,7 +2260,7 @@ impl<'a> Formatter<'a> { #[must_use] #[stable(feature = "fmt_flags", since = "1.5.0")] pub fn precision(&self) -> Option { - if self.options.flags & flags::PRECISION_FLAG == 0 { + if self.options.flags & rt::PRECISION_FLAG == 0 { None } else { Some(self.options.precision as usize) @@ -2144,7 +2296,7 @@ impl<'a> Formatter<'a> { #[must_use] #[stable(feature = "fmt_flags", since = "1.5.0")] pub fn sign_plus(&self) -> bool { - self.options.flags & flags::SIGN_PLUS_FLAG != 0 + self.options.flags & rt::SIGN_PLUS_FLAG != 0 } /// Determines if the `-` flag was specified. @@ -2173,7 +2325,7 @@ impl<'a> Formatter<'a> { #[must_use] #[stable(feature = "fmt_flags", since = "1.5.0")] pub fn sign_minus(&self) -> bool { - self.options.flags & flags::SIGN_MINUS_FLAG != 0 + self.options.flags & rt::SIGN_MINUS_FLAG != 0 } /// Determines if the `#` flag was specified. @@ -2201,7 +2353,7 @@ impl<'a> Formatter<'a> { #[must_use] #[stable(feature = "fmt_flags", since = "1.5.0")] pub fn alternate(&self) -> bool { - self.options.flags & flags::ALTERNATE_FLAG != 0 + self.options.flags & rt::ALTERNATE_FLAG != 0 } /// Determines if the `0` flag was specified. @@ -2227,16 +2379,16 @@ impl<'a> Formatter<'a> { #[must_use] #[stable(feature = "fmt_flags", since = "1.5.0")] pub fn sign_aware_zero_pad(&self) -> bool { - self.options.flags & flags::SIGN_AWARE_ZERO_PAD_FLAG != 0 + self.options.flags & rt::SIGN_AWARE_ZERO_PAD_FLAG != 0 } // FIXME: Decide what public API we want for these two flags. // https://github.com/rust-lang/rust/issues/48584 fn debug_lower_hex(&self) -> bool { - self.options.flags & flags::DEBUG_LOWER_HEX_FLAG != 0 + self.options.flags & rt::DEBUG_LOWER_HEX_FLAG != 0 } fn debug_upper_hex(&self) -> bool { - self.options.flags & flags::DEBUG_UPPER_HEX_FLAG != 0 + self.options.flags & rt::DEBUG_UPPER_HEX_FLAG != 0 } /// Creates a [`DebugStruct`] builder designed to assist with creation of @@ -2816,7 +2968,7 @@ impl Debug for char { #[stable(feature = "rust1", since = "1.0.0")] impl Display for char { fn fmt(&self, f: &mut Formatter<'_>) -> Result { - if f.options.flags & (flags::WIDTH_FLAG | flags::PRECISION_FLAG) == 0 { + if f.options.flags & (rt::WIDTH_FLAG | rt::PRECISION_FLAG) == 0 { f.write_char(*self) } else { f.pad(self.encode_utf8(&mut [0; 4])) diff --git a/library/core/src/fmt/rt.rs b/library/core/src/fmt/rt.rs index f6d5d2746b29e..0d7bc225507cb 100644 --- a/library/core/src/fmt/rt.rs +++ b/library/core/src/fmt/rt.rs @@ -5,23 +5,101 @@ use super::*; use crate::hint::unreachable_unchecked; +use crate::marker::PhantomData; use crate::ptr::NonNull; +#[cfg(not(bootstrap))] +#[lang = "format_template"] +#[derive(Copy, Clone)] +pub struct Template<'a> { + pub(super) pieces: NonNull, + lifetime: PhantomData<&'a rt::Piece>, +} + +#[cfg(not(bootstrap))] +unsafe impl Send for Template<'_> {} +#[cfg(not(bootstrap))] +unsafe impl Sync for Template<'_> {} + +#[cfg(not(bootstrap))] +impl<'a> Template<'a> { + #[inline] + pub const unsafe fn new(pieces: &'a [rt::Piece; N]) -> Self { + Self { pieces: NonNull::from_ref(pieces).cast(), lifetime: PhantomData } + } + + #[inline] + pub const unsafe fn next(&mut self) -> Piece { + // SAFETY: Guaranteed by caller. + unsafe { + let piece = *self.pieces.as_ref(); + self.pieces = self.pieces.add(1); + piece + } + } +} + +#[cfg(not(bootstrap))] +#[lang = "format_piece"] +#[derive(Copy, Clone)] +pub union Piece { + #[cfg(target_pointer_width = "64")] + pub i: u64, + #[cfg(not(target_pointer_width = "64"))] + pub i: u32, + pub p: *const u8, +} + +#[cfg(not(bootstrap))] +unsafe impl Send for Piece {} +#[cfg(not(bootstrap))] +unsafe impl Sync for Piece {} + +// These are marked as #[stable] because of #[rustc_promotable] and #[rustc_const_stable]. +// With #[rustc_const_unstable], many format_args!() invocations would result in errors. +// +// There is still no way to use these on stable, because Piece itself is #[unstable] and not +// reachable through any public path. (format_args!()'s expansion uses it as a lang item.) +#[cfg(not(bootstrap))] +impl Piece { + #[rustc_promotable] + #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_stable(feature = "rust1", since = "1.0.0")] + pub const fn str(s: &'static str) -> Self { + Self { p: s as *const str as *const u8 } + } + + #[cfg(target_pointer_width = "64")] + #[rustc_promotable] + #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_stable(feature = "rust1", since = "1.0.0")] + pub const fn num(i: u64) -> Self { + Self { i } + } + + #[cfg(not(target_pointer_width = "64"))] + #[rustc_promotable] + #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_stable(feature = "rust1", since = "1.0.0")] + pub const fn num(i: u32) -> Self { + Self { i } + } +} + +#[cfg(bootstrap)] #[lang = "format_placeholder"] #[derive(Copy, Clone)] pub struct Placeholder { pub position: usize, - #[cfg(bootstrap)] pub fill: char, - #[cfg(bootstrap)] pub align: Alignment, pub flags: u32, pub precision: Count, pub width: Count, } +#[cfg(bootstrap)] impl Placeholder { - #[cfg(bootstrap)] #[inline] pub const fn new( position: usize, @@ -33,12 +111,6 @@ impl Placeholder { ) -> Self { Self { position, fill, align, flags, precision, width } } - - #[cfg(not(bootstrap))] - #[inline] - pub const fn new(position: usize, flags: u32, precision: Count, width: Count) -> Self { - Self { position, flags, precision, width } - } } #[cfg(bootstrap)] @@ -53,21 +125,34 @@ pub enum Alignment { /// Used by [width](https://doc.rust-lang.org/std/fmt/#width) /// and [precision](https://doc.rust-lang.org/std/fmt/#precision) specifiers. +#[cfg(bootstrap)] #[lang = "format_count"] #[derive(Copy, Clone)] pub enum Count { /// Specified with a literal number, stores the value - #[cfg(bootstrap)] Is(usize), - /// Specified with a literal number, stores the value - #[cfg(not(bootstrap))] - Is(u16), /// Specified using `$` and `*` syntaxes, stores the index into `args` Param(usize), /// Not specified Implied, } +// This needs to match with compiler/rustc_ast_lowering/src/format.rs. +pub const SIGN_PLUS_FLAG: u32 = 1 << 21; +pub const SIGN_MINUS_FLAG: u32 = 1 << 22; +pub const ALTERNATE_FLAG: u32 = 1 << 23; +pub const SIGN_AWARE_ZERO_PAD_FLAG: u32 = 1 << 24; +pub const DEBUG_LOWER_HEX_FLAG: u32 = 1 << 25; +pub const DEBUG_UPPER_HEX_FLAG: u32 = 1 << 26; +pub const WIDTH_FLAG: u32 = 1 << 27; +pub const PRECISION_FLAG: u32 = 1 << 28; +pub const ALIGN_BITS: u32 = 0b11 << 29; +pub const ALIGN_LEFT: u32 = 0 << 29; +pub const ALIGN_RIGHT: u32 = 1 << 29; +pub const ALIGN_CENTER: u32 = 2 << 29; +pub const ALIGN_UNKNOWN: u32 = 3 << 29; +pub const ALWAYS_SET: u32 = 1 << 31; + #[derive(Copy, Clone)] enum ArgumentType<'a> { Placeholder { @@ -210,17 +295,14 @@ impl Argument<'_> { } } -/// This struct represents the unsafety of constructing an `Arguments`. -/// It exists, rather than an unsafe function, in order to simplify the expansion -/// of `format_args!(..)` and reduce the scope of the `unsafe` block. +#[cfg(bootstrap)] #[lang = "format_unsafe_arg"] pub struct UnsafeArg { _private: (), } +#[cfg(bootstrap)] impl UnsafeArg { - /// See documentation where `UnsafeArg` is required to know when it is safe to - /// create and use `UnsafeArg`. #[inline] pub const unsafe fn new() -> Self { Self { _private: () } diff --git a/library/core/src/panicking.rs b/library/core/src/panicking.rs index d36e677d21a18..3b438fd515cfb 100644 --- a/library/core/src/panicking.rs +++ b/library/core/src/panicking.rs @@ -142,7 +142,8 @@ pub const fn panic(expr: &'static str) -> ! { // payload without any allocation or copying. Shorter-lived strings would become invalid as // stack frames get popped during unwinding, and couldn't be directly referenced from the // payload. - panic_fmt(fmt::Arguments::new_const(&[expr])); + // SAFETY: This is the correct way to make a `fmt::Arguments` from a string literal. + panic_fmt(unsafe { fmt::Arguments::from_pieces(&fmt::Arguments::pieces_for_str(expr)) }); } // We generate functions for usage by compiler-generated assertions. @@ -169,13 +170,8 @@ macro_rules! panic_const { #[rustc_const_stable_indirect] // must follow stable const rules since it is exposed to stable #[lang = stringify!($lang)] pub const fn $lang() -> ! { - // Use Arguments::new_const instead of format_args!("{expr}") to potentially - // reduce size overhead. The format_args! macro uses str's Display trait to - // write expr, which calls Formatter::pad, which must accommodate string - // truncation and padding (even though none is used here). Using - // Arguments::new_const may allow the compiler to omit Formatter::pad from the - // output binary, saving up to a few kilobytes. - panic_fmt(fmt::Arguments::new_const(&[$message])); + // SAFETY: This is the correct way to make a `fmt::Arguments` from a string literal. + panic_fmt(unsafe { fmt::Arguments::from_pieces(&fmt::Arguments::pieces_for_str($message)) }); } )+ } @@ -215,7 +211,11 @@ panic_const! { #[rustc_nounwind] #[rustc_const_stable_indirect] // must follow stable const rules since it is exposed to stable pub const fn panic_nounwind(expr: &'static str) -> ! { - panic_nounwind_fmt(fmt::Arguments::new_const(&[expr]), /* force_no_backtrace */ false); + panic_nounwind_fmt( + // SAFETY: This is the correct way to make a `fmt::Arguments` from a string literal. + unsafe { fmt::Arguments::from_pieces(&fmt::Arguments::pieces_for_str(expr)) }, + /* force_no_backtrace */ false, + ); } /// Like `panic_nounwind`, but also inhibits showing a backtrace. @@ -223,7 +223,11 @@ pub const fn panic_nounwind(expr: &'static str) -> ! { #[cfg_attr(feature = "panic_immediate_abort", inline)] #[rustc_nounwind] pub fn panic_nounwind_nobacktrace(expr: &'static str) -> ! { - panic_nounwind_fmt(fmt::Arguments::new_const(&[expr]), /* force_no_backtrace */ true); + panic_nounwind_fmt( + // SAFETY: This is the correct way to make a `fmt::Arguments` from a string literal. + unsafe { fmt::Arguments::from_pieces(&fmt::Arguments::pieces_for_str(expr)) }, + /* force_no_backtrace */ true, + ); } #[track_caller]