Skip to content

Commit 8e54a21

Browse files
borsbors
committed
Auto merge of #81238 - RalfJung:copy-intrinsics, r=m-ou-se
directly expose copy and copy_nonoverlapping intrinsics This effectively un-does #57997. That should help with `ptr::read` codegen in debug builds (and any other of these low-level functions that bottoms out at `copy`/`copy_nonoverlapping`), where the wrapper function will not get inlined. See the discussion in #80290 and #81163. Cc `@bjorn3` `@therealprof`
2 parents 3c10a88 + 1a80635 commit 8e54a21

File tree

4 files changed

+180
-223
lines changed

4 files changed

+180
-223
lines changed

‎library/core/src/intrinsics.rs

+151-186
Original file line numberDiff line numberDiff line change
@@ -1730,6 +1730,157 @@ extern "rust-intrinsic" {
17301730
/// Allocate at compile time. Should not be called at runtime.
17311731
#[rustc_const_unstable(feature = "const_heap", issue = "79597")]
17321732
pub fn const_allocate(size: usize, align: usize) -> *mut u8;
1733+
1734+
/// Copies `count * size_of::<T>()` bytes from `src` to `dst`. The source
1735+
/// and destination must *not* overlap.
1736+
///
1737+
/// For regions of memory which might overlap, use [`copy`] instead.
1738+
///
1739+
/// `copy_nonoverlapping` is semantically equivalent to C's [`memcpy`], but
1740+
/// with the argument order swapped.
1741+
///
1742+
/// [`memcpy`]: https://en.cppreference.com/w/c/string/byte/memcpy
1743+
///
1744+
/// # Safety
1745+
///
1746+
/// Behavior is undefined if any of the following conditions are violated:
1747+
///
1748+
/// * `src` must be [valid] for reads of `count * size_of::<T>()` bytes.
1749+
///
1750+
/// * `dst` must be [valid] for writes of `count * size_of::<T>()` bytes.
1751+
///
1752+
/// * Both `src` and `dst` must be properly aligned.
1753+
///
1754+
/// * The region of memory beginning at `src` with a size of `count *
1755+
/// size_of::<T>()` bytes must *not* overlap with the region of memory
1756+
/// beginning at `dst` with the same size.
1757+
///
1758+
/// Like [`read`], `copy_nonoverlapping` creates a bitwise copy of `T`, regardless of
1759+
/// whether `T` is [`Copy`]. If `T` is not [`Copy`], using *both* the values
1760+
/// in the region beginning at `*src` and the region beginning at `*dst` can
1761+
/// [violate memory safety][read-ownership].
1762+
///
1763+
/// Note that even if the effectively copied size (`count * size_of::<T>()`) is
1764+
/// `0`, the pointers must be non-NULL and properly aligned.
1765+
///
1766+
/// [`read`]: crate::ptr::read
1767+
/// [read-ownership]: crate::ptr::read#ownership-of-the-returned-value
1768+
/// [valid]: crate::ptr#safety
1769+
///
1770+
/// # Examples
1771+
///
1772+
/// Manually implement [`Vec::append`]:
1773+
///
1774+
/// ```
1775+
/// use std::ptr;
1776+
///
1777+
/// /// Moves all the elements of `src` into `dst`, leaving `src` empty.
1778+
/// fn append<T>(dst: &mut Vec<T>, src: &mut Vec<T>) {
1779+
/// let src_len = src.len();
1780+
/// let dst_len = dst.len();
1781+
///
1782+
/// // Ensure that `dst` has enough capacity to hold all of `src`.
1783+
/// dst.reserve(src_len);
1784+
///
1785+
/// unsafe {
1786+
/// // The call to offset is always safe because `Vec` will never
1787+
/// // allocate more than `isize::MAX` bytes.
1788+
/// let dst_ptr = dst.as_mut_ptr().offset(dst_len as isize);
1789+
/// let src_ptr = src.as_ptr();
1790+
///
1791+
/// // Truncate `src` without dropping its contents. We do this first,
1792+
/// // to avoid problems in case something further down panics.
1793+
/// src.set_len(0);
1794+
///
1795+
/// // The two regions cannot overlap because mutable references do
1796+
/// // not alias, and two different vectors cannot own the same
1797+
/// // memory.
1798+
/// ptr::copy_nonoverlapping(src_ptr, dst_ptr, src_len);
1799+
///
1800+
/// // Notify `dst` that it now holds the contents of `src`.
1801+
/// dst.set_len(dst_len + src_len);
1802+
/// }
1803+
/// }
1804+
///
1805+
/// let mut a = vec!['r'];
1806+
/// let mut b = vec!['u', 's', 't'];
1807+
///
1808+
/// append(&mut a, &mut b);
1809+
///
1810+
/// assert_eq!(a, &['r', 'u', 's', 't']);
1811+
/// assert!(b.is_empty());
1812+
/// ```
1813+
///
1814+
/// [`Vec::append`]: ../../std/vec/struct.Vec.html#method.append
1815+
#[doc(alias = "memcpy")]
1816+
#[stable(feature = "rust1", since = "1.0.0")]
1817+
#[rustc_const_unstable(feature = "const_intrinsic_copy", issue = "80697")]
1818+
pub fn copy_nonoverlapping<T>(src: *const T, dst: *mut T, count: usize);
1819+
1820+
/// Copies `count * size_of::<T>()` bytes from `src` to `dst`. The source
1821+
/// and destination may overlap.
1822+
///
1823+
/// If the source and destination will *never* overlap,
1824+
/// [`copy_nonoverlapping`] can be used instead.
1825+
///
1826+
/// `copy` is semantically equivalent to C's [`memmove`], but with the argument
1827+
/// order swapped. Copying takes place as if the bytes were copied from `src`
1828+
/// to a temporary array and then copied from the array to `dst`.
1829+
///
1830+
/// [`memmove`]: https://en.cppreference.com/w/c/string/byte/memmove
1831+
///
1832+
/// # Safety
1833+
///
1834+
/// Behavior is undefined if any of the following conditions are violated:
1835+
///
1836+
/// * `src` must be [valid] for reads of `count * size_of::<T>()` bytes.
1837+
///
1838+
/// * `dst` must be [valid] for writes of `count * size_of::<T>()` bytes.
1839+
///
1840+
/// * Both `src` and `dst` must be properly aligned.
1841+
///
1842+
/// Like [`read`], `copy` creates a bitwise copy of `T`, regardless of
1843+
/// whether `T` is [`Copy`]. If `T` is not [`Copy`], using both the values
1844+
/// in the region beginning at `*src` and the region beginning at `*dst` can
1845+
/// [violate memory safety][read-ownership].
1846+
///
1847+
/// Note that even if the effectively copied size (`count * size_of::<T>()`) is
1848+
/// `0`, the pointers must be non-NULL and properly aligned.
1849+
///
1850+
/// [`read`]: crate::ptr::read
1851+
/// [read-ownership]: crate::ptr::read#ownership-of-the-returned-value
1852+
/// [valid]: crate::ptr#safety
1853+
///
1854+
/// # Examples
1855+
///
1856+
/// Efficiently create a Rust vector from an unsafe buffer:
1857+
///
1858+
/// ```
1859+
/// use std::ptr;
1860+
///
1861+
/// /// # Safety
1862+
/// ///
1863+
/// /// * `ptr` must be correctly aligned for its type and non-zero.
1864+
/// /// * `ptr` must be valid for reads of `elts` contiguous elements of type `T`.
1865+
/// /// * Those elements must not be used after calling this function unless `T: Copy`.
1866+
/// # #[allow(dead_code)]
1867+
/// unsafe fn from_buf_raw<T>(ptr: *const T, elts: usize) -> Vec<T> {
1868+
/// let mut dst = Vec::with_capacity(elts);
1869+
///
1870+
/// // SAFETY: Our precondition ensures the source is aligned and valid,
1871+
/// // and `Vec::with_capacity` ensures that we have usable space to write them.
1872+
/// ptr::copy(ptr, dst.as_mut_ptr(), elts);
1873+
///
1874+
/// // SAFETY: We created it with this much capacity earlier,
1875+
/// // and the previous `copy` has initialized these elements.
1876+
/// dst.set_len(elts);
1877+
/// dst
1878+
/// }
1879+
/// ```
1880+
#[doc(alias = "memmove")]
1881+
#[stable(feature = "rust1", since = "1.0.0")]
1882+
#[rustc_const_unstable(feature = "const_intrinsic_copy", issue = "80697")]
1883+
pub fn copy<T>(src: *const T, dst: *mut T, count: usize);
17331884
}
17341885

17351886
// Some functions are defined here because they accidentally got made
@@ -1755,192 +1906,6 @@ pub(crate) fn is_nonoverlapping<T>(src: *const T, dst: *const T, count: usize) -
17551906
diff >= size
17561907
}
17571908

1758-
/// Copies `count * size_of::<T>()` bytes from `src` to `dst`. The source
1759-
/// and destination must *not* overlap.
1760-
///
1761-
/// For regions of memory which might overlap, use [`copy`] instead.
1762-
///
1763-
/// `copy_nonoverlapping` is semantically equivalent to C's [`memcpy`], but
1764-
/// with the argument order swapped.
1765-
///
1766-
/// [`memcpy`]: https://en.cppreference.com/w/c/string/byte/memcpy
1767-
///
1768-
/// # Safety
1769-
///
1770-
/// Behavior is undefined if any of the following conditions are violated:
1771-
///
1772-
/// * `src` must be [valid] for reads of `count * size_of::<T>()` bytes.
1773-
///
1774-
/// * `dst` must be [valid] for writes of `count * size_of::<T>()` bytes.
1775-
///
1776-
/// * Both `src` and `dst` must be properly aligned.
1777-
///
1778-
/// * The region of memory beginning at `src` with a size of `count *
1779-
/// size_of::<T>()` bytes must *not* overlap with the region of memory
1780-
/// beginning at `dst` with the same size.
1781-
///
1782-
/// Like [`read`], `copy_nonoverlapping` creates a bitwise copy of `T`, regardless of
1783-
/// whether `T` is [`Copy`]. If `T` is not [`Copy`], using *both* the values
1784-
/// in the region beginning at `*src` and the region beginning at `*dst` can
1785-
/// [violate memory safety][read-ownership].
1786-
///
1787-
/// Note that even if the effectively copied size (`count * size_of::<T>()`) is
1788-
/// `0`, the pointers must be non-NULL and properly aligned.
1789-
///
1790-
/// [`read`]: crate::ptr::read
1791-
/// [read-ownership]: crate::ptr::read#ownership-of-the-returned-value
1792-
/// [valid]: crate::ptr#safety
1793-
///
1794-
/// # Examples
1795-
///
1796-
/// Manually implement [`Vec::append`]:
1797-
///
1798-
/// ```
1799-
/// use std::ptr;
1800-
///
1801-
/// /// Moves all the elements of `src` into `dst`, leaving `src` empty.
1802-
/// fn append<T>(dst: &mut Vec<T>, src: &mut Vec<T>) {
1803-
/// let src_len = src.len();
1804-
/// let dst_len = dst.len();
1805-
///
1806-
/// // Ensure that `dst` has enough capacity to hold all of `src`.
1807-
/// dst.reserve(src_len);
1808-
///
1809-
/// unsafe {
1810-
/// // The call to offset is always safe because `Vec` will never
1811-
/// // allocate more than `isize::MAX` bytes.
1812-
/// let dst_ptr = dst.as_mut_ptr().offset(dst_len as isize);
1813-
/// let src_ptr = src.as_ptr();
1814-
///
1815-
/// // Truncate `src` without dropping its contents. We do this first,
1816-
/// // to avoid problems in case something further down panics.
1817-
/// src.set_len(0);
1818-
///
1819-
/// // The two regions cannot overlap because mutable references do
1820-
/// // not alias, and two different vectors cannot own the same
1821-
/// // memory.
1822-
/// ptr::copy_nonoverlapping(src_ptr, dst_ptr, src_len);
1823-
///
1824-
/// // Notify `dst` that it now holds the contents of `src`.
1825-
/// dst.set_len(dst_len + src_len);
1826-
/// }
1827-
/// }
1828-
///
1829-
/// let mut a = vec!['r'];
1830-
/// let mut b = vec!['u', 's', 't'];
1831-
///
1832-
/// append(&mut a, &mut b);
1833-
///
1834-
/// assert_eq!(a, &['r', 'u', 's', 't']);
1835-
/// assert!(b.is_empty());
1836-
/// ```
1837-
///
1838-
/// [`Vec::append`]: ../../std/vec/struct.Vec.html#method.append
1839-
#[doc(alias = "memcpy")]
1840-
#[stable(feature = "rust1", since = "1.0.0")]
1841-
#[rustc_const_unstable(feature = "const_intrinsic_copy", issue = "80697")]
1842-
#[inline]
1843-
pub const unsafe fn copy_nonoverlapping<T>(src: *const T, dst: *mut T, count: usize) {
1844-
extern "rust-intrinsic" {
1845-
#[rustc_const_unstable(feature = "const_intrinsic_copy", issue = "80697")]
1846-
fn copy_nonoverlapping<T>(src: *const T, dst: *mut T, count: usize);
1847-
}
1848-
1849-
// FIXME: Perform these checks only at run time
1850-
/*if cfg!(debug_assertions)
1851-
&& !(is_aligned_and_not_null(src)
1852-
&& is_aligned_and_not_null(dst)
1853-
&& is_nonoverlapping(src, dst, count))
1854-
{
1855-
// Not panicking to keep codegen impact smaller.
1856-
abort();
1857-
}*/
1858-
1859-
// SAFETY: the safety contract for `copy_nonoverlapping` must be
1860-
// upheld by the caller.
1861-
unsafe { copy_nonoverlapping(src, dst, count) }
1862-
}
1863-
1864-
/// Copies `count * size_of::<T>()` bytes from `src` to `dst`. The source
1865-
/// and destination may overlap.
1866-
///
1867-
/// If the source and destination will *never* overlap,
1868-
/// [`copy_nonoverlapping`] can be used instead.
1869-
///
1870-
/// `copy` is semantically equivalent to C's [`memmove`], but with the argument
1871-
/// order swapped. Copying takes place as if the bytes were copied from `src`
1872-
/// to a temporary array and then copied from the array to `dst`.
1873-
///
1874-
/// [`memmove`]: https://en.cppreference.com/w/c/string/byte/memmove
1875-
///
1876-
/// # Safety
1877-
///
1878-
/// Behavior is undefined if any of the following conditions are violated:
1879-
///
1880-
/// * `src` must be [valid] for reads of `count * size_of::<T>()` bytes.
1881-
///
1882-
/// * `dst` must be [valid] for writes of `count * size_of::<T>()` bytes.
1883-
///
1884-
/// * Both `src` and `dst` must be properly aligned.
1885-
///
1886-
/// Like [`read`], `copy` creates a bitwise copy of `T`, regardless of
1887-
/// whether `T` is [`Copy`]. If `T` is not [`Copy`], using both the values
1888-
/// in the region beginning at `*src` and the region beginning at `*dst` can
1889-
/// [violate memory safety][read-ownership].
1890-
///
1891-
/// Note that even if the effectively copied size (`count * size_of::<T>()`) is
1892-
/// `0`, the pointers must be non-NULL and properly aligned.
1893-
///
1894-
/// [`read`]: crate::ptr::read
1895-
/// [read-ownership]: crate::ptr::read#ownership-of-the-returned-value
1896-
/// [valid]: crate::ptr#safety
1897-
///
1898-
/// # Examples
1899-
///
1900-
/// Efficiently create a Rust vector from an unsafe buffer:
1901-
///
1902-
/// ```
1903-
/// use std::ptr;
1904-
///
1905-
/// /// # Safety
1906-
/// ///
1907-
/// /// * `ptr` must be correctly aligned for its type and non-zero.
1908-
/// /// * `ptr` must be valid for reads of `elts` contiguous elements of type `T`.
1909-
/// /// * Those elements must not be used after calling this function unless `T: Copy`.
1910-
/// # #[allow(dead_code)]
1911-
/// unsafe fn from_buf_raw<T>(ptr: *const T, elts: usize) -> Vec<T> {
1912-
/// let mut dst = Vec::with_capacity(elts);
1913-
///
1914-
/// // SAFETY: Our precondition ensures the source is aligned and valid,
1915-
/// // and `Vec::with_capacity` ensures that we have usable space to write them.
1916-
/// ptr::copy(ptr, dst.as_mut_ptr(), elts);
1917-
///
1918-
/// // SAFETY: We created it with this much capacity earlier,
1919-
/// // and the previous `copy` has initialized these elements.
1920-
/// dst.set_len(elts);
1921-
/// dst
1922-
/// }
1923-
/// ```
1924-
#[doc(alias = "memmove")]
1925-
#[stable(feature = "rust1", since = "1.0.0")]
1926-
#[rustc_const_unstable(feature = "const_intrinsic_copy", issue = "80697")]
1927-
#[inline]
1928-
pub const unsafe fn copy<T>(src: *const T, dst: *mut T, count: usize) {
1929-
extern "rust-intrinsic" {
1930-
#[rustc_const_unstable(feature = "const_intrinsic_copy", issue = "80697")]
1931-
fn copy<T>(src: *const T, dst: *mut T, count: usize);
1932-
}
1933-
1934-
// FIXME: Perform these checks only at run time
1935-
/*if cfg!(debug_assertions) && !(is_aligned_and_not_null(src) && is_aligned_and_not_null(dst)) {
1936-
// Not panicking to keep codegen impact smaller.
1937-
abort();
1938-
}*/
1939-
1940-
// SAFETY: the safety contract for `copy` must be upheld by the caller.
1941-
unsafe { copy(src, dst, count) }
1942-
}
1943-
19441909
/// Sets `count * size_of::<T>()` bytes of memory starting at `dst` to
19451910
/// `val`.
19461911
///

‎library/core/src/ptr/mod.rs

+1-6
Original file line numberDiff line numberDiff line change
@@ -881,17 +881,12 @@ pub const unsafe fn read_unaligned<T>(src: *const T) -> T {
881881
#[inline]
882882
#[stable(feature = "rust1", since = "1.0.0")]
883883
pub unsafe fn write<T>(dst: *mut T, src: T) {
884-
// We are calling the intrinsics directly to avoid function calls in the generated code
885-
// as `intrinsics::copy_nonoverlapping` is a wrapper function.
886-
extern "rust-intrinsic" {
887-
fn copy_nonoverlapping<T>(src: *const T, dst: *mut T, count: usize);
888-
}
889-
890884
// SAFETY: the caller must guarantee that `dst` is valid for writes.
891885
// `dst` cannot overlap `src` because the caller has mutable access
892886
// to `dst` while `src` is owned by this function.
893887
unsafe {
894888
copy_nonoverlapping(&src as *const T, dst, 1);
889+
// We are calling the intrinsic directly to avoid function calls in the generated code.
895890
intrinsics::forget(src);
896891
}
897892
}

0 commit comments

Comments
 (0)